From c8a85fabe1562debaedc346c07a16ecedcfece44 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Tue, 15 Jan 2019 21:05:46 -0800 Subject: [PATCH 001/523] Update Grafana from v5.4.2 to v5.4.3 * https://github.com/grafana/grafana/releases/tag/v5.4.3 --- CHANGES.md | 1 + addons/grafana/deployment.yaml | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index ee2152cf5..cb505718f 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -15,6 +15,7 @@ Notable changes between versions. #### Addons * Update kube-state-metrics from v1.4.0 to v1.5.0 +* Update Grafana from v5.4.2 to v5.4.3 ## v1.13.2 diff --git a/addons/grafana/deployment.yaml b/addons/grafana/deployment.yaml index aacd23e59..409c4710d 100644 --- a/addons/grafana/deployment.yaml +++ b/addons/grafana/deployment.yaml @@ -23,7 +23,7 @@ spec: spec: containers: - name: grafana - image: grafana/grafana:5.4.2 + image: grafana/grafana:5.4.3 env: - name: GF_SERVER_HTTP_PORT value: "8080" From 67fb9602e7df5342ffd23a520eb5ceced8a19965 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Tue, 15 Jan 2019 21:13:40 -0800 Subject: [PATCH 002/523] Update Prometheus from v2.6.0 to v2.6.1 * https://github.com/prometheus/prometheus/releases/tag/v2.6.1 --- CHANGES.md | 1 + addons/prometheus/deployment.yaml | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index cb505718f..5fb9757db 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -14,6 +14,7 @@ Notable changes between versions. #### Addons +* Update Prometheus from v2.6.0 to v2.6.1 * Update kube-state-metrics from v1.4.0 to v1.5.0 * Update Grafana from v5.4.2 to v5.4.3 diff --git a/addons/prometheus/deployment.yaml b/addons/prometheus/deployment.yaml index 36f03f896..638eca1f8 100644 --- a/addons/prometheus/deployment.yaml +++ b/addons/prometheus/deployment.yaml @@ -20,7 +20,7 @@ spec: serviceAccountName: prometheus containers: - name: prometheus - image: quay.io/prometheus/prometheus:v2.6.0 + image: quay.io/prometheus/prometheus:v2.6.1 args: - --web.listen-address=0.0.0.0:9090 - --config.file=/etc/prometheus/prometheus.yaml From f4d3508578e4f73c609271dbbcff8036ff393de3 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Tue, 15 Jan 2019 21:24:35 -0800 Subject: [PATCH 003/523] Update CoreDNS from v1.3.0 to v1.3.1 * https://coredns.io/2019/01/13/coredns-1.3.1-release/ --- CHANGES.md | 1 + aws/container-linux/kubernetes/bootkube.tf | 2 +- aws/fedora-atomic/kubernetes/bootkube.tf | 2 +- azure/container-linux/kubernetes/bootkube.tf | 2 +- bare-metal/container-linux/kubernetes/bootkube.tf | 2 +- bare-metal/fedora-atomic/kubernetes/bootkube.tf | 2 +- digital-ocean/container-linux/kubernetes/bootkube.tf | 2 +- digital-ocean/fedora-atomic/kubernetes/bootkube.tf | 2 +- google-cloud/container-linux/kubernetes/bootkube.tf | 2 +- google-cloud/fedora-atomic/kubernetes/bootkube.tf | 2 +- 10 files changed, 10 insertions(+), 9 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 5fb9757db..5a21a44f2 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -5,6 +5,7 @@ Notable changes between versions. ## Latest * Update etcd from v3.3.10 to [v3.3.11](https://github.com/etcd-io/etcd/blob/master/CHANGELOG-3.3.md#v3311-2019-1-11) +* Update CoreDNS from v1.3.0 to [v1.3.1](https://coredns.io/2019/01/13/coredns-1.3.1-release/) * Fix instance shutdown automatic worker deletion on cloud platforms * Lowering Kubelet privileges in [#372](https://github.com/poseidon/typhoon/pull/372) dropped a needed node deletion authorization. Scale-in due to manual terraform apply (any cloud), AWS spot termination, or Azure low priority deletion left old nodes registered, requiring manual deletion (`kubectl delete node name`) diff --git a/aws/container-linux/kubernetes/bootkube.tf b/aws/container-linux/kubernetes/bootkube.tf index bb7981e6f..2c26f75ae 100644 --- a/aws/container-linux/kubernetes/bootkube.tf +++ b/aws/container-linux/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=e892e291b572655699aee8565c14c8446bab2104" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=ef99293eb2b5fd8ba0bb19fb49e795899dbe690d" cluster_name = "${var.cluster_name}" api_servers = ["${format("%s.%s", var.cluster_name, var.dns_zone)}"] diff --git a/aws/fedora-atomic/kubernetes/bootkube.tf b/aws/fedora-atomic/kubernetes/bootkube.tf index 81d5d0f51..0ddfee8c0 100644 --- a/aws/fedora-atomic/kubernetes/bootkube.tf +++ b/aws/fedora-atomic/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=e892e291b572655699aee8565c14c8446bab2104" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=ef99293eb2b5fd8ba0bb19fb49e795899dbe690d" cluster_name = "${var.cluster_name}" api_servers = ["${format("%s.%s", var.cluster_name, var.dns_zone)}"] diff --git a/azure/container-linux/kubernetes/bootkube.tf b/azure/container-linux/kubernetes/bootkube.tf index 06684fd59..dd19b97fb 100644 --- a/azure/container-linux/kubernetes/bootkube.tf +++ b/azure/container-linux/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=e892e291b572655699aee8565c14c8446bab2104" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=ef99293eb2b5fd8ba0bb19fb49e795899dbe690d" cluster_name = "${var.cluster_name}" api_servers = ["${format("%s.%s", var.cluster_name, var.dns_zone)}"] diff --git a/bare-metal/container-linux/kubernetes/bootkube.tf b/bare-metal/container-linux/kubernetes/bootkube.tf index e61fea497..573d96b75 100644 --- a/bare-metal/container-linux/kubernetes/bootkube.tf +++ b/bare-metal/container-linux/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=e892e291b572655699aee8565c14c8446bab2104" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=ef99293eb2b5fd8ba0bb19fb49e795899dbe690d" cluster_name = "${var.cluster_name}" api_servers = ["${var.k8s_domain_name}"] diff --git a/bare-metal/fedora-atomic/kubernetes/bootkube.tf b/bare-metal/fedora-atomic/kubernetes/bootkube.tf index 7b15ac8e2..ad75bab58 100644 --- a/bare-metal/fedora-atomic/kubernetes/bootkube.tf +++ b/bare-metal/fedora-atomic/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=e892e291b572655699aee8565c14c8446bab2104" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=ef99293eb2b5fd8ba0bb19fb49e795899dbe690d" cluster_name = "${var.cluster_name}" api_servers = ["${var.k8s_domain_name}"] diff --git a/digital-ocean/container-linux/kubernetes/bootkube.tf b/digital-ocean/container-linux/kubernetes/bootkube.tf index fd4e4a493..023fa204e 100644 --- a/digital-ocean/container-linux/kubernetes/bootkube.tf +++ b/digital-ocean/container-linux/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=e892e291b572655699aee8565c14c8446bab2104" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=ef99293eb2b5fd8ba0bb19fb49e795899dbe690d" cluster_name = "${var.cluster_name}" api_servers = ["${format("%s.%s", var.cluster_name, var.dns_zone)}"] diff --git a/digital-ocean/fedora-atomic/kubernetes/bootkube.tf b/digital-ocean/fedora-atomic/kubernetes/bootkube.tf index 07eb60e77..195e98523 100644 --- a/digital-ocean/fedora-atomic/kubernetes/bootkube.tf +++ b/digital-ocean/fedora-atomic/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=e892e291b572655699aee8565c14c8446bab2104" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=ef99293eb2b5fd8ba0bb19fb49e795899dbe690d" cluster_name = "${var.cluster_name}" api_servers = ["${format("%s.%s", var.cluster_name, var.dns_zone)}"] diff --git a/google-cloud/container-linux/kubernetes/bootkube.tf b/google-cloud/container-linux/kubernetes/bootkube.tf index 80de51a41..92513ff26 100644 --- a/google-cloud/container-linux/kubernetes/bootkube.tf +++ b/google-cloud/container-linux/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=e892e291b572655699aee8565c14c8446bab2104" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=ef99293eb2b5fd8ba0bb19fb49e795899dbe690d" cluster_name = "${var.cluster_name}" api_servers = ["${format("%s.%s", var.cluster_name, var.dns_zone)}"] diff --git a/google-cloud/fedora-atomic/kubernetes/bootkube.tf b/google-cloud/fedora-atomic/kubernetes/bootkube.tf index aa3906155..4fbccb789 100644 --- a/google-cloud/fedora-atomic/kubernetes/bootkube.tf +++ b/google-cloud/fedora-atomic/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=e892e291b572655699aee8565c14c8446bab2104" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=ef99293eb2b5fd8ba0bb19fb49e795899dbe690d" cluster_name = "${var.cluster_name}" api_servers = ["${format("%s.%s", var.cluster_name, var.dns_zone)}"] From 2f3097ebeab2821562aaae10482f1807aaff7d88 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Wed, 16 Jan 2019 22:59:46 -0800 Subject: [PATCH 004/523] Update nginx-ingress from v0.21.0 to v0.22.0 * https://github.com/kubernetes/ingress-nginx/releases/tag/nginx-0.22.0 --- CHANGES.md | 3 ++- addons/nginx-ingress/aws/deployment.yaml | 2 +- addons/nginx-ingress/azure/deployment.yaml | 2 +- addons/nginx-ingress/bare-metal/deployment.yaml | 2 +- addons/nginx-ingress/digital-ocean/daemonset.yaml | 2 +- addons/nginx-ingress/google-cloud/deployment.yaml | 2 +- 6 files changed, 7 insertions(+), 6 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 5a21a44f2..dfa98af85 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -6,7 +6,7 @@ Notable changes between versions. * Update etcd from v3.3.10 to [v3.3.11](https://github.com/etcd-io/etcd/blob/master/CHANGELOG-3.3.md#v3311-2019-1-11) * Update CoreDNS from v1.3.0 to [v1.3.1](https://coredns.io/2019/01/13/coredns-1.3.1-release/) -* Fix instance shutdown automatic worker deletion on cloud platforms +* Fix automatic worker deletion on shutdown for cloud platforms * Lowering Kubelet privileges in [#372](https://github.com/poseidon/typhoon/pull/372) dropped a needed node deletion authorization. Scale-in due to manual terraform apply (any cloud), AWS spot termination, or Azure low priority deletion left old nodes registered, requiring manual deletion (`kubectl delete node name`) #### AWS @@ -15,6 +15,7 @@ Notable changes between versions. #### Addons +* Update nginx-ingress from v0.21.0 to v0.22.0 * Update Prometheus from v2.6.0 to v2.6.1 * Update kube-state-metrics from v1.4.0 to v1.5.0 * Update Grafana from v5.4.2 to v5.4.3 diff --git a/addons/nginx-ingress/aws/deployment.yaml b/addons/nginx-ingress/aws/deployment.yaml index 633076e34..c06e798dc 100644 --- a/addons/nginx-ingress/aws/deployment.yaml +++ b/addons/nginx-ingress/aws/deployment.yaml @@ -24,7 +24,7 @@ spec: node-role.kubernetes.io/node: "" containers: - name: nginx-ingress-controller - image: quay.io/kubernetes-ingress-controller/nginx-ingress-controller:0.21.0 + image: quay.io/kubernetes-ingress-controller/nginx-ingress-controller:0.22.0 args: - /nginx-ingress-controller - --default-backend-service=$(POD_NAMESPACE)/default-backend diff --git a/addons/nginx-ingress/azure/deployment.yaml b/addons/nginx-ingress/azure/deployment.yaml index 633076e34..c06e798dc 100644 --- a/addons/nginx-ingress/azure/deployment.yaml +++ b/addons/nginx-ingress/azure/deployment.yaml @@ -24,7 +24,7 @@ spec: node-role.kubernetes.io/node: "" containers: - name: nginx-ingress-controller - image: quay.io/kubernetes-ingress-controller/nginx-ingress-controller:0.21.0 + image: quay.io/kubernetes-ingress-controller/nginx-ingress-controller:0.22.0 args: - /nginx-ingress-controller - --default-backend-service=$(POD_NAMESPACE)/default-backend diff --git a/addons/nginx-ingress/bare-metal/deployment.yaml b/addons/nginx-ingress/bare-metal/deployment.yaml index 561835ed5..5c22a8d26 100644 --- a/addons/nginx-ingress/bare-metal/deployment.yaml +++ b/addons/nginx-ingress/bare-metal/deployment.yaml @@ -22,7 +22,7 @@ spec: spec: containers: - name: nginx-ingress-controller - image: quay.io/kubernetes-ingress-controller/nginx-ingress-controller:0.21.0 + image: quay.io/kubernetes-ingress-controller/nginx-ingress-controller:0.22.0 args: - /nginx-ingress-controller - --default-backend-service=$(POD_NAMESPACE)/default-backend diff --git a/addons/nginx-ingress/digital-ocean/daemonset.yaml b/addons/nginx-ingress/digital-ocean/daemonset.yaml index 8dc97748d..5909112c9 100644 --- a/addons/nginx-ingress/digital-ocean/daemonset.yaml +++ b/addons/nginx-ingress/digital-ocean/daemonset.yaml @@ -24,7 +24,7 @@ spec: node-role.kubernetes.io/node: "" containers: - name: nginx-ingress-controller - image: quay.io/kubernetes-ingress-controller/nginx-ingress-controller:0.21.0 + image: quay.io/kubernetes-ingress-controller/nginx-ingress-controller:0.22.0 args: - /nginx-ingress-controller - --default-backend-service=$(POD_NAMESPACE)/default-backend diff --git a/addons/nginx-ingress/google-cloud/deployment.yaml b/addons/nginx-ingress/google-cloud/deployment.yaml index 633076e34..c06e798dc 100644 --- a/addons/nginx-ingress/google-cloud/deployment.yaml +++ b/addons/nginx-ingress/google-cloud/deployment.yaml @@ -24,7 +24,7 @@ spec: node-role.kubernetes.io/node: "" containers: - name: nginx-ingress-controller - image: quay.io/kubernetes-ingress-controller/nginx-ingress-controller:0.21.0 + image: quay.io/kubernetes-ingress-controller/nginx-ingress-controller:0.22.0 args: - /nginx-ingress-controller - --default-backend-service=$(POD_NAMESPACE)/default-backend From d697dd46dc5f05dd02d2f24a75deae8f41a57f9e Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Mon, 21 Jan 2019 20:07:31 -0800 Subject: [PATCH 005/523] Allow kube-state-metrics PodDisruptionBudget metrics * Update kube-state-metrics ClusterRole to allow collecting poddisruptionbudget metrics (exported as kube_poddisruptionbudget_*) * https://github.com/kubernetes/kube-state-metrics/pull/551 * Bump addon-resizer from v1.7 to v1.8.4 --- CHANGES.md | 1 + .../kube-state-metrics/cluster-role.yaml | 45 ++++++++++++++----- .../kube-state-metrics/deployment.yaml | 2 +- .../resizer-role-binding.yaml | 2 +- .../kube-state-metrics/resizer-role.yaml | 28 +++++++++--- 5 files changed, 60 insertions(+), 18 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index dfa98af85..c5f543e6f 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -18,6 +18,7 @@ Notable changes between versions. * Update nginx-ingress from v0.21.0 to v0.22.0 * Update Prometheus from v2.6.0 to v2.6.1 * Update kube-state-metrics from v1.4.0 to v1.5.0 + * Fix ClusterRole to collect and export PodDisruptionBudget metrics ([#383](https://github.com/poseidon/typhoon/pull/383)) * Update Grafana from v5.4.2 to v5.4.3 ## v1.13.2 diff --git a/addons/prometheus/exporters/kube-state-metrics/cluster-role.yaml b/addons/prometheus/exporters/kube-state-metrics/cluster-role.yaml index edb3bdf98..1b3f2cde8 100644 --- a/addons/prometheus/exporters/kube-state-metrics/cluster-role.yaml +++ b/addons/prometheus/exporters/kube-state-metrics/cluster-role.yaml @@ -3,7 +3,8 @@ kind: ClusterRole metadata: name: kube-state-metrics rules: -- apiGroups: [""] +- apiGroups: + - "" resources: - configmaps - secrets @@ -17,23 +18,47 @@ rules: - persistentvolumes - namespaces - endpoints - verbs: ["list", "watch"] -- apiGroups: ["extensions"] + verbs: + - list + - watch +- apiGroups: + - extensions resources: - daemonsets - deployments - replicasets - verbs: ["list", "watch"] -- apiGroups: ["apps"] + verbs: + - list + - watch +- apiGroups: + - apps resources: - statefulsets - verbs: ["list", "watch"] -- apiGroups: ["batch"] + - daemonsets + - deployments + - replicasets + verbs: + - list + - watch +- apiGroups: + - batch resources: - cronjobs - jobs - verbs: ["list", "watch"] -- apiGroups: ["autoscaling"] + verbs: + - list + - watch +- apiGroups: + - autoscaling resources: - horizontalpodautoscalers - verbs: ["list", "watch"] + verbs: + - list + - watch +- apiGroups: + - policy + resources: + - poddisruptionbudgets + verbs: + - list + - watch diff --git a/addons/prometheus/exporters/kube-state-metrics/deployment.yaml b/addons/prometheus/exporters/kube-state-metrics/deployment.yaml index 4e0158927..598b0f574 100644 --- a/addons/prometheus/exporters/kube-state-metrics/deployment.yaml +++ b/addons/prometheus/exporters/kube-state-metrics/deployment.yaml @@ -35,7 +35,7 @@ spec: initialDelaySeconds: 5 timeoutSeconds: 5 - name: addon-resizer - image: k8s.gcr.io/addon-resizer:1.7 + image: k8s.gcr.io/addon-resizer:1.8.4 resources: limits: cpu: 100m diff --git a/addons/prometheus/exporters/kube-state-metrics/resizer-role-binding.yaml b/addons/prometheus/exporters/kube-state-metrics/resizer-role-binding.yaml index 656e3db64..f10a2bccb 100644 --- a/addons/prometheus/exporters/kube-state-metrics/resizer-role-binding.yaml +++ b/addons/prometheus/exporters/kube-state-metrics/resizer-role-binding.yaml @@ -6,7 +6,7 @@ metadata: roleRef: apiGroup: rbac.authorization.k8s.io kind: Role - name: kube-state-metrics-resizer + name: kube-state-metrics subjects: - kind: ServiceAccount name: kube-state-metrics diff --git a/addons/prometheus/exporters/kube-state-metrics/resizer-role.yaml b/addons/prometheus/exporters/kube-state-metrics/resizer-role.yaml index 497460645..6d12b57c3 100644 --- a/addons/prometheus/exporters/kube-state-metrics/resizer-role.yaml +++ b/addons/prometheus/exporters/kube-state-metrics/resizer-role.yaml @@ -1,15 +1,31 @@ apiVersion: rbac.authorization.k8s.io/v1 kind: Role metadata: - name: kube-state-metrics-resizer + name: kube-state-metrics namespace: monitoring rules: -- apiGroups: [""] +- apiGroups: + - "" resources: - pods - verbs: ["get"] -- apiGroups: ["extensions"] + verbs: + - get +- apiGroups: + - extensions resources: - deployments - resourceNames: ["kube-state-metrics"] - verbs: ["get", "update"] + resourceNames: + - kube-state-metrics + verbs: + - get + - update +- apiGroups: + - apps + resources: + - deployments + resourceNames: + - kube-state-metrics + verbs: + - get + - update + From f5ff003d0ee790ddf7a37d1f6e57f531f3f92739 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Tue, 22 Jan 2019 00:19:55 -0800 Subject: [PATCH 006/523] Update node-exporter from v0.15.2 to v0.17.0 * node-exporter renamed multiple metrics that are reflected in changes to Prometheus rules and Grafana dashboard expressions --- CHANGES.md | 1 + addons/grafana/dashboards.yaml | 58 +++++++++---------- .../exporters/node-exporter/daemonset.yaml | 19 ++++-- addons/prometheus/rules.yaml | 20 +++---- 4 files changed, 54 insertions(+), 44 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index c5f543e6f..f26c5dae6 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -19,6 +19,7 @@ Notable changes between versions. * Update Prometheus from v2.6.0 to v2.6.1 * Update kube-state-metrics from v1.4.0 to v1.5.0 * Fix ClusterRole to collect and export PodDisruptionBudget metrics ([#383](https://github.com/poseidon/typhoon/pull/383)) +* Update node-exporter from v0.15.2 to v0.17.0 * Update Grafana from v5.4.2 to v5.4.3 ## v1.13.2 diff --git a/addons/grafana/dashboards.yaml b/addons/grafana/dashboards.yaml index 4212ddf5c..56a03ceb0 100644 --- a/addons/grafana/dashboards.yaml +++ b/addons/grafana/dashboards.yaml @@ -1963,7 +1963,7 @@ data: "steppedLine": false, "targets": [ { - "expr": "sum(rate(node_cpu{mode=\"idle\"}[2m])) * 100", + "expr": "sum(rate(node_cpu_seconds_total{mode=\"idle\"}[2m])) * 100", "hide": false, "intervalFactor": 10, "legendFormat": "", @@ -2138,7 +2138,7 @@ data: "renderer": "flot", "seriesOverrides": [ { - "alias": "node_memory_SwapFree{instance=\"172.17.0.1:9100\",job=\"prometheus\"}", + "alias": "node_memory_SwapFree_bytes{instance=\"172.17.0.1:9100\",job=\"prometheus\"}", "yaxis": 2 } ], @@ -2148,7 +2148,7 @@ data: "steppedLine": false, "targets": [ { - "expr": "sum(node_memory_MemTotal) - sum(node_memory_MemFree) - sum(node_memory_Buffers) - sum(node_memory_Cached)", + "expr": "sum(node_memory_MemTotal_bytes) - sum(node_memory_MemFree_bytes) - sum(node_memory_Buffers_bytes) - sum(node_memory_Cached_bytes)", "intervalFactor": 2, "legendFormat": "memory usage", "metric": "memo", @@ -2157,7 +2157,7 @@ data: "target": "" }, { - "expr": "sum(node_memory_Buffers)", + "expr": "sum(node_memory_Buffers_bytes)", "interval": "", "intervalFactor": 2, "legendFormat": "memory buffers", @@ -2167,7 +2167,7 @@ data: "target": "" }, { - "expr": "sum(node_memory_Cached)", + "expr": "sum(node_memory_Cached_bytes)", "interval": "", "intervalFactor": 2, "legendFormat": "memory cached", @@ -2177,7 +2177,7 @@ data: "target": "" }, { - "expr": "sum(node_memory_MemFree)", + "expr": "sum(node_memory_MemFree_bytes)", "interval": "", "intervalFactor": 2, "legendFormat": "memory free", @@ -2268,7 +2268,7 @@ data: }, "targets": [ { - "expr": "((sum(node_memory_MemTotal) - sum(node_memory_MemFree) - sum(node_memory_Buffers) - sum(node_memory_Cached)) / sum(node_memory_MemTotal)) * 100", + "expr": "((sum(node_memory_MemTotal_bytes) - sum(node_memory_MemFree_bytes) - sum(node_memory_Buffers_bytes) - sum(node_memory_Cached_bytes)) / sum(node_memory_MemTotal_bytes)) * 100", "intervalFactor": 2, "metric": "", "refId": "A", @@ -2355,7 +2355,7 @@ data: "steppedLine": false, "targets": [ { - "expr": "sum(rate(node_disk_bytes_read[5m]))", + "expr": "max(rate(node_disk_read_bytes_total[5m]))", "hide": false, "intervalFactor": 4, "legendFormat": "read", @@ -2364,14 +2364,14 @@ data: "target": "" }, { - "expr": "sum(rate(node_disk_bytes_written[5m]))", + "expr": "max(rate(node_disk_written_bytes_total[5m]))", "intervalFactor": 4, "legendFormat": "written", "refId": "B", "step": 20 }, { - "expr": "sum(rate(node_disk_io_time_ms[5m]))", + "expr": "max(rate(node_disk_io_time_seconds_total[5m]))", "intervalFactor": 4, "legendFormat": "io time", "refId": "C", @@ -2458,7 +2458,7 @@ data: }, "targets": [ { - "expr": "(sum(node_filesystem_size{device!=\"rootfs\"}) - sum(node_filesystem_free{device!=\"rootfs\"})) / sum(node_filesystem_size{device!=\"rootfs\"})", + "expr": "(sum(node_filesystem_size_bytes{device!=\"rootfs\"}) - sum(node_filesystem_free_bytes{device!=\"rootfs\"})) / sum(node_filesystem_size_bytes{device!=\"rootfs\"})", "intervalFactor": 2, "refId": "A", "step": 60, @@ -2536,7 +2536,7 @@ data: "steppedLine": false, "targets": [ { - "expr": "sum(rate(node_network_receive_bytes{device!~\"lo\"}[5m]))", + "expr": "sum(rate(node_network_receive_bytes_total{device!~\"lo\"}[5m]))", "hide": false, "intervalFactor": 2, "legendFormat": "", @@ -2618,7 +2618,7 @@ data: "steppedLine": false, "targets": [ { - "expr": "sum(rate(node_network_transmit_bytes{device!~\"lo\"}[5m]))", + "expr": "sum(rate(node_network_transmit_bytes_total{device!~\"lo\"}[5m]))", "hide": false, "intervalFactor": 2, "legendFormat": "", @@ -4093,7 +4093,7 @@ data: }, "targets": [ { - "expr": "sum(100 - (avg by (instance) (rate(node_cpu{job=\"node-exporter\",mode=\"idle\"}[5m])) * 100)) / count(node_cpu{job=\"node-exporter\",mode=\"idle\"})", + "expr": "sum(100 - (avg by (instance) (rate(node_cpu_seconds_total{job=\"node-exporter\",mode=\"idle\"}[5m])) * 100)) / count(node_cpu_seconds_total{job=\"node-exporter\",mode=\"idle\"})", "format": "time_series", "intervalFactor": 2, "refId": "A", @@ -4165,7 +4165,7 @@ data: }, "targets": [ { - "expr": "((sum(node_memory_MemTotal) - sum(node_memory_MemFree) - sum(node_memory_Buffers) - sum(node_memory_Cached)) / sum(node_memory_MemTotal)) * 100", + "expr": "((sum(node_memory_MemTotal_bytes) - sum(node_memory_MemFree_bytes) - sum(node_memory_Buffers_bytes) - sum(node_memory_Cached_bytes)) / sum(node_memory_MemTotal_bytes)) * 100", "format": "time_series", "intervalFactor": 2, "refId": "A", @@ -4237,7 +4237,7 @@ data: }, "targets": [ { - "expr": "(sum(node_filesystem_size{device!=\"rootfs\"}) - sum(node_filesystem_free{device!=\"rootfs\"})) / sum(node_filesystem_size{device!=\"rootfs\"})", + "expr": "(sum(node_filesystem_size_bytes{device!=\"rootfs\"}) - sum(node_filesystem_free_bytes{device!=\"rootfs\"})) / sum(node_filesystem_size_bytes{device!=\"rootfs\"})", "format": "time_series", "intervalFactor": 2, "refId": "A", @@ -5476,7 +5476,7 @@ data: "steppedLine": false, "targets": [ { - "expr": "100 - (avg by (cpu) (irate(node_cpu{mode=\"idle\", instance=\"$server\"}[5m])) * 100)", + "expr": "100 - (avg by (cpu) (irate(node_cpu_seconds_total{mode=\"idle\", instance=\"$server\"}[5m])) * 100)", "hide": false, "intervalFactor": 10, "legendFormat": "{{cpu}}", @@ -5652,7 +5652,7 @@ data: "renderer": "flot", "seriesOverrides": [ { - "alias": "node_memory_SwapFree{instance=\"172.17.0.1:9100\",job=\"prometheus\"}", + "alias": "node_memory_SwapFree_bytes{instance=\"172.17.0.1:9100\",job=\"prometheus\"}", "yaxis": 2 } ], @@ -5662,7 +5662,7 @@ data: "steppedLine": false, "targets": [ { - "expr": "node_memory_MemTotal{instance=\"$server\"} - node_memory_MemFree{instance=\"$server\"} - node_memory_Buffers{instance=\"$server\"} - node_memory_Cached{instance=\"$server\"}", + "expr": "node_memory_MemTotal_bytes{instance=\"$server\"} - node_memory_MemFree_bytes{instance=\"$server\"} - node_memory_Buffers_bytes{instance=\"$server\"} - node_memory_Cached_bytes{instance=\"$server\"}", "hide": false, "interval": "", "intervalFactor": 2, @@ -5672,7 +5672,7 @@ data: "step": 10 }, { - "expr": "node_memory_Buffers{instance=\"$server\"}", + "expr": "node_memory_Buffers_bytes{instance=\"$server\"}", "interval": "", "intervalFactor": 2, "legendFormat": "memory buffers", @@ -5689,7 +5689,7 @@ data: "step": 10 }, { - "expr": "node_memory_MemFree{instance=\"$server\"}", + "expr": "node_memory_MemFree_bytes{instance=\"$server\"}", "intervalFactor": 2, "legendFormat": "memory free", "metric": "", @@ -5778,7 +5778,7 @@ data: }, "targets": [ { - "expr": "((node_memory_MemTotal{instance=\"$server\"} - node_memory_MemFree{instance=\"$server\"} - node_memory_Buffers{instance=\"$server\"} - node_memory_Cached{instance=\"$server\"}) / node_memory_MemTotal{instance=\"$server\"}) * 100", + "expr": "((node_memory_MemTotal_bytes{instance=\"$server\"} - node_memory_MemFree_bytes{instance=\"$server\"} - node_memory_Buffers_bytes{instance=\"$server\"} - node_memory_Cached_bytes{instance=\"$server\"}) / node_memory_MemTotal_bytes{instance=\"$server\"}) * 100", "intervalFactor": 2, "refId": "A", "step": 60, @@ -5864,7 +5864,7 @@ data: "steppedLine": false, "targets": [ { - "expr": "sum by (instance) (rate(node_disk_bytes_read{instance=\"$server\"}[2m]))", + "expr": "sum by (instance) (rate(node_disk_read_bytes_total{instance=\"$server\"}[2m]))", "hide": false, "intervalFactor": 4, "legendFormat": "read", @@ -5873,14 +5873,14 @@ data: "target": "" }, { - "expr": "sum by (instance) (rate(node_disk_bytes_written{instance=\"$server\"}[2m]))", + "expr": "sum by (instance) (rate(node_disk_written_bytes_total{instance=\"$server\"}[2m]))", "intervalFactor": 4, "legendFormat": "written", "refId": "B", "step": 20 }, { - "expr": "sum by (instance) (rate(node_disk_io_time_ms{instance=\"$server\"}[2m]))", + "expr": "sum by (instance) (rate(node_disk_io_time_seconds_total{instance=\"$server\"}[2m]))", "intervalFactor": 4, "legendFormat": "io time", "refId": "C", @@ -5967,7 +5967,7 @@ data: }, "targets": [ { - "expr": "(sum(node_filesystem_size{device!=\"rootfs\",instance=\"$server\"}) - sum(node_filesystem_free{device!=\"rootfs\",instance=\"$server\"})) / sum(node_filesystem_size{device!=\"rootfs\",instance=\"$server\"})", + "expr": "(sum(node_filesystem_size_bytes{device!=\"rootfs\",instance=\"$server\"}) - sum(node_filesystem_free_bytes{device!=\"rootfs\",instance=\"$server\"})) / sum(node_filesystem_size_bytes{device!=\"rootfs\",instance=\"$server\"})", "intervalFactor": 2, "refId": "A", "step": 60, @@ -6045,7 +6045,7 @@ data: "steppedLine": false, "targets": [ { - "expr": "rate(node_network_receive_bytes{instance=\"$server\",device!~\"lo\"}[5m])", + "expr": "rate(node_network_receive_bytes_total{instance=\"$server\",device!~\"lo\"}[5m])", "hide": false, "intervalFactor": 2, "legendFormat": "{{device}}", @@ -6127,7 +6127,7 @@ data: "steppedLine": false, "targets": [ { - "expr": "rate(node_network_transmit_bytes{instance=\"$server\",device!~\"lo\"}[5m])", + "expr": "rate(node_network_transmit_bytes_total{instance=\"$server\",device!~\"lo\"}[5m])", "hide": false, "intervalFactor": 2, "legendFormat": "{{device}}", @@ -6184,7 +6184,7 @@ data: "multi": false, "name": "server", "options": [], - "query": "label_values(node_boot_time, instance)", + "query": "label_values(node_boot_time_seconds, instance)", "refresh": 1, "regex": "", "sort": 0, diff --git a/addons/prometheus/exporters/node-exporter/daemonset.yaml b/addons/prometheus/exporters/node-exporter/daemonset.yaml index 4164bd51a..2b631c931 100644 --- a/addons/prometheus/exporters/node-exporter/daemonset.yaml +++ b/addons/prometheus/exporters/node-exporter/daemonset.yaml @@ -28,21 +28,24 @@ spec: hostPID: true containers: - name: node-exporter - image: quay.io/prometheus/node-exporter:v0.15.2 + image: quay.io/prometheus/node-exporter:v0.17.0 args: - - "--path.procfs=/host/proc" - - "--path.sysfs=/host/sys" + - --path.procfs=/host/proc + - --path.sysfs=/host/sys + - --path.rootfs=/host/root + - --collector.filesystem.ignored-mount-points=^/(dev|proc|sys|var/lib/docker/.+)($|/) + - --collector.filesystem.ignored-fs-types=^(autofs|binfmt_misc|cgroup|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|mqueue|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|sysfs|tracefs)$ ports: - name: metrics containerPort: 9100 hostPort: 9100 resources: requests: - memory: 30Mi cpu: 100m - limits: memory: 50Mi + limits: cpu: 200m + memory: 100Mi volumeMounts: - name: proc mountPath: /host/proc @@ -50,6 +53,9 @@ spec: - name: sys mountPath: /host/sys readOnly: true + - name: root + mountPath: /host/root + readOnly: true tolerations: - effect: NoSchedule operator: Exists @@ -60,3 +66,6 @@ spec: - name: sys hostPath: path: /sys + - name: root + hostPath: + path: / diff --git a/addons/prometheus/rules.yaml b/addons/prometheus/rules.yaml index ed61ecdae..618cd4288 100644 --- a/addons/prometheus/rules.yaml +++ b/addons/prometheus/rules.yaml @@ -456,22 +456,22 @@ data: - name: node.rules rules: - record: instance:node_cpu:rate:sum - expr: sum(rate(node_cpu{mode!="idle",mode!="iowait",mode!~"^(?:guest.*)$"}[3m])) + expr: sum(rate(node_cpu_seconds_total{mode!="idle",mode!="iowait",mode!~"^(?:guest.*)$"}[3m])) BY (instance) - record: instance:node_filesystem_usage:sum - expr: sum((node_filesystem_size{mountpoint="/"} - node_filesystem_free{mountpoint="/"})) + expr: sum((node_filesystem_size_bytes{mountpoint="/"} - node_filesystem_free_bytes{mountpoint="/"})) BY (instance) - record: instance:node_network_receive_bytes:rate:sum - expr: sum(rate(node_network_receive_bytes[3m])) BY (instance) + expr: sum(rate(node_network_receive_bytes_total[3m])) BY (instance) - record: instance:node_network_transmit_bytes:rate:sum - expr: sum(rate(node_network_transmit_bytes[3m])) BY (instance) + expr: sum(rate(node_network_transmit_bytes_total[3m])) BY (instance) - record: instance:node_cpu:ratio - expr: sum(rate(node_cpu{mode!="idle"}[5m])) WITHOUT (cpu, mode) / ON(instance) - GROUP_LEFT() count(sum(node_cpu) BY (instance, cpu)) BY (instance) + expr: sum(rate(node_cpu_seconds_total{mode!="idle"}[5m])) WITHOUT (cpu, mode) / ON(instance) + GROUP_LEFT() count(sum(node_cpu_seconds_total) BY (instance, cpu)) BY (instance) - record: cluster:node_cpu:sum_rate5m - expr: sum(rate(node_cpu{mode!="idle"}[5m])) + expr: sum(rate(node_cpu_seconds_total{mode!="idle"}[5m])) - record: cluster:node_cpu:ratio - expr: cluster:node_cpu:rate5m / count(sum(node_cpu) BY (instance, cpu)) + expr: cluster:node_cpu:sum_rate5m / count(sum(node_cpu_seconds_total) BY (instance, cpu)) - alert: NodeExporterDown expr: absent(up{job="node-exporter"} == 1) for: 10m @@ -481,7 +481,7 @@ data: description: Prometheus could not scrape a node-exporter for more than 10m, or node-exporters have disappeared from discovery - alert: NodeDiskRunningFull - expr: predict_linear(node_filesystem_free[6h], 3600 * 24) < 0 + expr: predict_linear(node_filesystem_free_bytes[6h], 3600 * 24) < 0 for: 30m labels: severity: warning @@ -489,7 +489,7 @@ data: description: device {{$labels.device}} on node {{$labels.instance}} is running full within the next 24 hours (mounted at {{$labels.mountpoint}}) - alert: NodeDiskRunningFull - expr: predict_linear(node_filesystem_free[30m], 3600 * 2) < 0 + expr: predict_linear(node_filesystem_free_bytes[30m], 3600 * 2) < 0 for: 10m labels: severity: critical From 6b87132aa16d9f46e12c105f79f363a9f2f7ac0d Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Sat, 26 Jan 2019 16:35:22 -0800 Subject: [PATCH 007/523] Fix per platform/OS links on the docs home page * Considering the reader of each, the Github README module links can go to module source code and docs module links can go to the associated tutorial docs for the platform/OS --- docs/index.md | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/docs/index.md b/docs/index.md index 6f8c8ce1f..a4892674a 100644 --- a/docs/index.md +++ b/docs/index.md @@ -23,20 +23,20 @@ Typhoon provides a Terraform Module for each supported operating system and plat | Platform | Operating System | Terraform Module | Status | |---------------|------------------|------------------|--------| -| AWS | Container Linux | [aws/container-linux/kubernetes](aws/container-linux/kubernetes) | stable | +| AWS | Container Linux | [aws/container-linux/kubernetes](cl/aws.md) | stable | | Azure | Container Linux | [azure/container-linux/kubernetes](cl/azure.md) | alpha | -| Bare-Metal | Container Linux | [bare-metal/container-linux/kubernetes](bare-metal/container-linux/kubernetes) | stable | -| Digital Ocean | Container Linux | [digital-ocean/container-linux/kubernetes](digital-ocean/container-linux/kubernetes) | beta | -| Google Cloud | Container Linux | [google-cloud/container-linux/kubernetes](google-cloud/container-linux/kubernetes) | stable | +| Bare-Metal | Container Linux | [bare-metal/container-linux/kubernetes](cl/bare-metal.md) | stable | +| Digital Ocean | Container Linux | [digital-ocean/container-linux/kubernetes](cl/digital-ocean.md) | beta | +| Google Cloud | Container Linux | [google-cloud/container-linux/kubernetes](cl/google-cloud.md) | stable | Fedora Atomic support is alpha and will evolve as Fedora Atomic is replaced by Fedora CoreOS. | Platform | Operating System | Terraform Module | Status | |---------------|------------------|------------------|--------| -| AWS | Fedora Atomic | [aws/fedora-atomic/kubernetes](aws/fedora-atomic/kubernetes) | alpha | -| Bare-Metal | Fedora Atomic | [bare-metal/fedora-atomic/kubernetes](bare-metal/fedora-atomic/kubernetes) | alpha | -| Digital Ocean | Fedora Atomic | [digital-ocean/fedora-atomic/kubernetes](digital-ocean/fedora-atomic/kubernetes) | alpha | -| Google Cloud | Fedora Atomic | [google-cloud/fedora-atomic/kubernetes](google-cloud/fedora-atomic/kubernetes) | alpha | +| AWS | Fedora Atomic | [aws/fedora-atomic/kubernetes](atomic/aws.md) | alpha | +| Bare-Metal | Fedora Atomic | [bare-metal/fedora-atomic/kubernetes](atomic/bare-metal.md) | alpha | +| Digital Ocean | Fedora Atomic | [digital-ocean/fedora-atomic/kubernetes](atomic/digital-ocean.md) | alpha | +| Google Cloud | Fedora Atomic | [google-cloud/fedora-atomic/kubernetes](atomic/google-cloud.md) | alpha | ## Documentation From e9659a8539f1a8140061430ad98feaeecc0f4007 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Sun, 27 Jan 2019 16:34:30 -0800 Subject: [PATCH 008/523] Update Calico from v3.4.0 to v3.5.0 * https://docs.projectcalico.org/v3.5/releases/ --- CHANGES.md | 1 + aws/container-linux/kubernetes/bootkube.tf | 2 +- aws/fedora-atomic/kubernetes/bootkube.tf | 2 +- azure/container-linux/kubernetes/bootkube.tf | 2 +- bare-metal/container-linux/kubernetes/bootkube.tf | 2 +- bare-metal/fedora-atomic/kubernetes/bootkube.tf | 2 +- digital-ocean/container-linux/kubernetes/bootkube.tf | 2 +- digital-ocean/fedora-atomic/kubernetes/bootkube.tf | 2 +- google-cloud/container-linux/kubernetes/bootkube.tf | 2 +- google-cloud/fedora-atomic/kubernetes/bootkube.tf | 2 +- 10 files changed, 10 insertions(+), 9 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index f26c5dae6..efc9bdb5b 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -6,6 +6,7 @@ Notable changes between versions. * Update etcd from v3.3.10 to [v3.3.11](https://github.com/etcd-io/etcd/blob/master/CHANGELOG-3.3.md#v3311-2019-1-11) * Update CoreDNS from v1.3.0 to [v1.3.1](https://coredns.io/2019/01/13/coredns-1.3.1-release/) +* Update Calico from v3.4.0 to [v3.5.0](https://docs.projectcalico.org/v3.5/releases/) * Fix automatic worker deletion on shutdown for cloud platforms * Lowering Kubelet privileges in [#372](https://github.com/poseidon/typhoon/pull/372) dropped a needed node deletion authorization. Scale-in due to manual terraform apply (any cloud), AWS spot termination, or Azure low priority deletion left old nodes registered, requiring manual deletion (`kubectl delete node name`) diff --git a/aws/container-linux/kubernetes/bootkube.tf b/aws/container-linux/kubernetes/bootkube.tf index 2c26f75ae..fd4b586ad 100644 --- a/aws/container-linux/kubernetes/bootkube.tf +++ b/aws/container-linux/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=ef99293eb2b5fd8ba0bb19fb49e795899dbe690d" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=54f15b6c8c4fa14a3f8ae3f58036a7b51e47ef87" cluster_name = "${var.cluster_name}" api_servers = ["${format("%s.%s", var.cluster_name, var.dns_zone)}"] diff --git a/aws/fedora-atomic/kubernetes/bootkube.tf b/aws/fedora-atomic/kubernetes/bootkube.tf index 0ddfee8c0..8ed96511b 100644 --- a/aws/fedora-atomic/kubernetes/bootkube.tf +++ b/aws/fedora-atomic/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=ef99293eb2b5fd8ba0bb19fb49e795899dbe690d" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=54f15b6c8c4fa14a3f8ae3f58036a7b51e47ef87" cluster_name = "${var.cluster_name}" api_servers = ["${format("%s.%s", var.cluster_name, var.dns_zone)}"] diff --git a/azure/container-linux/kubernetes/bootkube.tf b/azure/container-linux/kubernetes/bootkube.tf index dd19b97fb..451e75ead 100644 --- a/azure/container-linux/kubernetes/bootkube.tf +++ b/azure/container-linux/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=ef99293eb2b5fd8ba0bb19fb49e795899dbe690d" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=54f15b6c8c4fa14a3f8ae3f58036a7b51e47ef87" cluster_name = "${var.cluster_name}" api_servers = ["${format("%s.%s", var.cluster_name, var.dns_zone)}"] diff --git a/bare-metal/container-linux/kubernetes/bootkube.tf b/bare-metal/container-linux/kubernetes/bootkube.tf index 573d96b75..8da77d3d0 100644 --- a/bare-metal/container-linux/kubernetes/bootkube.tf +++ b/bare-metal/container-linux/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=ef99293eb2b5fd8ba0bb19fb49e795899dbe690d" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=54f15b6c8c4fa14a3f8ae3f58036a7b51e47ef87" cluster_name = "${var.cluster_name}" api_servers = ["${var.k8s_domain_name}"] diff --git a/bare-metal/fedora-atomic/kubernetes/bootkube.tf b/bare-metal/fedora-atomic/kubernetes/bootkube.tf index ad75bab58..de328a56f 100644 --- a/bare-metal/fedora-atomic/kubernetes/bootkube.tf +++ b/bare-metal/fedora-atomic/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=ef99293eb2b5fd8ba0bb19fb49e795899dbe690d" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=54f15b6c8c4fa14a3f8ae3f58036a7b51e47ef87" cluster_name = "${var.cluster_name}" api_servers = ["${var.k8s_domain_name}"] diff --git a/digital-ocean/container-linux/kubernetes/bootkube.tf b/digital-ocean/container-linux/kubernetes/bootkube.tf index 023fa204e..719c751ab 100644 --- a/digital-ocean/container-linux/kubernetes/bootkube.tf +++ b/digital-ocean/container-linux/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=ef99293eb2b5fd8ba0bb19fb49e795899dbe690d" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=54f15b6c8c4fa14a3f8ae3f58036a7b51e47ef87" cluster_name = "${var.cluster_name}" api_servers = ["${format("%s.%s", var.cluster_name, var.dns_zone)}"] diff --git a/digital-ocean/fedora-atomic/kubernetes/bootkube.tf b/digital-ocean/fedora-atomic/kubernetes/bootkube.tf index 195e98523..1d80da926 100644 --- a/digital-ocean/fedora-atomic/kubernetes/bootkube.tf +++ b/digital-ocean/fedora-atomic/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=ef99293eb2b5fd8ba0bb19fb49e795899dbe690d" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=54f15b6c8c4fa14a3f8ae3f58036a7b51e47ef87" cluster_name = "${var.cluster_name}" api_servers = ["${format("%s.%s", var.cluster_name, var.dns_zone)}"] diff --git a/google-cloud/container-linux/kubernetes/bootkube.tf b/google-cloud/container-linux/kubernetes/bootkube.tf index 92513ff26..c58e20de6 100644 --- a/google-cloud/container-linux/kubernetes/bootkube.tf +++ b/google-cloud/container-linux/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=ef99293eb2b5fd8ba0bb19fb49e795899dbe690d" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=54f15b6c8c4fa14a3f8ae3f58036a7b51e47ef87" cluster_name = "${var.cluster_name}" api_servers = ["${format("%s.%s", var.cluster_name, var.dns_zone)}"] diff --git a/google-cloud/fedora-atomic/kubernetes/bootkube.tf b/google-cloud/fedora-atomic/kubernetes/bootkube.tf index 4fbccb789..59a6db94e 100644 --- a/google-cloud/fedora-atomic/kubernetes/bootkube.tf +++ b/google-cloud/fedora-atomic/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=ef99293eb2b5fd8ba0bb19fb49e795899dbe690d" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=54f15b6c8c4fa14a3f8ae3f58036a7b51e47ef87" cluster_name = "${var.cluster_name}" api_servers = ["${format("%s.%s", var.cluster_name, var.dns_zone)}"] From eb08593eae2df58b898b6d1346a26d6ef7ff28ac Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Sun, 27 Jan 2019 17:52:35 -0800 Subject: [PATCH 009/523] Fix azure provider warning, rename a public_ip field * azurerm_public_ip (used internally) added a field `allocation_method` to replace the field `public_ip_address_allocation` (deprecated) * Require terraform-provider-azurerm v1.21+ * https://github.com/terraform-providers/terraform-provider-azurerm/pull/2576 --- CHANGES.md | 5 +++++ azure/container-linux/kubernetes/controllers.tf | 8 ++++---- azure/container-linux/kubernetes/lb.tf | 16 ++++++++-------- azure/container-linux/kubernetes/require.tf | 2 +- 4 files changed, 18 insertions(+), 13 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index efc9bdb5b..eac5c847b 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -14,6 +14,11 @@ Notable changes between versions. * Add `ingress_zone_id` output with the NLB DNS name's Route53 zone for use in alias records ([#380](https://github.com/poseidon/typhoon/pull/380)) +#### Azure + +* Fix azure provider warning, `public_ip` `allocation_method` replaces `public_ip_address_allocation` + * Require `terraform-provider-azurerm` v1.21+ (action required) + #### Addons * Update nginx-ingress from v0.21.0 to v0.22.0 diff --git a/azure/container-linux/kubernetes/controllers.tf b/azure/container-linux/kubernetes/controllers.tf index b929b7789..ffa05a1ec 100644 --- a/azure/container-linux/kubernetes/controllers.tf +++ b/azure/container-linux/kubernetes/controllers.tf @@ -121,10 +121,10 @@ resource "azurerm_public_ip" "controllers" { count = "${var.controller_count}" resource_group_name = "${azurerm_resource_group.cluster.name}" - name = "${var.cluster_name}-controller-${count.index}" - location = "${azurerm_resource_group.cluster.location}" - sku = "Standard" - public_ip_address_allocation = "static" + name = "${var.cluster_name}-controller-${count.index}" + location = "${azurerm_resource_group.cluster.location}" + sku = "Standard" + allocation_method = "Static" } # Controller Ignition configs diff --git a/azure/container-linux/kubernetes/lb.tf b/azure/container-linux/kubernetes/lb.tf index c15b5906b..d010937fc 100644 --- a/azure/container-linux/kubernetes/lb.tf +++ b/azure/container-linux/kubernetes/lb.tf @@ -17,20 +17,20 @@ resource "azurerm_dns_a_record" "apiserver" { resource "azurerm_public_ip" "apiserver-ipv4" { resource_group_name = "${azurerm_resource_group.cluster.name}" - name = "${var.cluster_name}-apiserver-ipv4" - location = "${var.region}" - sku = "Standard" - public_ip_address_allocation = "static" + name = "${var.cluster_name}-apiserver-ipv4" + location = "${var.region}" + sku = "Standard" + allocation_method = "Static" } # Static IPv4 address for the ingress frontend resource "azurerm_public_ip" "ingress-ipv4" { resource_group_name = "${azurerm_resource_group.cluster.name}" - name = "${var.cluster_name}-ingress-ipv4" - location = "${var.region}" - sku = "Standard" - public_ip_address_allocation = "static" + name = "${var.cluster_name}-ingress-ipv4" + location = "${var.region}" + sku = "Standard" + allocation_method = "Static" } # Network Load Balancer for apiservers and ingress diff --git a/azure/container-linux/kubernetes/require.tf b/azure/container-linux/kubernetes/require.tf index 5d350636e..f85cf7b49 100644 --- a/azure/container-linux/kubernetes/require.tf +++ b/azure/container-linux/kubernetes/require.tf @@ -5,7 +5,7 @@ terraform { } provider "azurerm" { - version = "~> 1.19" + version = "~> 1.21" } provider "local" { From 1ab06f69d7db0324a8069090cc83ea3b1d45e790 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Tue, 29 Jan 2019 21:51:25 -0800 Subject: [PATCH 010/523] Update flannel from v0.10.0 to v0.11.0 * https://github.com/coreos/flannel/releases/tag/v0.11.0 --- CHANGES.md | 1 + aws/container-linux/kubernetes/bootkube.tf | 2 +- aws/fedora-atomic/kubernetes/bootkube.tf | 2 +- azure/container-linux/kubernetes/bootkube.tf | 2 +- bare-metal/container-linux/kubernetes/bootkube.tf | 2 +- bare-metal/fedora-atomic/kubernetes/bootkube.tf | 2 +- digital-ocean/container-linux/kubernetes/bootkube.tf | 2 +- digital-ocean/fedora-atomic/kubernetes/bootkube.tf | 2 +- google-cloud/container-linux/kubernetes/bootkube.tf | 2 +- google-cloud/fedora-atomic/kubernetes/bootkube.tf | 2 +- 10 files changed, 10 insertions(+), 9 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index eac5c847b..db0b3b3e1 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -7,6 +7,7 @@ Notable changes between versions. * Update etcd from v3.3.10 to [v3.3.11](https://github.com/etcd-io/etcd/blob/master/CHANGELOG-3.3.md#v3311-2019-1-11) * Update CoreDNS from v1.3.0 to [v1.3.1](https://coredns.io/2019/01/13/coredns-1.3.1-release/) * Update Calico from v3.4.0 to [v3.5.0](https://docs.projectcalico.org/v3.5/releases/) +* Update flannel from v0.10.0 to [v0.11.0](https://github.com/coreos/flannel/releases/tag/v0.11.0) * Fix automatic worker deletion on shutdown for cloud platforms * Lowering Kubelet privileges in [#372](https://github.com/poseidon/typhoon/pull/372) dropped a needed node deletion authorization. Scale-in due to manual terraform apply (any cloud), AWS spot termination, or Azure low priority deletion left old nodes registered, requiring manual deletion (`kubectl delete node name`) diff --git a/aws/container-linux/kubernetes/bootkube.tf b/aws/container-linux/kubernetes/bootkube.tf index fd4b586ad..d45685bcc 100644 --- a/aws/container-linux/kubernetes/bootkube.tf +++ b/aws/container-linux/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=54f15b6c8c4fa14a3f8ae3f58036a7b51e47ef87" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=c5bc23ef7a3f5d8e5be976ee60b58dbbfcc3e7c9" cluster_name = "${var.cluster_name}" api_servers = ["${format("%s.%s", var.cluster_name, var.dns_zone)}"] diff --git a/aws/fedora-atomic/kubernetes/bootkube.tf b/aws/fedora-atomic/kubernetes/bootkube.tf index 8ed96511b..15aac5764 100644 --- a/aws/fedora-atomic/kubernetes/bootkube.tf +++ b/aws/fedora-atomic/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=54f15b6c8c4fa14a3f8ae3f58036a7b51e47ef87" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=c5bc23ef7a3f5d8e5be976ee60b58dbbfcc3e7c9" cluster_name = "${var.cluster_name}" api_servers = ["${format("%s.%s", var.cluster_name, var.dns_zone)}"] diff --git a/azure/container-linux/kubernetes/bootkube.tf b/azure/container-linux/kubernetes/bootkube.tf index 451e75ead..02033b2d1 100644 --- a/azure/container-linux/kubernetes/bootkube.tf +++ b/azure/container-linux/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=54f15b6c8c4fa14a3f8ae3f58036a7b51e47ef87" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=c5bc23ef7a3f5d8e5be976ee60b58dbbfcc3e7c9" cluster_name = "${var.cluster_name}" api_servers = ["${format("%s.%s", var.cluster_name, var.dns_zone)}"] diff --git a/bare-metal/container-linux/kubernetes/bootkube.tf b/bare-metal/container-linux/kubernetes/bootkube.tf index 8da77d3d0..d593feb42 100644 --- a/bare-metal/container-linux/kubernetes/bootkube.tf +++ b/bare-metal/container-linux/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=54f15b6c8c4fa14a3f8ae3f58036a7b51e47ef87" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=c5bc23ef7a3f5d8e5be976ee60b58dbbfcc3e7c9" cluster_name = "${var.cluster_name}" api_servers = ["${var.k8s_domain_name}"] diff --git a/bare-metal/fedora-atomic/kubernetes/bootkube.tf b/bare-metal/fedora-atomic/kubernetes/bootkube.tf index de328a56f..286f1323e 100644 --- a/bare-metal/fedora-atomic/kubernetes/bootkube.tf +++ b/bare-metal/fedora-atomic/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=54f15b6c8c4fa14a3f8ae3f58036a7b51e47ef87" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=c5bc23ef7a3f5d8e5be976ee60b58dbbfcc3e7c9" cluster_name = "${var.cluster_name}" api_servers = ["${var.k8s_domain_name}"] diff --git a/digital-ocean/container-linux/kubernetes/bootkube.tf b/digital-ocean/container-linux/kubernetes/bootkube.tf index 719c751ab..cc29a55be 100644 --- a/digital-ocean/container-linux/kubernetes/bootkube.tf +++ b/digital-ocean/container-linux/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=54f15b6c8c4fa14a3f8ae3f58036a7b51e47ef87" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=c5bc23ef7a3f5d8e5be976ee60b58dbbfcc3e7c9" cluster_name = "${var.cluster_name}" api_servers = ["${format("%s.%s", var.cluster_name, var.dns_zone)}"] diff --git a/digital-ocean/fedora-atomic/kubernetes/bootkube.tf b/digital-ocean/fedora-atomic/kubernetes/bootkube.tf index 1d80da926..3c153cc69 100644 --- a/digital-ocean/fedora-atomic/kubernetes/bootkube.tf +++ b/digital-ocean/fedora-atomic/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=54f15b6c8c4fa14a3f8ae3f58036a7b51e47ef87" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=c5bc23ef7a3f5d8e5be976ee60b58dbbfcc3e7c9" cluster_name = "${var.cluster_name}" api_servers = ["${format("%s.%s", var.cluster_name, var.dns_zone)}"] diff --git a/google-cloud/container-linux/kubernetes/bootkube.tf b/google-cloud/container-linux/kubernetes/bootkube.tf index c58e20de6..5e9ddc01a 100644 --- a/google-cloud/container-linux/kubernetes/bootkube.tf +++ b/google-cloud/container-linux/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=54f15b6c8c4fa14a3f8ae3f58036a7b51e47ef87" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=c5bc23ef7a3f5d8e5be976ee60b58dbbfcc3e7c9" cluster_name = "${var.cluster_name}" api_servers = ["${format("%s.%s", var.cluster_name, var.dns_zone)}"] diff --git a/google-cloud/fedora-atomic/kubernetes/bootkube.tf b/google-cloud/fedora-atomic/kubernetes/bootkube.tf index 59a6db94e..f8bbec7ad 100644 --- a/google-cloud/fedora-atomic/kubernetes/bootkube.tf +++ b/google-cloud/fedora-atomic/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=54f15b6c8c4fa14a3f8ae3f58036a7b51e47ef87" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=c5bc23ef7a3f5d8e5be976ee60b58dbbfcc3e7c9" cluster_name = "${var.cluster_name}" api_servers = ["${format("%s.%s", var.cluster_name, var.dns_zone)}"] From 130daeac2639bfb2a30abf628dbfccf2f3b50b60 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Tue, 29 Jan 2019 22:30:47 -0800 Subject: [PATCH 011/523] Update Prometheus from v2.6.1 to v2.7.0 --- CHANGES.md | 2 +- addons/prometheus/deployment.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index db0b3b3e1..5de8b3322 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -23,7 +23,7 @@ Notable changes between versions. #### Addons * Update nginx-ingress from v0.21.0 to v0.22.0 -* Update Prometheus from v2.6.0 to v2.6.1 +* Update Prometheus from v2.6.0 to v2.7.0 * Update kube-state-metrics from v1.4.0 to v1.5.0 * Fix ClusterRole to collect and export PodDisruptionBudget metrics ([#383](https://github.com/poseidon/typhoon/pull/383)) * Update node-exporter from v0.15.2 to v0.17.0 diff --git a/addons/prometheus/deployment.yaml b/addons/prometheus/deployment.yaml index 638eca1f8..380a222eb 100644 --- a/addons/prometheus/deployment.yaml +++ b/addons/prometheus/deployment.yaml @@ -20,7 +20,7 @@ spec: serviceAccountName: prometheus containers: - name: prometheus - image: quay.io/prometheus/prometheus:v2.6.1 + image: quay.io/prometheus/prometheus:v2.7.0 args: - --web.listen-address=0.0.0.0:9090 - --config.file=/etc/prometheus/prometheus.yaml From d02af3d40d1dcfc42d0d4a0400d52ee16d8575a2 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Tue, 29 Jan 2019 22:56:03 -0800 Subject: [PATCH 012/523] Update mkdocs-material from v3.2.0 to v3.3.0 * Fix minor docs typos and errors * Allow a transient verison of the six PyPi package, the docs build system can use the 0.12.0 (0.11.0 broke sync tools so pinning to 0.10.0 was previously needed) --- docs/addons/heapster.md | 2 +- docs/announce.md | 2 +- docs/atomic/digital-ocean.md | 2 +- docs/cl/aws.md | 2 +- docs/cl/azure.md | 2 +- docs/cl/digital-ocean.md | 2 +- docs/index.md | 4 ++-- requirements.txt | 3 +-- 8 files changed, 9 insertions(+), 10 deletions(-) diff --git a/docs/addons/heapster.md b/docs/addons/heapster.md index 7078bcf32..e10135446 100644 --- a/docs/addons/heapster.md +++ b/docs/addons/heapster.md @@ -1,6 +1,6 @@ # Heapster -[Heapster](https://kubernetes.io/docs/user-guide/monitoring/) collects data from apiservers and kubelets and exposes it through a REST API. This API powers the `kubectl top` command and Kubernetes dashboard graphs. +[Heapster](https://kubernetes.io/docs/user-guide/monitoring/) collects data from apiservers and kubelets and exposes it through a REST API. This API powers the `kubectl top` command. ## Create diff --git a/docs/announce.md b/docs/announce.md index c1988966d..ab1467cb4 100644 --- a/docs/announce.md +++ b/docs/announce.md @@ -18,7 +18,7 @@ Fedora Atomic is a container-optimized operating system designed for large-scale For newcomers, Typhoon is a free (cost and freedom) Kubernetes distribution providing upstream Kubernetes, declarative configuration via [Terraform](https://www.terraform.io/intro/index.html), and support for AWS, Google Cloud, DigitalOcean, and bare-metal. Typhoon clusters use a [self-hosted](https://github.com/kubernetes-incubator/bootkube) control plane, support [Calico](https://www.projectcalico.org/blog/) and [flannel](https://coreos.com/flannel/docs/latest/) CNI networking, and enable etcd TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/), and network policy. -Typhoon for Fedora Atomic reflects many of the same principles that created Typhoon for Container Linux. Clusters are declared using plain Terraform configs that can be versioned. In lieu of Ignition, instances are declaratively provisioned with Cloud-Init and kickstart (bare-metal only). TLS assets are generated. Hosts run only a kubelet service, other components are scheduled (i.e. self-hosted). The upstream hyperkube is used directly[^1]. And clusters are kept minimal by offering optional addons for [Ingress](https://typhoon.psdn.io/addons/ingress/), [Prometheus](https://typhoon.psdn.io/addons/prometheus/), and [Grafana](https://typhoon.psdn.io/addons/grafana/). Typhoon compliments and enhances Fedora Atomic as a choice of operating system for Kubernetes. +Typhoon for Fedora Atomic reflects many of the same principles that created Typhoon for Container Linux. Clusters are declared using plain Terraform configs that can be versioned. In lieu of Ignition, instances are declaratively provisioned with Cloud-Init and kickstart (bare-metal only). TLS assets are generated. Hosts run only a kubelet service, other components are scheduled (i.e. self-hosted). The upstream hyperkube is used directly[^1]. And clusters are kept minimal by offering optional addons for [Ingress](/addons/ingress/), [Prometheus](/addons/prometheus/), and [Grafana](/addons/grafana/). Typhoon compliments and enhances Fedora Atomic as a choice of operating system for Kubernetes. Meanwhile, Fedora Atomic adds some promising new low-level technologies: diff --git a/docs/atomic/digital-ocean.md b/docs/atomic/digital-ocean.md index bf9c18f19..db78c16e4 100644 --- a/docs/atomic/digital-ocean.md +++ b/docs/atomic/digital-ocean.md @@ -204,9 +204,9 @@ Clusters create DNS A records `${cluster_name}.${dns_zone}` to resolve to contro You'll need a registered domain name or delegated subdomain in Digital Ocean Domains (i.e. DNS zones). You can set this up once and create many clusters with unique names. ```tf +# Declare a DigitalOcean record to also create a zone file resource "digitalocean_domain" "zone-for-clusters" { name = "do.example.com" - # Digital Ocean oddly requires an IP here. You may have to delete the A record it makes. :( ip_address = "8.8.8.8" } ``` diff --git a/docs/cl/aws.md b/docs/cl/aws.md index 7b1421de5..d3a6eb5f1 100644 --- a/docs/cl/aws.md +++ b/docs/cl/aws.md @@ -113,7 +113,7 @@ module "aws-tempest" { # optional worker_count = 2 - worker_type = "t2.medium" + worker_type = "t3.small" } ``` diff --git a/docs/cl/azure.md b/docs/cl/azure.md index bedf0082f..6f8d94002 100644 --- a/docs/cl/azure.md +++ b/docs/cl/azure.md @@ -249,7 +249,7 @@ Reference the DNS zone with `"${azurerm_dns_zone.clusters.name}"` and its resour | controller_type | Machine type for controllers | "Standard_DS1_v2" | See below | | worker_type | Machine type for workers | "Standard_F1" | See below | | os_image | Channel for a Container Linux derivative | coreos-stable | coreos-stable, coreos-beta, coreos-alpha | -| disk_size | Size of the disk GB | "40" | "100" | +| disk_size | Size of the disk in GB | "40" | "100" | | worker_priority | Set priority to Low to use reduced cost surplus capacity, with the tradeoff that instances can be deallocated at any time | Regular | Low | | controller_clc_snippets | Controller Container Linux Config snippets | [] | [example](/advanced/customization/#usage) | | worker_clc_snippets | Worker Container Linux Config snippets | [] | [example](/advanced/customization/#usage) | diff --git a/docs/cl/digital-ocean.md b/docs/cl/digital-ocean.md index a28c4c902..a70704f70 100644 --- a/docs/cl/digital-ocean.md +++ b/docs/cl/digital-ocean.md @@ -214,9 +214,9 @@ Clusters create DNS A records `${cluster_name}.${dns_zone}` to resolve to contro You'll need a registered domain name or delegated subdomain in Digital Ocean Domains (i.e. DNS zones). You can set this up once and create many clusters with unique names. ```tf +# Declare a DigitalOcean record to also create a zone file resource "digitalocean_domain" "zone-for-clusters" { name = "do.example.com" - # Digital Ocean oddly requires an IP here. You may have to delete the A record it makes. :( ip_address = "8.8.8.8" } ``` diff --git a/docs/index.md b/docs/index.md index a4892674a..f1d8ceed1 100644 --- a/docs/index.md +++ b/docs/index.md @@ -14,8 +14,8 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster * Kubernetes v1.13.2 (upstream, via [kubernetes-incubator/bootkube](https://github.com/kubernetes-incubator/bootkube)) * Single or multi-master, [Calico](https://www.projectcalico.org/) or [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/) -* Advanced features like [worker pools](https://typhoon.psdn.io/advanced/worker-pools/), [preemptible](https://typhoon.psdn.io/cl/google-cloud/#preemption) workers, and [snippets](https://typhoon.psdn.io/advanced/customization/#container-linux) customization -* Ready for Ingress, Prometheus, Grafana, CSI, or other [addons](https://typhoon.psdn.io/addons/overview/) +* Advanced features like [worker pools](advanced/worker-pools/), [preemptible](cl/google-cloud/#preemption) workers, and [snippets](advanced/customization/#container-linux) customization +* Ready for Ingress, Prometheus, Grafana, CSI, or other [addons](addons/overview/) ## Modules diff --git a/requirements.txt b/requirements.txt index 9fd009416..fc78f3c9e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,4 @@ mkdocs==1.0.4 -mkdocs-material==3.2.0 +mkdocs-material==3.3.0 pygments==2.2.0 pymdown-extensions==5.0.0 -six==1.10.0 From 244a1a601a187b97bfeb28e4306279d4291ecb37 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Wed, 30 Jan 2019 22:25:23 -0800 Subject: [PATCH 013/523] Switch CoreDNS to use the forward plugin instead of proxy * Use the forward plugin to forward to upstream resolvers, instead of the proxy plugin. The forward plugin is reported to be a faster alternative since it can re-use open sockets * https://coredns.io/explugins/forward/ * https://coredns.io/plugins/proxy/ * https://github.com/kubernetes/kubernetes/issues/73254 --- CHANGES.md | 1 + aws/container-linux/kubernetes/bootkube.tf | 2 +- aws/fedora-atomic/kubernetes/bootkube.tf | 2 +- azure/container-linux/kubernetes/bootkube.tf | 2 +- bare-metal/container-linux/kubernetes/bootkube.tf | 2 +- bare-metal/fedora-atomic/kubernetes/bootkube.tf | 2 +- digital-ocean/container-linux/kubernetes/bootkube.tf | 2 +- digital-ocean/fedora-atomic/kubernetes/bootkube.tf | 2 +- google-cloud/container-linux/kubernetes/bootkube.tf | 2 +- google-cloud/fedora-atomic/kubernetes/bootkube.tf | 2 +- 10 files changed, 10 insertions(+), 9 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 5de8b3322..5ab6988ea 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -6,6 +6,7 @@ Notable changes between versions. * Update etcd from v3.3.10 to [v3.3.11](https://github.com/etcd-io/etcd/blob/master/CHANGELOG-3.3.md#v3311-2019-1-11) * Update CoreDNS from v1.3.0 to [v1.3.1](https://coredns.io/2019/01/13/coredns-1.3.1-release/) + * Switch from the `proxy` plugin to the faster `forward` plugin for upsteam resolvers * Update Calico from v3.4.0 to [v3.5.0](https://docs.projectcalico.org/v3.5/releases/) * Update flannel from v0.10.0 to [v0.11.0](https://github.com/coreos/flannel/releases/tag/v0.11.0) * Fix automatic worker deletion on shutdown for cloud platforms diff --git a/aws/container-linux/kubernetes/bootkube.tf b/aws/container-linux/kubernetes/bootkube.tf index d45685bcc..a9455d932 100644 --- a/aws/container-linux/kubernetes/bootkube.tf +++ b/aws/container-linux/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=c5bc23ef7a3f5d8e5be976ee60b58dbbfcc3e7c9" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=7dc8f8bf8c1a6bdb9170dd19836520ce436e26d7" cluster_name = "${var.cluster_name}" api_servers = ["${format("%s.%s", var.cluster_name, var.dns_zone)}"] diff --git a/aws/fedora-atomic/kubernetes/bootkube.tf b/aws/fedora-atomic/kubernetes/bootkube.tf index 15aac5764..b33839d64 100644 --- a/aws/fedora-atomic/kubernetes/bootkube.tf +++ b/aws/fedora-atomic/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=c5bc23ef7a3f5d8e5be976ee60b58dbbfcc3e7c9" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=7dc8f8bf8c1a6bdb9170dd19836520ce436e26d7" cluster_name = "${var.cluster_name}" api_servers = ["${format("%s.%s", var.cluster_name, var.dns_zone)}"] diff --git a/azure/container-linux/kubernetes/bootkube.tf b/azure/container-linux/kubernetes/bootkube.tf index 02033b2d1..ea386d24b 100644 --- a/azure/container-linux/kubernetes/bootkube.tf +++ b/azure/container-linux/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=c5bc23ef7a3f5d8e5be976ee60b58dbbfcc3e7c9" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=7dc8f8bf8c1a6bdb9170dd19836520ce436e26d7" cluster_name = "${var.cluster_name}" api_servers = ["${format("%s.%s", var.cluster_name, var.dns_zone)}"] diff --git a/bare-metal/container-linux/kubernetes/bootkube.tf b/bare-metal/container-linux/kubernetes/bootkube.tf index d593feb42..34ba39cfc 100644 --- a/bare-metal/container-linux/kubernetes/bootkube.tf +++ b/bare-metal/container-linux/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=c5bc23ef7a3f5d8e5be976ee60b58dbbfcc3e7c9" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=7dc8f8bf8c1a6bdb9170dd19836520ce436e26d7" cluster_name = "${var.cluster_name}" api_servers = ["${var.k8s_domain_name}"] diff --git a/bare-metal/fedora-atomic/kubernetes/bootkube.tf b/bare-metal/fedora-atomic/kubernetes/bootkube.tf index 286f1323e..8a84cc932 100644 --- a/bare-metal/fedora-atomic/kubernetes/bootkube.tf +++ b/bare-metal/fedora-atomic/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=c5bc23ef7a3f5d8e5be976ee60b58dbbfcc3e7c9" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=7dc8f8bf8c1a6bdb9170dd19836520ce436e26d7" cluster_name = "${var.cluster_name}" api_servers = ["${var.k8s_domain_name}"] diff --git a/digital-ocean/container-linux/kubernetes/bootkube.tf b/digital-ocean/container-linux/kubernetes/bootkube.tf index cc29a55be..a340e9669 100644 --- a/digital-ocean/container-linux/kubernetes/bootkube.tf +++ b/digital-ocean/container-linux/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=c5bc23ef7a3f5d8e5be976ee60b58dbbfcc3e7c9" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=7dc8f8bf8c1a6bdb9170dd19836520ce436e26d7" cluster_name = "${var.cluster_name}" api_servers = ["${format("%s.%s", var.cluster_name, var.dns_zone)}"] diff --git a/digital-ocean/fedora-atomic/kubernetes/bootkube.tf b/digital-ocean/fedora-atomic/kubernetes/bootkube.tf index 3c153cc69..de02385b1 100644 --- a/digital-ocean/fedora-atomic/kubernetes/bootkube.tf +++ b/digital-ocean/fedora-atomic/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=c5bc23ef7a3f5d8e5be976ee60b58dbbfcc3e7c9" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=7dc8f8bf8c1a6bdb9170dd19836520ce436e26d7" cluster_name = "${var.cluster_name}" api_servers = ["${format("%s.%s", var.cluster_name, var.dns_zone)}"] diff --git a/google-cloud/container-linux/kubernetes/bootkube.tf b/google-cloud/container-linux/kubernetes/bootkube.tf index 5e9ddc01a..a9b18f8cd 100644 --- a/google-cloud/container-linux/kubernetes/bootkube.tf +++ b/google-cloud/container-linux/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=c5bc23ef7a3f5d8e5be976ee60b58dbbfcc3e7c9" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=7dc8f8bf8c1a6bdb9170dd19836520ce436e26d7" cluster_name = "${var.cluster_name}" api_servers = ["${format("%s.%s", var.cluster_name, var.dns_zone)}"] diff --git a/google-cloud/fedora-atomic/kubernetes/bootkube.tf b/google-cloud/fedora-atomic/kubernetes/bootkube.tf index f8bbec7ad..842e60326 100644 --- a/google-cloud/fedora-atomic/kubernetes/bootkube.tf +++ b/google-cloud/fedora-atomic/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=c5bc23ef7a3f5d8e5be976ee60b58dbbfcc3e7c9" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=7dc8f8bf8c1a6bdb9170dd19836520ce436e26d7" cluster_name = "${var.cluster_name}" api_servers = ["${format("%s.%s", var.cluster_name, var.dns_zone)}"] From acd539f8658df671c82138adc3d3643aca124e8c Mon Sep 17 00:00:00 2001 From: Carlos Cobo Date: Sat, 2 Feb 2019 08:20:06 +0100 Subject: [PATCH 014/523] Fix architecture title for DigitalOcean (#390) --- docs/architecture/digitalocean.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/architecture/digitalocean.md b/docs/architecture/digitalocean.md index d0305e2d4..f6f328fc2 100644 --- a/docs/architecture/digitalocean.md +++ b/docs/architecture/digitalocean.md @@ -1,4 +1,4 @@ -# AWS +# DigitalOcean ## IPv6 From ccd96c37daa56b2bdd6ec0b98d692aebd949c110 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Fri, 1 Feb 2019 23:26:13 -0800 Subject: [PATCH 015/523] Update Kubernetes from v1.13.2 to v1.13.3 * https://github.com/kubernetes/kubernetes/blob/master/CHANGELOG-1.13.md#v1133 --- CHANGES.md | 1 + README.md | 10 +++++----- aws/container-linux/kubernetes/README.md | 2 +- aws/container-linux/kubernetes/bootkube.tf | 2 +- .../kubernetes/cl/controller.yaml.tmpl | 2 +- .../kubernetes/workers/cl/worker.yaml.tmpl | 4 ++-- aws/fedora-atomic/kubernetes/README.md | 2 +- aws/fedora-atomic/kubernetes/bootkube.tf | 2 +- .../kubernetes/cloudinit/controller.yaml.tmpl | 2 +- .../workers/cloudinit/worker.yaml.tmpl | 2 +- azure/container-linux/kubernetes/README.md | 2 +- azure/container-linux/kubernetes/bootkube.tf | 2 +- .../kubernetes/cl/controller.yaml.tmpl | 2 +- .../kubernetes/workers/cl/worker.yaml.tmpl | 4 ++-- bare-metal/container-linux/kubernetes/README.md | 2 +- .../container-linux/kubernetes/bootkube.tf | 2 +- .../kubernetes/cl/controller.yaml.tmpl | 2 +- .../kubernetes/cl/worker.yaml.tmpl | 2 +- bare-metal/fedora-atomic/kubernetes/README.md | 2 +- bare-metal/fedora-atomic/kubernetes/bootkube.tf | 2 +- .../kubernetes/cloudinit/controller.yaml.tmpl | 2 +- .../kubernetes/cloudinit/worker.yaml.tmpl | 2 +- .../container-linux/kubernetes/README.md | 2 +- .../container-linux/kubernetes/bootkube.tf | 2 +- .../kubernetes/cl/controller.yaml.tmpl | 2 +- .../kubernetes/cl/worker.yaml.tmpl | 4 ++-- digital-ocean/fedora-atomic/kubernetes/README.md | 2 +- .../fedora-atomic/kubernetes/bootkube.tf | 2 +- .../kubernetes/cloudinit/controller.yaml.tmpl | 2 +- .../kubernetes/cloudinit/worker.yaml.tmpl | 2 +- docs/advanced/worker-pools.md | 16 ++++++++-------- docs/atomic/aws.md | 10 +++++----- docs/atomic/bare-metal.md | 10 +++++----- docs/atomic/digital-ocean.md | 10 +++++----- docs/atomic/google-cloud.md | 10 +++++----- docs/cl/aws.md | 10 +++++----- docs/cl/azure.md | 10 +++++----- docs/cl/bare-metal.md | 14 +++++++------- docs/cl/digital-ocean.md | 10 +++++----- docs/cl/google-cloud.md | 10 +++++----- docs/index.md | 10 +++++----- docs/topics/maintenance.md | 2 +- .../container-linux/kubernetes/README.md | 2 +- .../container-linux/kubernetes/bootkube.tf | 2 +- .../kubernetes/cl/controller.yaml.tmpl | 2 +- .../kubernetes/workers/cl/worker.yaml.tmpl | 4 ++-- google-cloud/fedora-atomic/kubernetes/README.md | 2 +- .../fedora-atomic/kubernetes/bootkube.tf | 2 +- .../kubernetes/cloudinit/controller.yaml.tmpl | 2 +- .../workers/cloudinit/worker.yaml.tmpl | 2 +- 50 files changed, 107 insertions(+), 106 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 5ab6988ea..55675a54d 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -4,6 +4,7 @@ Notable changes between versions. ## Latest +* Kubernetes [v1.13.3](https://github.com/kubernetes/kubernetes/blob/master/CHANGELOG-1.13.md#v1133) * Update etcd from v3.3.10 to [v3.3.11](https://github.com/etcd-io/etcd/blob/master/CHANGELOG-3.3.md#v3311-2019-1-11) * Update CoreDNS from v1.3.0 to [v1.3.1](https://coredns.io/2019/01/13/coredns-1.3.1-release/) * Switch from the `proxy` plugin to the faster `forward` plugin for upsteam resolvers diff --git a/README.md b/README.md index a634bfc77..cd30ad0ae 100644 --- a/README.md +++ b/README.md @@ -11,7 +11,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features -* Kubernetes v1.13.2 (upstream, via [kubernetes-incubator/bootkube](https://github.com/kubernetes-incubator/bootkube)) +* Kubernetes v1.13.3 (upstream, via [kubernetes-incubator/bootkube](https://github.com/kubernetes-incubator/bootkube)) * Single or multi-master, [Calico](https://www.projectcalico.org/) or [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/) * Advanced features like [worker pools](https://typhoon.psdn.io/advanced/worker-pools/), [preemptible](https://typhoon.psdn.io/cl/google-cloud/#preemption) workers, and [snippets](https://typhoon.psdn.io/advanced/customization/#container-linux) customization @@ -50,7 +50,7 @@ Define a Kubernetes cluster by using the Terraform module for your chosen platfo ```tf module "google-cloud-yavin" { - source = "git::https://github.com/poseidon/typhoon//google-cloud/container-linux/kubernetes?ref=v1.13.2" + source = "git::https://github.com/poseidon/typhoon//google-cloud/container-linux/kubernetes?ref=v1.13.3" providers = { google = "google.default" @@ -91,9 +91,9 @@ In 4-8 minutes (varies by platform), the cluster will be ready. This Google Clou $ export KUBECONFIG=/home/user/.secrets/clusters/yavin/auth/kubeconfig $ kubectl get nodes NAME ROLES STATUS AGE VERSION -yavin-controller-0.c.example-com.internal controller,master Ready 6m v1.13.2 -yavin-worker-jrbf.c.example-com.internal node Ready 5m v1.13.2 -yavin-worker-mzdm.c.example-com.internal node Ready 5m v1.13.2 +yavin-controller-0.c.example-com.internal controller,master Ready 6m v1.13.3 +yavin-worker-jrbf.c.example-com.internal node Ready 5m v1.13.3 +yavin-worker-mzdm.c.example-com.internal node Ready 5m v1.13.3 ``` List the pods. diff --git a/aws/container-linux/kubernetes/README.md b/aws/container-linux/kubernetes/README.md index 37e5da1a5..1d59ee293 100644 --- a/aws/container-linux/kubernetes/README.md +++ b/aws/container-linux/kubernetes/README.md @@ -11,7 +11,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features -* Kubernetes v1.13.2 (upstream, via [kubernetes-incubator/bootkube](https://github.com/kubernetes-incubator/bootkube)) +* Kubernetes v1.13.3 (upstream, via [kubernetes-incubator/bootkube](https://github.com/kubernetes-incubator/bootkube)) * Single or multi-master, [Calico](https://www.projectcalico.org/) or [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/) * Advanced features like [worker pools](https://typhoon.psdn.io/advanced/worker-pools/), [spot](https://typhoon.psdn.io/cl/aws/#spot) workers, and [snippets](https://typhoon.psdn.io/advanced/customization/#container-linux) customization diff --git a/aws/container-linux/kubernetes/bootkube.tf b/aws/container-linux/kubernetes/bootkube.tf index a9455d932..2f5de5240 100644 --- a/aws/container-linux/kubernetes/bootkube.tf +++ b/aws/container-linux/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=7dc8f8bf8c1a6bdb9170dd19836520ce436e26d7" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=c12a11c8006606b59335ecc994abe22358aaf68b" cluster_name = "${var.cluster_name}" api_servers = ["${format("%s.%s", var.cluster_name, var.dns_zone)}"] diff --git a/aws/container-linux/kubernetes/cl/controller.yaml.tmpl b/aws/container-linux/kubernetes/cl/controller.yaml.tmpl index 166be04c7..e8f847ec3 100644 --- a/aws/container-linux/kubernetes/cl/controller.yaml.tmpl +++ b/aws/container-linux/kubernetes/cl/controller.yaml.tmpl @@ -123,7 +123,7 @@ storage: contents: inline: | KUBELET_IMAGE_URL=docker://k8s.gcr.io/hyperkube - KUBELET_IMAGE_TAG=v1.13.2 + KUBELET_IMAGE_TAG=v1.13.3 - path: /etc/sysctl.d/max-user-watches.conf filesystem: root contents: diff --git a/aws/container-linux/kubernetes/workers/cl/worker.yaml.tmpl b/aws/container-linux/kubernetes/workers/cl/worker.yaml.tmpl index a7410ba37..5bb864dd4 100644 --- a/aws/container-linux/kubernetes/workers/cl/worker.yaml.tmpl +++ b/aws/container-linux/kubernetes/workers/cl/worker.yaml.tmpl @@ -93,7 +93,7 @@ storage: contents: inline: | KUBELET_IMAGE_URL=docker://k8s.gcr.io/hyperkube - KUBELET_IMAGE_TAG=v1.13.2 + KUBELET_IMAGE_TAG=v1.13.3 - path: /etc/sysctl.d/max-user-watches.conf filesystem: root contents: @@ -111,7 +111,7 @@ storage: --volume config,kind=host,source=/etc/kubernetes \ --mount volume=config,target=/etc/kubernetes \ --insecure-options=image \ - docker://k8s.gcr.io/hyperkube:v1.13.2 \ + docker://k8s.gcr.io/hyperkube:v1.13.3 \ --net=host \ --dns=host \ --exec=/kubectl -- --kubeconfig=/etc/kubernetes/kubeconfig delete node $(hostname) diff --git a/aws/fedora-atomic/kubernetes/README.md b/aws/fedora-atomic/kubernetes/README.md index 2c81fdb01..fb3363078 100644 --- a/aws/fedora-atomic/kubernetes/README.md +++ b/aws/fedora-atomic/kubernetes/README.md @@ -11,7 +11,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features -* Kubernetes v1.13.2 (upstream, via [kubernetes-incubator/bootkube](https://github.com/kubernetes-incubator/bootkube)) +* Kubernetes v1.13.3 (upstream, via [kubernetes-incubator/bootkube](https://github.com/kubernetes-incubator/bootkube)) * Single or multi-master, [Calico](https://www.projectcalico.org/) or [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/) * Advanced features like [worker pools](https://typhoon.psdn.io/advanced/worker-pools/) and [spot](https://typhoon.psdn.io/cl/aws/#spot) workers diff --git a/aws/fedora-atomic/kubernetes/bootkube.tf b/aws/fedora-atomic/kubernetes/bootkube.tf index b33839d64..52f631b9c 100644 --- a/aws/fedora-atomic/kubernetes/bootkube.tf +++ b/aws/fedora-atomic/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=7dc8f8bf8c1a6bdb9170dd19836520ce436e26d7" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=c12a11c8006606b59335ecc994abe22358aaf68b" cluster_name = "${var.cluster_name}" api_servers = ["${format("%s.%s", var.cluster_name, var.dns_zone)}"] diff --git a/aws/fedora-atomic/kubernetes/cloudinit/controller.yaml.tmpl b/aws/fedora-atomic/kubernetes/cloudinit/controller.yaml.tmpl index 2e20258f0..7225a1490 100644 --- a/aws/fedora-atomic/kubernetes/cloudinit/controller.yaml.tmpl +++ b/aws/fedora-atomic/kubernetes/cloudinit/controller.yaml.tmpl @@ -79,7 +79,7 @@ runcmd: - [systemctl, daemon-reload] - [systemctl, restart, NetworkManager] - "atomic install --system --name=etcd quay.io/poseidon/etcd:v3.3.11" - - "atomic install --system --name=kubelet quay.io/poseidon/kubelet:v1.13.2" + - "atomic install --system --name=kubelet quay.io/poseidon/kubelet:v1.13.3" - "atomic install --system --name=bootkube quay.io/poseidon/bootkube:v0.14.0" - [systemctl, start, --no-block, etcd.service] - [systemctl, start, --no-block, kubelet.service] diff --git a/aws/fedora-atomic/kubernetes/workers/cloudinit/worker.yaml.tmpl b/aws/fedora-atomic/kubernetes/workers/cloudinit/worker.yaml.tmpl index bcb2949be..c14590c19 100644 --- a/aws/fedora-atomic/kubernetes/workers/cloudinit/worker.yaml.tmpl +++ b/aws/fedora-atomic/kubernetes/workers/cloudinit/worker.yaml.tmpl @@ -54,7 +54,7 @@ bootcmd: runcmd: - [systemctl, daemon-reload] - [systemctl, restart, NetworkManager] - - "atomic install --system --name=kubelet quay.io/poseidon/kubelet:v1.13.2" + - "atomic install --system --name=kubelet quay.io/poseidon/kubelet:v1.13.3" - [systemctl, start, --no-block, kubelet.service] users: - default diff --git a/azure/container-linux/kubernetes/README.md b/azure/container-linux/kubernetes/README.md index bfe7bdaac..4735444da 100644 --- a/azure/container-linux/kubernetes/README.md +++ b/azure/container-linux/kubernetes/README.md @@ -11,7 +11,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features -* Kubernetes v1.13.2 (upstream, via [kubernetes-incubator/bootkube](https://github.com/kubernetes-incubator/bootkube)) +* Kubernetes v1.13.3 (upstream, via [kubernetes-incubator/bootkube](https://github.com/kubernetes-incubator/bootkube)) * Single or multi-master, [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled * Advanced features like [worker pools](https://typhoon.psdn.io/advanced/worker-pools/), [low-priority](https://typhoon.psdn.io/cl/azure/#low-priority) workers, and [snippets](https://typhoon.psdn.io/advanced/customization/#container-linux) customization diff --git a/azure/container-linux/kubernetes/bootkube.tf b/azure/container-linux/kubernetes/bootkube.tf index ea386d24b..6576dbfc8 100644 --- a/azure/container-linux/kubernetes/bootkube.tf +++ b/azure/container-linux/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=7dc8f8bf8c1a6bdb9170dd19836520ce436e26d7" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=c12a11c8006606b59335ecc994abe22358aaf68b" cluster_name = "${var.cluster_name}" api_servers = ["${format("%s.%s", var.cluster_name, var.dns_zone)}"] diff --git a/azure/container-linux/kubernetes/cl/controller.yaml.tmpl b/azure/container-linux/kubernetes/cl/controller.yaml.tmpl index 166be04c7..e8f847ec3 100644 --- a/azure/container-linux/kubernetes/cl/controller.yaml.tmpl +++ b/azure/container-linux/kubernetes/cl/controller.yaml.tmpl @@ -123,7 +123,7 @@ storage: contents: inline: | KUBELET_IMAGE_URL=docker://k8s.gcr.io/hyperkube - KUBELET_IMAGE_TAG=v1.13.2 + KUBELET_IMAGE_TAG=v1.13.3 - path: /etc/sysctl.d/max-user-watches.conf filesystem: root contents: diff --git a/azure/container-linux/kubernetes/workers/cl/worker.yaml.tmpl b/azure/container-linux/kubernetes/workers/cl/worker.yaml.tmpl index f6458dc4f..decec0564 100644 --- a/azure/container-linux/kubernetes/workers/cl/worker.yaml.tmpl +++ b/azure/container-linux/kubernetes/workers/cl/worker.yaml.tmpl @@ -93,7 +93,7 @@ storage: contents: inline: | KUBELET_IMAGE_URL=docker://k8s.gcr.io/hyperkube - KUBELET_IMAGE_TAG=v1.13.2 + KUBELET_IMAGE_TAG=v1.13.3 - path: /etc/sysctl.d/max-user-watches.conf filesystem: root contents: @@ -111,7 +111,7 @@ storage: --volume config,kind=host,source=/etc/kubernetes \ --mount volume=config,target=/etc/kubernetes \ --insecure-options=image \ - docker://k8s.gcr.io/hyperkube:v1.13.2 \ + docker://k8s.gcr.io/hyperkube:v1.13.3 \ --net=host \ --dns=host \ --exec=/kubectl -- --kubeconfig=/etc/kubernetes/kubeconfig delete node $(hostname | tr '[:upper:]' '[:lower:]') diff --git a/bare-metal/container-linux/kubernetes/README.md b/bare-metal/container-linux/kubernetes/README.md index 2751b6cbb..04af2f52d 100644 --- a/bare-metal/container-linux/kubernetes/README.md +++ b/bare-metal/container-linux/kubernetes/README.md @@ -11,7 +11,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features -* Kubernetes v1.13.2 (upstream, via [kubernetes-incubator/bootkube](https://github.com/kubernetes-incubator/bootkube)) +* Kubernetes v1.13.3 (upstream, via [kubernetes-incubator/bootkube](https://github.com/kubernetes-incubator/bootkube)) * Single or multi-master, [Calico](https://www.projectcalico.org/) or [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/) * Advanced features like [snippets](https://typhoon.psdn.io/advanced/customization/#container-linux) customization diff --git a/bare-metal/container-linux/kubernetes/bootkube.tf b/bare-metal/container-linux/kubernetes/bootkube.tf index 34ba39cfc..6ffe17641 100644 --- a/bare-metal/container-linux/kubernetes/bootkube.tf +++ b/bare-metal/container-linux/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=7dc8f8bf8c1a6bdb9170dd19836520ce436e26d7" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=c12a11c8006606b59335ecc994abe22358aaf68b" cluster_name = "${var.cluster_name}" api_servers = ["${var.k8s_domain_name}"] diff --git a/bare-metal/container-linux/kubernetes/cl/controller.yaml.tmpl b/bare-metal/container-linux/kubernetes/cl/controller.yaml.tmpl index da4080c84..f5de49c50 100644 --- a/bare-metal/container-linux/kubernetes/cl/controller.yaml.tmpl +++ b/bare-metal/container-linux/kubernetes/cl/controller.yaml.tmpl @@ -128,7 +128,7 @@ storage: contents: inline: | KUBELET_IMAGE_URL=docker://k8s.gcr.io/hyperkube - KUBELET_IMAGE_TAG=v1.13.2 + KUBELET_IMAGE_TAG=v1.13.3 - path: /etc/hostname filesystem: root mode: 0644 diff --git a/bare-metal/container-linux/kubernetes/cl/worker.yaml.tmpl b/bare-metal/container-linux/kubernetes/cl/worker.yaml.tmpl index a18c6923e..a3f13d03b 100644 --- a/bare-metal/container-linux/kubernetes/cl/worker.yaml.tmpl +++ b/bare-metal/container-linux/kubernetes/cl/worker.yaml.tmpl @@ -89,7 +89,7 @@ storage: contents: inline: | KUBELET_IMAGE_URL=docker://k8s.gcr.io/hyperkube - KUBELET_IMAGE_TAG=v1.13.2 + KUBELET_IMAGE_TAG=v1.13.3 - path: /etc/hostname filesystem: root mode: 0644 diff --git a/bare-metal/fedora-atomic/kubernetes/README.md b/bare-metal/fedora-atomic/kubernetes/README.md index 7eef1cffa..4d343afda 100644 --- a/bare-metal/fedora-atomic/kubernetes/README.md +++ b/bare-metal/fedora-atomic/kubernetes/README.md @@ -11,7 +11,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features -* Kubernetes v1.13.2 (upstream, via [kubernetes-incubator/bootkube](https://github.com/kubernetes-incubator/bootkube)) +* Kubernetes v1.13.3 (upstream, via [kubernetes-incubator/bootkube](https://github.com/kubernetes-incubator/bootkube)) * Single or multi-master, [Calico](https://www.projectcalico.org/) or [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/) * Ready for Ingress, Prometheus, Grafana, and other optional [addons](https://typhoon.psdn.io/addons/overview/) diff --git a/bare-metal/fedora-atomic/kubernetes/bootkube.tf b/bare-metal/fedora-atomic/kubernetes/bootkube.tf index 8a84cc932..2156518e2 100644 --- a/bare-metal/fedora-atomic/kubernetes/bootkube.tf +++ b/bare-metal/fedora-atomic/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=7dc8f8bf8c1a6bdb9170dd19836520ce436e26d7" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=c12a11c8006606b59335ecc994abe22358aaf68b" cluster_name = "${var.cluster_name}" api_servers = ["${var.k8s_domain_name}"] diff --git a/bare-metal/fedora-atomic/kubernetes/cloudinit/controller.yaml.tmpl b/bare-metal/fedora-atomic/kubernetes/cloudinit/controller.yaml.tmpl index 6c5a64c51..8416ee300 100644 --- a/bare-metal/fedora-atomic/kubernetes/cloudinit/controller.yaml.tmpl +++ b/bare-metal/fedora-atomic/kubernetes/cloudinit/controller.yaml.tmpl @@ -85,7 +85,7 @@ runcmd: - [systemctl, restart, NetworkManager] - [hostnamectl, set-hostname, ${domain_name}] - "atomic install --system --name=etcd quay.io/poseidon/etcd:v3.3.11" - - "atomic install --system --name=kubelet quay.io/poseidon/kubelet:v1.13.2" + - "atomic install --system --name=kubelet quay.io/poseidon/kubelet:v1.13.3" - "atomic install --system --name=bootkube quay.io/poseidon/bootkube:v0.14.0" - [systemctl, start, --no-block, etcd.service] - [systemctl, enable, kubelet.path] diff --git a/bare-metal/fedora-atomic/kubernetes/cloudinit/worker.yaml.tmpl b/bare-metal/fedora-atomic/kubernetes/cloudinit/worker.yaml.tmpl index 3c9854cb5..52614004e 100644 --- a/bare-metal/fedora-atomic/kubernetes/cloudinit/worker.yaml.tmpl +++ b/bare-metal/fedora-atomic/kubernetes/cloudinit/worker.yaml.tmpl @@ -60,7 +60,7 @@ runcmd: - [systemctl, daemon-reload] - [systemctl, restart, NetworkManager] - [hostnamectl, set-hostname, ${domain_name}] - - "atomic install --system --name=kubelet quay.io/poseidon/kubelet:v1.13.2" + - "atomic install --system --name=kubelet quay.io/poseidon/kubelet:v1.13.3" - [systemctl, enable, kubelet.path] - [systemctl, start, --no-block, kubelet.path] users: diff --git a/digital-ocean/container-linux/kubernetes/README.md b/digital-ocean/container-linux/kubernetes/README.md index e9a859ea2..be9f9c36b 100644 --- a/digital-ocean/container-linux/kubernetes/README.md +++ b/digital-ocean/container-linux/kubernetes/README.md @@ -11,7 +11,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features -* Kubernetes v1.13.2 (upstream, via [kubernetes-incubator/bootkube](https://github.com/kubernetes-incubator/bootkube)) +* Kubernetes v1.13.3 (upstream, via [kubernetes-incubator/bootkube](https://github.com/kubernetes-incubator/bootkube)) * Single or multi-master, [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled * Advanced features like [snippets](https://typhoon.psdn.io/advanced/customization/#container-linux) customization diff --git a/digital-ocean/container-linux/kubernetes/bootkube.tf b/digital-ocean/container-linux/kubernetes/bootkube.tf index a340e9669..3396991ba 100644 --- a/digital-ocean/container-linux/kubernetes/bootkube.tf +++ b/digital-ocean/container-linux/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=7dc8f8bf8c1a6bdb9170dd19836520ce436e26d7" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=c12a11c8006606b59335ecc994abe22358aaf68b" cluster_name = "${var.cluster_name}" api_servers = ["${format("%s.%s", var.cluster_name, var.dns_zone)}"] diff --git a/digital-ocean/container-linux/kubernetes/cl/controller.yaml.tmpl b/digital-ocean/container-linux/kubernetes/cl/controller.yaml.tmpl index bd14cfd0f..66d5fdb6d 100644 --- a/digital-ocean/container-linux/kubernetes/cl/controller.yaml.tmpl +++ b/digital-ocean/container-linux/kubernetes/cl/controller.yaml.tmpl @@ -125,7 +125,7 @@ storage: contents: inline: | KUBELET_IMAGE_URL=docker://k8s.gcr.io/hyperkube - KUBELET_IMAGE_TAG=v1.13.2 + KUBELET_IMAGE_TAG=v1.13.3 - path: /etc/sysctl.d/max-user-watches.conf filesystem: root contents: diff --git a/digital-ocean/container-linux/kubernetes/cl/worker.yaml.tmpl b/digital-ocean/container-linux/kubernetes/cl/worker.yaml.tmpl index 4ff9a0644..d10fadf5c 100644 --- a/digital-ocean/container-linux/kubernetes/cl/worker.yaml.tmpl +++ b/digital-ocean/container-linux/kubernetes/cl/worker.yaml.tmpl @@ -95,7 +95,7 @@ storage: contents: inline: | KUBELET_IMAGE_URL=docker://k8s.gcr.io/hyperkube - KUBELET_IMAGE_TAG=v1.13.2 + KUBELET_IMAGE_TAG=v1.13.3 - path: /etc/sysctl.d/max-user-watches.conf filesystem: root contents: @@ -113,7 +113,7 @@ storage: --volume config,kind=host,source=/etc/kubernetes \ --mount volume=config,target=/etc/kubernetes \ --insecure-options=image \ - docker://k8s.gcr.io/hyperkube:v1.13.2 \ + docker://k8s.gcr.io/hyperkube:v1.13.3 \ --net=host \ --dns=host \ --exec=/kubectl -- --kubeconfig=/etc/kubernetes/kubeconfig delete node $(hostname) diff --git a/digital-ocean/fedora-atomic/kubernetes/README.md b/digital-ocean/fedora-atomic/kubernetes/README.md index 2c720b8c7..f5b86bf4c 100644 --- a/digital-ocean/fedora-atomic/kubernetes/README.md +++ b/digital-ocean/fedora-atomic/kubernetes/README.md @@ -11,7 +11,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features -* Kubernetes v1.13.2 (upstream, via [kubernetes-incubator/bootkube](https://github.com/kubernetes-incubator/bootkube)) +* Kubernetes v1.13.3 (upstream, via [kubernetes-incubator/bootkube](https://github.com/kubernetes-incubator/bootkube)) * Single or multi-master, [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled * Ready for Ingress, Prometheus, Grafana, and other optional [addons](https://typhoon.psdn.io/addons/overview/) diff --git a/digital-ocean/fedora-atomic/kubernetes/bootkube.tf b/digital-ocean/fedora-atomic/kubernetes/bootkube.tf index de02385b1..3eeefee1c 100644 --- a/digital-ocean/fedora-atomic/kubernetes/bootkube.tf +++ b/digital-ocean/fedora-atomic/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=7dc8f8bf8c1a6bdb9170dd19836520ce436e26d7" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=c12a11c8006606b59335ecc994abe22358aaf68b" cluster_name = "${var.cluster_name}" api_servers = ["${format("%s.%s", var.cluster_name, var.dns_zone)}"] diff --git a/digital-ocean/fedora-atomic/kubernetes/cloudinit/controller.yaml.tmpl b/digital-ocean/fedora-atomic/kubernetes/cloudinit/controller.yaml.tmpl index 9a0323eb3..b6a62d581 100644 --- a/digital-ocean/fedora-atomic/kubernetes/cloudinit/controller.yaml.tmpl +++ b/digital-ocean/fedora-atomic/kubernetes/cloudinit/controller.yaml.tmpl @@ -76,7 +76,7 @@ bootcmd: runcmd: - [systemctl, daemon-reload] - "atomic install --system --name=etcd quay.io/poseidon/etcd:v3.3.11" - - "atomic install --system --name=kubelet quay.io/poseidon/kubelet:v1.13.2" + - "atomic install --system --name=kubelet quay.io/poseidon/kubelet:v1.13.3" - "atomic install --system --name=bootkube quay.io/poseidon/bootkube:v0.14.0" - [systemctl, start, --no-block, etcd.service] - [systemctl, enable, kubelet.path] diff --git a/digital-ocean/fedora-atomic/kubernetes/cloudinit/worker.yaml.tmpl b/digital-ocean/fedora-atomic/kubernetes/cloudinit/worker.yaml.tmpl index 76b708325..56f43b4d1 100644 --- a/digital-ocean/fedora-atomic/kubernetes/cloudinit/worker.yaml.tmpl +++ b/digital-ocean/fedora-atomic/kubernetes/cloudinit/worker.yaml.tmpl @@ -51,7 +51,7 @@ bootcmd: - [modprobe, ip_vs] runcmd: - [systemctl, daemon-reload] - - "atomic install --system --name=kubelet quay.io/poseidon/kubelet:v1.13.2" + - "atomic install --system --name=kubelet quay.io/poseidon/kubelet:v1.13.3" - [systemctl, enable, kubelet.path] - [systemctl, start, --no-block, kubelet.path] users: diff --git a/docs/advanced/worker-pools.md b/docs/advanced/worker-pools.md index 4dcc1f40b..36cadd37d 100644 --- a/docs/advanced/worker-pools.md +++ b/docs/advanced/worker-pools.md @@ -16,7 +16,7 @@ Create a cluster following the AWS [tutorial](../cl/aws.md#cluster). Define a wo ```tf module "tempest-worker-pool" { - source = "git::https://github.com/poseidon/typhoon//aws/container-linux/kubernetes/workers?ref=v1.13.2" + source = "git::https://github.com/poseidon/typhoon//aws/container-linux/kubernetes/workers?ref=v1.13.3" providers = { aws = "aws.default" @@ -82,7 +82,7 @@ Create a cluster following the Azure [tutorial](../cl/azure.md#cluster). Define ```tf module "ramius-worker-pool" { - source = "git::https://github.com/poseidon/typhoon//azure/container-linux/kubernetes/workers?ref=v1.13.2" + source = "git::https://github.com/poseidon/typhoon//azure/container-linux/kubernetes/workers?ref=v1.13.3" providers = { azurerm = "azurerm.default" @@ -152,7 +152,7 @@ Create a cluster following the Google Cloud [tutorial](../cl/google-cloud.md#clu ```tf module "yavin-worker-pool" { - source = "git::https://github.com/poseidon/typhoon//google-cloud/container-linux/kubernetes/workers?ref=v1.13.2" + source = "git::https://github.com/poseidon/typhoon//google-cloud/container-linux/kubernetes/workers?ref=v1.13.3" providers = { google = "google.default" @@ -187,11 +187,11 @@ Verify a managed instance group of workers joins the cluster within a few minute ``` $ kubectl get nodes NAME STATUS AGE VERSION -yavin-controller-0.c.example-com.internal Ready 6m v1.13.2 -yavin-worker-jrbf.c.example-com.internal Ready 5m v1.13.2 -yavin-worker-mzdm.c.example-com.internal Ready 5m v1.13.2 -yavin-16x-worker-jrbf.c.example-com.internal Ready 3m v1.13.2 -yavin-16x-worker-mzdm.c.example-com.internal Ready 3m v1.13.2 +yavin-controller-0.c.example-com.internal Ready 6m v1.13.3 +yavin-worker-jrbf.c.example-com.internal Ready 5m v1.13.3 +yavin-worker-mzdm.c.example-com.internal Ready 5m v1.13.3 +yavin-16x-worker-jrbf.c.example-com.internal Ready 3m v1.13.3 +yavin-16x-worker-mzdm.c.example-com.internal Ready 3m v1.13.3 ``` ### Variables diff --git a/docs/atomic/aws.md b/docs/atomic/aws.md index 2d85ceedf..5a4699571 100644 --- a/docs/atomic/aws.md +++ b/docs/atomic/aws.md @@ -3,7 +3,7 @@ !!! danger Typhoon for Fedora Atomic is alpha. Expect rough edges and changes. -In this tutorial, we'll create a Kubernetes v1.13.2 cluster on AWS with Fedora Atomic. +In this tutorial, we'll create a Kubernetes v1.13.3 cluster on AWS with Fedora Atomic. We'll declare a Kubernetes cluster using the Typhoon Terraform module. Then apply the changes to create a VPC, gateway, subnets, security groups, controller instances, worker auto-scaling group, network load balancer, and TLS assets. Instances are provisioned on first boot with cloud-init. @@ -83,7 +83,7 @@ Define a Kubernetes cluster using the module `aws/fedora-atomic/kubernetes`. ```tf module "aws-tempest" { - source = "git::https://github.com/poseidon/typhoon//aws/fedora-atomic/kubernetes?ref=v1.13.2" + source = "git::https://github.com/poseidon/typhoon//aws/fedora-atomic/kubernetes?ref=v1.13.3" providers = { aws = "aws.default" @@ -156,9 +156,9 @@ In 5-10 minutes, the Kubernetes cluster will be ready. $ export KUBECONFIG=/home/user/.secrets/clusters/tempest/auth/kubeconfig $ kubectl get nodes NAME STATUS ROLES AGE VERSION -ip-10-0-3-155 Ready controller,master 10m v1.13.2 -ip-10-0-26-65 Ready node 10m v1.13.2 -ip-10-0-41-21 Ready node 10m v1.13.2 +ip-10-0-3-155 Ready controller,master 10m v1.13.3 +ip-10-0-26-65 Ready node 10m v1.13.3 +ip-10-0-41-21 Ready node 10m v1.13.3 ``` List the pods. diff --git a/docs/atomic/bare-metal.md b/docs/atomic/bare-metal.md index 3f1ba7b5b..dffd5bb6e 100644 --- a/docs/atomic/bare-metal.md +++ b/docs/atomic/bare-metal.md @@ -3,7 +3,7 @@ !!! danger Typhoon for Fedora Atomic is alpha. Expect rough edges and changes. -In this tutorial, we'll network boot and provision a Kubernetes v1.13.2 cluster on bare-metal with Fedora Atomic. +In this tutorial, we'll network boot and provision a Kubernetes v1.13.3 cluster on bare-metal with Fedora Atomic. First, we'll deploy a [Matchbox](https://github.com/coreos/matchbox) service and setup a network boot environment. Then, we'll declare a Kubernetes cluster using the Typhoon Terraform module and power on machines. On PXE boot, machines will install Fedora Atomic via kickstart, reboot into the disk install, and provision themselves as Kubernetes controllers or workers via cloud-init. @@ -235,7 +235,7 @@ Define a Kubernetes cluster using the module `bare-metal/fedora-atomic/kubernete ```tf module "bare-metal-mercury" { - source = "git::https://github.com/poseidon/typhoon//bare-metal/fedora-atomic/kubernetes?ref=v1.13.2" + source = "git::https://github.com/poseidon/typhoon//bare-metal/fedora-atomic/kubernetes?ref=v1.13.3" providers = { local = "local.default" @@ -361,9 +361,9 @@ bootkube[5]: Tearing down temporary bootstrap control plane... $ export KUBECONFIG=/home/user/.secrets/clusters/mercury/auth/kubeconfig $ kubectl get nodes NAME STATUS ROLES AGE VERSION -node1.example.com Ready controller,master 10m v1.13.2 -node2.example.com Ready node 10m v1.13.2 -node3.example.com Ready node 10m v1.13.2 +node1.example.com Ready controller,master 10m v1.13.3 +node2.example.com Ready node 10m v1.13.3 +node3.example.com Ready node 10m v1.13.3 ``` List the pods. diff --git a/docs/atomic/digital-ocean.md b/docs/atomic/digital-ocean.md index db78c16e4..383de3979 100644 --- a/docs/atomic/digital-ocean.md +++ b/docs/atomic/digital-ocean.md @@ -3,7 +3,7 @@ !!! danger Typhoon for Fedora Atomic is alpha. Expect rough edges and changes. -In this tutorial, we'll create a Kubernetes v1.13.2 cluster on DigitalOcean with Fedora Atomic. +In this tutorial, we'll create a Kubernetes v1.13.3 cluster on DigitalOcean with Fedora Atomic. We'll declare a Kubernetes cluster using the Typhoon Terraform module. Then apply the changes to create controller droplets, worker droplets, DNS records, tags, and TLS assets. Instances are provisioned on first boot with cloud-init. @@ -77,7 +77,7 @@ Define a Kubernetes cluster using the module `digital-ocean/fedora-atomic/kubern ```tf module "digital-ocean-nemo" { - source = "git::https://github.com/poseidon/typhoon//digital-ocean/fedora-atomic/kubernetes?ref=v1.13.2" + source = "git::https://github.com/poseidon/typhoon//digital-ocean/fedora-atomic/kubernetes?ref=v1.13.3" providers = { digitalocean = "digitalocean.default" @@ -152,9 +152,9 @@ In 3-6 minutes, the Kubernetes cluster will be ready. $ export KUBECONFIG=/home/user/.secrets/clusters/nemo/auth/kubeconfig $ kubectl get nodes NAME STATUS ROLES AGE VERSION -nemo-controller-0 Ready controller,master 10m v1.13.2 -nemo-worker-0 Ready node 10m v1.13.2 -nemo-worker-1 Ready node 10m v1.13.2 +nemo-controller-0 Ready controller,master 10m v1.13.3 +nemo-worker-0 Ready node 10m v1.13.3 +nemo-worker-1 Ready node 10m v1.13.3 ``` List the pods. diff --git a/docs/atomic/google-cloud.md b/docs/atomic/google-cloud.md index 9d224e990..0e8620063 100644 --- a/docs/atomic/google-cloud.md +++ b/docs/atomic/google-cloud.md @@ -3,7 +3,7 @@ !!! danger Typhoon for Fedora Atomic is alpha. Fedora does not publish official images for Google Cloud so you must prepare them yourself. Expect rough edges and changes. -In this tutorial, we'll create a Kubernetes v1.13.2 cluster on Google Compute Engine with Fedora Atomic. +In this tutorial, we'll create a Kubernetes v1.13.3 cluster on Google Compute Engine with Fedora Atomic. We'll declare a Kubernetes cluster using the Typhoon Terraform module. Then apply the changes to create a network, firewall rules, health checks, controller instances, worker managed instance group, load balancers, and TLS assets. Instances are provisioned on first boot with cloud-init. @@ -121,7 +121,7 @@ Define a Kubernetes cluster using the module `google-cloud/fedora-atomic/kuberne ```tf module "google-cloud-yavin" { - source = "git::https://github.com/poseidon/typhoon//google-cloud/fedora-atomic/kubernetes?ref=v1.13.2" + source = "git::https://github.com/poseidon/typhoon//google-cloud/fedora-atomic/kubernetes?ref=v1.13.3" providers = { google = "google.default" @@ -197,9 +197,9 @@ In 5-10 minutes, the Kubernetes cluster will be ready. $ export KUBECONFIG=/home/user/.secrets/clusters/yavin/auth/kubeconfig $ kubectl get nodes NAME ROLES STATUS AGE VERSION -yavin-controller-0.c.example-com.internal controller,master Ready 6m v1.13.2 -yavin-worker-jrbf.c.example-com.internal node Ready 5m v1.13.2 -yavin-worker-mzdm.c.example-com.internal node Ready 5m v1.13.2 +yavin-controller-0.c.example-com.internal controller,master Ready 6m v1.13.3 +yavin-worker-jrbf.c.example-com.internal node Ready 5m v1.13.3 +yavin-worker-mzdm.c.example-com.internal node Ready 5m v1.13.3 ``` List the pods. diff --git a/docs/cl/aws.md b/docs/cl/aws.md index d3a6eb5f1..b1f8ab389 100644 --- a/docs/cl/aws.md +++ b/docs/cl/aws.md @@ -1,6 +1,6 @@ # AWS -In this tutorial, we'll create a Kubernetes v1.13.2 cluster on AWS with Container Linux. +In this tutorial, we'll create a Kubernetes v1.13.3 cluster on AWS with Container Linux. We'll declare a Kubernetes cluster using the Typhoon Terraform module. Then apply the changes to create a VPC, gateway, subnets, security groups, controller instances, worker auto-scaling group, network load balancer, and TLS assets. @@ -92,7 +92,7 @@ Define a Kubernetes cluster using the module `aws/container-linux/kubernetes`. ```tf module "aws-tempest" { - source = "git::https://github.com/poseidon/typhoon//aws/container-linux/kubernetes?ref=v1.13.2" + source = "git::https://github.com/poseidon/typhoon//aws/container-linux/kubernetes?ref=v1.13.3" providers = { aws = "aws.default" @@ -165,9 +165,9 @@ In 4-8 minutes, the Kubernetes cluster will be ready. $ export KUBECONFIG=/home/user/.secrets/clusters/tempest/auth/kubeconfig $ kubectl get nodes NAME STATUS ROLES AGE VERSION -ip-10-0-3-155 Ready controller,master 10m v1.13.2 -ip-10-0-26-65 Ready node 10m v1.13.2 -ip-10-0-41-21 Ready node 10m v1.13.2 +ip-10-0-3-155 Ready controller,master 10m v1.13.3 +ip-10-0-26-65 Ready node 10m v1.13.3 +ip-10-0-41-21 Ready node 10m v1.13.3 ``` List the pods. diff --git a/docs/cl/azure.md b/docs/cl/azure.md index 6f8d94002..b992412fd 100644 --- a/docs/cl/azure.md +++ b/docs/cl/azure.md @@ -3,7 +3,7 @@ !!! danger Typhoon for Azure is alpha. For production, use AWS, Google Cloud, or bare-metal. As Azure matures, check [errata](https://github.com/poseidon/typhoon/wiki/Errata) for known shortcomings. -In this tutorial, we'll create a Kubernetes v1.13.2 cluster on Azure with Container Linux. +In this tutorial, we'll create a Kubernetes v1.13.3 cluster on Azure with Container Linux. We'll declare a Kubernetes cluster using the Typhoon Terraform module. Then apply the changes to create a resource group, virtual network, subnets, security groups, controller availability set, worker scale set, load balancer, and TLS assets. @@ -87,7 +87,7 @@ Define a Kubernetes cluster using the module `azure/container-linux/kubernetes`. ```tf module "azure-ramius" { - source = "git::https://github.com/poseidon/typhoon//azure/container-linux/kubernetes?ref=v1.13.2" + source = "git::https://github.com/poseidon/typhoon//azure/container-linux/kubernetes?ref=v1.13.3" providers = { azurerm = "azurerm.default" @@ -161,9 +161,9 @@ In 4-8 minutes, the Kubernetes cluster will be ready. $ export KUBECONFIG=/home/user/.secrets/clusters/ramius/auth/kubeconfig $ kubectl get nodes NAME STATUS ROLES AGE VERSION -ramius-controller-0 Ready controller,master 24m v1.13.2 -ramius-worker-000001 Ready node 25m v1.13.2 -ramius-worker-000002 Ready node 24m v1.13.2 +ramius-controller-0 Ready controller,master 24m v1.13.3 +ramius-worker-000001 Ready node 25m v1.13.3 +ramius-worker-000002 Ready node 24m v1.13.3 ``` List the pods. diff --git a/docs/cl/bare-metal.md b/docs/cl/bare-metal.md index 64eaf5398..43db3f2f7 100644 --- a/docs/cl/bare-metal.md +++ b/docs/cl/bare-metal.md @@ -1,6 +1,6 @@ # Bare-Metal -In this tutorial, we'll network boot and provision a Kubernetes v1.13.2 cluster on bare-metal with Container Linux. +In this tutorial, we'll network boot and provision a Kubernetes v1.13.3 cluster on bare-metal with Container Linux. First, we'll deploy a [Matchbox](https://github.com/coreos/matchbox) service and setup a network boot environment. Then, we'll declare a Kubernetes cluster using the Typhoon Terraform module and power on machines. On PXE boot, machines will install Container Linux to disk, reboot into the disk install, and provision themselves as Kubernetes controllers or workers via Ignition. @@ -179,7 +179,7 @@ Define a Kubernetes cluster using the module `bare-metal/container-linux/kuberne ```tf module "bare-metal-mercury" { - source = "git::https://github.com/poseidon/typhoon//bare-metal/container-linux/kubernetes?ref=v1.13.2" + source = "git::https://github.com/poseidon/typhoon//bare-metal/container-linux/kubernetes?ref=v1.13.3" providers = { local = "local.default" @@ -288,9 +288,9 @@ Apply complete! Resources: 55 added, 0 changed, 0 destroyed. To watch the install to disk (until machines reboot from disk), SSH to port 2222. ``` -# before v1.13.2 +# before v1.13.3 $ ssh debug@node1.example.com -# after v1.13.2 +# after v1.13.3 $ ssh -p 2222 core@node1.example.com ``` @@ -315,9 +315,9 @@ bootkube[5]: Tearing down temporary bootstrap control plane... $ export KUBECONFIG=/home/user/.secrets/clusters/mercury/auth/kubeconfig $ kubectl get nodes NAME STATUS ROLES AGE VERSION -node1.example.com Ready controller,master 10m v1.13.2 -node2.example.com Ready node 10m v1.13.2 -node3.example.com Ready node 10m v1.13.2 +node1.example.com Ready controller,master 10m v1.13.3 +node2.example.com Ready node 10m v1.13.3 +node3.example.com Ready node 10m v1.13.3 ``` List the pods. diff --git a/docs/cl/digital-ocean.md b/docs/cl/digital-ocean.md index a70704f70..25d12a3b7 100644 --- a/docs/cl/digital-ocean.md +++ b/docs/cl/digital-ocean.md @@ -1,6 +1,6 @@ # Digital Ocean -In this tutorial, we'll create a Kubernetes v1.13.2 cluster on DigitalOcean with Container Linux. +In this tutorial, we'll create a Kubernetes v1.13.3 cluster on DigitalOcean with Container Linux. We'll declare a Kubernetes cluster using the Typhoon Terraform module. Then apply the changes to create controller droplets, worker droplets, DNS records, tags, and TLS assets. @@ -86,7 +86,7 @@ Define a Kubernetes cluster using the module `digital-ocean/container-linux/kube ```tf module "digital-ocean-nemo" { - source = "git::https://github.com/poseidon/typhoon//digital-ocean/container-linux/kubernetes?ref=v1.13.2" + source = "git::https://github.com/poseidon/typhoon//digital-ocean/container-linux/kubernetes?ref=v1.13.3" providers = { digitalocean = "digitalocean.default" @@ -160,9 +160,9 @@ In 3-6 minutes, the Kubernetes cluster will be ready. $ export KUBECONFIG=/home/user/.secrets/clusters/nemo/auth/kubeconfig $ kubectl get nodes NAME STATUS ROLES AGE VERSION -nemo-controller-0 Ready controller,master 10m v1.13.2 -nemo-worker-0 Ready node 10m v1.13.2 -nemo-worker-1 Ready node 10m v1.13.2 +nemo-controller-0 Ready controller,master 10m v1.13.3 +nemo-worker-0 Ready node 10m v1.13.3 +nemo-worker-1 Ready node 10m v1.13.3 ``` List the pods. diff --git a/docs/cl/google-cloud.md b/docs/cl/google-cloud.md index 2a3ed2c1b..e8f2abcfb 100644 --- a/docs/cl/google-cloud.md +++ b/docs/cl/google-cloud.md @@ -1,6 +1,6 @@ # Google Cloud -In this tutorial, we'll create a Kubernetes v1.13.2 cluster on Google Compute Engine with Container Linux. +In this tutorial, we'll create a Kubernetes v1.13.3 cluster on Google Compute Engine with Container Linux. We'll declare a Kubernetes cluster using the Typhoon Terraform module. Then apply the changes to create a network, firewall rules, health checks, controller instances, worker managed instance group, load balancers, and TLS assets. @@ -93,7 +93,7 @@ Define a Kubernetes cluster using the module `google-cloud/container-linux/kuber ```tf module "google-cloud-yavin" { - source = "git::https://github.com/poseidon/typhoon//google-cloud/container-linux/kubernetes?ref=v1.13.2" + source = "git::https://github.com/poseidon/typhoon//google-cloud/container-linux/kubernetes?ref=v1.13.3" providers = { google = "google.default" @@ -168,9 +168,9 @@ In 4-8 minutes, the Kubernetes cluster will be ready. $ export KUBECONFIG=/home/user/.secrets/clusters/yavin/auth/kubeconfig $ kubectl get nodes NAME ROLES STATUS AGE VERSION -yavin-controller-0.c.example-com.internal controller,master Ready 6m v1.13.2 -yavin-worker-jrbf.c.example-com.internal node Ready 5m v1.13.2 -yavin-worker-mzdm.c.example-com.internal node Ready 5m v1.13.2 +yavin-controller-0.c.example-com.internal controller,master Ready 6m v1.13.3 +yavin-worker-jrbf.c.example-com.internal node Ready 5m v1.13.3 +yavin-worker-mzdm.c.example-com.internal node Ready 5m v1.13.3 ``` List the pods. diff --git a/docs/index.md b/docs/index.md index f1d8ceed1..586a36133 100644 --- a/docs/index.md +++ b/docs/index.md @@ -11,7 +11,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features -* Kubernetes v1.13.2 (upstream, via [kubernetes-incubator/bootkube](https://github.com/kubernetes-incubator/bootkube)) +* Kubernetes v1.13.3 (upstream, via [kubernetes-incubator/bootkube](https://github.com/kubernetes-incubator/bootkube)) * Single or multi-master, [Calico](https://www.projectcalico.org/) or [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/) * Advanced features like [worker pools](advanced/worker-pools/), [preemptible](cl/google-cloud/#preemption) workers, and [snippets](advanced/customization/#container-linux) customization @@ -49,7 +49,7 @@ Define a Kubernetes cluster by using the Terraform module for your chosen platfo ```tf module "google-cloud-yavin" { - source = "git::https://github.com/poseidon/typhoon//google-cloud/container-linux/kubernetes?ref=v1.13.2" + source = "git::https://github.com/poseidon/typhoon//google-cloud/container-linux/kubernetes?ref=v1.13.3" providers = { google = "google.default" @@ -90,9 +90,9 @@ In 4-8 minutes (varies by platform), the cluster will be ready. This Google Clou $ export KUBECONFIG=/home/user/.secrets/clusters/yavin/auth/kubeconfig $ kubectl get nodes NAME ROLES STATUS AGE VERSION -yavin-controller-0.c.example-com.internal controller,master Ready 6m v1.13.2 -yavin-worker-jrbf.c.example-com.internal node Ready 5m v1.13.2 -yavin-worker-mzdm.c.example-com.internal node Ready 5m v1.13.2 +yavin-controller-0.c.example-com.internal controller,master Ready 6m v1.13.3 +yavin-worker-jrbf.c.example-com.internal node Ready 5m v1.13.3 +yavin-worker-mzdm.c.example-com.internal node Ready 5m v1.13.3 ``` List the pods. diff --git a/docs/topics/maintenance.md b/docs/topics/maintenance.md index 7dc5e5428..94d52935d 100644 --- a/docs/topics/maintenance.md +++ b/docs/topics/maintenance.md @@ -18,7 +18,7 @@ module "google-cloud-yavin" { } module "bare-metal-mercury" { - source = "git::https://github.com/poseidon/typhoon//bare-metal/container-linux/kubernetes?ref=v1.13.2" + source = "git::https://github.com/poseidon/typhoon//bare-metal/container-linux/kubernetes?ref=v1.13.3" ... } ``` diff --git a/google-cloud/container-linux/kubernetes/README.md b/google-cloud/container-linux/kubernetes/README.md index 951b8b140..2f4cdef51 100644 --- a/google-cloud/container-linux/kubernetes/README.md +++ b/google-cloud/container-linux/kubernetes/README.md @@ -11,7 +11,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features -* Kubernetes v1.13.2 (upstream, via [kubernetes-incubator/bootkube](https://github.com/kubernetes-incubator/bootkube)) +* Kubernetes v1.13.3 (upstream, via [kubernetes-incubator/bootkube](https://github.com/kubernetes-incubator/bootkube)) * Single or multi-master, [Calico](https://www.projectcalico.org/) or [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/) * Advanced features like [worker pools](https://typhoon.psdn.io/advanced/worker-pools/), [preemptible](https://typhoon.psdn.io/cl/google-cloud/#preemption) workers, and [snippets](https://typhoon.psdn.io/advanced/customization/#container-linux) customization diff --git a/google-cloud/container-linux/kubernetes/bootkube.tf b/google-cloud/container-linux/kubernetes/bootkube.tf index a9b18f8cd..afed043f4 100644 --- a/google-cloud/container-linux/kubernetes/bootkube.tf +++ b/google-cloud/container-linux/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=7dc8f8bf8c1a6bdb9170dd19836520ce436e26d7" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=c12a11c8006606b59335ecc994abe22358aaf68b" cluster_name = "${var.cluster_name}" api_servers = ["${format("%s.%s", var.cluster_name, var.dns_zone)}"] diff --git a/google-cloud/container-linux/kubernetes/cl/controller.yaml.tmpl b/google-cloud/container-linux/kubernetes/cl/controller.yaml.tmpl index ca17ff76d..7fcc5b2ef 100644 --- a/google-cloud/container-linux/kubernetes/cl/controller.yaml.tmpl +++ b/google-cloud/container-linux/kubernetes/cl/controller.yaml.tmpl @@ -124,7 +124,7 @@ storage: contents: inline: | KUBELET_IMAGE_URL=docker://k8s.gcr.io/hyperkube - KUBELET_IMAGE_TAG=v1.13.2 + KUBELET_IMAGE_TAG=v1.13.3 - path: /etc/sysctl.d/max-user-watches.conf filesystem: root contents: diff --git a/google-cloud/container-linux/kubernetes/workers/cl/worker.yaml.tmpl b/google-cloud/container-linux/kubernetes/workers/cl/worker.yaml.tmpl index c7464d8cf..6e114d0eb 100644 --- a/google-cloud/container-linux/kubernetes/workers/cl/worker.yaml.tmpl +++ b/google-cloud/container-linux/kubernetes/workers/cl/worker.yaml.tmpl @@ -94,7 +94,7 @@ storage: contents: inline: | KUBELET_IMAGE_URL=docker://k8s.gcr.io/hyperkube - KUBELET_IMAGE_TAG=v1.13.2 + KUBELET_IMAGE_TAG=v1.13.3 - path: /etc/sysctl.d/max-user-watches.conf filesystem: root contents: @@ -112,7 +112,7 @@ storage: --volume config,kind=host,source=/etc/kubernetes \ --mount volume=config,target=/etc/kubernetes \ --insecure-options=image \ - docker://k8s.gcr.io/hyperkube:v1.13.2 \ + docker://k8s.gcr.io/hyperkube:v1.13.3 \ --net=host \ --dns=host \ --exec=/kubectl -- --kubeconfig=/etc/kubernetes/kubeconfig delete node $(hostname) diff --git a/google-cloud/fedora-atomic/kubernetes/README.md b/google-cloud/fedora-atomic/kubernetes/README.md index b07079cd3..ae7e6758f 100644 --- a/google-cloud/fedora-atomic/kubernetes/README.md +++ b/google-cloud/fedora-atomic/kubernetes/README.md @@ -11,7 +11,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features -* Kubernetes v1.13.2 (upstream, via [kubernetes-incubator/bootkube](https://github.com/kubernetes-incubator/bootkube)) +* Kubernetes v1.13.3 (upstream, via [kubernetes-incubator/bootkube](https://github.com/kubernetes-incubator/bootkube)) * Single or multi-master, [Calico](https://www.projectcalico.org/) or [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/) * Advanced features like [worker pools](https://typhoon.psdn.io/advanced/worker-pools/) and [preemptible](https://typhoon.psdn.io/cl/google-cloud/#preemption) workers diff --git a/google-cloud/fedora-atomic/kubernetes/bootkube.tf b/google-cloud/fedora-atomic/kubernetes/bootkube.tf index 842e60326..ea1f3a975 100644 --- a/google-cloud/fedora-atomic/kubernetes/bootkube.tf +++ b/google-cloud/fedora-atomic/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=7dc8f8bf8c1a6bdb9170dd19836520ce436e26d7" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=c12a11c8006606b59335ecc994abe22358aaf68b" cluster_name = "${var.cluster_name}" api_servers = ["${format("%s.%s", var.cluster_name, var.dns_zone)}"] diff --git a/google-cloud/fedora-atomic/kubernetes/cloudinit/controller.yaml.tmpl b/google-cloud/fedora-atomic/kubernetes/cloudinit/controller.yaml.tmpl index 2e20258f0..7225a1490 100644 --- a/google-cloud/fedora-atomic/kubernetes/cloudinit/controller.yaml.tmpl +++ b/google-cloud/fedora-atomic/kubernetes/cloudinit/controller.yaml.tmpl @@ -79,7 +79,7 @@ runcmd: - [systemctl, daemon-reload] - [systemctl, restart, NetworkManager] - "atomic install --system --name=etcd quay.io/poseidon/etcd:v3.3.11" - - "atomic install --system --name=kubelet quay.io/poseidon/kubelet:v1.13.2" + - "atomic install --system --name=kubelet quay.io/poseidon/kubelet:v1.13.3" - "atomic install --system --name=bootkube quay.io/poseidon/bootkube:v0.14.0" - [systemctl, start, --no-block, etcd.service] - [systemctl, start, --no-block, kubelet.service] diff --git a/google-cloud/fedora-atomic/kubernetes/workers/cloudinit/worker.yaml.tmpl b/google-cloud/fedora-atomic/kubernetes/workers/cloudinit/worker.yaml.tmpl index bcb2949be..c14590c19 100644 --- a/google-cloud/fedora-atomic/kubernetes/workers/cloudinit/worker.yaml.tmpl +++ b/google-cloud/fedora-atomic/kubernetes/workers/cloudinit/worker.yaml.tmpl @@ -54,7 +54,7 @@ bootcmd: runcmd: - [systemctl, daemon-reload] - [systemctl, restart, NetworkManager] - - "atomic install --system --name=kubelet quay.io/poseidon/kubelet:v1.13.2" + - "atomic install --system --name=kubelet quay.io/poseidon/kubelet:v1.13.3" - [systemctl, start, --no-block, kubelet.service] users: - default From 949ce21fb251b750bb674c06817127589a05aec2 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Sat, 2 Feb 2019 00:13:24 -0800 Subject: [PATCH 016/523] Update Prometheus from v2.7.0 to v2.7.1 * https://github.com/prometheus/prometheus/releases/tag/v2.7.1 --- CHANGES.md | 2 +- addons/prometheus/deployment.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 55675a54d..c9635111a 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -25,7 +25,7 @@ Notable changes between versions. #### Addons * Update nginx-ingress from v0.21.0 to v0.22.0 -* Update Prometheus from v2.6.0 to v2.7.0 +* Update Prometheus from v2.6.0 to v2.7.1 * Update kube-state-metrics from v1.4.0 to v1.5.0 * Fix ClusterRole to collect and export PodDisruptionBudget metrics ([#383](https://github.com/poseidon/typhoon/pull/383)) * Update node-exporter from v0.15.2 to v0.17.0 diff --git a/addons/prometheus/deployment.yaml b/addons/prometheus/deployment.yaml index 380a222eb..8e9f26a79 100644 --- a/addons/prometheus/deployment.yaml +++ b/addons/prometheus/deployment.yaml @@ -20,7 +20,7 @@ spec: serviceAccountName: prometheus containers: - name: prometheus - image: quay.io/prometheus/prometheus:v2.7.0 + image: quay.io/prometheus/prometheus:v2.7.1 args: - --web.listen-address=0.0.0.0:9090 - --config.file=/etc/prometheus/prometheus.yaml From d5537405e1d62cfac31502f5cf10683fcc26f3b1 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Sat, 2 Feb 2019 14:54:18 -0800 Subject: [PATCH 017/523] Add CHANGES note about reducing the pod eviciton timeout --- CHANGES.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGES.md b/CHANGES.md index c9635111a..5ae186c1a 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -4,12 +4,16 @@ Notable changes between versions. ## Latest +## v1.13.3 + * Kubernetes [v1.13.3](https://github.com/kubernetes/kubernetes/blob/master/CHANGELOG-1.13.md#v1133) * Update etcd from v3.3.10 to [v3.3.11](https://github.com/etcd-io/etcd/blob/master/CHANGELOG-3.3.md#v3311-2019-1-11) * Update CoreDNS from v1.3.0 to [v1.3.1](https://coredns.io/2019/01/13/coredns-1.3.1-release/) * Switch from the `proxy` plugin to the faster `forward` plugin for upsteam resolvers * Update Calico from v3.4.0 to [v3.5.0](https://docs.projectcalico.org/v3.5/releases/) * Update flannel from v0.10.0 to [v0.11.0](https://github.com/coreos/flannel/releases/tag/v0.11.0) +* Reduce pod eviction timeout for deleting pods on unready nodes to 1 minute + * Respond more quickly to node preemption (previously 5 minutes) * Fix automatic worker deletion on shutdown for cloud platforms * Lowering Kubelet privileges in [#372](https://github.com/poseidon/typhoon/pull/372) dropped a needed node deletion authorization. Scale-in due to manual terraform apply (any cloud), AWS spot termination, or Azure low priority deletion left old nodes registered, requiring manual deletion (`kubectl delete node name`) From 0200058e0e1a3b5159876ce4b17009f562b6b7c0 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Sat, 9 Feb 2019 11:49:31 -0800 Subject: [PATCH 018/523] Update Calico from v3.5.0 to v3.5.1 * Fix in confd https://github.com/projectcalico/confd/pull/205 --- CHANGES.md | 2 ++ aws/container-linux/kubernetes/bootkube.tf | 2 +- aws/fedora-atomic/kubernetes/bootkube.tf | 2 +- azure/container-linux/kubernetes/bootkube.tf | 2 +- bare-metal/container-linux/kubernetes/bootkube.tf | 2 +- bare-metal/fedora-atomic/kubernetes/bootkube.tf | 2 +- digital-ocean/container-linux/kubernetes/bootkube.tf | 2 +- digital-ocean/fedora-atomic/kubernetes/bootkube.tf | 2 +- google-cloud/container-linux/kubernetes/bootkube.tf | 2 +- google-cloud/fedora-atomic/kubernetes/bootkube.tf | 2 +- 10 files changed, 11 insertions(+), 9 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 5ae186c1a..8ceb24ae6 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -4,6 +4,8 @@ Notable changes between versions. ## Latest +* Update Calico from v3.5.0 to v3.5.1 + ## v1.13.3 * Kubernetes [v1.13.3](https://github.com/kubernetes/kubernetes/blob/master/CHANGELOG-1.13.md#v1133) diff --git a/aws/container-linux/kubernetes/bootkube.tf b/aws/container-linux/kubernetes/bootkube.tf index 2f5de5240..fbaeb6616 100644 --- a/aws/container-linux/kubernetes/bootkube.tf +++ b/aws/container-linux/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=c12a11c8006606b59335ecc994abe22358aaf68b" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=4d315afd41dea46fdae90bb629dfc027a6eaa2ad" cluster_name = "${var.cluster_name}" api_servers = ["${format("%s.%s", var.cluster_name, var.dns_zone)}"] diff --git a/aws/fedora-atomic/kubernetes/bootkube.tf b/aws/fedora-atomic/kubernetes/bootkube.tf index 52f631b9c..e7ed6cda1 100644 --- a/aws/fedora-atomic/kubernetes/bootkube.tf +++ b/aws/fedora-atomic/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=c12a11c8006606b59335ecc994abe22358aaf68b" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=4d315afd41dea46fdae90bb629dfc027a6eaa2ad" cluster_name = "${var.cluster_name}" api_servers = ["${format("%s.%s", var.cluster_name, var.dns_zone)}"] diff --git a/azure/container-linux/kubernetes/bootkube.tf b/azure/container-linux/kubernetes/bootkube.tf index 6576dbfc8..d8b9d1e43 100644 --- a/azure/container-linux/kubernetes/bootkube.tf +++ b/azure/container-linux/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=c12a11c8006606b59335ecc994abe22358aaf68b" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=4d315afd41dea46fdae90bb629dfc027a6eaa2ad" cluster_name = "${var.cluster_name}" api_servers = ["${format("%s.%s", var.cluster_name, var.dns_zone)}"] diff --git a/bare-metal/container-linux/kubernetes/bootkube.tf b/bare-metal/container-linux/kubernetes/bootkube.tf index 6ffe17641..e56dc536d 100644 --- a/bare-metal/container-linux/kubernetes/bootkube.tf +++ b/bare-metal/container-linux/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=c12a11c8006606b59335ecc994abe22358aaf68b" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=4d315afd41dea46fdae90bb629dfc027a6eaa2ad" cluster_name = "${var.cluster_name}" api_servers = ["${var.k8s_domain_name}"] diff --git a/bare-metal/fedora-atomic/kubernetes/bootkube.tf b/bare-metal/fedora-atomic/kubernetes/bootkube.tf index 2156518e2..bc382e386 100644 --- a/bare-metal/fedora-atomic/kubernetes/bootkube.tf +++ b/bare-metal/fedora-atomic/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=c12a11c8006606b59335ecc994abe22358aaf68b" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=4d315afd41dea46fdae90bb629dfc027a6eaa2ad" cluster_name = "${var.cluster_name}" api_servers = ["${var.k8s_domain_name}"] diff --git a/digital-ocean/container-linux/kubernetes/bootkube.tf b/digital-ocean/container-linux/kubernetes/bootkube.tf index 3396991ba..f96d64cf6 100644 --- a/digital-ocean/container-linux/kubernetes/bootkube.tf +++ b/digital-ocean/container-linux/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=c12a11c8006606b59335ecc994abe22358aaf68b" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=4d315afd41dea46fdae90bb629dfc027a6eaa2ad" cluster_name = "${var.cluster_name}" api_servers = ["${format("%s.%s", var.cluster_name, var.dns_zone)}"] diff --git a/digital-ocean/fedora-atomic/kubernetes/bootkube.tf b/digital-ocean/fedora-atomic/kubernetes/bootkube.tf index 3eeefee1c..60ba58574 100644 --- a/digital-ocean/fedora-atomic/kubernetes/bootkube.tf +++ b/digital-ocean/fedora-atomic/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=c12a11c8006606b59335ecc994abe22358aaf68b" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=4d315afd41dea46fdae90bb629dfc027a6eaa2ad" cluster_name = "${var.cluster_name}" api_servers = ["${format("%s.%s", var.cluster_name, var.dns_zone)}"] diff --git a/google-cloud/container-linux/kubernetes/bootkube.tf b/google-cloud/container-linux/kubernetes/bootkube.tf index afed043f4..00e4be158 100644 --- a/google-cloud/container-linux/kubernetes/bootkube.tf +++ b/google-cloud/container-linux/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=c12a11c8006606b59335ecc994abe22358aaf68b" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=4d315afd41dea46fdae90bb629dfc027a6eaa2ad" cluster_name = "${var.cluster_name}" api_servers = ["${format("%s.%s", var.cluster_name, var.dns_zone)}"] diff --git a/google-cloud/fedora-atomic/kubernetes/bootkube.tf b/google-cloud/fedora-atomic/kubernetes/bootkube.tf index ea1f3a975..f0ad310a5 100644 --- a/google-cloud/fedora-atomic/kubernetes/bootkube.tf +++ b/google-cloud/fedora-atomic/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=c12a11c8006606b59335ecc994abe22358aaf68b" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=4d315afd41dea46fdae90bb629dfc027a6eaa2ad" cluster_name = "${var.cluster_name}" api_servers = ["${format("%s.%s", var.cluster_name, var.dns_zone)}"] From 584088397cda58637ac70e4013e5bade9b03ac70 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Sat, 9 Feb 2019 11:54:54 -0800 Subject: [PATCH 019/523] Update etcd from v3.3.11 to v3.3.12 * https://github.com/etcd-io/etcd/releases/tag/v3.3.12 --- CHANGES.md | 1 + aws/container-linux/kubernetes/cl/controller.yaml.tmpl | 2 +- aws/fedora-atomic/kubernetes/cloudinit/controller.yaml.tmpl | 2 +- azure/container-linux/kubernetes/cl/controller.yaml.tmpl | 2 +- bare-metal/container-linux/kubernetes/cl/controller.yaml.tmpl | 2 +- .../fedora-atomic/kubernetes/cloudinit/controller.yaml.tmpl | 2 +- .../container-linux/kubernetes/cl/controller.yaml.tmpl | 2 +- .../fedora-atomic/kubernetes/cloudinit/controller.yaml.tmpl | 2 +- google-cloud/container-linux/kubernetes/cl/controller.yaml.tmpl | 2 +- .../fedora-atomic/kubernetes/cloudinit/controller.yaml.tmpl | 2 +- 10 files changed, 10 insertions(+), 9 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 8ceb24ae6..f06daac38 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -4,6 +4,7 @@ Notable changes between versions. ## Latest +* Update etcd from v3.3.11 to [v3.3.12](https://github.com/etcd-io/etcd/releases/tag/v3.3.12) * Update Calico from v3.5.0 to v3.5.1 ## v1.13.3 diff --git a/aws/container-linux/kubernetes/cl/controller.yaml.tmpl b/aws/container-linux/kubernetes/cl/controller.yaml.tmpl index e8f847ec3..63285ee56 100644 --- a/aws/container-linux/kubernetes/cl/controller.yaml.tmpl +++ b/aws/container-linux/kubernetes/cl/controller.yaml.tmpl @@ -7,7 +7,7 @@ systemd: - name: 40-etcd-cluster.conf contents: | [Service] - Environment="ETCD_IMAGE_TAG=v3.3.11" + Environment="ETCD_IMAGE_TAG=v3.3.12" Environment="ETCD_NAME=${etcd_name}" Environment="ETCD_ADVERTISE_CLIENT_URLS=https://${etcd_domain}:2379" Environment="ETCD_INITIAL_ADVERTISE_PEER_URLS=https://${etcd_domain}:2380" diff --git a/aws/fedora-atomic/kubernetes/cloudinit/controller.yaml.tmpl b/aws/fedora-atomic/kubernetes/cloudinit/controller.yaml.tmpl index 7225a1490..483591e63 100644 --- a/aws/fedora-atomic/kubernetes/cloudinit/controller.yaml.tmpl +++ b/aws/fedora-atomic/kubernetes/cloudinit/controller.yaml.tmpl @@ -78,7 +78,7 @@ bootcmd: runcmd: - [systemctl, daemon-reload] - [systemctl, restart, NetworkManager] - - "atomic install --system --name=etcd quay.io/poseidon/etcd:v3.3.11" + - "atomic install --system --name=etcd quay.io/poseidon/etcd:v3.3.12" - "atomic install --system --name=kubelet quay.io/poseidon/kubelet:v1.13.3" - "atomic install --system --name=bootkube quay.io/poseidon/bootkube:v0.14.0" - [systemctl, start, --no-block, etcd.service] diff --git a/azure/container-linux/kubernetes/cl/controller.yaml.tmpl b/azure/container-linux/kubernetes/cl/controller.yaml.tmpl index e8f847ec3..63285ee56 100644 --- a/azure/container-linux/kubernetes/cl/controller.yaml.tmpl +++ b/azure/container-linux/kubernetes/cl/controller.yaml.tmpl @@ -7,7 +7,7 @@ systemd: - name: 40-etcd-cluster.conf contents: | [Service] - Environment="ETCD_IMAGE_TAG=v3.3.11" + Environment="ETCD_IMAGE_TAG=v3.3.12" Environment="ETCD_NAME=${etcd_name}" Environment="ETCD_ADVERTISE_CLIENT_URLS=https://${etcd_domain}:2379" Environment="ETCD_INITIAL_ADVERTISE_PEER_URLS=https://${etcd_domain}:2380" diff --git a/bare-metal/container-linux/kubernetes/cl/controller.yaml.tmpl b/bare-metal/container-linux/kubernetes/cl/controller.yaml.tmpl index f5de49c50..6a1c80e95 100644 --- a/bare-metal/container-linux/kubernetes/cl/controller.yaml.tmpl +++ b/bare-metal/container-linux/kubernetes/cl/controller.yaml.tmpl @@ -7,7 +7,7 @@ systemd: - name: 40-etcd-cluster.conf contents: | [Service] - Environment="ETCD_IMAGE_TAG=v3.3.11" + Environment="ETCD_IMAGE_TAG=v3.3.12" Environment="ETCD_NAME=${etcd_name}" Environment="ETCD_ADVERTISE_CLIENT_URLS=https://${domain_name}:2379" Environment="ETCD_INITIAL_ADVERTISE_PEER_URLS=https://${domain_name}:2380" diff --git a/bare-metal/fedora-atomic/kubernetes/cloudinit/controller.yaml.tmpl b/bare-metal/fedora-atomic/kubernetes/cloudinit/controller.yaml.tmpl index 8416ee300..2cba4d893 100644 --- a/bare-metal/fedora-atomic/kubernetes/cloudinit/controller.yaml.tmpl +++ b/bare-metal/fedora-atomic/kubernetes/cloudinit/controller.yaml.tmpl @@ -84,7 +84,7 @@ runcmd: - [systemctl, daemon-reload] - [systemctl, restart, NetworkManager] - [hostnamectl, set-hostname, ${domain_name}] - - "atomic install --system --name=etcd quay.io/poseidon/etcd:v3.3.11" + - "atomic install --system --name=etcd quay.io/poseidon/etcd:v3.3.12" - "atomic install --system --name=kubelet quay.io/poseidon/kubelet:v1.13.3" - "atomic install --system --name=bootkube quay.io/poseidon/bootkube:v0.14.0" - [systemctl, start, --no-block, etcd.service] diff --git a/digital-ocean/container-linux/kubernetes/cl/controller.yaml.tmpl b/digital-ocean/container-linux/kubernetes/cl/controller.yaml.tmpl index 66d5fdb6d..d8219f394 100644 --- a/digital-ocean/container-linux/kubernetes/cl/controller.yaml.tmpl +++ b/digital-ocean/container-linux/kubernetes/cl/controller.yaml.tmpl @@ -7,7 +7,7 @@ systemd: - name: 40-etcd-cluster.conf contents: | [Service] - Environment="ETCD_IMAGE_TAG=v3.3.11" + Environment="ETCD_IMAGE_TAG=v3.3.12" Environment="ETCD_NAME=${etcd_name}" Environment="ETCD_ADVERTISE_CLIENT_URLS=https://${etcd_domain}:2379" Environment="ETCD_INITIAL_ADVERTISE_PEER_URLS=https://${etcd_domain}:2380" diff --git a/digital-ocean/fedora-atomic/kubernetes/cloudinit/controller.yaml.tmpl b/digital-ocean/fedora-atomic/kubernetes/cloudinit/controller.yaml.tmpl index b6a62d581..74798986c 100644 --- a/digital-ocean/fedora-atomic/kubernetes/cloudinit/controller.yaml.tmpl +++ b/digital-ocean/fedora-atomic/kubernetes/cloudinit/controller.yaml.tmpl @@ -75,7 +75,7 @@ bootcmd: - [modprobe, ip_vs] runcmd: - [systemctl, daemon-reload] - - "atomic install --system --name=etcd quay.io/poseidon/etcd:v3.3.11" + - "atomic install --system --name=etcd quay.io/poseidon/etcd:v3.3.12" - "atomic install --system --name=kubelet quay.io/poseidon/kubelet:v1.13.3" - "atomic install --system --name=bootkube quay.io/poseidon/bootkube:v0.14.0" - [systemctl, start, --no-block, etcd.service] diff --git a/google-cloud/container-linux/kubernetes/cl/controller.yaml.tmpl b/google-cloud/container-linux/kubernetes/cl/controller.yaml.tmpl index 7fcc5b2ef..1fc03ce0c 100644 --- a/google-cloud/container-linux/kubernetes/cl/controller.yaml.tmpl +++ b/google-cloud/container-linux/kubernetes/cl/controller.yaml.tmpl @@ -7,7 +7,7 @@ systemd: - name: 40-etcd-cluster.conf contents: | [Service] - Environment="ETCD_IMAGE_TAG=v3.3.11" + Environment="ETCD_IMAGE_TAG=v3.3.12" Environment="ETCD_NAME=${etcd_name}" Environment="ETCD_ADVERTISE_CLIENT_URLS=https://${etcd_domain}:2379" Environment="ETCD_INITIAL_ADVERTISE_PEER_URLS=https://${etcd_domain}:2380" diff --git a/google-cloud/fedora-atomic/kubernetes/cloudinit/controller.yaml.tmpl b/google-cloud/fedora-atomic/kubernetes/cloudinit/controller.yaml.tmpl index 7225a1490..483591e63 100644 --- a/google-cloud/fedora-atomic/kubernetes/cloudinit/controller.yaml.tmpl +++ b/google-cloud/fedora-atomic/kubernetes/cloudinit/controller.yaml.tmpl @@ -78,7 +78,7 @@ bootcmd: runcmd: - [systemctl, daemon-reload] - [systemctl, restart, NetworkManager] - - "atomic install --system --name=etcd quay.io/poseidon/etcd:v3.3.11" + - "atomic install --system --name=etcd quay.io/poseidon/etcd:v3.3.12" - "atomic install --system --name=kubelet quay.io/poseidon/kubelet:v1.13.3" - "atomic install --system --name=bootkube quay.io/poseidon/bootkube:v0.14.0" - [systemctl, start, --no-block, etcd.service] From 3e4b3bfb048dfda4ddfccf015473e564daff9037 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Sat, 9 Feb 2019 12:23:12 -0800 Subject: [PATCH 020/523] Raise nginx-ingress liveness/readiness timeout * Under heavy load, avoid timeouts causing nginx-ingress restarts https://github.com/kubernetes/ingress-nginx/pull/3737 --- CHANGES.md | 4 ++++ addons/nginx-ingress/aws/deployment.yaml | 4 ++-- addons/nginx-ingress/azure/deployment.yaml | 4 ++-- addons/nginx-ingress/bare-metal/deployment.yaml | 4 ++-- addons/nginx-ingress/digital-ocean/daemonset.yaml | 4 ++-- addons/nginx-ingress/google-cloud/deployment.yaml | 4 ++-- 6 files changed, 14 insertions(+), 10 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index f06daac38..46faa2a4c 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -7,6 +7,10 @@ Notable changes between versions. * Update etcd from v3.3.11 to [v3.3.12](https://github.com/etcd-io/etcd/releases/tag/v3.3.12) * Update Calico from v3.5.0 to v3.5.1 +#### Addons + +* Raise nginx-ingress liveness/readiness timeout to 5 seconds + ## v1.13.3 * Kubernetes [v1.13.3](https://github.com/kubernetes/kubernetes/blob/master/CHANGELOG-1.13.md#v1133) diff --git a/addons/nginx-ingress/aws/deployment.yaml b/addons/nginx-ingress/aws/deployment.yaml index c06e798dc..cd841eb5a 100644 --- a/addons/nginx-ingress/aws/deployment.yaml +++ b/addons/nginx-ingress/aws/deployment.yaml @@ -58,7 +58,7 @@ spec: initialDelaySeconds: 10 periodSeconds: 10 successThreshold: 1 - timeoutSeconds: 1 + timeoutSeconds: 5 readinessProbe: failureThreshold: 3 httpGet: @@ -67,7 +67,7 @@ spec: scheme: HTTP periodSeconds: 10 successThreshold: 1 - timeoutSeconds: 1 + timeoutSeconds: 5 securityContext: capabilities: add: diff --git a/addons/nginx-ingress/azure/deployment.yaml b/addons/nginx-ingress/azure/deployment.yaml index c06e798dc..cd841eb5a 100644 --- a/addons/nginx-ingress/azure/deployment.yaml +++ b/addons/nginx-ingress/azure/deployment.yaml @@ -58,7 +58,7 @@ spec: initialDelaySeconds: 10 periodSeconds: 10 successThreshold: 1 - timeoutSeconds: 1 + timeoutSeconds: 5 readinessProbe: failureThreshold: 3 httpGet: @@ -67,7 +67,7 @@ spec: scheme: HTTP periodSeconds: 10 successThreshold: 1 - timeoutSeconds: 1 + timeoutSeconds: 5 securityContext: capabilities: add: diff --git a/addons/nginx-ingress/bare-metal/deployment.yaml b/addons/nginx-ingress/bare-metal/deployment.yaml index 5c22a8d26..7ba6b8667 100644 --- a/addons/nginx-ingress/bare-metal/deployment.yaml +++ b/addons/nginx-ingress/bare-metal/deployment.yaml @@ -53,7 +53,7 @@ spec: periodSeconds: 10 successThreshold: 1 failureThreshold: 3 - timeoutSeconds: 1 + timeoutSeconds: 5 readinessProbe: httpGet: path: /healthz @@ -62,7 +62,7 @@ spec: periodSeconds: 10 successThreshold: 1 failureThreshold: 3 - timeoutSeconds: 1 + timeoutSeconds: 5 securityContext: capabilities: add: diff --git a/addons/nginx-ingress/digital-ocean/daemonset.yaml b/addons/nginx-ingress/digital-ocean/daemonset.yaml index 5909112c9..e733b3b58 100644 --- a/addons/nginx-ingress/digital-ocean/daemonset.yaml +++ b/addons/nginx-ingress/digital-ocean/daemonset.yaml @@ -58,7 +58,7 @@ spec: initialDelaySeconds: 10 periodSeconds: 10 successThreshold: 1 - timeoutSeconds: 1 + timeoutSeconds: 5 readinessProbe: failureThreshold: 3 httpGet: @@ -67,7 +67,7 @@ spec: scheme: HTTP periodSeconds: 10 successThreshold: 1 - timeoutSeconds: 1 + timeoutSeconds: 5 securityContext: capabilities: add: diff --git a/addons/nginx-ingress/google-cloud/deployment.yaml b/addons/nginx-ingress/google-cloud/deployment.yaml index c06e798dc..cd841eb5a 100644 --- a/addons/nginx-ingress/google-cloud/deployment.yaml +++ b/addons/nginx-ingress/google-cloud/deployment.yaml @@ -58,7 +58,7 @@ spec: initialDelaySeconds: 10 periodSeconds: 10 successThreshold: 1 - timeoutSeconds: 1 + timeoutSeconds: 5 readinessProbe: failureThreshold: 3 httpGet: @@ -67,7 +67,7 @@ spec: scheme: HTTP periodSeconds: 10 successThreshold: 1 - timeoutSeconds: 1 + timeoutSeconds: 5 securityContext: capabilities: add: From 9c59f393a59c8e231417c7a911164ae5307a24b6 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Thu, 7 Feb 2019 23:26:59 -0800 Subject: [PATCH 021/523] Add Kubernetes pod name to metrics discovered from service endpoints * Prometheus queries from some upstreams use joins of node-exporter and kube-state-metrics metrics by (namespace,pod). Add the Kubernetes pod name to service endpoint metrics * Rename the kubernetes_namespace field to namespace * Honor labels since kube-state-metrics already include a `pod` field that should not be overridden --- CHANGES.md | 3 +++ addons/prometheus/config.yaml | 9 ++++++--- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 46faa2a4c..f89ff56f6 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -10,6 +10,9 @@ Notable changes between versions. #### Addons * Raise nginx-ingress liveness/readiness timeout to 5 seconds +* Improve Prometheus metrics labels + * Add `pod` name label to metrics discovered via service endpoints + * Rename `kubernetes_namespace` label to `namespace` ## v1.13.3 diff --git a/addons/prometheus/config.yaml b/addons/prometheus/config.yaml index b1d383071..2408b3de2 100644 --- a/addons/prometheus/config.yaml +++ b/addons/prometheus/config.yaml @@ -119,10 +119,10 @@ data: # * `prometheus.io/port`: If the metrics are exposed on a different port to the # service then set this appropriately. - job_name: 'kubernetes-service-endpoints' - kubernetes_sd_configs: - role: endpoints + honor_labels: true relabel_configs: - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_scrape] action: keep @@ -144,7 +144,10 @@ data: regex: __meta_kubernetes_service_label_(.+) - source_labels: [__meta_kubernetes_namespace] action: replace - target_label: kubernetes_namespace + target_label: namespace + - source_labels: [__meta_kubernetes_pod_name] + action: replace + target_label: pod - source_labels: [__meta_kubernetes_service_name] action: replace target_label: job @@ -177,7 +180,7 @@ data: - action: labelmap regex: __meta_kubernetes_service_label_(.+) - source_labels: [__meta_kubernetes_namespace] - target_label: kubernetes_namespace + target_label: namespace - source_labels: [__meta_kubernetes_service_name] target_label: job From b13a651cfe979f117466811f334e9fc558b696c5 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Sat, 9 Feb 2019 16:47:19 -0800 Subject: [PATCH 022/523] Drop metrics that are unset, high cardinality, or extraneous * https://github.com/coreos/prometheus-operator/pull/2387 * https://github.com/coreos/prometheus-operator/pull/1959 --- CHANGES.md | 2 +- addons/prometheus/config.yaml | 23 +++++++++++++++++++++++ 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index f89ff56f6..e2cb82c1e 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -10,7 +10,7 @@ Notable changes between versions. #### Addons * Raise nginx-ingress liveness/readiness timeout to 5 seconds -* Improve Prometheus metrics labels +* Improve Prometheus metrics labels and drop extraneous metrics ([#397](https://github.com/poseidon/typhoon/pull/397)) * Add `pod` name label to metrics discovered via service endpoints * Rename `kubernetes_namespace` label to `namespace` diff --git a/addons/prometheus/config.yaml b/addons/prometheus/config.yaml index 2408b3de2..3ea5b65dd 100644 --- a/addons/prometheus/config.yaml +++ b/addons/prometheus/config.yaml @@ -55,6 +55,17 @@ data: action: replace target_label: job + metric_relabel_configs: + - source_labels: [__name__] + action: drop + regex: etcd_(debugging|disk|request|server).* + - source_labels: [__name__] + action: drop + regex: apiserver_admission_controller_admission_latencies_seconds_.* + - source_labels: [__name__] + action: drop + regex: apiserver_admission_step_admission_latencies_seconds_.* + # Scrape config for node (i.e. kubelet) /metrics (e.g. 'kubelet_'). Explore # metrics from a node by scraping kubelet (127.0.0.1:10250/metrics). - job_name: 'kubelet' @@ -89,6 +100,13 @@ data: relabel_configs: - action: labelmap regex: __meta_kubernetes_node_label_(.+) + metric_relabel_configs: + - source_labels: [__name__, image] + action: drop + regex: container_([a-z_]+); + - source_labels: [__name__] + action: drop + regex: container_(network_tcp_usage_total|network_udp_usage_total|tasks_state|cpu_load_average_10s) # Scrap etcd metrics from controllers via listen-metrics-urls @@ -151,6 +169,11 @@ data: - source_labels: [__meta_kubernetes_service_name] action: replace target_label: job + + metric_relabel_configs: + - source_labels: [__name__] + action: drop + regex: etcd_(debugging|disk|request|server).* # Example scrape config for probing services via the Blackbox Exporter. # From 170ef74eea50fa1b8c0aa8012f23b40c522137e4 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Sat, 16 Feb 2019 14:13:08 -0800 Subject: [PATCH 023/523] Remove Nginx Ingress default backend * nginx-ingress no longer requires a configured default-backend, it will respond with its own 404 page starting in v0.21.0 * https://github.com/kubernetes/ingress-nginx/pull/3196 --- CHANGES.md | 2 + .../aws/default-backend/deployment.yaml | 42 ------------------- .../aws/default-backend/service.yaml | 15 ------- addons/nginx-ingress/aws/deployment.yaml | 1 - .../azure/default-backend/deployment.yaml | 42 ------------------- .../azure/default-backend/service.yaml | 15 ------- addons/nginx-ingress/azure/deployment.yaml | 1 - .../default-backend/deployment.yaml | 42 ------------------- .../bare-metal/default-backend/service.yaml | 15 ------- .../nginx-ingress/bare-metal/deployment.yaml | 1 - .../digital-ocean/daemonset.yaml | 1 - .../default-backend/deployment.yaml | 42 ------------------- .../default-backend/service.yaml | 15 ------- .../default-backend/deployment.yaml | 42 ------------------- .../google-cloud/default-backend/service.yaml | 15 ------- .../google-cloud/deployment.yaml | 1 - 16 files changed, 2 insertions(+), 290 deletions(-) delete mode 100644 addons/nginx-ingress/aws/default-backend/deployment.yaml delete mode 100644 addons/nginx-ingress/aws/default-backend/service.yaml delete mode 100644 addons/nginx-ingress/azure/default-backend/deployment.yaml delete mode 100644 addons/nginx-ingress/azure/default-backend/service.yaml delete mode 100644 addons/nginx-ingress/bare-metal/default-backend/deployment.yaml delete mode 100644 addons/nginx-ingress/bare-metal/default-backend/service.yaml delete mode 100644 addons/nginx-ingress/digital-ocean/default-backend/deployment.yaml delete mode 100644 addons/nginx-ingress/digital-ocean/default-backend/service.yaml delete mode 100644 addons/nginx-ingress/google-cloud/default-backend/deployment.yaml delete mode 100644 addons/nginx-ingress/google-cloud/default-backend/service.yaml diff --git a/CHANGES.md b/CHANGES.md index e2cb82c1e..32ffcf312 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -10,6 +10,8 @@ Notable changes between versions. #### Addons * Raise nginx-ingress liveness/readiness timeout to 5 seconds +* Remove nginx-ingess default-backend ([#401](https://github.com/poseidon/typhoon/pull/401)) + * nginx-ingress now responds with its own 404 page by default * Improve Prometheus metrics labels and drop extraneous metrics ([#397](https://github.com/poseidon/typhoon/pull/397)) * Add `pod` name label to metrics discovered via service endpoints * Rename `kubernetes_namespace` label to `namespace` diff --git a/addons/nginx-ingress/aws/default-backend/deployment.yaml b/addons/nginx-ingress/aws/default-backend/deployment.yaml deleted file mode 100644 index ce6401897..000000000 --- a/addons/nginx-ingress/aws/default-backend/deployment.yaml +++ /dev/null @@ -1,42 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: default-backend - namespace: ingress -spec: - replicas: 1 - selector: - matchLabels: - name: default-backend - phase: prod - template: - metadata: - labels: - name: default-backend - phase: prod - annotations: - seccomp.security.alpha.kubernetes.io/pod: 'docker/default' - spec: - containers: - - name: default-backend - # Any image is permissable as long as: - # 1. It serves a 404 page at / - # 2. It serves 200 on a /healthz endpoint - image: k8s.gcr.io/defaultbackend:1.4 - ports: - - containerPort: 8080 - resources: - limits: - cpu: 10m - memory: 20Mi - requests: - cpu: 10m - memory: 20Mi - livenessProbe: - httpGet: - path: /healthz - port: 8080 - scheme: HTTP - initialDelaySeconds: 30 - timeoutSeconds: 5 - terminationGracePeriodSeconds: 60 diff --git a/addons/nginx-ingress/aws/default-backend/service.yaml b/addons/nginx-ingress/aws/default-backend/service.yaml deleted file mode 100644 index 87997aba0..000000000 --- a/addons/nginx-ingress/aws/default-backend/service.yaml +++ /dev/null @@ -1,15 +0,0 @@ -apiVersion: v1 -kind: Service -metadata: - name: default-backend - namespace: ingress -spec: - type: ClusterIP - selector: - name: default-backend - phase: prod - ports: - - name: http - protocol: TCP - port: 80 - targetPort: 8080 diff --git a/addons/nginx-ingress/aws/deployment.yaml b/addons/nginx-ingress/aws/deployment.yaml index cd841eb5a..1f537e253 100644 --- a/addons/nginx-ingress/aws/deployment.yaml +++ b/addons/nginx-ingress/aws/deployment.yaml @@ -27,7 +27,6 @@ spec: image: quay.io/kubernetes-ingress-controller/nginx-ingress-controller:0.22.0 args: - /nginx-ingress-controller - - --default-backend-service=$(POD_NAMESPACE)/default-backend - --ingress-class=public # use downward API env: diff --git a/addons/nginx-ingress/azure/default-backend/deployment.yaml b/addons/nginx-ingress/azure/default-backend/deployment.yaml deleted file mode 100644 index ce6401897..000000000 --- a/addons/nginx-ingress/azure/default-backend/deployment.yaml +++ /dev/null @@ -1,42 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: default-backend - namespace: ingress -spec: - replicas: 1 - selector: - matchLabels: - name: default-backend - phase: prod - template: - metadata: - labels: - name: default-backend - phase: prod - annotations: - seccomp.security.alpha.kubernetes.io/pod: 'docker/default' - spec: - containers: - - name: default-backend - # Any image is permissable as long as: - # 1. It serves a 404 page at / - # 2. It serves 200 on a /healthz endpoint - image: k8s.gcr.io/defaultbackend:1.4 - ports: - - containerPort: 8080 - resources: - limits: - cpu: 10m - memory: 20Mi - requests: - cpu: 10m - memory: 20Mi - livenessProbe: - httpGet: - path: /healthz - port: 8080 - scheme: HTTP - initialDelaySeconds: 30 - timeoutSeconds: 5 - terminationGracePeriodSeconds: 60 diff --git a/addons/nginx-ingress/azure/default-backend/service.yaml b/addons/nginx-ingress/azure/default-backend/service.yaml deleted file mode 100644 index 87997aba0..000000000 --- a/addons/nginx-ingress/azure/default-backend/service.yaml +++ /dev/null @@ -1,15 +0,0 @@ -apiVersion: v1 -kind: Service -metadata: - name: default-backend - namespace: ingress -spec: - type: ClusterIP - selector: - name: default-backend - phase: prod - ports: - - name: http - protocol: TCP - port: 80 - targetPort: 8080 diff --git a/addons/nginx-ingress/azure/deployment.yaml b/addons/nginx-ingress/azure/deployment.yaml index cd841eb5a..1f537e253 100644 --- a/addons/nginx-ingress/azure/deployment.yaml +++ b/addons/nginx-ingress/azure/deployment.yaml @@ -27,7 +27,6 @@ spec: image: quay.io/kubernetes-ingress-controller/nginx-ingress-controller:0.22.0 args: - /nginx-ingress-controller - - --default-backend-service=$(POD_NAMESPACE)/default-backend - --ingress-class=public # use downward API env: diff --git a/addons/nginx-ingress/bare-metal/default-backend/deployment.yaml b/addons/nginx-ingress/bare-metal/default-backend/deployment.yaml deleted file mode 100644 index ce6401897..000000000 --- a/addons/nginx-ingress/bare-metal/default-backend/deployment.yaml +++ /dev/null @@ -1,42 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: default-backend - namespace: ingress -spec: - replicas: 1 - selector: - matchLabels: - name: default-backend - phase: prod - template: - metadata: - labels: - name: default-backend - phase: prod - annotations: - seccomp.security.alpha.kubernetes.io/pod: 'docker/default' - spec: - containers: - - name: default-backend - # Any image is permissable as long as: - # 1. It serves a 404 page at / - # 2. It serves 200 on a /healthz endpoint - image: k8s.gcr.io/defaultbackend:1.4 - ports: - - containerPort: 8080 - resources: - limits: - cpu: 10m - memory: 20Mi - requests: - cpu: 10m - memory: 20Mi - livenessProbe: - httpGet: - path: /healthz - port: 8080 - scheme: HTTP - initialDelaySeconds: 30 - timeoutSeconds: 5 - terminationGracePeriodSeconds: 60 diff --git a/addons/nginx-ingress/bare-metal/default-backend/service.yaml b/addons/nginx-ingress/bare-metal/default-backend/service.yaml deleted file mode 100644 index 87997aba0..000000000 --- a/addons/nginx-ingress/bare-metal/default-backend/service.yaml +++ /dev/null @@ -1,15 +0,0 @@ -apiVersion: v1 -kind: Service -metadata: - name: default-backend - namespace: ingress -spec: - type: ClusterIP - selector: - name: default-backend - phase: prod - ports: - - name: http - protocol: TCP - port: 80 - targetPort: 8080 diff --git a/addons/nginx-ingress/bare-metal/deployment.yaml b/addons/nginx-ingress/bare-metal/deployment.yaml index 7ba6b8667..8c61aa0f5 100644 --- a/addons/nginx-ingress/bare-metal/deployment.yaml +++ b/addons/nginx-ingress/bare-metal/deployment.yaml @@ -25,7 +25,6 @@ spec: image: quay.io/kubernetes-ingress-controller/nginx-ingress-controller:0.22.0 args: - /nginx-ingress-controller - - --default-backend-service=$(POD_NAMESPACE)/default-backend - --ingress-class=public # use downward API env: diff --git a/addons/nginx-ingress/digital-ocean/daemonset.yaml b/addons/nginx-ingress/digital-ocean/daemonset.yaml index e733b3b58..29885d6ac 100644 --- a/addons/nginx-ingress/digital-ocean/daemonset.yaml +++ b/addons/nginx-ingress/digital-ocean/daemonset.yaml @@ -27,7 +27,6 @@ spec: image: quay.io/kubernetes-ingress-controller/nginx-ingress-controller:0.22.0 args: - /nginx-ingress-controller - - --default-backend-service=$(POD_NAMESPACE)/default-backend - --ingress-class=public # use downward API env: diff --git a/addons/nginx-ingress/digital-ocean/default-backend/deployment.yaml b/addons/nginx-ingress/digital-ocean/default-backend/deployment.yaml deleted file mode 100644 index ce6401897..000000000 --- a/addons/nginx-ingress/digital-ocean/default-backend/deployment.yaml +++ /dev/null @@ -1,42 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: default-backend - namespace: ingress -spec: - replicas: 1 - selector: - matchLabels: - name: default-backend - phase: prod - template: - metadata: - labels: - name: default-backend - phase: prod - annotations: - seccomp.security.alpha.kubernetes.io/pod: 'docker/default' - spec: - containers: - - name: default-backend - # Any image is permissable as long as: - # 1. It serves a 404 page at / - # 2. It serves 200 on a /healthz endpoint - image: k8s.gcr.io/defaultbackend:1.4 - ports: - - containerPort: 8080 - resources: - limits: - cpu: 10m - memory: 20Mi - requests: - cpu: 10m - memory: 20Mi - livenessProbe: - httpGet: - path: /healthz - port: 8080 - scheme: HTTP - initialDelaySeconds: 30 - timeoutSeconds: 5 - terminationGracePeriodSeconds: 60 diff --git a/addons/nginx-ingress/digital-ocean/default-backend/service.yaml b/addons/nginx-ingress/digital-ocean/default-backend/service.yaml deleted file mode 100644 index 87997aba0..000000000 --- a/addons/nginx-ingress/digital-ocean/default-backend/service.yaml +++ /dev/null @@ -1,15 +0,0 @@ -apiVersion: v1 -kind: Service -metadata: - name: default-backend - namespace: ingress -spec: - type: ClusterIP - selector: - name: default-backend - phase: prod - ports: - - name: http - protocol: TCP - port: 80 - targetPort: 8080 diff --git a/addons/nginx-ingress/google-cloud/default-backend/deployment.yaml b/addons/nginx-ingress/google-cloud/default-backend/deployment.yaml deleted file mode 100644 index ce6401897..000000000 --- a/addons/nginx-ingress/google-cloud/default-backend/deployment.yaml +++ /dev/null @@ -1,42 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: default-backend - namespace: ingress -spec: - replicas: 1 - selector: - matchLabels: - name: default-backend - phase: prod - template: - metadata: - labels: - name: default-backend - phase: prod - annotations: - seccomp.security.alpha.kubernetes.io/pod: 'docker/default' - spec: - containers: - - name: default-backend - # Any image is permissable as long as: - # 1. It serves a 404 page at / - # 2. It serves 200 on a /healthz endpoint - image: k8s.gcr.io/defaultbackend:1.4 - ports: - - containerPort: 8080 - resources: - limits: - cpu: 10m - memory: 20Mi - requests: - cpu: 10m - memory: 20Mi - livenessProbe: - httpGet: - path: /healthz - port: 8080 - scheme: HTTP - initialDelaySeconds: 30 - timeoutSeconds: 5 - terminationGracePeriodSeconds: 60 diff --git a/addons/nginx-ingress/google-cloud/default-backend/service.yaml b/addons/nginx-ingress/google-cloud/default-backend/service.yaml deleted file mode 100644 index 87997aba0..000000000 --- a/addons/nginx-ingress/google-cloud/default-backend/service.yaml +++ /dev/null @@ -1,15 +0,0 @@ -apiVersion: v1 -kind: Service -metadata: - name: default-backend - namespace: ingress -spec: - type: ClusterIP - selector: - name: default-backend - phase: prod - ports: - - name: http - protocol: TCP - port: 80 - targetPort: 8080 diff --git a/addons/nginx-ingress/google-cloud/deployment.yaml b/addons/nginx-ingress/google-cloud/deployment.yaml index cd841eb5a..1f537e253 100644 --- a/addons/nginx-ingress/google-cloud/deployment.yaml +++ b/addons/nginx-ingress/google-cloud/deployment.yaml @@ -27,7 +27,6 @@ spec: image: quay.io/kubernetes-ingress-controller/nginx-ingress-controller:0.22.0 args: - /nginx-ingress-controller - - --default-backend-service=$(POD_NAMESPACE)/default-backend - --ingress-class=public # use downward API env: From d9888227415f7bf58d8d61df34dcd41096446d35 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Sat, 16 Feb 2019 15:01:44 -0800 Subject: [PATCH 024/523] Document and recommend terraform-provider-matchbox v0.2.3 * https://github.com/coreos/terraform-provider-matchbox/releases/tag/v0.2.3 --- CHANGES.md | 4 ++++ docs/atomic/bare-metal.md | 17 +++++------------ docs/cl/bare-metal.md | 8 ++++---- docs/topics/maintenance.md | 12 ++++++------ 4 files changed, 19 insertions(+), 22 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 32ffcf312..7fb373a7b 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -7,6 +7,10 @@ Notable changes between versions. * Update etcd from v3.3.11 to [v3.3.12](https://github.com/etcd-io/etcd/releases/tag/v3.3.12) * Update Calico from v3.5.0 to v3.5.1 +#### Bare-Metal + +* Recommend updating [terraform-provider-matchbox](https://github.com/coreos/terraform-provider-matchbox) plugin from v0.2.2 to [v0.2.3](https://github.com/coreos/terraform-provider-matchbox/releases/tag/v0.2.3) ([#402](https://github.com/poseidon/typhoon/pull/402)) + #### Addons * Raise nginx-ingress liveness/readiness timeout to 5 seconds diff --git a/docs/atomic/bare-metal.md b/docs/atomic/bare-metal.md index dffd5bb6e..df9bbb09d 100644 --- a/docs/atomic/bare-metal.md +++ b/docs/atomic/bare-metal.md @@ -174,20 +174,12 @@ $ terraform version Terraform v0.11.7 ``` -Add the [terraform-provider-matchbox](https://github.com/coreos/terraform-provider-matchbox) plugin binary for your system. +Add the [terraform-provider-matchbox](https://github.com/coreos/terraform-provider-matchbox) plugin binary for your system to `~/.terraform.d/plugins/`, noting the final name. ```sh -wget https://github.com/coreos/terraform-provider-matchbox/releases/download/v0.2.2/terraform-provider-matchbox-v0.2.2-linux-amd64.tar.gz -tar xzf terraform-provider-matchbox-v0.2.2-linux-amd64.tar.gz -sudo mv terraform-provider-matchbox-v0.2.2-linux-amd64/terraform-provider-matchbox /usr/local/bin/ -``` - -Add the plugin to your `~/.terraformrc`. - -``` -providers { - matchbox = "/usr/local/bin/terraform-provider-matchbox" -} +wget https://github.com/coreos/terraform-provider-matchbox/releases/download/v0.2.3/terraform-provider-matchbox-v0.2.3-linux-amd64.tar.gz +tar xzf terraform-provider-matchbox-v0.2.3-linux-amd64.tar.gz +mv terraform-provider-matchbox-v0.2.3-linux-amd64/terraform-provider-matchbox ~/.terraform.d/plugins/terraform-provider-matchbox_v0.2.3 ``` Read [concepts](/architecture/concepts/) to learn about Terraform, modules, and organizing resources. Change to your infrastructure repository (e.g. `infra`). @@ -202,6 +194,7 @@ Configure the Matchbox provider to use your Matchbox API endpoint and client cer ```tf provider "matchbox" { + version = "0.2.3" endpoint = "matchbox.example.com:8081" client_cert = "${file("~/.config/matchbox/client.crt")}" client_key = "${file("~/.config/matchbox/client.key")}" diff --git a/docs/cl/bare-metal.md b/docs/cl/bare-metal.md index 43db3f2f7..af1b518ec 100644 --- a/docs/cl/bare-metal.md +++ b/docs/cl/bare-metal.md @@ -116,9 +116,9 @@ Terraform v0.11.7 Add the [terraform-provider-matchbox](https://github.com/coreos/terraform-provider-matchbox) plugin binary for your system to `~/.terraform.d/plugins/`, noting the final name. ```sh -wget https://github.com/coreos/terraform-provider-matchbox/releases/download/v0.2.2/terraform-provider-matchbox-v0.2.2-linux-amd64.tar.gz -tar xzf terraform-provider-matchbox-v0.2.2-linux-amd64.tar.gz -mv terraform-provider-matchbox-v0.2.2-linux-amd64/terraform-provider-matchbox ~/.terraform.d/plugins/terraform-provider-matchbox_v0.2.2 +wget https://github.com/coreos/terraform-provider-matchbox/releases/download/v0.2.3/terraform-provider-matchbox-v0.2.3-linux-amd64.tar.gz +tar xzf terraform-provider-matchbox-v0.2.3-linux-amd64.tar.gz +mv terraform-provider-matchbox-v0.2.3-linux-amd64/terraform-provider-matchbox ~/.terraform.d/plugins/terraform-provider-matchbox_v0.2.3 ``` Add the [terraform-provider-ct](https://github.com/coreos/terraform-provider-ct) plugin binary for your system to `~/.terraform.d/plugins/`, noting the final name. @@ -141,7 +141,7 @@ Configure the Matchbox provider to use your Matchbox API endpoint and client cer ```tf provider "matchbox" { - version = "0.2.2" + version = "0.2.3" endpoint = "matchbox.example.com:8081" client_cert = "${file("~/.config/matchbox/client.crt")}" client_key = "${file("~/.config/matchbox/client.key")}" diff --git a/docs/topics/maintenance.md b/docs/topics/maintenance.md index 94d52935d..0675ea479 100644 --- a/docs/topics/maintenance.md +++ b/docs/topics/maintenance.md @@ -156,9 +156,9 @@ mv terraform-provider-ct-v0.2.1-linux-amd64/terraform-provider-ct ~/.terraform.d If you use bare-metal, add the [terraform-provider-matchbox](https://github.com/coreos/terraform-provider-matchbox) plugin binary for your system to `~/.terraform.d/plugins/`, noting the versioned name. ```sh -wget https://github.com/coreos/terraform-provider-matchbox/releases/download/v0.2.2/terraform-provider-matchbox-v0.2.2-linux-amd64.tar.gz -tar xzf terraform-provider-matchbox-v0.2.2-linux-amd64.tar.gz -mv terraform-provider-matchbox-v0.2.2-linux-amd64/terraform-provider-matchbox ~/.terraform.d/plugins/terraform-provider-matchbox_v0.2.2 +wget https://github.com/coreos/terraform-provider-matchbox/releases/download/v0.2.3/terraform-provider-matchbox-v0.2.3-linux-amd64.tar.gz +tar xzf terraform-provider-matchbox-v0.2.3-linux-amd64.tar.gz +mv terraform-provider-matchbox-v0.2.3-linux-amd64/terraform-provider-matchbox ~/.terraform.d/plugins/terraform-provider-matchbox_v0.2.3 ``` Binary names are versioned. This enables the ability to upgrade different plugins and have clusters pin different versions. @@ -168,7 +168,7 @@ $ tree ~/.terraform.d/ /home/user/.terraform.d/ └── plugins ├── terraform-provider-ct_v0.2.1 - └── terraform-provider-matchbox_v0.2.2 + └── terraform-provider-matchbox_v0.2.3 ``` In each Terraform working directory, set the version of each provider. @@ -177,7 +177,7 @@ In each Terraform working directory, set the version of each provider. # providers.tf provider "matchbox" { - version = "0.2.2" + version = "0.2.3" ... } @@ -215,7 +215,7 @@ $ tree ~/.terraform.d/ └── plugins ├── terraform-provider-ct_v0.2.1 ├── terraform-provider-ct_v0.3.0 - └── terraform-provider-matchbox_v0.2.2 + └── terraform-provider-matchbox_v0.2.3 ``` From ac95e8324948af4289cf1630609ebb4c8e8c882e Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Sat, 16 Feb 2019 15:55:38 -0800 Subject: [PATCH 025/523] Update mkdocs-material from v3.3.0 to v4.0.1 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index fc78f3c9e..37b4849cb 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ mkdocs==1.0.4 -mkdocs-material==3.3.0 +mkdocs-material==4.0.1 pygments==2.2.0 pymdown-extensions==5.0.0 From 6fa3b8a13f324bc0241edd051c6972b5c95f90ac Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Sun, 17 Feb 2019 12:36:40 -0800 Subject: [PATCH 026/523] Upgrade Grafana to v6.0.0-beta2 and enable Explore UI * Upgrade Grafana from v5.4.3 to v6.0.0-beta2 * Enable Grafana Explore UI while still using only the Viewer role (inspect/edit without saving) * http://docs.grafana.org/guides/whats-new-in-v6-0/ --- CHANGES.md | 3 ++ addons/grafana/config.yaml | 36 +++++++++++++++++++ addons/grafana/datasources.yaml | 10 +++++- addons/grafana/deployment.yaml | 29 +++++++-------- ...ashboard-providers.yaml => providers.yaml} | 6 ++-- 5 files changed, 63 insertions(+), 21 deletions(-) create mode 100644 addons/grafana/config.yaml rename addons/grafana/{dashboard-providers.yaml => providers.yaml} (63%) diff --git a/CHANGES.md b/CHANGES.md index 7fb373a7b..0b47c5a5b 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -19,6 +19,9 @@ Notable changes between versions. * Improve Prometheus metrics labels and drop extraneous metrics ([#397](https://github.com/poseidon/typhoon/pull/397)) * Add `pod` name label to metrics discovered via service endpoints * Rename `kubernetes_namespace` label to `namespace` +* Revamp the Grafana addon ([#403](https://github.com/poseidon/typhoon/pull/403)) + * Upgrade Grafana from v5.4.3 to 6.0.0-beta2 + * Enable Grafana [Explore](http://docs.grafana.org/guides/whats-new-in-v6-0/#explore) UI as a Viewer (inspect/edit without saving) ## v1.13.3 diff --git a/addons/grafana/config.yaml b/addons/grafana/config.yaml new file mode 100644 index 000000000..c6c2421ef --- /dev/null +++ b/addons/grafana/config.yaml @@ -0,0 +1,36 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: grafana-config + namespace: monitoring +data: + custom.ini: |+ + [server] + http_port = 8080 + + [paths] + data = /var/lib/grafana + plugins = /var/lib/grafana/plugins + provisioning = /etc/grafana/provisioning + + [users] + allow_sign_up = false + allow_org_create = false + # viewers can edit/inspect, but not save + viewers_can_edit = true + + # Disable login form, since Grafana always creates an admin user + [auth] + disable_login_form = true + + # Disable the user/pass login system + [auth.basic] + enabled = false + + # Allow anonymous authentication with view-only authorization + [auth.anonymous] + enabled = true + org_role = Viewer + + [analytics] + reporting_enabled = false diff --git a/addons/grafana/datasources.yaml b/addons/grafana/datasources.yaml index a13f52ef4..d235ace57 100644 --- a/addons/grafana/datasources.yaml +++ b/addons/grafana/datasources.yaml @@ -10,7 +10,15 @@ data: - name: prometheus type: prometheus access: proxy - orgId: 1 url: http://prometheus.monitoring.svc.cluster.local version: 1 editable: false + loki.yaml: |+ + apiVersion: 1 + datasources: + - name: loki + type: loki + access: proxy + url: http://loki.monitoring.svc.cluster.local + version: 1 + editable: false diff --git a/addons/grafana/deployment.yaml b/addons/grafana/deployment.yaml index 409c4710d..cfe9d608c 100644 --- a/addons/grafana/deployment.yaml +++ b/addons/grafana/deployment.yaml @@ -23,20 +23,10 @@ spec: spec: containers: - name: grafana - image: grafana/grafana:5.4.3 + image: grafana/grafana:6.0.0-beta2 env: - - name: GF_SERVER_HTTP_PORT - value: "8080" - - name: GF_AUTH_BASIC_ENABLED - value: "false" - - name: GF_AUTH_DISABLE_LOGIN_FORM - value: "true" - - name: GF_AUTH_ANONYMOUS_ENABLED - value: "true" - - name: GF_AUTH_ANONYMOUS_ORG_ROLE - value: Viewer - - name: GF_ANALYTICS_REPORTING_ENABLED - value: "false" + - name: GF_PATHS_CONFIG + value: "/etc/grafana/custom.ini" ports: - name: http containerPort: 8080 @@ -48,19 +38,24 @@ spec: memory: 200Mi cpu: 200m volumeMounts: + - name: config + mountPath: /etc/grafana - name: datasources mountPath: /etc/grafana/provisioning/datasources - - name: dashboard-providers + - name: providers mountPath: /etc/grafana/provisioning/dashboards - name: dashboards - mountPath: /var/lib/grafana/dashboards + mountPath: /etc/grafana/dashboards volumes: + - name: config + configMap: + name: grafana-config - name: datasources configMap: name: grafana-datasources - - name: dashboard-providers + - name: providers configMap: - name: grafana-dashboard-providers + name: grafana-providers - name: dashboards configMap: name: grafana-dashboards diff --git a/addons/grafana/dashboard-providers.yaml b/addons/grafana/providers.yaml similarity index 63% rename from addons/grafana/dashboard-providers.yaml rename to addons/grafana/providers.yaml index e48924e88..3ddaf4509 100644 --- a/addons/grafana/dashboard-providers.yaml +++ b/addons/grafana/providers.yaml @@ -1,10 +1,10 @@ apiVersion: v1 kind: ConfigMap metadata: - name: grafana-dashboard-providers + name: grafana-providers namespace: monitoring data: - dashboard-providers.yaml: |+ + providers.yaml: |+ apiVersion: 1 providers: - name: 'default' @@ -12,4 +12,4 @@ data: folder: '' type: file options: - path: /var/lib/grafana/dashboards + path: /etc/grafana/dashboards From e483c81ce9f7ab79444a684701bc8b5ba94df84d Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Mon, 18 Feb 2019 12:05:27 -0800 Subject: [PATCH 027/523] Improve Prometheus rules and alerts and Grafana dashboards * Collate upstream rules, alerts, and dashboards and tune for use in Typhoon * Previously, a well-chosen (but older) set of rules, alerts, and dashboards were maintained to reflect metric name changes --- CHANGES.md | 11 +- addons/grafana/dashboards.yaml | 10023 +++++++++++++---------- addons/prometheus/rules.yaml | 1665 ++-- docs/addons/grafana.md | 7 +- docs/img/grafana-capacity.png | Bin 239490 -> 0 bytes docs/img/grafana-control-plane.png | Bin 265550 -> 0 bytes docs/img/grafana-etcd.png | Bin 0 -> 90362 bytes docs/img/grafana-node.png | Bin 245965 -> 0 bytes docs/img/grafana-resources-cluster.png | Bin 0 -> 88285 bytes docs/img/grafana-usage-cluster.png | Bin 0 -> 94682 bytes docs/img/grafana-usage-node.png | Bin 0 -> 105880 bytes 11 files changed, 6971 insertions(+), 4735 deletions(-) delete mode 100644 docs/img/grafana-capacity.png delete mode 100644 docs/img/grafana-control-plane.png create mode 100644 docs/img/grafana-etcd.png delete mode 100644 docs/img/grafana-node.png create mode 100644 docs/img/grafana-resources-cluster.png create mode 100644 docs/img/grafana-usage-cluster.png create mode 100644 docs/img/grafana-usage-node.png diff --git a/CHANGES.md b/CHANGES.md index 0b47c5a5b..a3aa360eb 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -13,15 +13,16 @@ Notable changes between versions. #### Addons -* Raise nginx-ingress liveness/readiness timeout to 5 seconds -* Remove nginx-ingess default-backend ([#401](https://github.com/poseidon/typhoon/pull/401)) - * nginx-ingress now responds with its own 404 page by default -* Improve Prometheus metrics labels and drop extraneous metrics ([#397](https://github.com/poseidon/typhoon/pull/397)) +* Improve Prometheus rules and alerts ([#404](https://github.com/poseidon/typhoon/pull/404)) + * Drop extraneous metrics ([#397](https://github.com/poseidon/typhoon/pull/397)) * Add `pod` name label to metrics discovered via service endpoints * Rename `kubernetes_namespace` label to `namespace` -* Revamp the Grafana addon ([#403](https://github.com/poseidon/typhoon/pull/403)) +* Improve Grafana and dashboards ([#403](https://github.com/poseidon/typhoon/pull/403), [#404](https://github.com/poseidon/typhoon/pull/404)) * Upgrade Grafana from v5.4.3 to 6.0.0-beta2 * Enable Grafana [Explore](http://docs.grafana.org/guides/whats-new-in-v6-0/#explore) UI as a Viewer (inspect/edit without saving) +* Raise nginx-ingress liveness/readiness timeout to 5 seconds +* Remove nginx-ingess default-backend ([#401](https://github.com/poseidon/typhoon/pull/401)) + * nginx-ingress now responds with its own 404 page by default ## v1.13.3 diff --git a/addons/grafana/dashboards.yaml b/addons/grafana/dashboards.yaml index 56a03ceb0..181b288b7 100644 --- a/addons/grafana/dashboards.yaml +++ b/addons/grafana/dashboards.yaml @@ -4,32 +4,30 @@ metadata: name: grafana-dashboards namespace: monitoring data: - deployment-dashboard.json: |+ + etcd.json: |- { - "__inputs": [ - { - "description": "", - "label": "prometheus", - "name": "prometheus", - "pluginId": "prometheus", - "pluginName": "Prometheus", - "type": "datasource" - } - ], "annotations": { - "list": [] + "list": [ + + ] }, - "editable": false, - "graphTooltip": 1, + "description": "etcd sample Grafana dashboard with Prometheus", + "editable": true, + "gnetId": null, "hideControls": false, - "links": [], + "id": 6, + "links": [ + + ], + "refresh": false, "rows": [ { "collapse": false, - "editable": false, - "height": "200px", + "editable": true, + "height": "250px", "panels": [ { + "cacheTimeout": null, "colorBackground": false, "colorValue": false, "colors": [ @@ -37,8 +35,9 @@ data: "rgba(237, 129, 40, 0.89)", "rgba(50, 172, 45, 0.97)" ], - "datasource": "prometheus", - "editable": false, + "datasource": "$datasource", + "editable": true, + "error": false, "format": "none", "gauge": { "maxValue": 100, @@ -47,8 +46,12 @@ data: "thresholdLabels": false, "thresholdMarkers": true }, - "id": 8, - "links": [], + "id": 28, + "interval": null, + "isNew": true, + "links": [ + + ], "mappingType": 1, "mappingTypes": [ { @@ -62,7 +65,8 @@ data: ], "maxDataPoints": 100, "nullPointMode": "connected", - "postfix": "cores", + "nullText": null, + "postfix": "", "postfixFontSize": "50%", "prefix": "", "prefixFontSize": "50%", @@ -73,24 +77,27 @@ data: "to": "null" } ], - "span": 4, + "span": 3, "sparkline": { "fillColor": "rgba(31, 118, 189, 0.18)", "full": false, "lineColor": "rgb(31, 120, 193)", - "show": true + "show": false }, "targets": [ { - "expr": "sum(rate(container_cpu_usage_seconds_total{namespace=\"$deployment_namespace\",pod_name=~\"$deployment_name.*\"}[3m]))", + "expr": "sum(etcd_server_has_leader{job=\"$cluster\"})", "intervalFactor": 2, + "legendFormat": "", + "metric": "etcd_server_has_leader", "refId": "A", - "step": 600 + "step": 20 } ], - "title": "CPU", + "thresholds": "", + "title": "Up", "type": "singlestat", - "valueFontSize": "110%", + "valueFontSize": "200%", "valueMaps": [ { "op": "=", @@ -101,784 +108,579 @@ data: "valueName": "avg" }, { - "colorBackground": false, - "colorValue": false, - "colors": [ - "rgba(245, 54, 54, 0.9)", - "rgba(237, 129, 40, 0.89)", - "rgba(50, 172, 45, 0.97)" - ], - "datasource": "prometheus", - "editable": false, - "format": "none", - "gauge": { - "maxValue": 100, - "minValue": 0, + "aliasColors": { + + }, + "bars": false, + "datasource": "$datasource", + "editable": true, + "error": false, + "fill": 0, + "id": 23, + "isNew": true, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, "show": false, - "thresholdLabels": false, - "thresholdMarkers": true + "total": false, + "values": false }, - "id": 9, - "links": [], - "mappingType": 1, - "mappingTypes": [ - { - "name": "value to text", - "value": 1 - }, - { - "name": "range to text", - "value": 2 - } + "lines": true, + "linewidth": 2, + "links": [ + ], - "maxDataPoints": 100, "nullPointMode": "connected", - "postfix": "GB", - "postfixFontSize": "50%", - "prefix": "", - "prefixFontSize": "80%", - "rangeMaps": [ - { - "from": "null", - "text": "N/A", - "to": "null" - } + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + ], - "span": 4, - "sparkline": { - "fillColor": "rgba(31, 118, 189, 0.18)", - "full": false, - "lineColor": "rgb(31, 120, 193)", - "show": true - }, + "span": 5, + "stack": false, + "steppedLine": false, "targets": [ { - "expr": "sum(container_memory_usage_bytes{namespace=\"$deployment_namespace\",pod_name=~\"$deployment_name.*\"}) / 1024^3", + "expr": "sum(rate(grpc_server_started_total{job=\"$cluster\",grpc_type=\"unary\"}[5m]))", + "format": "time_series", "intervalFactor": 2, + "legendFormat": "RPC Rate", + "metric": "grpc_server_started_total", "refId": "A", - "step": 600 - } - ], - "title": "Memory", - "type": "singlestat", - "valueFontSize": "110%", - "valueMaps": [ + "step": 2 + }, { - "op": "=", - "text": "N/A", - "value": "null" + "expr": "sum(rate(grpc_server_handled_total{job=\"$cluster\",grpc_type=\"unary\",grpc_code!=\"OK\"}[5m]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "RPC Failed Rate", + "metric": "grpc_server_handled_total", + "refId": "B", + "step": 2 } ], - "valueName": "avg" - }, - { - "colorBackground": false, - "colorValue": false, - "colors": [ - "rgba(245, 54, 54, 0.9)", - "rgba(237, 129, 40, 0.89)", - "rgba(50, 172, 45, 0.97)" + "thresholds": [ + ], - "datasource": "prometheus", - "editable": false, - "format": "Bps", - "gauge": { - "maxValue": 100, - "minValue": 0, - "show": false, - "thresholdLabels": false, - "thresholdMarkers": false + "timeFrom": null, + "timeShift": null, + "title": "RPC Rate", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" }, - "id": 7, - "links": [], - "mappingType": 1, - "mappingTypes": [ + "type": "graph", + "xaxis": { + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ { - "name": "value to text", - "value": 1 + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true }, { - "name": "range to text", - "value": 2 + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "datasource": "$datasource", + "editable": true, + "error": false, + "fill": 0, + "id": 41, + "isNew": true, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [ + ], - "maxDataPoints": 100, "nullPointMode": "connected", - "postfix": "", - "postfixFontSize": "50%", - "prefix": "", - "prefixFontSize": "50%", - "rangeMaps": [ - { - "from": "null", - "text": "N/A", - "to": "null" - } + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + ], "span": 4, - "sparkline": { - "fillColor": "rgba(31, 118, 189, 0.18)", - "full": false, - "lineColor": "rgb(31, 120, 193)", - "show": true - }, + "stack": true, + "steppedLine": false, "targets": [ { - "expr": "sum(rate(container_network_transmit_bytes_total{namespace=\"$deployment_namespace\",pod_name=~\"$deployment_name.*\"}[3m])) + sum(rate(container_network_receive_bytes_total{namespace=\"$deployment_namespace\",pod_name=~\"$deployment_name.*\"}[3m]))", + "expr": "sum(grpc_server_started_total{job=\"$cluster\",grpc_service=\"etcdserverpb.Watch\",grpc_type=\"bidi_stream\"}) - sum(grpc_server_handled_total{job=\"$cluster\",grpc_service=\"etcdserverpb.Watch\",grpc_type=\"bidi_stream\"})", "intervalFactor": 2, + "legendFormat": "Watch Streams", + "metric": "grpc_server_handled_total", "refId": "A", - "step": 600 - } - ], - "title": "Network", - "type": "singlestat", - "valueFontSize": "80%", - "valueMaps": [ + "step": 4 + }, { - "op": "=", - "text": "N/A", - "value": "null" + "expr": "sum(grpc_server_started_total{job=\"$cluster\",grpc_service=\"etcdserverpb.Lease\",grpc_type=\"bidi_stream\"}) - sum(grpc_server_handled_total{job=\"$cluster\",grpc_service=\"etcdserverpb.Lease\",grpc_type=\"bidi_stream\"})", + "intervalFactor": 2, + "legendFormat": "Lease Streams", + "metric": "grpc_server_handled_total", + "refId": "B", + "step": 4 } ], - "valueName": "avg" - } - ], - "showTitle": false, - "title": "Dashboard Row", - "titleSize": "h6" - }, - { - "collapse": false, - "editable": false, - "height": "100px", - "panels": [ - { - "colorBackground": false, - "colorValue": false, - "colors": [ - "rgba(245, 54, 54, 0.9)", - "rgba(237, 129, 40, 0.89)", - "rgba(50, 172, 45, 0.97)" + "thresholds": [ + ], - "datasource": "prometheus", - "editable": false, - "format": "none", - "gauge": { - "maxValue": 100, - "minValue": 0, - "show": false, - "thresholdLabels": false, - "thresholdMarkers": false - }, - "id": 5, - "links": [], - "mappingType": 1, - "mappingTypes": [ - { - "name": "value to text", - "value": 1 + "timeFrom": null, + "timeShift": null, + "title": "Active Streams", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": "", + "logBase": 1, + "max": null, + "min": null, + "show": true }, { - "name": "range to text", - "value": 2 + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true } + ] + } + ], + "showTitle": false, + "title": "Row" + }, + { + "collapse": false, + "editable": true, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "datasource": "$datasource", + "decimals": null, + "editable": true, + "error": false, + "fill": 0, + "grid": { + + }, + "id": 1, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [ + ], - "maxDataPoints": 100, "nullPointMode": "connected", - "postfixFontSize": "50%", - "prefix": "", - "prefixFontSize": "50%", - "rangeMaps": [ - { - "from": "null", - "text": "N/A", - "to": "null" - } + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + ], - "span": 3, - "sparkline": { - "fillColor": "rgba(31, 118, 189, 0.18)", - "full": false, - "lineColor": "rgb(31, 120, 193)", - "show": false - }, + "span": 4, + "stack": false, + "steppedLine": false, "targets": [ { - "expr": "max(kube_deployment_spec_replicas{deployment=\"$deployment_name\",namespace=\"$deployment_namespace\"}) without (instance, pod)", + "expr": "etcd_debugging_mvcc_db_total_size_in_bytes{job=\"$cluster\"}", + "hide": false, + "interval": "", "intervalFactor": 2, - "metric": "kube_deployment_spec_replicas", + "legendFormat": "{{instance}} DB Size", + "metric": "", "refId": "A", - "step": 600 - } - ], - "title": "Desired Replicas", - "type": "singlestat", - "valueFontSize": "80%", - "valueMaps": [ - { - "op": "=", - "text": "N/A", - "value": "null" + "step": 4 } ], - "valueName": "avg" - }, - { - "colorBackground": false, - "colorValue": false, - "colors": [ - "rgba(245, 54, 54, 0.9)", - "rgba(237, 129, 40, 0.89)", - "rgba(50, 172, 45, 0.97)" + "thresholds": [ + ], - "datasource": "prometheus", - "editable": false, - "format": "none", - "gauge": { - "maxValue": 100, - "minValue": 0, - "show": false, - "thresholdLabels": false, - "thresholdMarkers": true + "timeFrom": null, + "timeShift": null, + "title": "DB Size", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" }, - "id": 6, - "links": [], - "mappingType": 1, - "mappingTypes": [ + "type": "graph", + "xaxis": { + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ { - "name": "value to text", - "value": 1 + "format": "bytes", + "logBase": 1, + "max": null, + "min": null, + "show": true }, { - "name": "range to text", - "value": 2 + "format": "short", + "logBase": 1, + "max": null, + "min": null, + "show": false } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "datasource": "$datasource", + "editable": true, + "error": false, + "fill": 0, + "grid": { + + }, + "id": 3, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [ + ], - "maxDataPoints": 100, "nullPointMode": "connected", - "postfixFontSize": "50%", - "prefix": "", - "prefixFontSize": "50%", - "rangeMaps": [ - { - "from": "null", - "text": "N/A", - "to": "null" - } + "percentage": false, + "pointradius": 1, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + ], - "span": 3, - "sparkline": { - "fillColor": "rgba(31, 118, 189, 0.18)", - "full": false, - "lineColor": "rgb(31, 120, 193)", - "show": false - }, + "span": 4, + "stack": false, + "steppedLine": true, "targets": [ { - "expr": "min(kube_deployment_status_replicas_available{deployment=\"$deployment_name\",namespace=\"$deployment_namespace\"}) without (instance, pod)", + "expr": "histogram_quantile(0.99, sum(rate(etcd_disk_wal_fsync_duration_seconds_bucket{job=\"$cluster\"}[5m])) by (instance, le))", + "hide": false, "intervalFactor": 2, + "legendFormat": "{{instance}} WAL fsync", + "metric": "etcd_disk_wal_fsync_duration_seconds_bucket", "refId": "A", - "step": 600 - } - ], - "title": "Available Replicas", - "type": "singlestat", - "valueFontSize": "80%", - "valueMaps": [ + "step": 4 + }, { - "op": "=", - "text": "N/A", - "value": "null" + "expr": "histogram_quantile(0.99, sum(rate(etcd_disk_backend_commit_duration_seconds_bucket{job=\"$cluster\"}[5m])) by (instance, le))", + "intervalFactor": 2, + "legendFormat": "{{instance}} DB fsync", + "metric": "etcd_disk_backend_commit_duration_seconds_bucket", + "refId": "B", + "step": 4 } ], - "valueName": "avg" - }, - { - "colorBackground": false, - "colorValue": false, - "colors": [ - "rgba(245, 54, 54, 0.9)", - "rgba(237, 129, 40, 0.89)", - "rgba(50, 172, 45, 0.97)" + "thresholds": [ + ], - "datasource": "prometheus", - "editable": false, - "format": "none", - "gauge": { - "maxValue": 100, - "minValue": 0, - "show": false, - "thresholdLabels": false, - "thresholdMarkers": true + "timeFrom": null, + "timeShift": null, + "title": "Disk Sync Duration", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" }, - "id": 3, - "links": [], - "mappingType": 1, - "mappingTypes": [ + "type": "graph", + "xaxis": { + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ { - "name": "value to text", - "value": 1 + "format": "s", + "logBase": 1, + "max": null, + "min": null, + "show": true }, { - "name": "range to text", - "value": 2 - } - ], - "maxDataPoints": 100, - "nullPointMode": "connected", - "postfixFontSize": "50%", - "prefix": "", - "prefixFontSize": "50%", - "rangeMaps": [ - { - "from": "null", - "text": "N/A", - "to": "null" - } - ], - "span": 3, - "sparkline": { - "fillColor": "rgba(31, 118, 189, 0.18)", - "full": false, - "lineColor": "rgb(31, 120, 193)", - "show": false - }, - "targets": [ - { - "expr": "max(kube_deployment_status_observed_generation{deployment=\"$deployment_name\",namespace=\"$deployment_namespace\"}) without (instance, pod)", - "intervalFactor": 2, - "refId": "A", - "step": 600 - } - ], - "title": "Observed Generation", - "type": "singlestat", - "valueFontSize": "80%", - "valueMaps": [ - { - "op": "=", - "text": "N/A", - "value": "null" + "format": "short", + "logBase": 1, + "max": null, + "min": null, + "show": false } - ], - "valueName": "avg" + ] }, { - "colorBackground": false, - "colorValue": false, - "colors": [ - "rgba(245, 54, 54, 0.9)", - "rgba(237, 129, 40, 0.89)", - "rgba(50, 172, 45, 0.97)" - ], - "datasource": "prometheus", - "editable": false, - "format": "none", - "gauge": { - "maxValue": 100, - "minValue": 0, + "aliasColors": { + + }, + "bars": false, + "datasource": "$datasource", + "editable": true, + "error": false, + "fill": 0, + "id": 29, + "isNew": true, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, "show": false, - "thresholdLabels": false, - "thresholdMarkers": true + "total": false, + "values": false }, - "id": 2, - "links": [], - "mappingType": 1, - "mappingTypes": [ - { - "name": "value to text", - "value": 1 - }, - { - "name": "range to text", - "value": 2 - } + "lines": true, + "linewidth": 2, + "links": [ + ], - "maxDataPoints": 100, "nullPointMode": "connected", - "postfixFontSize": "50%", - "prefix": "", - "prefixFontSize": "50%", - "rangeMaps": [ - { - "from": "null", - "text": "N/A", - "to": "null" - } + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + ], - "span": 3, - "sparkline": { - "fillColor": "rgba(31, 118, 189, 0.18)", - "full": false, - "lineColor": "rgb(31, 120, 193)", - "show": false - }, + "span": 4, + "stack": false, + "steppedLine": false, "targets": [ { - "expr": "max(kube_deployment_metadata_generation{deployment=\"$deployment_name\",namespace=\"$deployment_namespace\"}) without (instance, pod)", + "expr": "process_resident_memory_bytes{job=\"$cluster\"}", "intervalFactor": 2, + "legendFormat": "{{instance}} Resident Memory", + "metric": "process_resident_memory_bytes", "refId": "A", - "step": 600 + "step": 4 } ], - "title": "Metadata Generation", - "type": "singlestat", - "valueFontSize": "80%", - "valueMaps": [ + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Memory", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ { - "op": "=", - "text": "N/A", - "value": "null" + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true } - ], - "valueName": "avg" + ] } ], - "showTitle": false, - "title": "Dashboard Row", - "titleSize": "h6" + "title": "New row" }, { "collapse": false, - "editable": false, - "height": "350px", + "editable": true, + "height": "250px", "panels": [ { - "aliasColors": {}, + "aliasColors": { + + }, "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "prometheus", - "editable": false, + "datasource": "$datasource", + "editable": true, "error": false, - "fill": 1, - "grid": { - "threshold1Color": "rgba(216, 200, 27, 0.27)", - "threshold2Color": "rgba(234, 112, 112, 0.22)" - }, - "id": 1, + "fill": 5, + "id": 22, "isNew": true, "legend": { - "alignAsTable": false, "avg": false, "current": false, - "hideEmpty": false, - "hideZero": false, "max": false, "min": false, - "rightSide": false, - "show": true, - "total": false + "show": false, + "total": false, + "values": false }, "lines": true, "linewidth": 2, - "links": [], + "links": [ + + ], "nullPointMode": "connected", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "span": 12, - "stack": false, + "seriesOverrides": [ + + ], + "span": 3, + "stack": true, "steppedLine": false, "targets": [ { - "expr": "max(kube_deployment_status_replicas{deployment=\"$deployment_name\",namespace=\"$deployment_namespace\"}) without (instance, pod)", + "expr": "rate(etcd_network_client_grpc_received_bytes_total{job=\"$cluster\"}[5m])", "intervalFactor": 2, - "legendFormat": "current replicas", + "legendFormat": "{{instance}} Client Traffic In", + "metric": "etcd_network_client_grpc_received_bytes_total", "refId": "A", - "step": 30 - }, - { - "expr": "min(kube_deployment_status_replicas_available{deployment=\"$deployment_name\",namespace=\"$deployment_namespace\"}) without (instance, pod)", - "intervalFactor": 2, - "legendFormat": "available", - "refId": "B", - "step": 30 - }, - { - "expr": "max(kube_deployment_status_replicas_unavailable{deployment=\"$deployment_name\",namespace=\"$deployment_namespace\"}) without (instance, pod)", - "intervalFactor": 2, - "legendFormat": "unavailable", - "refId": "C", - "step": 30 - }, - { - "expr": "min(kube_deployment_status_replicas_updated{deployment=\"$deployment_name\",namespace=\"$deployment_namespace\"}) without (instance, pod)", - "intervalFactor": 2, - "legendFormat": "updated", - "refId": "D", - "step": 30 - }, - { - "expr": "max(kube_deployment_spec_replicas{deployment=\"$deployment_name\",namespace=\"$deployment_namespace\"}) without (instance, pod)", - "intervalFactor": 2, - "legendFormat": "desired", - "refId": "E", - "step": 30 + "step": 4 } ], - "title": "Replicas", + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Client Traffic In", "tooltip": { - "msResolution": true, + "msResolution": false, "shared": true, "sort": 0, - "value_type": "cumulative" + "value_type": "individual" }, "type": "graph", "xaxis": { "mode": "time", + "name": null, "show": true, - "values": [] + "values": [ + + ] }, "yaxes": [ { - "format": "none", - "label": "", + "format": "Bps", + "label": null, "logBase": 1, + "max": null, + "min": null, "show": true }, { "format": "short", - "label": "", + "label": null, "logBase": 1, - "show": false + "max": null, + "min": null, + "show": true } ] - } - ], - "showTitle": false, - "title": "Dashboard Row", - "titleSize": "h6" - } - ], - "schemaVersion": 14, - "sharedCrosshair": false, - "style": "dark", - "tags": [], - "templating": { - "list": [ - { - "allValue": ".*", - "current": {}, - "datasource": "prometheus", - "hide": 0, - "includeAll": false, - "label": "Namespace", - "multi": false, - "name": "deployment_namespace", - "options": [], - "query": "label_values(kube_deployment_metadata_generation, namespace)", - "refresh": 1, - "regex": "", - "sort": 0, - "tagValuesQuery": null, - "tags": [], - "tagsQuery": "", - "type": "query", - "useTags": false - }, - { - "allValue": null, - "current": {}, - "datasource": "prometheus", - "hide": 0, - "includeAll": false, - "label": "Deployment", - "multi": false, - "name": "deployment_name", - "options": [], - "query": "label_values(kube_deployment_metadata_generation{namespace=\"$deployment_namespace\"}, deployment)", - "refresh": 1, - "regex": "", - "sort": 0, - "tagValuesQuery": "", - "tags": [], - "tagsQuery": "deployment", - "type": "query", - "useTags": false - } - ] - }, - "time": { - "from": "now-6h", - "to": "now" - }, - "timepicker": { - "refresh_intervals": [ - "5s", - "10s", - "30s", - "1m", - "5m", - "15m", - "30m", - "1h", - "2h", - "1d" - ], - "time_options": [ - "5m", - "15m", - "1h", - "6h", - "12h", - "24h", - "2d", - "7d", - "30d" - ] - }, - "timezone": "browser", - "title": "Deployment", - "version": 1 - } - etcd-dashboard.json: |+ - { - "__inputs": [ - { - "name": "prometheus", - "label": "prometheus", - "description": "", - "type": "datasource", - "pluginId": "prometheus", - "pluginName": "Prometheus" - } - ], - "__requires": [ - { - "type": "grafana", - "id": "grafana", - "name": "Grafana", - "version": "4.5.2" - }, - { - "type": "panel", - "id": "graph", - "name": "Graph", - "version": "" - }, - { - "type": "datasource", - "id": "prometheus", - "name": "Prometheus", - "version": "1.0.0" - }, - { - "type": "panel", - "id": "singlestat", - "name": "Singlestat", - "version": "" - } - ], - "annotations": { - "list": [] - }, - "description": "etcd sample Grafana dashboard with Prometheus", - "editable": false, - "gnetId": null, - "graphTooltip": 0, - "hideControls": false, - "id": null, - "links": [], - "refresh": false, - "rows": [ - { - "collapse": false, - "height": "250px", - "panels": [ - { - "cacheTimeout": null, - "colorBackground": false, - "colorValue": false, - "colors": [ - "rgba(245, 54, 54, 0.9)", - "rgba(237, 129, 40, 0.89)", - "rgba(50, 172, 45, 0.97)" - ], - "datasource": "prometheus", - "editable": false, - "error": false, - "format": "none", - "gauge": { - "maxValue": 100, - "minValue": 0, - "show": false, - "thresholdLabels": false, - "thresholdMarkers": true - }, - "id": 28, - "interval": null, - "links": [], - "mappingType": 1, - "mappingTypes": [ - { - "name": "value to text", - "value": 1 - }, - { - "name": "range to text", - "value": 2 - } - ], - "maxDataPoints": 100, - "nullPointMode": "connected", - "nullText": null, - "postfix": "", - "postfixFontSize": "50%", - "prefix": "", - "prefixFontSize": "50%", - "rangeMaps": [ - { - "from": "null", - "text": "N/A", - "to": "null" - } - ], - "span": 3, - "sparkline": { - "fillColor": "rgba(31, 118, 189, 0.18)", - "full": false, - "lineColor": "rgb(31, 120, 193)", - "show": false - }, - "tableColumn": "", - "targets": [ - { - "expr": "sum(etcd_server_has_leader)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "", - "metric": "etcd_server_has_leader", - "refId": "A", - "step": 20 - } - ], - "thresholds": "", - "title": "Up", - "type": "singlestat", - "valueFontSize": "200%", - "valueMaps": [ - { - "op": "=", - "text": "N/A", - "value": "null" - } - ], - "valueName": "avg" }, { - "aliasColors": {}, + "aliasColors": { + + }, "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "prometheus", - "editable": false, + "datasource": "$datasource", + "editable": true, "error": false, - "fill": 0, - "id": 23, + "fill": 5, + "id": 21, + "isNew": true, "legend": { "avg": false, "current": false, @@ -890,41 +692,36 @@ data: }, "lines": true, "linewidth": 2, - "links": [], + "links": [ + + ], "nullPointMode": "connected", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "span": 5, - "stack": false, + "seriesOverrides": [ + + ], + "span": 3, + "stack": true, "steppedLine": false, "targets": [ { - "expr": "sum(rate(grpc_server_started_total{grpc_type=\"unary\"}[5m]))", - "format": "time_series", + "expr": "rate(etcd_network_client_grpc_sent_bytes_total{job=\"$cluster\"}[5m])", "intervalFactor": 2, - "legendFormat": "RPC Rate", - "metric": "grpc_server_started_total", + "legendFormat": "{{instance}} Client Traffic Out", + "metric": "etcd_network_client_grpc_sent_bytes_total", "refId": "A", "step": 4 - }, - { - "expr": "sum(rate(grpc_server_handled_total{grpc_type=\"unary\",grpc_code!=\"OK\"}[5m]))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "RPC Failed Rate", - "metric": "grpc_server_handled_total", - "refId": "B", - "step": 4 } ], - "thresholds": [], + "thresholds": [ + + ], "timeFrom": null, "timeShift": null, - "title": "RPC Rate", + "title": "Client Traffic Out", "tooltip": { "msResolution": false, "shared": true, @@ -933,15 +730,16 @@ data: }, "type": "graph", "xaxis": { - "buckets": null, "mode": "time", "name": null, "show": true, - "values": [] + "values": [ + + ] }, "yaxes": [ { - "format": "ops", + "format": "Bps", "label": null, "logBase": 1, "max": null, @@ -959,15 +757,16 @@ data: ] }, { - "aliasColors": {}, + "aliasColors": { + + }, "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "prometheus", - "editable": false, + "datasource": "$datasource", + "editable": true, "error": false, "fill": 0, - "id": 41, + "id": 20, + "isNew": true, "legend": { "avg": false, "current": false, @@ -979,41 +778,36 @@ data: }, "lines": true, "linewidth": 2, - "links": [], + "links": [ + + ], "nullPointMode": "connected", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "span": 4, - "stack": true, + "seriesOverrides": [ + + ], + "span": 3, + "stack": false, "steppedLine": false, "targets": [ { - "expr": "sum(grpc_server_started_total{grpc_service=\"etcdserverpb.Watch\",grpc_type=\"bidi_stream\"}) - sum(grpc_server_handled_total{grpc_service=\"etcdserverpb.Watch\",grpc_type=\"bidi_stream\"})", - "format": "time_series", + "expr": "sum(rate(etcd_network_peer_received_bytes_total{job=\"$cluster\"}[5m])) by (instance)", "intervalFactor": 2, - "legendFormat": "Watch Streams", - "metric": "grpc_server_handled_total", + "legendFormat": "{{instance}} Peer Traffic In", + "metric": "etcd_network_peer_received_bytes_total", "refId": "A", "step": 4 - }, - { - "expr": "sum(grpc_server_started_total{grpc_service=\"etcdserverpb.Lease\",grpc_type=\"bidi_stream\"}) - sum(grpc_server_handled_total{grpc_service=\"etcdserverpb.Lease\",grpc_type=\"bidi_stream\"})", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "Lease Streams", - "metric": "grpc_server_handled_total", - "refId": "B", - "step": 4 } ], - "thresholds": [], + "thresholds": [ + + ], "timeFrom": null, "timeShift": null, - "title": "Active Streams", + "title": "Peer Traffic In", "tooltip": { "msResolution": false, "shared": true, @@ -1022,16 +816,17 @@ data: }, "type": "graph", "xaxis": { - "buckets": null, "mode": "time", "name": null, "show": true, - "values": [] + "values": [ + + ] }, "yaxes": [ { - "format": "short", - "label": "", + "format": "Bps", + "label": null, "logBase": 1, "max": null, "min": null, @@ -1046,31 +841,21 @@ data: "show": true } ] - } - ], - "repeat": null, - "repeatIteration": null, - "repeatRowId": null, - "showTitle": false, - "title": "Row", - "titleSize": "h6" - }, - { - "collapse": false, - "height": "250px", - "panels": [ + }, { - "aliasColors": {}, + "aliasColors": { + + }, "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "prometheus", + "datasource": "$datasource", "decimals": null, - "editable": false, + "editable": true, "error": false, "fill": 0, - "grid": {}, - "id": 1, + "grid": { + + }, + "id": 16, "legend": { "avg": false, "current": false, @@ -1082,34 +867,38 @@ data: }, "lines": true, "linewidth": 2, - "links": [], + "links": [ + + ], "nullPointMode": "connected", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "span": 4, + "seriesOverrides": [ + + ], + "span": 3, "stack": false, "steppedLine": false, "targets": [ { - "expr": "etcd_debugging_mvcc_db_total_size_in_bytes", - "format": "time_series", + "expr": "sum(rate(etcd_network_peer_sent_bytes_total{job=\"$cluster\"}[5m])) by (instance)", "hide": false, "interval": "", "intervalFactor": 2, - "legendFormat": "{{instance}} DB Size", - "metric": "", + "legendFormat": "{{instance}} Peer Traffic Out", + "metric": "etcd_network_peer_sent_bytes_total", "refId": "A", "step": 4 } ], - "thresholds": [], + "thresholds": [ + + ], "timeFrom": null, "timeShift": null, - "title": "DB Size", + "title": "Peer Traffic Out", "tooltip": { "msResolution": false, "shared": true, @@ -1118,15 +907,16 @@ data: }, "type": "graph", "xaxis": { - "buckets": null, "mode": "time", "name": null, "show": true, - "values": [] + "values": [ + + ] }, "yaxes": [ { - "format": "bytes", + "format": "Bps", "logBase": 1, "max": null, "min": null, @@ -1137,22 +927,30 @@ data: "logBase": 1, "max": null, "min": null, - "show": false + "show": true } ] - }, + } + ], + "title": "New row" + }, + { + "collapse": false, + "editable": true, + "height": "250px", + "panels": [ { - "aliasColors": {}, + "aliasColors": { + + }, "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "prometheus", - "editable": false, + "datasource": "$datasource", + "editable": true, "error": false, "fill": 0, - "grid": {}, - "id": 3, - "legend": { + "id": 40, + "isNew": true, + "legend": { "avg": false, "current": false, "max": false, @@ -1163,59 +961,78 @@ data: }, "lines": true, "linewidth": 2, - "links": [], + "links": [ + + ], "nullPointMode": "connected", "percentage": false, - "pointradius": 1, + "pointradius": 5, "points": false, "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "span": 4, + "seriesOverrides": [ + + ], + "span": 6, "stack": false, - "steppedLine": true, + "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.99, sum(rate(etcd_disk_wal_fsync_duration_seconds_bucket[5m])) by (instance, le))", - "format": "time_series", - "hide": false, + "expr": "sum(rate(etcd_server_proposals_failed_total{job=\"$cluster\"}[5m]))", "intervalFactor": 2, - "legendFormat": "{{instance}} WAL fsync", - "metric": "etcd_disk_wal_fsync_duration_seconds_bucket", + "legendFormat": "Proposal Failure Rate", + "metric": "etcd_server_proposals_failed_total", "refId": "A", - "step": 4 + "step": 2 }, { - "expr": "histogram_quantile(0.99, sum(rate(etcd_disk_backend_commit_duration_seconds_bucket[5m])) by (instance, le))", - "format": "time_series", + "expr": "sum(etcd_server_proposals_pending{job=\"$cluster\"})", "intervalFactor": 2, - "legendFormat": "{{instance}} DB fsync", - "metric": "etcd_disk_backend_commit_duration_seconds_bucket", + "legendFormat": "Proposal Pending Total", + "metric": "etcd_server_proposals_pending", "refId": "B", - "step": 4 + "step": 2 + }, + { + "expr": "sum(rate(etcd_server_proposals_committed_total{job=\"$cluster\"}[5m]))", + "intervalFactor": 2, + "legendFormat": "Proposal Commit Rate", + "metric": "etcd_server_proposals_committed_total", + "refId": "C", + "step": 2 + }, + { + "expr": "sum(rate(etcd_server_proposals_applied_total{job=\"$cluster\"}[5m]))", + "intervalFactor": 2, + "legendFormat": "Proposal Apply Rate", + "refId": "D", + "step": 2 } ], - "thresholds": [], + "thresholds": [ + + ], "timeFrom": null, "timeShift": null, - "title": "Disk Sync Duration", + "title": "Raft Proposals", "tooltip": { "msResolution": false, "shared": true, "sort": 0, - "value_type": "cumulative" + "value_type": "individual" }, "type": "graph", "xaxis": { - "buckets": null, "mode": "time", "name": null, "show": true, - "values": [] + "values": [ + + ] }, "yaxes": [ { - "format": "s", + "format": "short", + "label": "", "logBase": 1, "max": null, "min": null, @@ -1223,60 +1040,69 @@ data: }, { "format": "short", + "label": null, "logBase": 1, "max": null, "min": null, - "show": false + "show": true } ] }, { - "aliasColors": {}, + "aliasColors": { + + }, "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "prometheus", - "editable": false, + "datasource": "$datasource", + "decimals": 0, + "editable": true, "error": false, "fill": 0, - "id": 29, + "id": 19, + "isNew": true, "legend": { + "alignAsTable": false, "avg": false, "current": false, "max": false, "min": false, + "rightSide": false, "show": false, "total": false, "values": false }, "lines": true, "linewidth": 2, - "links": [], + "links": [ + + ], "nullPointMode": "connected", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "span": 4, + "seriesOverrides": [ + + ], + "span": 6, "stack": false, "steppedLine": false, "targets": [ { - "expr": "process_resident_memory_bytes", - "format": "time_series", + "expr": "changes(etcd_server_leader_changes_seen_total{job=\"$cluster\"}[1d])", "intervalFactor": 2, - "legendFormat": "{{instance}} Resident Memory", - "metric": "process_resident_memory_bytes", + "legendFormat": "{{instance}} Total Leader Elections Per Day", + "metric": "etcd_server_leader_changes_seen_total", "refId": "A", - "step": 4 + "step": 2 } ], - "thresholds": [], + "thresholds": [ + + ], "timeFrom": null, "timeShift": null, - "title": "Memory", + "title": "Total Leader Elections Per Day", "tooltip": { "msResolution": false, "shared": true, @@ -1285,15 +1111,16 @@ data: }, "type": "graph", "xaxis": { - "buckets": null, "mode": "time", "name": null, "show": true, - "values": [] + "values": [ + + ] }, "yaxes": [ { - "format": "bytes", + "format": "short", "label": null, "logBase": 1, "max": null, @@ -1311,66 +1138,169 @@ data: ] } ], - "repeat": null, - "repeatIteration": null, - "repeatRowId": null, - "showTitle": false, - "title": "New row", - "titleSize": "h6" - }, + "title": "New row" + } + ], + "schemaVersion": 13, + "sharedCrosshair": false, + "style": "dark", + "tags": [ + + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": null, + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + "text": "prod", + "value": "prod" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": "cluster", + "multi": false, + "name": "cluster", + "options": [ + + ], + "query": "label_values(etcd_server_has_leader, job)", + "refresh": 1, + "regex": "", + "sort": 2, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-15m", + "to": "now" + }, + "timepicker": { + "now": true, + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "browser", + "title": "etcd", + "version": 215 + } + k8s-cluster-rsrc-use.json: |- + { + "annotations": { + "list": [ + + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "links": [ + + ], + "refresh": "10s", + "rows": [ { "collapse": false, "height": "250px", "panels": [ { - "aliasColors": {}, + "aliasColors": { + + }, "bars": false, "dashLength": 10, "dashes": false, - "datasource": "prometheus", - "editable": false, - "error": false, - "fill": 5, - "id": 22, + "datasource": "$datasource", + "fill": 10, + "id": 1, "legend": { "avg": false, "current": false, "max": false, "min": false, - "show": false, + "show": true, "total": false, "values": false }, "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "connected", + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", - "seriesOverrides": [], + "seriesOverrides": [ + + ], "spaceLength": 10, - "span": 3, + "span": 6, "stack": true, "steppedLine": false, "targets": [ { - "expr": "rate(etcd_network_client_grpc_received_bytes_total[5m])", + "expr": "node:cluster_cpu_utilisation:ratio", "format": "time_series", "intervalFactor": 2, - "legendFormat": "{{instance}} Client Traffic In", - "metric": "etcd_network_client_grpc_received_bytes_total", - "refId": "A", - "step": 4 + "legendFormat": "{{node}}", + "legendLink": "/d/4ac4f123aae0ff6dbaf4f4f66120033b/k8s-node-rsrc-use", + "step": 10 } ], - "thresholds": [], + "thresholds": [ + + ], "timeFrom": null, "timeShift": null, - "title": "Client Traffic In", + "title": "CPU Utilisation", "tooltip": { - "msResolution": false, "shared": true, "sort": 0, "value_type": "individual" @@ -1381,15 +1311,17 @@ data: "mode": "time", "name": null, "show": true, - "values": [] + "values": [ + + ] }, "yaxes": [ { - "format": "Bps", + "format": "percentunit", "label": null, "logBase": 1, - "max": null, - "min": null, + "max": 1, + "min": 0, "show": true }, { @@ -1398,59 +1330,63 @@ data: "logBase": 1, "max": null, "min": null, - "show": true + "show": false } ] }, { - "aliasColors": {}, + "aliasColors": { + + }, "bars": false, "dashLength": 10, "dashes": false, - "datasource": "prometheus", - "editable": false, - "error": false, - "fill": 5, - "id": 21, + "datasource": "$datasource", + "fill": 10, + "id": 2, "legend": { "avg": false, "current": false, "max": false, "min": false, - "show": false, + "show": true, "total": false, "values": false }, "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "connected", + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", - "seriesOverrides": [], + "seriesOverrides": [ + + ], "spaceLength": 10, - "span": 3, + "span": 6, "stack": true, "steppedLine": false, "targets": [ { - "expr": "rate(etcd_network_client_grpc_sent_bytes_total[5m])", + "expr": "node:node_cpu_saturation_load1: / scalar(sum(min(kube_pod_info) by (node)))", "format": "time_series", "intervalFactor": 2, - "legendFormat": "{{instance}} Client Traffic Out", - "metric": "etcd_network_client_grpc_sent_bytes_total", - "refId": "A", - "step": 4 + "legendFormat": "{{node}}", + "legendLink": "/d/4ac4f123aae0ff6dbaf4f4f66120033b/k8s-node-rsrc-use", + "step": 10 } ], - "thresholds": [], + "thresholds": [ + + ], "timeFrom": null, "timeShift": null, - "title": "Client Traffic Out", + "title": "CPU Saturation (Load1)", "tooltip": { - "msResolution": false, "shared": true, "sort": 0, "value_type": "individual" @@ -1461,15 +1397,17 @@ data: "mode": "time", "name": null, "show": true, - "values": [] + "values": [ + + ] }, "yaxes": [ { - "format": "Bps", + "format": "percentunit", "label": null, "logBase": 1, - "max": null, - "min": null, + "max": 1, + "min": 0, "show": true }, { @@ -1478,59 +1416,75 @@ data: "logBase": 1, "max": null, "min": null, - "show": true + "show": false } ] - }, + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "CPU", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ { - "aliasColors": {}, + "aliasColors": { + + }, "bars": false, "dashLength": 10, "dashes": false, - "datasource": "prometheus", - "editable": false, - "error": false, - "fill": 0, - "id": 20, + "datasource": "$datasource", + "fill": 10, + "id": 3, "legend": { "avg": false, "current": false, "max": false, "min": false, - "show": false, + "show": true, "total": false, "values": false }, "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "connected", + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", - "seriesOverrides": [], + "seriesOverrides": [ + + ], "spaceLength": 10, - "span": 3, - "stack": false, + "span": 6, + "stack": true, "steppedLine": false, "targets": [ { - "expr": "sum(rate(etcd_network_peer_received_bytes_total[5m])) by (instance)", + "expr": "node:cluster_memory_utilisation:ratio", "format": "time_series", "intervalFactor": 2, - "legendFormat": "{{instance}} Peer Traffic In", - "metric": "etcd_network_peer_received_bytes_total", - "refId": "A", - "step": 4 + "legendFormat": "{{node}}", + "legendLink": "/d/4ac4f123aae0ff6dbaf4f4f66120033b/k8s-node-rsrc-use", + "step": 10 } ], - "thresholds": [], + "thresholds": [ + + ], "timeFrom": null, "timeShift": null, - "title": "Peer Traffic In", + "title": "Memory Utilisation", "tooltip": { - "msResolution": false, "shared": true, "sort": 0, "value_type": "individual" @@ -1541,15 +1495,17 @@ data: "mode": "time", "name": null, "show": true, - "values": [] + "values": [ + + ] }, "yaxes": [ { - "format": "Bps", + "format": "percentunit", "label": null, "logBase": 1, - "max": null, - "min": null, + "max": 1, + "min": 0, "show": true }, { @@ -1558,66 +1514,66 @@ data: "logBase": 1, "max": null, "min": null, - "show": true + "show": false } ] }, { - "aliasColors": {}, + "aliasColors": { + + }, "bars": false, "dashLength": 10, "dashes": false, - "datasource": "prometheus", - "decimals": null, - "editable": false, - "error": false, - "fill": 0, - "grid": {}, - "id": 16, + "datasource": "$datasource", + "fill": 10, + "id": 4, "legend": { "avg": false, "current": false, "max": false, "min": false, - "show": false, + "show": true, "total": false, "values": false }, "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "connected", + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", - "seriesOverrides": [], + "seriesOverrides": [ + + ], "spaceLength": 10, - "span": 3, - "stack": false, + "span": 6, + "stack": true, "steppedLine": false, "targets": [ { - "expr": "sum(rate(etcd_network_peer_sent_bytes_total[5m])) by (instance)", + "expr": "node:node_memory_swap_io_bytes:sum_rate", "format": "time_series", - "hide": false, - "interval": "", "intervalFactor": 2, - "legendFormat": "{{instance}} Peer Traffic Out", - "metric": "etcd_network_peer_sent_bytes_total", - "refId": "A", - "step": 4 + "legendFormat": "{{node}}", + "legendLink": "/d/4ac4f123aae0ff6dbaf4f4f66120033b/k8s-node-rsrc-use", + "step": 10 } ], - "thresholds": [], + "thresholds": [ + + ], "timeFrom": null, "timeShift": null, - "title": "Peer Traffic Out", + "title": "Memory Saturation (Swap I/O)", "tooltip": { - "msResolution": false, "shared": true, "sort": 0, - "value_type": "cumulative" + "value_type": "individual" }, "type": "graph", "xaxis": { @@ -1625,22 +1581,26 @@ data: "mode": "time", "name": null, "show": true, - "values": [] + "values": [ + + ] }, "yaxes": [ { "format": "Bps", + "label": null, "logBase": 1, "max": null, - "min": null, + "min": 0, "show": true }, { "format": "short", + "label": null, "logBase": 1, "max": null, "min": null, - "show": true + "show": false } ] } @@ -1648,8 +1608,8 @@ data: "repeat": null, "repeatIteration": null, "repeatRowId": null, - "showTitle": false, - "title": "New row", + "showTitle": true, + "title": "Memory", "titleSize": "h6" }, { @@ -1657,80 +1617,58 @@ data: "height": "250px", "panels": [ { - "aliasColors": {}, + "aliasColors": { + + }, "bars": false, "dashLength": 10, "dashes": false, - "datasource": "prometheus", - "editable": false, - "error": false, - "fill": 0, - "id": 40, + "datasource": "$datasource", + "fill": 10, + "id": 5, "legend": { "avg": false, "current": false, "max": false, "min": false, - "show": false, + "show": true, "total": false, "values": false }, "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "connected", + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", - "seriesOverrides": [], + "seriesOverrides": [ + + ], "spaceLength": 10, "span": 6, - "stack": false, + "stack": true, "steppedLine": false, "targets": [ { - "expr": "sum(rate(etcd_server_proposals_failed_total[5m]))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "Proposal Failure Rate", - "metric": "etcd_server_proposals_failed_total", - "refId": "A", - "step": 2 - }, - { - "expr": "sum(etcd_server_proposals_pending)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "Proposal Pending Total", - "metric": "etcd_server_proposals_pending", - "refId": "B", - "step": 2 - }, - { - "expr": "sum(rate(etcd_server_proposals_committed_total[5m]))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "Proposal Commit Rate", - "metric": "etcd_server_proposals_committed_total", - "refId": "C", - "step": 2 - }, - { - "expr": "sum(rate(etcd_server_proposals_applied_total[5m]))", + "expr": "node:node_disk_utilisation:avg_irate / scalar(:kube_pod_info_node_count:)", "format": "time_series", "intervalFactor": 2, - "legendFormat": "Proposal Apply Rate", - "refId": "D", - "step": 2 + "legendFormat": "{{node}}", + "legendLink": "/d/4ac4f123aae0ff6dbaf4f4f66120033b/k8s-node-rsrc-use", + "step": 10 } ], - "thresholds": [], + "thresholds": [ + + ], "timeFrom": null, "timeShift": null, - "title": "Raft Proposals", + "title": "Disk IO Utilisation", "tooltip": { - "msResolution": false, "shared": true, "sort": 0, "value_type": "individual" @@ -1741,15 +1679,17 @@ data: "mode": "time", "name": null, "show": true, - "values": [] + "values": [ + + ] }, "yaxes": [ { - "format": "short", - "label": "", + "format": "percentunit", + "label": null, "logBase": 1, - "max": null, - "min": null, + "max": 1, + "min": 0, "show": true }, { @@ -1758,62 +1698,63 @@ data: "logBase": 1, "max": null, "min": null, - "show": true + "show": false } ] }, { - "aliasColors": {}, + "aliasColors": { + + }, "bars": false, "dashLength": 10, "dashes": false, - "datasource": "prometheus", - "decimals": 0, - "editable": false, - "error": false, - "fill": 0, - "id": 19, + "datasource": "$datasource", + "fill": 10, + "id": 6, "legend": { - "alignAsTable": false, "avg": false, "current": false, "max": false, "min": false, - "rightSide": false, - "show": false, + "show": true, "total": false, "values": false }, "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "connected", + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", - "seriesOverrides": [], + "seriesOverrides": [ + + ], "spaceLength": 10, "span": 6, - "stack": false, + "stack": true, "steppedLine": false, "targets": [ { - "expr": "changes(etcd_server_leader_changes_seen_total[1d])", + "expr": "node:node_disk_saturation:avg_irate / scalar(:kube_pod_info_node_count:)", "format": "time_series", "intervalFactor": 2, - "legendFormat": "{{instance}} Total Leader Elections Per Day", - "metric": "etcd_server_leader_changes_seen_total", - "refId": "A", - "step": 2 + "legendFormat": "{{node}}", + "legendLink": "/d/4ac4f123aae0ff6dbaf4f4f66120033b/k8s-node-rsrc-use", + "step": 10 } ], - "thresholds": [], + "thresholds": [ + + ], "timeFrom": null, "timeShift": null, - "title": "Total Leader Elections Per Day", + "title": "Disk IO Saturation", "tooltip": { - "msResolution": false, "shared": true, "sort": 0, "value_type": "individual" @@ -1824,15 +1765,17 @@ data: "mode": "time", "name": null, "show": true, - "values": [] + "values": [ + + ] }, "yaxes": [ { - "format": "short", + "format": "percentunit", "label": null, "logBase": 1, - "max": null, - "min": null, + "max": 1, + "min": 0, "show": true }, { @@ -1841,7 +1784,7 @@ data: "logBase": 1, "max": null, "min": null, - "show": true + "show": false } ] } @@ -1849,686 +1792,587 @@ data: "repeat": null, "repeatIteration": null, "repeatRowId": null, - "showTitle": false, - "title": "New row", + "showTitle": true, + "title": "Disk", "titleSize": "h6" - } - ], - "schemaVersion": 14, - "style": "dark", - "tags": [], - "templating": { - "list": [] - }, - "time": { - "from": "now-15m", - "to": "now" - }, - "timepicker": { - "now": true, - "refresh_intervals": [ - "5s", - "10s", - "30s", - "1m", - "5m", - "15m", - "30m", - "1h", - "2h", - "1d" - ], - "time_options": [ - "5m", - "15m", - "1h", - "6h", - "12h", - "24h", - "2d", - "7d", - "30d" - ] - }, - "timezone": "browser", - "title": "etcd", - "version": 4 - } - kubernetes-capacity-planning-dashboard.json: |+ - { - "__inputs": [ - { - "description": "", - "label": "prometheus", - "name": "prometheus", - "pluginId": "prometheus", - "pluginName": "Prometheus", - "type": "datasource" - } - ], - "annotations": { - "list": [] - }, - "editable": false, - "gnetId": 22, - "graphTooltip": 0, - "hideControls": false, - "links": [], - "refresh": false, - "rows": [ + }, { "collapse": false, - "editable": false, "height": "250px", "panels": [ { - "aliasColors": {}, + "aliasColors": { + + }, "bars": false, "dashLength": 10, "dashes": false, - "datasource": "prometheus", - "editable": false, - "error": false, - "fill": 1, - "grid": { - "threshold1Color": "rgba(216, 200, 27, 0.27)", - "threshold2Color": "rgba(234, 112, 112, 0.22)" - }, - "id": 3, - "isNew": false, + "datasource": "$datasource", + "fill": 10, + "id": 7, "legend": { - "alignAsTable": false, "avg": false, "current": false, - "hideEmpty": false, - "hideZero": false, "max": false, "min": false, - "rightSide": false, "show": true, - "total": false + "total": false, + "values": false }, "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "connected", + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", - "seriesOverrides": [], + "seriesOverrides": [ + + ], "spaceLength": 10, "span": 6, - "stack": false, + "stack": true, "steppedLine": false, "targets": [ { - "expr": "sum(rate(node_cpu_seconds_total{mode=\"idle\"}[2m])) * 100", - "hide": false, - "intervalFactor": 10, - "legendFormat": "", - "refId": "A", - "step": 50 + "expr": "node:node_net_utilisation:sum_irate", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{node}}", + "legendLink": "/d/4ac4f123aae0ff6dbaf4f4f66120033b/k8s-node-rsrc-use", + "step": 10 } ], - "title": "Idle CPU", + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Net Utilisation (Transmitted)", "tooltip": { - "msResolution": false, "shared": true, "sort": 0, - "value_type": "cumulative" + "value_type": "individual" }, "type": "graph", "xaxis": { + "buckets": null, "mode": "time", + "name": null, "show": true, - "values": [] + "values": [ + + ] }, "yaxes": [ { - "format": "percent", - "label": "cpu usage", + "format": "Bps", + "label": null, "logBase": 1, + "max": null, "min": 0, "show": true }, { "format": "short", + "label": null, "logBase": 1, - "show": true + "max": null, + "min": null, + "show": false } ] }, { - "aliasColors": {}, + "aliasColors": { + + }, "bars": false, "dashLength": 10, "dashes": false, - "datasource": "prometheus", - "editable": false, - "error": false, - "fill": 1, - "grid": { - "threshold1Color": "rgba(216, 200, 27, 0.27)", - "threshold2Color": "rgba(234, 112, 112, 0.22)" - }, - "id": 9, - "isNew": false, + "datasource": "$datasource", + "fill": 10, + "id": 8, "legend": { - "alignAsTable": false, "avg": false, "current": false, - "hideEmpty": false, - "hideZero": false, "max": false, "min": false, - "rightSide": false, "show": true, - "total": false + "total": false, + "values": false }, "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "connected", + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", - "seriesOverrides": [], + "seriesOverrides": [ + + ], "spaceLength": 10, "span": 6, - "stack": false, + "stack": true, "steppedLine": false, "targets": [ { - "expr": "sum(node_load1)", - "intervalFactor": 4, - "legendFormat": "load 1m", - "refId": "A", - "step": 20, - "target": "" - }, - { - "expr": "sum(node_load5)", - "intervalFactor": 4, - "legendFormat": "load 5m", - "refId": "B", - "step": 20, - "target": "" - }, - { - "expr": "sum(node_load15)", - "intervalFactor": 4, - "legendFormat": "load 15m", - "refId": "C", - "step": 20, - "target": "" - } - ], - "title": "System Load", + "expr": "node:node_net_saturation:sum_irate", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{node}}", + "legendLink": "/d/4ac4f123aae0ff6dbaf4f4f66120033b/k8s-node-rsrc-use", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Net Saturation (Dropped)", "tooltip": { - "msResolution": false, "shared": true, "sort": 0, - "value_type": "cumulative" + "value_type": "individual" }, "type": "graph", "xaxis": { + "buckets": null, "mode": "time", + "name": null, "show": true, - "values": [] + "values": [ + + ] }, "yaxes": [ { - "format": "percentunit", + "format": "Bps", + "label": null, "logBase": 1, + "max": null, + "min": 0, "show": true }, { "format": "short", + "label": null, "logBase": 1, - "show": true + "max": null, + "min": null, + "show": false } ] } ], - "showTitle": false, - "title": "New Row", + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Network", "titleSize": "h6" }, { "collapse": false, - "editable": false, "height": "250px", "panels": [ { - "aliasColors": {}, + "aliasColors": { + + }, "bars": false, "dashLength": 10, "dashes": false, - "datasource": "prometheus", - "editable": false, - "error": false, - "fill": 1, - "grid": { - "threshold1Color": "rgba(216, 200, 27, 0.27)", - "threshold2Color": "rgba(234, 112, 112, 0.22)" - }, - "id": 4, - "isNew": false, + "datasource": "$datasource", + "fill": 10, + "id": 9, "legend": { - "alignAsTable": false, "avg": false, "current": false, - "hideEmpty": false, - "hideZero": false, "max": false, "min": false, - "rightSide": false, "show": true, - "total": false + "total": false, + "values": false }, "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "connected", + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [ - { - "alias": "node_memory_SwapFree_bytes{instance=\"172.17.0.1:9100\",job=\"prometheus\"}", - "yaxis": 2 - } + ], "spaceLength": 10, - "span": 9, + "span": 12, "stack": true, "steppedLine": false, "targets": [ { - "expr": "sum(node_memory_MemTotal_bytes) - sum(node_memory_MemFree_bytes) - sum(node_memory_Buffers_bytes) - sum(node_memory_Cached_bytes)", - "intervalFactor": 2, - "legendFormat": "memory usage", - "metric": "memo", - "refId": "A", - "step": 10, - "target": "" - }, - { - "expr": "sum(node_memory_Buffers_bytes)", - "interval": "", - "intervalFactor": 2, - "legendFormat": "memory buffers", - "metric": "memo", - "refId": "B", - "step": 10, - "target": "" - }, - { - "expr": "sum(node_memory_Cached_bytes)", - "interval": "", - "intervalFactor": 2, - "legendFormat": "memory cached", - "metric": "memo", - "refId": "C", - "step": 10, - "target": "" - }, - { - "expr": "sum(node_memory_MemFree_bytes)", - "interval": "", + "expr": "sum(max(node_filesystem_size_bytes{fstype=~\"ext[234]|btrfs|xfs|zfs\"} - node_filesystem_avail_bytes{fstype=~\"ext[234]|btrfs|xfs|zfs\"}) by (device,pod,namespace)) by (pod,namespace)\n/ scalar(sum(max(node_filesystem_size_bytes{fstype=~\"ext[234]|btrfs|xfs|zfs\"}) by (device,pod,namespace)))\n* on (namespace, pod) group_left (node) node_namespace_pod:kube_pod_info:\n", + "format": "time_series", "intervalFactor": 2, - "legendFormat": "memory free", - "metric": "memo", - "refId": "D", - "step": 10, - "target": "" + "legendFormat": "{{node}}", + "legendLink": "/d/4ac4f123aae0ff6dbaf4f4f66120033b/k8s-node-rsrc-use", + "step": 10 } ], - "title": "Memory Usage", + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Disk Capacity", "tooltip": { - "msResolution": false, "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { + "buckets": null, "mode": "time", + "name": null, "show": true, - "values": [] + "values": [ + + ] }, "yaxes": [ { - "format": "bytes", + "format": "percentunit", + "label": null, "logBase": 1, - "min": "0", + "max": 1, + "min": 0, "show": true }, { "format": "short", + "label": null, "logBase": 1, - "show": true + "max": null, + "min": null, + "show": false } ] - }, - { - "colorBackground": false, - "colorValue": false, - "colors": [ - "rgba(50, 172, 45, 0.97)", - "rgba(237, 129, 40, 0.89)", - "rgba(245, 54, 54, 0.9)" - ], - "datasource": "prometheus", - "editable": false, - "format": "percent", - "gauge": { - "maxValue": 100, - "minValue": 0, - "show": true, - "thresholdLabels": false, - "thresholdMarkers": true - }, - "hideTimeOverride": false, - "id": 5, - "links": [], - "mappingType": 1, - "mappingTypes": [ - { - "name": "value to text", - "value": 1 - }, - { - "name": "range to text", - "value": 2 - } - ], - "maxDataPoints": 100, - "nullPointMode": "connected", - "postfix": "", - "postfixFontSize": "50%", - "prefix": "", - "prefixFontSize": "50%", - "rangeMaps": [ - { - "from": "null", - "text": "N/A", - "to": "null" - } - ], - "span": 3, - "sparkline": { - "fillColor": "rgba(31, 118, 189, 0.18)", - "full": false, - "lineColor": "rgb(31, 120, 193)", - "show": false - }, - "targets": [ - { - "expr": "((sum(node_memory_MemTotal_bytes) - sum(node_memory_MemFree_bytes) - sum(node_memory_Buffers_bytes) - sum(node_memory_Cached_bytes)) / sum(node_memory_MemTotal_bytes)) * 100", - "intervalFactor": 2, - "metric": "", - "refId": "A", - "step": 60, - "target": "" - } - ], - "thresholds": "80, 90", - "title": "Memory Usage", - "transparent": false, - "type": "singlestat", - "valueFontSize": "80%", - "valueMaps": [ - { - "op": "=", - "text": "N/A", - "value": "null" - } - ], - "valueName": "avg" } ], - "showTitle": false, - "title": "New Row", + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Storage", "titleSize": "h6" - }, + } + ], + "schemaVersion": 14, + "style": "dark", + "tags": [ + "kubernetes-mixin" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": null, + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "", + "title": "Kubernetes / USE Method / Cluster", + "uid": "a6e7d1362e1ddbb79db21d5bb40d7137", + "version": 0 + } + k8s-node-rsrc-use.json: |- + { + "annotations": { + "list": [ + + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "links": [ + + ], + "refresh": "10s", + "rows": [ { "collapse": false, - "editable": false, - "height": "246px", + "height": "250px", "panels": [ { - "aliasColors": {}, + "aliasColors": { + + }, "bars": false, "dashLength": 10, "dashes": false, - "datasource": "prometheus", - "editable": false, - "error": false, + "datasource": "$datasource", "fill": 1, - "grid": { - "threshold1Color": "rgba(216, 200, 27, 0.27)", - "threshold2Color": "rgba(234, 112, 112, 0.22)" - }, - "id": 6, - "isNew": false, + "id": 1, "legend": { - "alignAsTable": false, "avg": false, "current": false, - "hideEmpty": false, - "hideZero": false, "max": false, "min": false, - "rightSide": false, "show": true, - "total": false + "total": false, + "values": false }, "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "connected", + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [ - { - "alias": "read", - "yaxis": 1 - }, - { - "alias": "{instance=\"172.17.0.1:9100\"}", - "yaxis": 2 - }, - { - "alias": "io time", - "yaxis": 2 - } + ], "spaceLength": 10, - "span": 9, + "span": 6, "stack": false, "steppedLine": false, "targets": [ { - "expr": "max(rate(node_disk_read_bytes_total[5m]))", - "hide": false, - "intervalFactor": 4, - "legendFormat": "read", - "refId": "A", - "step": 20, - "target": "" - }, - { - "expr": "max(rate(node_disk_written_bytes_total[5m]))", - "intervalFactor": 4, - "legendFormat": "written", - "refId": "B", - "step": 20 - }, - { - "expr": "max(rate(node_disk_io_time_seconds_total[5m]))", - "intervalFactor": 4, - "legendFormat": "io time", - "refId": "C", - "step": 20 + "expr": "node:node_cpu_utilisation:avg1m{node=\"$node\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Utilisation", + "legendLink": null, + "step": 10 } ], - "title": "Disk I/O", + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU Utilisation", "tooltip": { - "msResolution": false, "shared": true, "sort": 0, - "value_type": "cumulative" + "value_type": "individual" }, "type": "graph", "xaxis": { + "buckets": null, "mode": "time", + "name": null, "show": true, - "values": [] + "values": [ + + ] }, "yaxes": [ { - "format": "bytes", + "format": "percentunit", + "label": null, "logBase": 1, + "max": null, + "min": 0, "show": true }, { - "format": "ms", + "format": "short", + "label": null, "logBase": 1, - "show": true + "max": null, + "min": null, + "show": false } ] }, { - "colorBackground": false, - "colorValue": false, - "colors": [ - "rgba(50, 172, 45, 0.97)", - "rgba(237, 129, 40, 0.89)", - "rgba(245, 54, 54, 0.9)" - ], - "datasource": "prometheus", - "editable": false, - "format": "percentunit", - "gauge": { - "maxValue": 1, - "minValue": 0, + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 2, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, "show": true, - "thresholdLabels": false, - "thresholdMarkers": true + "total": false, + "values": false }, - "hideTimeOverride": false, - "id": 12, - "links": [], - "mappingType": 1, - "mappingTypes": [ - { - "name": "value to text", - "value": 1 - }, - { - "name": "range to text", - "value": 2 - } + "lines": true, + "linewidth": 1, + "links": [ + ], - "maxDataPoints": 100, - "nullPointMode": "connected", - "postfix": "", - "postfixFontSize": "50%", - "prefix": "", - "prefixFontSize": "50%", - "rangeMaps": [ - { - "from": "null", - "text": "N/A", - "to": "null" - } + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + ], - "span": 3, - "sparkline": { - "fillColor": "rgba(31, 118, 189, 0.18)", - "full": false, - "lineColor": "rgb(31, 120, 193)", - "show": false - }, + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, "targets": [ { - "expr": "(sum(node_filesystem_size_bytes{device!=\"rootfs\"}) - sum(node_filesystem_free_bytes{device!=\"rootfs\"})) / sum(node_filesystem_size_bytes{device!=\"rootfs\"})", + "expr": "node:node_cpu_saturation_load1:{node=\"$node\"}", + "format": "time_series", "intervalFactor": 2, - "refId": "A", - "step": 60, - "target": "" + "legendFormat": "Saturation", + "legendLink": null, + "step": 10 } ], - "thresholds": "0.75, 0.9", - "title": "Disk Space Usage", - "transparent": false, - "type": "singlestat", - "valueFontSize": "80%", - "valueMaps": [ + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU Saturation (Load1)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ { - "op": "=", - "text": "N/A", - "value": "null" + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false } - ], - "valueName": "current" + ] } ], - "showTitle": false, - "title": "New Row", + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "CPU", "titleSize": "h6" }, { "collapse": false, - "editable": false, "height": "250px", "panels": [ { - "aliasColors": {}, + "aliasColors": { + + }, "bars": false, "dashLength": 10, "dashes": false, - "datasource": "prometheus", - "editable": false, - "error": false, + "datasource": "$datasource", "fill": 1, - "grid": { - "threshold1Color": "rgba(216, 200, 27, 0.27)", - "threshold2Color": "rgba(234, 112, 112, 0.22)" - }, - "id": 8, - "isNew": false, + "id": 3, "legend": { - "alignAsTable": false, "avg": false, "current": false, - "hideEmpty": false, - "hideZero": false, "max": false, "min": false, - "rightSide": false, "show": true, - "total": false + "total": false, + "values": false }, "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "connected", + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [ - { - "alias": "transmitted", - "yaxis": 2 - } + ], "spaceLength": 10, "span": 6, @@ -2536,81 +2380,85 @@ data: "steppedLine": false, "targets": [ { - "expr": "sum(rate(node_network_receive_bytes_total{device!~\"lo\"}[5m]))", - "hide": false, + "expr": "node:node_memory_utilisation:{node=\"$node\"}", + "format": "time_series", "intervalFactor": 2, - "legendFormat": "", - "refId": "A", - "step": 10, - "target": "" + "legendFormat": "Memory", + "legendLink": null, + "step": 10 } ], - "title": "Network Received", + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Memory Utilisation", "tooltip": { - "msResolution": false, "shared": true, "sort": 0, - "value_type": "cumulative" + "value_type": "individual" }, "type": "graph", "xaxis": { + "buckets": null, "mode": "time", + "name": null, "show": true, - "values": [] + "values": [ + + ] }, "yaxes": [ { - "format": "bytes", + "format": "percentunit", + "label": null, "logBase": 1, + "max": null, + "min": 0, "show": true }, { - "format": "bytes", + "format": "short", + "label": null, "logBase": 1, - "show": true + "max": null, + "min": null, + "show": false } ] }, { - "aliasColors": {}, + "aliasColors": { + + }, "bars": false, "dashLength": 10, "dashes": false, - "datasource": "prometheus", - "editable": false, - "error": false, + "datasource": "$datasource", "fill": 1, - "grid": { - "threshold1Color": "rgba(216, 200, 27, 0.27)", - "threshold2Color": "rgba(234, 112, 112, 0.22)" - }, - "id": 10, - "isNew": false, + "id": 4, "legend": { - "alignAsTable": false, "avg": false, "current": false, - "hideEmpty": false, - "hideZero": false, "max": false, "min": false, - "rightSide": false, "show": true, - "total": false + "total": false, + "values": false }, "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "connected", + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [ - { - "alias": "transmitted", - "yaxis": 2 - } + ], "spaceLength": 10, "span": 6, @@ -2618,921 +2466,1905 @@ data: "steppedLine": false, "targets": [ { - "expr": "sum(rate(node_network_transmit_bytes_total{device!~\"lo\"}[5m]))", - "hide": false, + "expr": "node:node_memory_swap_io_bytes:sum_rate{node=\"$node\"}", + "format": "time_series", "intervalFactor": 2, - "legendFormat": "", - "refId": "B", - "step": 10, - "target": "" + "legendFormat": "Swap IO", + "legendLink": null, + "step": 10 } ], - "title": "Network Transmitted", + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Memory Saturation (Swap I/O)", "tooltip": { - "msResolution": false, "shared": true, "sort": 0, - "value_type": "cumulative" + "value_type": "individual" }, "type": "graph", "xaxis": { + "buckets": null, "mode": "time", + "name": null, "show": true, - "values": [] + "values": [ + + ] }, "yaxes": [ { - "format": "bytes", + "format": "Bps", + "label": null, "logBase": 1, + "max": null, + "min": 0, "show": true }, { - "format": "bytes", + "format": "short", + "label": null, "logBase": 1, - "show": true + "max": null, + "min": null, + "show": false } ] } ], - "showTitle": false, - "title": "New Row", + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Memory", "titleSize": "h6" }, { "collapse": false, - "editable": false, - "height": "276px", + "height": "250px", "panels": [ { - "aliasColors": {}, + "aliasColors": { + + }, "bars": false, + "dashLength": 10, "dashes": false, - "datasource": "prometheus", - "editable": false, - "error": false, + "datasource": "$datasource", "fill": 1, - "grid": { - "threshold1Color": "rgba(216, 200, 27, 0.27)", - "threshold2Color": "rgba(234, 112, 112, 0.22)" - }, - "id": 11, - "isNew": true, + "id": 5, "legend": { - "alignAsTable": false, "avg": false, "current": false, - "hideEmpty": false, - "hideZero": false, "max": false, "min": false, - "rightSide": false, "show": true, - "total": false + "total": false, + "values": false }, "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "connected", + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 11, - "span": 9, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, "stack": false, "steppedLine": false, "targets": [ { - "expr": "sum(kube_pod_info)", + "expr": "node:node_disk_utilisation:avg_irate{node=\"$node\"}", "format": "time_series", "intervalFactor": 2, - "legendFormat": "Current number of Pods", - "refId": "A", - "step": 10 - }, - { - "expr": "sum(kube_node_status_capacity_pods)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "Maximum capacity of pods", - "refId": "B", + "legendFormat": "Utilisation", + "legendLink": null, "step": 10 } ], - "title": "Cluster Pod Utilization", + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Disk IO Utilisation", "tooltip": { - "msResolution": false, "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { + "buckets": null, "mode": "time", + "name": null, "show": true, - "values": [] + "values": [ + + ] }, "yaxes": [ { - "format": "short", + "format": "percentunit", + "label": null, "logBase": 1, + "max": null, + "min": 0, "show": true }, { "format": "short", + "label": null, "logBase": 1, - "show": true + "max": null, + "min": null, + "show": false } ] }, { - "colorBackground": false, - "colorValue": false, - "colors": [ - "rgba(50, 172, 45, 0.97)", - "rgba(237, 129, 40, 0.89)", - "rgba(245, 54, 54, 0.9)" - ], - "datasource": "prometheus", - "editable": false, - "format": "percent", - "gauge": { - "maxValue": 100, - "minValue": 0, + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 6, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, "show": true, - "thresholdLabels": false, - "thresholdMarkers": true + "total": false, + "values": false }, - "hideTimeOverride": false, - "id": 7, - "links": [], - "mappingType": 1, - "mappingTypes": [ - { - "name": "value to text", - "value": 1 - }, - { - "name": "range to text", - "value": 2 - } + "lines": true, + "linewidth": 1, + "links": [ + ], - "maxDataPoints": 100, - "nullPointMode": "connected", - "postfix": "", - "postfixFontSize": "50%", - "prefix": "", - "prefixFontSize": "50%", - "rangeMaps": [ - { - "from": "null", - "text": "N/A", - "to": "null" - } + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + ], - "span": 3, - "sparkline": { - "fillColor": "rgba(31, 118, 189, 0.18)", - "full": false, - "lineColor": "rgb(31, 120, 193)", - "show": false - }, + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, "targets": [ { - "expr": "100 - (sum(kube_node_status_capacity_pods) - sum(kube_pod_info)) / sum(kube_node_status_capacity_pods) * 100", + "expr": "node:node_disk_saturation:avg_irate{node=\"$node\"}", "format": "time_series", "intervalFactor": 2, - "legendFormat": "", - "refId": "A", - "step": 60, - "target": "" + "legendFormat": "Saturation", + "legendLink": null, + "step": 10 } ], - "thresholds": "80, 90", - "title": "Pod Utilization", - "transparent": false, - "type": "singlestat", - "valueFontSize": "80%", - "valueMaps": [ + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Disk IO Saturation", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ { - "op": "=", - "text": "N/A", - "value": "null" + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false } - ], - "valueName": "current" + ] } ], - "showTitle": false, - "title": "New Row", + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Disk", "titleSize": "h6" - } - ], - "schemaVersion": 14, - "sharedCrosshair": false, - "style": "dark", - "tags": [], - "templating": { - "list": [] - }, - "time": { - "from": "now-1h", - "to": "now" - }, - "timepicker": { - "refresh_intervals": [ - "5s", - "10s", - "30s", - "1m", - "5m", - "15m", - "30m", - "1h", - "2h", - "1d" - ], - "time_options": [ - "5m", - "15m", - "1h", - "6h", - "12h", - "24h", - "2d", - "7d", - "30d" - ] - }, - "timezone": "browser", - "title": "Kubernetes Capacity Planning", - "version": 4 - } - kubernetes-cluster-health-dashboard.json: |+ - { - "__inputs": [ - { - "description": "", - "label": "prometheus", - "name": "prometheus", - "pluginId": "prometheus", - "pluginName": "Prometheus", - "type": "datasource" - } - ], - "annotations": { - "list": [] - }, - "editable": false, - "graphTooltip": 0, - "hideControls": false, - "links": [], - "refresh": "10s", - "rows": [ + }, { "collapse": false, - "editable": false, - "height": "254px", + "height": "250px", "panels": [ { - "colorBackground": false, - "colorValue": true, - "colors": [ - "rgba(50, 172, 45, 0.97)", - "rgba(237, 129, 40, 0.89)", - "rgba(245, 54, 54, 0.9)" - ], - "datasource": "prometheus", - "editable": false, - "format": "none", - "gauge": { - "maxValue": 100, - "minValue": 0, - "show": false, - "thresholdLabels": false, - "thresholdMarkers": true + "aliasColors": { + }, - "hideTimeOverride": false, - "id": 1, - "links": [], - "mappingType": 1, - "mappingTypes": [ - { - "name": "value to text", - "value": 1 - }, - { - "name": "range to text", - "value": 2 - } + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 7, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + ], - "maxDataPoints": 100, - "nullPointMode": "connected", - "postfix": "", - "postfixFontSize": "50%", - "prefix": "", - "prefixFontSize": "50%", - "rangeMaps": [ - { - "from": "null", - "text": "N/A", - "to": "null" - } + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + ], - "span": 3, - "sparkline": { - "fillColor": "rgba(31, 118, 189, 0.18)", - "full": false, - "lineColor": "rgb(31, 120, 193)", - "show": false - }, + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, "targets": [ { - "expr": "sum(up{job=~\"apiserver|kube-scheduler|kube-controller-manager\"} == 0)", + "expr": "node:node_net_utilisation:sum_irate{node=\"$node\"}", "format": "time_series", "intervalFactor": 2, - "legendFormat": "", - "refId": "A", - "step": 600 + "legendFormat": "Utilisation", + "legendLink": null, + "step": 10 } ], - "thresholds": "1, 3", - "title": "Control Plane Components Down", - "transparent": false, - "type": "singlestat", - "valueFontSize": "80%", - "valueMaps": [ + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Net Utilisation (Transmitted)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ { - "op": "=", - "text": "Everything UP and healthy", - "value": "null" + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true }, { - "op": "=", - "text": "", - "value": "" + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false } - ], - "valueName": "avg" + ] }, { - "colorBackground": false, - "colorValue": true, - "colors": [ - "rgba(50, 172, 45, 0.97)", - "rgba(237, 129, 40, 0.89)", - "rgba(245, 54, 54, 0.9)" - ], - "datasource": "prometheus", - "editable": false, - "format": "none", - "gauge": { - "maxValue": 100, - "minValue": 0, - "show": false, - "thresholdLabels": false, - "thresholdMarkers": true + "aliasColors": { + }, - "hideTimeOverride": false, - "id": 2, - "links": [], - "mappingType": 1, - "mappingTypes": [ - { - "name": "value to text", - "value": 1 - }, - { - "name": "range to text", - "value": 2 - } + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 8, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + ], - "maxDataPoints": 100, - "nullPointMode": "connected", - "postfix": "", - "postfixFontSize": "50%", - "prefix": "", - "prefixFontSize": "50%", - "rangeMaps": [ - { - "from": "null", - "text": "N/A", - "to": "null" - } + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + ], - "span": 3, - "sparkline": { - "fillColor": "rgba(31, 118, 189, 0.18)", - "full": false, - "lineColor": "rgb(31, 120, 193)", - "show": false - }, + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, "targets": [ { - "expr": "sum(ALERTS{alertstate=\"firing\",alertname!=\"DeadMansSwitch\"})", + "expr": "node:node_net_saturation:sum_irate{node=\"$node\"}", "format": "time_series", "intervalFactor": 2, - "legendFormat": "", - "refId": "A", - "step": 600 - } - ], - "thresholds": "1, 3", - "title": "Alerts Firing", - "transparent": false, - "type": "singlestat", - "valueFontSize": "80%", - "valueMaps": [ - { - "op": "=", - "text": "0", - "value": "null" + "legendFormat": "Saturation", + "legendLink": null, + "step": 10 } ], - "valueName": "current" - }, - { - "colorBackground": false, - "colorValue": true, - "colors": [ - "rgba(50, 172, 45, 0.97)", - "rgba(237, 129, 40, 0.89)", - "rgba(245, 54, 54, 0.9)" + "thresholds": [ + ], - "datasource": "prometheus", - "editable": false, - "format": "none", - "gauge": { - "maxValue": 100, - "minValue": 0, - "show": false, - "thresholdLabels": false, - "thresholdMarkers": true + "timeFrom": null, + "timeShift": null, + "title": "Net Saturation (Dropped)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" }, - "hideTimeOverride": false, - "id": 3, - "links": [], - "mappingType": 1, - "mappingTypes": [ + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ { - "name": "value to text", - "value": 1 + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true }, { - "name": "range to text", - "value": 2 + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Net", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 9, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + ], - "maxDataPoints": 100, - "nullPointMode": "connected", - "postfix": "", - "postfixFontSize": "50%", - "prefix": "", - "prefixFontSize": "50%", - "rangeMaps": [ - { - "from": "null", - "text": "N/A", - "to": "null" - } + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + ], - "span": 3, - "sparkline": { - "fillColor": "rgba(31, 118, 189, 0.18)", - "full": false, - "lineColor": "rgb(31, 120, 193)", - "show": false - }, + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, "targets": [ { - "expr": "sum(ALERTS{alertstate=\"pending\",alertname!=\"DeadMansSwitch\"})", + "expr": "node:node_filesystem_usage:\n* on (namespace, pod) group_left (node) node_namespace_pod:kube_pod_info:{node=\"$node\"}\n", "format": "time_series", "intervalFactor": 2, - "legendFormat": "", - "refId": "A", - "step": 600 - } - ], - "thresholds": "3, 5", - "title": "Alerts Pending", - "transparent": false, - "type": "singlestat", - "valueFontSize": "80%", - "valueMaps": [ - { - "op": "=", - "text": "0", - "value": "null" + "legendFormat": "{{device}}", + "legendLink": null, + "step": 10 } ], - "valueName": "current" - }, - { - "colorBackground": false, - "colorValue": true, - "colors": [ - "rgba(50, 172, 45, 0.97)", - "rgba(237, 129, 40, 0.89)", - "rgba(245, 54, 54, 0.9)" + "thresholds": [ + ], - "datasource": "prometheus", - "editable": false, - "format": "none", - "gauge": { - "maxValue": 100, - "minValue": 0, - "show": false, - "thresholdLabels": false, - "thresholdMarkers": true - }, - "hideTimeOverride": false, - "id": 4, - "links": [], - "mappingType": 1, - "mappingTypes": [ + "timeFrom": null, + "timeShift": null, + "title": "Disk Utilisation", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Disk", + "titleSize": "h6" + } + ], + "schemaVersion": 14, + "style": "dark", + "tags": [ + "kubernetes-mixin" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": null, + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + "text": "prod", + "value": "prod" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": "node", + "multi": false, + "name": "node", + "options": [ + + ], + "query": "label_values(kube_node_info, node)", + "refresh": 1, + "regex": "", + "sort": 2, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "", + "title": "Kubernetes / USE Method / Node", + "uid": "4ac4f123aae0ff6dbaf4f4f66120033b", + "version": 0 + } + k8s-resources-cluster.json: |- + { + "annotations": { + "list": [ + + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "links": [ + + ], + "refresh": "10s", + "rows": [ + { + "collapse": false, + "height": "100px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "format": "percentunit", + "id": 1, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[1m]))", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "refId": "A" + } + ], + "thresholds": "70,80", + "timeFrom": null, + "timeShift": null, + "title": "CPU Utilisation", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "singlestat", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "format": "percentunit", + "id": 2, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(kube_pod_container_resource_requests_cpu_cores) / sum(node:node_num_cpu:sum)", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "refId": "A" + } + ], + "thresholds": "70,80", + "timeFrom": null, + "timeShift": null, + "title": "CPU Requests Commitment", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "singlestat", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "format": "percentunit", + "id": 3, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(kube_pod_container_resource_limits_cpu_cores) / sum(node:node_num_cpu:sum)", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "refId": "A" + } + ], + "thresholds": "70,80", + "timeFrom": null, + "timeShift": null, + "title": "CPU Limits Commitment", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "singlestat", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "format": "percentunit", + "id": 4, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "1 - sum(:node_memory_MemFreeCachedBuffers_bytes:sum) / sum(:node_memory_MemTotal_bytes:sum)", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "refId": "A" + } + ], + "thresholds": "70,80", + "timeFrom": null, + "timeShift": null, + "title": "Memory Utilisation", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "singlestat", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "format": "percentunit", + "id": 5, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(kube_pod_container_resource_requests_memory_bytes) / sum(:node_memory_MemTotal_bytes:sum)", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "refId": "A" + } + ], + "thresholds": "70,80", + "timeFrom": null, + "timeShift": null, + "title": "Memory Requests Commitment", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "singlestat", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "format": "percentunit", + "id": 6, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(kube_pod_container_resource_limits_memory_bytes) / sum(:node_memory_MemTotal_bytes:sum)", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "refId": "A" + } + ], + "thresholds": "70,80", + "timeFrom": null, + "timeShift": null, + "title": "Memory Limits Commitment", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "singlestat", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Headlines", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 7, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(namespace_pod_name_container_name:container_cpu_usage_seconds_total:sum_rate) by (namespace)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{namespace}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU Usage", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "CPU", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 8, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, { - "name": "value to text", - "value": 1 + "alias": "CPU Usage", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #A", + "thresholds": [ + + ], + "type": "number", + "unit": "short" }, { - "name": "range to text", - "value": 2 - } - ], - "maxDataPoints": 100, - "nullPointMode": "connected", - "postfix": "", - "postfixFontSize": "50%", - "prefix": "", - "prefixFontSize": "50%", - "rangeMaps": [ + "alias": "CPU Requests", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #B", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, { - "from": "null", - "text": "N/A", - "to": "null" + "alias": "CPU Requests %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #C", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "CPU Limits", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #D", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "CPU Limits %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #E", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "Namespace", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": true, + "linkTooltip": "Drill down", + "linkUrl": "/d/85a562078cdf77779eaa1add43ccec1e/k8s-resources-namespace?var-datasource=$datasource&var-namespace=$__cell", + "pattern": "namespace", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "/.*/", + "thresholds": [ + + ], + "type": "string", + "unit": "short" } ], - "span": 3, - "sparkline": { - "fillColor": "rgba(31, 118, 189, 0.18)", - "full": false, - "lineColor": "rgb(31, 120, 193)", - "show": false - }, "targets": [ { - "expr": "count(increase(kube_pod_container_status_restarts[1h]) > 5)", - "format": "time_series", + "expr": "sum(namespace_pod_name_container_name:container_cpu_usage_seconds_total:sum_rate) by (namespace)", + "format": "table", + "instant": true, "intervalFactor": 2, "legendFormat": "", "refId": "A", - "step": 600 + "step": 10 + }, + { + "expr": "sum(kube_pod_container_resource_requests_cpu_cores) by (namespace)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + }, + { + "expr": "sum(namespace_pod_name_container_name:container_cpu_usage_seconds_total:sum_rate) by (namespace) / sum(kube_pod_container_resource_requests_cpu_cores) by (namespace)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "C", + "step": 10 + }, + { + "expr": "sum(kube_pod_container_resource_limits_cpu_cores) by (namespace)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "D", + "step": 10 + }, + { + "expr": "sum(namespace_pod_name_container_name:container_cpu_usage_seconds_total:sum_rate) by (namespace) / sum(kube_pod_container_resource_limits_cpu_cores) by (namespace)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "E", + "step": 10 } ], - "thresholds": "1, 3", - "title": "Crashlooping Pods", - "transparent": false, - "type": "singlestat", - "valueFontSize": "80%", - "valueMaps": [ + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU Quota", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "transform": "table", + "type": "table", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ { - "op": "=", - "text": "0", - "value": "null" + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false } - ], - "valueName": "current" + ] } ], - "showTitle": false, - "title": "Row", + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "CPU Quota", "titleSize": "h6" }, { "collapse": false, - "editable": false, "height": "250px", "panels": [ { - "colorBackground": false, - "colorValue": true, - "colors": [ - "rgba(50, 172, 45, 0.97)", - "rgba(237, 129, 40, 0.89)", - "rgba(245, 54, 54, 0.9)" - ], - "datasource": "prometheus", - "editable": false, - "format": "none", - "gauge": { - "maxValue": 100, - "minValue": 0, - "show": false, - "thresholdLabels": false, - "thresholdMarkers": true + "aliasColors": { + }, - "hideTimeOverride": false, - "id": 5, - "links": [], - "mappingType": 1, - "mappingTypes": [ - { - "name": "value to text", - "value": 1 - }, - { - "name": "range to text", - "value": 2 - } + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 9, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + ], - "maxDataPoints": 100, - "nullPointMode": "connected", - "postfix": "", - "postfixFontSize": "50%", - "prefix": "", - "prefixFontSize": "50%", - "rangeMaps": [ - { - "from": "null", - "text": "N/A", - "to": "null" - } + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + ], - "span": 3, - "sparkline": { - "fillColor": "rgba(31, 118, 189, 0.18)", - "full": false, - "lineColor": "rgb(31, 120, 193)", - "show": false - }, + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, "targets": [ { - "expr": "sum(kube_node_status_condition{condition=\"Ready\",status!=\"true\"})", + "expr": "sum(container_memory_rss{container_name!=\"\"}) by (namespace)", "format": "time_series", "intervalFactor": 2, - "legendFormat": "", - "refId": "A", - "step": 600 + "legendFormat": "{{namespace}}", + "legendLink": null, + "step": 10 } ], - "thresholds": "1, 3", - "title": "Node Not Ready", - "transparent": false, - "type": "singlestat", - "valueFontSize": "80%", - "valueMaps": [ + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Memory Usage (w/o cache)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ { - "op": "=", - "text": "N/A", - "value": "null" + "format": "decbytes", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false } - ], - "valueName": "current" - }, + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Memory", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ { - "colorBackground": false, - "colorValue": true, - "colors": [ - "rgba(50, 172, 45, 0.97)", - "rgba(237, 129, 40, 0.89)", - "rgba(245, 54, 54, 0.9)" - ], - "datasource": "prometheus", - "editable": false, - "format": "none", - "gauge": { - "maxValue": 100, - "minValue": 0, - "show": false, - "thresholdLabels": false, - "thresholdMarkers": true + "aliasColors": { + }, - "hideTimeOverride": false, - "id": 6, - "links": [], - "mappingType": 1, - "mappingTypes": [ + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 10, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "styles": [ { - "name": "value to text", - "value": 1 + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" }, { - "name": "range to text", - "value": 2 - } - ], - "maxDataPoints": 100, - "nullPointMode": "connected", - "postfix": "", - "postfixFontSize": "50%", - "prefix": "", - "prefixFontSize": "50%", - "rangeMaps": [ + "alias": "Memory Usage", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #A", + "thresholds": [ + + ], + "type": "number", + "unit": "decbytes" + }, { - "from": "null", - "text": "N/A", - "to": "null" - } - ], - "span": 3, - "sparkline": { - "fillColor": "rgba(31, 118, 189, 0.18)", - "full": false, - "lineColor": "rgb(31, 120, 193)", - "show": false - }, - "targets": [ + "alias": "Memory Requests", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #B", + "thresholds": [ + + ], + "type": "number", + "unit": "decbytes" + }, { - "expr": "sum(kube_node_status_condition{condition=\"DiskPressure\",status=\"true\"})", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "", - "refId": "A", - "step": 600 - } - ], - "thresholds": "1, 3", - "title": "Node Disk Pressure", - "transparent": false, - "type": "singlestat", - "valueFontSize": "80%", - "valueMaps": [ + "alias": "Memory Requests %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #C", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, { - "op": "=", - "text": "N/A", - "value": "null" - } - ], - "valueName": "current" - }, - { - "colorBackground": false, - "colorValue": true, - "colors": [ - "rgba(50, 172, 45, 0.97)", - "rgba(237, 129, 40, 0.89)", - "rgba(245, 54, 54, 0.9)" - ], - "datasource": "prometheus", - "editable": false, - "format": "none", - "gauge": { - "maxValue": 100, - "minValue": 0, - "show": false, - "thresholdLabels": false, - "thresholdMarkers": true - }, - "hideTimeOverride": false, - "id": 7, - "links": [], - "mappingType": 1, - "mappingTypes": [ + "alias": "Memory Limits", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #D", + "thresholds": [ + + ], + "type": "number", + "unit": "decbytes" + }, { - "name": "value to text", - "value": 1 + "alias": "Memory Limits %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #E", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" }, { - "name": "range to text", - "value": 2 - } - ], - "maxDataPoints": 100, - "nullPointMode": "connected", - "postfix": "", - "postfixFontSize": "50%", - "prefix": "", - "prefixFontSize": "50%", - "rangeMaps": [ + "alias": "Namespace", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": true, + "linkTooltip": "Drill down", + "linkUrl": "/d/85a562078cdf77779eaa1add43ccec1e/k8s-resources-namespace?var-datasource=$datasource&var-namespace=$__cell", + "pattern": "namespace", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, { - "from": "null", - "text": "N/A", - "to": "null" + "alias": "", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "/.*/", + "thresholds": [ + + ], + "type": "string", + "unit": "short" } ], - "span": 3, - "sparkline": { - "fillColor": "rgba(31, 118, 189, 0.18)", - "full": false, - "lineColor": "rgb(31, 120, 193)", - "show": false - }, "targets": [ { - "expr": "sum(kube_node_status_condition{condition=\"MemoryPressure\",status=\"true\"})", - "format": "time_series", + "expr": "sum(container_memory_rss{container_name!=\"\"}) by (namespace)", + "format": "table", + "instant": true, "intervalFactor": 2, "legendFormat": "", "refId": "A", - "step": 600 - } - ], - "thresholds": "1, 3", - "title": "Node Memory Pressure", - "transparent": false, - "type": "singlestat", - "valueFontSize": "80%", - "valueMaps": [ - { - "op": "=", - "text": "N/A", - "value": "null" - } - ], - "valueName": "current" - }, - { - "colorBackground": false, - "colorValue": true, - "colors": [ - "rgba(50, 172, 45, 0.97)", - "rgba(237, 129, 40, 0.89)", - "rgba(245, 54, 54, 0.9)" - ], - "datasource": "prometheus", - "editable": false, - "format": "none", - "gauge": { - "maxValue": 100, - "minValue": 0, - "show": false, - "thresholdLabels": false, - "thresholdMarkers": true - }, - "hideTimeOverride": false, - "id": 8, - "links": [], - "mappingType": 1, - "mappingTypes": [ + "step": 10 + }, { - "name": "value to text", - "value": 1 + "expr": "sum(kube_pod_container_resource_requests_memory_bytes) by (namespace)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 }, { - "name": "range to text", - "value": 2 - } - ], - "maxDataPoints": 100, - "nullPointMode": "connected", - "postfix": "", - "postfixFontSize": "50%", - "prefix": "", - "prefixFontSize": "50%", - "rangeMaps": [ + "expr": "sum(container_memory_rss{container_name!=\"\"}) by (namespace) / sum(kube_pod_container_resource_requests_memory_bytes) by (namespace)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "C", + "step": 10 + }, { - "from": "null", - "text": "N/A", - "to": "null" - } - ], - "span": 3, - "sparkline": { - "fillColor": "rgba(31, 118, 189, 0.18)", - "full": false, - "lineColor": "rgb(31, 120, 193)", - "show": false - }, - "targets": [ + "expr": "sum(kube_pod_container_resource_limits_memory_bytes) by (namespace)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "D", + "step": 10 + }, { - "expr": "sum(kube_node_spec_unschedulable)", - "format": "time_series", + "expr": "sum(container_memory_rss{container_name!=\"\"}) by (namespace) / sum(kube_pod_container_resource_limits_memory_bytes) by (namespace)", + "format": "table", + "instant": true, "intervalFactor": 2, "legendFormat": "", - "refId": "A", - "step": 600 + "refId": "E", + "step": 10 } ], - "thresholds": "1, 3", - "title": "Nodes Unschedulable", - "transparent": false, - "type": "singlestat", - "valueFontSize": "80%", - "valueMaps": [ + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Requests by Namespace", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "transform": "table", + "type": "table", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ { - "op": "=", - "text": "N/A", - "value": "null" + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false } - ], - "valueName": "current" + ] } ], - "showTitle": false, - "title": "Row", + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Memory Requests", "titleSize": "h6" } ], "schemaVersion": 14, - "sharedCrosshair": false, "style": "dark", - "tags": [], + "tags": [ + "kubernetes-mixin" + ], "templating": { - "list": [] + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": null, + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + } + ] }, "time": { - "from": "now-6h", + "from": "now-1h", "to": "now" }, "timepicker": { @@ -3560,790 +4392,812 @@ data: "30d" ] }, - "timezone": "browser", - "title": "Kubernetes Cluster Health", - "version": 9 + "timezone": "", + "title": "Kubernetes / Compute Resources / Cluster", + "uid": "efa86fd1d0c121a26444b636a3f509a8", + "version": 0 } - kubernetes-cluster-status-dashboard.json: |+ + k8s-resources-namespace.json: |- { - "__inputs": [ - { - "description": "", - "label": "prometheus", - "name": "prometheus", - "pluginId": "prometheus", - "pluginName": "Prometheus", - "type": "datasource" - } - ], "annotations": { - "list": [] + "list": [ + + ] }, - "editable": false, + "editable": true, + "gnetId": null, "graphTooltip": 0, "hideControls": false, - "links": [], + "links": [ + + ], + "refresh": "10s", "rows": [ { "collapse": false, - "editable": false, - "height": "129px", + "height": "250px", "panels": [ { - "colorBackground": false, - "colorValue": true, - "colors": [ - "rgba(50, 172, 45, 0.97)", - "rgba(237, 129, 40, 0.89)", - "rgba(245, 54, 54, 0.9)" - ], - "datasource": "prometheus", - "editable": false, - "format": "none", - "gauge": { - "maxValue": 100, - "minValue": 0, - "show": false, - "thresholdLabels": false, - "thresholdMarkers": true + "aliasColors": { + }, - "id": 5, - "links": [], - "mappingType": 1, - "mappingTypes": [ - { - "name": "value to text", - "value": 1 - }, - { - "name": "range to text", - "value": 2 - } + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 1, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + ], - "maxDataPoints": 100, - "nullPointMode": "connected", - "postfixFontSize": "50%", - "prefix": "", - "prefixFontSize": "50%", - "rangeMaps": [ - { - "from": "null", - "text": "N/A", - "to": "null" - } + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + ], - "span": 6, - "sparkline": { - "fillColor": "rgba(31, 118, 189, 0.18)", - "full": false, - "lineColor": "rgb(31, 120, 193)", - "show": false - }, + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, "targets": [ { - "expr": "sum(up{job=~\"apiserver|kube-scheduler|kube-controller-manager\"} == 0)", + "expr": "sum(namespace_pod_name_container_name:container_cpu_usage_seconds_total:sum_rate{namespace=\"$namespace\"}) by (pod_name)", "format": "time_series", "intervalFactor": 2, - "refId": "A", - "step": 600 - } - ], - "thresholds": "1, 3", - "title": "Control Plane UP", - "type": "singlestat", - "valueFontSize": "80%", - "valueMaps": [ - { - "op": "=", - "text": "UP", - "value": "null" + "legendFormat": "{{pod_name}}", + "legendLink": null, + "step": 10 } ], - "valueName": "total" - }, - { - "colorBackground": false, - "colorValue": true, - "colors": [ - "rgba(50, 172, 45, 0.97)", - "rgba(237, 129, 40, 0.89)", - "rgba(245, 54, 54, 0.9)" + "thresholds": [ + ], - "datasource": "prometheus", - "editable": false, - "format": "none", - "gauge": { - "maxValue": 100, - "minValue": 0, - "show": false, - "thresholdLabels": false, - "thresholdMarkers": true + "timeFrom": null, + "timeShift": null, + "title": "CPU Usage", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" }, - "id": 6, - "links": [], - "mappingType": 1, - "mappingTypes": [ - { - "name": "value to text", - "value": 1 - }, - { - "name": "range to text", - "value": 2 - } - ], - "maxDataPoints": 100, - "nullPointMode": "connected", - "postfixFontSize": "50%", - "prefix": "", - "prefixFontSize": "50%", - "rangeMaps": [ - { - "from": "null", - "text": "N/A", - "to": "null" - } - ], - "span": 6, - "sparkline": { - "fillColor": "rgba(31, 118, 189, 0.18)", - "full": false, - "lineColor": "rgb(31, 120, 193)", - "show": false + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] }, - "targets": [ + "yaxes": [ { - "expr": "sum(ALERTS{alertstate=\"firing\",alertname!=\"DeadMansSwitch\"})", - "format": "time_series", - "intervalFactor": 2, - "refId": "A", - "step": 600 - } - ], - "thresholds": "3, 5", - "title": "Alerts Firing", - "type": "singlestat", - "valueFontSize": "80%", - "valueMaps": [ + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, { - "op": "=", - "text": "0", - "value": "null" + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false } - ], - "valueName": "current" + ] } ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, "showTitle": true, - "title": "Cluster Health", + "title": "CPU Usage", "titleSize": "h6" }, { "collapse": false, - "editable": false, - "height": "168px", + "height": "250px", "panels": [ { - "colorBackground": false, - "colorValue": false, - "colors": [ - "rgba(245, 54, 54, 0.9)", - "rgba(237, 129, 40, 0.89)", - "rgba(50, 172, 45, 0.97)" - ], - "datasource": "prometheus", - "editable": false, - "format": "percent", - "gauge": { - "maxValue": 100, - "minValue": 0, + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 2, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, "show": true, - "thresholdLabels": false, - "thresholdMarkers": true + "total": false, + "values": false }, - "id": 1, - "links": [], - "mappingType": 1, - "mappingTypes": [ + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "styles": [ { - "name": "value to text", - "value": 1 + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" }, { - "name": "range to text", - "value": 2 - } - ], - "maxDataPoints": 100, - "nullPointMode": "connected", - "postfixFontSize": "50%", - "prefix": "", - "prefixFontSize": "50%", - "rangeMaps": [ + "alias": "CPU Usage", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #A", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, { - "from": "null", - "text": "N/A", - "to": "null" - } - ], - "span": 3, - "sparkline": { - "fillColor": "rgba(31, 118, 189, 0.18)", - "full": false, - "lineColor": "rgb(31, 120, 193)", - "show": false - }, - "targets": [ + "alias": "CPU Requests", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #B", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, { - "expr": "(sum(up{job=\"apiserver\"} == 1) / count(up{job=\"apiserver\"})) * 100", - "format": "time_series", - "intervalFactor": 2, - "refId": "A", - "step": 600 - } - ], - "thresholds": "50, 80", - "title": "API Servers UP", - "type": "singlestat", - "valueFontSize": "80%", - "valueMaps": [ + "alias": "CPU Requests %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #C", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, { - "op": "=", - "text": "N/A", - "value": "null" - } - ], - "valueName": "current" - }, - { - "colorBackground": false, - "colorValue": false, - "colors": [ - "rgba(245, 54, 54, 0.9)", - "rgba(237, 129, 40, 0.89)", - "rgba(50, 172, 45, 0.97)" - ], - "datasource": "prometheus", - "editable": false, - "format": "percent", - "gauge": { - "maxValue": 100, - "minValue": 0, - "show": true, - "thresholdLabels": false, - "thresholdMarkers": true - }, - "id": 2, - "links": [], - "mappingType": 1, - "mappingTypes": [ + "alias": "CPU Limits", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #D", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, { - "name": "value to text", - "value": 1 + "alias": "CPU Limits %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #E", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" }, { - "name": "range to text", - "value": 2 - } - ], - "maxDataPoints": 100, - "nullPointMode": "connected", - "postfixFontSize": "50%", - "prefix": "", - "prefixFontSize": "50%", - "rangeMaps": [ + "alias": "Pod", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": true, + "linkTooltip": "Drill down", + "linkUrl": "/d/6581e46e4e5c7ba40a07646395ef7b23/k8s-resources-pod?var-datasource=$datasource&var-namespace=$namespace&var-pod=$__cell", + "pattern": "pod", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, { - "from": "null", - "text": "N/A", - "to": "null" + "alias": "", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "/.*/", + "thresholds": [ + + ], + "type": "string", + "unit": "short" } ], - "span": 3, - "sparkline": { - "fillColor": "rgba(31, 118, 189, 0.18)", - "full": false, - "lineColor": "rgb(31, 120, 193)", - "show": false - }, "targets": [ { - "expr": "(sum(up{job=\"kube-controller-manager\"} == 1) / count(up{job=\"kube-controller-manager\"})) * 100", - "format": "time_series", + "expr": "sum(label_replace(namespace_pod_name_container_name:container_cpu_usage_seconds_total:sum_rate{namespace=\"$namespace\"}, \"pod\", \"$1\", \"pod_name\", \"(.*)\")) by (pod)", + "format": "table", + "instant": true, "intervalFactor": 2, + "legendFormat": "", "refId": "A", - "step": 600 - } - ], - "thresholds": "50, 80", - "title": "Controller Managers UP", - "type": "singlestat", - "valueFontSize": "80%", - "valueMaps": [ - { - "op": "=", - "text": "N/A", - "value": "null" - } - ], - "valueName": "current" - }, - { - "colorBackground": false, - "colorValue": false, - "colors": [ - "rgba(245, 54, 54, 0.9)", - "rgba(237, 129, 40, 0.89)", - "rgba(50, 172, 45, 0.97)" - ], - "datasource": "prometheus", - "editable": false, - "format": "percent", - "gauge": { - "maxValue": 100, - "minValue": 0, - "show": true, - "thresholdLabels": false, - "thresholdMarkers": true - }, - "id": 3, - "links": [], - "mappingType": 1, - "mappingTypes": [ - { - "name": "value to text", - "value": 1 + "step": 10 }, { - "name": "range to text", - "value": 2 - } - ], - "maxDataPoints": 100, - "nullPointMode": "connected", - "postfixFontSize": "50%", - "prefix": "", - "prefixFontSize": "50%", - "rangeMaps": [ + "expr": "sum(kube_pod_container_resource_requests_cpu_cores{namespace=\"$namespace\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + }, { - "from": "null", - "text": "N/A", - "to": "null" - } - ], - "span": 3, - "sparkline": { - "fillColor": "rgba(31, 118, 189, 0.18)", - "full": false, - "lineColor": "rgb(31, 120, 193)", - "show": false - }, - "targets": [ + "expr": "sum(label_replace(namespace_pod_name_container_name:container_cpu_usage_seconds_total:sum_rate{namespace=\"$namespace\"}, \"pod\", \"$1\", \"pod_name\", \"(.*)\")) by (pod) / sum(kube_pod_container_resource_requests_cpu_cores{namespace=\"$namespace\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "C", + "step": 10 + }, { - "expr": "(sum(up{job=\"kube-scheduler\"} == 1) / count(up{job=\"kube-scheduler\"})) * 100", - "format": "time_series", + "expr": "sum(kube_pod_container_resource_limits_cpu_cores{namespace=\"$namespace\"}) by (pod)", + "format": "table", + "instant": true, "intervalFactor": 2, - "refId": "A", - "step": 600 - } - ], - "thresholds": "50, 80", - "title": "Schedulers UP", - "type": "singlestat", - "valueFontSize": "80%", - "valueMaps": [ + "legendFormat": "", + "refId": "D", + "step": 10 + }, { - "op": "=", - "text": "N/A", - "value": "null" + "expr": "sum(label_replace(namespace_pod_name_container_name:container_cpu_usage_seconds_total:sum_rate{namespace=\"$namespace\"}, \"pod\", \"$1\", \"pod_name\", \"(.*)\")) by (pod) / sum(kube_pod_container_resource_limits_cpu_cores{namespace=\"$namespace\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "E", + "step": 10 } ], - "valueName": "current" - }, - { - "colorBackground": false, - "colorValue": true, - "colors": [ - "rgba(50, 172, 45, 0.97)", - "rgba(237, 129, 40, 0.89)", - "rgba(245, 54, 54, 0.9)" + "thresholds": [ + ], - "datasource": "prometheus", - "editable": false, - "format": "none", - "gauge": { - "maxValue": 100, - "minValue": 0, - "show": false, - "thresholdLabels": false, - "thresholdMarkers": true + "timeFrom": null, + "timeShift": null, + "title": "CPU Quota", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" }, - "id": 4, - "links": [], - "mappingType": 1, - "mappingTypes": [ + "transform": "table", + "type": "table", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ { - "name": "value to text", - "value": 1 + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true }, { - "name": "range to text", - "value": 2 + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "CPU Quota", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 3, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + ], - "maxDataPoints": 100, - "nullPointMode": "connected", - "postfixFontSize": "50%", - "prefix": "", - "prefixFontSize": "50%", - "rangeMaps": [ - { - "from": "null", - "text": "N/A", - "to": "null" - } + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + ], - "span": 3, - "sparkline": { - "fillColor": "rgba(31, 118, 189, 0.18)", - "full": false, - "lineColor": "rgb(31, 120, 193)", - "show": false - }, + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, "targets": [ { - "expr": "count(increase(kube_pod_container_status_restarts{namespace=~\"kube-system|tectonic-system\"}[1h]) > 5)", + "expr": "sum(container_memory_usage_bytes{namespace=\"$namespace\", container_name!=\"\"}) by (pod_name)", "format": "time_series", "intervalFactor": 2, - "refId": "A", - "step": 600 + "legendFormat": "{{pod_name}}", + "legendLink": null, + "step": 10 } ], - "thresholds": "1, 3", - "title": "Crashlooping Control Plane Pods", - "type": "singlestat", - "valueFontSize": "80%", - "valueMaps": [ + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Memory Usage", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ { - "op": "=", - "text": "0", - "value": "null" + "format": "decbytes", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false } - ], - "valueName": "current" + ] } ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, "showTitle": true, - "title": "Control Plane Status", + "title": "Memory Usage", "titleSize": "h6" }, { "collapse": false, - "editable": false, - "height": "158px", + "height": "250px", "panels": [ { - "colorBackground": false, - "colorValue": false, - "colors": [ - "rgba(50, 172, 45, 0.97)", - "rgba(237, 129, 40, 0.89)", - "rgba(245, 54, 54, 0.9)" - ], - "datasource": "prometheus", - "editable": false, - "format": "percent", - "gauge": { - "maxValue": 100, - "minValue": 0, - "show": true, - "thresholdLabels": false, - "thresholdMarkers": true + "aliasColors": { + }, - "id": 8, - "links": [], - "mappingType": 1, - "mappingTypes": [ - { - "name": "value to text", - "value": 1 - }, - { - "name": "range to text", - "value": 2 - } - ], - "maxDataPoints": 100, - "nullPointMode": "connected", - "postfixFontSize": "50%", - "prefix": "", - "prefixFontSize": "50%", - "rangeMaps": [ - { - "from": "null", - "text": "N/A", - "to": "null" - } - ], - "span": 3, - "sparkline": { - "fillColor": "rgba(31, 118, 189, 0.18)", - "full": false, - "lineColor": "rgb(31, 120, 193)", - "show": false + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 4, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false }, - "targets": [ - { - "expr": "sum(100 - (avg by (instance) (rate(node_cpu_seconds_total{job=\"node-exporter\",mode=\"idle\"}[5m])) * 100)) / count(node_cpu_seconds_total{job=\"node-exporter\",mode=\"idle\"})", - "format": "time_series", - "intervalFactor": 2, - "refId": "A", - "step": 600 - } - ], - "thresholds": "80, 90", - "title": "CPU Utilization", - "type": "singlestat", - "valueFontSize": "80%", - "valueMaps": [ - { - "op": "=", - "text": "N/A", - "value": "null" - } + "lines": true, + "linewidth": 1, + "links": [ + ], - "valueName": "avg" - }, - { - "colorBackground": false, - "colorValue": false, - "colors": [ - "rgba(50, 172, 45, 0.97)", - "rgba(237, 129, 40, 0.89)", - "rgba(245, 54, 54, 0.9)" + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + ], - "datasource": "prometheus", - "editable": false, - "format": "percent", - "gauge": { - "maxValue": 100, - "minValue": 0, - "show": true, - "thresholdLabels": false, - "thresholdMarkers": true - }, - "id": 7, - "links": [], - "mappingType": 1, - "mappingTypes": [ + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "styles": [ { - "name": "value to text", - "value": 1 + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" }, { - "name": "range to text", - "value": 2 - } - ], - "maxDataPoints": 100, - "nullPointMode": "connected", - "postfixFontSize": "50%", - "prefix": "", - "prefixFontSize": "50%", - "rangeMaps": [ + "alias": "Memory Usage", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #A", + "thresholds": [ + + ], + "type": "number", + "unit": "decbytes" + }, { - "from": "null", - "text": "N/A", - "to": "null" - } - ], - "span": 3, - "sparkline": { - "fillColor": "rgba(31, 118, 189, 0.18)", - "full": false, - "lineColor": "rgb(31, 120, 193)", - "show": false - }, - "targets": [ + "alias": "Memory Requests", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #B", + "thresholds": [ + + ], + "type": "number", + "unit": "decbytes" + }, { - "expr": "((sum(node_memory_MemTotal_bytes) - sum(node_memory_MemFree_bytes) - sum(node_memory_Buffers_bytes) - sum(node_memory_Cached_bytes)) / sum(node_memory_MemTotal_bytes)) * 100", - "format": "time_series", - "intervalFactor": 2, - "refId": "A", - "step": 600 - } - ], - "thresholds": "80, 90", - "title": "Memory Utilization", - "type": "singlestat", - "valueFontSize": "80%", - "valueMaps": [ + "alias": "Memory Requests %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #C", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, { - "op": "=", - "text": "N/A", - "value": "null" - } - ], - "valueName": "avg" - }, - { - "colorBackground": false, - "colorValue": false, - "colors": [ - "rgba(50, 172, 45, 0.97)", - "rgba(237, 129, 40, 0.89)", - "rgba(245, 54, 54, 0.9)" - ], - "datasource": "prometheus", - "editable": false, - "format": "percent", - "gauge": { - "maxValue": 100, - "minValue": 0, - "show": true, - "thresholdLabels": false, - "thresholdMarkers": true - }, - "id": 9, - "links": [], - "mappingType": 1, - "mappingTypes": [ + "alias": "Memory Limits", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #D", + "thresholds": [ + + ], + "type": "number", + "unit": "decbytes" + }, { - "name": "value to text", - "value": 1 + "alias": "Memory Limits %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #E", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" }, { - "name": "range to text", - "value": 2 - } - ], - "maxDataPoints": 100, - "nullPointMode": "connected", - "postfixFontSize": "50%", - "prefix": "", - "prefixFontSize": "50%", - "rangeMaps": [ + "alias": "Pod", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": true, + "linkTooltip": "Drill down", + "linkUrl": "/d/6581e46e4e5c7ba40a07646395ef7b23/k8s-resources-pod?var-datasource=$datasource&var-namespace=$namespace&var-pod=$__cell", + "pattern": "pod", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, { - "from": "null", - "text": "N/A", - "to": "null" + "alias": "", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "/.*/", + "thresholds": [ + + ], + "type": "string", + "unit": "short" } ], - "span": 3, - "sparkline": { - "fillColor": "rgba(31, 118, 189, 0.18)", - "full": false, - "lineColor": "rgb(31, 120, 193)", - "show": false - }, "targets": [ { - "expr": "(sum(node_filesystem_size_bytes{device!=\"rootfs\"}) - sum(node_filesystem_free_bytes{device!=\"rootfs\"})) / sum(node_filesystem_size_bytes{device!=\"rootfs\"})", - "format": "time_series", + "expr": "sum(label_replace(container_memory_usage_bytes{namespace=\"$namespace\",container_name!=\"\"}, \"pod\", \"$1\", \"pod_name\", \"(.*)\")) by (pod)", + "format": "table", + "instant": true, "intervalFactor": 2, + "legendFormat": "", "refId": "A", - "step": 600 - } - ], - "thresholds": "80, 90", - "title": "Filesystem Utilization", - "type": "singlestat", - "valueFontSize": "80%", - "valueMaps": [ + "step": 10 + }, { - "op": "=", - "text": "N/A", - "value": "null" - } - ], - "valueName": "avg" - }, - { - "colorBackground": false, - "colorValue": false, - "colors": [ - "rgba(50, 172, 45, 0.97)", - "rgba(237, 129, 40, 0.89)", - "rgba(245, 54, 54, 0.9)" - ], - "datasource": "prometheus", - "editable": false, - "format": "percent", - "gauge": { - "maxValue": 100, - "minValue": 0, - "show": true, - "thresholdLabels": false, - "thresholdMarkers": true - }, - "id": 10, - "links": [], - "mappingType": 1, - "mappingTypes": [ + "expr": "sum(kube_pod_container_resource_requests_memory_bytes{namespace=\"$namespace\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + }, { - "name": "value to text", - "value": 1 + "expr": "sum(label_replace(container_memory_usage_bytes{namespace=\"$namespace\",container_name!=\"\"}, \"pod\", \"$1\", \"pod_name\", \"(.*)\")) by (pod) / sum(kube_pod_container_resource_requests_memory_bytes{namespace=\"$namespace\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "C", + "step": 10 }, { - "name": "range to text", - "value": 2 - } - ], - "maxDataPoints": 100, - "nullPointMode": "connected", - "postfixFontSize": "50%", - "prefix": "", - "prefixFontSize": "50%", - "rangeMaps": [ + "expr": "sum(kube_pod_container_resource_limits_memory_bytes{namespace=\"$namespace\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "D", + "step": 10 + }, { - "from": "null", - "text": "N/A", - "to": "null" + "expr": "sum(label_replace(container_memory_usage_bytes{namespace=\"$namespace\",container_name!=\"\"}, \"pod\", \"$1\", \"pod_name\", \"(.*)\")) by (pod) / sum(kube_pod_container_resource_limits_memory_bytes{namespace=\"$namespace\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "E", + "step": 10 } ], - "span": 3, - "sparkline": { - "fillColor": "rgba(31, 118, 189, 0.18)", - "full": false, - "lineColor": "rgb(31, 120, 193)", - "show": false + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Memory Quota", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" }, - "targets": [ + "transform": "table", + "type": "table", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ { - "expr": "100 - (sum(kube_node_status_capacity_pods) - sum(kube_pod_info)) / sum(kube_node_status_capacity_pods) * 100", - "format": "time_series", - "intervalFactor": 2, - "refId": "A", - "step": 600 - } - ], - "thresholds": "80, 90", - "title": "Pod Utilization", - "type": "singlestat", - "valueFontSize": "80%", - "valueMaps": [ + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, { - "op": "=", - "text": "N/A", - "value": "null" + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false } - ], - "valueName": "avg" + ] } ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, "showTitle": true, - "title": "Capacity Planning", + "title": "Memory Quota", "titleSize": "h6" } ], "schemaVersion": 14, - "sharedCrosshair": false, "style": "dark", - "tags": [], + "tags": [ + "kubernetes-mixin" + ], "templating": { - "list": [] + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": null, + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + "text": "prod", + "value": "prod" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": "namespace", + "multi": false, + "name": "namespace", + "options": [ + + ], + "query": "label_values(kube_pod_info, namespace)", + "refresh": 1, + "regex": "", + "sort": 2, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] }, "time": { - "from": "now-6h", + "from": "now-1h", "to": "now" }, "timepicker": { @@ -4371,750 +5225,1200 @@ data: "30d" ] }, - "timezone": "browser", - "title": "Kubernetes Cluster Status", - "version": 3 + "timezone": "", + "title": "Kubernetes / Compute Resources / Namespace", + "uid": "85a562078cdf77779eaa1add43ccec1e", + "version": 0 } - kubernetes-control-plane-status-dashboard.json: |+ + k8s-resources-pod.json: |- { - "__inputs": [ - { - "description": "", - "label": "prometheus", - "name": "prometheus", - "pluginId": "prometheus", - "pluginName": "Prometheus", - "type": "datasource" - } - ], "annotations": { - "list": [] + "list": [ + + ] }, - "editable": false, + "editable": true, + "gnetId": null, "graphTooltip": 0, "hideControls": false, - "links": [], + "links": [ + + ], + "refresh": "10s", "rows": [ { "collapse": false, - "editable": false, "height": "250px", "panels": [ { - "colorBackground": false, - "colorValue": false, - "colors": [ - "rgba(245, 54, 54, 0.9)", - "rgba(237, 129, 40, 0.89)", - "rgba(50, 172, 45, 0.97)" - ], - "datasource": "prometheus", - "editable": false, - "format": "percent", - "gauge": { - "maxValue": 100, - "minValue": 0, - "show": true, - "thresholdLabels": false, - "thresholdMarkers": true + "aliasColors": { + }, - "hideTimeOverride": false, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, "id": 1, - "links": [], - "mappingType": 1, - "mappingTypes": [ - { - "name": "value to text", - "value": 1 - }, - { - "name": "range to text", - "value": 2 - } + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + ], - "maxDataPoints": 100, - "nullPointMode": "connected", - "postfix": "", - "postfixFontSize": "50%", - "prefix": "", - "prefixFontSize": "50%", - "rangeMaps": [ - { - "from": "null", - "text": "N/A", - "to": "null" - } + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + ], - "span": 3, - "sparkline": { - "fillColor": "rgba(31, 118, 189, 0.18)", - "full": false, - "lineColor": "rgb(31, 120, 193)", - "show": false - }, + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, "targets": [ { - "expr": "(sum(up{job=\"apiserver\"} == 1) / sum(up{job=\"apiserver\"})) * 100", + "expr": "sum(namespace_pod_name_container_name:container_cpu_usage_seconds_total:sum_rate{namespace=\"$namespace\", pod_name=\"$pod\", container_name!=\"POD\"}) by (container_name)", "format": "time_series", "intervalFactor": 2, - "refId": "A", - "step": 600 + "legendFormat": "{{container_name}}", + "legendLink": null, + "step": 10 } ], - "thresholds": "50, 80", - "title": "API Servers UP", - "transparent": false, - "type": "singlestat", - "valueFontSize": "80%", - "valueMaps": [ + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU Usage", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ { - "op": "=", - "text": "N/A", - "value": "null" + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false } - ], - "valueName": "avg" - }, + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "CPU Usage", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ { - "colorBackground": false, - "colorValue": false, - "colors": [ - "rgba(245, 54, 54, 0.9)", - "rgba(237, 129, 40, 0.89)", - "rgba(50, 172, 45, 0.97)" - ], - "datasource": "prometheus", - "editable": false, - "format": "percent", - "gauge": { - "maxValue": 100, - "minValue": 0, - "show": true, - "thresholdLabels": false, - "thresholdMarkers": true + "aliasColors": { + }, - "hideTimeOverride": false, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, "id": 2, - "links": [], - "mappingType": 1, - "mappingTypes": [ + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "styles": [ { - "name": "value to text", - "value": 1 + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" }, { - "name": "range to text", - "value": 2 - } - ], - "maxDataPoints": 100, - "nullPointMode": "connected", - "postfix": "", - "postfixFontSize": "50%", - "prefix": "", - "prefixFontSize": "50%", - "rangeMaps": [ + "alias": "CPU Usage", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #A", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, { - "from": "null", - "text": "N/A", - "to": "null" - } - ], - "span": 3, - "sparkline": { - "fillColor": "rgba(31, 118, 189, 0.18)", - "full": false, - "lineColor": "rgb(31, 120, 193)", - "show": false - }, - "targets": [ + "alias": "CPU Requests", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #B", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, { - "expr": "(sum(up{job=\"kube-controller-manager\"} == 1) / sum(up{job=\"kube-controller-manager\"})) * 100", - "format": "time_series", - "intervalFactor": 2, - "refId": "A", - "step": 600 - } - ], - "thresholds": "50, 80", - "title": "Controller Managers UP", - "transparent": false, - "type": "singlestat", - "valueFontSize": "80%", - "valueMaps": [ + "alias": "CPU Requests %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #C", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, { - "op": "=", - "text": "N/A", - "value": "null" - } - ], - "valueName": "avg" - }, - { - "colorBackground": false, - "colorValue": false, - "colors": [ - "rgba(245, 54, 54, 0.9)", - "rgba(237, 129, 40, 0.89)", - "rgba(50, 172, 45, 0.97)" - ], - "datasource": "prometheus", - "editable": false, - "format": "percent", - "gauge": { - "maxValue": 100, - "minValue": 0, - "show": true, - "thresholdLabels": false, - "thresholdMarkers": true - }, - "hideTimeOverride": false, - "id": 3, - "links": [], - "mappingType": 1, - "mappingTypes": [ + "alias": "CPU Limits", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #D", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, { - "name": "value to text", - "value": 1 + "alias": "CPU Limits %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #E", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" }, { - "name": "range to text", - "value": 2 - } - ], - "maxDataPoints": 100, - "nullPointMode": "connected", - "postfix": "", - "postfixFontSize": "50%", - "prefix": "", - "prefixFontSize": "50%", - "rangeMaps": [ + "alias": "Container", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "container", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, { - "from": "null", - "text": "N/A", - "to": "null" + "alias": "", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "/.*/", + "thresholds": [ + + ], + "type": "string", + "unit": "short" } ], - "span": 3, - "sparkline": { - "fillColor": "rgba(31, 118, 189, 0.18)", - "full": false, - "lineColor": "rgb(31, 120, 193)", - "show": false - }, "targets": [ { - "expr": "(sum(up{job=\"kube-scheduler\"} == 1) / sum(up{job=\"kube-scheduler\"})) * 100", - "format": "time_series", + "expr": "sum(label_replace(namespace_pod_name_container_name:container_cpu_usage_seconds_total:sum_rate{namespace=\"$namespace\", pod_name=\"$pod\", container_name!=\"POD\"}, \"container\", \"$1\", \"container_name\", \"(.*)\")) by (container)", + "format": "table", + "instant": true, "intervalFactor": 2, + "legendFormat": "", "refId": "A", - "step": 600 - } - ], - "thresholds": "50, 80", - "title": "Schedulers UP", - "transparent": false, - "type": "singlestat", - "valueFontSize": "80%", - "valueMaps": [ + "step": 10 + }, { - "op": "=", - "text": "N/A", - "value": "null" + "expr": "sum(kube_pod_container_resource_requests_cpu_cores{namespace=\"$namespace\", pod=\"$pod\"}) by (container)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + }, + { + "expr": "sum(label_replace(namespace_pod_name_container_name:container_cpu_usage_seconds_total:sum_rate{namespace=\"$namespace\", pod_name=\"$pod\"}, \"container\", \"$1\", \"container_name\", \"(.*)\")) by (container) / sum(kube_pod_container_resource_requests_cpu_cores{namespace=\"$namespace\", pod=\"$pod\"}) by (container)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "C", + "step": 10 + }, + { + "expr": "sum(kube_pod_container_resource_limits_cpu_cores{namespace=\"$namespace\", pod=\"$pod\"}) by (container)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "D", + "step": 10 + }, + { + "expr": "sum(label_replace(namespace_pod_name_container_name:container_cpu_usage_seconds_total:sum_rate{namespace=\"$namespace\", pod_name=\"$pod\"}, \"container\", \"$1\", \"container_name\", \"(.*)\")) by (container) / sum(kube_pod_container_resource_limits_cpu_cores{namespace=\"$namespace\", pod=\"$pod\"}) by (container)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "E", + "step": 10 } ], - "valueName": "avg" - }, - { - "colorBackground": false, - "colorValue": false, - "colors": [ - "rgba(50, 172, 45, 0.97)", - "rgba(237, 129, 40, 0.89)", - "rgba(245, 54, 54, 0.9)" + "thresholds": [ + ], - "datasource": "prometheus", - "editable": false, - "format": "percent", - "gauge": { - "maxValue": 100, - "minValue": 0, + "timeFrom": null, + "timeShift": null, + "title": "CPU Quota", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "transform": "table", + "type": "table", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, "show": true, - "thresholdLabels": false, - "thresholdMarkers": true + "values": [ + + ] }, - "hideTimeOverride": false, - "id": 4, - "links": [], - "mappingType": 1, - "mappingTypes": [ + "yaxes": [ { - "name": "value to text", - "value": 1 + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true }, { - "name": "range to text", - "value": 2 + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "CPU Quota", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 3, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + ], - "maxDataPoints": 100, - "nullPointMode": "connected", - "postfix": "", - "postfixFontSize": "50%", - "prefix": "", - "prefixFontSize": "50%", - "rangeMaps": [ - { - "from": "null", - "text": "N/A", - "to": "null" - } + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + ], - "span": 3, - "sparkline": { - "fillColor": "rgba(31, 118, 189, 0.18)", - "full": false, - "lineColor": "rgb(31, 120, 193)", - "show": false - }, + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, "targets": [ { - "expr": "max(sum by(instance) (rate(apiserver_request_count{code=~\"5..\"}[5m])) / sum by(instance) (rate(apiserver_request_count[5m]))) * 100", + "expr": "sum(container_memory_usage_bytes{namespace=\"$namespace\", pod_name=\"$pod\", container_name!=\"POD\", container_name!=\"\"}) by (container_name)", "format": "time_series", "intervalFactor": 2, - "legendFormat": "", - "refId": "A", - "step": 600 + "legendFormat": "{{container_name}}", + "legendLink": null, + "step": 10 } ], - "thresholds": "5, 10", - "title": "API Server Request Error Rate", - "transparent": false, - "type": "singlestat", - "valueFontSize": "80%", - "valueMaps": [ + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Memory Usage", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, { - "op": "=", - "text": "0", - "value": "null" + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false } - ], - "valueName": "avg" + ] } ], - "showTitle": false, - "title": "Dashboard Row", + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Memory Usage", "titleSize": "h6" }, { "collapse": false, - "editable": false, "height": "250px", "panels": [ { - "aliasColors": {}, + "aliasColors": { + + }, "bars": false, "dashLength": 10, "dashes": false, - "datasource": "prometheus", - "editable": false, - "error": false, + "datasource": "$datasource", "fill": 1, - "grid": { - "threshold1Color": "rgba(216, 200, 27, 0.27)", - "threshold2Color": "rgba(234, 112, 112, 0.22)" - }, - "id": 7, - "isNew": false, + "id": 4, "legend": { - "alignAsTable": false, "avg": false, "current": false, - "hideEmpty": false, - "hideZero": false, "max": false, "min": false, - "rightSide": false, "show": true, - "total": false + "total": false, + "values": false }, "lines": true, "linewidth": 1, - "links": [], - "nullPointMode": "null", + "links": [ + + ], + "nullPointMode": "null as zero", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", - "seriesOverrides": [], + "seriesOverrides": [ + + ], "spaceLength": 10, "span": 12, "stack": false, "steppedLine": false, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "Memory Usage", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #A", + "thresholds": [ + + ], + "type": "number", + "unit": "decbytes" + }, + { + "alias": "Memory Requests", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #B", + "thresholds": [ + + ], + "type": "number", + "unit": "decbytes" + }, + { + "alias": "Memory Requests %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #C", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "Memory Limits", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #D", + "thresholds": [ + + ], + "type": "number", + "unit": "decbytes" + }, + { + "alias": "Memory Limits %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #E", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "Container", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "container", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "/.*/", + "thresholds": [ + + ], + "type": "string", + "unit": "short" + } + ], "targets": [ { - "expr": "sum by(verb) (rate(apiserver_latency_seconds:quantile[5m]) >= 0)", - "format": "time_series", + "expr": "sum(label_replace(container_memory_usage_bytes{namespace=\"$namespace\", pod_name=\"$pod\", container_name!=\"POD\", container_name!=\"\"}, \"container\", \"$1\", \"container_name\", \"(.*)\")) by (container)", + "format": "table", + "instant": true, "intervalFactor": 2, "legendFormat": "", "refId": "A", - "step": 30 + "step": 10 + }, + { + "expr": "sum(kube_pod_container_resource_requests_memory_bytes{namespace=\"$namespace\", pod=\"$pod\"}) by (container)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + }, + { + "expr": "sum(label_replace(container_memory_usage_bytes{namespace=\"$namespace\", pod_name=\"$pod\"}, \"container\", \"$1\", \"container_name\", \"(.*)\")) by (container) / sum(kube_pod_container_resource_requests_memory_bytes{namespace=\"$namespace\", pod=\"$pod\"}) by (container)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "C", + "step": 10 + }, + { + "expr": "sum(kube_pod_container_resource_limits_memory_bytes{namespace=\"$namespace\", pod=\"$pod\", container!=\"\"}) by (container)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "D", + "step": 10 + }, + { + "expr": "sum(label_replace(container_memory_usage_bytes{namespace=\"$namespace\", pod_name=\"$pod\", container_name!=\"\"}, \"container\", \"$1\", \"container_name\", \"(.*)\")) by (container) / sum(kube_pod_container_resource_limits_memory_bytes{namespace=\"$namespace\", pod=\"$pod\"}) by (container)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "E", + "step": 10 } ], - "title": "API Server Request Latency", + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Memory Quota", "tooltip": { - "msResolution": false, "shared": true, "sort": 0, "value_type": "individual" }, - "type": "graph", + "transform": "table", + "type": "table", "xaxis": { + "buckets": null, "mode": "time", + "name": null, "show": true, - "values": [] + "values": [ + + ] }, "yaxes": [ { "format": "short", + "label": null, "logBase": 1, + "max": null, + "min": 0, "show": true }, { "format": "short", + "label": null, "logBase": 1, - "show": true + "max": null, + "min": null, + "show": false } ] } ], - "showTitle": false, - "title": "Dashboard Row", + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Memory Quota", "titleSize": "h6" - }, + } + ], + "schemaVersion": 14, + "style": "dark", + "tags": [ + "kubernetes-mixin" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": null, + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + "text": "prod", + "value": "prod" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": "namespace", + "multi": false, + "name": "namespace", + "options": [ + + ], + "query": "label_values(kube_pod_info, namespace)", + "refresh": 1, + "regex": "", + "sort": 2, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + "text": "prod", + "value": "prod" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": "pod", + "multi": false, + "name": "pod", + "options": [ + + ], + "query": "label_values(kube_pod_info{namespace=\"$namespace\"}, pod)", + "refresh": 1, + "regex": "", + "sort": 2, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "", + "title": "Kubernetes / Compute Resources / Pod", + "uid": "6581e46e4e5c7ba40a07646395ef7b23", + "version": 0 + } + nodes.json: |- + { + "__inputs": [ + + ], + "__requires": [ + + ], + "annotations": { + "list": [ + + ] + }, + "editable": false, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "id": null, + "links": [ + + ], + "refresh": "", + "rows": [ { "collapse": false, - "editable": false, - "height": "250px", + "collapsed": false, "panels": [ { - "aliasColors": {}, + "aliasColors": { + + }, "bars": false, "dashLength": 10, "dashes": false, - "datasource": "prometheus", - "editable": false, - "error": false, + "datasource": "$datasource", "fill": 1, - "grid": { - "threshold1Color": "rgba(216, 200, 27, 0.27)", - "threshold2Color": "rgba(234, 112, 112, 0.22)" + "gridPos": { + }, - "id": 5, - "isNew": false, + "id": 2, "legend": { "alignAsTable": false, "avg": false, "current": false, - "hideEmpty": false, - "hideZero": false, "max": false, "min": false, "rightSide": false, "show": true, - "total": false + "total": false, + "values": false }, "lines": true, "linewidth": 1, - "links": [], + "links": [ + + ], "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", - "seriesOverrides": [], + "repeat": null, + "seriesOverrides": [ + + ], "spaceLength": 10, "span": 6, "stack": false, "steppedLine": false, "targets": [ { - "expr": "cluster:scheduler_e2e_scheduling_latency_seconds:quantile", + "expr": "max(node_load1{job=\"node-exporter\", instance=\"$instance\"})", "format": "time_series", "intervalFactor": 2, - "refId": "A", - "step": 60 + "legendFormat": "load 1m", + "refId": "A" + }, + { + "expr": "max(node_load5{job=\"node-exporter\", instance=\"$instance\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "load 5m", + "refId": "B" + }, + { + "expr": "max(node_load15{job=\"node-exporter\", instance=\"$instance\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "load 15m", + "refId": "C" } ], - "title": "End to End Scheduling Latency", + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "System load", "tooltip": { - "msResolution": false, "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { + "buckets": null, "mode": "time", + "name": null, "show": true, - "values": [] + "values": [ + + ] }, "yaxes": [ { "format": "short", + "label": null, "logBase": 1, + "max": null, + "min": null, "show": true }, { - "format": "dtdurations", + "format": "short", + "label": null, "logBase": 1, + "max": null, + "min": null, "show": true } ] }, { - "aliasColors": {}, + "aliasColors": { + + }, "bars": false, "dashLength": 10, "dashes": false, - "datasource": "prometheus", - "editable": false, - "error": false, + "datasource": "$datasource", "fill": 1, - "grid": { - "threshold1Color": "rgba(216, 200, 27, 0.27)", - "threshold2Color": "rgba(234, 112, 112, 0.22)" + "gridPos": { + }, - "id": 6, - "isNew": false, + "id": 3, "legend": { "alignAsTable": false, "avg": false, "current": false, - "hideEmpty": false, - "hideZero": false, "max": false, "min": false, "rightSide": false, "show": true, - "total": false + "total": false, + "values": false }, "lines": true, "linewidth": 1, - "links": [], + "links": [ + + ], "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", - "seriesOverrides": [], + "repeat": null, + "seriesOverrides": [ + + ], "spaceLength": 10, "span": 6, "stack": false, "steppedLine": false, "targets": [ { - "expr": "sum by(instance) (rate(apiserver_request_count{code!~\"2..\"}[5m]))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "Error Rate", - "refId": "A", - "step": 60 - }, - { - "expr": "sum by(instance) (rate(apiserver_request_count[5m]))", + "expr": "sum by (cpu) (irate(node_cpu_seconds_total{job=\"node-exporter\", mode!=\"idle\", instance=\"$instance\"}[5m]))", "format": "time_series", "intervalFactor": 2, - "legendFormat": "Request Rate", - "refId": "B", - "step": 60 + "legendFormat": "{{cpu}}", + "refId": "A" } ], - "title": "API Server Request Rates", + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Usage Per Core", "tooltip": { - "msResolution": false, "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { + "buckets": null, "mode": "time", + "name": null, "show": true, - "values": [] + "values": [ + + ] }, "yaxes": [ { - "format": "short", + "format": "percentunit", + "label": null, "logBase": 1, + "max": null, + "min": null, "show": true }, { - "format": "short", + "format": "percentunit", + "label": null, "logBase": 1, + "max": null, + "min": null, "show": true } ] } ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, "showTitle": false, "title": "Dashboard Row", - "titleSize": "h6" - } - ], - "schemaVersion": 14, - "sharedCrosshair": false, - "style": "dark", - "tags": [], - "templating": { - "list": [] - }, - "time": { - "from": "now-6h", - "to": "now" - }, - "timepicker": { - "refresh_intervals": [ - "5s", - "10s", - "30s", - "1m", - "5m", - "15m", - "30m", - "1h", - "2h", - "1d" - ], - "time_options": [ - "5m", - "15m", - "1h", - "6h", - "12h", - "24h", - "2d", - "7d", - "30d" - ] - }, - "timezone": "browser", - "title": "Kubernetes Control Plane Status", - "version": 3 - } - kubernetes-resource-requests-dashboard.json: |+ - { - "__inputs": [ - { - "description": "", - "label": "prometheus", - "name": "prometheus", - "pluginId": "prometheus", - "pluginName": "Prometheus", - "type": "datasource" - } - ], - "annotations": { - "list": [] - }, - "editable": false, - "graphTooltip": 0, - "hideControls": false, - "links": [], - "refresh": false, - "rows": [ + "titleSize": "h6", + "type": "row" + }, { "collapse": false, - "editable": false, - "height": "300px", + "collapsed": false, "panels": [ { - "aliasColors": {}, + "aliasColors": { + + }, "bars": false, "dashLength": 10, "dashes": false, - "datasource": "prometheus", - "description": "This represents the total [CPU resource requests](https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/#meaning-of-cpu) in the cluster.\nFor comparison the total [allocatable CPU cores](https://github.com/kubernetes/community/blob/master/contributors/design-proposals/node-allocatable.md) is also shown.", - "editable": false, - "error": false, + "datasource": "$datasource", "fill": 1, - "grid": { - "threshold1Color": "rgba(216, 200, 27, 0.27)", - "threshold2Color": "rgba(234, 112, 112, 0.22)" + "gridPos": { + }, - "id": 1, - "isNew": false, + "id": 4, "legend": { - "alignAsTable": false, - "avg": false, - "current": false, - "hideEmpty": false, - "hideZero": false, - "max": false, - "min": false, - "rightSide": false, - "show": true, - "total": false + "alignAsTable": "true", + "avg": "true", + "current": "true", + "max": "false", + "min": "false", + "rightSide": "true", + "show": "true", + "total": "false", + "values": "true" }, "lines": true, "linewidth": 1, - "links": [], + "links": [ + + ], "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", - "seriesOverrides": [], + "repeat": null, + "seriesOverrides": [ + + ], "spaceLength": 10, "span": 9, "stack": false, "steppedLine": false, "targets": [ { - "expr": "min(sum(kube_node_status_allocatable_cpu_cores) by (instance))", - "hide": false, - "intervalFactor": 2, - "legendFormat": "Allocatable CPU Cores", - "refId": "A", - "step": 20 - }, - { - "expr": "max(sum(kube_pod_container_resource_requests_cpu_cores) by (instance))", - "hide": false, - "intervalFactor": 2, - "legendFormat": "Requested CPU Cores", - "refId": "B", - "step": 20 + "expr": "max (sum by (cpu) (irate(node_cpu_seconds_total{job=\"node-exporter\", mode!=\"idle\", instance=\"$instance\"}[2m])) ) * 100\n", + "format": "time_series", + "intervalFactor": 10, + "legendFormat": "{{ cpu }}", + "refId": "A" } ], - "title": "CPU Cores", + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU Utilizaion", "tooltip": { - "msResolution": false, "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { + "buckets": null, "mode": "time", + "name": null, "show": true, - "values": [] + "values": [ + + ] }, "yaxes": [ { - "format": "short", - "label": "CPU Cores", + "format": "percent", + "label": null, "logBase": 1, + "max": 100, + "min": 0, "show": true }, { - "format": "short", + "format": "percent", + "label": null, "logBase": 1, + "max": 100, + "min": 0, "show": true } ] }, { + "cacheTimeout": null, "colorBackground": false, "colorValue": false, "colors": [ @@ -5122,8 +6426,7 @@ data: "rgba(237, 129, 40, 0.89)", "rgba(245, 54, 54, 0.9)" ], - "datasource": "prometheus", - "editable": false, + "datasource": "$datasource", "format": "percent", "gauge": { "maxValue": 100, @@ -5132,9 +6435,14 @@ data: "thresholdLabels": false, "thresholdMarkers": true }, - "hideTimeOverride": false, - "id": 2, - "links": [], + "gridPos": { + + }, + "id": 5, + "interval": null, + "links": [ + + ], "mappingType": 1, "mappingTypes": [ { @@ -5148,6 +6456,7 @@ data: ], "maxDataPoints": 100, "nullPointMode": "connected", + "nullText": null, "postfix": "", "postfixFontSize": "50%", "prefix": "", @@ -5164,22 +6473,22 @@ data: "fillColor": "rgba(31, 118, 189, 0.18)", "full": false, "lineColor": "rgb(31, 120, 193)", - "show": true + "show": false }, + "tableColumn": "", "targets": [ { - "expr": "max(sum(kube_pod_container_resource_requests_cpu_cores) by (instance)) / min(sum(kube_node_status_allocatable_cpu_cores) by (instance)) * 100", + "expr": "avg(sum by (cpu) (irate(node_cpu_seconds_total{job=\"node-exporter\", mode!=\"idle\", instance=\"$instance\"}[2m]))) * 100\n", + "format": "time_series", "intervalFactor": 2, "legendFormat": "", - "refId": "A", - "step": 240 + "refId": "A" } ], "thresholds": "80, 90", - "title": "CPU Cores", - "transparent": false, + "title": "CPU Usage", "type": "singlestat", - "valueFontSize": "110%", + "valueFontSize": "80%", "valueMaps": [ { "op": "=", @@ -5187,105 +6496,135 @@ data: "value": "null" } ], - "valueName": "avg" + "valueName": "current" } ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, "showTitle": false, - "title": "CPU Cores", - "titleSize": "h6" + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" }, { "collapse": false, - "editable": false, - "height": "300px", + "collapsed": false, "panels": [ { - "aliasColors": {}, + "aliasColors": { + + }, "bars": false, "dashLength": 10, "dashes": false, - "datasource": "prometheus", - "description": "This represents the total [memory resource requests](https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/#meaning-of-memory) in the cluster.\nFor comparison the total [allocatable memory](https://github.com/kubernetes/community/blob/master/contributors/design-proposals/node-allocatable.md) is also shown.", - "editable": false, - "error": false, + "datasource": "$datasource", "fill": 1, - "grid": { - "threshold1Color": "rgba(216, 200, 27, 0.27)", - "threshold2Color": "rgba(234, 112, 112, 0.22)" + "gridPos": { + }, - "id": 3, - "isNew": false, + "id": 6, "legend": { "alignAsTable": false, "avg": false, "current": false, - "hideEmpty": false, - "hideZero": false, "max": false, "min": false, "rightSide": false, "show": true, - "total": false + "total": false, + "values": false }, "lines": true, "linewidth": 1, - "links": [], + "links": [ + + ], "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", - "seriesOverrides": [], + "repeat": null, + "seriesOverrides": [ + + ], "spaceLength": 10, "span": 9, "stack": false, "steppedLine": false, "targets": [ { - "expr": "min(sum(kube_node_status_allocatable_memory_bytes) by (instance))", - "hide": false, + "expr": "max(\n node_memory_MemTotal_bytes{job=\"node-exporter\", instance=\"$instance\"}\n - node_memory_MemFree_bytes{job=\"node-exporter\", instance=\"$instance\"}\n - node_memory_Buffers_bytes{job=\"node-exporter\", instance=\"$instance\"}\n - node_memory_Cached_bytes{job=\"node-exporter\", instance=\"$instance\"}\n)\n", + "format": "time_series", "intervalFactor": 2, - "legendFormat": "Allocatable Memory", - "refId": "A", - "step": 20 + "legendFormat": "memory used", + "refId": "A" }, { - "expr": "max(sum(kube_pod_container_resource_requests_memory_bytes) by (instance))", - "hide": false, + "expr": "max(node_memory_Buffers_bytes{job=\"node-exporter\", instance=\"$instance\"})", + "format": "time_series", "intervalFactor": 2, - "legendFormat": "Requested Memory", - "refId": "B", - "step": 20 + "legendFormat": "memory buffers", + "refId": "B" + }, + { + "expr": "max(node_memory_Cached_bytes{job=\"node-exporter\", instance=\"$instance\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "memory cached", + "refId": "C" + }, + { + "expr": "max(node_memory_MemFree_bytes{job=\"node-exporter\", instance=\"$instance\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "memory free", + "refId": "D" } ], - "title": "Memory", + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Memory Usage", "tooltip": { - "msResolution": false, "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { + "buckets": null, "mode": "time", + "name": null, "show": true, - "values": [] + "values": [ + + ] }, "yaxes": [ { "format": "bytes", - "label": "Memory", + "label": null, "logBase": 1, + "max": null, + "min": null, "show": true }, { - "format": "short", + "format": "bytes", + "label": null, "logBase": 1, + "max": null, + "min": null, "show": true } ] }, { + "cacheTimeout": null, "colorBackground": false, "colorValue": false, "colors": [ @@ -5293,8 +6632,7 @@ data: "rgba(237, 129, 40, 0.89)", "rgba(245, 54, 54, 0.9)" ], - "datasource": "prometheus", - "editable": false, + "datasource": "$datasource", "format": "percent", "gauge": { "maxValue": 100, @@ -5303,9 +6641,14 @@ data: "thresholdLabels": false, "thresholdMarkers": true }, - "hideTimeOverride": false, - "id": 4, - "links": [], + "gridPos": { + + }, + "id": 7, + "interval": null, + "links": [ + + ], "mappingType": 1, "mappingTypes": [ { @@ -5319,6 +6662,7 @@ data: ], "maxDataPoints": 100, "nullPointMode": "connected", + "nullText": null, "postfix": "", "postfixFontSize": "50%", "prefix": "", @@ -5330,533 +6674,497 @@ data: "to": "null" } ], - "span": 3, - "sparkline": { - "fillColor": "rgba(31, 118, 189, 0.18)", - "full": false, - "lineColor": "rgb(31, 120, 193)", - "show": true - }, - "targets": [ - { - "expr": "max(sum(kube_pod_container_resource_requests_memory_bytes) by (instance)) / min(sum(kube_node_status_allocatable_memory_bytes) by (instance)) * 100", - "intervalFactor": 2, - "legendFormat": "", - "refId": "A", - "step": 240 - } - ], - "thresholds": "80, 90", - "title": "Memory", - "transparent": false, - "type": "singlestat", - "valueFontSize": "110%", - "valueMaps": [ - { - "op": "=", - "text": "N/A", - "value": "null" - } - ], - "valueName": "avg" - } - ], - "showTitle": false, - "title": "Memory", - "titleSize": "h6" - } - ], - "schemaVersion": 14, - "sharedCrosshair": false, - "style": "dark", - "tags": [], - "templating": { - "list": [] - }, - "time": { - "from": "now-3h", - "to": "now" - }, - "timepicker": { - "refresh_intervals": [ - "5s", - "10s", - "30s", - "1m", - "5m", - "15m", - "30m", - "1h", - "2h", - "1d" - ], - "time_options": [ - "5m", - "15m", - "1h", - "6h", - "12h", - "24h", - "2d", - "7d", - "30d" - ] - }, - "timezone": "browser", - "title": "Kubernetes Resource Requests", - "version": 2 - } - nodes-dashboard.json: |+ - { - "__inputs": [ - { - "description": "", - "label": "prometheus", - "name": "prometheus", - "pluginId": "prometheus", - "pluginName": "Prometheus", - "type": "datasource" - } - ], - "annotations": { - "list": [] - }, - "description": "Dashboard to get an overview of one server", - "editable": false, - "gnetId": 22, - "graphTooltip": 0, - "hideControls": false, - "links": [], - "refresh": false, - "rows": [ + "span": 3, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "max(\n (\n (\n node_memory_MemTotal_bytes{job=\"node-exporter\", instance=\"$instance\"}\n - node_memory_MemFree_bytes{job=\"node-exporter\", instance=\"$instance\"}\n - node_memory_Buffers_bytes{job=\"node-exporter\", instance=\"$instance\"}\n - node_memory_Cached_bytes{job=\"node-exporter\", instance=\"$instance\"}\n )\n / node_memory_MemTotal_bytes{job=\"node-exporter\", instance=\"$instance\"}\n ) * 100)\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": "80, 90", + "title": "Memory Usage", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + }, { "collapse": false, - "editable": false, - "height": "250px", + "collapsed": false, "panels": [ { - "aliasColors": {}, + "aliasColors": { + + }, "bars": false, "dashLength": 10, "dashes": false, - "datasource": "prometheus", - "editable": false, - "error": false, + "datasource": "$datasource", "fill": 1, - "grid": { - "threshold1Color": "rgba(216, 200, 27, 0.27)", - "threshold2Color": "rgba(234, 112, 112, 0.22)" + "gridPos": { + }, - "id": 3, - "isNew": false, + "id": 8, "legend": { "alignAsTable": false, "avg": false, "current": false, - "hideEmpty": false, - "hideZero": false, "max": false, "min": false, "rightSide": false, "show": true, - "total": false + "total": false, + "values": false }, "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "connected", + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", - "seriesOverrides": [], + "repeat": null, + "seriesOverrides": [ + { + "alias": "read", + "yaxis": 1 + }, + { + "alias": "io time", + "yaxis": 2 + } + ], "spaceLength": 10, "span": 6, "stack": false, "steppedLine": false, "targets": [ { - "expr": "100 - (avg by (cpu) (irate(node_cpu_seconds_total{mode=\"idle\", instance=\"$server\"}[5m])) * 100)", - "hide": false, - "intervalFactor": 10, - "legendFormat": "{{cpu}}", - "refId": "A", - "step": 50 + "expr": "max(rate(node_disk_read_bytes_total{job=\"node-exporter\", instance=\"$instance\"}[2m]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "read", + "refId": "A" + }, + { + "expr": "max(rate(node_disk_written_bytes_total{job=\"node-exporter\", instance=\"$instance\"}[2m]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "written", + "refId": "B" + }, + { + "expr": "max(rate(node_disk_io_time_seconds_total{job=\"node-exporter\", instance=\"$instance\"}[2m]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "io time", + "refId": "C" } ], - "title": "Idle CPU", + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Disk I/O", "tooltip": { - "msResolution": false, "shared": true, "sort": 0, - "value_type": "cumulative" + "value_type": "individual" }, "type": "graph", "xaxis": { + "buckets": null, "mode": "time", + "name": null, "show": true, - "values": [] + "values": [ + + ] }, "yaxes": [ { - "format": "percent", - "label": "cpu usage", + "format": "bytes", + "label": null, "logBase": 1, - "max": 100, - "min": 0, + "max": null, + "min": null, "show": true }, { - "format": "short", + "format": "ms", + "label": null, "logBase": 1, + "max": null, + "min": null, "show": true } ] }, { - "aliasColors": {}, + "aliasColors": { + + }, "bars": false, "dashLength": 10, "dashes": false, - "datasource": "prometheus", - "editable": false, - "error": false, + "datasource": "$datasource", "fill": 1, - "grid": { - "threshold1Color": "rgba(216, 200, 27, 0.27)", - "threshold2Color": "rgba(234, 112, 112, 0.22)" + "gridPos": { + }, "id": 9, - "isNew": false, "legend": { "alignAsTable": false, "avg": false, "current": false, - "hideEmpty": false, - "hideZero": false, "max": false, "min": false, "rightSide": false, "show": true, - "total": false + "total": false, + "values": false }, "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "connected", + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", - "seriesOverrides": [], + "repeat": null, + "seriesOverrides": [ + + ], "spaceLength": 10, "span": 6, "stack": false, "steppedLine": false, "targets": [ { - "expr": "node_load1{instance=\"$server\"}", - "intervalFactor": 4, - "legendFormat": "load 1m", - "refId": "A", - "step": 20, - "target": "" - }, - { - "expr": "node_load5{instance=\"$server\"}", - "intervalFactor": 4, - "legendFormat": "load 5m", - "refId": "B", - "step": 20, - "target": "" - }, - { - "expr": "node_load15{instance=\"$server\"}", - "intervalFactor": 4, - "legendFormat": "load 15m", - "refId": "C", - "step": 20, - "target": "" + "expr": "node:node_filesystem_usage:\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{device}}", + "refId": "A" } ], - "title": "System Load", + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Disk Space Usage", "tooltip": { - "msResolution": false, "shared": true, "sort": 0, - "value_type": "cumulative" + "value_type": "individual" }, "type": "graph", "xaxis": { + "buckets": null, "mode": "time", + "name": null, "show": true, - "values": [] + "values": [ + + ] }, "yaxes": [ { "format": "percentunit", + "label": null, "logBase": 1, + "max": null, + "min": null, "show": true }, { - "format": "short", + "format": "percentunit", + "label": null, "logBase": 1, + "max": null, + "min": null, "show": true } ] } ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, "showTitle": false, - "title": "New Row", - "titleSize": "h6" + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" }, { "collapse": false, - "editable": false, - "height": "250px", + "collapsed": false, "panels": [ { - "aliasColors": {}, + "aliasColors": { + + }, "bars": false, "dashLength": 10, "dashes": false, - "datasource": "prometheus", - "editable": false, - "error": false, + "datasource": "$datasource", "fill": 1, - "grid": { - "threshold1Color": "rgba(216, 200, 27, 0.27)", - "threshold2Color": "rgba(234, 112, 112, 0.22)" + "gridPos": { + }, - "id": 4, - "isNew": false, + "id": 10, "legend": { "alignAsTable": false, "avg": false, "current": false, - "hideEmpty": false, - "hideZero": false, "max": false, "min": false, "rightSide": false, "show": true, - "total": false + "total": false, + "values": false }, "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "connected", + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", + "repeat": null, "seriesOverrides": [ - { - "alias": "node_memory_SwapFree_bytes{instance=\"172.17.0.1:9100\",job=\"prometheus\"}", - "yaxis": 2 - } + ], "spaceLength": 10, - "span": 9, - "stack": true, + "span": 6, + "stack": false, "steppedLine": false, "targets": [ { - "expr": "node_memory_MemTotal_bytes{instance=\"$server\"} - node_memory_MemFree_bytes{instance=\"$server\"} - node_memory_Buffers_bytes{instance=\"$server\"} - node_memory_Cached_bytes{instance=\"$server\"}", - "hide": false, - "interval": "", - "intervalFactor": 2, - "legendFormat": "memory used", - "metric": "", - "refId": "C", - "step": 10 - }, - { - "expr": "node_memory_Buffers_bytes{instance=\"$server\"}", - "interval": "", - "intervalFactor": 2, - "legendFormat": "memory buffers", - "metric": "", - "refId": "E", - "step": 10 - }, - { - "expr": "node_memory_Cached{instance=\"$server\"}", - "intervalFactor": 2, - "legendFormat": "memory cached", - "metric": "", - "refId": "F", - "step": 10 - }, - { - "expr": "node_memory_MemFree_bytes{instance=\"$server\"}", + "expr": "max(rate(node_network_receive_bytes_total{job=\"node-exporter\", instance=\"$instance\", device!~\"lo\"}[5m]))", + "format": "time_series", "intervalFactor": 2, - "legendFormat": "memory free", - "metric": "", - "refId": "D", - "step": 10 + "legendFormat": "{{device}}", + "refId": "A" } ], - "title": "Memory Usage", + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Network Received", "tooltip": { - "msResolution": false, "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { + "buckets": null, "mode": "time", + "name": null, "show": true, - "values": [] + "values": [ + + ] }, "yaxes": [ { "format": "bytes", + "label": null, "logBase": 1, - "min": "0", + "max": null, + "min": null, "show": true }, { - "format": "short", + "format": "bytes", + "label": null, "logBase": 1, + "max": null, + "min": null, "show": true } ] }, { - "colorBackground": false, - "colorValue": false, - "colors": [ - "rgba(50, 172, 45, 0.97)", - "rgba(237, 129, 40, 0.89)", - "rgba(245, 54, 54, 0.9)" - ], - "datasource": "prometheus", - "editable": false, - "format": "percent", - "gauge": { - "maxValue": 100, - "minValue": 0, - "show": true, - "thresholdLabels": false, - "thresholdMarkers": true + "aliasColors": { + }, - "hideTimeOverride": false, - "id": 5, - "links": [], - "mappingType": 1, - "mappingTypes": [ - { - "name": "value to text", - "value": 1 - }, - { - "name": "range to text", - "value": 2 - } - ], - "maxDataPoints": 100, - "nullPointMode": "connected", - "postfix": "", - "postfixFontSize": "50%", - "prefix": "", - "prefixFontSize": "50%", - "rangeMaps": [ - { - "from": "null", - "text": "N/A", - "to": "null" - } - ], - "span": 3, - "sparkline": { - "fillColor": "rgba(31, 118, 189, 0.18)", - "full": false, - "lineColor": "rgb(31, 120, 193)", - "show": false + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "gridPos": { + + }, + "id": 11, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": false }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, "targets": [ { - "expr": "((node_memory_MemTotal_bytes{instance=\"$server\"} - node_memory_MemFree_bytes{instance=\"$server\"} - node_memory_Buffers_bytes{instance=\"$server\"} - node_memory_Cached_bytes{instance=\"$server\"}) / node_memory_MemTotal_bytes{instance=\"$server\"}) * 100", + "expr": "max(rate(node_network_transmit_bytes_total{job=\"node-exporter\", instance=\"$instance\", device!~\"lo\"}[5m]))", + "format": "time_series", "intervalFactor": 2, - "refId": "A", - "step": 60, - "target": "" + "legendFormat": "{{device}}", + "refId": "A" } ], - "thresholds": "80, 90", - "title": "Memory Usage", - "transparent": false, - "type": "singlestat", - "valueFontSize": "80%", - "valueMaps": [ + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Network Transmitted", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ { - "op": "=", - "text": "N/A", - "value": "null" + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true } - ], - "valueName": "avg" + ] } ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, "showTitle": false, - "title": "New Row", - "titleSize": "h6" + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" }, { "collapse": false, - "editable": false, - "height": "250px", + "collapsed": false, "panels": [ { - "aliasColors": {}, + "aliasColors": { + + }, "bars": false, "dashLength": 10, "dashes": false, - "datasource": "prometheus", - "editable": false, - "error": false, + "datasource": "$datasource", "fill": 1, - "grid": { - "threshold1Color": "rgba(216, 200, 27, 0.27)", - "threshold2Color": "rgba(234, 112, 112, 0.22)" + "gridPos": { + }, - "id": 6, - "isNew": true, + "id": 12, "legend": { "alignAsTable": false, "avg": false, "current": false, - "hideEmpty": false, - "hideZero": false, "max": false, "min": false, "rightSide": false, "show": true, - "total": false + "total": false, + "values": false }, "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "connected", + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", + "repeat": null, "seriesOverrides": [ - { - "alias": "read", - "yaxis": 1 - }, - { - "alias": "{instance=\"172.17.0.1:9100\"}", - "yaxis": 2 - }, - { - "alias": "io time", - "yaxis": 2 - } + ], "spaceLength": 10, "span": 9, @@ -5864,56 +7172,62 @@ data: "steppedLine": false, "targets": [ { - "expr": "sum by (instance) (rate(node_disk_read_bytes_total{instance=\"$server\"}[2m]))", - "hide": false, - "intervalFactor": 4, - "legendFormat": "read", - "refId": "A", - "step": 20, - "target": "" - }, - { - "expr": "sum by (instance) (rate(node_disk_written_bytes_total{instance=\"$server\"}[2m]))", - "intervalFactor": 4, - "legendFormat": "written", - "refId": "B", - "step": 20 + "expr": "max(\n node_filesystem_files{job=\"node-exporter\", instance=\"$instance\"}\n - node_filesystem_files_free{job=\"node-exporter\", instance=\"$instance\"}\n)\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "inodes used", + "refId": "A" }, { - "expr": "sum by (instance) (rate(node_disk_io_time_seconds_total{instance=\"$server\"}[2m]))", - "intervalFactor": 4, - "legendFormat": "io time", - "refId": "C", - "step": 20 + "expr": "max(node_filesystem_files_free{job=\"node-exporter\", instance=\"$instance\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "inodes free", + "refId": "B" } ], - "title": "Disk I/O", + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Inodes Usage", "tooltip": { - "msResolution": false, "shared": true, "sort": 0, - "value_type": "cumulative" + "value_type": "individual" }, "type": "graph", "xaxis": { + "buckets": null, "mode": "time", + "name": null, "show": true, - "values": [] + "values": [ + + ] }, "yaxes": [ { - "format": "bytes", + "format": "short", + "label": null, "logBase": 1, + "max": null, + "min": null, "show": true }, { - "format": "ms", + "format": "short", + "label": null, "logBase": 1, + "max": null, + "min": null, "show": true } ] }, { + "cacheTimeout": null, "colorBackground": false, "colorValue": false, "colors": [ @@ -5921,19 +7235,23 @@ data: "rgba(237, 129, 40, 0.89)", "rgba(245, 54, 54, 0.9)" ], - "datasource": "prometheus", - "editable": false, - "format": "percentunit", + "datasource": "$datasource", + "format": "percent", "gauge": { - "maxValue": 1, + "maxValue": 100, "minValue": 0, "show": true, "thresholdLabels": false, "thresholdMarkers": true }, - "hideTimeOverride": false, - "id": 7, - "links": [], + "gridPos": { + + }, + "id": 13, + "interval": null, + "links": [ + + ], "mappingType": 1, "mappingTypes": [ { @@ -5947,6 +7265,7 @@ data: ], "maxDataPoints": 100, "nullPointMode": "connected", + "nullText": null, "postfix": "", "postfixFontSize": "50%", "prefix": "", @@ -5965,18 +7284,18 @@ data: "lineColor": "rgb(31, 120, 193)", "show": false }, + "tableColumn": "", "targets": [ { - "expr": "(sum(node_filesystem_size_bytes{device!=\"rootfs\",instance=\"$server\"}) - sum(node_filesystem_free_bytes{device!=\"rootfs\",instance=\"$server\"})) / sum(node_filesystem_size_bytes{device!=\"rootfs\",instance=\"$server\"})", + "expr": "max(\n (\n (\n node_filesystem_files{job=\"node-exporter\", instance=\"$instance\"}\n - node_filesystem_files_free{job=\"node-exporter\", instance=\"$instance\"}\n )\n / node_filesystem_files{job=\"node-exporter\", instance=\"$instance\"}\n ) * 100)\n", + "format": "time_series", "intervalFactor": 2, - "refId": "A", - "step": 60, - "target": "" + "legendFormat": "", + "refId": "A" } ], - "thresholds": "0.75, 0.9", - "title": "Disk Space Usage", - "transparent": false, + "thresholds": "80, 90", + "title": "Inodes Usage", "type": "singlestat", "valueFontSize": "80%", "valueMaps": [ @@ -5989,207 +7308,403 @@ data: "valueName": "current" } ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, "showTitle": false, - "title": "New Row", - "titleSize": "h6" - }, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + } + ], + "schemaVersion": 14, + "style": "dark", + "tags": [ + "kubernetes-mixin" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": null, + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "instance", + "options": [ + + ], + "query": "label_values(node_boot_time_seconds{job=\"node-exporter\"}, instance)", + "refresh": 2, + "regex": "", + "sort": 0, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "", + "title": "Kubernetes / Nodes", + "uid": "fa49a4706d07a042595b664c87fb33ea", + "version": 0 + } + persistentvolumesusage.json: |- + { + "__inputs": [ + + ], + "__requires": [ + + ], + "annotations": { + "list": [ + + ] + }, + "editable": false, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "id": null, + "links": [ + + ], + "refresh": "", + "rows": [ { "collapse": false, - "editable": false, - "height": "250px", + "collapsed": false, "panels": [ { - "aliasColors": {}, + "aliasColors": { + + }, "bars": false, "dashLength": 10, "dashes": false, - "datasource": "prometheus", - "editable": false, - "error": false, + "datasource": "$datasource", "fill": 1, - "grid": { - "threshold1Color": "rgba(216, 200, 27, 0.27)", - "threshold2Color": "rgba(234, 112, 112, 0.22)" + "gridPos": { + }, - "id": 8, - "isNew": false, + "id": 2, "legend": { "alignAsTable": false, - "avg": false, - "current": false, - "hideEmpty": false, - "hideZero": false, - "max": false, - "min": false, + "avg": true, + "current": true, + "max": true, + "min": true, "rightSide": false, "show": true, - "total": false + "total": false, + "values": true }, "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "connected", + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", + "repeat": null, "seriesOverrides": [ - { - "alias": "transmitted", - "yaxis": 2 - } + ], "spaceLength": 10, - "span": 6, + "span": 12, "stack": false, "steppedLine": false, "targets": [ { - "expr": "rate(node_network_receive_bytes_total{instance=\"$server\",device!~\"lo\"}[5m])", - "hide": false, - "intervalFactor": 2, - "legendFormat": "{{device}}", - "refId": "A", - "step": 10, - "target": "" + "expr": "(kubelet_volume_stats_capacity_bytes{job=\"kubelet\", persistentvolumeclaim=\"$volume\"} - kubelet_volume_stats_available_bytes{job=\"kubelet\", persistentvolumeclaim=\"$volume\"}) / kubelet_volume_stats_capacity_bytes{job=\"kubelet\", persistentvolumeclaim=\"$volume\"} * 100\n", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{ Usage }}", + "refId": "A" } ], - "title": "Network Received", + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Volume Space Usage", "tooltip": { - "msResolution": false, "shared": true, "sort": 0, - "value_type": "cumulative" + "value_type": "individual" }, "type": "graph", "xaxis": { + "buckets": null, "mode": "time", + "name": null, "show": true, - "values": [] + "values": [ + + ] }, "yaxes": [ { - "format": "bytes", + "format": "percent", + "label": null, "logBase": 1, + "max": 100, + "min": 0, "show": true }, { - "format": "bytes", + "format": "percent", + "label": null, "logBase": 1, + "max": 100, + "min": 0, "show": true } ] - }, + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ { - "aliasColors": {}, + "aliasColors": { + + }, "bars": false, "dashLength": 10, "dashes": false, - "datasource": "prometheus", - "editable": false, - "error": false, + "datasource": "$datasource", "fill": 1, - "grid": { - "threshold1Color": "rgba(216, 200, 27, 0.27)", - "threshold2Color": "rgba(234, 112, 112, 0.22)" + "gridPos": { + }, - "id": 10, - "isNew": false, + "id": 3, "legend": { "alignAsTable": false, - "avg": false, - "current": false, - "hideEmpty": false, - "hideZero": false, - "max": false, - "min": false, + "avg": true, + "current": true, + "max": true, + "min": true, "rightSide": false, "show": true, - "total": false + "total": false, + "values": true }, "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "connected", + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", + "repeat": null, "seriesOverrides": [ - { - "alias": "transmitted", - "yaxis": 2 - } + ], "spaceLength": 10, - "span": 6, + "span": 12, "stack": false, "steppedLine": false, "targets": [ { - "expr": "rate(node_network_transmit_bytes_total{instance=\"$server\",device!~\"lo\"}[5m])", - "hide": false, - "intervalFactor": 2, - "legendFormat": "{{device}}", - "refId": "B", - "step": 10, - "target": "" + "expr": "kubelet_volume_stats_inodes_used{job=\"kubelet\", persistentvolumeclaim=\"$volume\"} / kubelet_volume_stats_inodes{job=\"kubelet\", persistentvolumeclaim=\"$volume\"} * 100\n", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{ Usage }}", + "refId": "A" } ], - "title": "Network Transmitted", + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Volume inodes Usage", "tooltip": { - "msResolution": false, "shared": true, "sort": 0, - "value_type": "cumulative" + "value_type": "individual" }, "type": "graph", "xaxis": { + "buckets": null, "mode": "time", + "name": null, "show": true, - "values": [] + "values": [ + + ] }, "yaxes": [ { - "format": "bytes", + "format": "percent", + "label": null, "logBase": 1, + "max": 100, + "min": 0, "show": true }, { - "format": "bytes", + "format": "percent", + "label": null, "logBase": 1, + "max": 100, + "min": 0, "show": true } ] } ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, "showTitle": false, - "title": "New Row", - "titleSize": "h6" + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" } ], "schemaVersion": 14, - "sharedCrosshair": false, "style": "dark", - "tags": [], + "tags": [ + "kubernetes-mixin" + ], "templating": { "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": null, + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, { "allValue": null, - "current": {}, - "datasource": "prometheus", + "current": { + + }, + "datasource": "$datasource", "hide": 0, "includeAll": false, - "label": null, + "label": "Namespace", "multi": false, - "name": "server", - "options": [], - "query": "label_values(node_boot_time_seconds, instance)", - "refresh": 1, + "name": "namespace", + "options": [ + + ], + "query": "label_values(kubelet_volume_stats_capacity_bytes{job=\"kubelet\"}, exported_namespace)", + "refresh": 2, + "regex": "", + "sort": 0, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": "PersistentVolumeClaim", + "multi": false, + "name": "volume", + "options": [ + + ], + "query": "label_values(kubelet_volume_stats_capacity_bytes{job=\"kubelet\", exported_namespace=\"$namespace\"}, persistentvolumeclaim)", + "refresh": 2, "regex": "", "sort": 0, "tagValuesQuery": "", - "tags": [], + "tags": [ + + ], "tagsQuery": "", "type": "query", "useTags": false @@ -6197,7 +7712,7 @@ data: ] }, "time": { - "from": "now-1h", + "from": "now-7d", "to": "now" }, "timepicker": { @@ -6225,390 +7740,455 @@ data: "30d" ] }, - "timezone": "browser", - "title": "Nodes", - "version": 2 + "timezone": "", + "title": "Kubernetes / Persistent Volumes", + "uid": "919b92a8e8041bd567af9edab12c840c", + "version": 0 } - pods-dashboard.json: |+ + pods.json: |- { "__inputs": [ - { - "description": "", - "label": "prometheus", - "name": "prometheus", - "pluginId": "prometheus", - "pluginName": "Prometheus", - "type": "datasource" - } + + ], + "__requires": [ + ], "annotations": { - "list": [] + "list": [ + + ] }, "editable": false, - "graphTooltip": 1, + "gnetId": null, + "graphTooltip": 0, "hideControls": false, - "links": [], - "refresh": false, + "id": null, + "links": [ + + ], + "refresh": "", "rows": [ { "collapse": false, - "editable": false, - "height": "250px", + "collapsed": false, "panels": [ { - "aliasColors": {}, + "aliasColors": { + + }, "bars": false, "dashLength": 10, "dashes": false, - "datasource": "prometheus", - "editable": false, - "error": false, + "datasource": "$datasource", "fill": 1, - "grid": { - "threshold1Color": "rgba(216, 200, 27, 0.27)", - "threshold2Color": "rgba(234, 112, 112, 0.22)" + "gridPos": { + }, - "id": 1, - "isNew": false, + "id": 2, "legend": { "alignAsTable": true, "avg": true, "current": true, - "hideEmpty": false, - "hideZero": false, "max": false, "min": false, "rightSide": true, "show": true, "total": false, - "values": true + "values": false }, "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "connected", + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", - "seriesOverrides": [], + "repeat": null, + "seriesOverrides": [ + + ], "spaceLength": 10, - "span": 12, "stack": false, "steppedLine": false, "targets": [ { - "expr": "sum by(container_name) (container_memory_usage_bytes{pod_name=\"$pod\", container_name=~\"$container\", container_name!=\"POD\"})", - "interval": "10s", - "intervalFactor": 1, + "expr": "sum by(container_name) (container_memory_usage_bytes{job=\"kubernetes-cadvisor\", namespace=\"$namespace\", pod_name=\"$pod\", container_name=~\"$container\", container_name!=\"POD\"})", + "format": "time_series", + "intervalFactor": 2, "legendFormat": "Current: {{ container_name }}", - "metric": "container_memory_usage_bytes", - "refId": "A", - "step": 15 + "refId": "A" }, { - "expr": "kube_pod_container_resource_requests_memory_bytes{pod=\"$pod\", container=~\"$container\"}", - "interval": "10s", + "expr": "sum by(container) (kube_pod_container_resource_requests_memory_bytes{job=\"kube-state-metrics\", namespace=\"$namespace\", pod=\"$pod\", container=~\"$container\"})", + "format": "time_series", "intervalFactor": 2, "legendFormat": "Requested: {{ container }}", - "metric": "kube_pod_container_resource_requests_memory_bytes", - "refId": "B", - "step": 20 + "refId": "B" }, { - "expr": "kube_pod_container_resource_limits_memory_bytes{pod=\"$pod\", container=~\"$container\"}", - "interval": "10s", + "expr": "sum by(container) (kube_pod_container_resource_limits_memory_bytes{job=\"kube-state-metrics\", namespace=\"$namespace\", pod=\"$pod\", container=~\"$container\"})", + "format": "time_series", "intervalFactor": 2, "legendFormat": "Limit: {{ container }}", - "metric": "kube_pod_container_resource_limits_memory_bytes", - "refId": "C", - "step": 20 + "refId": "C" } ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, "title": "Memory Usage", "tooltip": { - "msResolution": true, "shared": true, "sort": 0, - "value_type": "cumulative" + "value_type": "individual" }, "type": "graph", "xaxis": { + "buckets": null, "mode": "time", + "name": null, "show": true, - "values": [] + "values": [ + + ] }, "yaxes": [ { "format": "bytes", + "label": null, "logBase": 1, + "max": null, + "min": 0, "show": true }, { - "format": "short", + "format": "bytes", + "label": null, "logBase": 1, + "max": null, + "min": 0, "show": true } ] } ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, "showTitle": false, - "title": "Row", - "titleSize": "h6" + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" }, { "collapse": false, - "editable": false, - "height": "250px", + "collapsed": false, "panels": [ { - "aliasColors": {}, + "aliasColors": { + + }, "bars": false, "dashLength": 10, "dashes": false, - "datasource": "prometheus", - "editable": false, - "error": false, + "datasource": "$datasource", "fill": 1, - "grid": { - "threshold1Color": "rgba(216, 200, 27, 0.27)", - "threshold2Color": "rgba(234, 112, 112, 0.22)" + "gridPos": { + }, - "id": 2, - "isNew": false, + "id": 3, "legend": { "alignAsTable": true, "avg": true, "current": true, - "hideEmpty": false, - "hideZero": false, "max": false, "min": false, "rightSide": true, "show": true, "total": false, - "values": true + "values": false }, "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "connected", + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", - "seriesOverrides": [], + "repeat": null, + "seriesOverrides": [ + + ], "spaceLength": 10, - "span": 12, "stack": false, "steppedLine": false, "targets": [ { - "expr": "sum by (container_name)(rate(container_cpu_usage_seconds_total{image!=\"\",container_name!=\"POD\",pod_name=\"$pod\"}[1m]))", + "expr": "sum by (container_name) (rate(container_cpu_usage_seconds_total{job=\"kubernetes-cadvisor\", namespace=\"$namespace\", image!=\"\",container_name!=\"POD\",pod_name=\"$pod\"}[1m]))", + "format": "time_series", "intervalFactor": 2, "legendFormat": "{{ container_name }}", - "refId": "A", - "step": 30 - }, - { - "expr": "kube_pod_container_resource_requests_cpu_cores{pod=\"$pod\", container=~\"$container\"}", - "interval": "10s", - "intervalFactor": 2, - "legendFormat": "Requested: {{ container }}", - "metric": "kube_pod_container_resource_requests_cpu_cores", - "refId": "B", - "step": 20 - }, - { - "expr": "kube_pod_container_resource_limits_cpu_cores{pod=\"$pod\", container=~\"$container\"}", - "interval": "10s", - "intervalFactor": 2, - "legendFormat": "Limit: {{ container }}", - "metric": "kube_pod_container_resource_limits_memory_bytes", - "refId": "C", - "step": 20 + "refId": "A" } ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, "title": "CPU Usage", "tooltip": { - "msResolution": true, "shared": true, "sort": 0, - "value_type": "cumulative" + "value_type": "individual" }, "type": "graph", "xaxis": { + "buckets": null, "mode": "time", + "name": null, "show": true, - "values": [] + "values": [ + + ] }, "yaxes": [ { "format": "short", + "label": null, "logBase": 1, + "max": null, + "min": 0, "show": true }, { "format": "short", + "label": null, "logBase": 1, + "max": null, + "min": 0, "show": true } ] } ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, "showTitle": false, - "title": "Row", - "titleSize": "h6" + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" }, { "collapse": false, - "editable": false, - "height": "250px", + "collapsed": false, "panels": [ { - "aliasColors": {}, + "aliasColors": { + + }, "bars": false, "dashLength": 10, "dashes": false, - "datasource": "prometheus", - "editable": false, - "error": false, + "datasource": "$datasource", "fill": 1, - "grid": { - "threshold1Color": "rgba(216, 200, 27, 0.27)", - "threshold2Color": "rgba(234, 112, 112, 0.22)" + "gridPos": { + }, - "id": 3, - "isNew": false, + "id": 4, "legend": { "alignAsTable": true, "avg": true, "current": true, - "hideEmpty": false, - "hideZero": false, "max": false, "min": false, "rightSide": true, "show": true, "total": false, - "values": true + "values": false }, "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "connected", + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", - "seriesOverrides": [], + "repeat": null, + "seriesOverrides": [ + + ], "spaceLength": 10, - "span": 12, "stack": false, "steppedLine": false, "targets": [ { - "expr": "sort_desc(sum by (pod_name) (rate(container_network_receive_bytes_total{pod_name=\"$pod\"}[1m])))", + "expr": "sort_desc(sum by (pod_name) (rate(container_network_receive_bytes_total{job=\"kubernetes-cadvisor\", namespace=\"$namespace\", pod_name=\"$pod\"}[1m])))", + "format": "time_series", "intervalFactor": 2, "legendFormat": "{{ pod_name }}", - "refId": "A", - "step": 30 + "refId": "A" } ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, "title": "Network I/O", "tooltip": { - "msResolution": true, "shared": true, "sort": 0, - "value_type": "cumulative" + "value_type": "individual" }, "type": "graph", "xaxis": { + "buckets": null, "mode": "time", + "name": null, "show": true, - "values": [] + "values": [ + + ] }, "yaxes": [ { "format": "bytes", + "label": null, "logBase": 1, + "max": null, + "min": 0, "show": true }, { - "format": "short", + "format": "bytes", + "label": null, "logBase": 1, + "max": null, + "min": 0, "show": true } ] } ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, "showTitle": false, - "title": "New Row", - "titleSize": "h6" + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" } ], "schemaVersion": 14, - "sharedCrosshair": false, "style": "dark", - "tags": [], + "tags": [ + "kubernetes-mixin" + ], "templating": { "list": [ { - "allValue": ".*", - "current": {}, - "datasource": "prometheus", + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, "hide": 0, - "includeAll": true, + "label": null, + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, "label": "Namespace", "multi": false, "name": "namespace", - "options": [], + "options": [ + + ], "query": "label_values(kube_pod_info, namespace)", - "refresh": 1, + "refresh": 2, "regex": "", "sort": 0, "tagValuesQuery": "", - "tags": [], + "tags": [ + + ], "tagsQuery": "", "type": "query", "useTags": false }, { "allValue": null, - "current": {}, - "datasource": "prometheus", + "current": { + + }, + "datasource": "$datasource", "hide": 0, "includeAll": false, "label": "Pod", "multi": false, "name": "pod", - "options": [], + "options": [ + + ], "query": "label_values(kube_pod_info{namespace=~\"$namespace\"}, pod)", - "refresh": 1, + "refresh": 2, "regex": "", "sort": 0, "tagValuesQuery": "", - "tags": [], + "tags": [ + + ], "tagsQuery": "", "type": "query", "useTags": false }, { - "allValue": ".*", - "current": {}, - "datasource": "prometheus", + "allValue": null, + "current": { + + }, + "datasource": "$datasource", "hide": 0, "includeAll": true, "label": "Container", "multi": false, "name": "container", - "options": [], + "options": [ + + ], "query": "label_values(kube_pod_container_info{namespace=\"$namespace\", pod=\"$pod\"}, container)", - "refresh": 1, + "refresh": 2, "regex": "", "sort": 0, "tagValuesQuery": "", - "tags": [], + "tags": [ + + ], "tagsQuery": "", "type": "query", "useTags": false @@ -6616,7 +8196,7 @@ data: ] }, "time": { - "from": "now-6h", + "from": "now-1h", "to": "now" }, "timepicker": { @@ -6644,45 +8224,48 @@ data: "30d" ] }, - "timezone": "browser", - "title": "Pods", - "version": 1 + "timezone": "", + "title": "Kubernetes / Pods", + "uid": "ab4f13a9892a76a4d21ce8c2445bf4ea", + "version": 0 } - statefulset-dashboard.json: |+ + statefulset.json: |- { "__inputs": [ - { - "description": "", - "label": "prometheus", - "name": "prometheus", - "pluginId": "prometheus", - "pluginName": "Prometheus", - "type": "datasource" - } + + ], + "__requires": [ + ], "annotations": { - "list": [] + "list": [ + + ] }, "editable": false, - "graphTooltip": 1, + "gnetId": null, + "graphTooltip": 0, "hideControls": false, - "links": [], + "id": null, + "links": [ + + ], + "refresh": "", "rows": [ { "collapse": false, - "editable": false, - "height": "200px", + "collapsed": false, "panels": [ { + "cacheTimeout": null, "colorBackground": false, "colorValue": false, "colors": [ - "rgba(245, 54, 54, 0.9)", + "#299c46", "rgba(237, 129, 40, 0.89)", - "rgba(50, 172, 45, 0.97)" + "#d44a3a" ], - "datasource": "prometheus", - "editable": false, + "datasource": "$datasource", "format": "none", "gauge": { "maxValue": 100, @@ -6691,8 +8274,14 @@ data: "thresholdLabels": false, "thresholdMarkers": true }, - "id": 8, - "links": [], + "gridPos": { + + }, + "id": 2, + "interval": null, + "links": [ + + ], "mappingType": 1, "mappingTypes": [ { @@ -6706,6 +8295,7 @@ data: ], "maxDataPoints": 100, "nullPointMode": "connected", + "nullText": null, "postfix": "cores", "postfixFontSize": "50%", "prefix": "", @@ -6720,40 +8310,42 @@ data: "span": 4, "sparkline": { "fillColor": "rgba(31, 118, 189, 0.18)", - "full": false, "lineColor": "rgb(31, 120, 193)", "show": true }, + "tableColumn": "", "targets": [ { - "expr": "sum(rate(container_cpu_usage_seconds_total{namespace=\"$statefulset_namespace\",pod_name=~\"$statefulset_name.*\"}[3m]))", + "expr": "sum(rate(container_cpu_usage_seconds_total{job=\"kubernetes-cadvisor\", namespace=\"$namespace\", pod_name=~\"$statefulset.*\"}[3m]))", + "format": "time_series", "intervalFactor": 2, - "refId": "A", - "step": 600 + "legendFormat": "", + "refId": "A" } ], + "thresholds": "", "title": "CPU", "type": "singlestat", - "valueFontSize": "110%", + "valueFontSize": "80%", "valueMaps": [ { "op": "=", - "text": "N/A", + "text": "0", "value": "null" } ], - "valueName": "avg" + "valueName": "current" }, { + "cacheTimeout": null, "colorBackground": false, "colorValue": false, "colors": [ - "rgba(245, 54, 54, 0.9)", + "#299c46", "rgba(237, 129, 40, 0.89)", - "rgba(50, 172, 45, 0.97)" + "#d44a3a" ], - "datasource": "prometheus", - "editable": false, + "datasource": "$datasource", "format": "none", "gauge": { "maxValue": 100, @@ -6762,8 +8354,14 @@ data: "thresholdLabels": false, "thresholdMarkers": true }, - "id": 9, - "links": [], + "gridPos": { + + }, + "id": 3, + "interval": null, + "links": [ + + ], "mappingType": 1, "mappingTypes": [ { @@ -6777,10 +8375,11 @@ data: ], "maxDataPoints": 100, "nullPointMode": "connected", + "nullText": null, "postfix": "GB", "postfixFontSize": "50%", "prefix": "", - "prefixFontSize": "80%", + "prefixFontSize": "50%", "rangeMaps": [ { "from": "null", @@ -6791,50 +8390,58 @@ data: "span": 4, "sparkline": { "fillColor": "rgba(31, 118, 189, 0.18)", - "full": false, "lineColor": "rgb(31, 120, 193)", "show": true }, + "tableColumn": "", "targets": [ { - "expr": "sum(container_memory_usage_bytes{namespace=\"$statefulset_namespace\",pod_name=~\"$statefulset_name.*\"}) / 1024^3", + "expr": "sum(container_memory_usage_bytes{job=\"kubernetes-cadvisor\", namespace=\"$namespace\", pod_name=~\"$statefulset.*\"}) / 1024^3", + "format": "time_series", "intervalFactor": 2, - "refId": "A", - "step": 600 + "legendFormat": "", + "refId": "A" } ], + "thresholds": "", "title": "Memory", "type": "singlestat", - "valueFontSize": "110%", + "valueFontSize": "80%", "valueMaps": [ { "op": "=", - "text": "N/A", + "text": "0", "value": "null" } ], - "valueName": "avg" + "valueName": "current" }, { + "cacheTimeout": null, "colorBackground": false, "colorValue": false, "colors": [ - "rgba(245, 54, 54, 0.9)", + "#299c46", "rgba(237, 129, 40, 0.89)", - "rgba(50, 172, 45, 0.97)" + "#d44a3a" ], - "datasource": "prometheus", - "editable": false, - "format": "Bps", + "datasource": "$datasource", + "format": "none", "gauge": { "maxValue": 100, "minValue": 0, "show": false, "thresholdLabels": false, - "thresholdMarkers": false + "thresholdMarkers": true }, - "id": 7, - "links": [], + "gridPos": { + + }, + "id": 4, + "interval": null, + "links": [ + + ], "mappingType": 1, "mappingTypes": [ { @@ -6848,7 +8455,8 @@ data: ], "maxDataPoints": 100, "nullPointMode": "connected", - "postfix": "", + "nullText": null, + "postfix": "Bps", "postfixFontSize": "50%", "prefix": "", "prefixFontSize": "50%", @@ -6862,60 +8470,72 @@ data: "span": 4, "sparkline": { "fillColor": "rgba(31, 118, 189, 0.18)", - "full": false, "lineColor": "rgb(31, 120, 193)", "show": true }, + "tableColumn": "", "targets": [ { - "expr": "sum(rate(container_network_transmit_bytes_total{namespace=\"$statefulset_namespace\",pod_name=~\"$statefulset_name.*\"}[3m])) + sum(rate(container_network_receive_bytes_total{namespace=\"$statefulset_namespace\",pod_name=~\"$statefulset_name.*\"}[3m]))", + "expr": "sum(rate(container_network_transmit_bytes_total{job=\"kubernetes-cadvisor\", namespace=\"$namespace\", pod_name=~\"$statefulset.*\"}[3m])) + sum(rate(container_network_receive_bytes_total{namespace=\"$namespace\",pod_name=~\"$statefulset.*\"}[3m]))", + "format": "time_series", "intervalFactor": 2, - "refId": "A", - "step": 600 + "legendFormat": "", + "refId": "A" } ], + "thresholds": "", "title": "Network", "type": "singlestat", "valueFontSize": "80%", "valueMaps": [ { "op": "=", - "text": "N/A", + "text": "0", "value": "null" } ], - "valueName": "avg" + "valueName": "current" } ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, "showTitle": false, "title": "Dashboard Row", - "titleSize": "h6" + "titleSize": "h6", + "type": "row" }, { "collapse": false, - "editable": false, + "collapsed": false, "height": "100px", "panels": [ { + "cacheTimeout": null, "colorBackground": false, "colorValue": false, "colors": [ - "rgba(245, 54, 54, 0.9)", + "#299c46", "rgba(237, 129, 40, 0.89)", - "rgba(50, 172, 45, 0.97)" + "#d44a3a" ], - "datasource": "prometheus", - "editable": false, + "datasource": "$datasource", "format": "none", "gauge": { "maxValue": 100, "minValue": 0, "show": false, "thresholdLabels": false, - "thresholdMarkers": false + "thresholdMarkers": true + }, + "gridPos": { + }, "id": 5, - "links": [], + "interval": null, + "links": [ + + ], "mappingType": 1, "mappingTypes": [ { @@ -6929,6 +8549,8 @@ data: ], "maxDataPoints": 100, "nullPointMode": "connected", + "nullText": null, + "postfix": "", "postfixFontSize": "50%", "prefix": "", "prefixFontSize": "50%", @@ -6946,37 +8568,39 @@ data: "lineColor": "rgb(31, 120, 193)", "show": false }, + "tableColumn": "", "targets": [ { - "expr": "max(kube_statefulset_replicas{statefulset=\"$statefulset_name\",namespace=\"$statefulset_namespace\"}) without (instance, pod)", + "expr": "max(kube_statefulset_replicas{job=\"kube-state-metrics\", namespace=\"$namespace\", statefulset=\"$statefulset\"}) without (instance, pod)", + "format": "time_series", "intervalFactor": 2, - "metric": "kube_statefulset_replicas", - "refId": "A", - "step": 600 + "legendFormat": "", + "refId": "A" } ], + "thresholds": "", "title": "Desired Replicas", "type": "singlestat", "valueFontSize": "80%", "valueMaps": [ { "op": "=", - "text": "N/A", + "text": "0", "value": "null" } ], - "valueName": "avg" + "valueName": "current" }, { + "cacheTimeout": null, "colorBackground": false, "colorValue": false, "colors": [ - "rgba(245, 54, 54, 0.9)", + "#299c46", "rgba(237, 129, 40, 0.89)", - "rgba(50, 172, 45, 0.97)" + "#d44a3a" ], - "datasource": "prometheus", - "editable": false, + "datasource": "$datasource", "format": "none", "gauge": { "maxValue": 100, @@ -6984,9 +8608,15 @@ data: "show": false, "thresholdLabels": false, "thresholdMarkers": true + }, + "gridPos": { + }, "id": 6, - "links": [], + "interval": null, + "links": [ + + ], "mappingType": 1, "mappingTypes": [ { @@ -7000,6 +8630,8 @@ data: ], "maxDataPoints": 100, "nullPointMode": "connected", + "nullText": null, + "postfix": "", "postfixFontSize": "50%", "prefix": "", "prefixFontSize": "50%", @@ -7017,36 +8649,39 @@ data: "lineColor": "rgb(31, 120, 193)", "show": false }, + "tableColumn": "", "targets": [ { - "expr": "min(kube_statefulset_status_replicas{statefulset=\"$statefulset_name\",namespace=\"$statefulset_namespace\"}) without (instance, pod)", + "expr": "min(kube_statefulset_status_replicas_current{job=\"kube-state-metrics\", namespace=\"$namespace\", statefulset=\"$statefulset\"}) without (instance, pod)", + "format": "time_series", "intervalFactor": 2, - "refId": "A", - "step": 600 + "legendFormat": "", + "refId": "A" } ], - "title": "Available Replicas", + "thresholds": "", + "title": "Replicas of current version", "type": "singlestat", "valueFontSize": "80%", "valueMaps": [ { "op": "=", - "text": "N/A", + "text": "0", "value": "null" } ], - "valueName": "avg" + "valueName": "current" }, { + "cacheTimeout": null, "colorBackground": false, "colorValue": false, "colors": [ - "rgba(245, 54, 54, 0.9)", + "#299c46", "rgba(237, 129, 40, 0.89)", - "rgba(50, 172, 45, 0.97)" + "#d44a3a" ], - "datasource": "prometheus", - "editable": false, + "datasource": "$datasource", "format": "none", "gauge": { "maxValue": 100, @@ -7055,8 +8690,14 @@ data: "thresholdLabels": false, "thresholdMarkers": true }, - "id": 3, - "links": [], + "gridPos": { + + }, + "id": 7, + "interval": null, + "links": [ + + ], "mappingType": 1, "mappingTypes": [ { @@ -7070,6 +8711,8 @@ data: ], "maxDataPoints": 100, "nullPointMode": "connected", + "nullText": null, + "postfix": "", "postfixFontSize": "50%", "prefix": "", "prefixFontSize": "50%", @@ -7087,36 +8730,39 @@ data: "lineColor": "rgb(31, 120, 193)", "show": false }, + "tableColumn": "", "targets": [ { - "expr": "max(kube_statefulset_status_observed_generation{statefulset=\"$statefulset_name\",namespace=\"$statefulset_namespace\"}) without (instance, pod)", + "expr": "max(kube_statefulset_status_observed_generation{job=\"kube-state-metrics\", namespace=\"$namespace\", statefulset=\"$statefulset\"}) without (instance, pod)", + "format": "time_series", "intervalFactor": 2, - "refId": "A", - "step": 600 + "legendFormat": "", + "refId": "A" } ], + "thresholds": "", "title": "Observed Generation", "type": "singlestat", "valueFontSize": "80%", "valueMaps": [ { "op": "=", - "text": "N/A", + "text": "0", "value": "null" } ], - "valueName": "avg" + "valueName": "current" }, { + "cacheTimeout": null, "colorBackground": false, "colorValue": false, "colors": [ - "rgba(245, 54, 54, 0.9)", + "#299c46", "rgba(237, 129, 40, 0.89)", - "rgba(50, 172, 45, 0.97)" + "#d44a3a" ], - "datasource": "prometheus", - "editable": false, + "datasource": "$datasource", "format": "none", "gauge": { "maxValue": 100, @@ -7125,8 +8771,14 @@ data: "thresholdLabels": false, "thresholdMarkers": true }, - "id": 2, - "links": [], + "gridPos": { + + }, + "id": 8, + "interval": null, + "links": [ + + ], "mappingType": 1, "mappingTypes": [ { @@ -7140,6 +8792,8 @@ data: ], "maxDataPoints": 100, "nullPointMode": "connected", + "nullText": null, + "postfix": "", "postfixFontSize": "50%", "prefix": "", "prefixFontSize": "50%", @@ -7157,176 +8811,249 @@ data: "lineColor": "rgb(31, 120, 193)", "show": false }, + "tableColumn": "", "targets": [ { - "expr": "max(kube_statefulset_metadata_generation{statefulset=\"$statefulset_name\",namespace=\"$statefulset_namespace\"}) without (instance, pod)", + "expr": "max(kube_statefulset_metadata_generation{job=\"kube-state-metrics\", statefulset=\"$statefulset\", namespace=\"$namespace\"}) without (instance, pod)", + "format": "time_series", "intervalFactor": 2, - "refId": "A", - "step": 600 + "legendFormat": "", + "refId": "A" } ], + "thresholds": "", "title": "Metadata Generation", "type": "singlestat", "valueFontSize": "80%", "valueMaps": [ { "op": "=", - "text": "N/A", + "text": "0", "value": "null" } ], - "valueName": "avg" + "valueName": "current" } ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, "showTitle": false, "title": "Dashboard Row", - "titleSize": "h6" + "titleSize": "h6", + "type": "row" }, { "collapse": false, - "editable": false, - "height": "350px", + "collapsed": false, "panels": [ { - "aliasColors": {}, + "aliasColors": { + + }, "bars": false, "dashLength": 10, "dashes": false, - "datasource": "prometheus", - "editable": false, - "error": false, + "datasource": "$datasource", "fill": 1, - "grid": { - "threshold1Color": "rgba(216, 200, 27, 0.27)", - "threshold2Color": "rgba(234, 112, 112, 0.22)" + "gridPos": { + }, - "id": 1, - "isNew": true, + "id": 9, "legend": { "alignAsTable": false, "avg": false, "current": false, - "hideEmpty": false, - "hideZero": false, "max": false, "min": false, "rightSide": false, "show": true, - "total": false + "total": false, + "values": false }, "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "connected", + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", - "seriesOverrides": [], + "repeat": null, + "seriesOverrides": [ + + ], "spaceLength": 10, - "span": 12, "stack": false, "steppedLine": false, "targets": [ { - "expr": "min(kube_statefulset_status_replicas{statefulset=\"$statefulset_name\",namespace=\"$statefulset_namespace\"}) without (instance, pod)", + "expr": "max(kube_statefulset_replicas{job=\"kube-state-metrics\", statefulset=\"$statefulset\",namespace=\"$namespace\"}) without (instance, pod)", + "format": "time_series", "intervalFactor": 2, - "legendFormat": "available", - "refId": "B", - "step": 30 + "legendFormat": "replicas specified", + "refId": "A" }, { - "expr": "max(kube_statefulset_replicas{statefulset=\"$statefulset_name\",namespace=\"$statefulset_namespace\"}) without (instance, pod)", + "expr": "max(kube_statefulset_status_replicas{job=\"kube-state-metrics\", statefulset=\"$statefulset\",namespace=\"$namespace\"}) without (instance, pod)", + "format": "time_series", "intervalFactor": 2, - "legendFormat": "desired", - "refId": "E", - "step": 30 + "legendFormat": "replicas created", + "refId": "B" + }, + { + "expr": "min(kube_statefulset_status_replicas_ready{job=\"kube-state-metrics\", statefulset=\"$statefulset\",namespace=\"$namespace\"}) without (instance, pod)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "ready", + "refId": "C" + }, + { + "expr": "min(kube_statefulset_status_replicas_current{job=\"kube-state-metrics\", statefulset=\"$statefulset\",namespace=\"$namespace\"}) without (instance, pod)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "replicas of current version", + "refId": "D" + }, + { + "expr": "min(kube_statefulset_status_replicas_updated{job=\"kube-state-metrics\", statefulset=\"$statefulset\",namespace=\"$namespace\"}) without (instance, pod)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "updated", + "refId": "E" } ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, "title": "Replicas", "tooltip": { - "msResolution": true, "shared": true, "sort": 0, - "value_type": "cumulative" + "value_type": "individual" }, "type": "graph", "xaxis": { + "buckets": null, "mode": "time", + "name": null, "show": true, - "values": [] + "values": [ + + ] }, "yaxes": [ { - "format": "none", - "label": "", + "format": "short", + "label": null, "logBase": 1, + "max": null, + "min": null, "show": true }, { "format": "short", - "label": "", + "label": null, "logBase": 1, - "show": false + "max": null, + "min": null, + "show": true } ] } ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, "showTitle": false, "title": "Dashboard Row", - "titleSize": "h6" + "titleSize": "h6", + "type": "row" } ], "schemaVersion": 14, - "sharedCrosshair": false, "style": "dark", - "tags": [], + "tags": [ + "kubernetes-mixin" + ], "templating": { "list": [ { - "allValue": ".*", - "current": {}, - "datasource": "prometheus", + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": null, + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + + }, + "datasource": "$datasource", "hide": 0, "includeAll": false, "label": "Namespace", "multi": false, - "name": "statefulset_namespace", - "options": [], - "query": "label_values(kube_statefulset_metadata_generation, namespace)", - "refresh": 1, + "name": "namespace", + "options": [ + + ], + "query": "label_values(kube_statefulset_metadata_generation{job=\"kube-state-metrics\"}, namespace)", + "refresh": 2, "regex": "", "sort": 0, - "tagValuesQuery": null, - "tags": [], + "tagValuesQuery": "", + "tags": [ + + ], "tagsQuery": "", "type": "query", "useTags": false }, { "allValue": null, - "current": {}, - "datasource": "prometheus", + "current": { + + }, + "datasource": "$datasource", "hide": 0, "includeAll": false, - "label": "StatefulSet", + "label": "Name", "multi": false, - "name": "statefulset_name", - "options": [], - "query": "label_values(kube_statefulset_metadata_generation{namespace=\"$statefulset_namespace\"}, statefulset)", - "refresh": 1, + "name": "statefulset", + "options": [ + + ], + "query": "label_values(kube_statefulset_metadata_generation{job=\"kube-state-metrics\", namespace=\"$namespace\"}, statefulset)", + "refresh": 2, "regex": "", "sort": 0, "tagValuesQuery": "", - "tags": [], - "tagsQuery": "statefulset", + "tags": [ + + ], + "tagsQuery": "", "type": "query", "useTags": false } ] }, "time": { - "from": "now-6h", + "from": "now-1h", "to": "now" }, "timepicker": { @@ -7354,8 +9081,8 @@ data: "30d" ] }, - "timezone": "browser", - "title": "StatefulSet", - "version": 1 + "timezone": "", + "title": "Kubernetes / StatefulSets", + "uid": "a31c1f46e6f727cb37c0d731a7245005", + "version": 0 } ---- diff --git a/addons/prometheus/rules.yaml b/addons/prometheus/rules.yaml index 618cd4288..f88d94c06 100644 --- a/addons/prometheus/rules.yaml +++ b/addons/prometheus/rules.yaml @@ -4,582 +4,1089 @@ metadata: name: prometheus-rules namespace: monitoring data: - alertmanager.rules.yaml: | - groups: - - name: alertmanager.rules - rules: - - alert: AlertmanagerConfigInconsistent - expr: count_values("config_hash", alertmanager_config_hash) BY (service) / ON(service) - GROUP_LEFT() label_replace(prometheus_operator_alertmanager_spec_replicas, "service", - "alertmanager-$1", "alertmanager", "(.*)") != 1 - for: 5m - labels: - severity: critical - annotations: - description: The configuration of the instances of the Alertmanager cluster - `{{$labels.service}}` are out of sync. - - alert: AlertmanagerDownOrMissing - expr: label_replace(prometheus_operator_alertmanager_spec_replicas, "job", "alertmanager-$1", - "alertmanager", "(.*)") / ON(job) GROUP_RIGHT() sum(up) BY (job) != 1 - for: 5m - labels: - severity: warning - annotations: - description: An unexpected number of Alertmanagers are scraped or Alertmanagers - disappeared from discovery. - - alert: AlertmanagerFailedReload - expr: alertmanager_config_last_reload_successful == 0 - for: 10m - labels: - severity: warning - annotations: - description: Reloading Alertmanager's configuration has failed for {{ $labels.namespace - }}/{{ $labels.pod}}. - etcd3.rules.yaml: | - groups: - - name: ./etcd3.rules - rules: - - alert: InsufficientMembers - expr: count(up{job="etcd"} == 0) > (count(up{job="etcd"}) / 2 - 1) - for: 3m - labels: - severity: critical - annotations: - description: If one more etcd member goes down the cluster will be unavailable - summary: etcd cluster insufficient members - - alert: NoLeader - expr: etcd_server_has_leader{job="etcd"} == 0 - for: 1m - labels: - severity: critical - annotations: - description: etcd member {{ $labels.instance }} has no leader - summary: etcd member has no leader - - alert: HighNumberOfLeaderChanges - expr: increase(etcd_server_leader_changes_seen_total{job="etcd"}[1h]) > 3 - labels: - severity: warning - annotations: - description: etcd instance {{ $labels.instance }} has seen {{ $value }} leader - changes within the last hour - summary: a high number of leader changes within the etcd cluster are happening - - alert: GRPCRequestsSlow - expr: histogram_quantile(0.99, sum(rate(grpc_server_handling_seconds_bucket{job="etcd",grpc_type="unary"}[5m])) by (grpc_service, grpc_method, le)) - > 0.15 - for: 10m - labels: - severity: critical - annotations: - description: on etcd instance {{ $labels.instance }} gRPC requests to {{ $labels.grpc_method - }} are slow - summary: slow gRPC requests - - alert: HighNumberOfFailedHTTPRequests - expr: sum(rate(etcd_http_failed_total{job="etcd"}[5m])) BY (method) / sum(rate(etcd_http_received_total{job="etcd"}[5m])) - BY (method) > 0.01 - for: 10m - labels: - severity: warning - annotations: - description: '{{ $value }}% of requests for {{ $labels.method }} failed on etcd - instance {{ $labels.instance }}' - summary: a high number of HTTP requests are failing - - alert: HighNumberOfFailedHTTPRequests - expr: sum(rate(etcd_http_failed_total{job="etcd"}[5m])) BY (method) / sum(rate(etcd_http_received_total{job="etcd"}[5m])) - BY (method) > 0.05 - for: 5m - labels: - severity: critical - annotations: - description: '{{ $value }}% of requests for {{ $labels.method }} failed on etcd - instance {{ $labels.instance }}' - summary: a high number of HTTP requests are failing - - alert: HTTPRequestsSlow - expr: histogram_quantile(0.99, rate(etcd_http_successful_duration_seconds_bucket[5m])) - > 0.15 - for: 10m - labels: - severity: warning - annotations: - description: on etcd instance {{ $labels.instance }} HTTP requests to {{ $labels.method - }} are slow - summary: slow HTTP requests - - alert: EtcdMemberCommunicationSlow - expr: histogram_quantile(0.99, rate(etcd_network_peer_round_trip_time_seconds_bucket[5m])) - > 0.15 - for: 10m - labels: - severity: warning - annotations: - description: etcd instance {{ $labels.instance }} member communication with - {{ $labels.To }} is slow - summary: etcd member communication is slow - - alert: HighNumberOfFailedProposals - expr: increase(etcd_server_proposals_failed_total{job="etcd"}[1h]) > 5 - labels: - severity: warning - annotations: - description: etcd instance {{ $labels.instance }} has seen {{ $value }} proposal - failures within the last hour - summary: a high number of proposals within the etcd cluster are failing - - alert: HighFsyncDurations - expr: histogram_quantile(0.99, rate(etcd_disk_wal_fsync_duration_seconds_bucket[5m])) - > 0.5 - for: 10m - labels: - severity: warning - annotations: - description: etcd instance {{ $labels.instance }} fync durations are high - summary: high fsync durations - - alert: HighCommitDurations - expr: histogram_quantile(0.99, rate(etcd_disk_backend_commit_duration_seconds_bucket[5m])) - > 0.25 - for: 10m - labels: - severity: warning - annotations: - description: etcd instance {{ $labels.instance }} commit durations are high - summary: high commit durations - general.rules.yaml: | - groups: - - name: general.rules - rules: - - alert: TargetDown - expr: 100 * (count(up == 0) BY (job) / count(up) BY (job)) > 10 - for: 10m - labels: - severity: warning - annotations: - description: '{{ $value }}% of {{ $labels.job }} targets are down.' - summary: Targets are down - - record: fd_utilization - expr: process_open_fds / process_max_fds - - alert: FdExhaustionClose - expr: predict_linear(fd_utilization[1h], 3600 * 4) > 1 - for: 10m - labels: - severity: warning - annotations: - description: '{{ $labels.job }}: {{ $labels.namespace }}/{{ $labels.pod }} instance - will exhaust in file/socket descriptors within the next 4 hours' - summary: file descriptors soon exhausted - - alert: FdExhaustionClose - expr: predict_linear(fd_utilization[10m], 3600) > 1 - for: 10m - labels: - severity: critical - annotations: - description: '{{ $labels.job }}: {{ $labels.namespace }}/{{ $labels.pod }} instance - will exhaust in file/socket descriptors within the next hour' - summary: file descriptors soon exhausted - kube-controller-manager.rules.yaml: | - groups: - - name: kube-controller-manager.rules - rules: - - alert: K8SControllerManagerDown - expr: absent(up{job="kube-controller-manager"} == 1) - for: 5m - labels: - severity: critical - annotations: - description: There is no running K8S controller manager. Deployments and replication - controllers are not making progress. - summary: Controller manager is down - kube-scheduler.rules.yaml: | - groups: - - name: kube-scheduler.rules - rules: - - record: cluster:scheduler_e2e_scheduling_latency_seconds:quantile - expr: histogram_quantile(0.99, sum(scheduler_e2e_scheduling_latency_microseconds_bucket) - BY (le, cluster)) / 1e+06 - labels: - quantile: "0.99" - - record: cluster:scheduler_e2e_scheduling_latency_seconds:quantile - expr: histogram_quantile(0.9, sum(scheduler_e2e_scheduling_latency_microseconds_bucket) - BY (le, cluster)) / 1e+06 - labels: - quantile: "0.9" - - record: cluster:scheduler_e2e_scheduling_latency_seconds:quantile - expr: histogram_quantile(0.5, sum(scheduler_e2e_scheduling_latency_microseconds_bucket) - BY (le, cluster)) / 1e+06 - labels: - quantile: "0.5" - - record: cluster:scheduler_scheduling_algorithm_latency_seconds:quantile - expr: histogram_quantile(0.99, sum(scheduler_scheduling_algorithm_latency_microseconds_bucket) - BY (le, cluster)) / 1e+06 - labels: - quantile: "0.99" - - record: cluster:scheduler_scheduling_algorithm_latency_seconds:quantile - expr: histogram_quantile(0.9, sum(scheduler_scheduling_algorithm_latency_microseconds_bucket) - BY (le, cluster)) / 1e+06 - labels: - quantile: "0.9" - - record: cluster:scheduler_scheduling_algorithm_latency_seconds:quantile - expr: histogram_quantile(0.5, sum(scheduler_scheduling_algorithm_latency_microseconds_bucket) - BY (le, cluster)) / 1e+06 - labels: - quantile: "0.5" - - record: cluster:scheduler_binding_latency_seconds:quantile - expr: histogram_quantile(0.99, sum(scheduler_binding_latency_microseconds_bucket) - BY (le, cluster)) / 1e+06 - labels: - quantile: "0.99" - - record: cluster:scheduler_binding_latency_seconds:quantile - expr: histogram_quantile(0.9, sum(scheduler_binding_latency_microseconds_bucket) - BY (le, cluster)) / 1e+06 - labels: - quantile: "0.9" - - record: cluster:scheduler_binding_latency_seconds:quantile - expr: histogram_quantile(0.5, sum(scheduler_binding_latency_microseconds_bucket) - BY (le, cluster)) / 1e+06 - labels: - quantile: "0.5" - - alert: K8SSchedulerDown - expr: absent(up{job="kube-scheduler"} == 1) - for: 5m - labels: - severity: critical - annotations: - description: There is no running K8S scheduler. New pods are not being assigned - to nodes. - summary: Scheduler is down - kube-state-metrics.rules.yaml: | - groups: - - name: kube-state-metrics.rules - rules: - - alert: DeploymentGenerationMismatch - expr: kube_deployment_status_observed_generation != kube_deployment_metadata_generation - for: 15m - labels: - severity: warning - annotations: - description: Observed deployment generation does not match expected one for - deployment {{$labels.namespaces}}/{{$labels.deployment}} - summary: Deployment is outdated - - alert: DeploymentReplicasNotUpdated - expr: ((kube_deployment_status_replicas_updated != kube_deployment_spec_replicas) - or (kube_deployment_status_replicas_available != kube_deployment_spec_replicas)) - unless (kube_deployment_spec_paused == 1) - for: 15m - labels: - severity: warning - annotations: - description: Replicas are not updated and available for deployment {{$labels.namespaces}}/{{$labels.deployment}} - summary: Deployment replicas are outdated - - alert: DaemonSetRolloutStuck - expr: kube_daemonset_status_number_ready / kube_daemonset_status_desired_number_scheduled - * 100 < 100 - for: 15m - labels: - severity: warning - annotations: - description: Only {{$value}}% of desired pods scheduled and ready for daemon - set {{$labels.namespaces}}/{{$labels.daemonset}} - summary: DaemonSet is missing pods - - alert: K8SDaemonSetsNotScheduled - expr: kube_daemonset_status_desired_number_scheduled - kube_daemonset_status_current_number_scheduled - > 0 - for: 10m - labels: - severity: warning - annotations: - description: A number of daemonsets are not scheduled. - summary: Daemonsets are not scheduled correctly - - alert: DaemonSetsMissScheduled - expr: kube_daemonset_status_number_misscheduled > 0 - for: 10m - labels: - severity: warning - annotations: - description: A number of daemonsets are running where they are not supposed - to run. - summary: Daemonsets are not scheduled correctly - - alert: PodFrequentlyRestarting - expr: increase(kube_pod_container_status_restarts_total[1h]) > 5 - for: 10m - labels: - severity: warning - annotations: - description: Pod {{$labels.namespaces}}/{{$labels.pod}} restarted {{$value}} - times within the last hour - summary: Pod is restarting frequently - kubelet.rules.yaml: | - groups: - - name: kubelet.rules - rules: - - alert: K8SNodeNotReady - expr: kube_node_status_condition{condition="Ready",status="true"} == 0 - for: 1h - labels: - severity: warning - annotations: - description: The Kubelet on {{ $labels.node }} has not checked in with the API, - or has set itself to NotReady, for more than an hour - summary: Node status is NotReady - - alert: K8SManyNodesNotReady - expr: count(kube_node_status_condition{condition="Ready",status="true"} == 0) - > 1 and (count(kube_node_status_condition{condition="Ready",status="true"} == - 0) / count(kube_node_status_condition{condition="Ready",status="true"})) > 0.2 - for: 1m - labels: - severity: critical - annotations: - description: '{{ $value }}% of Kubernetes nodes are not ready' - - alert: K8SKubeletDown - expr: count(up{job="kubelet"} == 0) / count(up{job="kubelet"}) * 100 > 3 - for: 1h - labels: - severity: warning - annotations: - description: Prometheus failed to scrape {{ $value }}% of kubelets. - - alert: K8SKubeletDown - expr: (absent(up{job="kubelet"} == 1) or count(up{job="kubelet"} == 0) / count(up{job="kubelet"})) - * 100 > 10 - for: 1h - labels: - severity: critical - annotations: - description: Prometheus failed to scrape {{ $value }}% of kubelets, or all Kubelets - have disappeared from service discovery. - summary: Many Kubelets cannot be scraped - - alert: K8SKubeletTooManyPods - expr: kubelet_running_pod_count > 100 - for: 10m - labels: - severity: warning - annotations: - description: Kubelet {{$labels.instance}} is running {{$value}} pods, close - to the limit of 110 - summary: Kubelet is close to pod limit - kubernetes.rules.yaml: | - groups: - - name: kubernetes.rules - rules: - - record: pod_name:container_memory_usage_bytes:sum - expr: sum(container_memory_usage_bytes{container_name!="POD",pod_name!=""}) BY - (pod_name) - - record: pod_name:container_spec_cpu_shares:sum - expr: sum(container_spec_cpu_shares{container_name!="POD",pod_name!=""}) BY (pod_name) - - record: pod_name:container_cpu_usage:sum - expr: sum(rate(container_cpu_usage_seconds_total{container_name!="POD",pod_name!=""}[5m])) - BY (pod_name) - - record: pod_name:container_fs_usage_bytes:sum - expr: sum(container_fs_usage_bytes{container_name!="POD",pod_name!=""}) BY (pod_name) - - record: namespace:container_memory_usage_bytes:sum - expr: sum(container_memory_usage_bytes{container_name!=""}) BY (namespace) - - record: namespace:container_spec_cpu_shares:sum - expr: sum(container_spec_cpu_shares{container_name!=""}) BY (namespace) - - record: namespace:container_cpu_usage:sum - expr: sum(rate(container_cpu_usage_seconds_total{container_name!="POD"}[5m])) - BY (namespace) - - record: cluster:memory_usage:ratio - expr: sum(container_memory_usage_bytes{container_name!="POD",pod_name!=""}) BY - (cluster) / sum(machine_memory_bytes) BY (cluster) - - record: cluster:container_spec_cpu_shares:ratio - expr: sum(container_spec_cpu_shares{container_name!="POD",pod_name!=""}) / 1000 - / sum(machine_cpu_cores) - - record: cluster:container_cpu_usage:ratio - expr: sum(rate(container_cpu_usage_seconds_total{container_name!="POD",pod_name!=""}[5m])) - / sum(machine_cpu_cores) - - record: apiserver_latency_seconds:quantile - expr: histogram_quantile(0.99, rate(apiserver_request_latencies_bucket[5m])) / - 1e+06 - labels: - quantile: "0.99" - - record: apiserver_latency:quantile_seconds - expr: histogram_quantile(0.9, rate(apiserver_request_latencies_bucket[5m])) / - 1e+06 - labels: - quantile: "0.9" - - record: apiserver_latency_seconds:quantile - expr: histogram_quantile(0.5, rate(apiserver_request_latencies_bucket[5m])) / - 1e+06 - labels: - quantile: "0.5" - - alert: APIServerLatencyHigh - expr: apiserver_latency_seconds:quantile{quantile="0.99",subresource!="log",verb!~"^(?:WATCH|WATCHLIST|PROXY|CONNECT)$"} - > 1 - for: 10m - labels: - severity: warning - annotations: - description: the API server has a 99th percentile latency of {{ $value }} seconds - for {{$labels.verb}} {{$labels.resource}} - - alert: APIServerLatencyHigh - expr: apiserver_latency_seconds:quantile{quantile="0.99",subresource!="log",verb!~"^(?:WATCH|WATCHLIST|PROXY|CONNECT)$"} - > 4 - for: 10m - labels: - severity: critical - annotations: - description: the API server has a 99th percentile latency of {{ $value }} seconds - for {{$labels.verb}} {{$labels.resource}} - - alert: APIServerErrorsHigh - expr: rate(apiserver_request_count{code=~"^(?:5..)$"}[5m]) / rate(apiserver_request_count[5m]) - * 100 > 2 - for: 10m - labels: - severity: warning - annotations: - description: API server returns errors for {{ $value }}% of requests - - alert: APIServerErrorsHigh - expr: rate(apiserver_request_count{code=~"^(?:5..)$"}[5m]) / rate(apiserver_request_count[5m]) - * 100 > 5 - for: 10m - labels: - severity: critical - annotations: - description: API server returns errors for {{ $value }}% of requests - - alert: K8SApiserverDown - expr: absent(up{job="apiserver"} == 1) - for: 20m - labels: - severity: critical - annotations: - description: No API servers are reachable or all have disappeared from service - discovery - - - alert: K8sCertificateExpirationNotice - labels: - severity: warning - annotations: - description: Kubernetes API Certificate is expiring soon (less than 7 days) - expr: sum(apiserver_client_certificate_expiration_seconds_bucket{le="604800"}) > 0 - - - alert: K8sCertificateExpirationNotice - labels: - severity: critical - annotations: - description: Kubernetes API Certificate is expiring in less than 1 day - expr: sum(apiserver_client_certificate_expiration_seconds_bucket{le="86400"}) > 0 - node.rules.yaml: | - groups: - - name: node.rules - rules: - - record: instance:node_cpu:rate:sum - expr: sum(rate(node_cpu_seconds_total{mode!="idle",mode!="iowait",mode!~"^(?:guest.*)$"}[3m])) - BY (instance) - - record: instance:node_filesystem_usage:sum - expr: sum((node_filesystem_size_bytes{mountpoint="/"} - node_filesystem_free_bytes{mountpoint="/"})) - BY (instance) - - record: instance:node_network_receive_bytes:rate:sum - expr: sum(rate(node_network_receive_bytes_total[3m])) BY (instance) - - record: instance:node_network_transmit_bytes:rate:sum - expr: sum(rate(node_network_transmit_bytes_total[3m])) BY (instance) - - record: instance:node_cpu:ratio - expr: sum(rate(node_cpu_seconds_total{mode!="idle"}[5m])) WITHOUT (cpu, mode) / ON(instance) - GROUP_LEFT() count(sum(node_cpu_seconds_total) BY (instance, cpu)) BY (instance) - - record: cluster:node_cpu:sum_rate5m - expr: sum(rate(node_cpu_seconds_total{mode!="idle"}[5m])) - - record: cluster:node_cpu:ratio - expr: cluster:node_cpu:sum_rate5m / count(sum(node_cpu_seconds_total) BY (instance, cpu)) - - alert: NodeExporterDown - expr: absent(up{job="node-exporter"} == 1) - for: 10m - labels: - severity: warning - annotations: - description: Prometheus could not scrape a node-exporter for more than 10m, - or node-exporters have disappeared from discovery - - alert: NodeDiskRunningFull - expr: predict_linear(node_filesystem_free_bytes[6h], 3600 * 24) < 0 - for: 30m - labels: - severity: warning - annotations: - description: device {{$labels.device}} on node {{$labels.instance}} is running - full within the next 24 hours (mounted at {{$labels.mountpoint}}) - - alert: NodeDiskRunningFull - expr: predict_linear(node_filesystem_free_bytes[30m], 3600 * 2) < 0 - for: 10m - labels: - severity: critical - annotations: - description: device {{$labels.device}} on node {{$labels.instance}} is running - full within the next 2 hours (mounted at {{$labels.mountpoint}}) - - alert: InactiveRAIDDisk - expr: node_md_disks - node_md_disks_active > 0 - for: 10m - labels: - severity: warning - annotations: - description: '{{$value}} RAID disk(s) on node {{$labels.instance}} are inactive' - prometheus.rules.yaml: | - groups: - - name: prometheus.rules - rules: - - alert: PrometheusConfigReloadFailed - expr: prometheus_config_last_reload_successful == 0 - for: 10m - labels: - severity: warning - annotations: - description: Reloading Prometheus' configuration has failed for {{$labels.namespace}}/{{$labels.pod}} - - alert: PrometheusNotificationQueueRunningFull - expr: predict_linear(prometheus_notifications_queue_length[5m], 60 * 30) > prometheus_notifications_queue_capacity - for: 10m - labels: - severity: warning - annotations: - description: Prometheus' alert notification queue is running full for {{$labels.namespace}}/{{ - $labels.pod}} - - alert: PrometheusErrorSendingAlerts - expr: rate(prometheus_notifications_errors_total[5m]) / rate(prometheus_notifications_sent_total[5m]) - > 0.01 - for: 10m - labels: - severity: warning - annotations: - description: Errors while sending alerts from Prometheus {{$labels.namespace}}/{{ - $labels.pod}} to Alertmanager {{$labels.Alertmanager}} - - alert: PrometheusErrorSendingAlerts - expr: rate(prometheus_notifications_errors_total[5m]) / rate(prometheus_notifications_sent_total[5m]) - > 0.03 - for: 10m - labels: - severity: critical - annotations: - description: Errors while sending alerts from Prometheus {{$labels.namespace}}/{{ - $labels.pod}} to Alertmanager {{$labels.Alertmanager}} - - alert: PrometheusNotConnectedToAlertmanagers - expr: prometheus_notifications_alertmanagers_discovered < 1 - for: 10m - labels: - severity: warning - annotations: - description: Prometheus {{ $labels.namespace }}/{{ $labels.pod}} is not connected - to any Alertmanagers - - alert: PrometheusTSDBReloadsFailing - expr: increase(prometheus_tsdb_reloads_failures_total[2h]) > 0 - for: 12h - labels: - severity: warning - annotations: - description: '{{$labels.job}} at {{$labels.instance}} had {{$value | humanize}} - reload failures over the last four hours.' - summary: Prometheus has issues reloading data blocks from disk - - alert: PrometheusTSDBCompactionsFailing - expr: increase(prometheus_tsdb_compactions_failed_total[2h]) > 0 - for: 12h - labels: - severity: warning - annotations: - description: '{{$labels.job}} at {{$labels.instance}} had {{$value | humanize}} - compaction failures over the last four hours.' - summary: Prometheus has issues compacting sample blocks - - alert: PrometheusTSDBWALCorruptions - expr: tsdb_wal_corruptions_total > 0 - for: 4h - labels: - severity: warning - annotations: - description: '{{$labels.job}} at {{$labels.instance}} has a corrupted write-ahead - log (WAL).' - summary: Prometheus write-ahead log is corrupted - - alert: PrometheusNotIngestingSamples - expr: rate(prometheus_tsdb_head_samples_appended_total[5m]) <= 0 - for: 10m - labels: - severity: warning - annotations: - description: "Prometheus {{ $labels.namespace }}/{{ $labels.pod}} isn't ingesting samples." - summary: "Prometheus isn't ingesting samples" + etcd.yaml: |- + { + "groups": [ + { + "name": "etcd", + "rules": [ + { + "alert": "etcdInsufficientMembers", + "annotations": { + "message": "etcd cluster \"{{ $labels.job }}\": insufficient members ({{ $value }})." + }, + "expr": "sum(up{job=~\".*etcd.*\"} == bool 1) by (job) < ((count(up{job=~\".*etcd.*\"}) by (job) + 1) / 2)\n", + "for": "3m", + "labels": { + "severity": "critical" + } + }, + { + "alert": "etcdNoLeader", + "annotations": { + "message": "etcd cluster \"{{ $labels.job }}\": member {{ $labels.instance }} has no leader." + }, + "expr": "etcd_server_has_leader{job=~\".*etcd.*\"} == 0\n", + "for": "1m", + "labels": { + "severity": "critical" + } + }, + { + "alert": "etcdHighNumberOfLeaderChanges", + "annotations": { + "message": "etcd cluster \"{{ $labels.job }}\": instance {{ $labels.instance }} has seen {{ $value }} leader changes within the last 30 minutes." + }, + "expr": "rate(etcd_server_leader_changes_seen_total{job=~\".*etcd.*\"}[15m]) > 3\n", + "for": "15m", + "labels": { + "severity": "warning" + } + }, + { + "alert": "etcdGRPCRequestsSlow", + "annotations": { + "message": "etcd cluster \"{{ $labels.job }}\": gRPC requests to {{ $labels.grpc_method }} are taking {{ $value }}s on etcd instance {{ $labels.instance }}." + }, + "expr": "histogram_quantile(0.99, sum(rate(grpc_server_handling_seconds_bucket{job=~\".*etcd.*\", grpc_type=\"unary\"}[5m])) by (job, instance, grpc_service, grpc_method, le))\n> 0.15\n", + "for": "10m", + "labels": { + "severity": "critical" + } + }, + { + "alert": "etcdMemberCommunicationSlow", + "annotations": { + "message": "etcd cluster \"{{ $labels.job }}\": member communication with {{ $labels.To }} is taking {{ $value }}s on etcd instance {{ $labels.instance }}." + }, + "expr": "histogram_quantile(0.99, rate(etcd_network_peer_round_trip_time_seconds_bucket{job=~\".*etcd.*\"}[5m]))\n> 0.15\n", + "for": "10m", + "labels": { + "severity": "warning" + } + }, + { + "alert": "etcdHighNumberOfFailedProposals", + "annotations": { + "message": "etcd cluster \"{{ $labels.job }}\": {{ $value }} proposal failures within the last 30 minutes on etcd instance {{ $labels.instance }}." + }, + "expr": "rate(etcd_server_proposals_failed_total{job=~\".*etcd.*\"}[15m]) > 5\n", + "for": "15m", + "labels": { + "severity": "warning" + } + }, + { + "alert": "etcdHighFsyncDurations", + "annotations": { + "message": "etcd cluster \"{{ $labels.job }}\": 99th percentile fync durations are {{ $value }}s on etcd instance {{ $labels.instance }}." + }, + "expr": "histogram_quantile(0.99, rate(etcd_disk_wal_fsync_duration_seconds_bucket{job=~\".*etcd.*\"}[5m]))\n> 0.5\n", + "for": "10m", + "labels": { + "severity": "warning" + } + }, + { + "alert": "etcdHighCommitDurations", + "annotations": { + "message": "etcd cluster \"{{ $labels.job }}\": 99th percentile commit durations {{ $value }}s on etcd instance {{ $labels.instance }}." + }, + "expr": "histogram_quantile(0.99, rate(etcd_disk_backend_commit_duration_seconds_bucket{job=~\".*etcd.*\"}[5m]))\n> 0.25\n", + "for": "10m", + "labels": { + "severity": "warning" + } + }, + { + "alert": "etcdHighNumberOfFailedHTTPRequests", + "annotations": { + "message": "{{ $value }}% of requests for {{ $labels.method }} failed on etcd instance {{ $labels.instance }}" + }, + "expr": "sum(rate(etcd_http_failed_total{job=~\".*etcd.*\", code!=\"404\"}[5m])) BY (method) / sum(rate(etcd_http_received_total{job=~\".*etcd.*\"}[5m]))\nBY (method) > 0.01\n", + "for": "10m", + "labels": { + "severity": "warning" + } + }, + { + "alert": "etcdHighNumberOfFailedHTTPRequests", + "annotations": { + "message": "{{ $value }}% of requests for {{ $labels.method }} failed on etcd instance {{ $labels.instance }}." + }, + "expr": "sum(rate(etcd_http_failed_total{job=~\".*etcd.*\", code!=\"404\"}[5m])) BY (method) / sum(rate(etcd_http_received_total{job=~\".*etcd.*\"}[5m]))\nBY (method) > 0.05\n", + "for": "10m", + "labels": { + "severity": "critical" + } + }, + { + "alert": "etcdHTTPRequestsSlow", + "annotations": { + "message": "etcd instance {{ $labels.instance }} HTTP requests to {{ $labels.method }} are slow." + }, + "expr": "histogram_quantile(0.99, rate(etcd_http_successful_duration_seconds_bucket[5m]))\n> 0.15\n", + "for": "10m", + "labels": { + "severity": "warning" + } + } + ] + } + ] + } + extra.yaml: |- + { + "groups": [ + { + "name": "extra.rules", + "rules": [ + { + "alert": "InactiveRAIDDisk", + "annotations": { + "message": "{{ $value }} RAID disk(s) on node {{ $labels.instance }} are inactive." + }, + "expr": "node_md_disks - node_md_disks_active > 0", + "for": "10m", + "labels": { + "severity": "warning" + } + } + ] + } + ] + } + kube.yaml: |- + { + "groups": [ + { + "name": "k8s.rules", + "rules": [ + { + "expr": "sum(rate(container_cpu_usage_seconds_total{job=\"kubernetes-cadvisor\", image!=\"\", container_name!=\"\"}[5m])) by (namespace)\n", + "record": "namespace:container_cpu_usage_seconds_total:sum_rate" + }, + { + "expr": "sum by (namespace, pod_name, container_name) (\n rate(container_cpu_usage_seconds_total{job=\"kubernetes-cadvisor\", image!=\"\", container_name!=\"\"}[5m])\n)\n", + "record": "namespace_pod_name_container_name:container_cpu_usage_seconds_total:sum_rate" + }, + { + "expr": "sum(container_memory_usage_bytes{job=\"kubernetes-cadvisor\", image!=\"\", container_name!=\"\"}) by (namespace)\n", + "record": "namespace:container_memory_usage_bytes:sum" + }, + { + "expr": "sum by (namespace, label_name) (\n sum(rate(container_cpu_usage_seconds_total{job=\"kubernetes-cadvisor\", image!=\"\", container_name!=\"\"}[5m])) by (namespace, pod_name)\n * on (namespace, pod_name) group_left(label_name)\n label_replace(kube_pod_labels{job=\"kube-state-metrics\"}, \"pod_name\", \"$1\", \"pod\", \"(.*)\")\n)\n", + "record": "namespace_name:container_cpu_usage_seconds_total:sum_rate" + }, + { + "expr": "sum by (namespace, label_name) (\n sum(container_memory_usage_bytes{job=\"kubernetes-cadvisor\",image!=\"\", container_name!=\"\"}) by (pod_name, namespace)\n* on (namespace, pod_name) group_left(label_name)\n label_replace(kube_pod_labels{job=\"kube-state-metrics\"}, \"pod_name\", \"$1\", \"pod\", \"(.*)\")\n)\n", + "record": "namespace_name:container_memory_usage_bytes:sum" + }, + { + "expr": "sum by (namespace, label_name) (\n sum(kube_pod_container_resource_requests_memory_bytes{job=\"kube-state-metrics\"}) by (namespace, pod)\n* on (namespace, pod) group_left(label_name)\n label_replace(kube_pod_labels{job=\"kube-state-metrics\"}, \"pod_name\", \"$1\", \"pod\", \"(.*)\")\n)\n", + "record": "namespace_name:kube_pod_container_resource_requests_memory_bytes:sum" + }, + { + "expr": "sum by (namespace, label_name) (\n sum(kube_pod_container_resource_requests_cpu_cores{job=\"kube-state-metrics\"} and on(pod) kube_pod_status_scheduled{condition=\"true\"}) by (namespace, pod)\n* on (namespace, pod) group_left(label_name)\n label_replace(kube_pod_labels{job=\"kube-state-metrics\"}, \"pod_name\", \"$1\", \"pod\", \"(.*)\")\n)\n", + "record": "namespace_name:kube_pod_container_resource_requests_cpu_cores:sum" + } + ] + }, + { + "name": "kube-scheduler.rules", + "rules": [ + { + "expr": "histogram_quantile(0.99, sum(rate(scheduler_e2e_scheduling_latency_microseconds_bucket{job=\"kube-scheduler\"}[5m])) without(instance, pod)) / 1e+06\n", + "labels": { + "quantile": "0.99" + }, + "record": "cluster_quantile:scheduler_e2e_scheduling_latency:histogram_quantile" + }, + { + "expr": "histogram_quantile(0.99, sum(rate(scheduler_scheduling_algorithm_latency_microseconds_bucket{job=\"kube-scheduler\"}[5m])) without(instance, pod)) / 1e+06\n", + "labels": { + "quantile": "0.99" + }, + "record": "cluster_quantile:scheduler_scheduling_algorithm_latency:histogram_quantile" + }, + { + "expr": "histogram_quantile(0.99, sum(rate(scheduler_binding_latency_microseconds_bucket{job=\"kube-scheduler\"}[5m])) without(instance, pod)) / 1e+06\n", + "labels": { + "quantile": "0.99" + }, + "record": "cluster_quantile:scheduler_binding_latency:histogram_quantile" + }, + { + "expr": "histogram_quantile(0.9, sum(rate(scheduler_e2e_scheduling_latency_microseconds_bucket{job=\"kube-scheduler\"}[5m])) without(instance, pod)) / 1e+06\n", + "labels": { + "quantile": "0.9" + }, + "record": "cluster_quantile:scheduler_e2e_scheduling_latency:histogram_quantile" + }, + { + "expr": "histogram_quantile(0.9, sum(rate(scheduler_scheduling_algorithm_latency_microseconds_bucket{job=\"kube-scheduler\"}[5m])) without(instance, pod)) / 1e+06\n", + "labels": { + "quantile": "0.9" + }, + "record": "cluster_quantile:scheduler_scheduling_algorithm_latency:histogram_quantile" + }, + { + "expr": "histogram_quantile(0.9, sum(rate(scheduler_binding_latency_microseconds_bucket{job=\"kube-scheduler\"}[5m])) without(instance, pod)) / 1e+06\n", + "labels": { + "quantile": "0.9" + }, + "record": "cluster_quantile:scheduler_binding_latency:histogram_quantile" + }, + { + "expr": "histogram_quantile(0.5, sum(rate(scheduler_e2e_scheduling_latency_microseconds_bucket{job=\"kube-scheduler\"}[5m])) without(instance, pod)) / 1e+06\n", + "labels": { + "quantile": "0.5" + }, + "record": "cluster_quantile:scheduler_e2e_scheduling_latency:histogram_quantile" + }, + { + "expr": "histogram_quantile(0.5, sum(rate(scheduler_scheduling_algorithm_latency_microseconds_bucket{job=\"kube-scheduler\"}[5m])) without(instance, pod)) / 1e+06\n", + "labels": { + "quantile": "0.5" + }, + "record": "cluster_quantile:scheduler_scheduling_algorithm_latency:histogram_quantile" + }, + { + "expr": "histogram_quantile(0.5, sum(rate(scheduler_binding_latency_microseconds_bucket{job=\"kube-scheduler\"}[5m])) without(instance, pod)) / 1e+06\n", + "labels": { + "quantile": "0.5" + }, + "record": "cluster_quantile:scheduler_binding_latency:histogram_quantile" + } + ] + }, + { + "name": "kube-apiserver.rules", + "rules": [ + { + "expr": "histogram_quantile(0.99, sum(rate(apiserver_request_latencies_bucket{job=\"apiserver\"}[5m])) without(instance, pod)) / 1e+06\n", + "labels": { + "quantile": "0.99" + }, + "record": "cluster_quantile:apiserver_request_latencies:histogram_quantile" + }, + { + "expr": "histogram_quantile(0.9, sum(rate(apiserver_request_latencies_bucket{job=\"apiserver\"}[5m])) without(instance, pod)) / 1e+06\n", + "labels": { + "quantile": "0.9" + }, + "record": "cluster_quantile:apiserver_request_latencies:histogram_quantile" + }, + { + "expr": "histogram_quantile(0.5, sum(rate(apiserver_request_latencies_bucket{job=\"apiserver\"}[5m])) without(instance, pod)) / 1e+06\n", + "labels": { + "quantile": "0.5" + }, + "record": "cluster_quantile:apiserver_request_latencies:histogram_quantile" + } + ] + }, + { + "name": "node.rules", + "rules": [ + { + "expr": "sum(min(kube_pod_info) by (node))", + "record": ":kube_pod_info_node_count:" + }, + { + "expr": "max(label_replace(kube_pod_info{job=\"kube-state-metrics\"}, \"pod\", \"$1\", \"pod\", \"(.*)\")) by (node, namespace, pod)\n", + "record": "node_namespace_pod:kube_pod_info:" + }, + { + "expr": "count by (node) (sum by (node, cpu) (\n node_cpu_seconds_total{job=\"node-exporter\"}\n* on (namespace, pod) group_left(node)\n node_namespace_pod:kube_pod_info:\n))\n", + "record": "node:node_num_cpu:sum" + }, + { + "expr": "1 - avg(rate(node_cpu_seconds_total{job=\"node-exporter\",mode=\"idle\"}[1m]))\n", + "record": ":node_cpu_utilisation:avg1m" + }, + { + "expr": "1 - avg by (node) (\n rate(node_cpu_seconds_total{job=\"node-exporter\",mode=\"idle\"}[1m])\n* on (namespace, pod) group_left(node)\n node_namespace_pod:kube_pod_info:)\n", + "record": "node:node_cpu_utilisation:avg1m" + }, + { + "expr": "node:node_cpu_utilisation:avg1m\n *\nnode:node_num_cpu:sum\n /\nscalar(sum(node:node_num_cpu:sum))\n", + "record": "node:cluster_cpu_utilisation:ratio" + }, + { + "expr": "sum(node_load1{job=\"node-exporter\"})\n/\nsum(node:node_num_cpu:sum)\n", + "record": ":node_cpu_saturation_load1:" + }, + { + "expr": "sum by (node) (\n node_load1{job=\"node-exporter\"}\n* on (namespace, pod) group_left(node)\n node_namespace_pod:kube_pod_info:\n)\n/\nnode:node_num_cpu:sum\n", + "record": "node:node_cpu_saturation_load1:" + }, + { + "expr": "1 -\nsum(node_memory_MemFree_bytes{job=\"node-exporter\"} + node_memory_Cached_bytes{job=\"node-exporter\"} + node_memory_Buffers_bytes{job=\"node-exporter\"})\n/\nsum(node_memory_MemTotal_bytes{job=\"node-exporter\"})\n", + "record": ":node_memory_utilisation:" + }, + { + "expr": "sum(node_memory_MemFree_bytes{job=\"node-exporter\"} + node_memory_Cached_bytes{job=\"node-exporter\"} + node_memory_Buffers_bytes{job=\"node-exporter\"})\n", + "record": ":node_memory_MemFreeCachedBuffers_bytes:sum" + }, + { + "expr": "sum(node_memory_MemTotal_bytes{job=\"node-exporter\"})\n", + "record": ":node_memory_MemTotal_bytes:sum" + }, + { + "expr": "sum by (node) (\n (node_memory_MemFree_bytes{job=\"node-exporter\"} + node_memory_Cached_bytes{job=\"node-exporter\"} + node_memory_Buffers_bytes{job=\"node-exporter\"})\n * on (namespace, pod) group_left(node)\n node_namespace_pod:kube_pod_info:\n)\n", + "record": "node:node_memory_bytes_available:sum" + }, + { + "expr": "sum by (node) (\n node_memory_MemTotal_bytes{job=\"node-exporter\"}\n * on (namespace, pod) group_left(node)\n node_namespace_pod:kube_pod_info:\n)\n", + "record": "node:node_memory_bytes_total:sum" + }, + { + "expr": "(node:node_memory_bytes_total:sum - node:node_memory_bytes_available:sum)\n/\nnode:node_memory_bytes_total:sum\n", + "record": "node:node_memory_utilisation:ratio" + }, + { + "expr": "(node:node_memory_bytes_total:sum - node:node_memory_bytes_available:sum)\n/\nscalar(sum(node:node_memory_bytes_total:sum))\n", + "record": "node:cluster_memory_utilisation:ratio" + }, + { + "expr": "1e3 * sum(\n (rate(node_vmstat_pgpgin{job=\"node-exporter\"}[1m])\n + rate(node_vmstat_pgpgout{job=\"node-exporter\"}[1m]))\n)\n", + "record": ":node_memory_swap_io_bytes:sum_rate" + }, + { + "expr": "1 -\nsum by (node) (\n (node_memory_MemFree_bytes{job=\"node-exporter\"} + node_memory_Cached_bytes{job=\"node-exporter\"} + node_memory_Buffers_bytes{job=\"node-exporter\"})\n* on (namespace, pod) group_left(node)\n node_namespace_pod:kube_pod_info:\n)\n/\nsum by (node) (\n node_memory_MemTotal_bytes{job=\"node-exporter\"}\n* on (namespace, pod) group_left(node)\n node_namespace_pod:kube_pod_info:\n)\n", + "record": "node:node_memory_utilisation:" + }, + { + "expr": "1 - (node:node_memory_bytes_available:sum / node:node_memory_bytes_total:sum)\n", + "record": "node:node_memory_utilisation_2:" + }, + { + "expr": "1e3 * sum by (node) (\n (rate(node_vmstat_pgpgin{job=\"node-exporter\"}[1m])\n + rate(node_vmstat_pgpgout{job=\"node-exporter\"}[1m]))\n * on (namespace, pod) group_left(node)\n node_namespace_pod:kube_pod_info:\n)\n", + "record": "node:node_memory_swap_io_bytes:sum_rate" + }, + { + "expr": "avg(irate(node_disk_io_time_seconds_total{job=\"node-exporter\",device=~\"nvme.+|rbd.+|sd.+|vd.+|xvd.+\"}[1m]))\n", + "record": ":node_disk_utilisation:avg_irate" + }, + { + "expr": "avg by (node) (\n irate(node_disk_io_time_seconds_total{job=\"node-exporter\",device=~\"nvme.+|rbd.+|sd.+|vd.+|xvd.+\"}[1m])\n* on (namespace, pod) group_left(node)\n node_namespace_pod:kube_pod_info:\n)\n", + "record": "node:node_disk_utilisation:avg_irate" + }, + { + "expr": "avg(irate(node_disk_io_time_weighted_seconds_total{job=\"node-exporter\",device=~\"nvme.+|rbd.+|sd.+|vd.+|xvd.+\"}[1m]) / 1e3)\n", + "record": ":node_disk_saturation:avg_irate" + }, + { + "expr": "avg by (node) (\n irate(node_disk_io_time_weighted_seconds_total{job=\"node-exporter\",device=~\"nvme.+|rbd.+|sd.+|vd.+|xvd.+\"}[1m]) / 1e3\n* on (namespace, pod) group_left(node)\n node_namespace_pod:kube_pod_info:\n)\n", + "record": "node:node_disk_saturation:avg_irate" + }, + { + "expr": "max by (namespace, pod, device) ((node_filesystem_size_bytes{fstype=~\"ext[234]|btrfs|xfs|zfs\"}\n- node_filesystem_avail_bytes{fstype=~\"ext[234]|btrfs|xfs|zfs\"})\n/ node_filesystem_size_bytes{fstype=~\"ext[234]|btrfs|xfs|zfs\"})\n", + "record": "node:node_filesystem_usage:" + }, + { + "expr": "max by (namespace, pod, device) (node_filesystem_avail_bytes{fstype=~\"ext[234]|btrfs|xfs|zfs\"} / node_filesystem_size_bytes{fstype=~\"ext[234]|btrfs|xfs|zfs\"})\n", + "record": "node:node_filesystem_avail:" + }, + { + "expr": "sum(irate(node_network_receive_bytes_total{job=\"node-exporter\",device!~\"veth.+\"}[1m])) +\nsum(irate(node_network_transmit_bytes_total{job=\"node-exporter\",device!~\"veth.+\"}[1m]))\n", + "record": ":node_net_utilisation:sum_irate" + }, + { + "expr": "sum by (node) (\n (irate(node_network_receive_bytes_total{job=\"node-exporter\",device!~\"veth.+\"}[1m]) +\n irate(node_network_transmit_bytes_total{job=\"node-exporter\",device!~\"veth.+\"}[1m]))\n* on (namespace, pod) group_left(node)\n node_namespace_pod:kube_pod_info:\n)\n", + "record": "node:node_net_utilisation:sum_irate" + }, + { + "expr": "sum(irate(node_network_receive_drop_total{job=\"node-exporter\",device!~\"veth.+\"}[1m])) +\nsum(irate(node_network_transmit_drop_total{job=\"node-exporter\",device!~\"veth.+\"}[1m]))\n", + "record": ":node_net_saturation:sum_irate" + }, + { + "expr": "sum by (node) (\n (irate(node_network_receive_drop_total{job=\"node-exporter\",device!~\"veth.+\"}[1m]) +\n irate(node_network_transmit_drop_total{job=\"node-exporter\",device!~\"veth.+\"}[1m]))\n* on (namespace, pod) group_left(node)\n node_namespace_pod:kube_pod_info:\n)\n", + "record": "node:node_net_saturation:sum_irate" + }, + { + "expr": "max(\n max(\n kube_pod_info{job=\"kube-state-metrics\", host_ip!=\"\"}\n ) by (node, host_ip)\n * on (host_ip) group_right (node)\n label_replace(\n (max(node_filesystem_files{job=\"node-exporter\", mountpoint=\"/\"}) by (instance)), \"host_ip\", \"$1\", \"instance\", \"(.*):.*\"\n )\n) by (node)\n", + "record": "node:node_inodes_total:" + }, + { + "expr": "max(\n max(\n kube_pod_info{job=\"kube-state-metrics\", host_ip!=\"\"}\n ) by (node, host_ip)\n * on (host_ip) group_right (node)\n label_replace(\n (max(node_filesystem_files_free{job=\"node-exporter\", mountpoint=\"/\"}) by (instance)), \"host_ip\", \"$1\", \"instance\", \"(.*):.*\"\n )\n) by (node)\n", + "record": "node:node_inodes_free:" + } + ] + }, + { + "name": "kubernetes-absent", + "rules": [ + { + "alert": "KubeAPIDown", + "annotations": { + "message": "KubeAPI has disappeared from Prometheus target discovery.", + "runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubeapidown" + }, + "expr": "absent(up{job=\"apiserver\"} == 1)\n", + "for": "15m", + "labels": { + "severity": "critical" + } + }, + { + "alert": "KubeControllerManagerDown", + "annotations": { + "message": "KubeControllerManager has disappeared from Prometheus target discovery.", + "runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubecontrollermanagerdown" + }, + "expr": "absent(up{job=\"kube-controller-manager\"} == 1)\n", + "for": "15m", + "labels": { + "severity": "critical" + } + }, + { + "alert": "KubeSchedulerDown", + "annotations": { + "message": "KubeScheduler has disappeared from Prometheus target discovery.", + "runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubeschedulerdown" + }, + "expr": "absent(up{job=\"kube-scheduler\"} == 1)\n", + "for": "15m", + "labels": { + "severity": "critical" + } + }, + { + "alert": "KubeletDown", + "annotations": { + "message": "Kubelet has disappeared from Prometheus target discovery.", + "runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubeletdown" + }, + "expr": "absent(up{job=\"kubelet\"} == 1)\n", + "for": "15m", + "labels": { + "severity": "critical" + } + } + ] + }, + { + "name": "kubernetes-apps", + "rules": [ + { + "alert": "KubePodCrashLooping", + "annotations": { + "message": "Pod {{ $labels.namespace }}/{{ $labels.pod }} ({{ $labels.container }}) is restarting {{ printf \"%.2f\" $value }} times / 5 minutes.", + "runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubepodcrashlooping" + }, + "expr": "rate(kube_pod_container_status_restarts_total{job=\"kube-state-metrics\"}[15m]) * 60 * 5 > 0\n", + "for": "1h", + "labels": { + "severity": "critical" + } + }, + { + "alert": "KubePodNotReady", + "annotations": { + "message": "Pod {{ $labels.namespace }}/{{ $labels.pod }} has been in a non-ready state for longer than an hour.", + "runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubepodnotready" + }, + "expr": "sum by (namespace, pod) (kube_pod_status_phase{job=\"kube-state-metrics\", phase=~\"Pending|Unknown\"}) > 0\n", + "for": "1h", + "labels": { + "severity": "critical" + } + }, + { + "alert": "KubeDeploymentGenerationMismatch", + "annotations": { + "message": "Deployment generation for {{ $labels.namespace }}/{{ $labels.deployment }} does not match, this indicates that the Deployment has failed but has not been rolled back.", + "runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubedeploymentgenerationmismatch" + }, + "expr": "kube_deployment_status_observed_generation{job=\"kube-state-metrics\"}\n !=\nkube_deployment_metadata_generation{job=\"kube-state-metrics\"}\n", + "for": "15m", + "labels": { + "severity": "critical" + } + }, + { + "alert": "KubeDeploymentReplicasMismatch", + "annotations": { + "message": "Deployment {{ $labels.namespace }}/{{ $labels.deployment }} has not matched the expected number of replicas for longer than an hour.", + "runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubedeploymentreplicasmismatch" + }, + "expr": "kube_deployment_spec_replicas{job=\"kube-state-metrics\"}\n !=\nkube_deployment_status_replicas_available{job=\"kube-state-metrics\"}\n", + "for": "1h", + "labels": { + "severity": "critical" + } + }, + { + "alert": "KubeStatefulSetReplicasMismatch", + "annotations": { + "message": "StatefulSet {{ $labels.namespace }}/{{ $labels.statefulset }} has not matched the expected number of replicas for longer than 15 minutes.", + "runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubestatefulsetreplicasmismatch" + }, + "expr": "kube_statefulset_status_replicas_ready{job=\"kube-state-metrics\"}\n !=\nkube_statefulset_status_replicas{job=\"kube-state-metrics\"}\n", + "for": "15m", + "labels": { + "severity": "critical" + } + }, + { + "alert": "KubeStatefulSetGenerationMismatch", + "annotations": { + "message": "StatefulSet generation for {{ $labels.namespace }}/{{ $labels.statefulset }} does not match, this indicates that the StatefulSet has failed but has not been rolled back.", + "runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubestatefulsetgenerationmismatch" + }, + "expr": "kube_statefulset_status_observed_generation{job=\"kube-state-metrics\"}\n !=\nkube_statefulset_metadata_generation{job=\"kube-state-metrics\"}\n", + "for": "15m", + "labels": { + "severity": "critical" + } + }, + { + "alert": "KubeStatefulSetUpdateNotRolledOut", + "annotations": { + "message": "StatefulSet {{ $labels.namespace }}/{{ $labels.statefulset }} update has not been rolled out.", + "runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubestatefulsetupdatenotrolledout" + }, + "expr": "max without (revision) (\n kube_statefulset_status_current_revision{job=\"kube-state-metrics\"}\n unless\n kube_statefulset_status_update_revision{job=\"kube-state-metrics\"}\n)\n *\n(\n kube_statefulset_replicas{job=\"kube-state-metrics\"}\n !=\n kube_statefulset_status_replicas_updated{job=\"kube-state-metrics\"}\n)\n", + "for": "15m", + "labels": { + "severity": "critical" + } + }, + { + "alert": "KubeDaemonSetRolloutStuck", + "annotations": { + "message": "Only {{ $value }}% of the desired Pods of DaemonSet {{ $labels.namespace }}/{{ $labels.daemonset }} are scheduled and ready.", + "runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubedaemonsetrolloutstuck" + }, + "expr": "kube_daemonset_status_number_ready{job=\"kube-state-metrics\"}\n /\nkube_daemonset_status_desired_number_scheduled{job=\"kube-state-metrics\"} * 100 < 100\n", + "for": "15m", + "labels": { + "severity": "critical" + } + }, + { + "alert": "KubeDaemonSetNotScheduled", + "annotations": { + "message": "{{ $value }} Pods of DaemonSet {{ $labels.namespace }}/{{ $labels.daemonset }} are not scheduled.", + "runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubedaemonsetnotscheduled" + }, + "expr": "kube_daemonset_status_desired_number_scheduled{job=\"kube-state-metrics\"}\n -\nkube_daemonset_status_current_number_scheduled{job=\"kube-state-metrics\"} > 0\n", + "for": "10m", + "labels": { + "severity": "warning" + } + }, + { + "alert": "KubeDaemonSetMisScheduled", + "annotations": { + "message": "{{ $value }} Pods of DaemonSet {{ $labels.namespace }}/{{ $labels.daemonset }} are running where they are not supposed to run.", + "runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubedaemonsetmisscheduled" + }, + "expr": "kube_daemonset_status_number_misscheduled{job=\"kube-state-metrics\"} > 0\n", + "for": "10m", + "labels": { + "severity": "warning" + } + }, + { + "alert": "KubeCronJobRunning", + "annotations": { + "message": "CronJob {{ $labels.namespace }}/{{ $labels.cronjob }} is taking more than 1h to complete.", + "runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubecronjobrunning" + }, + "expr": "time() - kube_cronjob_next_schedule_time{job=\"kube-state-metrics\"} > 3600\n", + "for": "1h", + "labels": { + "severity": "warning" + } + }, + { + "alert": "KubeJobCompletion", + "annotations": { + "message": "Job {{ $labels.namespace }}/{{ $labels.job_name }} is taking more than one hour to complete.", + "runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubejobcompletion" + }, + "expr": "kube_job_spec_completions{job=\"kube-state-metrics\"} - kube_job_status_succeeded{job=\"kube-state-metrics\"} > 0\n", + "for": "1h", + "labels": { + "severity": "warning" + } + }, + { + "alert": "KubeJobFailed", + "annotations": { + "message": "Job {{ $labels.namespace }}/{{ $labels.job_name }} failed to complete.", + "runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubejobfailed" + }, + "expr": "kube_job_status_failed{job=\"kube-state-metrics\"} > 0\n", + "for": "1h", + "labels": { + "severity": "warning" + } + } + ] + }, + { + "name": "kubernetes-resources", + "rules": [ + { + "alert": "KubeCPUOvercommit", + "annotations": { + "message": "Cluster has overcommitted CPU resource requests for Pods and cannot tolerate node failure.", + "runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubecpuovercommit" + }, + "expr": "sum(namespace_name:kube_pod_container_resource_requests_cpu_cores:sum)\n /\nsum(node:node_num_cpu:sum)\n >\n(count(node:node_num_cpu:sum)-1) / count(node:node_num_cpu:sum)\n", + "for": "5m", + "labels": { + "severity": "warning" + } + }, + { + "alert": "KubeMemOvercommit", + "annotations": { + "message": "Cluster has overcommitted memory resource requests for Pods and cannot tolerate node failure.", + "runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubememovercommit" + }, + "expr": "sum(namespace_name:kube_pod_container_resource_requests_memory_bytes:sum)\n /\nsum(node_memory_MemTotal_bytes)\n >\n(count(node:node_num_cpu:sum)-1)\n /\ncount(node:node_num_cpu:sum)\n", + "for": "5m", + "labels": { + "severity": "warning" + } + }, + { + "alert": "KubeCPUOvercommit", + "annotations": { + "message": "Cluster has overcommitted CPU resource requests for Namespaces.", + "runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubecpuovercommit" + }, + "expr": "sum(kube_resourcequota{job=\"kube-state-metrics\", type=\"hard\", resource=\"requests.cpu\"})\n /\nsum(node:node_num_cpu:sum)\n > 1.5\n", + "for": "5m", + "labels": { + "severity": "warning" + } + }, + { + "alert": "KubeMemOvercommit", + "annotations": { + "message": "Cluster has overcommitted memory resource requests for Namespaces.", + "runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubememovercommit" + }, + "expr": "sum(kube_resourcequota{job=\"kube-state-metrics\", type=\"hard\", resource=\"requests.memory\"})\n /\nsum(node_memory_MemTotal_bytes{job=\"node-exporter\"})\n > 1.5\n", + "for": "5m", + "labels": { + "severity": "warning" + } + }, + { + "alert": "KubeQuotaExceeded", + "annotations": { + "message": "Namespace {{ $labels.namespace }} is using {{ printf \"%0.0f\" $value }}% of its {{ $labels.resource }} quota.", + "runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubequotaexceeded" + }, + "expr": "100 * kube_resourcequota{job=\"kube-state-metrics\", type=\"used\"}\n / ignoring(instance, job, type)\n(kube_resourcequota{job=\"kube-state-metrics\", type=\"hard\"} > 0)\n > 90\n", + "for": "15m", + "labels": { + "severity": "warning" + } + }, + { + "alert": "CPUThrottlingHigh", + "annotations": { + "message": "{{ printf \"%0.0f\" $value }}% throttling of CPU in namespace {{ $labels.namespace }} for container {{ $labels.container_name }} in pod {{ $labels.pod_name }}.", + "runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-cputhrottlinghigh" + }, + "expr": "100 * sum(increase(container_cpu_cfs_throttled_periods_total{container_name!=\"\", }[5m])) by (container_name, pod_name, namespace)\n /\nsum(increase(container_cpu_cfs_periods_total{}[5m])) by (container_name, pod_name, namespace)\n > 100 \n", + "for": "15m", + "labels": { + "severity": "warning" + } + } + ] + }, + { + "name": "kubernetes-storage", + "rules": [ + { + "alert": "KubePersistentVolumeUsageCritical", + "annotations": { + "message": "The PersistentVolume claimed by {{ $labels.persistentvolumeclaim }} in Namespace {{ $labels.namespace }} is only {{ printf \"%0.2f\" $value }}% free.", + "runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubepersistentvolumeusagecritical" + }, + "expr": "100 * kubelet_volume_stats_available_bytes{job=\"kubelet\"}\n /\nkubelet_volume_stats_capacity_bytes{job=\"kubelet\"}\n < 3\n", + "for": "1m", + "labels": { + "severity": "critical" + } + }, + { + "alert": "KubePersistentVolumeFullInFourDays", + "annotations": { + "message": "Based on recent sampling, the PersistentVolume claimed by {{ $labels.persistentvolumeclaim }} in Namespace {{ $labels.namespace }} is expected to fill up within four days. Currently {{ printf \"%0.2f\" $value }}% is available.", + "runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubepersistentvolumefullinfourdays" + }, + "expr": "100 * (\n kubelet_volume_stats_available_bytes{job=\"kubelet\"}\n /\n kubelet_volume_stats_capacity_bytes{job=\"kubelet\"}\n) < 15\nand\npredict_linear(kubelet_volume_stats_available_bytes{job=\"kubelet\"}[6h], 4 * 24 * 3600) < 0\n", + "for": "5m", + "labels": { + "severity": "critical" + } + }, + { + "alert": "KubePersistentVolumeErrors", + "annotations": { + "message": "The persistent volume {{ $labels.persistentvolume }} has status {{ $labels.phase }}.", + "runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubepersistentvolumeerrors" + }, + "expr": "kube_persistentvolume_status_phase{phase=~\"Failed|Pending\",job=\"kube-state-metrics\"} > 0\n", + "for": "5m", + "labels": { + "severity": "critical" + } + } + ] + }, + { + "name": "kubernetes-system", + "rules": [ + { + "alert": "KubeNodeNotReady", + "annotations": { + "message": "{{ $labels.node }} has been unready for more than an hour.", + "runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubenodenotready" + }, + "expr": "kube_node_status_condition{job=\"kube-state-metrics\",condition=\"Ready\",status=\"true\"} == 0\n", + "for": "1h", + "labels": { + "severity": "warning" + } + }, + { + "alert": "KubeVersionMismatch", + "annotations": { + "message": "There are {{ $value }} different semantic versions of Kubernetes components running.", + "runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubeversionmismatch" + }, + "expr": "count(count by (gitVersion) (label_replace(kubernetes_build_info{job!=\"coredns\"},\"gitVersion\",\"$1\",\"gitVersion\",\"(v[0-9]*.[0-9]*.[0-9]*).*\"))) > 1\n", + "for": "1h", + "labels": { + "severity": "warning" + } + }, + { + "alert": "KubeClientErrors", + "annotations": { + "message": "Kubernetes API server client '{{ $labels.job }}/{{ $labels.instance }}' is experiencing {{ printf \"%0.0f\" $value }}% errors.'", + "runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubeclienterrors" + }, + "expr": "(sum(rate(rest_client_requests_total{code=~\"5..\"}[5m])) by (instance, job)\n /\nsum(rate(rest_client_requests_total[5m])) by (instance, job))\n* 100 > 1\n", + "for": "15m", + "labels": { + "severity": "warning" + } + }, + { + "alert": "KubeClientErrors", + "annotations": { + "message": "Kubernetes API server client '{{ $labels.job }}/{{ $labels.instance }}' is experiencing {{ printf \"%0.0f\" $value }} errors / second.", + "runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubeclienterrors" + }, + "expr": "sum(rate(ksm_scrape_error_total{job=\"kube-state-metrics\"}[5m])) by (instance, job) > 0.1\n", + "for": "15m", + "labels": { + "severity": "warning" + } + }, + { + "alert": "KubeletTooManyPods", + "annotations": { + "message": "Kubelet {{ $labels.instance }} is running {{ $value }} Pods, close to the limit of 110.", + "runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubelettoomanypods" + }, + "expr": "kubelet_running_pod_count{job=\"kubelet\"} > 110 * 0.9\n", + "for": "15m", + "labels": { + "severity": "warning" + } + }, + { + "alert": "KubeAPILatencyHigh", + "annotations": { + "message": "The API server has a 99th percentile latency of {{ $value }} seconds for {{ $labels.verb }} {{ $labels.resource }}.", + "runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubeapilatencyhigh" + }, + "expr": "cluster_quantile:apiserver_request_latencies:histogram_quantile{job=\"apiserver\",quantile=\"0.99\",subresource!=\"log\",verb!~\"^(?:LIST|WATCH|WATCHLIST|PROXY|CONNECT)$\"} > 1\n", + "for": "10m", + "labels": { + "severity": "warning" + } + }, + { + "alert": "KubeAPILatencyHigh", + "annotations": { + "message": "The API server has a 99th percentile latency of {{ $value }} seconds for {{ $labels.verb }} {{ $labels.resource }}.", + "runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubeapilatencyhigh" + }, + "expr": "cluster_quantile:apiserver_request_latencies:histogram_quantile{job=\"apiserver\",quantile=\"0.99\",subresource!=\"log\",verb!~\"^(?:LIST|WATCH|WATCHLIST|PROXY|CONNECT)$\"} > 4\n", + "for": "10m", + "labels": { + "severity": "critical" + } + }, + { + "alert": "KubeAPIErrorsHigh", + "annotations": { + "message": "API server is returning errors for {{ $value }}% of requests.", + "runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubeapierrorshigh" + }, + "expr": "sum(rate(apiserver_request_count{job=\"apiserver\",code=~\"^(?:5..)$\"}[5m])) without(instance, pod)\n /\nsum(rate(apiserver_request_count{job=\"apiserver\"}[5m])) without(instance, pod) * 100 > 10\n", + "for": "10m", + "labels": { + "severity": "critical" + } + }, + { + "alert": "KubeAPIErrorsHigh", + "annotations": { + "message": "API server is returning errors for {{ $value }}% of requests.", + "runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubeapierrorshigh" + }, + "expr": "sum(rate(apiserver_request_count{job=\"apiserver\",code=~\"^(?:5..)$\"}[5m])) without(instance, pod)\n /\nsum(rate(apiserver_request_count{job=\"apiserver\"}[5m])) without(instance, pod) * 100 > 5\n", + "for": "10m", + "labels": { + "severity": "warning" + } + }, + { + "alert": "KubeClientCertificateExpiration", + "annotations": { + "message": "A client certificate used to authenticate to the apiserver is expiring in less than 7 days.", + "runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubeclientcertificateexpiration" + }, + "expr": "histogram_quantile(0.01, sum by (job, le) (rate(apiserver_client_certificate_expiration_seconds_bucket{job=\"apiserver\"}[5m]))) < 604800\n", + "labels": { + "severity": "warning" + } + }, + { + "alert": "KubeClientCertificateExpiration", + "annotations": { + "message": "A client certificate used to authenticate to the apiserver is expiring in less than 24 hours.", + "runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubeclientcertificateexpiration" + }, + "expr": "histogram_quantile(0.01, sum by (job, le) (rate(apiserver_client_certificate_expiration_seconds_bucket{job=\"apiserver\"}[5m]))) < 86400\n", + "labels": { + "severity": "critical" + } + } + ] + } + ] + } + kubeprom.yaml: |- + { + "groups": [ + { + "name": "kube-prometheus-node-recording.rules", + "rules": [ + { + "expr": "sum(rate(node_cpu_seconds_total{mode!=\"idle\",mode!=\"iowait\"}[3m])) BY (instance)", + "record": "instance:node_cpu:rate:sum" + }, + { + "expr": "sum((node_filesystem_size_bytes{mountpoint=\"/\"} - node_filesystem_free_bytes{mountpoint=\"/\"})) BY (instance)", + "record": "instance:node_filesystem_usage:sum" + }, + { + "expr": "sum(rate(node_network_receive_bytes_total[3m])) BY (instance)", + "record": "instance:node_network_receive_bytes:rate:sum" + }, + { + "expr": "sum(rate(node_network_transmit_bytes_total[3m])) BY (instance)", + "record": "instance:node_network_transmit_bytes:rate:sum" + }, + { + "expr": "sum(rate(node_cpu_seconds_total{mode!=\"idle\",mode!=\"iowait\"}[5m])) WITHOUT (cpu, mode) / ON(instance) GROUP_LEFT() count(sum(node_cpu_seconds_total) BY (instance, cpu)) BY (instance)", + "record": "instance:node_cpu:ratio" + }, + { + "expr": "sum(rate(node_cpu_seconds_total{mode!=\"idle\",mode!=\"iowait\"}[5m]))", + "record": "cluster:node_cpu:sum_rate5m" + }, + { + "expr": "cluster:node_cpu_seconds_total:rate5m / count(sum(node_cpu_seconds_total) BY (instance, cpu))", + "record": "cluster:node_cpu:ratio" + } + ] + }, + { + "name": "kube-prometheus-node-alerting.rules", + "rules": [ + { + "alert": "NodeDiskRunningFull", + "annotations": { + "message": "Device {{ $labels.device }} of node-exporter {{ $labels.namespace }}/{{ $labels.pod }} will be full within the next 24 hours." + }, + "expr": "(node:node_filesystem_usage: > 0.85) and (predict_linear(node:node_filesystem_avail:[6h], 3600 * 24) < 0)\n", + "for": "30m", + "labels": { + "severity": "warning" + } + }, + { + "alert": "NodeDiskRunningFull", + "annotations": { + "message": "Device {{ $labels.device }} of node-exporter {{ $labels.namespace }}/{{ $labels.pod }} will be full within the next 2 hours." + }, + "expr": "(node:node_filesystem_usage: > 0.85) and (predict_linear(node:node_filesystem_avail:[30m], 3600 * 2) < 0)\n", + "for": "10m", + "labels": { + "severity": "critical" + } + } + ] + }, + { + "name": "prometheus.rules", + "rules": [ + { + "alert": "PrometheusConfigReloadFailed", + "annotations": { + "description": "Reloading Prometheus' configuration has failed for {{$labels.namespace}}/{{$labels.pod}}", + "summary": "Reloading Prometheus' configuration failed" + }, + "expr": "prometheus_config_last_reload_successful{job=\"prometheus\"} == 0\n", + "for": "10m", + "labels": { + "severity": "warning" + } + }, + { + "alert": "PrometheusNotificationQueueRunningFull", + "annotations": { + "description": "Prometheus' alert notification queue is running full for {{$labels.namespace}}/{{ $labels.pod}}", + "summary": "Prometheus' alert notification queue is running full" + }, + "expr": "predict_linear(prometheus_notifications_queue_length{job=\"prometheus\"}[5m], 60 * 30) > prometheus_notifications_queue_capacity{job=\"prometheus\"}\n", + "for": "10m", + "labels": { + "severity": "warning" + } + }, + { + "alert": "PrometheusErrorSendingAlerts", + "annotations": { + "description": "Errors while sending alerts from Prometheus {{$labels.namespace}}/{{ $labels.pod}} to Alertmanager {{$labels.Alertmanager}}", + "summary": "Errors while sending alert from Prometheus" + }, + "expr": "rate(prometheus_notifications_errors_total{job=\"prometheus\"}[5m]) / rate(prometheus_notifications_sent_total{job=\"prometheus\"}[5m]) > 0.01\n", + "for": "10m", + "labels": { + "severity": "warning" + } + }, + { + "alert": "PrometheusErrorSendingAlerts", + "annotations": { + "description": "Errors while sending alerts from Prometheus {{$labels.namespace}}/{{ $labels.pod}} to Alertmanager {{$labels.Alertmanager}}", + "summary": "Errors while sending alerts from Prometheus" + }, + "expr": "rate(prometheus_notifications_errors_total{job=\"prometheus\"}[5m]) / rate(prometheus_notifications_sent_total{job=\"prometheus\"}[5m]) > 0.03\n", + "for": "10m", + "labels": { + "severity": "critical" + } + }, + { + "alert": "PrometheusNotConnectedToAlertmanagers", + "annotations": { + "description": "Prometheus {{ $labels.namespace }}/{{ $labels.pod}} is not connected to any Alertmanagers", + "summary": "Prometheus is not connected to any Alertmanagers" + }, + "expr": "prometheus_notifications_alertmanagers_discovered{job=\"prometheus\"} < 1\n", + "for": "10m", + "labels": { + "severity": "warning" + } + }, + { + "alert": "PrometheusTSDBReloadsFailing", + "annotations": { + "description": "{{$labels.job}} at {{$labels.instance}} had {{$value | humanize}} reload failures over the last four hours.", + "summary": "Prometheus has issues reloading data blocks from disk" + }, + "expr": "increase(prometheus_tsdb_reloads_failures_total{job=\"prometheus\"}[2h]) > 0\n", + "for": "12h", + "labels": { + "severity": "warning" + } + }, + { + "alert": "PrometheusTSDBCompactionsFailing", + "annotations": { + "description": "{{$labels.job}} at {{$labels.instance}} had {{$value | humanize}} compaction failures over the last four hours.", + "summary": "Prometheus has issues compacting sample blocks" + }, + "expr": "increase(prometheus_tsdb_compactions_failed_total{job=\"prometheus\"}[2h]) > 0\n", + "for": "12h", + "labels": { + "severity": "warning" + } + }, + { + "alert": "PrometheusTSDBWALCorruptions", + "annotations": { + "description": "{{$labels.job}} at {{$labels.instance}} has a corrupted write-ahead log (WAL).", + "summary": "Prometheus write-ahead log is corrupted" + }, + "expr": "tsdb_wal_corruptions_total{job=\"prometheus\"} > 0\n", + "for": "4h", + "labels": { + "severity": "warning" + } + }, + { + "alert": "PrometheusNotIngestingSamples", + "annotations": { + "description": "Prometheus {{ $labels.namespace }}/{{ $labels.pod}} isn't ingesting samples.", + "summary": "Prometheus isn't ingesting samples" + }, + "expr": "rate(prometheus_tsdb_head_samples_appended_total{job=\"prometheus\"}[5m]) <= 0\n", + "for": "10m", + "labels": { + "severity": "warning" + } + }, + { + "alert": "PrometheusTargetScrapesDuplicate", + "annotations": { + "description": "{{$labels.namespace}}/{{$labels.pod}} has many samples rejected due to duplicate timestamps but different values", + "summary": "Prometheus has many samples rejected" + }, + "expr": "increase(prometheus_target_scrapes_sample_duplicate_timestamp_total{job=\"prometheus\"}[5m]) > 0\n", + "for": "10m", + "labels": { + "severity": "warning" + } + } + ] + }, + { + "name": "general.rules", + "rules": [ + { + "alert": "TargetDown", + "annotations": { + "message": "{{ $value }}% of the {{ $labels.job }} targets are down." + }, + "expr": "100 * (count(up == 0) BY (job) / count(up) BY (job)) > 10", + "for": "10m", + "labels": { + "severity": "warning" + } + } + ] + } + ] + } diff --git a/docs/addons/grafana.md b/docs/addons/grafana.md index 724b5aa30..f60061c0f 100644 --- a/docs/addons/grafana.md +++ b/docs/addons/grafana.md @@ -14,7 +14,8 @@ kubectl port-forward grafana-POD-ID 8080 -n monitoring Visit [127.0.0.1:8080](http://127.0.0.1:8080) to view the bundled dashboards. -![Grafana Capacity Planning](../img/grafana-capacity.png) -![Grafana Control Plane](../img/grafana-control-plane.png) -![Grafana Node View](../img/grafana-node.png) +![Grafana etcd](../img/grafana-etcd.png) +![Grafana resources cluster](../img/grafana-resources-cluster.png) +![Grafana usage cluster](../img/grafana-usage-cluster.png) +![Grafana usage node](../img/grafana-usage-node.png) diff --git a/docs/img/grafana-capacity.png b/docs/img/grafana-capacity.png deleted file mode 100644 index e85ac696386306222ac66b0f9adc18b980a91ec7..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 239490 zcmY&<1yEI8*e(c?(%mT`-3^L#gLHSNbgOg=NSAcCba#g!(k8%m`?DQ3;_;&g`mCN2nq^4*h^K@Ma9US-0`!6 zxs|OMxr?Wx8M&EjxG8o z{7sm_h<*I7d(6zf{Bn1a^c+D(!mncIJI6Wl_(!Xd@frVa$HK2;n8Bk5Yfq-5WA)$%!ii zp8h9Kk!SDsW7?tp_bYfWCzu*nE$mnhSBq&+NHQ^3FZYgubTXXgcNbR&>$@80|JrDM&1@%|H|+0$b} z<6{z$-^Ja<#)`mZ+k=pj23M7Nhtj3@g$T^qnmI{zSik!Ijmq=$U8ye%RUWF`L*?v9 z+U%&ljGwqiLND1_ZeMuPxHnVjuakwf=QaP3*#~{?=1|qx&-v$%TMJKGG9?&)vOFJn zF3h%LSbs z;pq5W*Z2x^wNxYR^9tk2)o!&*W~NZaZi0{8ys*tw&xO_zC$(301$~qn-i3FqYO8{ye22E1+kgB{`0rdd5G_m>11_Z0nt)lnmFz}*VpveVR~N#TIF0?k7Mf;4#qT%VUO*a-CMhlED*K-`}!stDiJLj2|VbZapPY1TW$uDdZ+B_PT zTIR1u5Th;A;F0eTcFn#+cdo*Hdunt0_M~{2!k=xcP*_lAYeu+cKgB>F8%GzIl>6#m z_l}t))Y_~Y4W8L0cLsT)GC^|Qk6Mi?Ui2j~(fjlyV@m7FhzexOY159ppri`-+1|( z*%h(4mk6e1AQ{LFS@mLTUl+>~VosqJ3{1nZKUkj5iaF6d= z#$gdIbA_6l_)I?6zc0iw)ZUh(eH}UglX>{TxOdo>_m1mpM4))9D88bi{C)tnmsZsv z`J7=CxdYNE3^)ep+e`+ZNIvj%%M!(JhR1C$_*! z3ULp-*@{fbf8Lx)v02$LyDxPkMca9;=w={-bAwd^}=_^`~$^(IPaT} zNh{|#a!wh;yZxb7LpoKOB(>FHa2npUYuGFS{^)wR=dDzBBq=Je1t;k%hTMtnen!UEGvCCzIj=?BvQcUsbFEO5agVcdx zHFO_h`^OLHUwrQykM{Q>9xaZC;7KHcmy0G_l9sShaganUk)??>i!xY(VyyZReo#cS`vKmXgnY%W z&iibIEE4XPso35xxFtMS?wy5s5!Bz7Nm-mpI~V+=^M-qpzx3`d^bHaf6hV59Y54t| zVY%PcuI~u*R^PujG_}q3aJP-D{%bggMPs7hEAGC-t%Q8I*Nm>pA7psx@XI%)b9wOH z79;vWzH!;+DMpTZMdj05Q(DT5V@15?}iwY;V$G6bo882kFKw$ zOiUQr8l)J8+UF-990bvHD-zJ%?F0{n_+Jm&Dg5!Y(j0qJyii!7y6hqL%~h}@q^=f9 z<^qnBcEr!>2ejhy-yhoF&~I>O<9|GJ(AY~5_X(Rcul#UV%cBfeg>wb6XAsH^R-)?BerY|1>x!_ zC$r&ui|k^)mR#m!{CkECWA4GVaaf||wKO=Yv;hCy#^PY)3+Vpevr4v9 zdg$JH>8#L=Rk}Od!!aGbxVDORZ8GKLH@vJ%jvrP8Oq)tH-n|axV6Nw(;=`&>0eA3kEe9^_ z`rJH^#6Cz*+aon@J!8b0d-xUcNS>-lSLi0vH+{o{fbc2tK>^RsPnR~gEe*&Q;h1H8 ze?29gZoa&sdHX&?Eu3Ms!IAZinA2?V*GZ;$MlpeM4rTt=kJ)grFe$WZd1WL~$6wER zsNNc?1+%jGJx{t<-_gC1;drG%dlfgy>K1Ofhw?G9FH;OLD*&ZS=~Y>SW!)TsD~^!d z8lFY3@P`g=rJxp9?a*17+>FWahR(iDflqI7pX!zkQwTh9SrFhi8D{fPk&eTkCA#0% zLb}hIS$n?#%L^@N|kEXCxUGD%z%5KvQZ0IH^ zPArjsl%{SQPTjp~y`NYQ=Onx;Q5n1can1AG4hs-^B2M(f)rpIds+%A;zGd7U*^7@7 zMlXYEpmkG)ffD?p;i@Z?8~YOFnNJcJ>XU5LimcW4^U&4O@i)9-06F&+sv-GHttN6g z;nZKYh!{9%xS$KS&R;?Q#=Pd%3o3G|Td}Jg(Xw8j(WRLO^kKd5o1t$0&X0@PeKl8_ zTVev|E`~==W|>RXKb`H1OZj0WAoX^VbU-_d7l$9&nuHbx(f`^$=;_0+pTP_7Ut3O! z7p`#%{w6nuPxg?Tf{M3owNPB2ejOjsysmV_AgVOCo7MF?zgae5Cb<&Ji%>e>=+tnQ z2wV6aQSiyT&#FrF5vaJBQz42>lHTI58m6S{g?3%SA9Jp%TV)<_F@tZ0-%c{Y&z$z2 z@8r#V75@3=$7btSy-M5!v;?u`YKOSZh`C-XXy%WIH1G4j7}0JU*_aafhkqJGx;;xS zf5+bxD@q>N*7;aB`A!a2#rgDs69%o?tuka!kuiTXuwOMXu;-^^zz-<~gb1{$H$PY> zI7#TLdW}%3U47QFwVg?_OHw@No}w;4t=UHX+UBSA~v^OX5yW#`&A3*HITi!L%eHf_T!G$H|-5YNpW@u&-5- zg4IQa5e#(Emn0B6xc`z3J78J%mPr0U*tK;njIc@MHO%21+WzT`Qrwejgg1w;fHRAX zP-v9oRV|x_9x_LFm)%DN4KuY`EEYwQh~F0Vx;T?bRPp1VJr^lN6r8PSqo04jy)s1m z7FgUyw%NCLoE*n47{>Y0wA9#pvT+1ob>l_|Np>t>wai7T7 z9GlK;!(PjfIF}q6t{`_ghtZk07vXL)P*qaoiXZb%iJ2QO#4#wwo4AVM{_g(k^?D0F zOpHvLZ*z3}JSB9HmWZhZ<$}#G3vZVt2a7Wp{{CTD-+*l@TY|m;9XMF80F)ti&OrNh zHOoSCQB3Q*wz|`@Ug+pt>B84tYCXjV{VCn~zFt4_vaOveg6PYX=O_6k($%qv!+72+ z)V}WOqu&=>`j|=^{pufvyi_PD?!%duaMkPkuR{UmtUsj&WDiBY6GkyK#FF-2`i`W1 z`ALDh?a5P-0|U1Xy^T|@k{m%Q`30}SkLvey9qUuF6p}Q{?I4=LuYL|WgC{~7eoFCD zl8#q@HuA!cNpT}`!?R(=8HE0gQmcrWudLr+9@tz-~PiH-T0-z-hv6YKIEq@1!w;kNuW)f5~iEl@G! zR7iJG^2j_AxZJ@T^UR9yza(%}Bo?LMx;1`7!W!>0@9cw!V<7#VU;!&#UkOj;gG80* zuMS({Tl)xylAKldkFgxsc(jD@=TFgNUo8pesv=b~B3J^6xkG%S8-)X$^`_9towzfBSKbB0WWLl(ON!$1h(HH9b= zQU5v~$#XhRkKdlJ)8`^(-gVYL3PZx9nB@&)k-bV8IK}q8I|691F)V|+qZ_R6xpsQx zgZzf372j7EPkQ;Vb(6I6SKQt`Uir%V=tx3Gv1G=mcnBaRidDE+mEn?*BN<~{htOrN z9OUR^i@i#6ik3R^OqU%X*erF@mxhktJ~F}xv%I&F;y9G*iD9Yiin=km%G~(0J%p+IOvC-*e>b`tY{qR0fFR{IodC^^Q{*%S_&A>dt zME4Z7?ahUcpa;~>XiCb2#qiG4ed~xMA;0ktRDJ^*7rg=FYR8&$YOQ&#xg%5;y@xVI zS#orjFBJ!xfJm;?W*f=dH5%voxVbYUDTx-(rTL zBL$s4JoRomsj7)vKj_Dq=`!Q#SK_6O1yAo`i~fHvdq<`NB^wu5Qo_l@Gd(vKx_Tn1 zHdC*=c-^fok&~Yvw6LH}j0#UcK#-T8Z_=`;=@2@RI_y39 zdskPSY$XPA3|Va*4IAf`fYu9dOEbNbXuN)$|L z=cN=C9T@Vi=5wtGYZ%kA=WEg(lZkN3-b%AK{N_XvR~Ex>_`lcYqtH|Y#eMEaL_~CU z=ko=2*MkYkpH%CcZS9u|S|kx0!rh-%{220^VyxZA!RjSFY^I137M{ED`_z3UKM&Tv zWASx`Gh#&a6_u9uY<)JW^>v%cMxiLRB18=d@DPlqmX?YWqr(9n(~9TPqEMOvDzmYU@0S8gQPcWgzt%4928V~) zh03yMnk&H8-CL+*fPqJ)rlk$RqE(n|^LCFW6_WY%32U*z!QyzS>DRAc-&0bg!D7|a z*f}_~>TL;V6*F7kQO=>ScNe{!Q;IEa#XkD6(u_uK&11%^D2(gJ9a$TSg@x{-L3Xl_ zZX}z0H9r*aOqmGl^11*w2zIEr&=0`kM6%+&~CMG%yXKAHNSTw+t zk3k<+Yu)yAarn3%C`Z4h8jK56@WxogkE*RHN%5607o4OD5e!L~Rpd!Q0g&_K2R373 zOq&?ncUBo4ZEeSvs;>CO zMsTi$Bh4YUM=R7`U{zESUQ$95JVS&5tI?7%MDdt~hb>Nl|0d^vU!f37ewx|CFSP(8 zOiXEpmpo;nv7{s~FOSu5?R`mUX@xcs*nYH(-RrEW1x9t+V<-Qe*+x}rX-dK<%!ocE z?5&!9j2?T|{`}G?r}R?3zkAIJoP^?PGYs}OG~)uwp*#TA!o))RjLl(-LjOJb7%w@T zd8BJHN-*Mb9a)JhLor(m)E^;0PE*OdsHmu4^>Eqpqr-7>y0h=&8jEG&jO6=l8mfxN zCwz6HF=jXcE|eV5#fb$k%Veq2Qm9GVlw#_wqdEWN>qUd#zA$%1p2^l+TzNY+H6>?q zzIpBh>&S>pB_NOnZnV?s+J|QMlksYk0kFlocP=0YpsXyfstUbwR#s1s#M9H0SlB07 zzs>8c3!cnzcQPBb%>y^$b>XzMAgP19Al1&Dk6Kk!(gt`ADO@(VEpA8ma(!zv5{YIx zzv-GQ;^X4J%^vg_SN|;8w{TP7PCH6ZDTtYVacdy~A0Q)xXb5m=8Z5 zj^s%{am#I%kuOJH8T&$4__$hRugRa32KG%zcE!PuA3s?Ax?v>@1cJM}MZa~a6Q&U= zo4!sOF}7u5Rj>Y|9uDpmbh^S0-_9!y9?o8L z0~{Qj%Y)gN&4C!eG$JN;0P*TLeA5*)tD8Y&^V% zv@MR-T#K97v$W`Tj>hky0uSy(z7CJoi$`xr1(UgJ8`f#lGcj##B^l7*Mx>x^&Oh+A2qjIS;n_HF5^m|N-s6G`VbrP#%*mm= zzd4zBIO$NN#@st?KisYL5FB52UmcsPHYqMH?gGe-6r|bgLf_(kBB6}U?|vLPy(d^G zL{e9qn>63o&U5eH`2~73g(Jk^aRcw@{=(-eZpmV&r9KO{InzV+o_8dpW5n0*+uuUO8=Te)ko=q|Je76@7h) zEMedI76-IKh0J?78v@9zSi6W}9QjpW&+aqF+J3Qo5}TF=PJoa@+Z7ExJ^fBb5%-j@ zASqg|WK93MBYMO@*nshm#Keu8hgh7q%DExY1J0Wl$YDXXc4^?l)YWlKDqB!X`-azh zLU`Q{@liv1mNd?*0STPi{>@GN9`7JViLUp{R~XOTbxF*6IDQ-T%KptWpBzrMkB|G! z<9hDPV|wnZ^5EpVy4IPRnQ?dyV_zRFj&LZ7$lKYK(;uQRnaWKX>^Iu~T5VLX)_l{l z>^JUKUS3`>{+Tahjs*3BXKZY2+qHd?y7Lb9ctGf;*59E{o27X2gcf4~Z^1&4bYRbg z^`@dFh3^8F5f}OEdnmBi=&(2Plk}-sc+etdhB*{pa2pnVe|vA`yy69*8YMwN|3eFo z^QTT~K)M`JGu&dylWoztQS4je)bT1^G#m3v%4sXJyLE}2)c#OGG@hG@fA{wEg3V2F zFDn{7S;N+MUEc+vTrlxk>+0&3Kc*aVap^8=aQv#an8-}$b^4Z`PV6I0h8hARKW%Ai z+k0~JNkc<&wnC5Z^FM?(udAH0vdHJBhg#cqO!8c;LPj<6WOn&SY&2b zRG>!miPB+P-#Cg6u?vjAe===Yl+PxPk{C-7*mMs&xc`l; zG|kC~D=Tnf!b@y_!|u#W3|?7NTT39~CunA77AQ&~NIJo{NYvLIX(C1$>o&k4SlZui zQjOo#)FgO4XOb@H8Ml+=Puj7nsGtyi_nGTDp19wpc7C*;;7GO|Zphcmjf?=TuG*KE zn7cs@`*lQL_izG(m7Se<95p*TJ3fy+F%gfwn3`Ixe-vK46G`MV&-jOF&U6LZ#d^C< z@GvNOc#_UG2QJ1O{Nl-8ZU?px%g;J9D?1|H2)nuNggNd%sj21sF47tu>$Gn$sn+Cm z-N#18W9EG5YAlR93HawHeXL=^s*e4-bmqQTlF~9Yx@Y}oQDdmT?<4#hLczcjpPd$f z3u#BnY6*s{p^PJ=Ehm1%sKjfxXPev3k4~{>d~9D?9m!*auVGwB2#JV72TV+N{+;%a zJtfD*joU8PC%imvu`DiI{0HpRCgH2=J)W1QdjXwE$2ojA`Sa(fmjPoo0CdUTvwDBhizzUp zSdDufxozhumnDNTf&BgvkAEQBgx+8BO&)Xn-y_WOAhW0)* z(iaWC`(DLR@!Y|1QC?#6$)6v1GZfs8m*(w7g)q8Or21qO0>TG20e8t6+~xQ9q(Vne zZ*-C=q{)~_%~s4|d+QjCMigzVJlAeVU(~Iq~){E;!Ah}{wQ~P_R@?=>p~$RhO;GTTWU$L;49;ZskwP-_mt9;29*!TT^9o*V_s7e z0hp9X(Fz&-PX7im_+9sV!KU4tDaC_FBh7O^St6oK~2d{ZT!})m2WL5`QZV zV)~3Ja3h??xMo@H79d9@1dYsmZz})RNBFa|bJJNr(R{TDJmBQLM{XAz5iIOZ8)9JN z4oyrHf;o{jF#t{=pVuXwdT9cfN+>)_orAGdvD9Hliw(8*%luQ*(@i%k-h01u;bVlZ z@c~w~UQWpZHt%>b#>3&Vt3)krJd(&%X3#+@9S79MlG~!4%gG9f{4_6kQwEI+YWz2; zb`cV}a#_M8;L|uJ1Woi~N=n{+Qfgkdr;$$;iy;+qY|{OakWl@3OXkr}1P1PnA)s^+ z3{=jQFshxOYxCx@nXTZel%(vn#MZA&+TJ!#WY&{$b>*sIG^x{0;d7>Wy1)9Uj9u@r zI)1)2LLltJ8-hwgiH+nmA#|gps@gl@|LhE&hBz1@)tZs{vPr4&Fw=#Kv_e7|cm_Vz z%OeZuX@J!hl?*dfwswDkhS^j6!gZ`p0G39; zW(*6t+kc9bfo*YpygUL(f2bHGguSqt^jBF-kb(;g>@zlcd44L>Yf1e3R~?{f)AfQ) zaZ%CFy1IlAGQZ@hDHX4)eQ|w#Qh9m#GR^95U|}#yy1JwCJj-HUUi_pn@Bk;nuxLH* zwvvGIZZ=+-krj_KHLm7;{W{3!=($o^Q&Y^ugsRJsHMS&M*3`@*n+}OVqy2-LS^^)d zKW1do>6ys@uhRyskoPq;3rh@OLWgbwuV3GR5wGG@^(`?G*I7jhVt0HqrIu#zJ28n7 zQn1%&PUCUd-dlaf&6;rk{oUYWIyV;=+e_BP*@F*+WSPk(eO~K?K0iNi^t^u)j)ctj z`BJJ|;r#7k-a%ytsaEX#?Qsi0#3=et?V|dAq|AlLU&24)Tplf=$*XDA`w*bGy3QTC zed~g%HD4WG5%SjhWk~^^4xs()&f83fiw!X!0zwQM<~7@W2tJ0O<{P{`6VnNG9?sWV zS=$=#)#Up$I;_67gZmf?|LFI8IUtgA0NIomYXNB42UX@*@0RRu2F^|UjoE|9@43VQ$xvYRXO;&lOq;sc%F08Kv^yIeoCa_?Nr3+=>3ou9#V035#flji8Xe68 zFDK?#{UAuX^H;}#3OA;9CjhPSrt}5_;g-+w=Z}`-=7^4$C-HahP*X9AgF_b99oNo0 zz?`SX3{z54dM`*?SX?~jeRvIp_|WHwke^?O6(M0|Ys7z1h37WYK3kIcOtQV)YcNX9P_2 zX-+S*X8hTi5wzJXQ*T$O9&evvvh~=880BPpGl>pHQnHMRdl@p8Bpz~E%rKtpfz%{+Q5V^pmP{EUxkYQT~jO{oSH(cYf9RK0xTPw}V2)s8Hq zo?xJ7BqlP2Ag83UF&W5Ea`atRGSN6{%2))z{~=VkF9NR?9I#KJ`2__kN=kZghoL@;pFB9ds$8*hI9-7%eU#he_$c+ zr#V+VHc_@mQ$8vvpd#Zkk_!kBRn9JsKiIS)++QD!Z1hF^b_G(@M);q^3txr;we$uU z0RZLM999GYLdqsG0?oi?HAM{pyvMr>$Q2$Kyznrk zp_`?iPl`g~f}|EJt)4DN3kd)LH$FF60^+5n1~5wkOcyXUZC6@T0M4A>bi8B$XA%yD zu=jFrTK=q<4k=e6%J|@K`Su%~np1Z3{>YU1=$!rM~AD4xlq=LYL@3pWrZH)*8gq$g_*r=gQpW4~On zja}ce(a45}hYt=8=9TPY`#&tM-WuN61Nyw&a2=&A}8U~pSFJxGI#rE?BORF zO*{mo{g=qdH!FS*c78WYwSl79veD}7wy$Fr$JRasG`d~lL%uv%)BVee2lEZw?fgXM zzpwD^UAJdg9_~GG_L+Vh$j^6*qS@#MZ4RVrYBT?Kb~dlPJdWor?ylY2M^4wNZf8Ed z3;~==5wv1wX+7ncvg1~no|2Z9HhPFY0=^HBo_sDvud*f-m6b6RLIVE$QvJlcwSxyh z&w3d3M>Svu!y_ZiR^3aEeqg4Nz3iY_rVgXbDsHYzl0xh4lsBwAYGUZ67X0q;_Xm3C zr1b(9Ag>{-6fbfA(e?L4AeN(uAdpGGZ%a3l*dAO#OS9awUPXY2KsIlWMI&bl1Zm^4 zJ&?jZIpW9!f|jg2a%sUc)_prS7w6yZ-Vm?{9xu0WSd0_@Ez>f&4q832_zmdGcVZF0 zSM>TU8T_vOn@$&nBF|{R1*GPojj@q5kQMLyK#oDNP-~%R-*V)JO2#MZ_STQ82clP(!2QNx>fJZ_YPYGF3~bDamzbVf@iNz&4>o)3X&rRhcW=1T=7 z1s5XT)=kC!{8Y^zbTV~LRlesbH@k zR$qjHi`nLVlgfe9A7C0tO@xmFO@U!PIXT%`TG=iGh*6i}_g}v_3*=I|OVmn}+0Bqn zI$ne$@K}H&k(!!%JEZ6*TBwk5dgo(mVNq*4&ki}=pETcvcmFNd(Q2?K0nR{~PD3={ zY_VX3lA{|gH99?BWW7L;W!Ec*Zl=*`(_=G=r>MLfpboGtL8|eViz@-l`}1yOmMWlR z!33kC3INpU6ae+cl^0@24}h^$jwFS`=|dJZHK!(B;JE@%W?{=3|H-OO8@$C9(1Gya z+5oA6Alc3F^4Jp?hLF{PWfv6{k#lgw1E8<*d-4MJjX}MP^+5oP{38c{BA|}}0t3Ri zwpW-{(ArAevTPshLjL&vOaQL@+!?1vNx8nceYu~sIcYk#-kjSKi&&$e&Rp)tO z4soqwktDo900i_E6<7t3H*Nvna_9i?LoO;Ag!KWy1<($3ZZ2?^&7+%R9gGoL%5@rQtfpyzpAQJ|)^AC&z9YBI z&rPQuf@S(`so-3wmuvSGDd&zBh1O>Eh&hE_8|`ZFt7^x&P$R0!!)O`Ll&FB+ihMuuKM??wFaiCbJqbYVF12oC4Os!h#~6xv<^^{N2xQM zoPXlu(?S+E%@^y-!R>Q9TIu`p472;Y79Q;3^BVR;RQN*0j2w+heURHI`xc~HGWJ(* z4xDywv2q--_gVbzGod^zM&I*K$3=ROLoa+z6U(=f%UI5sb0|JK_io)F35S zoV31gJw`imI12C&qvJ!_)zz^D1?DQP`t8?O@Vl!WBJd+4@(+jL9`j~Y)C*1pqR@_f z4!KfU@pj;-Wb#ai9F_CC&Gk0{Lm-k=ATa<3{%(`(`90tsXYh|;JEO)7jKqsP(eDxL z0Zki2Z1Os238+b6(fz@_Z|f@K00=)cIJgF=%O%-lhaQ3_u(3s0vVaSciFR|ewP%r; zUsM%J_Ou%U!57bqV~eX2H7PrcXn?ohzKOaWE{z*JU(yDnkc|hR35^m}R=(HZiW}pU zpLT54Ez_$F-s4f+aqoB_8Hgb>x!nDIc`}si{d9rJ9dx<6v$dQRfsA9#@y?d!vg*g;~)wqR5ut~E)ljWGG zg>urt0pY>uQ<6G$$gjD?Cj1`<{K=&;xwyDA$_RDYC5{b&kM6$Y6rvUTck@i?)2EL> zL5&HH0oa5<3}9#YLSCHU$H5c6 zej3DlAiCxNJU@|9s~ZMQuoEcPo_f1QkDFz8MA%nH`gHxsmAOdG!IoTttALa4HY@Cm zXV5b+aGXqoLCT8HnN(I%Svg={yYAKk-2Ek&Y56z!tWW@RtgTDr97j3?LOmOehvQ$%9}M zz^A6ANz2M|#1H_{F);zMRla}Q+pi|U0ChW@GxGWlV#hJFmbF|TR4IgAAU?l!kuxVn zSaL4d^U-hmM0Nn@?&|7F(<3GN|8xNaNo%8kmDIAV01RK@tQhHdnreVR{m~?{)RL6(2uZhZEb1%!78!h=T6J`l#uY}?_bA)`!~n+&JCK3oSd9W%1+<; zoVOv2@+dselBj>UIKJ?9n(Srp6oYCIEP3sx4V{Rlf{_sgm`yZ7USDH+?nZDN@K*uF3*h-6p{!`yCIq~c7bMm|>3*r0{xcVvqv_a$85D|L8U*fS_xkDE`IBHF z3TaG(mwujPE^qEw$G{HZ7Y%*?C)Y!_hXEO@IO?$AdpIx=P8)^NBUM%O)Wlt9tWZh# zO$(lZo2m>Zj167j;?c3ANJ)uttu_I-ExzylwfHQXMwM2B8xw@cygbKeRkeS+fCRLtH!vVucti3VFPpoBZgq% zDJ5{ixjIeL&WdZvhrToTVPIFb(NN2u%N#^?WyCIoyHfdvQku1BFlqzJbT?tg&@Slr zdUHyIm0@oK4<{#-&xR;B9wli^JfI!am=sJ*W0&lx_^e-m8O~ugj15#~val~dFocXj zJOno7caUZP4hK9^aFrl=89#%SjS~azxkLLM4Vm99g`)5sH89Nn0nx=**xTfEks-85 zo#A>xRLdZ7RuV^kKS%&M6G&-LvnDKXbKCQt<+&zusXo|Y*;hW#o*U%VPsI$2P>z4ua;Oyb) z5zr_qV$>|E<2N7t05X13RSl$;KAfu#0KSm({dIf}9g^D2OlzGzNwvqQuE9KSOxd9= z5)!iV48uhQYlQ)5d`yVEXz;rpLtKD{6)z%4i-3zO3Z!O$t^q!@m7t;Dwe>md)B?gf z(gO;t0N3TVTL`4?0&|fLi$+0gdVRwGp6u1v&esul_%m@Wt_S@`4XZ;s4&|G`e(Bl! ze7@p+qTJl^1CrFn#SH_8bMBWFRtPFtPJ8=T&x>sgTf4qvsX1y$dX}9{s;5ycKx`ge zye`OUi;_0hAHFrxcvZhRI4BqR6P1|vyI#vmVM$4NbF((^_eN2s?M*M|s=**LuGY*E z9=%M{s`hz(b>)h8h}T2v%%3ru;*Hm^#2z8V+l}o1Ks1uba!?h1o=-W#4v+c+{2NcB zXnX_T0?6@fwRSl;V4=0hGBmG!`)b=|5zX*mdEY7Uq`&LD5jFHj&8ZT^6mnvK>HX)G7^j}&1flg@OCP3L=+=bFLW5h9i zW0cysrCikx@*k(Ua(+wYOeW4iEIW_mCX1orVc=pzAzJ*d^uJ=DQB3LXi4p8Jcsa)_ zDJiK)#xmT$noTRND%B{bL`NW(R!OT>*{ocswtkUgF_T3+dmQ6k3cuJ>O^b z)`W}uM43}0$jz0-tmVe3CrODxm=+f|b(}-3LEEmqfrlyK>g>z`^8K_i^NI@{W#tbI z_RDlEEVMvwfWe~zW-$m7n+o;;isAD*8n%Qt58;0sI=K3L{!e;)B5ML-!2>b+_5Bwq z>Tf;ktfhuFspi|TMEcETWsEg7HJ}g%`wF_*W$)YS$-vHqGN=rInFgjK&@W)7Ntl`C z?|BM%4^ihDOgxVcHi2;REjj`u6|h2KRA(~{sXLqbw~W9 z5zq=|lTBha1I!D+Uh+-2(;E9>C70ZI@*wDLY8{HI%7 z0%K9@oHl6ygCOu>Vv@_a#{o{4n2pWB>sbT&`Yv@w12}$QqQ}O?fy@o*q`l=Ulg>(R zQDZVdr!-KlpiL)4Hps54!x2~3Y_-GyrvP9BAx zUY}FGWv$UN@=UMU#q54;8WcbtkM25ZT=r~u7ZTy`8-4A704^?G&qChjFzN3&1-^GY zs91rHB54eV)f6_V?E*(@aLsVt7mf{xgx3qN4S=GaCyja23p#fgmhBHP@h_$tgS6`L%O^--GT;6e|>#@fNa-x9_0ifBi04eF(12}U3sv;^mJug69U=z z`1nefnMcU$`d>BKBg3ZxIisHYtHY)0p7hQpe7q~DMcexN`QR8C<&#S5t?Lfb9Pj9W z5ZdkY(_Y=2l3@~)8q3K3yXTRCuws6e4ugqOZ$fD&I!;*3rm}{ZLF4N|xeZ*BmRXrX zRR$ik;k6qFHL4Lrhm7?ph3#nsDo#;b6bfBcRa0hHRRA7I%zAWyrFRD+0`>)Zhk)9o z192tndtU`IEL-lhCLiURs%lL}a7E5SmGWuazY7(~+ONtT&T6xsB7s{}pqMp2#N#PVG_RIGo1po z3}{_=`t(e9fNKL5OFwX?x}r%0EUm5CV>$p60!RTdD~E=LvH{wF@Gs;1{Btc~dP&KW z>#SZYNDqN!>DYt9!|MO+13V6RRAR&7AN1ypCtS0-CJKhC8XC2}4{q&eQ4YZO344VM zoTvl_jSaV!vA3as^_+RNK~lwnLHId9WfTR^U|HI4jayXChJe6ZvNK|p2)&>8b!b@R zbJ>MNGE2aVhU8Andd=UHk|gEjqxcAZ5GR_1Dw^c%tF-fU&$;UHObh{xyAWpWE|iQS zG-F0q)6$X_7w_YPNntZ`yT7#ZJBu&?jkwYDNQSFNLYU|7pi0NZqXy@T%RRf_(4URt z>W^HkZEWsVlk}T&_AvWDYrvx=)*5z+gh^0ZHjktEokB2jXa2=2*hVtO=pd0Pj6fCR znGq7QT^P;@D_C@6*;Z>pjs@U29O64cel`fr;t{wsGZkdoHh% z`#3)gY@Tg6^3pK~d|?~`o#|*2A?t)G|%iq7d0EZrrBtFWizO^+HBu}qZnT!SqcV46iR3pTV_Tpxx6y;5TRV$Y% zm{L#xF`NKm6AQTgD9rLB>{!kE)tWB{JRva3r`sgah*1RwqDjr3MYb>M;7QkR{BOzX z?Un%{AR`~a1lwmb*lHR`0ub6&JFM!18u!)~yyr%tBKr&33*fyfP_}LYFYL}V3L#i( zpZGNO$AR072p7O;P@PV|@f-wfB=ZS)veOmsw6XR?YS6JTuQSt9NY7ml{G^6sARCe5 zXMwGA_26sn(Nu8&jvzPz5LuqW;m$1Ldv3w`XZB*f-8b+f1+e#l12)=rzziYPHkEDt zQ~9z@AVmR@%AdVYt*wU8pi8s;Tc(@HmQcv&$S#&4g1YN6R_%N+DEI7Zj$t1nYNKg(@j=8Pss6OP#g!8d6OUFz zBI85*_JGv| zW&vd$e?=$ynyqx)yIP6F|$!~SCINRZT6p?kFd~+A}w?q95)|8SadINVj8zo zZ6MtnV1W2OT&Juof^|yv2K?o97k)r= zK;{V`;qKw$(L42Mfi)4C0AAmam>R|pt!D?S(Ukl=2ArgSW+9}sJZod^Zq z+Yo#h1dt$YC(x1rop6w%LdDZ67Bn|6=qmCsFhpd%JnjfS9M%GH{gN);2#PkqzM#TJ zs>AtcDS~X-un7Do?vwThW^l>*d3k|E_Kjab``GPhK^)jFU=%Xyw+R?S973cxQRFz9pI1rJg) zDCxMt(b0beY4*l_;W*$o7tALz?Y5<0u(7eL0P^B8YEG^jnwOE#j-0B9K>7!n=;xrhD6 zA|eqO8++_ZadI->Vbf4fon2f)A3h#UD7zxeH^sn*GF~Mc_u|`bl1ZYR=p(m@-+xUa zo|b{ulWmk&tyhfwE>{TDjzIv-2HK~<>Hq-%2>Hnz+MCA9sN$EHIeK}A3Rn+8!B2r2 zlZ^yB7>+GmcSwS5jaz485~|-1%kEds0U_wVUk(#u?Sz%jof8&H@mxB4hk^c>yFTwtbwp0`xdcf$6F1eN+dDT;9qix+K4V zG-EUerr!dz#1UXeuL7dpCPg;-!vGloZdG5la@Xv-BlVb)w7ILnX1-Cw;@C+**H40a zqjVh`8;C)?w0(F4H639g_}Je>xZW%wT(x+_U1~||;6ObI1Qlg$sAon04oMK=ftT?- zZSE_#&pMlDjRchTwet&GifAPKKNcN2GJw-9Y2M^z;^dL*+wX@X-nHkFnmyBxOmWxP z?-0;s*y0y1k!${yDtB-WkYIe!)KL_=QVoGgqL$v~b5*52I&4?8FI=|gLH^XVY=0pv z-JS$abZUX#sdw(F#CaQ?bR6Hbm9B>T^r-hCsP%G}l$`(hq?qy>1)MMw;DA2f4v28W zH$+$D{;r5P1HJOk%-47A9dUdEBu8YdS$yKkfURxcbYCz)Htfjy=l-!5WZki6)L&7LP!}M1w~u!Pzfq(G1l!V2|g?JD(Rw?MMWtKXpHeV*x8wjKhMr|vUc0p*myo& z&8D!Kz=29qA%HnZHwK^@2*Lmj%?HCEUD!7j`hQXNm0?ju-`64t2#Az0v@}R}hjc0+ zE!{|WgS65ybT=r1fYRNaN_Tg64DZ2z{oYSL4)@NTbMA?~*IsKKctjvgK$HX}YEY*F z)CrzLF~E)*?Ca<=9}CI$xh?dxmOZo7|2Q4&A~^9 z<{RS}-d#c;l$B?K@To{AuYQnG#+mhmR{;|F2;ftIy}jH28R{3wT;hdxNol1MpIF}d zL;g>N_76@@%=nwR7BeQ^WxF@LliEr+0CWY|FNg@)L75XU01P%Xq5x<4a?v+)Vq)S{ zs7&4SNi!60|E~a&(P9_7>jlFz z&8WK4Q7GRtH}{nP<1bgrmEp@-dI^VLMF!o37Or^pHk+%l0xgID3QrTzoHo^vqb&V8 zbOl#zwNtPe{k^I2EFM1sj9=)TzG4KN{4(3F*vn+D|vst|LXaPW}-6^b#1 z^*{IH0?}4V3IPa~M>$@}#Ev>g%sCO--hJcl{zhR%e_mtV4DL%uqu>|8KF-r3vR*FVb;ex51*#PwHM82(BxFOL6kaAezn@CyJc zjGE(wwA*(cEDjS&N?5H5Rcrky*nE(EjqpEeuBC5|#bhJamjd=~5D8;QxxJ~Bd4Ku@4$Noy)(8@=EkDm<6_`Wus?fH;p zb@oX`I}0ZqQMC`f*!&CCcY)xSgGy0s4U*LH*{gOI56g+Vh<-R<+vqSJks(E-qZ4 z;!W=K|LOfuNWXuIBhyHKM+5e7P=yClG!P>Jlw{@Dt^T|-9P2O8@`SX%1?xiny5&EW z4p@aATT?URl~d6f>~`+a2&!5jWR8)}n{VrhMoz8L`mvvIg4w42pL#@Ub>sPrwR5py z-DXaALNc#s|J@_tpCe8CPQ23nBGD3!A@Tpbhf`hqNdQK}|6K_zP4eY$@MNVZF;vVbHg4JrxXS7Jf^ppOtuZm2i~kA4@ck z4X`r-8=^CVJ!d2oTA6C)rYq0mvk;*o6RY@_xv&NgxycH0nCd0~!_NddzkjzZ$%T~C zApB(L-7?=74Q7@?@Ljmh$!r!m@_lhUfb!pcQ_5q_RbX}Z;FWVJs;g^I^go}2KQD#( zm+aQj84uIA2p?t-{YSGfw9%%K-MPM%5*Vtxx&CL@n21h{#$f2`J|fBg)$bTywJ|bd zjlncE6wE&{dj8{EuE^t%{Wj*J%`j0I|L5YzA%~q{Tvm2m&Cu`GxNr*F3TqPjui>Bh zs)BO)B?8b;Z4$qC)}f)IdM2UNk@V*A-bW=W_U@w;wA5LkMzU=g9m$OB^S@tK;{}8e zHr0B{PVtm7;uM&y1^?P;M`e!PQV|)+0JN!A=oNLP?d}uL{&&*Ee(C28w>R zXpDy5(?P=v&Hrr`GJ>8HgIaxj$r(p9CCIr!NOZ?GeVD3Eto0w)_%jpWmUYX4q$1@T zm^=!!I8U)w)Hf+G89K^Dj<#RZqr{eGf8wh!d^@K4=X7oRzqFuN&0&=EJ+YwNlzCnZ z9t1IV8c}(rFh28j&iEMYs-+CEl)1Dgyu(R>?|LSsKd=N-ZvU#{7#Sl7=KM^FQMv^39?d)F_2yrM76_h{e9fMHwLo&523E zcd2XC_@(C6ilJrrQo{;en+Yithh~IGyt?j)hjB}K>lk^^kHw`7ub!ODU-)sX9tDX+ zjn80F5>0|fhYcn}c1K3TTjgV;;eJ?Ozn6(^fqbBP&WZar-g=4=OY7rVe7Vla)UA@A z#&a1qAz@ek)g%uAS-GrN-)WwOv5OMi3 zxa*3kjI-Q;h0FvO1}Ini$AIxTYJTRwi-m&V|2~-VEe95r?V5m(050tE`kKhrS)W3Z zcB@cE)?jUa(deevck{2#4{g`UOUQL=b2%(PmO+jG*e=>mUZR}OZiZ0?z9cYEk`iUU zd%ka=ES!A20Q;M8>r9aEru2jpN%OtOSl;V5n34M$-Ilgi1=PxIg2HKGzrCI_m-2;* zMXtWVAFMtg9KH~~_hU2@ow7D0R(_mrUXd}BBP-@rLJk~`q7%@pG70>G1g9%XL#w~F)TLt&{NIG_%jbfa0ven4#iNogo z+CROWToKQ)H`b&kiO7;sOM6GC#URe}&9I!m{NpQg>}f5IKu%wl#oJ4KXycjZ1nH%b zEnz#5LWfBqOBufYXwS3PW9_dM*7-%!s7P94Ke8jw>ElO=7}};_p4_g*)^>{Bsd^jw zw_&Jq6`#SxBPOrcu>s7^-oBx(-E-ZRN=or^5wL%+5R~4*!bztyaZ*z-ew7N`yug<| z97_H*q56UJORh9u>xKtGfW5H{1;Gayx~<-$7>X@bsH^~i*d+FFKErNcI~KgHJ5=x9 zjpW%e0XivvM69N00;4~B8D6f`dDeRSyviS*+HFhWmDc3O=hxO}%`Ry0^#zz=yFdwF zfWu;V2SL%njq&^kEDZ}{@x>$!eer~mzSLQa+6bliWzIAQ{Z8iyCPhubd-X9L+&z5wn%kI=DY-=6|KS4Ex?UIuqi;n{7GR=?sU`Y= zv^g-Af&17Td446k=mTpqKK>bmBQ*P-&tROh;aU)%%YjFSeSYgx+CygJX=L1P5!Sgn zw^PP;Z>~&Ac|{2Wy~a=%B%nN^oC+zYQ_|Mw ztKqCu5F{eXev|~sRuuM>G*7;H&+iMQ^VOUb5hp$Dxy6TO%c@J0c`{-Qn2y?;32)Bp>-|N=2W?No&49P0(rCK}}F5 zt+vk>K4i}@>D+5t`RwJ9bmZ@sQ+nlN<{X87 zPm&ebk}zr_Lv%cZd~eogYa6ILF9eC9oGQ`YvWAeS?Z1#UqK9Zr^<3yPVOOuBrLiM; z?ruCEI&_AAV<02OOdFKUs9Ie$?wU6r#@%&tXNm^5LE6Mku^6Co~ z^A2xVCrh>Eu3ddPN9>+5uWnolthfEbf)_K+AInxDh>*-pzM~CNrj7_RWv^L(2RjdY zkkp$-^Vf{3_TFi}%1ku5nv;4!YLv=!;!X>Ve1nV;_~O{xD1BPUe8_8U2MGF zAd7C}6`@Nbz20y=fOC7@wfI0WOd^eiHob<^EM}-59sV7oEeiRRHu=;==zKbA45#O{ z7@1Z^Q$U&bAuAtSe8asM>1r1z`;j=Ny&2wv%dPsAf-OGjRVDMuX!YOY4Z@H1vYS|Y z_l(GmK}xy2n&GY4Zx`n369G0gy0}<&`NWdL_E}pC2ZDtZ+h)qk)6yM5ul-@+Hays= zP@M#(0cGr*Xq>~CB3sDfZSD(7@Zjd_RpHgr2ogfr)=P6_Y3*XWN<}5s5MC-qf#dI{ z;*{^~mBMp!AaCg}Q>!TKKc#HgEx)L?yKYdfyVy5wsVGPDGYFUNF@ZpgD7U~cAc!zk zCP%YsU*>+e|G?I7yS{jMXSU#bEU?1bd3t%Drk0!GZ28GPAgtEIuN^x|=&jYhGXu*0 zYU;!3`|I)d-pOVuv(vD8yeSX;FN~Moe!|xwhhv}0R^JP`(AE1~sKVBmA0jhJt=3-J z)gg*fqqJDRiWJ)B0J=6h<@oDpwU?zj3bN5S@7xTfiA4d&(T*okyyhp4vts0tAB z!i;bo_i2P$-F2dllZ6}No3Fdz|1@l_mgs4H?YTHP5pz9RANCGzzFzSiQ+CHcOwt<) zcTYU^v)rlKybxX=ez-tSsn;P_@;h_tNRV`_&;ZO9{PXpqbu zH7`Jrr7IDM9!m7-FcWSJu;jK{s%pM9WAZqfy<7FY_wT;+3A9WoGtHkB*Q`jNtll%0 zI}B)O*n;r!zpLip8~mDu|EH&HIp@Zb3(vWCbbSW zudZ**YH*dIopNe;&WF_8+!-RW!KJjJGl#uaWjDn<@J-gUwKe%E*c(=MxY33xmK4ks zkT;sfphk;BuBH;G5^BuWyR`KB^MXj(Xqp04R92C83_r?F;LA}3Pr_>)H1fQGOlr8V ztH!K9aLVqkcJDSFtpmkyd#wP+J29&X?SnEgeTYEcZ5cmiyX za?2Km4(kxN4rKI$te?J-nW&^g#mzbQ{H?o21q?UNwo;N1$cm+TQME*{oRH9G>7l85D+nIM_`a zZ1&pKL!OS6=L@KKgDjIMuZDp#1!)_<+$R{M(>JNq)Si@}lc0|44bpzP@Pv?JU=9x7S}MVF?q3Rt{utt7Q-{W zdyCm3kE=c|Q}~`Q(#$AG6peVQ&*-Jinl0=q>~1w&YI-_8zo9Pt$K^e}HMun9(x{8R zaQeGI4UTY8G6~dXFBiPb`t7KzTNO@z#83^_c7>W$pSFu3xB(FasIKY+SUv`gC7zIE zFl;tHtd^O_GIF@`Ff>>__x@29+b4aKnz^#RcJ5{O_sQbjg(I5BTc(C%_%za8<5j|X z-aqNt1trKvTNv>8y<^qhpD!;TViQv{*YR#kH)k#%;C}3IBM_Py{cdQB8|Ftas%4Z^ zmJNJGiB*mvo%fUYB~kNDr>RgitbL9Tqn-Dm zv4nB9S6z3%MZ>@VLM@>{?{*lZUSaG)S2i0vK}&Ik%a-@8aLOv3Z`Q?FLxe5o&cYPK2sw(p)C|VdVyQ`iZPWM=P z*0r!GwY`2_vA1xBg*0YTqR|-2Te(Av+inG$cRZ6YW3SgfnX%O?P)~-?LZ9{rPgtj# zpZ94t+Hj(p4(cC8NL^j{F0{i8;yxGWzU4?c?y6uQniMRJd0-PrjudP%1Zlb= z0bkBtHAWNI9yPgOnf<2gp{bly$})^BbVGX}=y>q%cD~u5hm$td6VA4IhgiGn!fTA5 zWV7D&J~Z3u`Y%*tZ!E``?{-SvQ6th5nHuku3B=OpF4G^@l8K4Y&YYtonKshZ%Q|K&lP6tB4S(8lGdCX*+}@&?ep(y zja**zr^Z|geSV5O;Z&5#DxoDbtc>)V_M8FYPZ5(Sh&ZncBVj&zktvM}D5NO>$$-EZ5qrS*Q15hmwlwDfq$chb9jwh_gzIU`K-v(}mH;otgXb2mjY!)8dfZJNnFm$H05dg>$OldA#L+?tDr8 z=LxQ8_{^`v!?zc^SeaW%iNYI)*#yqIu%u1*+@VpKr;p#_SegZGI~5QZCUl+?-XIlD zSP(c7RPIx{ameeI_ZEQKO&;nN0mN@ByH(*#H1ho1$0rLb6HJd^x6l=DM=3DB&9tkJJu0opP zP?*s5pgXKRmOQwFT|O#>{&Ip+s1JO$Vp3}>+(FYW6R2TS&z->Kl+*UEvA8aJa~F`+ zxVPT|d5UpXb7tn-9qDing z?xA>z9o89nB4qj48|=7o*2^l$eycnTAItrLS``xoVS8(Dsr2F2w-yMsAM1fz`@4np z^P~M*^~8ZSO(;?cLq= z+r?WFbscX`AXcCu@kE=gii!=H$bM`(Z)$iUiBJCveKLfM-ADc$B?0KDSM4lE0#W_U zjcJjQ=uNkJ51o3xCvl*z@4xfylZS`L)mUOvkD1V|Cg9Xbore?T?Ve2fUgO>U)w}1! z)A3>rHn7uwfb+52yuAKnCfpwyPp|pY4(@W%l~gW)$pn=0|NNbFynMKa%N`lk)t_|k zR`9$1zTMRA!M6Z0$6rf2a5%d4TDu6^euE_3;8b@`T2O^{w|HdysZtqboP+nzSR3); zJ(KtSz1QhtlZ2)P-Fkfd-CcHw+uCvnD#;Sz;;rt?%*@rwe)VbZd*75W=Gs6hQujc8 zmY!$d)s%E~Tk(V*+)I6jJ>6z!%M8u!qVe?HG63)ns;_Dvt0Khz^l-wc4IgNLUl5Q;y|*Qh=qpP8AhIiFbEgS)xCrBO`{!Z^FQSm8l; z1T|w;j-HmK*R-9prFEgIcqN9LL&T3Ia!}fE;p8^b!fkA9R8dvS&g(c8AlTWRZuG>H z=R5M8~W*zwR_-W$1t7^*qB#%1QQhU}lFb`ID3HUSs`b9P}bApNXfc+kzLCmhv*NG%9ZW+8NJHf6$jl5M6a5!u*{6O_06!84 zlyNTJE706Yd^mOQR(hUd6aKjYc^SIdpol4xwRB)9d{=E>TiYAn!8YX$K@gA zl{ylg`FRH2y1;aP7o2dN2g%Cczn$l9*6Wz4sHi#&%Pj<5Z?2jjCBj7A+26lgY7OO5 z9G{0mV~2)@ESO*VhHE59ynp|)y}f<>U3Yj#Vu%3*4GoP2a8Q4&FR!|j02b}V#f827 zP2=C5n7SRbJ`^^a1%zQWs1p|z(Q$9Jd0;~bW_fUTNeU!Mvjfl>{J-qk;yXVrj0Zm<;| zDXCc>F2UrLXcbf$ynH6_lL+%}dpn(KJ1f5BkkV_mqs;f*0>kb0Y5qi#!Bz@*V5ch$ zTjdF2*dNzHhq)!$;BR-wwyF>y#T`qwo(ce0mD=r%^Sis&%<|*E>R&eF49?HaMkXhv z_|m(ab+Eo7IRAE*rYkNg$pQpjVDSJZlc&q!awvTnnCvW;Lqs<>w;-QhdAPrPSqr%3 zk3!NgT;_u6!bCvRe%h+Pyy39sC7f}ea^Scr-F34sZ@eY$2D7C1!%uQgXKzS|Mws{H%|YnM;M2pf+U;fMLB>bVkwrwCm1P%!A#`p-5WJtLZ2hj|`K$o7IQp?v=dA$OCkf;FS9Z};E5IT&I< zK!AAeXk~)5a#Y;t*5Z))FM&wOeq#O}H1bOYi2y931K!egX1B%>EtG)PA4=^Dq<9e|#mYt3- z#s2>J^AkA!DUmIV2gQ($++H4LR^3($4CfXVeF6R{fV0y*kmgakDkTPr*tyU*f!`F= zc?Z&Xa;fO8DrX2S`*&_mXDa2SWsj7_zPcOOSX8v(nwWVz?rf?9mVb@2d;n6 zH+#Jbh2x)JLs0ycfh$DYxz#M7<*^G0X!rQCj(L96)|P*>$kfil!iYL)kJ0u7r&aNb z6}96;d+U9prw{U(QyRPE%EKca%#JAJ;NZZ5F)qeKLRJ&`d1mR!=XN%*UV;K;h%p=Q zzSDrTmpho9EG#S{tc(8P9hBEBR?*lM7_3)d^@Qo%bEu1VmXfzd5rx&WU9R3-0^*&W{#7A$HB!9 z?D_kY1OlPmQV_;WE(`D${5Vn^B+kd%P9EP}oRPq-@Gr73OR|H%l9qTA2w8u1boqzm zaH@x8qs?6^kn48Sd{zDNi#JiaPd*Xt>j-Pvl3Xv8JMHv(CH#vvKRo|2_+R1ZW1d}; zl`GB>M{Erc!++tLIY$b+*gCn)TnYCmPt6ffjYPTXe9=@17C_>xWO&WY`(cwAkG-M8k*0^BuBTM8C{E1Rs4jPdj!l#G+x~ z@u{oG5Ov_GIF#yLmhYOyiEmF@H6WOKjk6JedD^Ce67|6W<;7!E?T(6x=oXn*#3xz9 zBVEBQv7htFMfidG-ApUXv+)}FN>i+y7y1ftlgEk~ftb#u?~(Fx`kr#{F6R;F6)VbS zw7t3;CbfQ@cBFP)gxxVZ+_mfTF@lPWb0>S#9U$AAZZ(>C+2Yv^(#uxPYq#C8jkI^J zJ8Y2KB{y+OI!+f$DH2T4B~u-N?*6~8h~?0wboBFxhK4kHe(#Ip@jm}k-t>EPShBBm zu&zeaRYfMbyt}P72m>*4b#3z*w=j2FUjs{dQUY&2n#*HTzAK_w&E}M3L*?1zPRlMU z?lvT#6Y(Qexy0=i-O3jPgMrzgEq)a#SmgO+#;wRg3eYwdg}%PknivAKcAnp}?)i#e z=8TKc36Urw%pbBv-!8D{;!q3?3o4uJ{_`OkeSg2n<`8wYOwOlez1iMA2SBs)5xqyL?!>e~w+-@A9~9odlyVZIg0 zA?N)u4iF17+U6m%1Gm=CA3~Ttrq%{r4}S8XyucOB64yl!TCE)-^`SiyNOaS*FSTWj zkAvT_%amA-i%e+mDOcQzB)Ct1)g<*Sw(auZWu2%V*#lp$No@0i(g{;|P`h4J4-z}h zTJ+&}W~43%YHbKFFU}yXC=ROOc8>wP#?>fZQ)1d^)G#JOtwi=oC>D9~>tkaaR*9WH zl&og!FDm5T@4RWih<5BkjxMFdKvh&$mp_CFd0d>ukPM7Cy%i*`uXjR-FdYM(`T|+8 z)Nmq2azn?NBw5@A>bi4QMxXSZ0N>zUMzoQ~=pXwbLPuReE6PyJ!dC*$-DzR>c#)%q z_P^d&9J!(4RoV{F^4eMr=kwh3PAzn_OD@GQ65qCbSs)<`Q7X(Bffo!0G~v9cI-7%_ zT&gv~TiI`;h=Gu6{j63wvubzfuMxpBR zJd`gJhFu#`P^*zi&KaY|_|7g{5;sMPtO>gIU5ek3Xsvr@y+R36CO`DfU-CPDfxk%q zoYywjHdBVGoG~?v*x1~hwi(YO&~@z6cq6kvX7VzEhKvPe;Voy59))1gQ?aXzEzB~< zn!~=OpG1;b$0UYPOo)@c+|GpI%wHx%0la?18n;_z7*@9S-Uh|({RVs4l>qFa ziz8M+ep?}&Oh~%<$2A8mWM+}dirT8;d6XUcQcu_E>Rc`rn=2A-OgOE)za{H;Ol0T} z=Dh_F!fV%nNni8m?8r4GlBy4qeg*_8Dk>wR3-&YLeY&kXkLNSHy|x1d`TgLN%5{ah zYL#Wlmyt&`!iR>?8TyCtdAnVhNULRNF56g#gffusH8!IH&>llCISUKAC*I!PNmb4R zh(w1J@0;r5Y#ERZmg>-foDhu$i!zQ5&2f)5tk?i6gAb-rROBE-6-OBow~B3XqW1cA zYOg$gfDTx=7v4aXA;DQ*rgUEqD2rh@E*%A)wxnN&3G)jZk-;v=X+f__|wF!Cz zQndPp>8VDOU$a9yf1=I(>d*c`JkYozU^?(U&-o*v%M{bv;fOhPtc=AIRI*4moT0U( z4N9eONd_;^Ol{kgTghl{Km)+&|Fm&(`-)5D~N%!8gRU!dQx zQ4#4GWYn~979~R`734_z!ME<9$niJqk2wbgrs!)y8BOo!YAuFE>~ONL-iYzEqlZ-e z`cPk#qM?a1e14{eHJGQbjNg*6X)w$6;sty0w^B*jubb`~2{ORd?0ChkwCdZgp(ZYH zlgR_g)3&=?MuD?Yo{R)4uCXT62GnqmNq>l4eWK6e*@(Tu$W@YL|{@= z;fQJcYt9k(0&eUBl&iy@jY*97x%{knZ#$|bW9sbJ$&Mt}YOM8FZ04Uhu>10!`z0uY zuX8&b-2ECTiqKA0W#yAzm;c zqJ%FXej*~wl%R8;KxycN%h0xE5&5wp2}(tN`dZXqhY-^1J(9fbDQufOmcmaG;M{DiG(~$=`b2G>e$zy2a z|Nb4w5KIMl4FVVm3XUp-U2@5r70dX*fdrrttY}7kbF>KlT*18P`>M5v93YrZg9y2f z&tyEw$4M{&V-MsUA3iaJ)4*16MflQ(5(83}Bo&^3C2JHJB;MWj1tyTB$56q4mFL4d z#EjJyo`afS)BjOE=+-HEB^oW8?*^+~JXvGTSejUTzyK*3`R=5xKFDmnR(JI_bq?zi zK-munQMR`aHooh@*zkUQl#-C3`dHl%i2;BE^-;yemPFWb7=jp?T8Xv{(A@y8LsG$8 zpa5N%tG!$KN+Vr3t}!ucxw*<)Ekg&CGay+*L_#XupX0?EJ_ViTmLbKKoOZ!>ED&YR z5vu%2Yj-jRd?r9blPupvb^0A6%a{qpWywXl-q!+eHvqJ!E}=$t=UE$zmcSPB;4s z^Hyp?#(Vfp< z{Hm_TMhk|JkVwi#Sxqe8Nre5hPUSt_7#*GH8r+>c-Jvh5FzMgbNKjS$1=L+M4fwp% zO#org)WmQ3)9ZjTMTW}Gf{yQTs+k3J7A?OReM($b7rR7ud%IGMWo`r!s``nZlK!P*tyfCby9PnPb$@lxZb9T{-) z?1;mH3&FOh4Wbdui!Ex{B!JVT2{TCntMM6St6~x!KrH2$>xpJ7q|s@bv~u`_hjdSj%Ah$8VO`Hfq70m4(| z-nBe7A2OP8d1^6WX2m_R+a#_#yP$g=ID(fQ$$Wyli=Y>bCXtv6RF>Tkk42TsW)jro5CBTFbSVU9_qI+1N ztV;D>`_9kLk84pk3ETzR=;-K-YmwM#P>Ui1kb5EaS~baEo?PyO47^{?6Gq!#qx172 z7Ev&517x-zns3yg%3~_x!;0L+ksMu`;fSZ}*-%4${mqnzh0&gqYaic zqR3!jE&fs5T4gE;;paAw&AS(w?GzEerAZiD!OrVLTIc&JlO3M^%X$wLH zds|*vS=8-8eWLKyQHcAtD|5w(zpC-LCR8OU!CB?Yx|>g;$AJI=W8A2@JJH}rrkBrY zB`mC23#?hDgb=z$GrtQ284&FG)&_BsYQNiG`Q#$hm@2Zjw}CaXx(d)AbauJ~F{mh>GeSAP4kYs@J_5PQvTC|^aMLC26F+9n zZ7o;F`kC{;SAxAh+k({L+o-%NR0V+?PIgxgVoumL=m9abs%-m(c60eMnxIFq+8ohO z`U&0NUxS#r$!_BJ1JwBf+?SM@6>}_ThND~d+!3V1);@LLQDo`zpn%e9zzvA>RpVh2={r{`K+@xKuY4qeK z2Pw*Z{xED&fM`xzA&&n-A?vTQjvF)uoA@8IsC={gEe79@Q3jA8jJnFN=zb&-4}xHz zZDE+B_viBIF?|RTA$<%US3hDuWfZB4wNv_1{qT4FY;S)(g2siIyF0{*G6HX+@`%{+ zs#924>=LG1vNT=Ge6kL#ma@dMq+aLe6&G843K7Yqdj!AGU;d23cskC3D*3}?R)oo@ zpxL*Vv<2oa`bDYx?p}c+;)Pz!ZE91L%H2OSNnXLBjxlu**QwAVCFmK__<#i8^$ol0 zfPgGgB>3%lkswko1Gz~7_fx+!oMpG9u{y(>kBHHw0#)r|pRq<7>j7eOF1L#kS!{2P zCaP#>6#*l(pW|71KMOaWB2(OZ_m=l0gmpqhaG`v=0jmg%yOizDo+4_!hxC#Kf#gXL z{&q=aV?6C6l(i?0GPd2hqC$U_RMl!{8!GmZKiOLe-lapdm%UycFIa>wTA8d-`z&m8 z1E$^G?)a*7b=g!W?N?Zwi`^LdG&EpShS*;)Vl-9%|B$lYY=x`_IS3roFCC z%U@`bA&kX1?eAo}C~-QYeww=7@!4D24k_cO3lZV@Y>}MXP#;W(EF+W05Qu_{`Q>Ah znowOQMp8nnzM21@k+a#zyj}oF{5KQ_@C#X$Av&7TbV@;JYsKIyL51v_l4yy|A)Daa z*ny^+uik_o-(9ock65g~F)gz+Jm{=sLGxpI*2aP%St=f9fGpoHitJ!8RuDtqXpCHv zC1d8u-Q2iJRgt`R+&G!yi1nX{Aa>mQ&#Ec}6lx^-j#AvZ6YVHzJgxZ{{#I|~x-Xba zOTXn>nLeMt^(=qShM#~@>#;71lF;Bt7FL$}K8L?oag&8M$cbufkgD0ak||rHNKyD> zG(C8gL9fb_oDC>WnNm9oLaMOBGv;IEQ+h-G6LEZgjIAF(#L(#|fXI)PG|?M`Nspt} zlC%DsYeRcRe0~MrX@6f5nJ4C+4$W`32BZt|habBd=!c9J54EOXQ}5&RW2UrYwXII%NfRlf8HWB{T~jBO(9Z8ViI95 zpw2cG)4~!Fcbc_viC%~}=CrqCx>3QGldmKoMI!t*lgw@$eRKsm(PvL|L)(XBmUOXm zBE%(2g(+_ujp*KN3emKsM|fDc3aund;PeMwec+D!E33QX z-96Z#5E^}RnYz8KZ?FZ+T%7Ud2TxvGYrz*`mUir=T%QXu|Br}Z&e1LL7~|o@OFZvx zKdVM>Ew`RD%3_fZXcZGXBPLpE_EJ)3}D zlEr&*`wnZI%Y1rH^0G-Nq0vmqhV*BlSXLqG|MwWF&rX!(-)LKNo_HhOU>5lC1>y9` z`WfKoA+kH|ZRMEuqJ{Q{MK3I47?U~Z>I}y))Oax^dRcMyF5ocFUB9_ULi*`HVGwm- zk)<+(?5fOEpWzdivjYh$)BQ5=Oc-$BTWh`{!4ke-(0$W)JKj!`^ya z{h8n)M8Ov$h(G_A6T-{&G!N`nT*ouy9vA0ig6=a5$`VX-ZV&V!UsS~b!2b5rYXwbf zI(4rVasu_q%4^tvqzcp2df0;AX6H)w#a(Wt15Ngsy! z2A%a!hKP)_`y$-@KX2Xx| zr7Ydu;F|L!={1hE;`*rA11p!OhX3NF!>Y-%QB9HlZk6ZVfpb1Urr1Aa{(Co|f?U1s zWPnabRs2X#mElX5ES^*Nw#t+1wA*8eb7rOMdoxLtFHCGkUI%s_YB*WCKm$egve?!k zgzZ+2V2iXEOSLa^p8T#(MjHML_xBB6mzHQ%FARdZXr}-6X63b^Xq`1oOZ9iDZRqi) zh88%aK^-L}f3hI-eW=mgSIDqRi-og-AcDf~I(@>Uopu8A$D!-6AsQm|N|^T{MLgiy zo8fXC@_^*={81-gapC}S3t$lgf-~h09|AVrc_PKYX%)w@RG@B2ksq>Ihv5GfmsxiJ z7&@Y6Yy%$xC1B#hmD*Xw@aFqB#csNqAOAu(l@>h~E;LY6Pevw-2FHbmBn#UL0kds` zi#^_{Dh1HfIXSW5A)ro&8qD~4w+`^h5*T!3t#ybaSM^h<0WSl1Jq2&NCl$!JOLQV! zg36IRt*p|7N}rv2?3q4dJI7ED?{1>FtkcWWxd->4Nm8$XGkq?zyQa>sMF6Kvczas5 z*x{_5pj(f@Z%*;PkeJ-;j*l);pRy&~UDb>+@S~%LMxjFm3mSN_csfq3kIv=CZmf@| zcz^@~fL8*CXD{RyD;qA_P&EPh17z7#XEW5nh)AHuyhxM&4m7-1@bV5S_Tz_@+ahC9 zf=mO?P;Qif=;;t~$>L8xX-ZiZ$eMbyM`*UV|87}*2CPZQ=b{CkLEA|vB0}MAtJznY z16S;m*uSFK2b|$kKw;8vzqQG(s!H&ExTv10wk1u}b-`EH_IZW`a(;8W&yRX`(C7B| z=iJfn3S(#wu6y4@2nawv0*d0K4d+~GR>6JW-W)XDhNvVY+HbF`v!oGi)GuD6Pd0A~ zx@C07MG3!q4e{z1Uw^ZVB+@&2Aww&95z<^lVP6aViUa~2B^^)9sR(EPHlK>cc^f$LD|Q3QZmK2o{-;o z_u3K5P8RWJY^ccVp2&3m+M5(hJ2nddEIsCuAV0uHLwrv^GSBS;{HbbJLg}ws(_JgT zAtg|#w&%Pvm~2fKf6E;1UvU5QI497uL^ssrdbsMGtv$He)vU9^FR)-U>OzC93_l>L zH?I;f)m~xM)zx1CNN!-IQ%>WzDd2|u7C1Q)L-23S4PQK-o8@p|{;M+0&bhFk9?Ax> zjfA#3LEB@zCmoWSRt;qP^|83eS>#3foV621@H2R{i2&gROyb#~UUg0xUf+=3zc+ zyc<;YR^W&Cw9Lp~0(iOUcfi3qyN1L4(-~KnXp$k%`Y59s6pFIYW(CiB`wFXzrd&c{-UuWxmyb7N&y_oK7y5wvMji1PEgY@<4{KNb5$1&aL!5JeB z^}fA4?l6jq8#HE7!0zWeukog%qRP)#QBx~7T$lAZoS1zibbZ0Y8y|Cs1}@2bGqaiv zZqL_W#h+cc0QTJ0;Vw8clO?w*_~HT`R6%<4#=$TfS;DZKD0vG9QrKU=Nmddrn=>QI zY>ZdC)L94To}eKH0C&?B_XYfyh>v*TIfMc@gw} z+HF_q|6fFf@E(H9NGApJEF5Q`%906atp(X?d5KdnPY@C^P{B zJ>;u6#RpR#9MO?=XVx@ng>Qt2fbO$pzS)U~HITthqy7>6_waBtR*uPJKjk+O-=98k zR7WU=KOkI(hK1r2$SP{9pshLCs+v$Cz1(~BC&#gdeh2 zM`)uD93&oz9$*p6gVN>$(5H)J1#> zD16l=Q2i|&=h7$4^xIQ^`u#5l49_Fzr*FT*YqS~-i?w0!;3Lge&}OR3uA2NtScT)O zavFuna=ZL7g+(Zoct1R|YW7thY8Qm76Q`3fF)>*_RuM&RwPTZUb-k>~LlCBdpJG7+ z^orWEihe+bT3YZt$o|ntJZWEZ6)UF!W?z)x2{JIw;WG)!hqiH`2@J2A0ZZQ@V*c6< zox_~m#t18F!1k$4rJ}OeL7J^P7Pz#lh;&L3d0jm$BP+v$1BaX_(+8NhldG%IW1^bCz36oc4RS5<0Y!#Y9^leo+ed3dhEVQ?!_R`AjqeUW zbT|c%xkp%gdh5D zZV-^K_EKUl@h}Axnnw-0OseK1;u1voaSL{qMTGSWl~6wEjd=X&p+olZAcj2Go4 zmhM&nc8mRP+@?FQi7_mPFfuYuU7VrgpJ69m%k%<-u<*CMmwTQvzzCe2ofRzITMUwE z^S8FQ@h`7cAprI2AE+J%sm1LgPc-*R67;4a%Q4^Bji=CY@NVU{hx(SiuJ0jNvY|P* zF+kD!b27cq8sU$0%Ydo^(JSkBFMxI^Z7OC}*$Hb<@&F4uKpQEr;Yiifp#nBm0jDJz>T9v{3Qq<=@T}b3r=?_Ja zVTjPStbE6Hfbnk_K6|m?){q=fM$)EG3yPn^A82P+S_-(nZ9vQS=25p7X4_k>tr6N* zD?@Hn)^mKh3+s?H=mIU+ppcd3!@AUNQcq~})oyb^arxKm*0-?LkBck2i+9w38R&Gg zrrsa_$UgwQ(?Rqx$)pP#F9EhhAV*Udomf&3MFUVQzED=qGc$49Rwn19{g`vB3L4L0TTdMX&lac7Cs!TQw_fOA}?**%U%oC%UZF?TM{rw^G zo!i##V#IHYc{z*k)r}O=G9c zd-pu=Isg4(K1`Cy%su$*C-x(p&kFHmg^RscCH$T_mPRISv1&lfeC$w!Em zVbCQFS`$NGp5)4c)Md_ADQ06v587YqM=%T*Jq#Hq2UugpPxDFga9|%Rsi02z=Vq8=&+qP)`MW6ai1PQ6WgZ zjE?J*&s7BNhbSwz3n<3L04xA=z#GHo-$<%(`YKf^M?R9IMU)bv( z?h~TdEZvrqoT6<(z1m6`q8uzyzv2 zn}IvsUh3*vTGOM#k1pIjuk>e{=gk=CtF1!k9f3`(w|qczKCZ{q6FG_Z6mS_Es}kjX z`BhT>onQGpQN=N1Hz_=bR9Ow;f0p+22&Lt5A%>dt9L~#bRRd@N$9GkD3EY(BBX&`g z78+OR4M-&j@gOqS5yxdMFZL&cPYw0`=WAfvoKU)|PHe?P7Sfy4*yz6;J-uCYBCbSo z3%ZSub3D~TLGsB^Twt~Oc!iAAr^HF_C~`g#l+#4W*aj!6=%r6|V?_L%@ zB$@p_a|B#Ikds^})k7@B`zFGv6-sBO*ZaXCBby>sWXBnlX&~2G$4VCeA?LL-%BbKH zu#dej)E^E08+g9lBkI-Gu3rvaYMTGkJ;Kbc-kU3nlQ-kr3{Var8hLTjFBo;-7gzX$ z(o-=SuU2^c3{fK3;Wt9?T#`H4Lyv%#_@~&UgRU$0{HK~#TUX>y(0rkgu&lvwoa)|mlT`YGe~N~cV2&(5$;qp%{asrT@T0RT;NdNjI9bS^iTf+ASI+5S6?bZKf0dN==55lC<&6;9%N+{fuzcBCqERea%kooZHD;@u|qVp_fxj|KJ zx8P;cH0@vr=J``#^NBC}?+@+k10IDg*CN>+=fv?f)RiZ>4D?Aj~+REq8d?)(vK27n#Sq_AAlM8(^QNtU~%-eb`fUM2Am(C zbGiBazWLOh$IHD<|Ik%<_WO_66RdO)ctn{5G>xB?JiL-LxnXp3YB?>j?kt z&Xm!gCMP49@q(Mu*2#lTzd2qGX1QFO+omqcCI|(yPrP~u!_^{L5sMzQm&@dWt2c3U z@T$U7YeZyS6=j33o`K9gNM<=S>~CoLn9)R$KEJwgy-GLA2H;6upg`rg(GqhN@I*Ip z#6Hc$M9AlwL6i5qkF__+_W#Bt;Gqr$midbLklT{u+Ymp8GLLvkB%xEK?66c}z}i85 z(nyd0=aYeoR_S7_18+J|I? z>9$WWY*%3x*n(@sr}YLfjs~es*2dSEoqso)Dr6&GPdsiTuHDlit~7Nf-|fo;=Liof zs>02?&CIyL%Y_3TGbi{|;gV~kx6ZpTO-@a05<}lb*a6`B(rBDwF=?;Xcw26?Ra>(_ z*X@cvVu$$XC&I~?QI@zJdSTQlc#Mw|MSQIQaGxVaz_0)@Rsd?f`6%+a-Notouc4=I zeU3tY>i>>oi7;|IH_aWbNS#|cD~9k-J7VduV0k1R6b>cIZjsx0Q*g^U2-gu+z3b4UH@ zsz}iD**x31y^`Yv+soJ^q^F}|5nQmXBn}uSUX+8Ocn>Cf$^%~BBVO(;UM4(lE;4Se zD_$mj@^rZ35ySv1sjWPsA?e|Jc+eAsn(ed;d0!IT%LRI$7%Gs-@jlM9Wi8JGJ|%J729q8<`>~HLjJC zW25(I(u%qdqo(}j5E}t|`RDpvVN=zp#S?$>8_$~Tu5QIpwD z$7~mni>1jQY8ft7HnPRq4ELkU5`~k(LUpq|y2!{x2=z8c>HIxsUHhDW5(o8ustTro z^Rtw)g-*_2uUMF>V%CPUg!5cINl|=3+q%AIv4|N@h$V3$Bkz`P^}@NjZs_JtN*Srs zfY1%@{nKnd$wf5ZkHfu%qNGkjUa+qdW*am-jW>xci21lDL-EQQQ)!8vho_z@3AS?c z9{K`A8ylxR#W4z_(x)PF>a9Zp4`iZyy-ry-}jB-=LkELTZN#*}l%O?#Z zjioofz^6nM8%UyL6kCV|Wrmq(w2cc9pub;?;%Cq;b8ug}*)ANMIGvp3ry(Ls4|jyf ze|s5lKJ(n%u0kBhX)>z##VA?IWL$8)l>o}4{_1ZE!)4e)__}S&{HqtTaDbId94f0O zIh*n3F@Dx{Zx?kdn_$h$iHCs4iDGA`e{~$q6++__{CpvaLxR|v8Cy3R*?1)#qT2Sb ztkLh@ymvTpFcMJH8zl+MQGiJI+8?KRne2T=Sjbeg@Ejfb-(VDh3=@-8(umOX4{1m~ zfQ*ZGv~^yy?ZH&DJf7XkJpAo}ZS337=gu}XS0uQZ6;?t`CPxG5L5HMJ$KN}WgYm@% zTtt&0$UZe>@$Vf9~*NE!DQ3yO%XDQnnkJJu9`e}zJ)Ea^YVz*G7 z<5{9?n%Z-d+|pXyIwf5xELMt;)I1!X4wx7&++3(N(cisN!4|sjL_*Y0Uz$Jf*fx4t zV$dOs2(Jvz?$F=eB}*xzU}oeXWzaz;2IvPprb1XW3M6TTRkHd<>4<$8djd{Mb2AgQx-rQdaS z%7n01xQ|c436F;XOI=n=w7+_V=TY@CIy_D9M%BRK1lpDXx^u;*c^R+T$Z!zkfMhYC zRxO@*JWCMP%*sZ zIHJFqT+TDW1X#e{%gM_xpl$VZm39_jJ-=+WN8S1R+jJ5~ivGU|$Y>1-X(T#0!9O6H zNHAV}#!esWfhSh}ye}lD4%*eP!(yde1!cI|4<9j5ySgI8ksE7cS@ZPGLE`Bfs4YE9 zdvHWo%g?i3!D(waGq>k^2h2_$rh7F?9Wp$ZkciOJND=quI@nLj#3wo!mGI>8*nwtrB_=tGMN_IvM>}ojXVYU z1rnK*VrGHZ*o0?L0TU%`2DTLR3V{tbsVCx9G40&+Lo)*+<=Rj*m)8tdkXz`gH^Bzr4&D% z;H*B6={4e5o%%nJTMe`?7*x~l4A)+3E$hD&7nyBSDv24Qk06BX(dEZr2uIGOy&TWF zWpt97!ZS*koX`Jchis**M+&xtM~~=hEyC7lC6islkGtUD+>?-@7-up>d`#zi*oU%` zK?u{7bQ3MjUGgI_*TB&^L@}PHRyYmoQP~qu(XPMS?;dA#43&EDRN{H3VSq@Mwc`|szE`W&ITu~I^yzKWN9rqHB=r3art8(?i5 zQ6b(ZLQHFRew$^e@6qAwjZmzaf+FbbmC-qa&+TV{v=*}%`(%~a87n?(IZTwgM7U{_ z2UW|IS|Q4aKBJ}6?PEH?37MuwM~N-ous@@nua8x%6K0~5V~Q1`|0uh>_9egqsjAh( z_3tz6iShX;6O2u#D1^HNDI&Imj8;l-b`T7ddO;??sqDRQI_=_PC!{lO1SL8gAoxFU zwpd30JyilQo|o0p@g)39_`j{HDF+SJi>v}f`L#xA=Py{80pZ_jxeO*c;x$v3E9jVy ztF_P4WfBWfw42<=BYbaWB$>;c0{N@brj6gKIaMBAk%vD=y3X`xdeBa%(H$9Vb`eBP zMElncoF*^0

!Vt0Vt$c|Ir#Dlb%>amn+MdgFxv4Tm+!$jLS`G0>YJNb>Z+QMBH z;2Hr$EofOotcN+&U1zLcvAnm$nGk6@S;cJ)&gHH+s}!?CuYfB_Nfcl#5Ij)LqB}SwY{UU&cCkvhREFt0&nps5 zC#+x^2Xrtngozf?;t0BHP;4a?Ihb=WczT+aDp^HXdEXJpjDN?PQ)JQ`WoMPF6M*E; zpS|0aKxwd^%}tS)5(NNbUe{;T2$xje2Lj27)AL~U)UaRj<5%aeb=G+Gr+;0aP0ao? zYQxVPoJ^78?1iLzHtq^t->b2yqT=(}*hqXd%Yjo?#BU#Y4yp-0npYv+>xU=5;e#I- zUyMz!d*8P>Ep-xgRiEjPM^x!6;d2QX*o9K^J)KeqWbrfLAez{@lAyIGB-=cuN`e!A zjyHqO)|Z0yuW>~L`iEX@sb{UnzvGw$$LL|T>NHv75F_}F%5ZS;x3w;`Hbgzv)YyPV zR@dZ4jPophO+n^i6lAX_J<0{tb^IpR1=+gZ3_uyI4BrWsq)aK!&h2;-Pd28Nz@}k_ zm^FMe#l?*XKJiYGr$QX;YjybKw^OD`UwFaa>M(o6fEgp@Zb*|TTaZ$u%UEQK-|Bz? z2?oHQpSAr^9Ne}VVXq>dm^q1wF@(+N%6Af z&n5VgHkc@kGKo4V|H*$2Evs)1dyEs!FvIzl`RFqZdb#NlNR)@GgqiJ2u4FEtmkG_^ z+PctFn!4Z~wU$T-Arp}ZC59UY8Gy<+?qhMOgCv7R@3$f&8=T0h+ffxdv$0-b3y8=g z_V5jiRlvC(Ky4O9no9k?)e?D=QtLMpvuV-g{QR6#j=gLdHF|D0+s#+2|A`=?zi&7u zM>eE(D|9)8jJ&+O(nO?&1kk0N`z2oo?nO^TaA2FOu7BE0_UZzezuxvC0q z9Q(?_$2b0R*S*?0oyBE8eq@D*hX;_QK#x2g0fGMcT|!1i1`4X$O$jQ4!$N&g+M>uM z)qMzUmsW6!sw-yTvkCM@SlKr1K0`Q}+0#>Hm%iPpfGy2%QO-rjqjXlrBwWw_i~Jt8 z&;(y(W+9<}om6+YKx?!5qT1k$$Mkcp%AW>A3YyVz6-?S`URkKg z>`u?Grj(q~YwZ4Y$gKB|w2u8Wh!UKJgNm70p70SnMk~y5VZIt`FgE z8pkK$pUBYEXH>Yx)kjv0J(26x;gweA?}u5CLUut?r6AiwS3bv`j+mXd+w_+=C%;e# zn8!Q&-Wvi$6ePDD2$bt?pb`FhfvV?vDESb)a^Dc$Gi9;YLJ@@<@bHeLD387H9q-uQhX-Nw-oXiIMrIRMPO0SA|@U)@eDz+3@L zj_%Tp{~&bgRPO>#G?W+z7@IwAG&?VkZR5(yz}_E&;reET3&QX({WxKN)mJrgwff1XVY4W~F={PUJK_w2$Nv0YQ zbQ?_i4X}Sp`x@ssz?rBGo+wXNO*tS>>Of#hh58lUG=tP*jx(%Q=B_htj&mr)o0Eu) zQcnqKrb#3MClI>xW(EhjHt_38zwLtn#4V;4z%4^mg7}PZQU4{IS7u zWWXl#{I7GBR3YO98|rRAAH1M;pn~7dLEqm$07ybS$Vrzz@h@}7(*4Yl_xfAo{pw@m zqd%=i|Lp8+ujXL%<=i+RRpcf4Bh&YHN2JrAL7oQ2>>pPhk$%jx`G3v=_bC7uIe&nd z9(v|w>AoiiXyVNuo-ACMh$0_rvvZu6l`n$|9*E(meg&`DpOwNpj{*d6RR8?ej3;Y* z&0r*qXGL^+xd|FJSO2qZuySR;)UUCdX0X28+IWYskwL_~dnkxFDY{t1x)MfoJM61Y zyIa9CQG)KHDW&Vnvc2{l?TDf3<={I=ig{nXy3HrXEQunR0Gf8EF18pF4=F;{ij`ld zaEYtPkK~Atb{1@f%Dve8h@W

^|o-pB5{Zp9yoJTpo;p>nUeqYM*saYHvVol9Uc| z0JWV%3i@$&7v)FFAo|%OXYPjH=Z`*l`L-5dZAwC-3!`rLa?hC8z`z6%f|nMra3z0f zqL`1)G44${bub)Tw~}O!Qd}Zk;2P6Jl8u_957Ayk1E$)&?A*EZz1Q%bRP=-hne zk{=(xhLExf_-h7Me24_;i@8LuYv(jZc3kEscE)@9RBdGpq&R1SSFjQ;f>% zN}4#5uaF_6h+`J)pKLl?t9vjMq?T%`YHENEH^R}5qaGZDQ6Mb%mDK&^5sD15kRWLm zOGGt48)OS)ig!Dl8XexaLr6(8<@eu+!^IOLBBZ6IWo#U5hv$$0hKwr{KRTQoX!H3d zoDBGhM?^x~+B#4P@X7I~9{V}50PdL*D_p<_tpAd&0<^>Z@%Qh#-;be8k{e*n5jghv z`juG@3H2K@F-X0lLyx%aaCi_()2*2RAHT?r%J;?3bDN>E!daei@OWYN$tJPx!}C8;0tG*zyX@(efMogMm`E4DFBn94bR^>Rhxd@K;z@2CGpFB z@uvS(cglnm;6Tap=g*%**Cs_}ML=q(6NDB)7I$Hi0iAauO+ziO#)w_d6l2yP3mjuK z=pL^ELy6k9l-=Ag)}ma1|MPGA^1h!RGwKA?|ckrB5x|u^XHzndo~Z^9?D`NoY;n2=zAb3w_XEfA%dd8)wv6Sg4~oHh6J^Q&cDtNHe? zA_@d)VWwlNtE)^x#H_vXw51t2IVGyJ28&nrhc+QPkFraps}CKQ)g5ig!4!iz?>Bro zAu&Hsszf5?Zw^ngdAVp``v1~JD}SR-pGwikr^mYdLn;au?w6?EfeE)OPki3xrGJ{k zjB0>zrL!!DT2#t`Ygm=|y~alA@>I#eM#%wqJau4D?G$4bmdqfcI`z*saZPciCxAPa z+MA|#mGFQt&4As4EZ&bjhZg5SjbQs|2T@FMmf+LRa-c;7&(+Xsu5uPOA}+>3mR_K~ z5>I&cyWRMs2&R)F!fvup+!gs#>rnIX5k#HO$!LY!>=qiJblDJ1;qSt*k(NzQ>53zdK?8&#Ql~&cIF&1Xe3X{^2zF@8p0Nl=;@HM{{dyp=!)+U^>8% zEH-zb)}Ft^5RuXU8FIWPA0gLt4?YZZ4sqR{~&Z zTy()d8b@@XVEk%{PY^ir8*l}4F3k56o~^3u84&rn+&PBX`M45%o)bR5xFFMX6F1+x zT2qe@MIgf_l5rZ2iA1O{%1L(>Zn5de(eJ{UD*a-+Ui4zgL5vWf@)P}N+Q@pW24+w{$}|}#m)4?L=$RtL9I}H1&Tvz80Si^zGCt4!_Oz;ETr=A zZ;x+o*%Kn!@tqR`xk5M?IU-tKjPs=Ij#vcQ!W`OysJ+d=>8aTAPYNXj(=`)4*hOhQ z?tx~Cr=QBwE?ZiKgb}cVl9NBl7zq*;F^;XSVP+3}(-W~a&RBRg zi&Ig_|D!PaICSEI8KYs}PWyz>lbDsBN6Cc2Rn6BMAfVaO-*>@$+Br%jhuS-|yF~`CJcKd!LS2 zIN13oo_e2z!cf1gb*9GPFtf8dM>Sc`HrlT+Zwr2nRj-h2LZT28asI0<5@+Nf z=y=5TIZmp+lIAKHXE(K&RfWMp8ilzZWIXFBEeo`7oOX^Z%W&5paOjVzV0#_aA+rgw zhFaOUF}aikB@ryFmESNt(eV8|xxLdsw$(@Nx&pm{kFwe+D&u-#$t;Ajic&8E5t6hy zLs2opk6O5<3Wz3OB%dd}Fw0_js1*I$Zz5V~AqbK?IkcLpaBSLnF|L!mqdR>h9bqEw z!K@+#rQk3Kgh!RUT#6-@9|NOtRY9bT z*vCZc=Wd)3&30le4Z2M%J}Bq(O$h0R$9XJJZdaaXsD=#MbXbzcL*Y6J3l*5P7ecI(P-7S1$n`YrLg+UuAsc1+*(Qq{ zhnNzF@y?%Q=c&O#0Djm#ry~UJ3Tg=)>+9p96!ib$&O}dP!wLYGB^`ZjW|^zeM#hPw&`6)i|FQtLSfgv_HynwwlA{JYGAj?+ovZ## znr_33XPm$xFIu92VXp>wUjSVTB-xmRsF!YR92}?+B}PU^MG_9!PHwelVjx%~Y0GV+ z^rk-Lym*ro%pqQaHh0f6{@IuUo}!-b{#fFYY6;4sN(xAAfmR4rkM3@JeN@UqzQ{CJ zg~r+l((tsM(K{cy(K|k@gF1iNCp+dpX#~cL=i@}POHnE_6v=B5PTJL0a3bEz^ENJT z#w~9KZ?A?duSP5{1eZHS4gDTs?H_{uJ47q?HI`IBJ|nvp^YbnF%`e}u04YTn;t7x9 zj}}o;c(UVY130A^cpgKewU=mdEgm7#72I3j78<5)fC&Ds^j*=i3_H-+aU|DYr2I6x z;s$R7n#PeCQcs%gZRO~b4wT(4jNfrnzB!;{2}_odnYl+0_~y~L_Dd;ujuCSfglC9_ zFo?R*N?}jLrQ<%>$oNIzrgpM&=^BVGvyG8;;LZOpsRuIcqWuL@jl+eF%#oM z4rA@>z&Z;KXKCje_wYZA^0OeZCQqw53s`bBGHE@hGjT~7hjxm zJ)9C%vGQvdde+5B*4k3kK(4}tnWD!cqa!|O@c)G#AK)BrFwM!drA>#Ogi5)RG40HP zc5^&28`Z>Blf5Sw(?7b++3PqupEzNLJIB}YMN+BkY`O|p_%YSa48~Y$VPzFDDsqP> zKTQ5{Bl7MWmHYcs8 z3aYRmJN?#DN7op(LX7Xdrfy5@EZU_|Dik;=&I#yCz71c0Coh^)3m;(cE=N^=fV11M zKiys)j~3vUU8bQ`P9OVz=>3*IlTfWv|8j6qVJm-kzJkKW!5wN!ppb^sWph_TdJWF0 zWPVY`-S}sI`cT@f>#t5^W+!i`qm=7zG`2Oo_-=k6rBor|?R_wMD!$KEb$*z^uQ@ub zaP%rxS7!#Yk#c93ybAb4(D&-*d9*Y9`(y6d8RGkyCEP80v@^s?UD4E-a@VN(gux2| zC0187C$AurFR({^uzhZZ)gwM3W|@5tiJcL$CC7S{-NhGp2t)cI1eJPZsJ}$&^ZWbB z+zn3vV)8hOrdPH?bD+_o`3QX(`~T1GyTt%N-xK(`p1p2EQ}Vk-aAyB!P^23#6;6d* z33kUI!r}rc_K1!uR(ByeqByw75GT_5O54A@W-HQkbG7}DCwPQP-29um2!_AL;mED5 zl0-NeaK;f3Zp!pG?uZ16^rYt92dAw)7Sd*1hieSpKRtiCjGlPT0d@30Z^a2Gt}{{a zhK5H?OOS^|!&?_hj}CIzD=dc{ym+d~UXh!|WX1FY%6t=_Yre^vW{gRqo0cX5d6>0% z&P2&oxiBlqjvp;Zr-ni{NKB~-$H>=)@AG0I<7o0S#?%?O6gl`bQ*mgD-!TrYiu^Sf z8BE8QTE)>#;aMHR>QKoN$9lpU!bcg8J0bs*z01tW*_|~0n@-5@j#?`I_^TsXAsza; z#rVS`gmjli$eN5)HK7^u<5wYP=;wSAV!A}mI`RO9y}?RNNh>?q zt2e@tiIxOeDdpN)9eM(tTdR}fhZokvKA6e4N_u0#NZ(goO$$3}1zkqeG+_FH9eKkV z#)}tm5oOj!wHERNL98AN!c#Zbvg1{P7^b zaY!V!fXV0+l|s8C0SWd9+87sn@jb2CdX~VU53dx|WQ#N&E1vMyBva9yk}+%5B&R(p zcD5>$)MX4hkshu+7B9}Ms${IDBqzg;q#2EKoI-L1qR1ua2N`6d{r$dA>abba*=L`- zv&!7XX6YIKS7tUK5r^$+obBwc`uwDdQWm&AIksVL6Cu5Gk$&p`Lax1D01}c= z(vMi5LCoD)U%j7y;Pw8O73Yv>Qbe>Ee^lbw<_Wg4ZBq_>gW8)p>XNvdkZ$!iqX0JF z6)AD+at^A3Qa6O#M4oCsv5CTm7Dee zmzvkAwqJRviB#`$S>Ai#KyGaieUC0ra@3_SSm8;0c}|xVzQ_H0EoIg77<8)Bw$JF(rkl${bJda84gO(5=kw4aedu-Eza>EzCvc>M{@=`~ z9r4GMwfDo4j-1vy&HrkF@9|Rq(7WbZ2h*qIHa0)AST`2KZ}22=teix@WCbnX$})M= zr8u~%o1=0Y$}cJly$vyd5?pAU|a z>hQC74d~PYB-zXC&QJ8Y`2KYjRzq!;`fk81a%!wo{ zMRs#k@?Zh5R4nHV_5=WaxBD9j`>mBycLS8%3KjSvkIWJlhP~7%Zb!>wSD3c` z2lZ_Bo!ZBoVE7~-biIHA=eDQ@WjQFB_4UD!*=sO;s}6a9rIQ4sX~@76w%Io8{xR?1 zal^^l-93yHw*6$=7idnBke5RpO1J*M0;HK)ZA|4^OW4WZrSsmi2u{yG=gK)DgHa^8 z5vpYOW8GRy+zjCE8ZZp#xyswcf)}hp83zM@$x6s7E~o14RB$$kw`AW&tdNC0VjH&M zO3?W76v(w19ILLN!U9l_2BR-fpp4)vaZrHGb?MQu)!&EWf#Dq2$2X;E_K96UPxKso z@aM_U>yOSBw?KMLdM5?Ej$-S)t71+=tG9z}uT$gg$tq(3*{{x(>eY5Ptx!fk?J8=! z)0RAIEVdwoY&u6yD_mxJR%J}PV9{eGF*E7t8E8AA8_UX7_ak8r~wizl$11A`EKzf@1SAUFJjjJ@0}X2Y{!siz7K%YxhyRs{BLR0QD6F6h*R|K zR;tRhiKlIw)qd}h{xVlnIZdmTw(0+XrkNa{shXKL!OIu7jpi)A7C2G6kO2WD{@ zHE@@63EtL`><)2Qdta<2w^N}U5Urney^YK`Ih{%kx8dt7Yi#71UQeSLtPHZYRCN|8 zC|l}_iwrTman{(FOpbB`-w6-LU=!i{F`Kk7E(s>NfDC8RRiPvbkvb}@iu2(BJxl*W)c1V)T?~fVhuY<6{h9lYfgmPgxLO@ z@Sj}$$&F{=TN|LT1K8#j7Z=-PMSyIthPg-ne+A!e2qbL4l>(o-mgv=EAhWKXZ8}Q|tMZUgszLTLFwzSqhljj%fd`~7b-)l$d)FM+$3eWQB7k(i#sbpwkmmn_> z0J{L$nBEAMP7QHO6;49`F1q&|O`39_pJC>6i~1vfb>h#*K66~ag_3fwUmU6wh*@d2 zyGqy17)AU0^l1lUo!ajM)`q zdgqc^b3q(d4MaB1UE48y95m(rd7%}#=+pQfO|=cniNS5(mVdUUlo zN5v`jy@=OO3UqR%>OdYsDz%Ua;hXQ!e^*)q1o3~_#*#1G0qz*tA`M{EbG+}DJbVvL zZjfTmgCijc;Jv?^%=Bs6{J(Pid4mCXO5CWOY_$Y>S2`s9GC{-RQ_mf%E}qGe$d&8s z-p+WQ-oTJWmWO?8!+TGg29CDxsl+WuU7?ox&5kyfms?f39y&aE7aT5Mlg|1-YhM|T zQ+%9*8NIB7jm#<)ZQKgk+G^C~Rm;h(EPwqP@KxYpiz_yV@Z@LH2@p5TAtG_08H%E& z*590VOO9BtY492yUN2Ka5k25f<9$zM!q@LaZ2d}5ZO`K@D1|Crr|>yx+@SA;Ek9T; zKX^PSI4bB1n}yxT(NB$0-3BM33z6Q?0%i$Y(9o;DeMJQaUA(f}-b>TrD~)Xu*%rf> z=HHN9AQk&=WrgpO$@LA3CK8e3*D13m+=nl1(Os_Vre4%J7&j5j(X9-?dBPKottN|_X9ncp#CoElC#F~aJMZ^ErZTvb;JX%$8OusP@l>3 z{nlYB4x9JSRL6@}*CgGB!6|bpxhBG(r`Cf|&`-`VRRf?o>b{Th;_1P86a&yu*zQj( z+cA5-XYH(@z&?a#sn-70LELFu|5&?Jy}hM$68BYbw(RP=LTnDMd1GK!9}1pGzU8Pj z_gzEa-6xqmpBN36JSnJDY%!jSAC=?=P|2aj8*b#l}9xh6nz+`=)E3F0%#a1>0xb{uD5hHviizu` zHh%3Q2;9 zb!8YEP3tepKc{u(j_Y#3H-wr02l?+9>437f_z&|vf+oQZGB5)w?4JLw{8w?P^CUZP z0JC#RvekDSUo(gTj{IlPB_b}q2wcjMayCt+wygC)*y!y7BKAQkNSK8<@J1w<6_)Kw zsG;#{TqZl;{WOmYJaK2mbbg!GGF9cBR=)yf=LO6z44Elvk%E+X-DWJU&Jsc&0;JKW z1ysfODPXbgmfLAi;|??v^8AST!S`ptd#1}d&b2U13+={nO$H2zH{3?$mw#UtwhD!w zcoDwz%K>jleF{5K-js?{5rX=a+Ivr=xBpnh(d6Q!E%A5d!FMO(5l=CW2ig^_MOdxj zY0dNKyT$XlsE#-AFmh)A%$9%bY7y(Os4{1#C#d+=K1T+(_JL+gxWuRKNKk z(8G|v?}cU?WChDi*e>bV)fYGGxJU7E47UCJgCfVbG@Vs5#usGtKq3|H=_%NnI#?)%r9JI@k?UtE4v)ALC)9R~zJx0U7+PQo?dgJm%r2$lh?4X1YVQI3#4ttXrC>Q;r99N83A3>~m zMYN(0mA8=nxs+rpyZAGOTqAp65krwK8_e=9Nq)E3wD!IsJf2Qa(Qjd(OzinvUs(!^ zMh25cqenFiQtQ=?EFr!mnQlWx>iRP0M5r83AH%av4Ya*TmM8~OiERkzpzy$Km*Q+- z7%|#w82#l2FT(QM16mAL8%{lPZn3o$p%%S*R+c5VfP^=-jA;Q%ou)XMERDOJIy9U@ zLMxS=hN`~(LQ7?uR(6)nFZtwr{1PuVnNSghEr^g2{uqmOYOQSXrsP;YY>e7zcN2~R zbj)r_s6>->dDa?zL>wfmlJM;>=39BBna$&?a3xq+-;%z9|DqgdE^@xE)E6RgLjKl4 z3kIfkQ&Hf02ce0&`$ilV{aRHbrc)PHQeNF;Hct;F8zs9Ug(K8x9;R^+wusRV(H*v! z9QN{gU=!2SF`_IZO%%De@YFf~wO0ReTO&=yRoE#Ts-eVZqfaes20t zHr6s>K1l?@gWtPD7s?&~JZ|YyrgI-8U7P!srcCh*GYaI>`P)F|ppzLy*neLh`7o~1 z^s;h}_BH9#cTq{GZ=6TP7^v9rhcSE+_LM}ua4jfG&S$%P>KGWbW#2>uBy?%jvnGnx zsI?s+#+RX&0o8T5OwyfIjQsw~+>!iTF-&Gs2plH*@0u?JTjG=xtbO%!myp`*!I@tq+o|bU(ITYoXubmI z+JbxBf+idu@3%e)NK|Zv=RpmxBX&Ti&WCKOhA0I+1Ekz2{9Q{2MLI&APD(c)bT_^Q zJ4;2|L0Yu0ag-FF%z_FHCN7VtA)&p_YpZUO6|x>CuF(HY-)xGG4By}vl5hb3FdaLl zBs>jjioY|11`VoLFsW)%(3tJA2a;hru2Eu|dR#$5Jmldut}@buQ=^?{iP!+IA(Z6$sY4Of4_e zl_=$8Mkzd{Kuf%+G@@Q3^~#$|k5t$??#1~g7`;y%w$*V<-d>-eiG!od`<=Jvuo#cm&9Z7h6zvdm@Ti_Z7~;mt`+?#WCJ zYXGe*c9_PW$rS)5SOe^TcA^(mHQ;#d`$P=b)QI6a-;bdI>}&Z*QAhEcO23Z$6#Qn_X95P*3?8l=vLT?_l1fhaX=n4$94S*)oSY8Oc9+aZ|<%5 zWr=zLZ?sMl7N&qaVJ-Yhj`yC*0KGM0^nhZmYG3XgGmfbL)Qx3WmRFbh6e?J;yiR7c zg=k6^d<#NulyV(ww7XBzjVaon8;dME!3tx695P(2l119dGt8Yb8t;3;ukYwradS&( zWJFP<3*+X<3EAUZmJr7<<24^A9Xn{`P2+TsROT_V_n>c{)T*h;0)aG5V~vx@)Cpro znvktWsLV(j&hxVJqp73@xjn_uwkt7Z?YJ8R64H%HESV}a$X(A@MgrP-hf~&Jh(g(Bdw*zpB1vpJ+-{sEybju#Z z<)lW7kcxjS99cb6wx~DVnGGUEZom#3JtkFhh=CgWkHx>DAD6^2=wp|v3X~El93x;~ z2_pgbv*#!bIRfZp>U;$`Te^$o+M_%wg;jtWUYX91hu|a;o?>d-2IO%$7C|Z!PhJ&U z9iban?V-!d+|6~VROu(0m6 zHqJk@5HH`{Tp1lWdY9{s-%Snyt*7Wd22g< z^Ke!bwwieO+36!lb3-jo@TP1E zd9!baAVHJP4f;-3+9bG-oe7@)9Wh@$WR$^X!!3+rBb&{qDCLEbUV=JRECx+1`8G(& zwoqKEFq*tT$ZQd>;Y`_%bJSfA1Ap2dK5@Xc(mJ-dfL-#f#|U}pYq5`ic>qbAE!$&~ zd>0gc!{Xy2l=qPghc@DnWT3(Xz0K$)U9utF8Wn#(yS7+;;r)PdIQ4#+GXU zlegrCgAp`XNUuYcz@n1o6hJzIpady26b7Zy>B{JMsyjlHe)w1^n;p5qvaoWL9p2jG z)TFil;^Fwdy!N8Z|F^p#&%yH=cJ!>u^iBYZ5N6TKiU^gOam=+>UjxSpj|0xAOT!l) zjc^$)XDujAD5YuM!6l7}OkRJ)@H6C`|Bt7)45*@ezlISh0Rg2urMtVkK~h>;x*McL zx}`frx*KU}0qN$@-5u}d{yqQqi-^vdIkRWamFrronOyDq!ePIO7EmPKh=`!d=o3#>KwyvU(Ws0`Ia z#FP97Y|nmX)`_7LnRi{VW~?D3|Az(OeU%krM0a3O?e6ibcTF*Q*(AndLIiVl3g#&{ z9VY6~FcVJOi+^s6D1GcB25fXcSlbJNgMv;AlP->J=6kuC*~dXa-m7coqmnNrg>Sp} z&;K@cq&-#;kdCMHIH=Vco>onT8^48@keX^!CV}((-v4ysP|f3%P78QzpcT}|Io{4V zgg7Vkg~pCpf^pjBa>(qcwpBO^#eSamd#AnZiDONR`PHT}^N82-&nWq(+7Y+{Av~I6 zn@!tEY^H-4pLC*CS*&q@>cx0oain8D3DsAwGssWUG zt=Z%JTR(aPp1^nV5eE8Ctv>`@ytF!e31fDViK0V|xuCjT411j^gv$F@suRgqc zzggc9zm`xzzzkxZ=+owV(^4Kjw%>hxZTxA=0S+=3AZ*CT2@4BmMsFb($YgVDN#K{_ zlJ7HaSLu1ftjciB<#}Egt-DB~2*U}%_+eS&V}S1be0 zo7?uaGp5c>qRNK_y{x;O4+q>(mt}wd!6xOc<>m@+`HDUI^=?KutP0ZB$Md8eH7c6r zaekO>sE#$tqA?w8#lukO{DybI?mAh+74VKlC747P1Mdg%-s1pQ(?xOzVn_h>_ka|l z`F;#RjMx;ZfE<#NLNiX|Q8SMV5GMN@LM>FE4-&cPVXJ?X5mybwQg`U%{yk@+`8~J# zW?E;i-3K3^)7o%7UuLISksfR4Af20&hCH}`zmr)tr8J>jjvnj7DlzzTKMO|Gdj5hD zNZ_dCNlq+-Tm@sJN{V>1e!U`FTMY(@z| zL_`JQ#cEL#hdw3Z^}AZw*$7!_E)n#TuA-tL!#epe5Z`qf<|4pOvdZA6EY)OYX4bgE z&sM*XcoT)kOH-Wn-Qv?%vjRmp6S5SU#snr-Fvy>N8}3oQzkcdDr(-a7Ri*=^QU2p2 z-zwDx5YkqCCZBPH)WRHU#Gh)23-#cCZ+v_K|0J-cSu09TlSa!+#cZAz7W}D-#g$*D zerd9`7l!O^XY}u>|Dr|?b!xs}?FmX-_yas1!Ei!korqUu!;2T5;NQvar` zg2SIu7M{Bjh_#^Ihzd`dW0w55+>xu&6HG4E=>Yq)RzgPRU9c}Us?1>bC$l1V z&o@)*ymf?@oYs2~OEQOrlF??E&J%| zYDYj-5GF~c>#>=r>%JB&pUxdtSlE5zEm+>VM>}1i(?5cQ&0;xWQX-|{w7M!1qzv*7 z)8QZE1OlE3&XYz!g^-#W!Rf{x;xJ~*>_TmlD_-qnb~XEZH;qqESuI^W;4IvXc}?dz za5jX4;8SzL*f36~pSG5G%8W=cV%NyO(_Z;_`FR2!!^ywqlSB@d;myu{y6;26jhgez zhOgD;N4T3ygg=t(pf^=Z^nbeW<(-fqcM=~iq6_j4P@B(T`0Ug;*;`AnY`bUlual(~dJsZj+1udid-*#PTbm0p!+xsH|aPVxlXx&T4o6aPxA;z|-H!XSe1CfJvL=kdU_4W={@ssVEb1 z7mg64lLNM)M%yR;^(ve(MxBrsQd@~2L?n;X+wF0ej5KV<+@hj1y%yJZJ4_ruOqsJb zsd(HFa}0}AgF>QL)1U7IhRa>A9@pF=2+>#h!n=Y>VXPWbYOl4N#R+*KC>birS`z|{ z3Dbu{VzlCYM(Qv-i0-e++*|)ZFsr2c8Mp*{CNVV5ugGiIoP&EQ}FxzFSaX&FJcOrS2Sfn-aK#1`6h^}MmpAg~X zgv38+7EzHYaTD)xp#`A8XaNJ=$O76k*p(ky2mIw9EI_2G6@5^!Td#m)fHuQLr zFCeU&q)$J zFDXp6vCH#WqDU?-*Y#@zda33g;M1%L+%3kEK)@B8CtV$MJY;)lQdu(`P<=u(4|Od4 zw%)XRII%d5p`M89#HjCkj$505Ggky6Q*kxuyHIlfeUOI_4Vo&W&)}2f2vzc9r8tf*QJzT$(?VsxMc(h(W0YS9>`H^o)xNT7=(82O17vrccKc z&$U7N=kd+nQW?zCFM)E`nVf1$zKL)CTkD_XgoZEQQkp`HI7fKia9HnJRq&be>iP6~ zk>!lpO%d4x|AxJ1r(=dU0Bw@Q|*+rGvL2Cg5-g`P3-vb>^h zNAZ$$^@dZ4gme-RKTUIe7FR`3Co#?f$D8bs{r^|PmgrQZd=p(}^dPpfxpTFoJAHv+ z&|LqmA!^6^@@-K~d1QH@KrxZRH0H~)kVgIg7gPxPT}KaHP#}NTLX5_y&NzFU^)RA~ zTEQ`EtxU_DM}uPbMUom?)3f7cRW@S(Z&gTvd}J!+2W*4y>Vm0ZY0vQ$dhgdpc+)iR zyD$>q=bpDjCS;H#&>gQe!D5U;*DBq^{qOlnE5euWXL|||Xg%V6971mQ)DC)kZ)h3V z<-rx5eVRe(87Gcfm26gML^842>m*`9;UR~3BXrQ}g1zZ$DIV*lHE((qa@&-VSm zUx}}kHiIG9%xfhMwa5GUb@hZJ>R%OV-bv|RUd5M)8AnLuN!OGV58$B?4y!vF$n8{<~)y6x!Ld! z7XGuM=YF58j;uiVE4I?=IH190I%PjNG?c<&g=MwuiPX&u^m_V7OkX$^rn8rlO|`X> z5-0QKI!|!?w#uZXO<|_cZ~EVJ>T9rJ39mX4k+I2^I~FRaz~w4Wf%f$r=BQG@*701=X=8mW5*Ix zaOS4E)QHVnIn=uHnbrZQyyg43`$VnHq3gNt)g*Z62n`n;5ZoVAkR+fb-kBG?w&p|8 zUqX82uX(+t;QYssMHx<3xHHe=ObSl)ulZ%%H~d?tv7)cJ)cKQepx9|V*^>6cc(220 zXSUuGhO^5KhPXP}Cv|;$F@a)$jLaJ#(*}5wb1XmE`fEcTcNFt-cY6&$uD6vPe{P*1 z);=gKOq)0w-t>?MwnS>l=TtZlfq=a`vu&j~!@W=%bg z(7-TmiQW(NkLOvp>#tFjwX~%9iKJ#+=!bwHn2+)?&A!$)I_^E$GhFbC(bQiY7PTUh zxJXA#>E%g0I?_i(3b(McE6T6M8DgIU!hcfH1V*D%n(?xQ(%3041i!d!X38KeV@(-W z*9DB~QBhGM9UYz1c^(~KS)@Z&Vo2|~3*@6n@Bf@id$cC=Ck+^CkmI~ZsqBa3R8_@l zvG=V!&DecZ)g1;>TGEv7BXC#(V#%e>Hsbx(a>JRno|t@NE$C%|0i>}bg=sGL8!Q77 zB<}}OWZM}mG7+O7XrRKlU&f`?dV?hjOG>Gb+A~q0v?-d|AAvhm@pPNK=z0&7X1DNi zeS84Rd+Y1>GO~oUz(&PQju`G4)iLW{h&dWj^j9-_@)KDv_^x11vRQrrdl=;V3JMAs z7#NjyD}tP@=P3FW*wOuBL$Tzw>|B$RlYxa+*2}Fa(iD!m2+98i5m_*)g+`yI;rFzl zvc^Io>B}2BB_$I}OA=D%RG_R?s@IYPMC#ZXK0Xz<`=Wy2HR{SWqP7z>s=y63!J7OR zikyX%R_F+PiP)dlc;<-ZtH42olPOwx9-f4!IaD$(&_(dKOa)z2c7>s-SEFX}7x#}q z7?O;rfBQOb|1L&%Jo&%H3EyvaZ_|H2umQddpaKr<)++RvcTulT6{Sra{E!9BeE(eE z5816iO7GgglzpUmS&s*CmESI*k_hZ3Y)5-QG4a)g#t4Uk*d#kazlgdzV)t6Kdg@MN zz9g~zVNX_u)(~@=z1+a$@Nex~vxNuW!{IX#$Dtz6f!Db1&vV;1Dhm`re-dQ}XU>ED zA_){->zNjc0I#)oBl2=u{$|e=$^jD9ubNa;?-&9>3*R7PNA5X}f!$sgy$4?*-;QVQ zTl4PJnv_f=L`1lP+7-inm2tbj8+3uB(K(#9inT1Ofu_*?7rQd|(wRfuE(E%Qnf!29 zvEFrD)#>sxzu|?B(fbuNR=A_$ug|{5H$0j^sE0yF$i+k`iF;xkzOLQCbDhk?KR)F7 zwjYhFU?!^3CbFry=#7(MlukZZJt_3L&0Of`M0zMWY*#rt9pS8?)vinrEI8jN#1%q?gsiDItHXaG-Xzg}nE z<~p#uX_7M?_)`epcU0IXCIi0uk>Zz5jZ{K)0iwEkFCYNw-tn^mrFj+wwi&y#*a*Ef z5p@hOT=m=G>`zo0YP^)$qB|Ni@W}(`4)Jv?aFjad?)+4Bb=C`#Dj4#U!vi!62Cx00 z0KeC920_IbJVM+6%=g@lC|a}rcQIC88bnE!TNi(Xy)a2^QX7kI-FtU8#P8`Au)PHh zMo~uQO|+>rXI$77x?KtACx_=q{e?-Wy1n(N--!~;TtPNo<;M*epFt5UjP6+J^#dON z$>9+;?JNr7!?S2nUu~_d_p%+IQ<)vefO75!G~|it`MycHtj}FRZ*)DT)OVUg`}zXu z{kCvb3xTZUOMGEf(ZX0$a@0ugvq!_rJKtn_$`HY`O+Y5l#@}{e$$M}0K5Lf{C|euh z=<+&IV(WRc_;zH?>(!NddVHuc8#BAR*mdXE2SupgX?Cst5N<%%6CuZBI$b*VU+ohT z{d@~gsgT}t=(Z?AkCg{%)22h2g$)f{Ojxh06^C5Tw;U7|-hy*->HB5a=rp{Ls4MH~ zMM_F*{KhYTO3U+nKT_o3%xT!j3bNTth_ib*+{9uW2?2o#t z0m79RKdCHjZTr|2K#K^dCSy;I2}Ng?+twd1c78X{+MGyXOW|{41T|sXCEn>Wz3>^Y zts#>A;-rKG`|e1t_l&e%>K5_1IsbB{V6BpyWJBr$SZFA0rZgPIjr2bGa63sAB!I8OE)g!1Pa-G*TuJnQ^c6j zvTO3<`-!{&y&{--b~#6YdkwgmDj!nJzg=9$Di+}ieum*%c4mWxC3RNVE=Xjma_UF1 z)o=V3hE7Ei)CHV`Lva-g(^dq%wnxOTpGHTEO)#%ty{H=;j7$A2!5e+p$3{bj+dVqk znll{&au1#C3ba56XRR=e1`!)XR2m&;i1=LX-Kc&S%tU%XY3Lk0%xdHQtJXMpf;2hC z#uG9#T~8Xpicp{XtJ8~#0$o;!s+m6TYd_}S%Xe+9t&Sf+I`B0h06I4o0R(#QzED&W_1#5hzrvtLLl2`S~8GfJ8%|r*lrw@d*~2*=d^ofru$GGLru`nW}L$H~fds zoGB=mpd1PP4rgi?<|pcRS&HD#RVh?R8^tDe|C;rb<(He67a01yEwoWxCv@5asb%Ld zCCKLABiCl^SeGr7=fT#yu5^IF>Fc$n!p<0sft_~>4{hzfO$Z*U99b+dbq=?in)dgE zZhyzeMU<4dw(H1>sGBB`$b`SPe$D#$2lB@0YlIN23Mr(H`Pj-mvt>=bI66u;; zm-!17Grq4Jn@<)hv|etd0sj@D_Fay#Ey3$ZTU1I49&~Q}Jl5ZUj?2n(Cn`^+pHPzN zX_3&Q(0Y~DveaxVu^8E#X{z9o66=NL$n7kh(N>T1O2W`37Z>ZL>fp=$8IV_)TYj-E zEUAtgH}3ljLLZ%aYopzGCD!XyD-cyABqp=mF4A}(`*|mW1DIX**um=@qtnC_KQNtC zLm28}HGMLzW9KloePndJ>={*Cdva%0;m~-B1{(fSOGiucX@VKb)IDorp;|Rt4Enwv zAKaGCd0da@dy83&xo2np2tA)an*!b;4{YdCkYQ-0t21>BKRw&P>=2G?2yDt zS6ZBY4tOG|(ux38cwN`c#D}Amr$lgYUO~ZaBXW&7F9=TN<_6Bk1f*CKH~R2xH$XkM z2)tC_7~_pUu`hkTq%AYE(R3MLG9GU5GClWbh`cUTqKJ3`#)$;4^DJHc&xCvQ;I_Y zJi-_Ut~(TUyP@Ou?O27Dh>{!EW;bmyu281I!-IM0r@FZEa!_qYDjG~lfZ2~HTToP2 znQb;j8Y|NpWoTF$9nX{6J=ie;(4SJR_H?WACafV=0J$G-yif(fWdCy8CB4_3Wk@uR zh{6cZ#l;n?+Y!eDM9DWGmh%V4J*(^CVhYms5WM4=vU-Ea8O}`$&O_HP+scx5q##>kD8RvIQLwvzJB2L=q)cb`h)(5gW?F!!;O?szUPKCrTmXSj@pg| zT@PkO_+wX>+*VAdhM1i02^*XYpM638iYsIUeUFFtqDkC&nI%CTP^MGAiV9O)TDrbQ z{0zjfmMLuV&-+Gkq+Q2(@FJ`CHj{bWPoUfuUAREni#}BCemJE$S8s!Vz2YZS(ecEa ztmn#O*==~h1L1buKzIi5h1{H+SKWX~)$mol>ABz2F4c>HL+lDZD61Hw%ZOi6mhOR+ z`fg&r$cZdvwS?ALEq7JZFS|dwfSZgR1Lw7xDP4<`Gh1C(6>y|z6A*YffnPDVd(;QD z`@%xFN%`|U;-?s!g?dq zT7@HcdO8otJ9PyIUtBmlol0Pz@2@;z`TZW-8cBP(caVH>N6A@P(Fs;0D^n+68~K9l zxS00P#)jg(Lti8y_E2I1-EeSrGvxVqa&lGl;iu@7%AYjA?n{c51sp|0K+LC_6O#DW}nRIUifsH#iil9I46Byxbbl)5?% zLTD4vs=jSG_Du%5A<+I%mXrCvTJXE2Czs}AOieM9Ofya$|1j$Jer5p{3?Z~TAu-8u z=PxO+>&wgfh=@p_^I7<_DNL9|n418Ov=~n?9lqmEz5OcE4v4mi0Ir`e?wAH1CnD`n zm!ZWarNc^*3@=hun}>idgAy-tVrE7TI2+&zElR*@A;AX2Ht`tWe0%rs zfZbsN;sEgC2c)K+67btYsegQN9K8|}9@rmZ5~bWa_(Rk+*10)j+XcyD1wETveiCSe zjY0C7n!=PtzQs*M@Y9CAg%k5?X0FAzdvb3IKs z02>iZAnDb=DHT1v3^wocH>5CxgoL1>QS$2-Nb~?PQ!_6g=r2YPaipq|Ntl~cNdWys z=vP5#x;~e>`ktq-si=gb1X3z$l2HT$D?ZzNi_g9AXRZzmznu8^ctKhLRNm$n=0VDR zH9Z|c?xy&*OiD+FG9JjPt)jw7Q^vhiWF8(-{r)`Mp1&$4(2&tvvVG_z@!G#;wmoQU z#pe=5u$_(8Rsm>MFmwCt>=r_AzVJl|$Aysa5S3lj&Z0VqJ}1Cm`{K0fe4 zzW7lAQ1o+mgGgQ-{LFKmVcgR9dRtNgRIX#V=rBs!H!$M__jV9o!OREhn^n0AQG?D~eZJ&qrTi3WW+IrOVe>SB{{H3H-5wlheoL zgq=a|PRlKQCkDU>_plj8b>c1np72rgzO6N14Ve`VoKAza5h!vlI&5dOw)qNtER@gj z{73CNGhOX6goSr?RM65As$jdB46aMUO{8Y(@Y)Zhe%4m9duN zb~!R4b2l3X!7S8nW`LB$F0^EEnRjk(Vf&DF^=}4p?%8w_fHc?Y!M5wnTR|4KIvX|p zXyE;Isl{rp-qh)+bbaZ1Q4Am|_h%}o5@oBbwnx+X_x4=TqkB_W&Q)JJ?wQTJz~6<9 z9Yem;l^br4W)RPJjEYyAdkoRhYB{d>ylDH?5(xlFri7%VvnH>jwMed&)sq~tm!3Qb z!mCUM+7gly9f0ND8g5G6O2PsGeVAk(fvt6ZetwWlJmvZ5g57qc?FbnL><}w3s+;Ga zG^}1xn!9;oq?-+rk`LF*LwA?^vmJgygoKCxj_11d+?EukzOAtzEDfdeXiiQpbo;-) zpKEqOYH@q<@GA0YoIpFPVLOBW>*Cyv)0jZ=*s?vj68|YTP@Ao_1oExj07>wY9{~9| zU@tC->-J6dlNf*(-3GPy)$7~80=!F0O9TRrK`MDtA5~Z)+_-FqKfE5sR+AvjQ2fZ| z%a4T3xc-ysS(GgHH@iX~D0~5M>Wd(f7-&VjlmZtPv7Pmd3+lA9dOB%buD3MgZeNu$O3(MN@3m_!Sb(G-k4N&nYXF{RrKKCU<@oQK!ci*L1?|wo&h9;z*%+Fl&A! zdp9jDP~6zsG6Rv8gqs^GY)(KcuAz33-NeSgaJoR+%G1^i0srhb%hcgifwZda2*}Ly z8B=XtJ)rk0C@8=P5(OKa^$SoeQcRJZI#~n3K}3RxiV6;>?a-wV9%i|oE2bRW=7Y3F z(&ZAt_clBWJyL>Y($cnZmvW+T;P~lr1-xqR{6rz@DBZAo*9S&-wYw2~MIrB|roc9v zm|Pn1e+^%}pch`U-*nRE&AbeH*lcE_`Gtic22W?$AnuG&#HYVGxGZiS?ReU^3zIYk zXB!+G{L3CGP0`gJ0tW!PIVE_7M%fos5PQN|ZC>Q^4_G=CJn z`CZnqqA1oaNaXi0*5m;YrR+8MfK>W1Q-J+oA$06mY{nT_K}!n@Ll8qfJP9*~$p~J+ z4!{FEmwt3IurrZvD@(sYm^;AJgM`hHYkO_`aw>qP&5^Uz#%(SHQf^{4hqiclcsN3W zHTODt0&(K}ic=@F>tXJUy9&N*AnmnnJLGh=UKyQK5(wTm!;-db_8^5vAy|!-A`(IH zpV==cZ6(kf3KE-{ngT}mcd<&|JJE~iv^KEU&=CM@ZC2j^AeaXD8Z1WLo{6H&^~1}3 z2fgID302!qjWLOI1{+gTzns(sfVjO%Az{Lm( z$e*6hQJ*_U_4wY2Dr;y+GUN0xBLin9Zga>8G=~9Rz>;^dr_t%l&e4(CVxt2A&qSOX zgwiTH3Lq}6D%zq}o{!vsIJ1#9R-R<^=jDf6{v%pS#&sA2{kOW10Gv1N4?$Fsf zE9{I68{o!2r&1~aEK5;&6bNbo;J1V7*PZg`KmT3{W`HU`S?uchwV(g~)(E7GYbCoj zKS=$ThC^?nCIBT-fO!g0&<_|Byr&15r7}ih=8^4-pYri(12>(-peO6)#n-Un0N+hZ zC)0!e;vG}D_5%j;*;$GDqSwmytK_~w_;L0~sx0p~ zx{AgCfy+xZbq$B3e>C}VGS&-gLeG%vbiYnmUEf>G9`rW=PGX+Skk@cLtQ3$qqK}UN z2UL)sVly*|qKE`UX;VH=ZYg*l?yqE5okCSfCuFt@>U%$V*tYIcYE%zSaZ3Yb9s~DP zIcrZh^5*i7nhl{q$e7iy6Z$1{oxkf&+!}2QS?m7gelad}Yq#>$vRDrMN^`4pK%yQL z4$l{i=aY^nuDTZ2SVjDoLjR&;(2JULnN9=A*GG7i%C(HQ^o_o+nE0RhowkN^YHN|z zCyST#h6b?>Or{U!Km`)iG)-p8GeG8($fge`%!ssWnrtx2)p2Q{b-tAbBPPOWGcV4q z+tVe1^pX_=)Ga6v5CYl8bib$D*1J7G`FTD9U=!Pyr}I|7epajuCAUAA(oc3lF00ov z8a%cSa}r^OT*g{?e`mpf9DZh|f2RHEEl5-yLC7KnWWYD&m{^R~DXj>*%$aMa`$SkoY~3UuV31Y7A(N4 z8B+flqSJH)Q#M$P`Vsw*&SP*Z`xVv3_ff6V)TOUU5KIj!~@Pqq;ZP`CjJwsPGP8-;G)b2 zB#=sZ)D$E1TaO(-KceDrcw&H{ixh+QB|x-aYV`nU`QFwxTS8y}_^)P-|3zd8H&PE1 z`##@4?ck@qPOhd`b;Q&D6bXcPB176PZysMgL9ROhjI9^Y#J>jeyLWp{Kr0r^84V>dNW7!HUV8S> z?FW3NXE(;tMmUoCvxeH$QtZ#WMMB-uA1tk{#SuEcZw}oK2KtE=lZY=JeG<)qemUhf zP;Sl6$?dV+>dv41T?n5HEmG3oxEeWFlmw>ABEp0ums9R#^>G571x}_B-tC?~7gUXqgobN3R^iXIE5!s19_(VFEiB>zsLg`uI9t ztEh2I5(fqJ?~~dTvZzXy`vKs#5oJg zb?dSgd|Ine&o+JeAu^)y&5N>`{oeY+#aU(sBnbqthjeF1@ghItNR}t4aubZ2 zYL7{oGu>O)oB>pd7Co2&8;MNuKl`&ag%B)?+wlR5JW>8VGTa-OKm-!J$T(Tb^Kt?R z4JqMjbd=3PQ@yFC9^AAm>@uXsG+yWDV?3sbMrAbzYfV9j9fMMJG4o4q08 zJSGbXOJzg)A2B{?zII*pTGLWR(#5`eyXL)~H%M|Xfd7fTvTg3J|CaJK$qXg;^wz=t z!8N)Sl==2QdA*b2^w>kB&%f((dgfjLcLJWQkFSnbloY3_s^P2zujAwAFn>E3?{;s* zGVr*sYYl2vpBs*rtE#7?R@TAsGQ7UDX$O4diV6LpbD31}QW{F8ckuxwC>errb#-ui zMkHLEaP>m>Ept|LRe75^guF5F1KkeRy4MtEXwY|ujHEDMo(m&v7C^5t&FRcz*qdbH z6@ZDo58H^}wEll^sM~)#ZeP#sDh&R^75xu#7S{p&e`q(bIl$qi!o0u#!wJ3o2B1lF z{|9RO-#?Bb{vT9ogPb&|%fO&gAw6Ytxu?H{>6B%~uNh5rt)in$K5evB;{_~bptQ{U zTU6!#W6G~$53L4J!+&Y<_I${Y{mlD-85IlF-zAO9oxZB|23FytP)NSB|u1|??$T=J`*@nj~F?lu3Ye@*kyNb(BM3KE`Db5wYbSmIh* zmL>WT0JAm{20A{+n>fqyE=#c<=9Zu&trt=6Bo>hU?|#Y}8+VX}%-J>ijd(w1zu`7L z+@GcP|J;`VyUQ~nzBsO8FG7vLC!ty4FND$c>y4mH;t?;(@_K9pVj=h9V&Gpj#Y?y# zsD&W=O=`qMZmH<2^&LD!l+$I!bsBW+zpstHi!OhBT8xekj~uLIfvDs%5FFNwD>Z53 zZqHnX;PO?+EZ2i%CDy?@IKY15;;Ls{4EDwa^8iC&=^gSAteR-A1k45LCn+_u*bM=X z1{Yck=_1`eUC-BUOif`;GfZolbp6I@lt;KYp8rE?{uPEoNy6W%K^5#tBqy;%3bHaz z!KxQygqd}*2Kv0NtSHKq5!1cjr2dwSZnwh=_TvbM4yveN`o~8ZV=8X6kFCfmq(}}r zzx%q+3tl3dmahxX^SFx|Z~RjfKRS$;J&fpKLwY8u5y5|&@}&H=fSNh%Fxw`0l6O)d zTcV6FZwmLa<0k%)`db|~=_iZvxaQ4TH1XD2sNeA8%=GFHT=Utg!^LeA6Vt3avaX-? zLoU*5#z%3Xji;^9WHi{A*hB19gx3ONgL}!p(4b*v!w`P1%Pc-8-v3bEi;)k^rEc7X zu^f}e@^N^Btp>*uNnz$r=H~tA#s2^a^|)EHW3h67r7=JJXISuRPVuO&b_VRaqW+>I zC?tvO;z4*nQ)5dE%!xPYC1du-88h#91ce{pH6C1zvEaybO-w?D&%F6q( zWZtB(PYy(KM0YwzI^3Zu-QM|=C?(~&C4yZA1wmqX^?9mVpCq|AdaE)}opFMp6flYJ zKESnh%20(-O^{g2OXXD722_&?m0PYuiB8eb)8w`0qfNO0+}J|(ICQUC0(M@$v3R^$ z-<+E@L0dZRjIQ)b5|>un72#DwA6iLBIkmf(L_%WFxGp{iT45=BL{#9p5nti=??4Wt z#IzmtVNf66o*ukB6rc0uv8~ZHk(4>DabWdfCu`q*-9l3Nq_61U4?p8zZ;g+i5qF08 z{UN}DB2{})8K&GOH;Bq{2%BhqD61vuv^(4$ouzg!ywNkaFQh@E<|;$anSw!0PfSGs z10zxS{ngqm8a-_R#Q}Ujo^$AnnNdts5`dXk#7U{4IX-v7MoSHuDKgUtWI#92hV@)| zHQibhM##YO^^x9{ilo8o&vOrBa zb1{087C{#cdg%aCXX+=O;f32={#iUeLi+~xsL7E(V?<94`Fe7G!hbi3)wOhWMCb{h z$=XipdiVHk4W{Aq^TCo^x}L<+&FR>1x|8ifoWgN1MFLC=R4gSTH8d^tH!>`731Ju? z418BLwS&#^{<9i^=;*uR9b+FDi5L`rihML#gx!T|A3HwNt?T|Kw;Fs-5sH|%-YXTF z`b}QPP3GI90$y9*W5eE5)fXDpcqY+zO`Vn!#`^KNsr_G=;kp`s%jx^^Fd48DnX{wn zJ+t{fgtl1nTeD0q_+`;FbQtLe~~75(L&6>)14we9maA$k&OKb_p)rHV&EBHrAu z3~ZxNvB^9wPs`Waq85ry9y{{>_ zs+L`H@gk7vyykwbRgP)9w}t>ee1t%4cKDL>u+z7iJgVnsEV*sQ>oSKx(}5 zjQQ-!-YMO1uekTq3tsg#OggQrd*A$n{BYbytf%B*{c?=@j3xR_V9xwirRwVY-db!1 zv)!tZS^EU%I230?99>A2OE0agQSfj?y!$bqlKH(|dtXu}a^##1$biiRZ?3K<=APd? zRdgs$Fqwx8MHvs@_7&4slT&aIKR-y@{bqbA*1tf=+e@@Al0iNy`8 zZLwcfe^#eLr-Uulxh2_CXR{dQa{4zky1gEL#y*xd5~V|cZWdW|7@LOsM4?}SaP0Ew09L{sl z*TMYv2_gEe1)jX{bsxM(9k`M=edk{X%Y!9;;%H|44ys}L+$o3Jdzt+E;LKDz ze2`P6jl-zF+J(a5PR|lEe)e=YZOeU?cz;0AHs>-E^CHumd-6f@lm3}xc_VdriZ3(0 z&Op8;gLAoGkm`i^Y)2+MrvlXajMrsqGrWi7Q%y2UnqxMq5%* z)18rrG&N?ced*MZDR&?_Q>0ZD%H^_Es>1h(xqh3gn4;f`%vMnRr@x>Ki>zwP0=BI#uMgA~hceIkFWG2EDt86}3|~!Q-H|_=2YW zDlWET-@#?;ao{tc-+bcjzgIqZlVX3uy}ryjIEO25r6E3WVz#oTn;O})9K@IWm6JQo zbsmq&N0+W^M>mzxq>}3M)E!y8%r|9fE=PQGj1<-2EBkG8*U{XF+?KNF!uYMY!v$HZ zBaAH$%%%~7z7KXniHQdHT;(-I=WW-7!3>x`cPI{z@?tO zc21&~#aB}$jnHd;h_Bb$|2yAJ9j=~DAeLp9V29NDV=yFiR|jIhc}DFY);IrP;dNdqM=I?rFq@8-s zqC;#`fI)NCo~{;*2Hto_6m#T@i|RXbx9axJ1qH}YWf}RZmWXqK+Hje8T@JSLzr9(0 zUf9Im-MzYPH~d@l^d}7!E1g%kAII4Ydicu!OjNp)-SVMNuQ-wGQ5lTZ8Kf($Bp9cZ z5{ak1gTy&lRNwOZ7gcg*Y$c3&uD*AU`H*}WO0@0tDo??^ditHo+h~EmL+XdG*?+Ka z&6}EVSxD%9pPMy56;QzN=X|BWz~Ic(;jd$GC0PD2yS+m*OzD1ux>!}7C)guKGZ3ktY)uW3v94(fyHzT>|rLa*NG=$gk3@ zGIpr?NACY&0k}9fzBuHNCXcOIR*uFnY@Ci*%ow92Z2R`(%u=;v^XLicpq zY~_P1XT}4g;8WyS^?**#qBp#3+JzPluQx`DHxZ_LPa-XPGyDm&O= zWcj-w!gBR@FsD+ScBqyH9pRpDb{v0#-INz1pE5m72orC{I$lxPRu^yBWlec!I?wA> z`Q@S~8SzV7SBYg1V zuPNdfUm2G*NHE`uzygAollW#SJWl+*qnvlY-gy3lh!)Jn#|sRnxMcGa7fiyKVRZ-n zoMh?ctas7)!F+9qlEM`>aTV%e=tGw?z-HV@h9$3NJwGj@bx@G^D|$HSHTOaGZh{xz z&f!r(QAP6SI!-kLsq`E9RF*efe2>X)Mnt-z`lqXGjj`v#CV%{cB3!74 zq!zN)sZ4?{dA$+d$29hg!^5CaIrN-2q&;lEe>&$!rgE5PlH;aC>02q}D3g?GWIaJ? zz3nJ!To9PHm8Yw%Ruj$28}nw6D@vN1

KLZu36oazo!UyciA>MIh}FEPO=9MYWH5 z`kZ1x5A(ayDPutZM;k6VX=w<}{d=WuzOKR-f%;dCKV*N3TX6Uv=Ltj}-_BM_R}mNk z!TCvd{jWrvoZ3cjx^XsdZzSWz)jubXa@DCM&RPci?w7zMOFr^avShjxVrjH-Z%0cO z4+1!ns;x@7h0 zq(gWOfzFid7Yi?`qpv4fGWt*OCb9Gzfib~|IeE(oWM`LZ2QO5I_mPoMHwM`WvY{#; z)8uq@0;onhbCjFwtw;ZQ>o+*H+FNBt(m}mQcWUu#brbscZAIlc*o848xikUK zL19DEdwN_bd7|5M)TXUHWM*k6`ueE9gvp#^-mR`CqFYGwv1ZzD`O(GjkdH{iK&tioN~5KSJ@Cb(b}2oe)a_ipqP## zm&6#de1MB}D)5;D{zi#$8CDDM6|l4rc8a7aK@kWPIMxW?llDA=P=sJ=K7WB?7c{|1 zXEx`loR#}E?XuW+)`>3*T{?ST<46?++(LZ!bDiZd&l`E;>0!Cg?W0kY^b#uYbGGQX zMrMPV31cU+f9Nd_&&Gx~^2NobYjX`dp61^NDrJ6P7;O%ZUNq!SHbmWtKoI52zAjM~ zTIEm7lK^f&k;`yI;@{|RZXepzPj{#hSHvS4fl8}R`q+IN1YL0M$6Fai1PB>sfEOWM z89tmynSkJ?{33c~@7GhnWXae#60<(tNnZrWczIgTO`g%)3@eRy5I>x9!2UJeMzlX_nv0?XT|tPWer`Y>n2T-m;tK+T*W^{{DPAA&THnunrG=ssH}K^TTZ5_v!d; z_Qn=(HyXzOjkr;G0){N*rD9qU?3=IV2Bf+40kr@_OnjG1A{46(4aPFE(8%qGpWv*Y z=QuF4vRZu%J;VF8Y_;WqY=2e*3BcO{cW<`{eGy3@LqI?{WrtCfoLXI_UFa8zBENO^ zI}y%3@vm}3DrA%VvP}iyQi;g(66^KQ+S8W6@AA(SF0Fnh^}4O2+Y}^`l&hMMV-gQW zcz~UX2kI@R1IT=#B={dYiUnLCRbdFkmQFOg znEHQO30ll?%RXSLEJ5`Hr~;J<@7fup$p5E|QxjbP$Dz)2yqinxJr)JC9fO%`|?8A zGTY9(-s*(n85d`w5pm3aKOW(wG=EdGO_t6}&qN+C$?|V=Y%r(q%f?lpzilS!TBkof zcl@gq&p^c#LlDwPO@3ZYdO32qk{JnrqyKArW^wUbuI1V8WNC%XBH!I45^{q-V}1wj z-HW{8ror=)^Evf@;rJjCk4V(VeF_LD*)yvdv3A#_jabDt$nn1A z7X`j5H-|9*U5`l7rlB=o1$Kz0qtq4!!OTO{dQ3*k{x z?QEHgDOVJ&oalID1_Mc+{A`ZK)==`8Bt>Ip>;#UKlLOxjp)XPm%l%(I>kf*T5{OKs zuvwk^mlG;D)NveSy$?Q8;t33q(wJ32LITgLK%+#mo+dCdMc?iZkb)^AhQ*nPYt*w3 zVE$OJxU?QjtdQwf(l9Yzbe65!uOOcA^I2(m*5fC>Jywx*$@C_PVTa$>j_2sH`X3ri zg+UUf^>_|&w%()W7;07dED$E7rE_sWB4*mGuv$VIBjj6+%xK}vUY#~tp}mG-jf8Zj z-$DAaX(`$O5?w)icKh;1{=(wq?^*Mi>IUVrLPhhhD^iF+3TzD{W)b^4zCGGO=ea(d z7HDgoa9uC~D_}VED&t!3&lj0k`+H|!eP>I;_;E&?W9`9%yyA}JY{j5dOJl=zd0LvH z3B1sGnqKG%GR)Fx?9oM01xxNN;ZWx1dK@NnNZ2s7E z;1c;^A3d>XL&@d~>|Zdhih5(R6_@2tc%)>#F_jWm9{$jsC=708Anz73aA%Fw5?(h! z3uhF`>K~tecT&y=YS?=eqx8%S4X-&IvQ2W!qR;dDeuar2q%+@xWxS>tk`G>EzMlP9 zQh-fkwMN=;h*%$2ZH-8Ys`^O@SdD(&yd=_?Ydo)0^Y;CpJadSn>eq*_WXk58+v?-I z2hL-J?a6I#_mt(wNSAd`-^YG@qrm=I;SIefp7`hMRHna0c^Z>BguN|p#hQxd_c0@F zl$oV@!7ui|IEbgf6M=#$eUjFdmBZNX>+2}OoV>+3F@tHRD^dSQ>{Cz?hW`{Fx~VM1 z(Me$!UY@d0ud(h!1k2*$o0^a70upl&#&_E7s5;r*`1dEdb5Nah8f&ly9n`i%_}d$S zq>5Cd0neYEjNch?m5s6=`>?FliCHD`xez3A!^Pl@GvG5?$^n7a|;}n+M z@)qT&R6-C(OwBx!iC>5TY_%w?mn%LRuu)HZ8pl>{kLCu2ne**J(VA%wmY#WRW%yU|zl5_^khg zw?CmJ@O_2N1+tj6VJe+7tz{|H zy}3C^gm>d0!qP|7^BUIc0$xvlnU`=UYojw(TQsj4(TZ4IKIPw#@74`NZw=$41X1<=7A;6my5=!?t*3^;lSh8SdiDF7VSy3G2-+C9I)xa<1Gs`Si;}aUZ*+k zPK~u}*{C|pZqE(T6Ln^=)QOQ?hHVG4$~HR*5N;h+<3wq&UAbnE7tAV ze6^f(jL~lprhSd}-?k)bPLfi6q(*N2GM)&4Hux3bkE z{NYeOM<$CuMq}<h+N-ysViUSK7J?kCEj@4#(uenULSa;t(umXr}{j# zu_9HIF`Ho!j7W2Hizxp^Oq%CGdHp^6E8`!j3-}bRE?>lrS202c?-`ZMw)*sbTOr%l zcuT%#9ro`rn^$G@2Vh-xRZDqLTq7`#CfH?0VK zpW3RrT3^G?fgPIli;AGq1sKPg4};QhT?Y)Qonr^E*@pUblplCYCWfB_VI} z?LB=#rfqk!B4mABZ~t^?i|2h7a3 z;{(U{CjOB^HSZj|w@A*t3!cY=tl@zJy8ZpCuAI^JTjKL^+CBWw9ht)hglm=Iq)0jf z&wum|j0oRwy1KSNta#68`@6McX}(yI6la_FzS@B-Z}Ph((Skn=zEU&{Uh`TtjLjNN zY!*UeS3cjgxeq<_4Kvs-2jxVoKQHKA6~^ifV}c%;e++f`vFI^k^B$;=`_7TrJiU-- zzE?8T7%F|GdAoU6UvS`3(|+)>d7l?+e%QXRqjrV(+_i-vU4kGfm4N(r;|+EW5duo0 z*E*bJI5S9#SaT^S4t{@NS#^E?XviR^h{$=9Ysp&e99160=%XZjk#`#v0FYP+8~*SBA&;Bj{Pb5gd4G`0VXe}6gkY`M|V z%XNOjZoigcASyxd%NVE$1xoueeIQ8I@!!|ORPUzV=4nBKCtdwn#S zY5BdJ7C+3=&~u-w2+OgXmTJFx%HH|tKk5bw3@wr>!PQOiXbvXC$@Ot&O-M_MY3zaC==`Tx> zCZ$%fZ+MV-acq~S=#YRRzQF?woZn}$T7W_-SpLI>g!t1UaXB?MkvLgPA(k!jM; z4iiPBX?zbTz5tuym6EBU&IkBU(v)U5&6@>!y&1H@#$d^o6sBJJ=Q68J9^@+?EP&M? ze7ZB?X*S2*>!oN!9yG?qNpHJw4;r=IOlgRnnqTXobLH#j&UPhBoRi5lQhmAWdf1_h zRs4-@^SZ{Y`~KF)c^F6Pz9ooQ-tovzK(l9NR8bmbFV+dGZ;B%17TL&D4H2fL3K76L+U7#%_>KEgL2s%D>^IM3mkEF8Qyrn zK`?EbJEv*;H3mIz)kt?MuG_DTP4i&Kltup%t39=i&J-2i2muwW!{h#OE-39VB|acM zf*q)e!+)}nBCjYQjiphaL&^wiFp%)a#I8~jBT$uTk7Ljc3TKc_X}LAy$GK0?Lg&*T z>0wrf1>5tyVyoY*ACS;e<&hB4;W7-(!L+2Q@Dj*(%(weVm!FXNF3%hAyhk+J_ww&A zrY^xZS}mDDoi_t7m!zrJ1#CNyi)wrW%|iZWuN|vypilH1XVL9jHL5l*Z^aLfBYgYg zZ1QV^Kv^HlDq9+{_{dH22w5;5R}Hg;IN*+~Z_%nk?Urgan=VP}dW>6gn0m5Q(D?Qb zW)~Vq^)17&BG|NN+4??F-h5?6PklN%J0b77z{;P@j~3f>e-OuZW|T*z3q?Z0*O=60aj+!tJ8< zN3W~rg-H}LtM+S#)s`KLaj%&li|rOBda1N7@j|NG%>+NF6pcO%`!|+kP{c~i%hRa48*jr-^I<-q8+Nz{0KA6T;%~i~eS$r0 zI&)5YcR0@a2UufshM9&4hT1;~&*9WAA>fE@5# z^W018dY$8PJ|{i$IE+>}eTlV-Bfr34m*|GCe${F|8Vu!lDrD%lYK`Y+C*Sq19?!2w zp7MC>Y){>p=!hEKaQei+4??5&UcpXEeUE12HvE+1Mt_jyL-ZqMWAOM4b*8u}j?Q?> z05dhGn~8xZkbK6wG_Rn>1SzuE^89g>@L{8CRmKyWN6p}mOj9lrZ?KY_wzSfybPVT~ zdJ!wM`A2i}&>Rd~JG;100uLBJj+iM;e)`4aO*O!?$V)P+cErW^#w?w4Ie!>p*5zo9 zH_>v@FaB0zUvI4_H$rP9q~;%YSd1c zlHQ2a1nW2N*o(UNd;Zm0XNc=J9~je@6%{SCeo) z!XMVke1}Z=Z_zuig6*{(deXIh_Kv;^cw4PcOCIi-cicW}b00zI-d{z!R~=w^o!YRkVj2~IH`-bkVg*#; z58!Jp9e=c24*m6QrHC#mO6>V+<0{y;>0+OSVKYmxKZM^Wknjcc!5B9t-|_j=c8?P* zP2?avw?^AOU*mnwNU7HO_iijWdVHpnN#hr0-`8=rT?Q}Q;v7%#<-z*Or!P5gQ-nuS z2#)ZH?mbHfj)NLZ$@mM`S$eA@$0MpfZMz-X50&@!_tvhLLmR8I{Mc7eIh_tMN*i|A zL#k}iG9jM6H`tv|2cWB^@CykAybS-1D=gF8c~5tR7=(&~wJLeVNg=en(t za36Z5Z6h}tBRuh)sRV{$awrJbN}aeqLwZoGC5Tay(Z{cN=B#(opq*8I0PjU6@mn<8Dcop2J!_$zkzrFF${wKZ)0X@ohwbg$4d0a z9aH)PAZ}tkz!<(jmHW%$Vcnr0+Mxg9*F8gKpH>FxsYn?cZwIRs4w+Q%p6 zO!qytmYZMB?I%}N7mNDY=nStAgxqYq1-8Bnqf(qaKPvOm-XM@S#J3TN`yC?{;da~s z)^E_Ahq|SVkrC;fq=?5sHmew(2*(+^Tf{F-azS)R@DWiU<$nDkN)o0FRrO1hs`?2q zM`GeA+H8;!M!$W9%q@?dId^#=m|ROezG$AkdhrF|BBAwaYMo&RUwJY6kPdY{Qq%L* z1|nScu!%OTO@yUW#NVt6&?#yMdeTCUvS&Q8_op>6j&% zfa(fLR%FG1D?kb-==^go4rKBct4cJhidNh!%W@Wy6iEqUH!WZplW;tEl9cWGJw?ww z7w#7}3aq4;SnUV3_y^l`*?(~R+#+_~CHl($F`lwn&{bm_DRr)#=Kwt|@A!62NBqKI zHaXJJcA26vgN6RpkF=rO{OtNZ%<%rk$b}oF4LYvy0ycNkFSw zxBY=m4^;7(QMD^dJFU$os?TE}0JWwy^3eH}WIQ45L;?rS&ve3h(fH8)U|JUf767+1 z)N|xOh~A-8kT>DtZVPkryJtn@R8`N1y!w-v#?$HN!W7t{P?!kAuQJWeR=GRg3%_MgrK&Oz)YeCU3VCKcgzfn%8ht^u&Di(#%2Le`p zG6nHdR>4hv?`WH7#khV{g(M1irtDY{Zqm&i95Eyj_=RIkU62CgauxZEhliGMFloee ziIk){4}Uj{SqH;AhmnPi*9@&J~1cfdVPRr#z1s}=qnqb2W(Ii3X+M# z#{Q;7i4%DjfwXUv=&}JD9b=x`GJn(0N)t|C)ZkS>@|kHj9Z7(E)7yi0?G@AK%5Vd18sg zo67j1%`b3=m?RQ5p}{*rK-RX~Z9U96JdmjSnkRuQUf_S}-~Pla21Nf#=!!%Npd_Hm z4GI7d#E>HQc^%c8X7=GJ&T;wZgCbA;`Eg0kcLc!w<2IGCFyksT&ev>$PKr2IaKiViV3^?iapqtVgNc_no}HF zqL{v$nK>np#nPOxNR&9)TNciSH3$t+AFTS=&*=M>E_gg1`-&O0lrz@a3~_NHW%E62 zt<8&}o*_HKpMDsy1%`lB#46kUoF7W#_^}@2hHyf4Hffp?)w{)eB|);D9?FY z8o4>N2Y7>6z!At{oi{5h>*VTUf2;F}ctX&QEQkN+jzcnltwY`U{WXaOQ}-eO-TnQU zU%W=yaBlmNa$LsA7OK6!bin&Fh(yRetgn3h4I>%MY_?XkC!4mrETS6a_@O zb7s0T^DR@eq!UDm8#cxQ{#pfDR5=fTCmGt=*-59?`QTx|L6tWojqWjd<8d5Z@WfiN zx9Q^g8JDJDr40(S{`oPL6X0*3#od!OLJ2+%ILf0d20^{hzb?r;TE^in=y_lg9Fl4MxY>ADb83 z_a~6H+@|<~nG;2;1{sJ0lmNDFJ3rDm%t`d#rc;5A4bmtfuMN{3BizTe_qP<0$WQjO zQ&KIy$FEKzMaM}4O9A-5;{tdX z0N0}+dG`7C`1wt~!z26a_fgHN6)ScCI$0pr3_1C?uGbtDc3yc-j$hPr&U86~VEthroGSEYqJ*oQQ)SQX%v!*!CRK z6ZKvU*PrGiABO&Xb+|3&8}Png3ssB&5U|iyg_|(S6#O5YZnd;g^L2RYi7ZIQ@yigI#89>gZ~?hUIqtBs|@u=_iTO z7j_zUe?ER)*T1HYra9?oO3C8wk7C}|K*Sy4@d{jn{FzaY;IYU~Uq)0MPJv~J!48J$ ze-y2t^GU@eEnlQz;IM+pM=T)1qOT5nUP^nT7_!eZ`o9Bipy;b?og4Q<-hg|=PmLY8 z5W7}lF0yW!J9!P6>0G-`A#Qn9f9J!Io%`G7_4&1zaY`>6SCfWY!Wl~G686m2;Xd|2 z7IV`EYE9*w!UVr13{)P)0Vhr6slqBwJD-YTb%?DFp}*eSUv>5`GPMuQ`NQGM3bPKx zNh8{!BWwlu8D|JS;vCGa8wzsgj#T3_GK0_A$%!$RxIFqUUf(iPTktM|`47rY>l39L z!o-f{l7u$#^{!;(y7oU+sP394Lt=OJnu1mt14} zgI7WcflEdxvpHCa3nE1*$h27pU1rXAVzOPUfd$>kyUoBcVC|3r97!#Q=QPA5wbIs1 z?9r;=hJ;4EPKgG}CnJc4PaB;u&ki%PiEXf+JT02Ihg2a$d2~L&An*o(U*i8iI!jak zS1)ULvi1Box{yP6@L@rXgzv$lX7wMk7?Z*!qtT?1unEF&>~4*fBn7wSdqK;nZUQ@cO&je#!3}VYS;s0K>>HNiBgb;KM zX(sySK2~oaa}cx}Rl(Hl`np@{ea-bN>7B;ADMsUbU)e{rn>)xlQvU>Dl^M=PkIDQe zbf@7wNhBRv*P_)*`=6vbFy;$-py_oZMD{9T=*HX{cadqcTw05Rra0bLNn zW@wcdkLGm6lkkr>M~&dDEmT=sGOnp@UhRSM*h}`12W$2--gRep@=K2YQ(!iBUkIlW z{v6*#vP3K!rbk&v6`^u8NX$>bVr zE2U_#pCVP(-S{~|2BxHCrlWaP1unKAb>Z~Fu?hBFtwS!fzYjEh4`*ntfwebzHK%w6 zt4pRRo7pxj!LU=TafrWk>w5K{Lss=F-;ax&t{RBXRd1T2sQB^Z+WCxHq+|PO`Xw?J?>cBo(xLFK|SOcRb%^+ z&n-7oNM-H!nI))%|KJ$B2z(8_U|}F_9MV=e>|MRy(OS=fw`9-{nI7xzIcGetr_VD; zp`mt`FG}{ElR;-ujoS_r-X!$A!EFUi9n;Ve<?}R{A^+#GU4ws!JBW?J%J4b26}*iIRB?l&z$f5@&_YR*ueTd1t%DvF&8R( zLRp&%;qct7517pVNVgsWb8Qy1fmxu5?lhGLk#%>Pq2pn8(uPl<1}fXMO}9xK^UKrUtC&g?NTaDVOsXPs7S0w71z(CfB|5U_#HfQm||mLf#FFW@g~Wh#yB3eP}-v`QK}1D z6Dm>D8J!X}9vN|mj9Q+JUy0r(-5NdoGz?9ii`i=PE5L^aT+a%bI<<4;A%8{ltoc!{ zlnGejA%7XMx#-(gXQb7OE+#hiQLYz|)EV?pxnp;=hqF<0dEZIn+dZ3Vm3GKD)bnr) zyoJddWu*y*e*Nl;@A4cAQYg}j)vxS~!dkLG8)o5aNB?-eAo8;_DCOOY#vnIKp_@UJ zDmqb`EFbqaBwK>Mp*`K5kb&9;u5M20-QRS`_GQA`O9XQ*e3uFuX7e0!1qnJH!It2a z4!O)3$KjmYj|6XPm=2?5D4cFFh>@x+K*EJy!t?vdQAfi(kZcMv;5b%)dTNyX5~E%l zYlA(`3$Gaff|29$QL*D;UMOk^ao5A+1b}lw0RsV4 z0%5b5+V5+aCNN-J@ct$Fx%vTMQ~oZ9&39gLVNRz7d2c*7yDl*U$lSj(*eoBf3E%xL z+p~^-V|%Z@vlJ5imdoMG0s>b(y$!6>5*PK8-=bg>I8M4+nmQa9hPYA z6Hr8^_+GPf!W@w%t}nb}9a?zyZKpX7et*7F9!W;^l`@KVjIm-5Y)E0mdmi#<3X!JB zRi=(Tsueop_nSE2No#nktT#5*8u!yusI9o z(}glsV-}+>UE+!rFM2sPgiz73Xq9o!m+m1m;X*}hAX**Wb4-%jTkEX!SA^i#T^S8I zNAoYvc95?Es`Ln6UtilQNGsR?gjVJIL=UeKr{$SL9uX&ffxTV4;i&o?84b24c~dC% zGI?tzz@Y0}V|K0Of1F0mb+a&Ci$0S|GM$&tH;0bz%U~^V;h7-SP9RS|7wq}T+f{(? z(LlS*Uhftwx@$wfV_j0Xe&_Cj9667Z`Mlr(6cVQ3;^2J2fU{a^#jIen$E$I>G?W}b zQ`38e<+}`*bC~RbP8|Nu`QknSpm)BYy}o(_f#K#epFLjU=mF>Z%e6+QA*b)dD+q|? z2M3GlU0bvZH7l4a3xoIW*M^4p?oeZR_tn>Zh6`U_2XRYOX%3$}bOi8*o=?e?2#a(G zM|iz{->K(eCiMo1px0@;n4!=0PknL|#$EwZDBbtnT+t10o%G}Hrl3CsyUpj%>l2{$ z(qm^A6i8SUBG|Ab@$h*`O_BngB%|SF+R@spE7{Wn^2^Ak1G1Rc?Zk(rs}$FSOb!Ow z%-#N)26gy`(G+sY`E3z0@$h_c$<=QVWZ(!NyURP2)z+RpeUn%%J)`cpM_tB7ftYZ2WR-WR6Pld8X0&S1=LJh-C-KCdzVU71?T%$c> zdc{(jDUr2mpimE5{(-&B%8R=U!@nXJ!D6a*A5PPP1^e4)?ufS8gmrb&j!MO`MxC65 z(NbT^0DAWXTW5)LGjx0hm{*69#mX0Tw>nu>?K)rO22D&u)6 zHHJAf2vJ@1NdmiCg)HU1c~uMF>LYuW=6vcneqV~=p#kR!Lv!TMR$?zbSmf z5LxwkK6@!3>`Hlm(tF2c8H&9g(6ZfI1Gyu^J6^vm#&+Nuk@CHk$W2=`{H~3nsccRV zE$A`jf5oP6HKU9c4U{eJ+ukN&@Lam}dOo*h;&seS)$!%+y2R}o29Rzar&BwB_xAGK zdf-P*qFu@%ho2! zX{tJOd80M6IAYSWawuyXXLAf-q-CQVk7rlYb4Vg03{+{b%<~3jXRl3nTtulHCQ0)= zm7;I@JYjyUSWbp^px zALpeC)6(ck^V+BA=Yph<{PoA)Q$@tyGJ!86LC0!bWXQ&^t>BH;tPkZs^x1yMh^bz-lK{!VfBUbzYzOe$nf{$+W3$Xw(S*+3gV!z+r`@}xBXiV!$q78Z7b8qbVzQw+eUstw-EOegVdL48=|HXQhP(-+Q z7{BaCm$TBqIUs_<$(^%EW`^Qo?j5e|i-}PxIp1AB`V?AZ`YTBj>@zg{nf8%OYyEP? zG(K(lpr>4gYQdBlBMXagJ-tkq)i3H2&F`{fNz3CsTxqAAS3IT$26;{k3l_@BWU^Q% z`_dD5l(l<0(1EAPv$)?)O^2@>ic`4{uyAqv(t6jK^^5|QA5<%))r@9#0&Ua@qRtZy&;7 z6z~9~OjLjl@5jc2XeYpK@lxh{Yx!~xdgUW6D9Ba(xs4@UJhL56nVq|CHi>7>%*8cm z6wBMF-BeyW3gJY9<@LMbje3RRr0|f;Rf4o=#`)*C(QIIf`%1h|DEh+tOzQKd%hKT7 zUCQF7U}|>lN=rME>T_fNiA%KJ==la>WexL2MERZRl6RSA`HabiO@pAR;T(>-V@2}p z=H7khsZa10&^KZL3Sc|Am3S>JY@@{0R4wEwUQjddDnvHcOfNq^ckL1FLLUqo#InyHIeqb{|Eqgn8aGPwU}(H`KX}BQ6W_Z zb%6@FvA1{hmsvVv0nnC=H5%9L-7S6JM5@Tze$93+7CFdCK5bKXq%K5^Py1N2V$n&u zUL8=>57EsrG)Z)(-@zX*v4fU0Vg)4=f?_wWrt+0+JI3?G2)DXkoA^n>clPW>2eJ5W zObLH3Sxui6{WNF78TORJ8}b5qZtU>C_W1tPwauk6Z<0ze@w!UPQI9!)yViRn-{u(f zoB%i(GxpW|i$$ph2~MZgat1D&6T0c8A*6VTMRXUY(V4X#vHH z#FV=qIJ<46rFv!tt*`bHD*`^&l8|oBTJKa5;%z#uS%x=5lCNRR{QR*@wM9k7lieen zLoZ%a{kew%RF=9;Ge?s)P2iF`&uZV>sZ}hOX2KqxoKnYD@(I3|A;SNa6A(8kR&d0z z^rlksu(Kwnjg>mT^Y`+vD*2zI^qi%>E988OJ&pBYPZ@K>+ue`fCs!4>VVtu^czI86 zc-GYBhpQ?CRO04tR|3lq`r|s=BM84LcP>)6^N1yGeeU@|;-^;pgv9l&>dp0CW3<=V zdsp@OPxU6sV8*YR_qr=PDe0hl_I< zHc)OGF$=mlrTeyVrue8C`ryfF2w79)MDG8Y%Mejrn03d@%Om_|i(qYmT|boWI6!UA zL8?na)yEm?>-Q}wx`Z}Q5yx_pPR=B&2rwiBXgFzBwETX?8E4Dsw-5iQ0l*RlLlBp< zM-)C_YarQcF^E7FJ5kCu68wbMz|bUqF3E&Xlww#u6$kh8EC#IqW2RzDnV1vM09zyL zq638%xN9jAKy^5Pde#)ZX-QYw)a=A+!Ya zUY?5bUgs$u=U~+Z+0nY#dhkYmtuE(Ztr?Ct1W~97`xAws&QdX-G398Ga3xK-o8V`i zy58w#p;hD}3O`%Rc&Yq)m4TM_y_t^dSP%r$qq>>FptlqqyGwe^rj!(Va{N!1xv}7i z7X>Nvv8==QL!-9^#dFUTLQbx!%Q*}Navo&0#B1&_sE~$%&iY6$Oe~yh3v71K2@RnH z-l$OmPxrX;{>MPwD1q<-F&D0%~)~vK(=8o0p>s7IuVMW5T+9*K6=Vq56qH3 z)|B(t$ZVn0-R#{mIvhfc{%m=&#=J_KFzj!ECF^Rhe%^5pV%OTTy=CiUTs(DQ{aa=MlfmTpBcI-Eyg` z4@EGPrxT|U!o#MM?~upbEBjYQ9w;FEum7!pHN~N@D87Lgr(mHtF>!eVq5) zr<%BC`8tBqZ^Z{feuIJ|U=(Y#R@Ph4A`8!BIp5 z4+Z^(rpbfQpQ~eyMabN;U?>9C?-vf+Ya1Iqox))t9)tuiYeINQnPig_bz%|mtN&Zj z^t3z={na;t&Q}NhX-LG9#qOVd0_7aJml68bE_l=qeZVCZs2p)4H_XRz{F$7}hD-eK=R;(N zq>y_(LjCBhV(cAV^ogzO?gzewo|(~1KDc7o%STmG%`uq(ECSd&Kg`xD z^U0RZ2rH+_+ThFvuurJ9w8CG17yqfXdgNU9oVNK4dJXAa^pwj0wn54412Euly7S_D z)q<9fSHU|?H517uPm~&hQTSF(VY~DR!~X=SJc{L()JvH3bVo!V))d_C8Pf0iq4+Em z`it-par@URWa3pkBoF0~)uqKNXUt4Y zf(c~BuruM|;g40q)&HvnF!9$en`?fFp!v0M!f}uyQXMvVa{EiQ>~zJI^ZD+~$if2d z?A@zFSL|E!Zh};^6-f8)bSbdmyD%~SV%loow>d^tR3wIdgE!lMEn#o18 z#;XyQW%c}(|9wY}Qn73T8LWAR!)7-)5Ng;8`IE)CpVkaL^i%TnynTS?aJiuZqDH-^ zJ>NmKt}9IX7N^g#d>&!3zBef{4n23GC=Rz#lqtjgcJ+HBnz01`8LdTWgh{2<&am+x z8D=x(n8XD2B#YQ$I|Y&7C^1Vv!G4YKFZQQ08_+N2ZRj$aavy?WJzJ)9zew$`x3yus zni3KJf^qn%vc{Couj7991ks~|;4g1O$-xIGd6OMiN8{MY-*VsT{@^fi73*_}#@AWwKarNL0LJ$iZpFDzOk_3@U7VVtF zE$+Va{;T`rs9sO(d%s+qGWCtem%+5X-`A?eqGiYCdRj38`#g!5dAiXY?MWX#JU6bJ z%7rRE-6B{STRxr@-na;mnYV)ouTS7N7lrqc-u;58TJ z-3B72)fUHcM?h&f%<|U?IQ0wjhHZ4S^yyDRCO|Xalv%ghS_GsfM=62t5WjcAouy|E z9&l+9gn7hW9bpU+z*nwZ?aGmC=&b+-IA8Nzw4V~G*P@FdB%UH9=AeD0K)-sq*LY8FD8t#w{RQf`X4ir|;^Y^jNO#r1TDvM-cDr};Hj z8W<1din-t!e<)tt`o8Llk?Z)#ka8I6at#3Q+G6WLH z+P4tDUawkakAmLELCjyjiY-e?Bm@EQ`#;_EHN$BbWQ&@YZAWF69vi7ly4W8yIN%Tqhu|&8x+JZ*^7s z=oLoZs=C${n~9c|Htud(QBQB(6VQxAf*n9_pjh=mJspLah2=WQ$GQoABlYoq zNf|+0?MNqe(o^KDe#hTOj+R=_B5`#lP%`Zro@NO$&_NJfIjBb>E(~KsjrZ&s@T(Dy zbDvVJU(|!ziXfIGcWCm=B9c2>V|cUC3CE{KJ~{?XeE7#`U|FJ~=2LCoRc(kCEa@8> z2K@en8W&3FllI{`r%%uOGLi?>vT)Il9-DZ;nJn20&{di>t+ek|*zPd!-*fsrcXSQd z0&hprmX*>&kIq&2hQ?QTgM;fbmVx@Pm%n zS)Ek!V5Ba3m z`V5(tO#gY7y+YtB%T-kcso#RZG*s@(YPz?gmh~E|(=i|(AvlHAeHRzr;q6-0lW%>I zzMZv#kadj;`mj*_J2^Uu`l|!TK6#Q9KRtdrBg7UFLtu`xgh&W%qt0_QN(KyMH59D3 zDY+kXLsQPfB3&4aJ)Tm9YU;+z`&H{eZS9n z)91``mnXe0k@I$NuX)kydVnAOBBzm(%tu#D>lPe=2}CS-ndkAIqfi!lR-GjG zYoiFe>t$yhx})%oe3g0XV)x6E9K63Tek?w;KRB~Xiw3jSv9EBCTKF_R8-nG(+KSXR zJ?a@;I~+sgyG!Jf~0}BA)(3$7^7q1yWD1FfS@eb4MSVexzIt4E0Pc0`0P7 z0{rQxZwk7+!n`)v*kGR!k8eDB1Xr!}@J&iTz{v2jfkja*IC`2p$c6#2;#~8BY7a*?G$iQ$#{@yEF_vq zXe1o+NZ6Th241|Q0b0Qa`c~GXI#$A+l-xOJy)<_=QTB4n&hb=M~ z|6me`_&4>{<1+b=1f8m_rS5UHzWNsBz+6<8#C8^h-`}^3_yquM9=fVj;)SUnyJ0Z@ zsP}pd{fk*)GrX6M&TmsUBg#VO<_JtjFr89$8J%5iK2*?;c)v}hv~k-|-n6A3r1h3^ z9tC@BHux*kKZfSNs#TDvcU@Ys&=hbi0s>FVxu+@QJ`Z%T?8Kh#OEI>?S(9<0skuos zLm_zEzmaXDfFNq_TR08t#Hw{G%%sq*3;r%8UOe1Y`<5W9@+?OExvC;xo3y zd+Q~6x;RcM1J&vuR+f~0ewzJ&7G6%PP1Pr)c78&el zXEs>W*U(BF?f-5boa24p^j3>wZDYSWu16>G^dNCL5i1q=yn_-Ga5-5!k*=LZ52^=e zf{zGDNx#WZYWj%Xs&@eb=M)8=)*v_ybF{__KV>$EA0qyn6qcIt7k>>gx) z*Q^#15!e2Ax{eg(aGP=h4-wnAvBPR+V}L?<|GVw80(l2TkDk4I!T1o;)&emXw`WvU z@_-Fe=MPyQBNX99B55A81$nOJ0hy;El8WAa|2FR2CVv0QY}5uEuxoZBM*i|_5$*L* zjbpCWG?G5GPk`I~zi1VU$QN{pb)TU`Uv+EwA-{|HEbQt6Sa_bkg>V9H_i1HEZ9|-wbF4^yd<{s=^GP;-;@`g30N(>E|q!5~@AXpA? zu9+{J0+Wm~Tns~t1D6!-|JDkTZ6y;CexZjBc+>aZ^0W7Hked(Q-9FKyquK~PBGxu; z$Ujf6uTiy9=W4lF5&bUAy7yU$KFkC+?;2BPFloK@3c78ShoDczJT(RV8L89NCI9@z zNkFL7a#~sjh#LT*YpO*90DZKTESx%WTLBu@+#GNWU0I3Kqb_YN2u_qpm% za?UuhDzJ22G{(Q08OS&LBZFnpyEi>(k^)^mj${0xCQK3nnh5thJ+5}ptD2F)ov&6V z{=vZi;6^e@iMhlMkEh$krj95chC>amtLFs%Ps4AWu9O|R!60AnYMRr=o9KlN z-a~{?)9g0oP)lr<>o;sX|3qGVP4v>5>!z4TPEVr4*`k5xPeWfd*Ng1b7@lR)yNstV z>vTXi1z@8G;Gxc!?U29jb3W-9JWj>{ttdZH*WHE$`QCRB0mLUXARq9>gn6|Z^h9vP zYln8X*wmgRPUw~R^f8jN-BHXmE8=TWfiO#6n+yqPLHln|#q05SJ;#3+gPntQG+&ai ztU`&a<+jTfYF72NN>ggrpIrl!Wi7WxZ361ixH?!Yyz>6443|&=8r(B!pap5$I$>D`XcR?yhvjaiEXiTT9vQ$(R{J@6p=r2SDsOuLKIVy_(2avB%rLAbuDEm=S zQ2|v5ckT6riSp$EaVDrt!BpW=7h0kg^>jh$@_(_xercktrFrsA`*#`X+th$4A8Eq34qW59O^qeEbQGM8UTe2sGyDg7c@V( zXXSkGe^GO+Uw?QWjOm#;0&Iox+_LYE^JAspzli4TQ&@O;byhp1k?fmhL+!1tEAPFd@=LChKG zA07@midTLCGIRi9neKrbD7iRDhA_uls;lw)O8N!!kS1AkfL_)@p?fsB%ql%Eg^HUh)``$Jr0(|HwR4vlIz6rJ()Y!>oVM)kVa ztWpf;S><#F4zOyt#JMmc-!R8jdxA^nnK+l{Gou|I3paQ86W|cO=?q<^knFuVF^|UM z0{C3tWxbyeenDUPY`rZvUD@w?7asuwX2PUSvO$wR-2gkFmfZqf@wZy7fkoU-tyq6g zTB1x)!4{WQgJ@qYvob~(pHA?|B5W+Z&yhHQRZJ-U0%*zz62 z@6e5j*{~nGom!^aHEi-s~Rbkh2`oLJOs-9R9G?tWwX zcy~O)lZ?Ms(|i4CV1#y4)^$reR>B#u1pmty%^X-MipK6|>IrA&5@ksg^ndIt3b4DB zvkC|*@_tz>KkPPdl8xh=f~^#_uNP8RVNcmlsb}3kLcfBPtPX6VjkHgMtLDFW?LMyU(eag$!vQ&4O>nvB9ESTu* zjT`s>}YtrvI zRf7cf9}v2pi2*ulBTWru%tiPbB` zv2F*m!3W?<$oMYKPThh)Ho|BJ;WMEx${G@H)YAvrNB<2=lGpTvh<3Cx`=$BP0p~SL)R=k&M>dcd?4mk1$>`%Ogm>%;pI9 zt?uZz&^oSc(t?XN`%E%r(A1#C=^;ztYb!@^GMY?{UIOqXww!@l9|+^dqdGKc>|g3E z7(S}!s=Clk8YgjK^eKF6DffoL@w3&G%~92 zpBn7FFd(kDxuxos%PjHR?e=+lLXiJ)+6F>Ekk-XPr=>W0;0^v<@rKFoEAT?Rn17OK z;;~%aLnWxs8fh1=yQ#%#=jPING>{P@F+rYJsI~#)2o1uiGDw4rniNX8>bBxe@^?3Z zWy3&remgE0C|@Ll5kcag7uJ-TIdLqDQBqDc5yU7E!ZRHQeiMWT}r!MO1ITCbZ`FHL21^OC==hQdrAREzgWrMiF9wzn~k>l3k z=5+~A=zp`-kQa!I)8jI95TgD+O1_zMF5t0k%g(MekD{xb0W`B%=oQ zcMmba<{O*LiJJmJ?m6O0H>Q0|zoq!*dNoaTy0+Sv;#pC8TB|qq%d`d|ob9(;^ptxj z!_`YxY2NTC_ccI+N8;~deSkAor&wI=wzP!^21^6uF`fGdcC%U>(-zk1A~ zpG;MgL^nMz-Fe+8A3or_!o21yjol((jCH0$www4#Bj0C~L#ajMUh-U)h)$iO@SD2C z4q~ldu-oC~mpZm>eERw8R{-2IE8MJ;jUBnO4t$o*oB&iGkR4uKef|7aV3Jf}>wBvo z2r}{AsPckhnI?gjBQAqJ^8A7FAjKcu#+W#dBHI-jH&JtQf9rIa^`J z6V7uZH=C%#uS_}yQ_~sXO^?$Fib1HCXQL7?*nuMrr(cmX?sfIv=-qBVyb7S9tbdly zd#kAA4J+c~8v;;pSMN5#2ys`0+8;qFCYAuRoDLm{-mCUQksu zINv!PJ%&rSWvFii=$^BX8_ynZ~vVoLF~F*-@V}q4W*m00P~zD(`QAaFHk) z_3>gYB+vE8yg`%a{aQE_Y7mP%3n3vPp&ei@se>{3wBf2oC$`^qdyJ&8b3o*n!PffRcRU3Q>=rNa`%EB>uC1l^HU+u1kdWT85wX=|r)92JS4rqSV(H!QCmhsYG8vSX*hbSTNkOZ^*XwUsEh z)MG0QPJqV!GFJ>874ww=5}lH2tGEqWEXNG^j;^OiE_v8KEfRL>nLB~bwKlJT`H`r& zS=E4H8&1Ifk*Ud55v;x?eMCfr_S^f>-5viI?E4BN|8Q8*&{(<)q)XE z-66u0%r@f!Y)Zq zT|eyi>-&?l6h$cgpkW-0+swBk`^pW28SjXE!Mm-JQbrWm7QHY*UVL^H0JjBL=KrK3 zM%6mf1Ddv?nR){d*QWvyd4|`|sl6{Tz5PN6-2r&2dQq`L*=-K7Gp+6F6NB*OI>gRR zu0vr+bF9CgY{NPvC|j_?cwZT4M|%a>EnwiDeY?vzV`~X zC^|0Ymi9{uvGPp|c$2V5qeAiggxE3lH0Xyv zGN~T8eS5nWvc79@I-iex(D>Ulz4D5TzP$m3f?$}XC*7NbUr8bYWt3udJc%guM{#WQ7xUfNWjkzhaUL-a zDwe3ebYgY}>!fJ+{v+eqs2q^m~Se&r%KZaZ*=2q@=P)p+kex7fU@oz0b>7n)Y4c zzGJ)zN(ue>hDPN4`v2x>BzLjQBauW8D!>UDnYjE^OMl?_p>6jYh8))?Ga7$tUw*+< zQWPW7pzDtq5=@Oe4VbAUD`gHm;7%flQDFB(Y1D7wt?B$4snmSQx@afm-icKr*iy_R zADHACZx12Yk<5QA58Y%bP)J!!@w4)inAva28ncIIq~nk_EwwP~uU-aygPUw(mj!G= z^Yq(Sj&?t_&RdlN^U(*6_3qtXk}Z#YWCTL@#FX}Davs~Sk4#QGf54I0x?z1|pdaYo zF*q(0;a=Ah(|NXm$M2n4SU5l;q*8XX4T^aE^|`hMs8qTg`#RUVzExWvuE_b~G2Gh& zREOPVUU^-4WIUYllx^{kG1t=NFIX^faJX!eD1#X3VpoZ2yYD68@vI)!I@ zlj~_4O=)d~t$A_9eDrh~etID0rs$2_7>Z9tL`^04^!T3@I<{`)h5>Qlz?+zuNcp%| zmi}?bOilY&6fu3~4EWI?;xHhWA?|URABdu0>RE=)&dyXM-GDWU%yS&RDI#AQ? z8M1z%%FoZ|JPO@QP1_rCyPpwmKNH`YfZY57NKRU&?!1;dkPPYVGJWs!l@n04omVz@ zjhH1R+D^=IS5%7y5Urid8$=Wwb-}A0MqTqqMOc2eUf5xkBa6UwE*TnK@j1V5BRcHk zx>UO^3C15n`6s^)0;^jmHhz|kmp3)Y;EuRMkPePo19CzE3*Pi?K9C+u7FuJT0emu@ z8^eo=B>t7P49`w?%Vt+JcMsrw57>>XT|W+%YTvxD!L#Zm>QQ#UU!B_gPXsnu(O$BD z+;0B5KYPsi)_Qv7MjTy79dTYga7cyP9g|w zgBy6u+ABnivSTzfGnyn3Uam$O`yEU6H!|!cwzX1*E}-1{Sb^ZXXDX|#WIW7;bHvMw zTV>Y{g!S9&R!w!YMBR#Z_8l%Ibk6hhQ1iKJO@biZ;xtdDEIB zH~!`w;wQ&x6!+7n10uix$M(f3*~>APy(j+X=W2Ew*hJEbV{F||`74Eo^Y)hsM{!E7 z+nX&>DZa)37-N4C08SOX$v2FOO=|SdmGDV6oxL>$RTYmyH;R?p_>sS;K1}P(^{}FY zBgjA3unsv$^UO@Tt%w5$&qx!g!=3S7e)39#t1fINfwa&Xi>jE#q#Df)cg`ZO%|K zCRtqT{C%bs`QakEGmll8a5}dF5_z^ia}x`hrV{zOcdqk4m>Oa)o}H2;z9qy9Yvu!D;8id zs-B-~=-2^c4jM@(Z+u~4e#yj`EXU|8ZOwhg-|iUV3V9;9S~utKdaWVy{vodLk66Aq z`$ah{A8N_MZbr7MN7oBq{Oet5zm77NjCv~XFRKWJM)eFP5YD}!;)?eDY$n(*&WSX> z5o9ihVSUCv5C(KDKJPawK2+QNid^BSXAxPa_(5L8 z%1tuiPQOcM-}#m|xRTB9k3R6Yq$=jR)k%YDx71(wjBw5fYuvs&HBSFyZeo{PCp#6_ zR!ynirJ6e@+4_vm=7ju~w`psg$*O<|&NGG2AeLBp_A2=^B7qw7X&p40hVigj1Iu@z~S&(2=ylq~|mh3Lf*U(@1V=@@r*LgW~jNbvQ+eDUFT@qOG- zX$=a9RNiN4zE%6vxq6bsLhZ7<&bo;-q*AsU@2Y`W5?M7Jn^ZFJp{ocN>XNem7~)dA z#~fvu)s0!N2$f@kup|XYv4$}U(@bC4q8gs*3fIPLCov!D22m){X~IO`wmg`>gB336 zl1%_Vq&5pxXs)y-m4!=5x_i*3LbW2oCQ;I{nbr`r!z{ukwDcXP7oGf&1mwtk1w-nm z_*jQFM<~LkN-kE0`ZIh2#reK8X8X2;<6-{%1Z(9jR7D=d5n8P-{AB!=UZ5d%CiF8% zon(A&b-oee3Vxw9R{_&~B9w~xPx6wkig7)H28&Qbdm(t)lHeuS4f0~L%Y9fW%24pk zCwDcuE@Ti;G*(H#6+m%I^N_ur$+(IXiR`I*X(@wV&K9|tyGD1jN$x*JDux^6m5vyr zH^=;z)U%1sqzP#ktov(>Npm00)y9k!U`)ARLjP}I8sKzoWUUgkQg~;&R3}D#YR>WO zssvtlY|x*F99!2=GE};pU~Nef_>*zsMFa1TrfW4+rm0d?LO7_!(kfN3WEvSv1CF;|@C7}wOo@NWW0AtZqlvs$?5<^+L zG4=$7Ekp|@mLC-v-EP1*BF&XI6Z(A?Z^T?@EGuY$g%G_racqWIfcRdY45feghUII{ zxjX`t*6wWlu<9-ETOy5jB1S{ocT`BjDr-Z0*%L`|W}YfmJAm*_H2sIzuYs6k)|;Lg zWgvmkaqS~8BByUwG=&1@Aisi`!A6m&6&))&34qfpkRIf|yP&o{x0aw8d|#BAKkeV2 zqT8*DIJ8r00%AV-oftNx*Yg9Dx2LXcteR)h_zKWSDfWiSPL0^b`>;+1n;(|8DZwYM_fnv-Uq>M)FU$=WcKP93CFo~0SUQ{0Od7^e? z3rX}~BPJ5%renk7C>xU3LnC1JCz2%|l7=(LyK^YaxJS$a{S^%D;PGW|#=0mK6#S5( zeisOnRbufDPw-FO?M$(7Vz40zjX7?D3gk63?rBQj-pKpTG4GzaPh0UQ1!wtyET7C5 zoC)ZTaD8tkh;5EG;@v{1?OCPZAl6z zDup6#B)b}+Pg8bq&VicFy8YJ21XmfWeJjQr&NW=n%Z4b{cmn%}<%K7V6 zhh8FOyrH=i$j<6eK6hR=;(&OU>_Z>D_2Bz`k^%?WaaHOC`{8(Kh!i0j-j7}tN#Ma8 z{QwP+cXL0A*v6#S^SdFI?{?HiIYSg=7t*aR=qD}Gy7;@t5P ztm!l6F(qaVeoJSCB{k5wI_5rrzV?2K@%oGs4xH<&LHtU@g}rAFM1e$j2&U~Z(7DJ^ z&*r~=jUP8iO9f3Gfj575td5gvm;#VAn7q zmdWL@=2-`c7QC5lSU4+a$6s|wzA|J~Sp47@!5LI9Jzf5W31}FKJ%&mS&k?UbiZ`6hTdpEmJZ+}Wldl@4mDUzkA zkIVm5mQI>+%{j;BVN=NkghcW&;$Gfc9mKmqvYY5 zvZ}I4r7l;KbIBw^RFo{)?oH$)12s9Y573ygf|63gj583u!l@iW5_%G0ncDYs%{AuN zJ1Y5B2`5_KA98}ytXhRQK)Md~?41@qS{OnC%m^(VFXLO@F;hP-BJdGV!z|UUG2@Dm zM$ZQ1PYp>6)4xG;s7LBNQ#sh22^G{Ji_bCXe5+R{*IKgPQb|jsjr|&x@iL2+OWqK*YEJO1`g`ZK56(9Pm;6Z+`4ru;)1-V%uLC@3;`tH`yW7o6iU}dm*-gK_$?i?z*vW z;%2|fUGC;@-A55r_ZnFpQ~V5!8=;}%EFf|-a9Z;Xmc_(uoC^}MlB2CHu#)3$G62>; zni1t`bVo1FzeS`rWhlFk)&z@C^>;!sG@OvqvQ%mC{H)E*G!Ng@(}9Y;=nR*>0Sif= zpaXW91=i6!t>MFHkY$R&iG;CHy^kdI4E;rGO9(Wv=EvFIJ&_-2e|?8 zn8xx2BXyoni}hep>0m@%w_ga zoW9nu!St+#>=o1fb^BK-bL;HNm(0S8js3wHlKb#T|Cqh8NZ5)iNe+b?TuP~bkDYD72`ZnK8Mvhd{~DZUm?n%7yv8;j z_sTo;?dn{%w~@(^()HkF08Nx0L{~IejZ_DeYX3k+@7=uc=AZY5ucBG7;J_=JNcay0 zpI16B4~XYS5dNn7nU|MFva0fm!4k_sM_lZ+55@?(+F_D^F1ugMXuG$DHudv$f25&r z8+sRDkZ200;b1kq0#mZTsHe);g>+{74<3Gc_fpj>qD&l+xH~g+y?i#=a!Hbb$cC)$ zy;XbiIm!|j3Yp=2Yl<*Z!It$Uw3UaF{Yi-@L6bPtf5A3W*t^J@X%-o}N5rTYZWm9s z%Y-K@ED1ShYS20{FPM)nc;*Zl!^@Y@ZUkp;{>$e^W6@JSX6tM~l0re~H*`f|%hJ#U z@~0!I*Rz8ijN@sGLzF_5h&bSpsr`y@18HePgRI21R{eZ34~m4v1AN>_$5-JSnls6Q z*9445Y%ny8;za+w3<^T~VW@@fi^e!u-$o24a&MQRfvYO!X%NI8Kt69>c_!V)GuuKA z-$5yee9ZBK4aZK3DlvUKl2Db|y%cENv_^D2AmdLv)=f6km7JM$bEg zTWdRlk`W#sUX%o5Fqxa63K!l@1-Dt}g6L~Vkf^#PaUTX4np)EQC4p4hm!Djos=%a1 z--386)nt}vC!>nJ#?phH4>mE^^#A{iMMA!zw22OB#}?c}YK4~mRkgp4NGI?c%0%K> z@hNfg$~6pL?2$%$UX&i|VOaZmQ}#Mz%cLmLD@64*2|V50;P5zt7{1}ed`}}f^ko?O zaREu|X8HtOFcg6ijS@Cw2S`Gs?cDAxI?jfjVsR%Q2&tjun)mN9#mi-?i_H40cJMeZ z=^L#N=1V~k4Dov>@3DdCb$2^k zl>9L#+etN>6&t$}qMX}iMUnM8U_Y5AM>Lm)gFUshfL7tq$B@(C$$y$%h$M9i*kItF zNI&iL{21{u+6>LclFAEAPTwGhGiuIs(S4`p|Lj`}EPxem
wi2#xY=Mo~Cw2ZBk zSom6`rA*562ER&C08?Ljm@j9bj=L3#;a&=3WUIT~J^HE=1{yF{@b01=jIgvUsfgTp zvZ$yV{JW+9-Pih1ND4vmDbD_b3fq`}Tp?I{K&HV)Jn>~4We}Qi3d-?Qa|GhhFh0a& zj6WlVEQIW0OJ*|ca?L7fO%`e~HC_)@X@pUl#E0y_zOjt^k&22KlHM4SzJFl5XkoH< zdHdps=bu&dwO9%GO7KLl!P&JwQ&C#9ufJ4GF#}pQHCN<@tKE&C4bVMj5Z-S8@w7O@ znGuaYlxK9CfunFtBZD!6`;a6Y*9O7lg?v(woKC5tlDRldP0E zj3ns>Se21k;l?aQtX){MIZ~51u(6eVvz6L$Zo6^i$;qzt@u5fr>6_+sZT4efgAu0Y zxukoU_;(2Yxini73G{0n0_)u;g5h-qBij?jg(F2+VilnZFi&+mS$*3oLZbRP0YAT( z#u|tZrz1S)q@0(%yGL1KW^bE85sfd#p+iL8x+XjDcDAC+9I)cG9{7tviSNk?XJd?W z6>op?4P$mo1_{-=JQ%gT?UX)pjo{Fs&PKHVSG#H|ruPe7uAf8N_eNTHwuj~pN%pRL zG=7gy`t&>TYqhOUj5)|-Rxw_g_i?ZC)oVh$XbG7MnoN%R@y!V^3B|yL1)P-5>$xSG zu>H{@dt)?UUE8Y;$E%ruimR@vPOrCWI@qG?||o7y#DWU}+LRwa5(^ zx-(fXv3Xej+vkTxHzTVy#BWxkM_Z!*cks9R%M0A@21^mi_>Hnm|Dc#jCud%g=(xlY zgqGKhi$7~cY=)wX$xb=f3ljW#Cx3O@6}huHtn=YV5l-kQMc!4UGz-*t>iw#I{pHemn;&w#XD^YCQLOrv zc+?n}cK5?&S|QNP@jzfkN>{fwxwaWm?S>*-hbNlps~RXL+|ZF=;92YTdAq~CyliHk zUD!2{NYvNM>{zIcY(axsv?Eah4PqCS5Kh|xr2LE<;ImHWxsObj64A8SG2#yn`=_F! zu%G~pnjHV``)sj|UiSOdS6;Oef}w}qu8HMbi`(^^y_N+H^a(d{0?vK*3-{u;ZY z6yA{e$BTs^b*v)k31`l;x&(C)DT~K+DiX+$oxD_`BW>0gtKbw~!Q4}lB0$7HEh1)0 zQ((%&Jv;JG)J~)|N{BZ!QvRM9@jFo{#k~ywxAA4B6tT$JQ_f?J4A(;k-mJ4P$)1-C z^@3tm0kg*i%F#*EnQ&G5JhNVv_9sE|$fwgK0R_t@0#V^sJ-%nAF3XF9HV`n0b4fW4 zOja*Z3GNUPqImz>K=wW*jA>UQWk!dbQ={Nz%pa$|x|w z%i7VYAp~C*?F=KA?n1w|fVJArS=!pTHU*anZ84i=#ryt%((WE87X^})+SN8-?`Naz z{s*fR1DxMntxizbt=IqYYCoPUMl_wuwjxiCPACwMA-F%ENtm5gJhdp(I`dCb#0nPl z_Woqk*~S~{Il42zf=D*bHD6U}YgGJg4pO!C7KTYQS3P_Kxu#alk1FK5C!t*g>(@OE z7uf9X-l0fEUPA{v=7>B#BD&Nd!zI$jlW0*BkQtfGVg+eE7BHVR@9I}}tfvN7I1P_K zRu6RuHp9G(5GY~gPT$vi1xu8WV#%M*6|1;w=Qa~K>`ABuBRj_kAE8f_+G3dkbRxm? z4H1Z0+_+QM37>ZEVYH=TjG6vjm*09WYv!{svH!x^5KHXv4U<|WJy>*_@Exk72w*d| zAwh>Tb_^QID#GIw>urFfzP)dTq=KZVRt0(&B1oO66@Ukyo=H2#-;s>8OVR-y7|*Jl znb&&oNEwZ9w6<2}X1a>{KGsB#TZc`FTp7|W%j3Ca#7^kumrj<7qfU@DTTyj)fq!Bjb9ivH;z%bHl5saO@vu!Mj3t&? zRD={48jbPGz#&)jbPu7oBA-=|u?i_hQLXSwo#G`-8HRAXMJ7-UlSI_FMce*91W?0X zJ$68X5#{d>D5Msem}u6%Tuk6CL?eg^>Fu4|cNrI-tG`SCnH2zlwuwMzt#V3AfbMxR zbM2=~YEsDN;hwuo931SX>|vaUGxDrXH0go}F9xB1u%y`XK1S$as=lRX^lE%JL!>e* zw`{d`#I+9p9cI~)|7sMN* zU5fb&o#b~nuVwb%y(o4w&boh+%`CMAW$a|9yuhfSXY>8^Kq;XY7t8}6j)YdB<4^8) zlCe92jSbW<9SS6%SXp{@CZshQ9JqbHcY-z;h%)2k?@g~^#kTpEj#8OXZyKn7gELn0 z{>fn)%}GT*?TDHJ-r59tNAN+{cZ&{vl4pN3_Wl{QL6w6j+;W{tlq8@QFVZ{cEC7{s z>Oq#!ze^A59TkrBWjwOXm>_la&!`3s) z8fD9wiSt^1Hb1`@yOj%yj($)<-Jfi1SNbP(i|@{oRrctO|M`bCE?4+cGJU>q2tvil z)WN~Q((|0Qo&+JIHX>)&#mj^5nRL@qOt~88;YIqM`|QO78Lh3(O0QG?+WqAYHxha! zX@|Yn2#hDqhxAw20nnWB8-A&DC`Q7Un_m=}H128Rl( z%)*ro1~apGy)Kir7_xpY_z)c9X&a` zFEhdQ5v9rON`wdkG#kC*7|3uCc5NGY-`(M2hUTQ=t5oLM{_n*}5-~pLFbB7u&AIyNvfT0t$CZprt#w?ad@MAb%SgDe>LNmx`h}(R2ZY%Dkm;WIX?>~4NT#$No}r-atDKoRLKRLthoE&zS8c-0=-sL zUCknm$pA>$L5$TR3CZiI42&MoUcY$y>c5;-DJVOlq>MTB_&aaVS^vM-Tf@kX53S^W zt<=$>2?Hg56jb!p>AaF#%orsumO9w8qOlXC8ojj5Z*=ars{H`nzSM6r$s_H&vLfpE zM(&hJPXYkP&TO7f-r&&TFN4oZuzSA4n>YxGD$y@##vXmk_{ai_-E!2mtc%=d1%(Fl zZweN!Xk#@_f-admavEM)mfEzBNs;R~<)vDPNIAZmz-TjXsJWvJSL>A3t2+KD#fsG2 zl_^nV4lK@8w^yioj_Cq%S>p;@mOY>Rgu&iPtwW>9a*;T$J-)s1YGzh zj3V`K@bqA5Ggg`~!PE8uFdE}{2=bY#-=M5UK)?{18R}N6&B$wwH!q&*$w7%IvU8!L z`Q_@L?lfv$g#^n^o0;4Hq3nM{(al^iGbV6oa00t;M@s&-uttcedjfd$2+@fB4dGxB z$~g5KEB)gvjAbZP5s$>(Fo8QH+jjX4+5&TEMTL7$shGM-(CeRg{w+*&DJz;|7L>bC zWqx~aO|@71AVrVNX5q9V%woTvSUc~wz{_<%*dgUqa%K3vf}@Wr&G(M22PL^d$noiN zdeXc+IMtC2+Av5KxlNCHqZ++q~)x#s%^2RhfQWn4dxe`;&n+}b@* zo*Yg2Y%G*Qwyrzzovx|@+@Je%NN86>TI^vajc7P@|au%z*ww7%HI7^nN_l+ZUpR#Iv(xKz+Sw8-4o z#JHckm-b;uGFspnF~G-J+rUx@6T!l}a)3ZXdk0wRz?Fkdi-w?HkcLl#Ig<(Q__jT- zHW3z=ZA8+uvI@{tx3~O}0Le9nB9`nA82>zdN;Q(6#lPI(B0!*X3(x7eMTndIXm2!v z@B6yceb)qqE{v;Bc=kXz%4I;bIY<_QGGg}2Ql`#3gV}~5{CQ9+J3m0jqdP#ZEMgWL z`P<&+=Zq=lv`WA2Is_yp|I{tLimRt?LBcC(iH2*Q%K1iPRxNf@>=i4Kd(=)GB%xUO znjnuUW6U^qKn7>@4ypT3wktgAe%8pxi?D=IdYd6wPy2$?Mx}?Y563POy7aAf@h?*=ca(k5=Iu?O(fWU zO7JfdG(WGNTrQss=t+ecV40;m_ZSGd?HN3u9wf= z78CxMQkrM2I4}*X#I2Yrd|1^Tcr*XbsTYH}Ca9@fsA&SMX9NiolvSZ2pQ|2Wx~O5wdsw6kkRiddULKM#pTC^mJpp0-9fud)sMpekEU-9 ztmBKeZj45af3a=eIBjg(w$0l#wynl$lE${(G`4L!`KIrC@6TZFojJ45+H2$NwI>ay z>Z<3^upw)G?xJN$f{xfRD}E)~aT#3+nHm zl2p@DVtbkNXrefcZJt|l{G_)RM^ZMF-Oo-Gzjy7y{EcsR=}g$Gv$qO9Q(@ zL>9(iV%DL4o;U2N_WGQ#v1pyq`3DX3Rfs?gopr;#VuaU$+jr5-f+*cfO8d0iPrZpU z-IR_=!3tS>=?5v!yhuzQFGRcW%Tp!Qm_Jm@d}96_Rv%F$S#x?JJ0|5zOZ^p?sfPYf z>^u8^b>>sW3LBlct&`d+e~N503X>RgTB8!`EF`HCo7e1%WitmizXtws zOYHFRsIx!Qwr}2oHJ?h4T-n+0h{j*4HIvD-C*hC-WD1?e{uo8!3lUW>jP(SPdo;^qPqIz z^@0I)d|1=E<-2a7!+lFj%XkiZ=xYoi0yp=~aYgmEds{lY1(SbQ?((vB3WI;LC5JRW zcEQizap`sD%9@KXNycXxa|Arj(0VhDEo-l98}E$x>t3mHnOuM~USUuXh7U-TWivZg zwI|?!AFK5|nP>ue8Vc3zkLLk1>ER9M%GCgNQhztde>KVXAD1m|h?Vp0rP2H2kWl^! z?+p%v;U+Q^4XptE{5~r|m0HAS|l+alO@JyI}7YOQeBcvMAYP`_qG>rlur_z=96XybW|=&K%Ph_uykGyGgOUsD}(h%9!@*aP}6SN z+zTBeov8O@YUS;Hd(K}FgVVrMkg!O&-Qc$aR9rz{&MnT>Y!Qco2Kt7{ z!K1Y{?wO0$NMU)DBFZAbnuGk`o>{$)RdrJ$F{l-OTMr=GuGpWseTfu5nrR}=1zP*P zo;A7(^Yeea5P5z-Uu(sP9MA+N-Dfcgt`&1tsFqLf{(5@&1M~wIDz`V<{)c>bH{NzY z#OppzO%z%8IHyroR@T#K2Ymd$>vcjRXL&Dk6 zz@XKh)+?Xg?6WV+Z;wh@MA_cxN(-2iC>apiOLRW=V|aGx+w%3)7Rm6qzk*uPcJ~f$ zQVpKyQ@R?XfGGv~e-7*9O!rsy#TVSvD|JVf*exc{%m_w+|EHzpiQEAlwoYd8xctXw z&wzY~Od#l-W+s!GfPv)v0fqXvKms*}K(HU*p?$izknPL_utk&8W~WDA{riiJsa8%K zT!DRe@ff^tD{b42&gf`-u1{rs178DSZypbbsAp$IDXl!9e3W(LV8fw8;TkE>B%p$Z%CyfAW65sgLj1`6B$CmPCPVdpkGY zKogF4BqN_ZIv+MA9cLC;C4n&S@aZFyuo!IYjNth!PRYG~Ux>E@I?=rlhZ?B1x zW3__U=J5#9EqIa()Di_s^HutCejA@$hIXvjx5->h?=|;z% zcJLh>hZbu-iIRVTMJ1Bevty)=$=#gO&@Xm5n2f^X%#g$92J){D49%&puP8}bjCswE zu9*_lf2R|hHngXXW=cE(d6K2tJ--Z*iPx)WZWg(~6?^;3B0gXY*h)pXK?|v_uZRU)17E@FNC@#mh zR)ULdscUWStS<}W^0}I5JT44k^q!yXy8P` z=ZXUSnQ;y7&Hhx0i9f{Mom&>KTLd$BkN)*y3OpGObX9Nmz%9M5`&3SUZxp5@OL_L% z8H@(R7Gi@ViHshq);Ku0uIDoy(Cty!jkZNm2zmSExHqDTtyEQ2S*&j@Lr{rhKDnPS z2AH)hcG%!JbamvsTy*9B{fkaVPq*?bv*TZDOKEwP`CJ6nuE*(c+H`@}$J+ydZAO4^ zTID@eZxtr5x$HzwM*u`R!m}?J_&~PFd7{cICKx`Q@MCQYqv_Rh8BqCPrK%Iy*z&o-QTU*0S-s9(MoYak*?2IDIiBUh{Uf=nwnyy2jo0wK_-O zKTdBBhiU@_5;RMdZId)Aphcs-#u_W5nAL8XgR7${yf{Q$oxwo<<*3wWcG+`u<0h<#hB;Eo=N&I0oAfQ$j-j(kT$j%4h8K!LX!_^pSIZ3XSfVu^pM3_Qw-FTreor(~ zX!|DmhWARnv8)ysc+Oyu-!lhOYK5GcTiY6DHt)bF7Jc(slYt4Hl z_rH&nlam8dx7GR`e>kG$vw5#>azCP))_oH#IWi)o@n9b|K0W~%LiT}yq@<+!BLP|3 zVE=Xf;+mRxpYxVbxr>HVGxc9T0UL)!B9NRONnG_lsT3vq0uZG*PMi7b*SaZXb@h;> zgr^4vlAN%c*;s#3Cnu*i)4HMuwl?>3)4sNg>+V2QMd!w#hzRFK`=!yVouOu*L0MpA zpmeUF-(~X}CboM%@O{J|k3**$4-o%cX*M^p-on!X_!+KPAvZ>dIuunK1?Qb$4Utqk z{8{2GQI82{GMUKHhGd)Mh6BB+zvh0^UhG)G;oG z3+^acRXx+l(0IlBihA{_cQ*5N2bZ*?qtY~jYNEhW5C}M@z4>_5nC*D0cRuRmcscRB zjAZ~VHm+_hPi>IV6ix!6JtCor_^*Xn)};z|L45fU+L7I0qa-%$j;nG56HwCUNodFShXC*vAW zG%~WMfIgC3NmXOxnZsbKybFz}iV1m$(dNSwZ8Z(`>Z+`n8D$3AOem@UpAy3;$I4Y(%a0N410zik%^apS86=MkB80RZH)i$+LJ=WGT>#06+wall{i5 zqhn)X;$(pHnf{{~D@%2>QcawUg~g$!c5~Sqyy|*0Ve9wdcWixv>%Rhjf3Y%UlKVnX z5*e}B;$o4RP=A{zXJ!V|m@R8poQMM4HSh?Ke;RBwIOGZ%i!-JRhDbP~p=Q(=WSZ5fonK0$-olSPu8oGc1xw9(VY zDJNiz8%?bXJiZA&-=B>EQ4cPkJ;ByK3;w!asG)rv&;rPSU6S9P)&A;B77w4z=TTf+ zTPx=$3m*-*9H4FOXuX3E*$o-kbYxpBfSMV!OCSd~HwN&Bl_qAOUhB$0W4QRohye_o z$(Buz_`@+ersAKmL7MByA?{MzB6lV{!vTEomLog-R~9sY(k3{9WX;4VmFb*+dew8v$soSr|5jR2U7}({DiVTZ^i;c$=D(l^+&bWCdeYek z_-`+JQzoucxP(kOkODJ~t%3KeUyX_!{oKlncYMMsmzb4BQfvj#h!HvNHBk~=(}gNM z*)AVgoegj<5JB!(^QaPVT4R3Po`J=~T;Au3m4rxv=IAT|m8cz%>+nQDZGM6!g|wI# z&q$#NOWIgE`-@|X?!V`&qqGz zKp`|9iXjC2mGO$ZbS^^Z;NT#afrXo1NmUgJEhSUx=Ts|TYsmrimJw6*84gFERaI5> zKb1Bbh`?Y__ibq44C*#8Mi~+d3#OQRj&DdK>juGVP>HV4o_=V=BLzdT$x zsGw)f+j(hOo#<0x+b@l&Y9s=?*yAE(2gEVEz;3-ODXlYV7=qlE{QBw(0K~tjyuIs5 z;Jo8Ou=8dOOi2`5XR^ZOD`>DU4xafV97!j-+b%!F`1F%aLIj2rWJE4j{zb8kw2}%r z38GiMwD4O+avIz=?w*urOQ6!hPljnGQC-^LbG9;}j&S1lH|;+O=LbYfBR`F;k)w#=_URU7>1y_=F4R zX%RvC6H`rwx@gdT2?>^2B3Q3a5O7!m(YYo3#HX@v!ig<8W3v-Psa%!=&!>D0cs2BB zM52{QH)Hy7Lg|IM^k*flg(m0A7K@YU9Yv%Aebt}7!EZNCR5qbltUg}x-((u>+<7B} zW~Q^cP{F{!wAUOX8&Bf`5jvI}{Hi^-PGh^mRl0Eq(to{x+*P<(0WM>tE14eZ@*r$> z=LifW5C7{hS)<*Cycz&<#nV8Z$*X#{sW513(M^l)wuTplC&j9^E*++~~IRZAP*m9f`n`++!-MhuHrE zvb72$Tr`aZm_7@Ov>SYo+b2(@)o3bZLp1=<6JvDR^m zoP&$Y>Gm+5OJOb<9cVd@W$;cSmxxhTRSh!Jdh`_}@_sbpviD52WEk9^$W)Tvx_RO8 zd&ZobQN72oH1K=ckw$1!PF3w_pi>vQ_BnG6fz;A9rXufX%6Cb)i1AI=a`5-MB@tpC>w0RGI= zLjc@`w2=|1eG@+E=2_E*`Btw4d}!b4QpZa|jh7vS9=E6G);DQs869?0hMC>zAtIBL zlm`S(&Kipcd%VmH&%>?Iqd$324Gj$d1&)k}KpY(#y#j{C&CO32SSY-n9U0W%0x^Th zk+AFY7Scvc>@VLT?r6@IPJjCp1^cAGwTYwU)l|b-vP!Xp92aoIvh`DkM-oA zJQwh}^zJCTyEs&m&jKwKo3UDH&D_;L2wjrI16`^sO*^R2iptlE7Q_}GoqnnsYDxwg zp*tWT^OyrZ*ZMRVOFT7eHA72FYcw3E^)(!~o?WAYbv70^R5+ff7hkMC^@Sz4*D(F_^x;`T%%3xO0bw5W3`RN{-6^ zNCI?oC#9q`9+7%^JH#a>_VtQIE;icH&&|y-e)%98&*aYKv{Byyp*VCtQ7KCUdl|rO z8pHMWC(&LbIy$=Fx=hFT_=kYkCW4yF#1s?x5^ym#WOB{WIL|&#oFXrRNZfsd$ zpv&}dVNwSoLo903!(Us{B24LG$t!AByWnn&!|%*#D9JJe68^7BHOe&5DGRYH)!7;w zznH9$d^w#ux&F3FK1B0$ske3%;r>idJv4yINTvuP3v1qHodvo(StS=Jka1bQ1ElA2 zCYN6fz%HWd)M*p@5{aoC{<|IGf1Crbr4%b;>cpTvqln1J5k99Md>2_s)^vH@K&B8O z^k})^vyHaA_KLDPbvh85u~?kzuCLSw+?{NYadW4lJi&}-2!swq8kkMx2w-AjZaS5~ z`mac3Vo=AZ=;(}HZjm2O*Ws)*+L_#EwE!OV`k0c6LU~5dYiwFF5^pe}gAY;{yu_5;j(+_a0gs2u5&K;v| zhu`1boIqf6=z*NC(&__Hab#-Bfl|+%Z3MtO;Pe3y==A_5Sm=_<1RM;wH28SS<-hC> zl+EPICJ#kzdNR(E?EjC)8dIRv?I2Xhx3(eWy{gT^kWb40sxH?g4YsAl3px-MAWIrL zL)nQ`Gt~|Tt3;?5_MSKu!IyNjz_*o@x3anIhZ8;tOcO+11xf~GliXvF|>&#f4acn zpemPrTUHC1Nj9r%)91U3*n|imHR*D!`XyzKtB|d7K|VM;&HrKTbu0LQ$MKRRK=?ZZ z6darN_}S|<&WDVZ6)oU}af`#N#j4+49)^hF|L5-3J;{Ny5|~)o=2JOqt9}ni*Ef9m z%Zpuq-+6(gr?j-RX00bHpw=KeGxq9pVRGY>soCW(@I=UKsEtji&4wO?23TX}y5F0ng)=|)%A#7qQ6IH(DUa5S15WgG1ONU`OJ+q5!JIHCd+ z>R(T(g}wsq<1L=;`lq5lYJqO49XZZ>}#791YRs`fX*)MQvew91E^d8 z=8rXIU;%Ie=)kYu_s0Xfv?`Bl4!K*8S3CbUgnOG4L8u~)DDcSfl0kffF(o3Qz8s`! z02P2kB`_SgUQluWgEH|#Fl2OGeEc&tVP3w{H%}GDOj7vM=iFJY(zEeqaNZ+xeEp|2nY* zxMFbU%JBuD0hK-;Io`*;9kRWRWxsxH4##_6JKv6KtD@#kx^6!!w^1c z>AFe7h$T`Poq9=@u%W;4yg{*>laDpUSKlouFuE&hobb~YOs)f>ytS@5i0nG+kg@s zf!%hI<@v5F0N6;qm(R6D<7qsJVzGkKZf;n>#ZYqazz$fJ)hC6n(Trj>u|Kfcc!X6; z##-gELiY!M2T0Xb`V_yYh=_nniJZuS8^CLU!p=`16*ENiK>!r$j2A1LY60fI?tjAg zuR`RL$+dO!#N+)4-{x`gFOi8*BkSPVAxQl3+yNdsc$jW+Aq7aGDBa`oLRXAy0uliz zj;ecYnC|S$=&7J=k-NgzUpYjvsj4kBqdGjC)P-S?p(f9rzU$Dl%Nx z#DcR-VgzD(dLUwW_Hhw|AxpCTsoKt4=|4#jd10=tt(`WmQ~~A7oRDYlNze_${gMSS zm3&4&)We2?fdjG6)Ou#e{bFP~dy~ofDvztHYm`<=(_h*Zr;_aE=M0;}iwlc;!Mgtz zR>$Xc9Qt5z^oP5PPwe#0ubg+Ghz3-Rv*vlK&@n~ z0KGr%NS4g3I=!6nf2_Lv(rx)sJSgt}@$xZrAou};viR}I`H5hN$b%pd3hMUmE?SDP zV%RNJtM2i7PGe?cVF=I#mM}3PpDCHzG&#tf`gg-T6y+|R7cQ??LpT2>tPGQ*0(Bo1 z&0;wTAN;1Zdv5*o55!oZcTmjUZ)-LBoBc`Bbb?H`O{^B}lmU(EH(r2VmbA7;KmoU$ zptBu8FBEB(E}PC4+#z%cp1tb<;-|kmNc54ZIhv9f0 ze+HISRc*(I66dQjkPQwa^ExjvJC2r>I3FKb{8?BTo0cGsKJg|1_lBQ#eY`j4j44SF z4lFl7*|PiL{}V5_1Ha%;jc1Ovd)=gQSfT$L!Thfrae+E+UfpqTK1ArT{dACnuadeQ zK_t84jbHWlyf9*3d)m5E1vumA;$kP==Q0IW5U9KJ<$SF9YGU-P_4O!yb(EDbe^}Hg^ZQ2XqJ7(0T*&61 zJ{0oQF{dT7+~TUnl)+*u6y8e`ZH>z9F!C4@B#~nIY~cBF2H)XMW6iW)UkpBWS8pbF z7Dpd07EJ(EZ&x?BXf1qoEn~;sX$hCM6_MmIAV?8u6v5p<0vFfWdm^`8CCmu1 z!D##;BA@e-?(R<(xh_m_1fC=-c3lDrf>+QA{8u0VjLZPr1vcZ=#=$>Z6EoR&Q#%^t z;N-rEcF>o=g&xo12_*DbjrSz$!P4+g(d8N=YP*=^VmJRYv(_g?ohvIL0T)W>4w4{r z;Q$805Lc|IvpSsXUI#-C5%>_iT!2Y(4zBx6Vo}<}W5l%RAqEBC(#Mu8HvDL?A_?4P zm6Qzj^CN*oHR6cr4-r&>sK!Pv6wu};ATf03c9-Qen0^ZGhEUbeaQX0ih@U$6cjcnX zlt6m>9A=L9HNx}D%ZG));xC);5J)V}h~SKHp*3-J?~i-}gjRL*OdSdi%JUqQ+Wt)T z?{i_uB1moAJM`n~s4pXs_gQ4~?U0idOoN8FPgxmexi(szgV^U%RZIL)vlc~|@LxgF zjX57h2J`{OxNJHZ1UnR5&;3EtSNPmhcv#oP)-D6{Yu2M2MUdd;pUJ9_Wv1v0ZW{>@ z?E}@7o*1f-BeUEbNNrpPH(u9I{56_LLrx)3GPe(RN_u)gYf8AdB4KDZ>fz}z?&J%n z{i~<#O0nNW#+Ku|i5yToQs=q0EyV=(=wf4_S@J_L>6(B5XirOehNG;kQtXr8n_pq* z-@bwqAbX1o3k3y*gn|-d<&~^lSYBQ#UBrR}wbJ+>9$v%`lO^}1WF;05hbU<-b6fU- zrM-O&FL5xWz}{VU@-4hPLkOY>VsLL-a@`NB3=R$sl0p)qNEHU`5%ao!;~LB8CM#uU zJ#QbMva0fk`M~*EH2PS1iIcN@4xE3Y_8J;9Ycq1BsS+`H)SpCa?JO<38|`dY#Nm4Y zb1OD8N-a2`D>^(n8sH#b%ENR+Mus;uH2j{INBZd>Pb#aV%pIFvnW@tn&5)!3!mzJ+ zurYRtBc}P5^)Z1vAW4w~DT`rKwnAlEfVPCVO*h&VCYjd#7ZfMUAYmRmPZwjSMSA?m z#l=Nxb}Ik~)w}XW@AUt+`#jco37l0qw^uI1958lz)S)Y)Ac{?#Fb5LbeYFgy&Wbsm z>-x?ut7&trMYF#jM2q~@=BZG|)bI+~t!fK~IT5ibXfUr&jx}0-J^A{}j)Kr>DWm1& z`Zlx{+@vhnl-}s3H(0cKk98Zgs*!G|mHw5bL87gZ+-V|)M3O;mN}eCrfePI54IB&nu`b>y1y{QSK2@Mmt0)3*Zw zgNm&1CtyJn3G(9bUrkJQF1t%EUXW0bzqn+Tb|%3cw6(Pr3@;CmrI;z{=zIpg2=&P~ zXMcbHt`q$B_GZeK&YEf`Cx;v^3OU2UD(BLiU2mJI={4U3KGXJ{>I2SRw(tXskd`7v zm0et2?eK`Nj!=#T4FRn1_^1|b#SUn8VIegzNbQ*o#<=Ft$1sEH$v5Er-M`$X8~{vJ zjfFT_s!PAz$;nBh)bZkd^Mr6{Jw3f7>7t4J&ww%E7EAqFb@-2fpr7d^z+^?qpqSw2 z!~;L{kLQrXC*VPc&FlIWq}0^nrAmhSAA=~PE@#p=wYvh8&4~JJ&=ELUp|E-@2+}Akj@tgvwc^_b^d102qmrLBB?goxTN*o~@3-22^4~xl!`z)R`PUUS@kZW_SOm^F@S`iT3;@ z-jC6?n$Z`!=<{KMq~vRjYccg7owhM|`pH>gSWD{XthMdQ{NUtz?y3_=rJdR>St+x6 zUl)!+rSm>Z-23R`uLGOGZXtN07c`fC3@b$En6aGXcR$(sxRk4sd$Qvn))i4h#nJk9n`9lmkLI2Y^z+U zJ1;O`mFS`(2qZcciZC0^Dj$4;ivnpLZi_d_kQ-i!JwiQHhB0l0`<#P(%swb8^1HfiBQO!^^1~8TtZQ> z*f*}H6%=|=$<(oLXWSvx*<^_Hxrhb^qDy+Vw)!kiT?w%eO_J1emfl&7bRrYQs{(nG zd&u(h4tIk?--h_8Nnz#!pWgiRR3PaxIfz#D1$4=s%v>+%Ut^uGHl356C$W2(wyDFM3?|+hH&Uy*5?2Hb!jdMC5+s~ z+HYaEmJ}ZlF{D$4pZ|+Ks7ze25=#a;!0M;4-hCbPss3E;1y|-y}y{3oe5ZH3?@c0rP_ZQp>zh5z*T4o1Mkrmlmu@h*_zKDigESg8oTCStzpLVeG^D)fF zQk~I@A@#WSiAGRH>Ku>2FQNk=jngya57kM?YI8**sRs42Ue(F1}$-x32R$nmL74>LrRWh7uNytQX6D z?Mc*+pZ#EY`MmsT-u5dzMCU<%VYpS;!kg#kZpM8_s6f%)9{PPJC@KCe+av_x8*H?n zki^VU%F^xW$=G=Z=!h6&o+l^l>zwcqc?yCCqMEG6fE=hc@P6?oUgB6H9&H;QRo|@P z>zKO_nM&52MUzUJ(63s2+r@A6Nhv7^>2`672)KS|g-CsqT=qiX((2^s%5^52*6ow&HuT2@m zMM6Q^z8Q=%EFxKA3D9H_1uu))Ukh~Npp}vcgBnijcQ_6WMk3CUz8#X{{g=o;^-SQeZ0sUXo1>w-9%&AC6+s&TKxKMd47aUTB8!Z*nVBTBSRh}UfP zv6yDt7(Yp3p`fPFV#PRWV4j?OT& zjdumOXf#|6#Bjx5#7+Ow3&k3~J#=3>O|v`D3;{DZMGbKw(tB`2a~@~~TOTl2Wr-*U zN@JNQnqgx1wIm#muyCA)nRM9w7RZ}-*>PNU*H;qQ@-3aM3!GKfHT*-6*T>Qd#D}Hq zE>H1?eFTH(n&)pEM!jgcUhq2bpalV1LI!T1gg?a$Qk#XCSB=35Nxofx^1`G|>6X$<@%@c#~s^ z5wkRrAW?-rC6U2@lcf%k3Oj+*lt)qtfe!dq`e^}`H5=5J=O{poR{Ce?bGNfGZqT0m zS?lCU>o+^YIC{lAg`X7i{4sXvec9H1wc$%tMw?&gf-hhs$9lMs3yqBb+w>tyThhx_Oz*e-tSYJ5C=dci8-@fi#i)4M9X`MUfBCB9 z{3S$CtbEaIO5i`=XKX=mUsu_N3A3|{|CC188xAjIFdD4Iw;hK9{kEmJYPPnEbQ_V!V*D{?ct5*Y7>#SLUp_meM-O{ZpI-1_}#ZR)CGNL}KBI57=lF7r^wRN|tV zVdrpdFfK2axLyBj{(k_5NMNVJE0;Yby=~nI=kVty3O-LkK4x$IZxdKpqQLSTjH&i8 z)XCdJhB?Ip>Bcj4+qCx2Ta368;D9l6jhW=+ zC>s{7mY;#;l~inb>P{OILc2LdR1&!!5JRV2Fd`^y4^v&69?WT3KKl}BJmet;*2W3C zeSRJrgLJq@BNTbFcXiKyXf_KI+{Aqe;<~4TqIR-JZ>^FcO&LDcu>m;C!9(? zAe8T5?apfRpE4lUf|e(k9qjY}M*DZ^6E{JQnq*6crAW`3#|J^GaYvs-R?PKiQ@A{WIih9{o4Dl_6AyGPpMf`^3f3VIUhLOYu6kv zzRxbF#h7wBEw4KssH{hasO&z{SpFNv?~jo>nn_K+_d+0|plMLSEJHT(RdfOIzrJas zzoPq}aZI*vCfZgk)6B^VBfCzsN;(aC&aZpLVI`O{7#Ry}Il(djSP?E`WyoJ+XULDQ7MX3XJebr%)w7mIAB|p(yUv=}%07Gcjo0R<1ANR0M z?DO^~3I3%9{11;lDNG%oGQ5STChsqVHCN+ViV6oa;5`elPsj){Z}evc{_mo)ESJ4{ zubahi>Hf#T1Y9O`$X8_cei%@(8-orXcP!_(Gf#h)W}5E?4QkraOk({|%ns~4%uf{j zu|Fp5&9j@Kp+!=jU#jGRHpJfGYmyVwd@W-0jVD1Xy*UJ_qm< zjyfiEa*^);5z2`nKI=~vit)?-Y>8CVq)6N`7L~%dKfd{J+sU{X3Ll23Dx^nfdt(@U z43IIk9bvoMd@vmtymK?X%yXXd*w{p6B&Mx@Xb3vrByMK^UoXHwL6+A}HRx?v;<2LC zTz5{p)Bowt_AOoK@%lXBGjns6RqAV$%YEh~dl?~*6I^Ppt{YQ@h5qmZvtaNL&c;{c zK_XeaXN*)mKO7(ZD@OT6H#`pk9mCk{p_#j3I<}l(GjB;gDYwz^{4beKICTBSWrPAQc|xN~%3LGY_bGvk8)3C1a_AL$AKXLd=aAIyV%a~Lr}>Q?7Hg0BDE zIsf~c%ah5y$m@PaVtRw;e^ahO6xnytFnO3cR?e55w2jrNo;`z1$Bk2=o*?s!NZSb*u zcIy8Wo#ouy`FgJQl`Xwr5kh(yXwZtYdtycvx+81Z*7NmT{<;QT}CEKr=@U^IT0yR+SCrWteWEJ$eW zSb^+?w&$qI*`PqmU*qAgP4@+)i>t#Z`cdVIZD+>IH}t2XnWJ_Ny53-qOK`4^gzN@9 zsstjlo3dqI(=diV4rH=?n|kiLo)IO^SYq>L)eHZ2?Tl7WZfzU9h5IFR?f24ydwGPO zcwX#Qpj8A%&j)36->x>;-?L6yuIu~Kz$b{s=NrFp>JHdt?zuNZ7xtE_hkgjzrQo< zIn!(P8LzHeu#u+kF9^M%PsxEoiFG|csTiDY|1?20e4i4Wl&g5y9@=YeOX- zO%MWy6HmxpM;ojQk0)hDg`Lt{T1z_5L*vUt+doP{wCIMzsbh^GLgaEyu?rlk$G@muy(2#SeJ=B+<_^rw=MmMH zqmM3+)70iFE-x4yQgQe9NQSFMGjTlux{kC^VlSAD0F4~~bPV~m0D%X0%lo#+sfsbB8ttHi zABdN4XjU1LU7}L`d3Q$sACIz=UFR0{{BHPNZ8j***S3PbbKk1(IA* zCb}wEA)|^yHSxtxIxdgF21XLazvWC)3bRGa`3)6Wae~k`Lx;sfWhS%wdvqWux!i40 zp;bfqAIH=M{S+XWA&T>GPxZuPDir9r`pZ)W>Kozu$4Nt+nat0ii#ZqA3$`Hre;15J z(PCU?ZtsSu*ZuAw7K)0u(;FYl=1C$>nB0d?aFb12l|1<2bNIyxaZLZG%|>53FQfd+ zHclE}Np#xT<){pE-NRR(>wq=5va$AF1MGb%^`Q|YytMpKoE)t{3>*giW@xuzb&4$X zDtr=TjIZBMo=K2pRgBZHQ_PHU(MgdKfKInGw)AYOoId2X;zbkf2x=8MOgN33*&T(1 z$T>sC&Q!i!zF6k>(aAOVVX6yG1u|Dt zF}aL;btd05ZX|bzB2IUuX=osA68oMI==|8lM0q=m&r(S>-uDdT zA)B1Fn){W-ok~;?fyW*FJI}6%-W5k0GcxcO;jQJb%>I6cRStKFxMXS-4 z_G}H&Vw>d1HGJ27MLAV#@XU@$&4}1VUwbTXhWdsdM&E1ODh129G}*&&gAeWwYh*>s z7ykGubH*hH{nk;4fBYX#qN!eLH>x5PZa7`LnZk|l(M^`mttjtGuCW&!MFw=|XX!Le zbIHbv%>Mg^SzanCtDN^4kF=Lq!j|pKRA+H*vWE|r^S6boz@nL_{ka3f=Su$fagto2 zhs0S%gUN}m&cw<7ugyzbL|r@BM4HDc$w|#(scDSdD1+qrmK`>{k+*bTmsMX3cwYNew-0*iOw* z^ou_wJjJNXh=ug5tX(hRVIHPUJ`+(KKAq?T8^Lx=zTFj>_q|XC#^h4=&mejSeUKSIU;c1E@f9aT3!Y2#Ehm%0l#!HiYL8Dv!DaxB_)K}vh}n+rk5*aaR&J|(z%RF< zR^n(}kbe`bb9k9BD}AY&(_HNr5lFmy;tq{5{5GFr@0v_~kLtHhUSiofG0p(fPx%C=G>CiRWW=u|m}J4X`g zE?dDRweDWj9@VyR)G(OB!O%uL@4~p7a3pg0TdmN+5bERLG=q&>Xm>x(0of3NNuhvq z7}U!X1A*}LD3L=GRi08|3C{oVe&Pj7tL-MRzm*NsNbMfpQpay0GajIh=WBHvJS9J*(6wsJhCnOMg zaFIT=tQir)BAgBzIA&mWnb!BTTEzAKD=_WQb#vijd`Qq6R3jUp-ISZ^9a=j2->KUs zD}j(7K{RVPl3sZRes~4SD#^gOo`>-?IrG(}%*##B&Qt+F*b?=}yNJC*zHaoRok%2N zhlt*#J}wc@Urymk;YP7#hIGS2sk?eTHufW71^kv&D;W(5kSZ^)wL_~eFBr5vcz?+` zs2d`hI+8nEbkAt}c&BscM=dPA=`$ZVNilj@dpAH1@p+R+k<7gq0uEe~E1Fdq}~=bT-KZCvUN3;9zkS zVa`VFpA`$fN1vZ1=xvny50awBGlGK32;QAcY&6nj_u2D!x+TkJ_$W)K4lxpfsM7<5 zO-bZ$#1F!OlJWi9NRQ&h6Ab3Y839b=VdVVLvQL0$kp&8U7a$bPArTI(>mGn|gQ-1F zCdZz^iA^<9PG?I_(M^#qy_JSWBBegJRccSv_TVMxH@=irr-l=fzLO`k5iUEM;0$eq zYfN1s$*Z(U$*w=t{7eS+-K0fn-4$i(@}|ykUryJ!_(0GLHV0;xv5JK;Cs4o|Y%R@M zT!D!%TRu|9Sjw%H0(#l%#{C?L>J(m(7VPU1&hLwmkj>MY$G-YDr&$bz;C@7FwLc$1 zVOH}-o>Z-6H>yQyDmzLR+9F}0AtoL_^Y%8#c%GK23p)1>g*Tby{TY1o5a<_~%I;79 z@Zg-N#sry`IJs6CCibi_h+Ov83NjtD*GRkwqD51i-VjA5cS9gb8}UjhOTm33KBADM zBnZ0xqMk4q0srO+Yp9Bs%j7Y^WBToRP^Z=L-HWgoII9jTQ+D61MEw+*ohQ zabqmhdD%>()&z{@GuwEbS)!51-xO2G0i@){oHXBh33?Uu6nxjA7*3F+X1{4FhmF2q zlj1s+S75+Xh9~X?QxjXnvJ3SOO|BtMU3+d#e_Zy0{~u9r84yPoBznh^7_~Vwkz77^>GYX#hx?MUZcfB5E5ml_={B^WB-1xB9=WZ&&L$#ZLtQYp)SQ+a zL}K0HzpQ!)H-u01I|ZiZVoF4ji)Z^lRqL(`gyeCZ=D$;cMgDT+h1;g4kU3%L3LT^8 z`&rUYWz-j2l)LyF9g4j7Wz?en}@~2Cg?y(td zjC0ETM=A84AVCNzgO}VbwMNs~ZCZL8ey$Q3p))voTHgDBcMuSd-Ib}MGvOC)k6jJw zldN|-u#iAOu8VGS0R06NegHY$A8#@%q5+mMw@bf2{XRnwSW)zNb_KJ)a94Ils2Pv8 zrX*Nf)hAIJn#wST8&3`2j7b7)Sn|SkVY{>zn!crk9VeX{136b-v3=v~OA0opcU8&+ z$eyipUF9j<%#6BAa|(kA%TL)}kYAYYNP9o~GP>!s;aT>66UBoVTxXHrH0n9S2Hghx=%r6Ltak+4&eo(*Iuikyt#mT3g>5fn#MScJ#fCc zIj5lV%mBP<@*T|DC688%^hQ>lmY%@ViId@E@iIARiKljUb3FK{MzSj8KHakBlw=4M ztJ6VbW+ZTRG$={rS2q!@DMl3HaKrZ&W(!yAe4ihM!AoXGGrsXbBJrAf)0E&}v&A!9 zo$+N%I%}V^8YVmdeE9@MdaaI}%6d52Z1yrvSC_wTrw{sTP;Tn!{KE2NcD}G44ohLX zjd{1d&q=<&kF73lC#YfW+=*(<=7Y7;SOxQOvsG$;Ta8LRl9g0Tc`4WG^dxp|4Rb_C zR*+tL`+NbCD`#&@OWWMS=3uf^HN5%K_jxi$W@eAt3scQ0c>%a4AF=p0?5PvHAA4}V z>*@7{_JswONI3JjBXS)V0I|1Umk(pylES}r4&8R(Szc_Mxv?IGtQeAS%ImQgVkDI!3=SeAzoM$^>N z^Jnt&N8zOBgG!7qYhC46Eb)_Rp0oE&OT1{=`amSUk#419(_NEH5sk^CT|wO z5@{Af?9EoDH2h~RH)ssqW9t>^?d1={xdrkN8aYhN$c;pff1*Pilb^}6qB*LnaOM}6PE5iMFa8F za$~h>Lud(Dbpe|OwRS$gl|vBe(hQIVT`ce%_1TG#5P`U7UeWB2Ro|$- zrP8tynWcwnGoj&m`3lZYbeQj8yIZ(ZN8}YUV9P|2L?$O#uB}c$@3-eqXITeFmjyq4 z49+T`ww&69UltF^nN4Q)%WR~gmr0yDf@gCza=8#8>DFb`Q?Ji`-ABQXA7F+q(jP&gZp@T8 z!FGibV>&AAfh;3iuYaLfbNDEd$~QrUps&j0vl=67G-?)-7Y#zg{z9T96dEbRlL7?Q z?NwnRYz#2s+(gFM>S%C(H3v(~?ljQbKTNa-%NX|{HYX!0Mb15Y?PX4u=^#$6Xv5QA z$LhNI;!-nXOi;1wf9#i`R@$4uVz+OB`0wP@K@KlHj=mTxVV+B_Vj<@oRe`?h+czLo z)DB9gWDP^rWMoh0Aqhfpr{0=Zj$RHR7}aw&wN9KlALZvJa=2w_*7q_`J!%fIxz(pt zx{++7v`M9MYDUu?H>k$CjCe}!sBzizpFAwAPU{ZU2K%Ox+d5&{%ggc)rD{i*QcYFj z?OuN>#j?bYOnrg&CkbMDp0*K1NN77I*~{-_M2nBZGV668*NIi*pB(g--mXgx&Li@r zY+@DJZ#ZXMK^77>r;y15qmOyc>^g&m6^usMq&l2JB%+&~(zF@0Z`bAdDN5&~!J0t{ zlBs z`ECMBwcd9Xz`lPThw|)sEVGgV`bb*BQzM&Q{P!r277XO)el>M29^SDjlw|s%tHs)`>NchjQPb3 zrv3L{54v$6d@Nw2u4}1UtN1`Ny~KnOAp?}3_+(Es)N|anN(BDgV@~H`G)+oJIN?zL$P(fvA~tIHiNBq)YqV5u<^xtJpfh#(lP$ zivIE@Run;{xuV#rs%CM*-rXctT%ND-xL-L$mwn@p38S;f6DLuLCIa7jBdA3O6D4JM z_h7naHwIfapu9Y9Kn)LNOxXYL&^bs9!s=_et5PEA0OfC4dv~o8UtTn3g%RbsUpfkz zvpX4|#CY*x^uRYfG{^yCTQ#Z?Vh#9p-DSn_0q=P%BmS6WmffAIrXlZdRi*#V7}R3P z;MpCaHeA_W_~siMkt1#Kd_phj?YZ=M$i>i1WRV1M;12inc-juli=}zJ7;jP;N(`{U zabU1w!+#+!1aU{bUqOh&a-f_Rx+BDcCVWY!=w!x3tC-~DaCSWmkvQpQ>cZ5auVt0> zoD)`G4w`kt90*vQD5hWRXFnty?GJ47x>Om9 zE;V}Ifq5V&7m1X7HijIN=@?k zZn|0;qd&|rT^FTbT_Gbk#yW!PZ4`hmE-@qw&IQe z5Y_|~@Put-lb-WLohtMIj}r!n!qoKoSU@J7tFyLc(f9ucJso|DLJK2<2eJ}!xZEPq zfKYRcsND?%0brELl|q-XF|zywP403Jh|6o>?RBzzz;@R{Y@BQRe<4~|fQL9U{cqQJ zgsE(!>QBx=g6VYe(GOaZW6muq7=Xf`wpkp*&`|T^kCs)a>FfV{%)o&hb{k%7W0{h< z9w-fiLe0wS0OyXd-}2x7pK!Cm6jn?)C3WTaA%lZrv7=@!ZD3-S%>SJz|6R;ubOEdp zke-DwbryX02&BG#u)(zkgt2A-A7cuyfrb{(vqwi2MXnYq@^tl|=zB8g+fz)@*zA1B z$=Rp+&%8i)R*9x~7|;-=XgkkuIc}lLgLVI0{R{B;_>CZ~H-<(`=m!|srxnok=Zw;? ze2p*u#v(ArjxW#Grfz&u!%xo)wD1u`zQLUn_JD#1^p{t&nxL+2X@sb+FYFU6Jxob^ z%H-_AHm|5_if= zU&ujiROBR(O~*Lw&Q$7lKRW%q`A1Pv5d;h{=4euUC63S5jDa4Leoz57U($WfSBzh63QG^N)4m69$UzK7{c!7FO} z_9IqckSN32z(>JplgB_b1{Wm!TWV5MolJ_ta#Ng&d{4f$nCbF0es4i2^Doh*IAMBVY`tzRfNJ_D)5K-m5)e_;5+&i{v1@+GJ z>@z-75Gi=`mCewI+n@9ez0Y+MsB|*qt(%Ii14_XB7n*sLC%OxVqi*G-;(jgE-}lwwU$YP-sI)JTm1?Z z6sux2#;4P00#V(yfY*q%Z|yL&7vHZzM;>aypa($`91 zSy#+&CFLBi06fD(LzWlNUdZ85X z%nzdcY;?PlLz5>;iM^~QM_`G(nZMvNmGfQ{Q!Og4r$KpVMtE>yh2cp%QQsLk4c6t1 zg~I)y+66bdyoo=#L6lqCQ2JnI)*XWw(p2Tu7}7%AW|}6^OYr)zd2RpgKMs&(s-`jb zFz%i2b-`>q+oQdA&kq-#)~@qT!TbK55)Vi1B0p*~@&*DqXLf%x#t%Ci$E^v)YX^a) zoK6uNnW1=%%_c)}Z(r+)Cx>~1Vvdc}_{1giHH~5nge6VU(&sg1g}>&30)O4k(pG+Y zAW&XczR9Q-8D~AVkd1+C7wJqke%WKpy&v3om?vgz#B6gS`fn!27xf6)r_UbF1i!-B zZcUi8qTc3^=1+yT3lCscg?GNeeRdzt!fiA;ZhlxlB&BLD&(m-W8-MeG@@|P(We*3H zfB#`#w-Z%;e_XKL5u1@7MlBlolg*dN=y>7f;>fr`QA0gNk*sXEu_7px5e)D9vW7#@ zww6V|XW17x?d{`L8I_Rh2WaM2u4iqgW=HgO&todSz6~)ctj`R`q^m#H$qUYt;U2b7 z`c3C23ij`4on_v$g9vB}#uoa!leR@Vh(I^rqYysULVh}aosm56@W}h zn=%)J3HAQ|P|Ei^$RNKdDw3`{9>8BnE$}47U@BQvo#!9yhyR}X?0R@rw<{rYIeoVx zGO_K{5k(tj^yk6+aSkl}odh#zcIe3bVNtI3*Zp6GhkjjZY3snQ=Fe_7(iI5-R@ zFZw%T{U-x0?)kz=4ZWxq^xhR$-YS|Bg5pzH9x&dX`=&0ov%V9f&X|s)z9v&Pe|e~{ ze+?;fq;*+B6`r<5{WSUhVveuO6DpU~%i=|oV%ptUOINGg?Zq4z7^u@`NQ#Tnen(&* zI%vZ7a%Mw}jz9>9#0s(@N0SBVPMrI{eK#&Im^u#j=VD}n1~C}bdL2cRNd^q&`zeN_@guuI~iJD#nI6V&HKK}B)wA$Mp zZv_9rf}$2vSLZ~ikSsHsYctHwuS0y!cSXM{SRGVh?M!rjyH=?g9GlOUFT(ZL;Czfw zs?r%rzz_}IGf_+~yOM?=I}Wy^+# z9OaG8m}dU@b!)csSjpmi#^3Gw>!(c51#Y1?JzS;FmJ`Pd5lN$U4RteNG3BT_pS&-@&!@ZufhvE(^_uRO%S@x%*JKfpJjT{=S}4N%YUXpZDf* zb?4N~Gzx7t58;JxI=|>-uP2}?{@z<(Lpl*68<~ke-T@a(iuo7Jd|ZQmmlg2n4IrJ3 z+&*qLeuEYK!OTgcH&oyp7_w3D&Q|cWm~lEKB)Q6i;Qd$OSKx6Ec6N1Xto?s%Qj$>A=et!Yu4!(mMZEitjG$A+e<_~?% zF;69mn!l!H^I7jJOKW1<+PHmJD=_tEU6pjPn}^HluF`mf$gsmt(SbHC4c~d&Im-DJ z)X(^uia(D#y;Ao-EMTUVhNlsXM)a6nx~nvg<^2eHo;KP3R1PwF_|X4!VSMT;%~>~U zIhwmB^nT0uhkt+7v1uwd>yA#JM>1ApO2^HQdv^BBvcm9I$p0#^XB~fsk0+gEC#|Bu z)N*m8#a7+c-c6VN&MC<2qCf|P)zQWV`)}=7MgAc$;GAwJQst067Dtn3Yi4j}gAs(A zu;s?w-@DJxFi)N2XzC!m6Io$3S3od3_?hH;ez$Vj_0ZU{ayit31qocC5 z*@c|#xZZ3yAJ-WQ20nP~-LLs0@e36keU4mqql!*1f3LDSnGUw&{YCcWk$eyTBEPp) zPf_x|Ci~|b$d2D0p0u5wcWywZPd_oLFrSopKSCa{8vJ;HsmAAaMD@yBRGdAlzuwF3 z?D_LTB>y;Z-oHZfb;>@y_XYAU#aI6|=shY??;3M_aMh|}p(g|Nt& zGdy_0YlRRa5tUg~CYMv6;I3OFgbSo?;?U6is1dQhTW%RkzI6Tuvd4GR!_`WGUb*I6{dJsFL4-|Y69WJ8A3w#6+SCb-} zB;P4ZhRA>w3vp_IgP)F1=eyL$Bu0}mHjn)Tw->}^$r$?@=))l;oFWN;al=4K$+*We zX!A6A(-@kN@$XxIb4^7?f^**`mU*~Sno>#^SR{o<#gjx4en~=VM0+fhV&Eicm+O2( zV`#vn=Qzbmzo*Am@F{|%a$-;tSi=io6opGYJBH7-$o80*D!yiRDR z$K`(-kAou~#LF@{Gm>sCX>V1~eDdGbt&W14zB-B)Slc!D+Gq_45qfIS;?s)W7IFdD z;ND!>t6=xmd0Vov3;`Whk|AHHt>z&>Q)xu=On+av-H%n%pER_^;sw7&njV|Bmq0uq z9;i~m{cV>Mk^W4#_za8%sW2AU_f6RcF4Ux=M0gV5=!q?d$wt;>f$?}>w~3i9;Kq~S zqv9yHE?J!wN?Af^mF6nNkQ$Z==*!Rt{%y*@at=Gp_VvRSVkOIC*I@u}Hd}PVFX%ev zIfjw?2fC@Tbp1(dJO~Y#29)qHar;QRmgZ&qX@0t&tH4$$#uBSS0IEa-VAisej3B!>_CFx9kdn_EmL zt`$Bo%vK{}Y`+S*o$&exV%&3EF$vf=|9d%owBhkYJ^O&|gp@V-(EToWa`@pyHssG= zMgHxqd^yY^t;?%z)0yn%E|>`NqZFHln`A{*Wsw2GD%#exWuU4py4JQf5FHjg;j$#A z_=p^&5bEq5kO1!rMN4gph9O06zOmj=8VN$TU#^J>yq`uc_+O!FeL)MOi~Vi>-KAPI7+oBVsVtO~5cHy$&a6KPK&gQj3?nP7>db1MeS@xN$xPVq%6&hbZSY%? zA8TpDh=712;tS^#zD2)P^8F~}0$U;zXZ6`VZD*WBZ|3Zo*%VIAgrl;+iJLoBew0S>X5I}=_0Bob#(utJofCK;r z+~9UN`SiMQ$|ZQ!!^x;sv$LqKvVSv9vcG0sU|EspRa{Z=`C`m@wm1^$@v(IU#DLBJ z`+JG+-UMybkHx|6_CKI~pfXxH&lG?18pb5NzB0e*0jr$V@7r2y1-j#ORcW@v0LK6R zqQx(B`(WOY_Scx+dcQ-RUWz57tg{nFL61qdDFR?~H#}eZ|C4iF-;>?Zbe=+|#vUc< zE1tsoM(0s~@CYDzQNWhEDVF;16@OupeVlkawMw1u&(GoS$9LzAD@H_aCq;k$`~lG+ zyX?u)x*jCS01QqI99E;RAiI*iWq6ZtQ>4dw6>%JanC3To5;gd#}3vo^Nw>>m&jIP?Sy65zG}V#N*^S)AO5~ zpUn1U(m+%xasZv?)2B~>2i*|=}TWT^S2&(Y) z2f_0OO3{Gsgm$aLYjr`qyG>%`N3@0>zxBT6o9oy$tR$FcCF305Q1ZR9g6<2-W=u`#1(H0O~-WY66-+b5ty*qez*!elSmHe9rg3 z9lD!s_FH0LNJGp&PH}GY-HRFTyo(q7$4V*?01F+>WO0_do}ZtWR#%@XX^tGJDomnD zp_hSTwzjrBnHDHR_4p&vJhb+kWY$Hi8d<)KQUG>2Wgm;6fPi$5Y6hsuY&4eK6-*h) zT*gA~bj37f29yZ8dXU`c7iV_eX|kM>cEGpy>AVNL6W`Pf&W!irYC(JO13d&dKo_=x#;yZ0TwkDLYm z@w&}cHwatL`QZ7DmCva-YFHFyl2wCXOwiA*zD$1iqr%kU-rhXlyQ5@zY5-Tt zz^GMM2z<~BUi+M39F!FwACLa5&y9!doX?9bo1iV%Z#~M2WSb`z8QVpi$ zspFLmx?X+akY&F-N2=o1I$A%xszWZ%$7PP5TuO9644+!v;x!Zv`#cwayOh48a;LkN zHc>AfrvKFLTRN;!%3T7uIH~f~J>S52jvGvVj_BRzO^ojWNBCabj^0L@uK}+C^wa=F zbAzW!1F^cxO%6z1YGMi_+ZiKtGWp$7We_k?X7JcMb$WR@#`9=}995e<{WM$GnR?zZ z9Nz=Z1gkON+=Pd8Uv`A2r>o_^W=VBHhy*z#TZ;SMy98p+CDqQ!j zsX&^Om?8cZRbb&q8orxnS=e8|hO8#v_)OyO@>2{BQwqHrfMR7?=L7N+erim(%eyj~ znJ=s7e2!cL>Hr`6Mu>?T8LY{yZ4U+YHv;lZAFgBh=VDz| zp1yk|P_?!U%M>p@6TdS%XBIdEY=_4^nE(n=0MMOc!@}_d(tfE9QI&k5EP!i+e%7Y% z9-1%Y%fYB!M>NOsld=~W@ZA8Fy-BL$0$|-Htr!Ofj-=mgep7&mkl%;vVzD`Uev&H= zVk!eTq;frd76{_8QSPv_xFdCG&V+=^r#E++?VF9}Uk0{ZqhP zYKOr+L17O-*4_AQs&gfn`mZwb)w4v;NBiYhJqPep{Qig*PT$oMfM~nNXZ}wSFPSiN z?<6uR<1h;bN1di>M&U<0&fr&Qtk0zX4AG_{_g*IEg68kc! zWr*5eVY5oGUE=EOv*N3uR5eWab=p>x$1~1070@$t#NcC=5ihj&vP3sxU?Oex+l-*; zpfd`V#*+NY5h#CZMpjmftD*8Qql~LB`{qThq=6oa%5}6-1C$zz>3p-3aYa}(xujIo z?kOgP|E^5-l^@tM^cDB-U`8O8nX3uYeK+4FkwROPYNtUi>7rAnm*NSDt#_G&H$!U2 zGLZ|#DmDr#Yrzl3N6s1e%GpjqFj@&ofhNC_Fc$)~j$;y<4(S-07=qQ`K8VXxh#0lI-hLK_1>HV5 z@|IL1fPV~FpA9GfGzR6mF4|a~=N@D|T6Q6#YlWZP3gE2$#+;B0i#MKqPAu^IvwHtFOc5E9d9VbG>HKa^hhWA@iS z6Qzb$I~+blmJ%{f9$q6QQXb(uyoPibWAr@6Ek+(@WsOuSED=P!=mrPQcP*x`W%D-B z6h8q13}j;H7$R3!VG?!Zx%SWchDm~5dPy}(1%gS$wQ95?Sp7vQ@YAHD&oevZDsY=u z4NiQiG{n=nlB_>fNVbBf5{=LLGz!mg{gOzuZSV;@o&nqQ$WF-j7?Y z20akHxU>-#%gr*Rl|kG~vaz+J@k`^kA-u0?4~`>%);8FnNf{w(t9l5@+I$J#TedDBva;_Qu<^}-xar3gFX*D+ zSAXj4z`1nYu=gN!I{pfQ;X-Z4^wX-u5&na&R&*FmESFZnkR7-&BVv^p5SH1O#t#=- zksgL2gj)<>KLB=zJ}anr2(5^sP4Gyv-1v`WOVqu6U7_=lx+`26H|Fdk)9S}BL9d)G zmXr=ij!c94T#(licxmnY&!47$7#`g$zHr22>qqi(kCRIheIb+%*DcvTc}C>_gU|jF z;aLC!>t44w!l5x$Xt$@gVywSFSd=#Ag~iNFuT_6Ko~-ps^zE3MNVYSKc)a^%9AWz9 zZT{xvx=qCGTDh_NZ?pmM~cC^siTq z25e~g-C%!lNn*@#FbO#|AQ*KG=%J>sf`8J#hLA1xcozsgD=J)1i+HG2(s9T=wguJx z9Vr?v(7y^-qO>#5*Rwkr(?8`)@1SvGR;ZyoZ;hvg%=cf~pVuS~_#LPea7I~6)MTH{ z=yYmw^9>`b^sDQpUwSC7ScAPnALJW-4@Mw?oZ$w;N z7G|rxb~UVL?)%)W7jcNn+~(oB z-6S6VwoZjKRH(F!qR)x^=dmz7D<-%iOpiUFQL+T`#-tY@;F|CA1$6?)rzHG zF1al@q*f@w;ULAX9VA)a!J)G{vCri5O>^siSoA66ZQbxctB~@>>V9DmYA#fqPtv#e zRc&-!_@r7NtWVmRTBxdK;I56@jH})8aDe0VHVNJ5ey8`w(_BxGz!dWp5g3zKG)phY z#YEi@GO`DPv;P+hpp}du^y;$=rgcc&D4PAJOjo}LklY!4#WnZPmnmE& zcLXn-=wSDo?TkUAy%eAFu=D5!SPg^nR5o!RPPYG6+)o-S6yw!Dy+-`Ag61-ks*n}l z!FFBwKG3IpIx$o>QfOltd^|d*T!sn4A@Nu#!tMw|jk*sDq=iEtPWi?#dLm8b#GKXK zx73srNzSfcMZgQ%^IT>7e*@a3C^VJ6TMmY^rs;Gr>=a|>blx{_xbIWy1_d1Jbv*}g z$nkglN&&ncJ(!Clg^+4Lcf=b~6xh~iR$1j1?I`WTIOd~^N$jd>8-*EbKIHn)3!UpE ztP`WVk%?=I`Vj0^>!|t@sA)wfhxNhhF2dWNxXCtJ^5Nu7LQ+*)Atfc9q)K)|UZ7cL z&B$wL6)s@xB3^qqoTB`FV*VL-RRm#7`H8sSIE89eFs9uZ(cV*bFA{Ni9<5?i#-|Vy zg=M}RVw>upuP>?=%C;x~RbQT_AW(RJwku?QFOunb0$jxugiV`A4v$?#iY}E+JT4eP z&SXTmvyQ==Qgj^iSI)!!!*wW4A=#<7c2yjBNZ}f}`106+$1lqV>f+DAaWstd$JK+e>JxR!>)DIlhmgKW>Gy1JDwA)@1rZJ|R(IA*MX({8MI`FA{; z6ilDAhE%z#8QYPh7Pyr=WCc@PXe3Sm@h5|qik6o{+7d#jGNOlkaSG&Za>Eiu?jDAt zbCDe>C3*Go-nTNHK|kj8BIyq}1ao$3Z=xRorKCHdnT(8oLjsvDLep?%RVR`+_P?n% zC8JmAmUOVneYeiD4|nqHXovc^j8kf9yyhb`9Fq5M*2YC!?xL!Rqlg6XfQx^+)Ob#QKsWp+3L`i`PIXv|w{%*!l9T@I=Z2!!4(P3sAS5p)^Bj83-K3 zT6&dyrNdl~sfwkF9ntlzo|g*I93@c+6bz`)OJ(okWREzD=FtGgTMMR|^L}NS z-ZUMw{^7qd$DCpJxil8)>)Jb6dEgrdhU31@eNMsL>vdp+jt0eNgp%D5;)NdPD=fH+ zR5M%a0ZLZQpEk<;Y~!xaR-J!iAW>&n8k=aP{w(gJNhx-oa737MUVYvcgAvQcV~pid zqZ{s)V!m_N|1zvA!+W!si_yIC$w0qH0qR5J<$$AkPaedi*-41<=|h!+_^ny@Y@~?s zl<)Xa=eb15n!jFHwUgmg*dcb4+)W$qG@!LnE;jAJiB=+1toc0Cz<-o2F$|6N$o=!J zjmOD6{U{1g&oQ*g9Dyk#vZGr3dqiR82e1|Pi@8E zh9o6lX;r;$QuSe_y$*J;(}jIaDEuZ#VgPaj^e&zp-x77MNl}}(YD(VfH=<>;7{aZ} zK_$IZGhEv|T#fi?Req+(XGMH4HmbENyfRTqPfI!U_Wv+*65^4lcy+u3ZK9= z5?@`IsMZen(%0#Z7Q>aY;h-!)HXSNRlPNs2yOaj8@@r}Mm!)uhJpwugiErbN=X)jh ztw#R%ZPrm^>+}xma+AN9Drix$c`TjC53G~HT_lIk@U}~sHu%ywjnBj_wbS z+fRT2;_$6&F(EbDFFjdlQz4`UEghRwn^ku^t@w@B=$j22Wg28z^=ws~brk_c`il9b z4(d&^48hO`;s+Z`#{d+2dh|B^_t#~EV)eSh1Nyqo>I~eCPb~btSiHtOSyRrZJFF_6 z=K4FUR-IIUYM8Y+qPMiPw!Jkb!X(vDjWmr@T!U^DUf^2NsHtwKqFuk13 zuT5x)9IS0dU1c`=!F0o94Spl$6RZOqaPlG;^l^M}hR5_UC$W>JCV*3`4nNCz(9TZ! zD7*1?I^4NPgXnV|wzw@9A~Ai$vJG)}r!FK7>|o(fEe2sN#9GRT#E4IZf42RR(ngUI zPk$tiILVJAG&vxk{$Oc#In-m}@658FaWo&Bp$~G?#;q2ZvPg0JPB8lKNJGvd*G67H ztYLac5a>jI$h7vo)vaQq{+wpW8lA=8L6N--BYst3&#|91W11zuphYl}6Cz2f>Bq|s zM{D^Gi)W*2wma~)-A}Sa_O6fTU6s76(GXDH+^ZCo3q&wuB&A2oYr;>B5dsyF5%W?b zo-3lgluUZhll(d&me5D#dWqT|A58w!9MdH2+2Bla6HkrNBnb$>%pYB^#K z3a~~MgRVnKLOWrjVz(8zRQ6xoTy2g0ChnFQ$&;TvK6Ci}@4XhB#?6@obI* z*Syl`5KqeRaGG1s)tU5PPn2hmGjPeW z1~X)v(!sL?OyGq)`OhL5yW?Kji$2!8mB4~MBo#`|HF63a?N1*$r) zQt0c|Y;PSYc<-mI=&6j*{Vyz}N-R4&cjN4>05#q%!T$31VOp6mV>hIMexfq+82d-k zjujHYwt?2&w3Wak*1NUWN9YQo%v})?_N2AD^y`BK5$hRp+@?F;I*VOw<8juEMhpaV z1QG~Y2uox3R^~xCRNKACQS5v}9r82)B}hPfM1${si7R&z$TnF&l0vVxWWc7l`1`lJ z9{rfscFEf(d#P#UZ}yO@UT70k;yRUYM#Z7R8jYyjKUG#c34$l8E5$|BOh0H6i29SP z*B8F7Z^=B8tb?npy?vo<%78*E0qU4}#Kf{wdMiSV&^D1vLQ!hgh9j#AtQBY8B33&| z{^&6lfB!yYXU}-&t=heHe}3|{teT~+Xt~~C{d(Nh55Ar1_X6EOCD;AxkZA|@37)D} zV=ARe;<)B7-9CJ{?ls)o;GnG7plHk=-k3JgaNvl4J-~Mv|QzJaV;HzSMYi zfE=jOOqH}#`pu8}8W3sX7HOjm=PXT-hSSLF@x4&9gIrlhi;EBAYJ_PpAK{lZ&}o0s ziln}Lw?zc#6>sg3_2Qsrd6}=AWUPPs*JI|OB+hudy`C3HBLjEl_Lq&w^Xr1` z<5tGLcw;6rT1>hW2C$(Nru`Zws-4x77mr9!ue}Qz4fVI5G7C2q`--{BEA_$A{SInG zNc!fVoTkO$Zsdv`?UC6&v%N)e5?ZW6kmZfk+U=lMC zVCZx6sJH$?v<=0Jc1e%um`!cWfCg4I%8CmUdWdm%w93qDj$}{S*_u`$L@NF{f?IZvrb*)0<;n@~oSK(h(eg)R-Zt)5J5-Xs*4Q z9lc3psMic%@r&yY83|VXB-?9%1fVD&bWSU=iQPKw$SFqMdI#JB}F}VRDwrT_;2z(4)+SUI(A_-2_Mb3D^q1 z8l2BLdTe?ye=`uJ0{!JX(yM9-^AKlK1sQniqq?U*8Mqj5@&;Q{0K+0jpY%%FCXzZ5 zeGmvaQLNQxwJo-iJ8a;3`$|PY-)J}F>BGq&+6_r;`|FsI>P>e{7cGQ&AUWEil~{9$ z={sy;dZV7`7$l6AZc;##_+A#rU_Q$ri`-Bk1|7nDy`jeD6d!sPT9{3;!Y<_JgMDg@ zgw#YBV*Rytt0_OI{}kbNIC9uX3D0`M8WTPH)ML#}u}|*w$cpKqnaXzn2KYwP3wMeL*?}=uT?vkc_!kVhNrhQ5^bjge#Ow);R05-ptANI zZ)=aWkNl&rjlu2HjArAtcI=j5u*zah8Lh69^JwMK0poswlgbQVt=#8 z0^-FSl2}ReXjVt5eO1uCLrL|+9oziu8vrYAxJ`Q4B*>+<2PMB}YgnnMwNBsF6R)U* ztqrRap~eD(F=b4?JaGamYx7>TJV%w;+_gPN(r(@Y#LR4w7hO{=yOzxcTHR#KdRQrN zIsUQY+qzJEF2%>grMa2AT~{j9Di%nCkQW*4Z;am3d#^$>-HD<6jwtxPqM7?LkoL>- z9pkN&uef5z9PqwF((gZL(2L54))1Wje>9y{R9#K5t%C*)?hxGF-F?HxLU4k+yF+ky z_uvxT-6goY1}C^X-1Xmc4ucnXU<_8T-Q897%{dup<5ZAZ{dCDt>d6R{YYlcxVlgq+ zRI*&0F%%~muXYS$<;8|}xRGiywEhtL@$Dbx_UQ=_|842`d*|KMM511~3;$Chz<>Bd zVS1ytRPje&OSLX|xH0vchs{^EY~2!HOU^hs+0`Xz@4IJ(Nngbn#LcbotfH^Yf}65j zl+ESkY(Opf2_io~JcDs`xajf$M505lU{4vEO)ywB`H25}gZe2JeX@@P+YBBt9<}y} z?d(stl%P$}&#Y*r?a4a_>=$>hw#U#S_~A7f%5sO5g_Ck4zGI*)6~7u6cxN|ZLrF!& zimvh@)~2=7`gCGRhN>B zc-3IgSqt{77LMMD!xSM`-NvI|-jFNdy3W_%9)zxQG^Z)bg<5O&!!i9|IkC;;30zNH zd5oIL_vy5pg&Es&z2_C@y9ujrBvq%hob9M@UC^A}>R#_`b94}AU21~KvX{NN4<|%v zYbcC$!FEacFT~6<9nbBze%HTSsC^@e&)>;&GFt=unxs`TNmSuE(h1MUYD7L$tpR0V=T_8SD}TTh-QhhQ$;CU{G{9Kq=BcDbh@ zhVlg|DbLbBkd`NpRoMQy&=Fx#YUwAwoMHFBWRxgQ1czdRRM~-piQS3o_f~1Ti97Xx0+c*^r;{ zxGHOLX<%zWuq@wCH}Dmsb>( zU2_;-(RG|dwDEZzog5XFG|MOv{!+a>s)X|#4FKU%A3NxT8b?^BY&K8rw6j{CFJL3^ zXBz1Lg$&U1iCsDLPR^-Ky$*)@$V4GtlpaZa^zTFD^rg5IyA;0^1Ea#z`RA*K)2RMx z#+*@Cy-g;2jzEK=l=PFG(NjZ8!t{a36{B6&cR!4yMS~E9_ATXh_dRn$^3eJC=0&I& zGLRVay}B%fWoZai-MeGqHbLzP$PPth@Ik z0~x&G4@6E#BN7QDa4mO>!ZQmi@FL%w*omJnLt9T`_>WFbb~_N6wgFJ7-vy&Ww>{-} zuEc5mWortEN*ys~6L`D*Ch)#q4APV1Xm(Z?BidudZxOPS;(M?>BbxnEIb^sjJk zxwW`l#zlsc)4;55d4)lZDl~b7lpQZi9-BUun%9!U2m`)C{tJINN{7K)|SizgDP z1PDViL&l=d{Ib@@-^ky}-)Q)yki4d7D4{V6>#CNVBZW~mg-;ge2jxO{#@;CBvf%II zbB4O&H6!I_X=z1~R#nn1wvfq9vS3G|1of z$s>Omb)y~^9yEO4xgXnZU|zR$W!2pG>bX{+_b|{r)Z^tWGteWlep|>|48_i(!orc3 zW1xJQb*wFv&_W+eSZ<=2O<3Zak7JcQ(`PB>5mG@Y+*hJgq>kdq3sAD?tMMeX=##;a zP`bj}2T7FW(>_JimC$(MGtWCee1{X0EH(VCR|w3wBxbgEWh_+hazNrEOB!PF&WvOe zu*l=OV zAFIekNu;%I|G8u_syT_b4d=J(mj>@g!lA4WtgIQM8(9->vQPNxl!FhA&JiO+derKHNn*gO^OF^I# zseXgxzY9s@ZZYa+rqrW}CcyCFGVrI@MQiBN53gvaNx@hkxhz1X zY#aaxTkaH_?aw_9uW}wbLZv^hP<(+<;#vUu@wLB;UG_NCb$8zHZ>fq*N(jpAskNCD zxM9E^&z!1AXHRD}g56gE+-JtfQ3U@)bFcT1hJGE=G|L{`jTPi7WKib(cFMnjHcthK z-yFXjw={|;soa@G;3VnC&EnEJ!z~E|ajaahe0-xv+5FM_Cnz#|1W?WFe^*Loi$Bck z1iiMjp~1R$Zj^%fFz%e%l?@ci zd%yB2zJBdfE2Q{E&Ve>Ru(YP#ZUu0F(_JGVE~#oM6Qzwx#yRg8&_B&$8$)hgxh9~hD|rj2Wm zm*f+LO&q!Ib~*exM`nLV5n?EnK)(b~%uZd6W4(ooo|sj07%k2>C|g9gQ{k-w%8j;Y5v zZ!c26@OnQ`q~>siXjWq%;a%F>unu5|fHXfQr5$y(Qob{rowdeG?S3d!3Q24661+I+ zC+;Vy#4K+G=Ru&E0m0plWm!q2S*Ij&C$-P$?flxhXvd!;DRQbvtRYs)7u+bE|E;07 z(FJ|Oriot2`=h%h{S=3}`7#@qbOjBIDCyrLP9@4v z?6^x{wLAKr??ig+Tn)DZ%pX8}owi^2wp7X0s_rct#TZ;C(aNW>_ zAdv>qSZ$k{5ey{*-d~w$N{$#t2VGei9;m7D+}Ekn!QK9QoLRBFy7Fxi_BaF{}YJI9xjL_ZDZTom9m zFT1>^D7A6(c_@k?v?kk6mJopycr&VqV+bihPJA zE?%#hA2@AGU)o}q*73tPHakUbLQ5#O-rOQTcw7XQgz@7Q{*bEa(^3K#fjgbAR){0UtRevo zAsRRRy$#c%Pg40b?ChUn^}~;gJ2$-_)el-qN}?IRNWz?DekxJqrJ{ml#5@&*or+J> zbr+xCYt0Ul0N6Nz*PvUk+GAo|h~4SFV&$wp=kQ5@c->9-K=4t-MU5xFvNQWTy*iuK zn$2+*i@%sq?8S>J9Ry<~6?4ZWq}!GYcYRa#ph3Oa&Fjt75_48z(w-yl_0y}Vust9Q z=trfc?7Cp*;aQHm8{u@Y(8LBJ^#I^81S6GvxK5QWerz8fu!eemzV!LfC6XhhbSV3i z6gysg)1p}S{7iq`{bm|uLY4%Q|J6XE(nz3@LU95TkltxTFm(=*KVw<31%=p9X`QmH z6c+H7DI=4CQRxheuIM8+EiZ({{*MLl9zQuTY;!{E6lWD|P3!WOr<^y>mN2%885?P= z=_y5ErR^={BVJo@(!~@jlwN{Ei8Vo=U;#&Z`cm%>@Tx@%D+`h!f3B##4}4kV zX4oJD!};JM5#?-q9CE&Cgz%cI!iiW5v^(c-Kc3NhM~fcEurcl*L|UVm?($yI2O#6j5Kcl{_u+B^~A#Y`x*Ls|{k!J9}zab_K*puVrQ4G;PGj%%lY_HfK5re4rh zR_|n#_gjc{g6d#2Z9JKGG}MH~k0P+fq*D+*Ze1Gx7&KbKUBCx@-Y>+4edMr7XHfuS ze?h5YNtru!6RGz~xYpFVm$|hAM97lunrJlJ5^d<}654Nvrn9+# zHdacV&xM$hnah5ESd#Z}Az zLpp(GF)c)2EYcYGHjL~IQVXQDft}++DsCc>znGV|urjK_oy(3q9{b$UGG4+>Jjae( z&y>3UaELV+QJ#T^oQc1XE57kb79sm}LVup&S5-0n@QgP95~XrLyv0Q{z`UP44JnV& zpryy~mp?E!4r?Xe`j@JoQ$g+Hgu}l4o{}yd{AV*3_3fJYHn>U6&IO` zcUty@fEI0`!@^)C4TH)+AuEK{fzI?$qOQIv&n4G~m0CeG*(aZq9&V|MM58&Q#)?sS z3TT&C?WLT^2{qFJ=9}MBE|-<`;^!JHB8lIyJ{sqFivGP!4Z-P) zi{ZpMAnO? za2GL|q+tSeQN_nWv_MMZt`{nFsb%%G6nTtM07+@};h#yo=)OM=73qatE1q2cxUO=f z1K>EyB~|#CJjn9J6(al8MBQCv%d9cajbA zlL&G5Ld0WxC0LT5qDYE07gJ=pvZW^*qhsYEDCxeEg5TB}aJv5+>NMG!dywaWa(N;A zj5a0k#&L5vbr>+^i#>IxQR~f%h|RElR1o_&kocA4$q-_!uH~tiKsubD5)(MV)}42A zpPngG?e~!Y0MCCp?+g{o`)RXQ2EV9nEKqUit|7hbeL@>0 zp*n`^+ID0$V9QQl^^S&-fyr+(Hy-t6q(YYJ8^3&A70C%GLs!;GN~N48#Z9{66xqYJ zXe8nL8xpqRwXb|$1?^)5sFby^w3r4F37+xCweawd{0hGbvJk!Iyc4`DY{rnG5xXC| zjMW8;Cw?FC7n&Duc0&3)Hn)b@Vn28NcYg)ra`AkVWi}Q26wQPM_u@GnnG?(-<%wt9 z1hcXpFf$*(Ye2UcWm8)`mG2SjgJz4JOe}$QQ)-ywUX!c8&ld1q4qm-WVYxGf3a!rC zCip^UV~ShRa#uY!cQVN}^7qBbAXnn%Fv#xF&F?p^iW0+chu;g*F!A!x#uTPz8xz?H z946!roYju1R*V4r$I4R@4zrl&im~f~jji^sL;Je%Nf{>Dnna5$rC|4~8{cF{*ofW> zcf|=4rP~r1ZcTwor2NZF3jEt(+SVrbz=z*CG!u_0`p3hPtyTwx)t;bWN@m)X{WziB z@O=`pl1(&*A_q-&9OdeYs4ADz1-+=|_J;aEy0#b{D%kYjxxf#l@Zz}%^p{3dCyy^5 zMqt%1&hd?Z-oo1SfmYpZSO&!e_g;LL6o1rM+j%s4Pa|eiNv>xE^N-tB&&+(4vj0Rt zDVcUs^pXJKpDrI#lH-|>`a=s^EN%)#`94$|59`{a>bXBzLOJN9O~-e|geOKEH^jyN zrO#=K@&FUWI=!?MM=MNAn;D=rz&RVRvyIxG<;@2EOf*v1+I_H29_g+BU_HOVyWLFxm8*DLkgFGyF++(?&2v4gKKnu%%ime=*#ZOXMn zY+IT*WJL+#ncswGW2}0?v3s>q|9G`Psxu`|1B&E2seMq*>fKIl26mVrQo`647KyD3 zKhO#7;f%w%ItxGb+(*gS44ninwt2A1^q$N?x9UL?HG=oSAFgvhrYX6xNOxBw6M~|` z!dEDYZL53>s`nA_z?w$*+|3_eBp@s@Tb*M=v*(9WNR-6#MNcS+NhB#p94iZ?TpO*qUF{ ze{|-V(Aq(zr|AyX%OjLD|DwYGQ=7fZQCczJ&Pd{=4Pm7jH~-uP&J9k26^BrpU7F2Q zmvSC{Ea4E(^rwks5x^i33v1>a@#{)H7<1gM1JClSB&v)eFAB$CmT?*gT)P;dfEv3O zB+i#r(^YSat!?JXI<=^6Od1rRPb$jr6xO@L16^-goG$F957o^59%0|#f(0Vmu})yg zC}c|CPoPeFpegbq`e?uAr-@Y{M zdvFM!(m6UU=qK8yS(U{;pf7sWh_RMF&6X|a+Z@9w7#!bw)EtE>%XcP8oY@ms-SUvG zl8``2vWVZ1f80aPvPM1QH5H4N(rNL}VC%#@dp>UMi@4icy8Ext`}(edB)eB?LEzZz zZQ~5HlufV^Q)fEhc0<1IR|Ibl1ZuUqDlk`qEiidZcFW+k0Hx>|doZdh9aaDdAaG(B zgCv22q)P_*Xwn62!NLO6_;PRlZ5frh7u3}W-03i=r5T46#D~_+ht;hOv%p3v0Oh9q zlMuJi*_K|OBaclCS`HOr;2T3o*Sgf4Nnl#%t{im9C>x~-y9#@1L^9RkutsrjltN~w zn>8rtB4uG7l+_u`NsbNJP4#n}5nj>p@ za{l9`F3*~^^oyn*iAF87-weAkJ9z!}0n#Tp?ILG&Bv~(WGS#m>LPe6i2zO2E9+?#y zbBy)nY<>K+vu^!LB1c{QbAu^mE%p)3z-gAm7;A}=`y*4I3;mYsT0s(bp5-WY2g4u? z3i{o{?3E%8`xE~M_+hF`(d5YrZcUO_auIF)qhpG+J@s+aA$#1fY+y780r{XkjZL96 zea514#brtNKYyNm$8??;pfgAHtpvP(x1?9nok7cE^Xyy<&K$blA3Hudt286*(j;W+q8)=^2@96tt6plEFz-(r0s{ z55kU)q;7WM(dddyl4@t`(8#putE<2{Ut55R=6Ls$H(ofq9WP257AGSa& z8W15#Ssoe$T2}AZ8W22kIA<$Z#>^(lfkSou&fsood&0 zay6UL5R&(n8K5mbxMiQzKg@RB8E2pB8#7Mhw892{%g2A-f^nGT^aaqEN+_#-WC6FQ ziWO8wD;$LpcuAlL+q!QSwsqy!^=VL616PY%8a2YFYge!P?BhaXs z*P6B(BqjeHrAGOyL;&*hK9-6lHg5dJJ8LVe{s*7FnyXdMD55b?(d?y>CSCGBYIV|( zf_B*#b!9tZ)i0dNNolv%{42Zb{rr!M)YMmH;%m!p)S+gn$*rg7q*dL1KVS5)j_SK3 ziIR`H4^rW1rCjkn*MKanQIl@cxt@duE=j6HN;taGX!G!TUW>ij3eupS8$dZ(G$D|J zZ;)21;&1 z%YbsV`J3>K%5k?qd|dlzR|AqU;iy{BgD?Bg2kRq!m`1>v+Ysu_a*IjZmSOAbEQ>u6 zx8~G@!}vVD0q+2mV9=OefyKSU@?3M#?W6btv5G$4bYdi}8s;JK%Fu5say8`D$3a57 z33A^t{S2xVYp&z6^`D2txaaDpJYU4c#gW&TlxTr}N)#lUr{q3&$AMI^`Rr&d42%nN zP5?91VhccSi{RInc7M9E7aD?#kH0fNr+7$C%o~*Lx*D#o z=K)hRpZNm{aN1U-vwOxLb_n~u1dlA+c)-2BLgn`Zu@8^j_jk22!g{M@M>_)C)beA_ zF6}Nto~Q;SS@fObCy|cM?AO`@R9$fBB4dKRI>AdYLVjVd<#fa2zpmvp>BpobSkmK4 zJ_cKi`mzD%(KuN|Qd)GkRhmUrv1mQz$Ne;K+G5Ne;8I-hV6~g7mo{=HydHuWYdc)K z9ed-Lb(Prox`zBF@M@TzBb1w-`^-_ve=hPt>=#IvnxIBbOS=nEiCQIX@?;2W_lUgt z9y;rKTBdX#j%KXeo8cd3_mLNB_5JW+F&7{E98@;YzY<-ew+U`u%K#6 zi(HqEq^qm@pSbqCiiyhEbv9{Yc5SVHZzK^AIWrOwW#1iDpLllIHE(Ft=tC$9-cbM% zZkT}boFo#Pai<`ZXsDrVZrNj)sP!SIz4fhWZBzBtmJ{w5W{b4apf^&yw-^ zpWO|0(DTr!>MNi?Bw>J}h2DG%qUcu_;x6PX< zXM?H@6ara`B*-F{1sRZ1nFP{PhJOH=9^8m2p%xo9p7iu7LfsF;>39d2(6g1kzZX+A za}1d#^ia0hdiuTeCOJQ}^3W{_F_6XvE^mxcc!Ex6a*ekSe#517MqOqks7@v&T?Nd- zaf@GFc`Vnv=AX?^p;r8mGUyeL1LQtfgLONHZuFk82;MJAU#Ob!UzehFB^;tmB9vFZ z!<7bQ__$48hreaEyq=4;!5XnOi6Dkw%kiw8N!Us}F|r31cb}&df%gFn!q@m%z&Qf+{(e+&@t(Ss5LTECUZL{f22sWIi zV0$`Xq3Yp!SfG95ZfiNBC^lNKH@@TZWe`y{tTdQHpMha*zzI!J`Ue!Br*g!!VKdla zn=JJ4^)vPHAcK3j^6E9Om&Z(eg{g;s{}xkwkTyrwRM*E_hR#Y!VvM`@?n-pewKnJRJ_v24P8Bl~6KP=%9gzEG^Ygqn4l&lk2-H6K zP^HzcFVy2?vOy^KNwn~3eCDj;EH>@MyX?p6Zx=&|4d+W`8D{Z!|8B~eEm#HRt$#|g z^L|I?1guoxqTM+?N}YDdj|H4?-Pno(jfVU_7nCm!_tM%eHoojALi{pLLlg>D0pH4_ zusR6n*k`l=F?@&t;9M}4!f2F>az6IEx}%S&{cLER*9n&p*SNa#-gG8kvcs>s)BPyV zqyM9y<2BHM_WymPwsA(9(R( zm7J(cG|$F;!~d@4?uKojTO~s0`;CJHrjO&#y1JNL7|KYDEjyXN2!@TFvOr__Y3?<_ zB%S%EranP_%6(#T$-1d8*kn{qn%>skYWHiD;#N|#Ih<+JhO;vitYA@8QB12o9$BXt zLnSR#DWJRdhy`|IGOj7br5QlN#2QR6{DoUz&RuLqLlwqR91o`_Iy)&Tz9Cb(+knWz zHgWs*v^C8q71j_eH_9Vv6=abYz~-~`8X>#&s7RG{W778i@cD>GVKfAKxp;51)x$Fs zb4#Eodom^6`^N3JX47QJik!;FP1nw4r#7#7Gt!bU$3Vn+SX0K06RsgRLuyQ!S~<;& zKzWKUXut4f!j#&fn7HE#;dF#O&3@)V<0V&*`rjFutAGH3t#`v8G%s$at8fgBN6&O1 z282W6`CHMv2mL03m0C4$Au(LzLOhr{48l;{qsP%32cGox=oqt62GyLI{GpXzcaS7A zlkwlaV8Gs{NTv6|N3BA`KPf?_Y#;)Qo&}t!QugPhx_fdU9E0~g2qSza?B%>Q6R>lx z{Pwy?V+AVVLc2Jlyvrbv`F?khVvm!QIDe%2S}eke&Jd?H?Q9pG0MXl--JE2a9&B6dwTXCtAIQPY3H=uxj*@z(XFjf5z7M3pWV1kQD z1eWBC`rcSXKs8Mg=U@bxI^0N;b6t~J*tJ`8s|;asIxXcW z^Q_~aT4NO6koT<*6oUu+pv*i|`P2k9CS?QzUJ2dbxE<#ze8L}stHMAc*3v5iVe@%* zybDaTux24Q@fEX~{B}h3#NJ7_c7MH`cik8Fs*!YJn-UGRb-r$y#rOe)I7lMc2A@}? z2EF{p$H0+~$_eWtYavUFuTO+;OYaHcdQa*=h^bB6`=M7U%K!_QE^h|~#iGkEVS-oM z`O=TOx}OdI+>@rrKVHT0M9;35157xar&4A=v7o-RhQk{ZxR@dQ!)?QJjBwas;@`r$ z{o9>;&M>eZ8g<6-E7mC{fDYLG%8+4fC$;<0LlFIpXA=8;=mH&G;>Rv=YM!GxY&tQy zFQ++1Rx4#kXgOzh*&`S)99BL&3^To5j)oQv<#zD|kCCG^);;NkY&~@B$l9n#tOSy0 zU(i1({PK<--vaK-#_wom?~Xu}bHEW*GQIk}DN}tZC_MgqA$$SFoyv*C04P38=Np0{ z7V89Pz&epWUIi(a;I_2g(kl&FCsBREE9)aDm=8jZ0b0j%7k8oK4@3P z(^cr6Xld>`ToJ8GYiN6Y4b{-GGuLGiYuvA>Z^?IS0!4UXBpGLFFSq*N4s$G${VN0` z;nh1*cbn7Ct^l%KI1Y3oRr}_(^zrvlmDd-YfbU~*D=7`8zH)h(eJ*7Te$ns0{S*#y zx}*>BaAv(0rj%EEYAKi*VjpkFC7Jk&qd(`+*Vt>4I$SHDxY6370cTESLJ*Ltfv4XR zjb77hqGg5DPm(wSBsXw?8OF3Ou0l53zmtk2VZNU-6cyCt2Hy=Wvt#HRk+x%+*YWFKg`mZM|WSRlt{=x~;<_%6%u_*(Ttvkl zjpDOq#hvq7=0Lx;ES{xM8DwvjCO&7jzk!C^A!9Q|WZ5`60pm^i1vR$4U#kO%H^vFREbLpD=aMRh@2&Wt(W;m zrQGtbiW<{gQC~=%>g7Y>#p3MsTyC^Dba@P3FY)47>Q&$|9=TIE49jN`j8ad&gADh5u>JPp3knD6=y zNWXm6vmXlY_7oGlU#MPja;v0jX6wPDYGI3T{Z#lzc^_?h26oNt_u+^61vAH^wUEQu znDh}JEw#DY2ZtUGl06+b-VEYYhuV-%&R-_M}WmxUR~_!i=>OQ{*Hi{QIcm z5}5LvoUIuLHANoU(NhNO5eWxFW~6`nd*H4W6-7Fcgmn<%S6#c@2P>ZHG%kU<=c??A zrOdOVl_RC(A}6oYJ+$pxZIPx5O=YU)>w@pe^S0iZOXicanUMiFmub zT$sfvgG~acWPMzV?ZoE{a(}TLs_pc0IOYGH#+S#?y*loG#xXBSbN}1SnxQqV9Ap4j z)xs@}N!#TxN`W*dzs6Nk*rAO}5i|AKUBZk14|+lMD@>_UVr#+AmEUNtbiUHc{LSW_ zf3-mYkSoT9jM4UoNm4dOuu3U_LJ(1S+u_gomwl1?qThNd5Ls;JWhT>4Sdvk_w4dGV z2M5T)CeDGJo=*1IJHt!3*e(mnX~!L0Zuv>I#%%G{&&Bifcry}Nw;Z4&g8 zO0h#Rl#Sq9w&V9XAS`MN(Jwdf`Gs&*q4VddsI*wwPpQi_FDIZgs>f$b z7Ews4Qtl6gbwEG{RbQfvpxW7ygUGeMTJ)gfR)4p@p!Fwj*5YPJ&aEKH<(zq0LIlSI=U)IOGDEbL4p{_qO( zeOz6yJ^6=(wTa9cRhARtK2DBtAB^7c=CRLmd0&D{1pWEg)kt6_uXQ!{t4K-cAkaPq zao+KI=Yl<4VbP+GAFKkHhFhAk;<}`)gNxN0X)0oN+qTzd$-22_v@!m2igqj%$pMjP z{xih88`=~xoF*KNV$?;Vc~7rDnC93LKpiG^ezV`eqbm;eske7cH?_N-0%3hC`c4{g~gIUPX-m`c1QS5X6H4FO=y~r_lp$i+mbFQ zi$H(JtUGvSHilfx)^GH1rO4#{sUck6ydIuBTk5~X#*S~u4%dbbjx|9G!PQ;*jV>>JE!sRZ^r^3h78|SuE~AfE>PL|z)S*7i6oG7GE{=dv{LH{CXT1z3O@`x zN3Uv<&51Q(++E~1DwT?lwFTd)ByeO5$3BY3k&TV9hlH*_WHQ>HCCIGd(Kybgh>>Eg zjQVnfEr+M!~|yi(An821p!!k9ltmH`$P&bt_|{ zNs_AIn^yveo96N9$70m4#aXxrtYA`@65`d7m3p9Q>=pW=M{||fjp=SE3$ylVN_O%W zauIE?jJ&>eXiR-~5EItOhU53yI!fTH422U(wP?JjvD1)(I-Njbil2E1jDM6oo%_BwQ9KTNlu_2wLUDH|%F~2X_F3w4I7M_ zRu1s(2fKC-+QqdtQmd(K(Dan2FEUV_EX>ZLbgUt`2jp%JR{S;@eg>S^&}^nyEr zBPT_e#=7bFzibeoQT;OvQNNy&`b1X&7#OMwi6iSKk%uvigj$r>(xDbYt?9`nicE|o z{tj-CNYpMKSy692Ok^mTtq!66->*z{88&(^mH8SscPQ>zZh8jwk?(HKoK=mc0Ia(I zpQWjtUwg7-B=*ea<|F`FgT2f?)2ml(GjneY?2UYTEJ0)-%BnFp} zk;%KiY}D#EN$Bb6`FQ(uc=PHsWuVE`*(oTW&X!yd%NyeF55X{Ie4fSIak=i41|Yj+ zTPn(b<7fWW#-B(T3RIZcWE+2fN`VYEBJg&uR%3d0r+zuzu5$`&w??7 zt<+gAOUBad|GFJL?mgY=I?>jZ!GEXQIcsG;uezXqXg&#Zgu(?++(iaX)W+7`9x=Y;d9sq8Wae?s0@QvSlZ zFY@gRr|+-QpzY5Qj=#!+G8_wO)iFiWI*JLY#@-_GHT?4oC7ARel(Aw#OToq!2`@6D zsMH~?Ae0YX+V8QpixO8dwOTl(Q9l$K@0I@CNxu9U4)|+@$GYQ(B(0P9AV>R;dkl5i zbU{G=k1}L;S4<{4a+6J*3nQVNp*G|a^e{G0%0MFwVD@GcZbHMS#pYfmN;pn&g3m+6 z6Vp=<-9%|V1zJR=@pa$eYx@jk{q5ImYh|?bPo<1&13c$?3FxR*SVP2)cDd!>h7yUq zRf88aOQuaCGRIY&Fqq9w-u_N;1O!T`42bNM&sBfhSiN{?-MX ztP+Vfq&JNLz=s%Ik_rbm_l>{LeQ%wOz!Z6e*cX0pu1+5x9hU`So|B6Dl?(9~_9O!1 zmAWDwr@I9J{5QZ&E0)C#Nh=;n8u>O2a(F&AQ@Z!b@w(T%Up7K<3=Hh}c=Ng$|EB18 z*0|)@1M%7M`MBi2H!WH#mnNc|x@4jq@jpA}ny)Y@{WYXckHM(WqAZqz1U6>qVyM!j zzLdr2viB>s;ZuIzWc3fX8f{E_-L|I(U5B6`G#*pOS>(P6;l+MU_vwS38F4g_e8&^>@1$i@g9}y5D!i44 zjJ)&ka*~QJO446JTK9#@H$t~L^i40d!U~)c?Dt!TCVu%TH^f~AAF#FfxBjZTOm zplt{j7vfJU!I|9TsRyp-+&|hcmLD}@Aa7+(;S}%6-=+s)N#-FvIm+{5N@(2Pf{QfJ z9!0Mo40)+iQQ(Zgs1^Zm(`}m(ilck{i*w#+ceg((d+R8`cmxo*Aw3nv&Y4nmwjmH9 z5KW}DNYOTwJBw^EP4GbXQEu(i7~tpcZ$tpssY@mSu8|Kn3NQ)K^|8j!^lir?Yb}j7 z3~Q7cM5PB;kjA+9u#u|BW>Kr5N-}{E4U3c!46`MClOTno(DJdx6;{m4htzv*x&D6r zVz^fn0&4>|u1?>A@+7-qx0^GxiHUr-C>iakvspO5N@o$x5#{@bqr#-a0~60EyQ z!Pqpw7NYolP|0bL>B~bkg1cbJh!(0{NMP@O#2OudI39!uzsEaG@e^C%(iDxD)>VAf z0Q>m~`xcEewV#g9O$qeJ@ShK?9A@=jFXg_6W>%`iu zc^`?t)Z70RyABrkOze!>v|D%9RG-Fv*rgo51x&(O3?Oq3ZCufGFnTbk6$*8j=GNDv zW_}WwgB!qEJZQrXG^u0AIQO)oA=RS-Pvh**KOy$G~e))B-$ zwGmwnK!t7G7rxEu^`2{mTiXn;J?QskRu>i&;aR)Y6MY&Q)5s{6m2+aRRYP{tAs#!b zEiBJB+6Z+6y&-KgQ>G`Dou~D(qL+L3b6(%uG38_YHxR{g{QWd$H(bFjJ zZ$eSsjO=egrK~axIR!em`L?aKt%1LW>CPl?HRE2~QxCIf%aj8XlN@F{hpmqQhnXz0 z`*}nn&>+mSvM#AVABVJuM0qbRELww7Ou}D<3@w&UlhDdvC4;3(D3(-JmV++>-5-xO zEQut76JNHJ(ib&vbmYX4^4~@5&T5_8;R8~EF&RljqRSU@Fs@Y}%~Q8QlEkJBHCoqO zpa&+bK~{x!DMOA*PlL+Ao)Xr(Mc$=uY(NO1vC_lQKcVeNUP{5sd2GuoF}U zJOl5p2LIfPMtjQpa-)jPL*ehvAj(2jnbedDA5_ah3bmKW;i7m!J;s|RP`7UTeEneS zZdj=k*;GboP(fI8sIev}r`~CGI#a*!`lA^l$nIT8nlsZi-p{hIBC(fjLDfh!=ilE4kg z?5tJ;e=@7r1i%4v^U^#RdnYIMC-1LY56zFkDvZe$=;ma|>{OoKS9@GSSE8St2V8DK zS|0Fny8;GWkQksqMT~8ucXu)QX^y;46M-b9OXf!ru`{cp!z+42yhk)42V07N7p>y< z(Qyw<)z7u8M)#5S>$c*Q$5x@$0O^Si69%kk|JP*#9|kKpQkr(#o!03zabAHGUyEB- zZ5e0G9X&7FGrdP4#*bTM)iRM5N4(aOui!fBorxtjLw5!_IYOp!A%m27Qe>sI5q*4oc$zA@LOdI|c4g{fNOmcXhr*}l!$qP+2g8-3 ze?J%q=-~V=xA4=V-jZh38AI&n?DX6-LIy=XKPXWQ%TJAMw|(6Z^uZ0`rTeZO`xNDN z@<5nutLw+`?|4;29a%db+wE2@xnku~iVDIU2VnMmQ1bDHp|kYYDYrX}QckZx zhl$>VmFTu5WV`-9XvgRyGSjN1 zQR`-55y!NZ8d=o`@>%X1_Uu?f`?Vj<-5p5>k}6%MjbP~Ttld#9M*-YgU1InIEOrk8 zp`1leL=lqCjU2FN5v?aqmb`EaONFBKar`tc=0H=cbHz#)NHKgA8WB~<*A`Rjq)4!@ zh(jlmJaUBBy;l~6K{f9e8QH^@fntRAXJjD+Pbr02;JO%;4fB91?Hul6v6ZS1iR#Vg z#cKVRV!WeCT5S`gRD`glA;POIXZc2dWqha9Z`pa=SozAugOA@ZTS`NQKqmoBQ+fFr z)c9&wyHL>2mpS!G zH`r}qpn8a^wy=^{WY|I8%8I@u$CYnIK7<6${We}2$S^>;&O`ADb1afKJA}4vKeH<^ zMX(v`yAh~reWvSro7&pHB#$A9AhTYRfa|Rt_Khl=D`vHrrNAEdi^|I*wO*^z6R=@r zRj&-LYII{mDlmYtw-K~DX=xp`7~8VtNS4SK==t*zji__}V<0X#L zoUg=`Xj7M`GVvTK8RO7`9pV>9%0*+v(s+;7_66RL-!{7Qcz#x37mIbONR&Yq`v*bld4YWyqtjYDXf=MN&nA!)qz=KoaWCr_C6f<;2 zZ?zYCqcUN_D_RY|OkqSX@F$LV|1-ah?n*@Q{M~=#QjM)(B=@;mTxBY@oj|2VnDf}? z#$%_2VkqK*Uv1s-hV*DWOlneBo0}^IF1uBP*bvEf?&F`kcdgtlgPA<|^OHtD-Ar2C zsMdryR_Q6aSSk_IUT$jtbv7~4fT%1~kb8K9c&oO`7O##pV14#5lwKxi0iH+S z3~7Lg)milcg@jon`eHgGt@)2U{JGna=??fdieMBt>0lW?&2`4ejHEc(tzW{Ndcvb* zcbp)&k3WjWDg?p=Ae91_IuRyn*g|o#J9tZCHQx1Ayh0ihF&H?o4m`~!${QFfb7>4i z3<_fGaY;>m0y1S_r04Iyo1AVJV&VXCfI+Sw&gZJzqt8?_x-ak`OtARDpfv5tc2T$rqyjWW<{2M0*(_hg;#H zGKLTa$RM^5S`x$*Y`C#16hARf25pM|H0n{$8e(kh?rNg^J(AapXz#auU5P`mowYI@ zGa9b(3TXTn3jDSk^Si|w3SS2?rzPEjSBH;n(jTuPQM^KoX*NW6tEmIQ+bY3V-E7V~ z7uGXKBkSOOWAbcXEU1;|#xdixV``?RYoY4Y(hQu)+U8Hx@e|QsW)K;iK_S7H#0h3H zzW(i&V%Suh?28V!y?|kCBT9s>uCCjazZ_^@uaEn%UkwO>q!1F4yp9gk|G8Ue>46t) zU^?2{Ih1kNT1I!BfmDItmQl@%H(fJ&Zvi>`yeD?PV^#?Act3UG>Xm|}?rpXU;yoJP zrZA@2(I3*q@2>9C$K)fUXgNs|m1rCf@dLPvq@QVz8>=uKXh$OoxjN}AxF02mH^YiZ+3^hcBj2o zw!>DZ(k;O2)MRYYARye0`6!0D)h@TDJa=A9b9PKec0#ZJGwJK?mGM>?uMmoQs`Eb? zyn;dw`JOmP1SA4#_L(HVu;QTx^%Bg2+V75)hTam~L@NEy!qhAa`@6-DMZO<*7DC!TE( zN5iNurdSFeL?WfUH}?%i00e02uv@X1(8|obO6!o6pMi^I6_TI&ixj(Ib%R-HnZu}R z{M!0F?ykX6;iB*^_>;)=SttA}+d25K)h(V&B>DXEC{4U2^o}K&PoKH-InWBbt6Dn4 zunKaf^j(@r9)&0eyvH?&vS_xmz=1ocoXDr^6Z>)5z3R_qIxTEFvH?fX{Ludh6N{ud z%{Eq(I-+G|Ca@kyNBRV$nclKXm|Bx=>?z#ZNJONS`TH4)gU>Wc6Ok7g1td&!U z1B1lA*$CsOK!&l??B~VIiTpm0SGFN0T0<-vMwGZ4%HJ0jv$i$2m?~#~j{QofuChe~ z(OSjmn}i`EML6J)2*-B?Lv(z|^7`_aBr3OHm}M~!T|;|n+8EJ1-ic5pfZoZ$P`=!x`IGtnDrF~!3MSEMFB&irKk3k;yXwGJTZ=s~tWI}=`dbb=PKNPveDR7T z+6|d}ff+gP;ib5_CN%;i!!7+!dq)M5_2)=E*K&yi!eqJXiaS~eeZfovE+;)Y5ZXhq zQdUBVDNZma-AdNyo>-%YKVrzMSVr{O-5}(YQiSl(Ld`taGY?a1qENx>Bc-bT2>tXw zIH5Ro<3QWb0zit6Dq$GbLYeaq=?v&W^*MiY`<^aS3(2GJgHuwUfuAT4TvwqXuTfCB zts_VMXlO{s$rc5rNn#x;vWPSI=}NvKfPf{@RQRE(D>QNG0DEeB6~-|re#_R10R79e zu*OS?9SK|#R096_lZ3`rVm(c?3pVQFhY1_9oCh8nJjkG=Cab$cYNtTn;!nM0X*(QP zfU;S#2$%oy6NvG#SLL&ccv1G-@VZaC`p4{d|wo)bSbTuhNN}4Ey3`6viOq)L( z?j-ao+xN<3^xcfANlW}~Dh@dN%~1X}hkA6%Ar-U?limyrBzQO)=|q^@bqk)|8-vC6 zfD@NEix32exdmobe2y27%ie@oPvg|OZSDC}v#qM5x#gDu(rLVh$Hos&QAA7FO%}LY z!unm^81EQoQ*l&=!R-zY_PT7XNmp0<>hT+jZVShofZg*il|Ka0gIo$6KKdw9Pgo0s znHrJwzuk_TbB~zoyd7#Hp+%DCn9l)7|FK35MWy! z3|xhkDSh&e2SqJ1yU3T1eGJ~}{c~OPi;kC+`F!kn=-y{NZP1Wy!1D}Utw%`gBP?K= zW5Z?)i3KBZ!FkvR{$vWQQt&%y#P10wEOdMNz>(lVdtdXe)u=+YH1`JCQHfC(INF1Q75*?EQsk{zsuYdu;ZSnYfD2ZK%T4}^B+Wiwxl%)rpX#Y38ndh}^@ z5F8)or7~v5-=$9zux;(S{($U>TRIo8AAjI;Ll|>-AYpN>R-N=ztrb0JQcofvg(7td zPnp%07Tz@At5>}DguStq2LYgFx*iwNrVPD&2 zMgD#W3+5w27)bo(*A;feo1v1}B%{EHRP`VV2d++z?M&n|69>cRBzp#>pMeB@qLQT> z6O{Oy)m-KT(h*JZY>`cTX)!wHCs2&WeM+$7I0Sw_q7r&Izg~J3EMGwBkscNraD<A`>R#7z=sIkYW1)SVnxaCM#Q^u>kDNtJTx+>TOPA=p!1+)iv4oT24)<$n zZy8~4%KZwct#CS1XREKVG=^CK5AALqt%wCY`+l!-c_@^rhG`+;h*jHH$m5h#`H{SD zYU$?u_mI^UbKuXdT~J33^PFg&6aLOWvuNLsp1zb`!9zw0h+M-gMaT02fdLbjc;sE_ zuh0HbHED?}ES>2j%>OL`aLxUaqWAnic1=nI9M{rk`Vv0tN`6A4V%F?(%&*?Rk1%ag zamzK5mFFWsS9N$N7!akX)D_qk=nS${a?V)GU)G&_IJ$^3Vh&_IA6w6#qMxwY(%g6d zx-!ZtUB4wxcD%op&TO}epENG#e5MfpsM_sW9kOwsp6SBl|RkYzBYxNdw&UGJ=7 zV}(1)hdRzpM#(Q)*Efq1*QK7fXX24CY@uO-q)D1X!hh;Z!fG+jAMFt9%zMx<+&(FA zc-Wxk_OvEr<|Yh-`=|X4HiL?3-EW2%5Kp~E%d+N+;7EaM>SUY;yFk-N*u=Arscufwl7@ z4wlO1O~SR@Z9A)ScMxcd&4EmGDMtOd)lba!q#pW-O1b%afI%)gz8K4`zh zOn4-CdOh~R%Y8D}=;g|4#Lr;`%8f~V3=3QY#gCJM(pCZ5#?mM5JnG;^Hl+}!u#b!T zJh2K}jCiK!mUFuASHTxFCk(`CXHf!9e{%W^AQG%xB=f}r{#Dy6_MB73f0pXumbnm2 zDPnl)6G`|RLy4!4H|GYuVuJL@kM|RI@%c{BMjs1#e}AS~$POlg2sQ!NT21JmGVuaf zX?3T(`uHo=olO=66aog-y1)_-a(#=C@hNTRuS^tXI!5*WCuv<+C~1jssZaQxIA-~} zbrkgDr;WgIf6=WvN_gSrO0jZ?pPJPke5u__a}dE7=jqizTMI8&bLmD}jVpv!_m*Kz zy)iQqr{!qNaxT6t@ZaQYm{UUm5b(!Afx}k0 zR;8JXpznq%M0BI}cUMn|G7i~3H6*M_jB9AqA?_;*(yFu#;@9?~C1gx{G%j`_HqC4*zZBlLIBNPIzK@ajW=0V=?dtx&HRphU zwgu?Y#{rn8bbfACp;^QH@hW|#%ZDE|UOWe|7;6ML?~Ap1;rm|7*nchvFkk(@#K+Qw za7xdU#vq&29im7}LfI-!dg|yl8s)-@caD{*#;OF!3d2t zdOGj6uW}^Em1rw$!S(GTx4gR@!GMvn;nA#c)M&8wf49AVWIeyW!La@lNV579v=tFm z06M|_x!u)T6OUcLH6dF33fs_c7r&-^PuJghE&gz&8Fg2{s`_0&*02-D5Y{kf92EHL z8pt4Qtr2dAIGdKSPEqK;YxcgigqI0g#ow}vO}^O2t67j)v*>?*Z-5!9vgTErVn#97 zxj@|bHKw2h7w}2RXQmdY(R7k6B&f#kP3sSx-4v>`@EK_tMplCpy27!S$-m&uW2P-S zPR#BIyt8$r;`V`ca^&v_xaleHY)NU=m+*csT{9SE_`i}X17nJovrjpggXmTL$rhzV z;6EI}=y=gypG0IYc2NFQB*J@)27F1tP|P7>G1*ExI22f4}?)Ub74% z@0$UqVoBWf;9f|G}Zq%vb6-_t=)dd(R0U&F%{Vo~Oza1QTSX!vI=1Xdb* z#Z{tZyAfGwhUf zG^GoD7OVVJZF1OPj&^f<`TXSx$1=x@E))WAh#3e_%^E)2G#`;A(i-(Y-69yvRTI=% zEZ8(H;vu1Ua*MRstxQb6KKBS%&r%Vv8V>mNJRt6By!-wMvknQ0a`Q3t5~08hM=S5w z1DuB`NJ}|MKyYpdV=!vNmFmbEIo36pClB3>bhJWj()KU*bvvHA=#Jp;nQeKC7KNM* zhiIeEVk00}hzS#|8piAILcr8m#(6k5ijDBF>z`~&x>zjR4YfYzcSl6hH7d0KRh1%8 zF<#9vU+A0>Hc=QQ5Qn!3$OMXfme_{kb9*ba9A`Jyh7T>XN8?C^GkT0*F-0FJfB|M% z&iNYf6u{^&6*o8YUZOpdb)P4Xnt{O#b!MYD4y{{Ir7XT!K-w0nBx)(31;WA}fYNw3 z1fvKL4YVb%jdqa zROU-Ra)Zj*wY70T?+}Oe?Qe%iqsibg;Dn&6ipi`^Gu?5p9gVGVF{j9WwU_0%{pc^` zqJ%B=p85IXu z_?0LV1M*>X0EW0dUaSYQNkyii8$EBLy?Gy%0&p7cU?{A9Zl!-ekRXrk&aMm z@5_x{oLtmQyD@1R0wrcn%fojkb+&TvHGI#M_mZdH73B*f(B`OQee80HKzWD&8Bv{=YG|wKv1%W75gs$cI=lkx6U#nT0tNbi4pb=`DdXmS6cWG5iGO$+Fr{YV zYPRXiz*MX^PTu=@dz`CerX}yVm^H5S+qcjv|L*^R4X4RS9YW*the~(4JtD199P(#L z>T;>4;N}?lU55vi2s@(#BH!8w=J`x+21LL8+*EqH&&Z2U%CM@~TeOh5Ex}Tm>-=3R;ys~k=xA4h9Kz$r$ zv)h9|+*hs9(C|DrU!PaHI>N8nTN6z8$ra1hc7l-4fDlj2Vmt_lt^L4+&whR4%3QJS zn|aq1{)KxZ;&TYCv44)YQJ;5YQ3!Md#r>gt&r3R-1E z67_hnuL_9cS~=sCL{Fg!=qqp1e^3W1i-JDue86;kdFKnu)*UNIJa}w3YYj5Qf!3j8 zGGlv!eRN1y&39&=PC71C`WcaT*S+sz?VgOkZCHO@R!BIo3VWr_safIKV-DvoGuh~!59e53sjo*<93;dhl)@kzO{Q1Y1 zd&Mqp{ci^_R#nOpi|u(^-N;YbiSK?Hw)@lcz1j8?w4H4TT);q-h+1=*Q^-JtCe_EG zWX>?zeey^9(8V52ohGtiD_5!5Iie3sla10K{^Z-BtDzh9*@S5X>c5lj=T8cXIS|4p z4hRLX0eMiOg)dAMG0282W(veb6uRE62aDz#p!>`Z&D6)7>X0E3SyLC9u*e#%6F`iR z?q__{_hpQg@T5WC=jwk)X4)?^k~W^-#J3;dYs4&fIrIETcTUJI^CPq%vL?DNUTj}s z=FfH?8gPo?FzInefmKhLa#Fzz^Ni|Wqhoz14cup_ymS-*Ac;`Qii1~&a3s{x6E~;lL7fZ1e4d5XG2L(1Ccxq)pk^pQciB;s!67B|VIOCla8k!_u zwpRs7`skq!#m*fX!*j^Dt0{IPp32D4-ZKV<1`7Nh&fFN>$`OWBh2^fg`@4an(jsbeGl_SfB+{lp)jh((7!juHw70sZF;2rv_Xm}skNAf!WJ5lH6bzRxgSuJb3|b$-9(7EYqa z7^;DxK)c=0TpjD&CQ@XSw&*R5@D=JNyM2q95|$l2Y#jinLmbJs7r~Z5WK2MZ#;G1M zwrACKV8dmpAMsr%%c!5o+wm32d%;%iM9Si7(;Ox^Qr0~NJ@$Gs?|#0Wl9fmNJY}`b zlI7Na&8R4Jp!+6n*QjZ>yk!{_q`R{Nwd}+SJla8JpkJ z6Cf>>M?Bc!?c)>vu}TFMT^NWF2J+FR_%M+QWAnRsgE6|Oxxb?XC7>8o2gZFB8cr16 z)xSz#4%IFh$*o;+z^BXx8%l%c2nxj;~G$QYZSm_qLhPeoMXS zE=EP@Dr_h(z-D3ERj3f{)ld-GDAXuFx##8|;&8tMmt=D$=A3pMeEvy1?7`vf*~weD z+vzGw$Vd|!wqEVTf`bDHo|_{@q8d^L*6`4Vq)&tu*d%sg zbH@0+=(T5k(0I#}jQR=DJusQi3ARmh;2?eO1bXfR0>A?^0lWhzaSkCMk?H<>u9q8n z@vYj|rvN8_!H5%v-KL)dFvqwpF}^%xZw$JT4#g4=Cj&L-{8>KP-ilzJ$bqQ#Pyizi zy4mhvgUB({H0$>FABqvs_}_mmTaxrsw7A$e*#;Z1B?#IeJq}vVde3UL;+AR{)Tac6 zpPOukmLAyI1~bT*HH@mHaLOB3rBH8Z8Nrh19D~2*WQxO1R>xMQmfH9J z>**51oPTZIU0zbJ1uqLxKYKqjqn)s9d-hZ3$V_{cXSyqtb66`>+0B3z8P&^Ml0BL= zL26?y<16G;d;_NVCCXGt=N*|eZ(nYT?YQ|qz>PU${Vr^n`zp%U*(-{#EU2(bDcSIZ z!lN{b+RpV29Wj_)kYUn!xl)4I6!Z-y5INuxrc%3B2J}m+=LsK|UKucKZ0_u|TS?d# zf0?NTjsctPY(Lk%QK3U|g{7AK#C%oAn|oOSA^VR43Qqf^9(C@ML@!DfBiq(R?rcQH zzQH|nBSFhvI;XYTW_%E%K9~ZK#|w=v`-?~$W_GZ!m(YTC+Kk6A>+f}6L<7ce>5`se z2MS5hEUGPFN-IMy#K$mgydK(EP5{67YC#=$OoM2)j!20xhb^aFVj@|`9wYDUP3(TA zy39OXJf~+q*k38|8f7YY*5K9oa;;rXU=vx&U3||Py$c@=W5Qv8(T6{C0BZ-Mo!1gJ zk}0peYbckFJbSe+KkqjM{#u;ZVR67_yZTbZtI>$#sxr5$xOgWEa{Kx(5Yva;$P{fH z$h#*@XZba4He%5THr1&&K^#x-iNQ+p6%*f)>9Sd1O3?sP-rGIg8vTy>NR~d>0-E^$ z4B>BIbft!-H!T~L&ueLmc~;+RF}KPpm!9^M7w`U1#I8}T^QP_azY_iC=7_aaAvZ0% zyEY>^c}AfN41I%PNNp@~0fEvxR+b)1T6JDX+qvY$dyQnhpQ;mL-FLS}nEoUqRMEB)Q%euh)YPX;sDFa# z!IV>^WRZ|RKt)>{%XAlytihMSlS*eFX>(KeO&AtGfdfI~fp8Ux)FqIXXI$U%fODWHromeswL5x#4aybj3HweWBvYDs4FVtS~JCF(1@O zz{w{u<4X@QIJnW`Zqi{X#HBw*U<=vrT%61VNR?=U8>3uZRC9fCC@067~+z>sX;Cs zd%R=tNYj(j+swyrCb7KJb`p<=mo%SQ$gL3ae(lhh zAy6rxJV_dWIwV#oq$n_p%p&au&z}QR?94K}ze=fU;o1f>Bc?9Tl=a~dWvsJZe$>D_ zHWHwMTfTyoVp77B0B@7=Cf6K`$?k<{zFX=VtZ{s$ZH~2-I4-J7J&x1C{1QEQ^;+t5 zo#XP0B?PbP^;U@a4Ul4!! z9kM_-$zAu)^5Vp8bDGLr-S~7Dfx7}hgyKL0N4G*Qrby049Tw_vq-@*+l~5kId7V8) zEPQnQ?19h{gbt^>SBatlc(g!lhIeaGoR@|My%2QVYaDF|7P&rFIWY3OS238#%tKO` zech_G?s!4^xUYlc?}j{BN=`?%Y=G%zVO7=$%qvM?L*#VH+*FQ@7C&<@H`pZV~iYsKa3L(S_I}^=P=be4KAcvEYf+ zBa(;8?zoysRESPxH3NSE*w9@=K3;}J&lmM+WU^d*0S&e4aG4(MIZg4uTtW#bnw2Jf z2@{>r?=RYKkR{7*=~gdz;n%bH_nsFRuFWwM&2)t73T!mMT8ha8#_>8#8Lx+1Y96sQ z8jo76ZSRwE3{j-yeM0lP&R;MxdMzGKuph^gb@JZc1TZZsq7gYILyX7~;H;s5q{UZ$ z39}}6=O?Eirb%mLDOf^;iPI}#hIX2x6qQGAMr-vS6cmT$)0Z9n&r{Qa;|w*Ruc+sR z4kG8>{ib^JvUx1s)q&-Yl#jPuF+~tw49uGvMAb~_@ML`aWqfvPM2L;jU8xH$r5p%&ZRbf14bks6V521yN4*O-Km z$oGu`Z8RqVJIwulGu}}ukM%s}BbIDNUYL!2;q?)*wcr~{2-c*?7r@%wxvdCKPr*GF z7*!&}Q&ON%(XnRqrr^?DEt=Xgo-xtpPrHkGd!Px_NKJ*>7!^QD7SdGD*B-F?7Hj@L!rkD=nl>RiVNb>2QwGG z;U$xLG+_uz0Y!A|GMekXYMirn<{S; z*0E*>x@hxdY>Hw}J-je%iZWEwIqV|i7%xlV5DAp;0{FL%SzykVUD2LT`{O zQN!dNeMX3cQA9scDg{Lpk%=Znv2+_*_x^SnP2#oaGzY}!8%~E}b1-Oy7fj>d68eLE zR)!*y0W+m6?5X!d9w!B0j#us3@Jbfai(u|OuC;D%yJ;Wp%q@8J*-?$BSOO%4w^+6S zSHOV13Xw~JZd>~5({7mi6M(3q{64_h_XcKJQ|7we6E}ZvnN0l}o_7Em8{*-|Zvb_w zokS*h#sfg#N-$OigS58EpLd!Etxw!Lsy%^aKeKYqUqn;!*^T?K5^d{Q|9dtmrApFPvIX&zh^w%8 z*E3>Y3keq=g$WaH24KNBCG5jFUm90Ds2;NG&sXc0O;6^N#4BuL#`U~bga^XK@sYj! z-t$C7AFBnT^@K*g?WFk_4v+dgpCnPl2u3J}_N(@PQ(G(~M?(A{2i5=|cdg$6usIC~ zRy|~1EN)7Xt?;x#6-Mo0cbY-uQPH(Hva%$GFWJu2Tg z5Y>s=VdBG&6y+@+(JyWN^q%PPWNUsU@>@ennnV=tCK$&ahACNJGr!nghoN%_!7A-n z{P~kx8Wb4^*CZYp@4uz!`$QdmIPv6J6P*NSiYA2dj=d+2*rR4|U;KgK?RNS2K#9z$ z_gFD#7JL1n?T5?yBi53qBiu5js<>Y6KUIjm8Xc80Fi3yUXA`N;4$}D3HYFqE|IpNda+xDQ{J2^qwm1wv)d zsQxrimYyfFT*^=F`mA2VD;S^7Jm6uc_rgz{fA5W@vJ>@1kHG#oEK5 zwHd?TnD^>$;+D#8($s4vYg=^U7;a-3wj%tzTckUOiPOXo4ie7 zAdRM09xhCb&Qyry6l?N@{GWMj65Ot*z_TsCIT|?f9fat^HDVsNnV)324n-WQ6%B-e zWO_w)>r5JE$p&lC>F*?Cm~(;kl`AN5h@H_T%vzi5MhpmWpn!KDnExVwezEQYa>4J+|rVQ z5L~8&iH2Qt{vGqr09oi%_V*|1(VR$!@(LYeRlc6QsI@lfoRFw^W;P=h5Y+Q<{Sl!q0jYb5a6z5#V1oGf14P-Vf7gHpMsu4fBJ{NZaKYN3V( zmVGKXP~Lvpc!W3n^3PJFU1dHK`;qJWZU!<%^5y*6UH;huG$S{v1@I3p{98{LTUQ zQQlt#AI5tD3J_U{1c1A0cjb8ch1QV)EpHN02$pnnL^-{x@!X43=Up#xiVw3u&ZMg7 z3HYn;hm8{dPht&Wi0MD0c`|1OU$Kn3t7*Nqm(LfdET(Yb?QHoRB#Q{En zM5l7w9(a61RwV3F8;<@X%DJ7lq}53Z8muXTN8hHedtUc%WAf3M6up*EJDcpdyU(#? zIsF&=r@gJR@oJq(`mcn$2gSu3-fi4(a9(||tu;hYG>uUeJI)U&CnR(&KhvGM?|HiK z+}arhcWo%)nDR7PaPAtes%8w=YYrLN@c!xC8+<<7drUocYWVy~#1FYg-(N7mQNgXUXt?_z2++$QZI>Q9a(f@L5^0l2C?6(>J zrEG$2v2SAa$^VFRW@pz5^vm+taN8Uue$K=$)1sM^<3DE`OZeZ7Hft_m|FR0c!v9<8 zqx0{#5FHtJTHF4V64l$BI`yE5*G56=uY)e>8n{H<2oKgO9ypLZ$?uG>S6)9j*6Aa7 z8RC_eC;$C?A-?Zy&C$if0pjyqo;_M`FJm9x6H@Y|>ZU z25a|k3ro{yr+Nq4^Q|Xi(HR-<%05I@arYMfT<@#p50&4l0E=w&-~aXAHAYNfz-Jn$ zm$aRqK<1;frp5#K1(n8_Rwk$OnU`u@mmpLMCPKzfNmdG?2Ldv-P3tZ{+ON|nQb>4> z;j*jWLs*^qva58z$Or=ZuYPq-;!pOlve>dwcX#qiIRzm2ShXYp2B@ld^z>iXcc)J1 z!;Du0ssSU=%CRQ1-yJ^7^E_>+5qDGE^w__qDD@G69x}`%2HgXI!OHusf}xCi1|s1a zj-lv^e3V7#gQDC29U;e=*H#|p2uiYj@#YCsDKSp{`b-2(KuLx@VWReiM{U1`5^qHo zc_7 zEA*%cCTQD{OV1=D6d@7Xc-pr1xCLLEr|X=jJg5C|kpm-(0iVy|cn?DyeC?3hz5z=N z_aCt<&6QdrJWOzZie@7fn%bd_rDa-Vt4D^8iUjB>0{n3eqlf=oTVmtc*csI0@*fAP zwA-vs19wyjj5WK~^$~6+%GJR3cBrZkLHZTVLgjsMB~^vY=12yqXx8#7>%~=ASc|KY zw9}HF-Jv$i1w{-Ml_k~il+B?DvW`I-DJdkv)WmfN7TAG~(q8^M46=`V7 zEz4I@x~5oZ`iEsNS==Ec%T2AZwF6Y)L$N zV&s}&xF)fSJ5A~!Sf*!AH4;c!3x4l^sJc85cw9hX(${&Zq$l!y0hquT>A~(+y0XG; z?1O+%Op8g)?UM8xIkTXQ<_8{x+BuLBV#c!+pk-rb3VqR5X{u)Uo`RFrQ9d39rzN7?&>V$~KPEKE&avnpj{Aok1pwfZgKouQ)V+S9o-9jl=0j2pTlO&1FYlN?)1p zW!oeOC_NX#R5I?gEmdF?6G81_YC7&&?Bt zXcpStl<@Whc1?SR7qks~ow!S97Ps?*i3b1bd0jWBdmbe|d+n?&NE>2c24itbf^i1F zp%JC1%AA!`*@p}`iQr+qi;OEMy8WorpY2V=|Mx+CXbi?|04m<`{R``c(&paRwP6l- zeX{o{2P;9uqt;Y|jE46SBcVbRk zEFC$tbhVJ~e!dj`12#>FqhSf}yu2VH(C3WDtr3h{SN6keNk3Ryo$kG(5{gJ7`RUD- zW!L1AII8Vjm*~e%pjd%TVG>+n6GPG-Nu#7;z4ANMiK)!d`@2rz6>F0Pmz!#d1Xx+; zur!21wd+H?{D7hkDq}jlnZd$)dn@vd&=ed$xepl#3etF$@23D8H#UKqV4Ly?=q`h! zZ)WbxJ?@PUYg2|O*+)~nn%sUjQ>71Z*ohfq^gS>1F~?(?%=qOCV*`+a&QB$50>Ogx z*;8$nY}8;rl>F*OQh?^Bpw5cV zaC3uO#_9Vo-!r+w?e|SiL&u$;|Lx@cQ1+#@n9J>F64842B{wV~Dwg}kr()UGfBTq? zZ7df*Zkk(dm*?3&G%n!Lyvgd$oAMxMy-P)bWf{mZFzwRbqWv6S>xkAT%vby$iDCRX zf=1gK7)#G48lQ4!$wB5rx-es4z_Ih&w_o#p$#okOe_7)=?RYD9U~x2;7;7Iqe>r z+0sVI%x!=2x~a*%=!=U;`$6<^f59kbU~=axsU~}1)i3+}h1uT_scus=`#!9XiYa5T zlEo6uEZ~j{uI{USY`=h~_)cBCCL^r(xj*)K`(4n#DJe1^5YZ)DPAv1elBltzfeMI1 zyHQ$A+g^UYX6Sz5e{)`{6c8L-W>N2na~*ql<%@ zU2!ATzWR(OO2tUr{v4ulubaOd?W2?EaMT8N(bhY*+Z3-`LdVS~jJ6t<#~PBA|9-RJ?#G)huFOYPV26@^ zh!QnddcPwhRbHypR-1G)^O5Tk3pKIIN}Af#!3>PPG~K;7$;0KTvXPrT>LXAG)nZEB z)18?%3;RKTuan-4HDbcP#Y3o^lR=rUbh^ha`(W*X7Iz+VkQ zHDaeJFAay)U-+aD^W3h4DUUP;z$Bl)Y+zakzL0TZ=O68K+KNrSgXYgvI$e>btapYB zllj4Kc}w;C#Y!@VD|N^Iz_Th(@_H9}{Gm$k3B+ErtfXv~!T0u2Mbw{8MW1#tY)&GP zbJ%c%Ui=Qt(Qp5qs|*FVSX)edMSLt4+^`D}Msf8J%wAXiQ>Mn2D0)gD`+GLV0>!s7 z>D;63D^r*ziCmX3gD?)Q_%rup}s`x}k6SM7!^nx720EAKuwKH!_u$FW)T zAzI+=1(g90<&WPIpsI9*dXd4y!~32<$t_=BpuqnR1omn=&P?|2>=}mNbIr!4{PVfW zp6dt{R}&7!0^5UZXTN4;>^y#;RL_gF!8T?kYJg@DgkTL^QV0h#q~;3Jpe``Xgfugx z3h%kIV2aCk+nMZJfye)`$^e9cxtJ{8u)^sdLnY}gGmZWpbLSocgmr~R)o5rv*|%`q zhIH2NHU4N>80leiw7y4X-{0EPzk)h9soI&J#HjrEG5!#03aY?1X3VQ=l-o#yMQXt> z!?y(+IPzXOoC?PM*5g9Tzterde-0<%faa^$`YRDWzx5~AY;#ra#9wrLyg;ukam;Rsx&X&-CYjN36WC%^PQvur-Wb>Ceo@7Grq1t?%}tn618vYn zri&d1p}x<=8@u0swrG}xw65^ck^5l0L8o6c8cUx)O6BQFlejSvvg8aZW~H0{_$%^t zEx={5PaHaM8CBr}m;0P-t-nq_!|A}1!{NCivX=W#Y{SP0negpwNzn3B4nx{U!$-l5 zwrkqBwzK0OTx=$JJ6rhco2740S`I!I%yh!3*#Vcl-JXw5c+pYG>>}xH_u}K@9VT23 z-Zhx$t`zq!QK(Ot#-(Nh2z6NBbgPGU7&TjO=Efh+Jh~@epnxUZaqYg|+tq&=bH-rj zn_1cGWopdp&0x%{m1)1YarT}c0dKGOYExwMg$$olFz$=!LV&21rbTIWX)FW*6{a6Y zQeZnva2SGQohrLG$sog*Io@VdX}cI`#A1uc_uj!;uMxZxiPKoDZ#uF~E&EQZ(%jCJ zv0ziH(-BEAqWTZJ!u6IcE@gZXCpu(Y5RmSw^T7AGkMZV;vKYz7%jbQ6jS@7RPh|Hv z>bi1c+3jT(^#ai7A*Y{bim^4{(wECIJKaXMy#GDzQ<5e)B&An3=ZmQQx=B6U;OlK! za;GRZ+itoHagZ3^aR;90nCykokVoiOkq>TpGQgSq=_dihDO>!=w{>tZRAA>?aN7=9 zp@f|rLn2Fh$IB4OIh!GMDgtD00+3cTIxcmbiZbjs>C&xQfbH zEnIIQpC2&UN)_U+j=~?pd+r~|d!88yl*kqJmdxV5Gn5lN9g-jHPg(}_^}Npa;F3PS z6AM@{F#~Uh=sm<|Ihwi^2P>+44=)RA{Ty#Uo?gFGoZX}>hSKcAJT3A}wPf%W2Uq$X zuesQ;Dp5c4+RvbUa!;!v>>(qo4-{TLYJu>0F=7WD>k$>a)x5Yvs(Ct0my3Fi+t2S+ z41KwPP=*lWh?FFS;=a++oUw;X+ze4449wT7pb|?CY-LWbCR&S zm*5Mhg%jH=`CT(~nU=@Obg$_sFxb~NY?Y&rj7Xt3!VrMIX~fWT_1|TlzlWWZ>q{EK zT}ebG=euR6O!5gY@)G2SP$~-B;ziTC*RllS?Fe!0bU5z}nT@(WHK`sMZTCgCJ|;4K z{p1*Kq9ppl-nA58KAcsuu9Ex%{M;}PR?8`s@q$yYO*FNu{JaJRvxpiVIMTlvDTMUYH_EM`wH3`J7o14Vpj|A)Up}p4k+G!l8_*9KT~pb2FP@(b~ToI?XpNi1krCQTc;U!QZyh zW8brMj-q_k@JV$gJRa=f3UL!wIcl)SZcBOg#;!;ta)=Zm0++CP3xb*!6=NXEQq3zL zOXvbAf`w&eet)A#7@@#Y!rqw|(<<=>Yi(=O?lSspcV>q#V1r4gSqWiz;Qq(Dp&(`+ zM!|gFnWtfUDhRM(naDGK>F-WYS*Y2bh&tweQB{CD=Y{UKtaf2vf?1gC8);jm4ko6T zf_OujMd{1FF#nYAMytvK#rEy*pKpti1{66R`Zm*XgO*X(=!zQKrUZLG!ZxEqPoX7a zvM#<$f3q0~;SSIJP4VT+V;-wLV9aO5Pfys^*#q+oIHL7reA%;dXx!Pb{@Nmj>xDNQ4u~|%@p8aZ3 z&V7e}W|~oHaz8N%Xk)7H2EoE&@+tG7$*(hKFgr@e^8R_mBlFZ)X08h>f-gmu#>p01 zVon7oPp7jeD3La9Fti-ajfT)wK`v@kZMVD>!nUoY@gYbc1P2tn_VppvVNUEa zGBVA4=zY%LTNMI1lrxfU1zeaZ-j$@IJgX&4?$q zqs3s>ST?|qk$=|*kfr@d$id3vK`kmNfmy8yAKDETCFxd(Ed=qGzL}8e=cEV|e?qmk zv3Y9pf71*_#G7pDydn7a^hz;DFl)tRV}Or9otPB0a6|-3jg`+A8isny1EZ`>LLKG1 zngBIo!yk5b6Pn-1v)#&Sfp)yp-|D#KR!nq^<&xrs`BpGx7D*!l%Q&4HYf)BJz6zAS z!0q$eRJ&%LK9q=l#}R8h>+61hh*Qd>gdkKW@$nPlLz28YepXo3m4#nbzIp~fB4i4P zRcx<`vs{y=+11=ogMnTr5~nCkgay-x4>oFbODBNOwe)-g^LhUyh^ku|W4zf7-<$7w z`O+>uG>UvozAIih-rfX6q3nTAA$x_4v}J)PR+Q5YP*eQu;1ktXKO)<&)*O~LiD8en zV5X8Ff}{)Vw(Iw1p@w9-(wp-pHW{+uxMtLIzkdCO|0pAz^fkxX?4!3uSdQk(lV449 z@vFM}z(eQ9m!sCnXo6K11TqegjBCtf?@(87w@A?OMh73Bpyx%at-r_1$O9IvD9p5w z1C8@ANf>_fj;J>l!!VC2QAAnnRor;P9T6ru7em-SH&X7`A6b0v%!fVDPa6-NN=8N~ z7Zti7_0FWP)QKaPnJFnL%Va05iZAzPI0EMMD!hQ-W82FsrgrXytTfk49d5$3AsL_7 z+MXs2fw1LwCEXTxF5&O!yo80xVi+9QWalF4U zal3695?2@!#p!E4s*X+WAzUn_mR~@=Y+~C!2ZZ#hI%|ym`b3D7)U=E`b37F2%$wPK z+>!J3#5euyla@?kprCvP2WpNLa@-cICwG?(QE@rbucIe>-QRzX*!E#5?`TPlzQGJc z+GiW>>zwk;8SYr6VTrC_lM1(0J}7G##@V71s|C-F&&1HykRNefmboX1K&!%xhRU1B zmXl9}BKArZKiw?X?|!-n9BT=FfTn$8mDGoXG8!I>3 ztp}EsJ-!9Vhb^#2rKej@X7V65C+l}@et;q}fkSvN_P>AnEI!dtv{tI>sP?7J6|3S@ zS2fql((r(GAoQ!SRvUTTuE~KP##l=$D#Yj5@BJQ)4g{YW)(kIbVUUx>#jB!4Fu;Qm z6|*C5w^~K~^`9)h?Ny=|3By5eU!m}|#5E8x?BV6hG3QY#(+xjLef$h<&iN9p-+hJ7 zY<$i(PCG?rPA+nXTFqE(d_y_OVf6_2NyivI>72~srhiqR$1O7fJ3*d?-w`%D5u2i9SR(tVBV{Vt(SY57Ce(_S8wwx#uSXNqBg z5PY~9#I#2qN=mZVkdnGXo34pu7I-xwp^pXIiyoiA7dzU2=Y1uHz|=$vz!$~}l(kBb z59#sWMmHo**du0Q`e-HHKZYGUy=(oT+K2zZ>2b=G2Nl-OKS?fDz6r)UO*x&ee#0^Q zsEeO#v@38;_)%K+?7iPU(_p(R$+|8(dAm8gk31`FA=;Z%_CL7*4V|jtLzPK16AGVGaSfOo0xQt9aGbNJa` zWvs@rF(edO!${~-9R5D&uFte0Bv_SHp08+ny4Y1woj;XTFL;4*oOhrf6YcoSTD0FD z(WG{rqT^(xqha0`|L*PCN19@$soZx9I&{X3%r2W2@msoY({rQ&jhk=k3jqIdyvuA3JTURQU9C14@EoY4-DxL`O=2+UqnAY=#5e#Hc3o65@7NtyIH#0(TSseQ~ z9<{~!>u+y?M0cx@dV=8CU)IB|-^zAa3Yt4_EuPH^rZ2}}_%u2vqpu!VD-%*2|8#T4 zd#--jve_gWWJ?ZU;P=D`)1p_LjsGsn%p`o5diR&iZ~$g@xgPbT$9}tN)P6m z4JNURj}8l9`}h%8M*%xGn19MW!hL0m3LRbs4YPFUdDX<^^lPCX>vpqSPDEtjm$vK9 zah1SChlF7%ullL~2wi#1E~0L!13fwh1+AKD@ADo(v8Sj+3f={rY`qH#fHv zEZ5Y)$c9GMh5M(J`pDQ=-{}oXe1q)eMnE zMMZq5npw#-gpnxW;+LNMdxwYl-W}q-MyTQ9=3pN9LvNjnkxB719gamN{c~udk;C zc;kjATG$mL4-P21*RdO{=W$nStZXftjqHwA2O7@2=@{n{I?kF+MrteMdFmH_>$JVK zwcx}ICCu}PC?}qpUZ3znLPAnUh^nfhHQE_8%zS=eo+N@XBu|j0sk+UuO!TBrb~6*` zA|}%NFxBOa4%)mJVpp(To&1>tz^s;Jg|%3@Dn$_^k_cq3LlxQA_fMg5FMQ2NAZ7Pf;zm z8}VbWui&oh1Vr18siv&qi>E+!@L)@J*}q7)V>-^q(&_2Cj2sGM^&Upb$|oPJY(#~N zBlT;`$#c6`U$xXARA!f!U{rQ)o?sw?w2~!UtfOScN~z(1K9;11+G*RrQrlx{eHSK{ zCPkS__xV&-WWLf=Qwn=X%SZrxxzc{T5N|hKLp(B44jH_6+i^io+>4(uA|fKzig6pw z#VU;w^aT0nqYqRxWp#O*xO}+}Z+duNso{Co{DY zPgjDR%^+I(Nn7tE|Bpp1?ySr02X3uK;v=PMx0#qHE5Fz(A@XSO8b-1^k=g zLnA&RkQbIo+DMSZ*KYHDSAr=c$l$fgk5oE*NAL&-_i5UBFIBbL%>CRW92*6^WjXhRn=gupQ!-i90K;rS$Z$rBwBK^dgFr6z#nE3nRsmT>yl}lr7e1Q?L z%=!yP609~m1^oQKi-jT80_8Q$$1`P$XH~!hKDD`@o6ZeJX)Ch_BLjxwk=;FvZDJTCG+R%vp^8Dyfb&WbOwx=~AN`U89|< zu1c?g|AT{8mpg;aV&w>RUlU!i5w&K`P~uu@UTvp_yNcc` z4cIHBaldSu?B%mKqrM9XGh>^crO*Fet_#~HlVk?`P>pA>GdS%`Ki=P(vfNlF%w%L> zzl>)qFk=JAE1BL8ZLBes&4ZCp;WI>}zF!*9Lcr$vY^A~Ec{~jr6PPQg<$4esU!#M` z#%q|>ToFfEnbT!A*g};-ZGFwY3e%<;plkju;yOHA)hP(}!D0 z@Cp6n!QlN%Qj$d~s~fLaC_-*&O&s7k;Cygvb(_dRh3AZfCz#nI7P|HDQo^e_u_~9s z3@ltA>q4+ykVpzxxp$*UH~)lXp!Qnf-XRRKGiC6m(Xu%;L>MIEcf=J$kA61ZWXKP;2Rh3a~rXQDZs4H-$G zDn>#uX>WN%OwY)u%Y!IB%&##SnyJ&|=`oD$3qz{0Kl?VQfUUI1s0RxXWV_m$G?Bp> zCr$2%A10GP$?}jJ#;plT4jq)#*S=+q$U}}C@Vq{@I9h0+6MEk6N|2{?SgmDL*(a9z zi9PbGC+PC!O)XnsthQ;RwYP9;C|4{@>ur(`hcOI+U{qB}Kw-Ycr5_De|9w5Of5*vu zB3mpbTcAbFBTk)8(-|m;L|=IbWik{~+-{)+-a{OQL}>cf>h?2-IgU|p6u53|*K0!< zGPn|?$zxSmfqizs|44jDJ2kuXr{BT<2LCe@g6V58tQKG$zS{o2Ql9pC4`1agG;B6X4Y0o~FWdnPJmak1nMApv{9E|#<&@n^Ub5W&X+ zM6}wa7X@_kd+G&;#?7C6w&T^`p9eyIU!KtRvdD;&{V@>it68@JU2!=$7fkkEs5({B zVvt`Rrc46{7q^zPZUNDlfN!BSg;sO;ANB^0EZNoRvLzD3<&ps79SaQnRnZ8Yi?%|t zt0&i!`;Gx2p9lJzoXn|`&V5%)9Z?Y(WIc7{-X7UOEBlJWzsI!qoiCJ;$^P8~!O;dv zrxmDDkC(h|^e-cuF84=d^r;-#=UJ=Dx?hEjgrzXRhd5>Wd;b_z?v2j~&>P(1;(b}n zdweaqex_d(g{cq^j1O>mJa5V_(8R-_(;l)B0%bE$8AFJQ6wF-<1K|`wmv5Yz(8uQE zxRAyDQfEC4?JwU44a0s*d&`37M-5D&2lxei>p`MRLq4P*ZTZOY3>0?3fd(+Rh5da= z2-6n$kz)*%zPKe%kRczbm3-5)5{ip=I@!1zXB-X=&{ip#kyuQvaApb|@PvBZFBe@y zPdA6!t){Rb5a{-7F-4V1i4`=8j*ExK*I*-RW{@8{e{E$;BrycI;rf*9632t(GOKRnM>s+NC-^s~I zD!Ucx;Z)AyQ5+e)K_`A6b1FNC6_3l77(crDAWL0MP3^KDP3dyO`@+nPLLr9Y zsL1^K<{F#X@Kdp3mhnfYFTGxEaOc~z@>ex!H+2knEWt^5rq^WPz@Y$YOK?R}V4+Lo z?mRmEYhpA3?{t^HKd0?d_i+N1_WivM0s$uig*u-Ta71SQuGpLI38L3)4c7RrGgt26 z5KSO3x(2#=>=Ak=JG8M*{Wv14@H->`4-j5xhUf`+gZ8vJMuqx9D1WN)8)zjouo;a3 z4f0;B*4Y4C2;d6JNXYnlqH!*NE7hKy?=(V{Q0@o@q3Db4$ zRJ(%;#A|EgX>7$Ux&}%ES8&OU+M*y35AdMc;^qD~L{KihEiV|v%L9?&@hk-b0dL|? zJgWQoQcrFWw04s@BH--=R0YoN?G=4~xR?q@qp}1*0Gs(ZIq=aCcr0$W6ZEsyCX4$_ z9(FDcWq<@zP;k@W9K>NH2-D;9r~!Pc0ku_xG;OatAn32G6OYNDJT2YSbcYUCtLw3- zgaqV%hZl?cU+%m(7+?yE+)g>=vsu$}ULS?Dy8H-6Oj&K5yshZ$XPK93t=LW%fA7}} z5$?*HQBWynZUsGT9xlz800rTL@`s{X9jA#SA|w8_s4o=pu=C{^EoNZ3#RYnoN*)L- z-;bd`h99qXXFBwOG5fyh?;R$s>C%#bF_yx16Oh(#!h(d)KRijmx7_KA2Xt@`7_-}> znIQMGm0WMP_z`lU=b10dt#0s9Q3~U!EI~W*#_#72AIawrYhqRU9ZMbFni{g*rp@w{ z%e7W$7nM3gfkT+i&*N4A<#+51`pXT>*WvIC83G||urRT(vN8(C{5;Uj_aMNL$c$ed zAd^TOy6!Lm8BkmqJsO|Kc~!S4ml`h4G5BHAJ0^?I;}R$;RRb`GW##1{i+S1;CIj={ z;Bb6F*B`)LxxK_m!0T?hDfROBH03`E%-+?@8*uGPM&nNSKCH_rxN#IK0Q5!I)YUEqV1yEsFNMTrI*bfZ_yYq2YFT_k{_l@M zOKlHg@!=3AL(%zCG2V|4{x6yUttQJGr`EVYM1tPC&3w`E!r111ws%~dV-9GbIE;8F zc-TUI;Q#g}06CUu7=@l4nS?%`lz6;usU=xQ0z=?=s{Mt8Ubnky412%K#^ltO&WLDu%8lg{Xpo<u`o^6%B98ehEC z#*3|dvf}XcNp%$B zoJPOl^LhP%O-qq-06pv?;F-S@DU{Pe_QX$ZEw$1FW1<) zduB#R0HS|>!Y^1;ImHSYjeiyv%m?5y1*7b2*@x+0;HGtpce%y$QC6B9lzN_J$aW_RgfMhfCH z82W(Q;kbq|yM*!&@jR_$8EtK>2)Kx)wnH)cF;dJ(Wep8UW^~|8m~*wMd@*u5ad*5&FI66R80I!$_IrGc=<4dtXeo`<=C75vq%6U*LTF*lctbkE5kfv>jTM zq}5{O=ijUK=68G-rp-UaNfT32ZZtPvF@cM&{qohJw5rMiSR=0=AJ_geqPMH}FdxE?CKCaX*Ti5ui%>T|$9DG28i|R7@W~@z7u~>=u^qBWGgD zX>E<9;rhDRQCnRNFcfh~o|5Y7R@S!t>}z@98JBEA{qK zX4*hpAQ?kAw!@`wa`LAXJ5+=nwTB&n*Zo9%b-`}A(bqn8oy1ayG8nUoorXl0aRI`)0Ry5aNEt! zKu_mg8E3_a*!N5GVq;4ffV9|>sH=@MOcx>IDyn)4fD0jus%|u{K>C0~`{Yj?HYLamGZa^WX|3_$!8&C$w?nIKcCyJKVkU)M|2P@V_qhA7y$!)%>H^-dz-h2QX)2 zvhLczV97fzw|Nj1FqD>+^-5FLFpPCjQZ#VPyB~TeE_pJ$t9JjffR0Xaibt#X5dtL!CO1G?hY+QMNzvG8EscvPt*xBNg zn*?NN47SXaeh0Uv_6XladxBHI*>W9KYmqAGKwjgJj0EBi@ouSc;?jt$gjYv`9Mx&v zRXs_+!&_C;Bfyb6Gh9dmJ%r{#$>?&rM-9K+5FiQ?lagjyTr4mwR`q^R}>*M)MGHhvKqC}dK~8|7H@z_oNffOR~R4;_*B z4;*eOb7)j(DFQ6niHMV!uRydbHoqmz;bit0{z;qe{m#xnWN@ER@D43UL>{x*$r}O= zXYj66OpW=Sb#m7mr;@-Mg2`a?{-2Qq{E3}_p%g^xjwtOiJ%Z43*>OhP&M4tB;i)0X zm0ozm!W~Zx-H-*U*md=I-}WU5xLV;JS~a~1u}_VzNo&o#;Uzv2aoPPIM zDU3S_e>ot@gt~Y5HfBH9?@kT_DWN{&>-U-q9((UL$^^qsAKmeROH^5-lPEF^vmZ9y zVDq~hceccQLawXdRV5i$@{I2ZB*DLWua0J+g~tt;_lTKE>7*CJ=uGn z#LAzlvWm+4N=v7?w%2rF6OpKSbAJ^)Pn#?HSQ_&oiBgxOM++en?@`_SO@0BRp_pxC zgWI*xc3w!1{~pX(=B}s|RnR??bU)W3@p*ooIPnD)0P5!O3Hh|=LkY=%5HzW|Hi5^mCF$Hi`JeukQN*^~u zIIOnVWRlT&)zz`EXjE4x<&oIzo-pC!fxa(-0R3Suf#aO8q6J9qCNTnorE|sHj+dVZ z^{9|xF8dRdt~Z|Vf-^Eoe~5DfkioAj2Y??=kC#S8YeGp^Xq1is34xwlZ9jnvg-AF) z4sCuFne#z!lktUS9rj^6jEDpuvCeLFVVtD9Zc1=0ThG1}T0*&`qQbR>-y=Hb^)Lp9 z*B0AkFlrZA!}S-_0l#$`rvaG$e0w%FlP^`*x?Nwvo&wL~bV446MAVyq8XPFHIJUKZ z2dsBRHMMHMQ8OEKukE$w@&%yiayL63@~m+ISZDvnmsYlrz5n`p+*cmhD@v$$iOuJJ z{;RAEPD1(UPiZ1Fkb1QSh~>&we=7KeQa@mmKoaKczI@X|Er}BGD9gVs{zeX%Fpe+ ze7to5BCFU%r|*%rz9I<3BtJ)v8B+%gXea`KWD27mkMo}6!U1W@m@%PV^>@3~CS_IC zps(n%W266Azp>aCg(`Aq&aQ__IS)6zZhX4oPme0d{GP4E8Y^>4l`3rV;Doeh0y(LOnQO#A*7vzZE7MDYApG)Fslbb(tQ_RkP zs26_g$pHvGMxZga<3(_1ZzMq*PsoaA_lnyWpwC|W!|A=AJOB&PS>{w`C(;Mki^)i) z&L3rE!2O1)uHNE$jH<)wAi3z4?RS84!l<|RPBX_1Iw1#sxzJ)NJ2B;9r~BPkJAoxm zdb*|Jr?$&MtOmiW;7d)*cMZx*=qKQEIF~ zNjktbi)QD!<|4`64ih$9cn7^)>o>cp4rE;lbUuBOR4=ei3qH{Z!}3udD*X9|=j#C8 zumA-e%n+9@9GXanxrZ4Wy-S%kIz~0!+pw^X##`Fgj3-8V2^6evw)T~!QtEhLh4}(W z@w=)>oLB2Usv^JXd*V`~_>9lq{!5iRY%*$izf-{}w@=u>H?kf6eI#E zRx9;ZVnFpj71@*1b=8CKn=w|}05Ssnf-Os`iuGRZ|M86T@|lfxYae+jUz5RU@gizt$*xWvBR#WPaJAt@Pum#PxPjBMioI%H^4;$W01bps! zzAx^tb6?)$2nD>Z&n|pYILbBAVul>}KHH?rb!|?G-5j_8U=txF1R#5A4Z$utJ_J{r z?8wmJ;&O77oSaywXcdWr0j;1CkSCQ@RL0}_!iJ}&hIVIYtn>m&ArKJ}(b3V@+`)_M zjkR@k$Hi`l%ukaiY?rD7zEVpAa>v2AZ!CpF=g4+n7)*#LIwqzHAX#+!z5qut8bFu0 zRAHe57Z&I-PO4eUBY+|j3vD!>qM9P*W^aN?_J48#4yF#pB_QcE8L6-ll(n?F$>1@s z0hYVgQ>6(yKwP2U>UN<>%@mW@E6PuZILBT)dys6V41iEz zHl_{~GM1X1KIa<7&d#csPx;0UL~d>k$B_YF0CYTRMw;#X7(rR5<)p97(lX`$q<|LB zE7FOH33hhQ*;1E7`OPV)oFXfgZ{J2MmR$gzN#&ksNvZpy*kH3$K9xNUOW!lQtc>>h z`g%=VNV?Hs^`B;7W@Z*F`t*S4dV0Lt!~`Sq zf<*9RXjeR68DO?*%}!#3y)CIeu@C^HqK_E`V)`viw{@OX>36Cs}eZ zY!+*Dx@0DcyAzMco-gldz;bgw|GUhm5wZghFYzmjQ)X#0DM+>nnKqiX8V)j>668yH zR9~Ahy6`h7V|^G*InNMjnq{?oT7Q80W~-$44G$H4+dRFI+&Uz{eN_B+<7Fm0vP`W0 zwlyiOGy}!l@6htZ%IjO2t)+V^N2jDEjeFy~Lf2v0Y-t)NQ+HHFS*x34edUeo>TNWb zSf@C5J5h0%39f0GB zj9?_TC+K*+jjOft=20_Sm>0=JGNwr9!Mmi4cW4}%@K5rDn8}Ffk!}*z^}Vw6Oub(kU@pk6Qu9f%^dCGs{&XnI>xj3oOKfRL~yP#c@sc1t8BfxZUJgp?c#*L7}8Yqa2$p1F3 ziNoRyA>e%{JAik#14yxdR~q1fBMBfnmjE7EgC6oYm~pO1;V>6;BTc>oaDvds+<+1l zu`L$1ee;z1)J;zs2SYJ1aC^LpHOCGDP%sXsF*gC9oAKESM<%~hV0$2N*1CcLG-r*~ zF4Sc9@Afn9LH3ls&@YF_7TFZIQOqs}@&FL;@Om&GO?*M={+Y(MKF1TcQ=1!Lk^M#s zNLRlA4&`B?3-Hg9I-0H_19Vz*K+O8P)Di@s;hh=#%?b92{ac*5w3{t4@*Ssx@vI<| z)t~qR=|*BO?oAa1YHf6S3@{#=72rS!DN~kJ*LUv=P%RwQXar1mnP^7#nEv1 zFHY2Jbv1gbHN)q3{0=~znNDACjY_Q^AZ6BbXQcRtWZrnXK%_8nH=v_AcxTJI@R%IP zn@^?5k;WMUwE>or2U2THa9B;5!MbA7zVHjXxTFF*7dW9@F;af3T=nWS*>4}sZN+?2WYE=AN&9sv)1l znRRl4>-BJ~0MHu9qL+KBN-XAowc1-#9u}&KYp>Y_yl(e^Gm!+M9fv(1NLyG66^O3o3l1xpGa11EE)DGRdf&-Vpc-y}D3fJnLKz6&W*yVk%G5F7sBm z*(M5ixqWu0J~GeOFLJA^zgx`}quC?X8E%ULOYwN65FU+Eaqmy5>Q7t_I0<#jxmJc4 z2A@pj{2}B>iREUa&q)?jvt1vb{$1W*nY#gK@ngVvc~@oe=^kh&fX?+=lUcmRu~EYz zl=z>}1@#VrNQ1>yD`E`Dx3IS_^0?R-_kJ{zjKY^tutUTXToNWFw3yu+kcf^v0GF_T4Ay5*;TXI~)b8b+DSFpqBIZ_o zTa#c2_^X+tfy^REO&vpFu9}`sJ!nSc;e?uIOx}xan;R*;$n3;raQ8|cL|(NRSyra4 z9c)i&@w}XD*1`fC9aMRW6S7fT@iVhhVwNF1k?|&<8ZIJl>&mm*jt@;WnW0(3VcP9M zyU5DiTDRzwKcgtb&LFO?@HIS<&Y*R3z4RKYtSm6`h$e0J49V1Z7GciS#=kL9Rhc<{ zMBd+Vto4!2mn5Tj#p(;V-l9WKq0Y|1H%5gqfF;wz2@r?~GfWsE&YK(n3@ z$ENN+;)D;R8th&b?6FrqPc-<>F(Uw!$m05*D$rRsg!QkIG|H7b*VZ@!gfXY!84-9G zy}$4N-aN6U1qo>~76vE2e!5ayb9?^E?YquCAy2SsOZ>Z{o`0=L_0Ek$L2Rymyfr18 zZSIsr7%&5xtx4w zuIoq7qY8g+F;gX2nQ65KWY9vfuhv>5jNRVxEprLMrf~IT?Xc=NZyXUW*PEIqjPZpt zsb)0!oJZ~1@-+Ep@4kKm&DJ*culMgaEb5H<77tEQC9w&3Y?R(H#PL0fVYmMp)1M=Yxj+20pF~=K;p*zePq>5vU$`}Dg(QdhJWIt=o(+8K)A(U^xOitTg zYO>Ah)hGM0w&Cf5@5t`Iq2Q0oh&pVG8neMD;uG-uw(+t%3Zhy(%h+~r2|mTxY0I5( z=>IngzNN$>ZKh<#NGdRzXvoC!;p@vmbBcW}wrIYbn-gpF5ablH5CvR?e<&$f&wpv8 zdAd_wWi~0}u6gId7YNNGO(t0!#P$_SnClk*bV&nA!zAiZ_rW21;17a)_GsX)67R5d z0Xbv@Wr!4yrr8^TBmu6bYYC;JI=ZaMygZ3VZbj;}ny?R`WhvL=5_D!G;DP7@fQ7=clv}}lI zX`7llQ|78FFdjjC-(W`J7=)N%+G3JUv-lS0lrRY5o z1??v!xzR(u0*O~R6G|@KC44}qIZ7fj*})Um_IW40b-GajRYr3_EUj!u39^z(1#`DK zG1GOeRtwItO(N_6AI7NI@0!2Vv5{S>zm=9KhkMNmmo{CU%l_&oCst-!w$G!B`NUMY z17t?l1MQL8{sac$ZI{p$$-&mQTPQz6y~rMcp43s)gEQ2d0eQ;w3P#s0s@(153~luP zbYv|{I!RQ8iLZnY&9Y3mihYJwA<+x3z4Ti0`)#A6!z4{7k53OVm;}s5T|&DF#Sy96 zvkkA9`WGs@Nch>Fp<;S}SHyQ=3_}@GT#2janWU)^JaOt0IPS(>sDNVD!WKrb_$lgR zr~k7&`VK#ne8mVb`S1%8Z!J*q`Y>JfHw3Goh1iVB)k39@?fka_3=|=dgM+tF~#W_gERt* zieMVQI?AKb-673#vYcV!^g7%kk-ph`h+qESHNyu_(k(mt2fZi0RA53uj!cGr4kGtE zPJRtdXkQWpuVzwkdn5f`RHd&7NwCg>G}J(n@NnjAMqdg7lHv9@tH_Y<493#r0Sg~6 z48x}GHj^}RG+276UohcuKhTtLhyMg0X+ed1W)>I~R>F3Zr2Rl`2$c6dQNmuHq%?<5 zq-UWez*~(whcMGgbw8;yG2!EkCUe3M(jt%8IMFB6W)2C|LVHKbq8_A7mi~p|THxDF zoLr9*(ank4zbljZX9p<#lWYZ$eA4NYL$vf+|Gi52Z(TMC!$4XMqS(D$55K-xoi;Js zsd#5YG|D?TtaW7Afh<>R1Xx#s|1!r99mY_aQn3&~uS&;Dlh+jXbFNQm6#XsfWnkXN+HIX|SZ;L4;bhCJ?M4uNOE52E6 zWFS?stx^wWWkPN7j}h#bfIRnlS~w#qCYN`W1aE0daM!;ZI0FkmxHwRo?0wUJQ4@*r zB%_Wwv0Wbr__!J&N2ea;`do*|s4jKmjfzPb^H4_5M$96`S0BgL9RkpAMJv(xs+ckR3hX9mQB=!Cxp0S*^TT7tFpC0adol3#Jdc%4oJH^O;B^twdz}hUj9d?fo z3Y{Iu3Damw(&QvRUTi{ilk!2pDy;G#PPc5xIO-2!PwppR!(H_HmW7ALZV$0 zO!WHVXYO}%fjvXnJ0qKLzAn&ybCYxR9!}}1H!*;C_`hF8i6P~DhR|~QIruoi0FNpx zGBGAK4>8L+9l^`&YRH9TyCyvat8_iY=54Hv`m$r6{Zcicrp-HhKkIvm47J7;+x?l! zbJvvQ^Yg;Zq{huhjmPG;6UM{N$0l2`t!mw#{fP4bB33%d_ZqMaMHtu)URaLY!-JGP zR9R8RCyJE#Y++oI5KK{pBN}up-D;D?poSFz zKE9Rpw5i0Y%OnSVu{;pP>VyaaD=Rza0@^K}SdN{8^QKaEH-S5xLF)-UEyA#!LGOj% z9ktCIl=5fMb&J{hpoI3Tv)^x@6)I9!0lIeI8QT|{c=0X7 z`}~>V_=@dm2Db2ZdvXlaa0^HHa-rmM)c%t8cB=%s%RT$qcF9;NXpG>i0jo%qvk$$v z)*Bz;@^t*6A4uZ;evkIO7-C52Vr*GVevQyP0>cn5fI*jCPYTM*bvk&bXY%&4d(|vRl9W=%^j%R8C)Dp-ahA&lJe3Ba-Jp z%X+gtD5q;C=L2L)E9d7&_hqtbZzyJC@HThMiA;ih!avdb+&>A&<2AW12Di6Mqbrg$ zK>tg=ReF)tiocrFJ8J$#974xCI;mtg;~cJR06U55*u4cI)$&43zcw~DZTDY+2)5!F zvq`0@T`72I$Cn_3!}`+5$7QxtYFcM)W^a-)DZ9h>3jz7($0|p&UUCG{id5t69KtQB z49ctFDu=>@CJ)EWXPMWN02drBWtsisV$#Ep2syvzz$(F8;$i~6Xri}$U$rWTk|6r< z}jEOBw*!QxbOXJOzn)U2(a7*$8L8qY7 z5sgr6zQO{g)j#$%h%3P4+b&&v%6LWEjgecD;AIeSB`e)AbS46B?-#G$56D$;1wwzq zeHCTh?HAN#9SjDFjhFl?`g8ciGf{xb$k7YWj5Ipill3hKvAu@G zutKM=ec28D?mUFyW~_BW8h!QWR~3E5szLNdGc+ce{PquIsN_V}*IMt#i6oTgVf-6g zwDvRGrP#vudh8^ZtEAi}FdJap(;HJ@VEFf@t<((OA^&zuloIkPNJ7N@9>{n=w^M-2 z-{lhdMa_6D$F4aRYA%!G*_}S`hQDZmEb;E_BTrfblT1*Y=fM+@Js1%o9)o->S^A-|*~aE8i1+u&!&D2g)XBwYA6uesDBg1`4CsFoxcBZ!E$ z3=@IHUI?rq6w*~z6i2B0AtchO#@Hkb6F4dg<(|(yZJs7#!t};$+wBgQ%VM0@`{jP) z!By`+$DY$oPcy<1uoXk<()O>ee&VMZiY(n6<$Qu5Dc2~}iVoi`)w>4|gUd@ceDX@E z$c-(EnrTkEn*b}$aYZfn_hG1d;LG{AxGgo+5FExG>cu2xfX6UZCXgZsku{Qouj&XG zEa?A%4YACQxN2=>X=8LlbiS|Yf>&~+D&E+@Wxz}U*noBAcZw3cS^qbjP1MLxM!R0= z8(DH{&ux>R@`b;}ZparcOJSQ2@if5Ekfhm?+aUZZ`*Rq=5piM^G|T`B`e}sLCk1J^ zLNZbWHEX}bb6FKE91%;5z)}!G0{;4y1OY8cF+pLz?&yxH<#4+xTq6-JX8DIF`)1r9 z_OoARa3uG#l-sY?*aHiToB98XsJ9G@D_Gigfe<`lAZT!RcXx+if#9ye-66QUdvJGm zf;$0%Yp}rvcRefn-TOOS!w&{#(W_T=b$4~u{kT*IFRj~WDY?_YE6U36eIjLMz*D8J zjl5XwnnXq(a{Ohp33gs2YrCe%(QJrHQJvm8zDqGijfh|VJgocn*>96Ox<=sKc%j$` zg&h`IeA+7(c@#It%a+b5f_F;e_-_o1h1H$CwLd|I_@t!zf>~>K&Gvi-#M+{?;Go^t z+Q&g4Mp-Q+b_QHuRn_kaFRsRu>L-#YWPg^Ppm6cA^2a;?q>AX zinKlS@?=nxT4+pdzSe~>c|V9y7NSICVLspL(}5(E3+)|_QfgQDGx>VFkb698!a(pe z6wNjy@2Sbv!9Bmi$f+?WpI_|!%%3@O!LJUv?|X8e@vQ~r4TIWhTGa^jhdn@>l3z4h~h!=xWR!43Z@b!wL z8;4KX1nABK2+fM!o}9O{i9=5tL|Us4@uv z&B-ch8O?6|4kCg~{?2=`Z?Va5rYLmWRU#c_sN9^Dt@%ZUyF?eV+MTVyp&(8>23jVE z;HFww+HIRyDDXaVlPk=zE6m*zLf^l_lkZNzclIR_cuk7de!8ZumuX^8M5KTGfGgSK zF1#FPD!|O5Dp>VD!tUVfJZ2v4J(KTpDGlcGKD5SI#W`v z6B=H6KiK(lRNvos_ti3pAN)0OlID{gfTcSAIf1~kI^e|mK)+lp!Tuf1)AzLr4}H%I zR$5+c8s@(q)90X^eJ}=rN629do+QMQL_&JX;T)#6fQ4S{$4F9&izFtn5#q2tpG~uZ(1NhN`rD!uHqwDjK9S z4Gqw{w)69+aJO;*(PS(rKLB#3RIa_%R>1vW(B}1k1l@SRd2~0L#P)_CY~{vqJmu21 z1CDVldma%tZ1w^#DyLk@Tz z_+fkR`sN}jvrhfU3#4HK@B}(iL)heryif~`FaAhkLAnD!j=o1kV-z2esQsF9#fCj& z=twbLWE#sEf67I`>JXAzbUe{1CJ=J-nK&it8J}Kx{ZU*wjLG)N>05Wb zTk?C#c>D4IWtj%rmu@LymWN-w5?LWSZqF!wF{kcj{?OM;F`pxjQ`VLFUhCiI|6M{x zmZS>S2mV&fOv|3(-5fJZ(F9f90=vy71#A#BtCr}SFJ`GcAJ5zXyAApJZLMp|RlP-5 zkl6oyoNvp8+)`~f(F~CSBuZNNCtQldM(F2PX`GM;r_taEaCQP~?tpCzYCafN>1oB$ z;y?eAU13%AD}s7{lHMTavI!DZrUbX9V_vik@4X1Z?9%GsA9fqAB$cIb*0D(W1PJb$ zbqaCfAaa3*FDDU{8|Am7Ll|#&N|XOxY7-Fy8m=y1WsGjygdvbwFT{ohzaz+qO_VpC z5^_9BtguRjsN_$sw-TvWL|KgaD}tPub_BNwL)%pl3Xp}f0~-q&c!cU9RsDO2fHQQ5 z4M7=g=1|VH>Yp@=M}tov-%8h`y+6UM2*Z_p78l12ZZu9PRskaD)u>1M--|=+E{D|vkOXEAb$ri$Cc=^L zvKFFA&ru=>a(_rsDQ{q^>Nik$Lj44Txi4*gDFpY0^9?nbqzJ=oajO}0LR(~EWhExN zyvN&-Q`X!e{0+YMKTWn=@jnV|M#b{oTv;hM_vf(DCp8u9@FGEWUKGZ;uK)1go;n`w znNwWwu~b;)8ttpw4u)#A0zDhThaaOh|GV-3yqbFsbar$!U_J=IH}fvd|DTEcvl4Ta z6EmyVDd?Jw#fty^aWQ1Nox%T<7rFBPy<-WFPIHiRU(S5B>H?<$KltCv0*lYrFSK7< z2EM%~O4-o;uj&1}E|u9LQId)@jk2t=j?UUnowRP?b`A`<&*eY=dNmm%z=Sp!?9oAB zp)L!g2Lu@6v@85;R--`P4!%J{Eo?jJp0)_Ynv2g1ZeJ&Ca5_g8fbVDr3>f$uU78{w z8T0KHH=k#5-lqKDEkY0ppliSG2I?QM7F1xs2%c;FbI@`AGim?J1^Ay`<)%^drW#_x z82fY!4>5>6P81FHFClIEzef4sljb@zq9ELL_}szt#}Sxr zs-`BTg#cW@+m55(ji@BaDnW+QIIuy6_kJs96`!0e;3@qL({MsgTr=5Gc0+ym+p~(g z%p=K_+EA(#s&@aJ!j}Xk;S`4(;^2h%KmUe#(~tN$LaA-30Nyh0lq4ZewJj)Oh@O3T z5A@PUUeV8KW_O%w(#9pOEyGg2q8^s8)Umhwl(TIHT+{JqcxlDU%TpP};dSt<%dwI_ z#%2~~MTd~lNOL<%R^^r+z^Gk?#Mi9s98G`U6IF~ZHu@x>MN*wknXv*%E(xP1<{;}; zPD7CM@$b7N!0mjbhskqnY-x_a!EhZAB?gU=&F2iHayBd9^N*ZD7lT0VyD+sAF=`m1 z&Tti?V+8&cAZe0Iwl-1XLh;kmIb!A_*u3^NBFAM6 zh>Z;EmlW%TU*FKj+1--zhPz&k78hstx?%i5B-Pk|e}DS2Hv%n)*W753wjU$HOafAI zKFL0K@I>%uC-=uwaRCmI2sT5jj+?jKs<0LK5PQ{EMi8H*(5KNaMS?;=4+ zT#1DY>Cki$rn{?5Oqry6uuILsKjrYqmu8wlTb|G%mivL=Y8&fC*CsNh(=y2M%17|3 zX82b@!$zjzTo>M#dKYV(Q6mkr)MgqY6H4t>X;W72bQDc0Im$#zY#edx3bym4;O29& zm0MJ6Hwo0c)OzDy;S+ZO7zFPyHOFmDRT`NE<;_zPQB-~0G=g_XV{R9s;ijM0(bsPO z5XGsW!$Dtk@9;b|A{HKT&SV6~*hUvaHhBegicK{!urc$X7G3!6pxnAt6|YZfTzk=A z&{|t55?ditax#@(!J$Mx$&H2yU!t}mxqMS@Qwg`zlVyz1!ZQ^Tuf*i}BVZvVvq)hC^PqyC68c5|YrHH?Www)Sg zW=H~Wldm3f+_+b_U%(Q29_Ly+i#aZ%bVq@r)1>o8ieW!}>4`v|;z!{-20hZGJYV7F zybL??vjvcd>QK!o?^Dl&g(Ha|1s|JTbV6BbXv8C=z%qficD)O6^Qdj(t`t7g8vLUu|QeHpav$l>f_%Vu&RpPNykIg#;XvH3xaRZg6ZyMOssV-)wJ7In&p++Rzmtg(4PV5mCxyMw$sC+~ zk1$7x6-|tAA&?6Cq#D9K>S|IGp5?VO`De2u=)IM894m2Vpay*af>YRlxNX;yO2{(O zFe;P?d4|FG+UCU^mDx@hP6~p(6SO*cM4n?S52_ICTnwmAL5U>iIKL3FI}#58s2y&m zfey}fvUd`cem5RtfE$suLna@7#vK%*K>z*Iemyr&FlMA-8OZQ%KZwn&)_YvXtjgN_ zdRJyZd$Hcx2U%Tw*6ZGcHc+jzqc~hVA7?x$D{&H)a!EPAIarA#EG}y&;rp}_?}fW< z`whaW?;F7a2OVk3Zy!I%4?{|HMBvC<3@5}rM~XK2=px2*=o{ZTqLQHa6Z8HE4ZcsG z@B))ZuF!PZFFGdkwOwP2tSf{uy8(`f11@X+cV5el*1E$(kM!!5L?wC;6oTt+9Bop1 z`4q^-V>7ibbcf6$H))2yH+oYZT~F#|LvB+3*lb8=9!;K$(a|(J{#mp(@v{0zrzb!E z6Fnh1df+jZbf!CFK|Z0lSlt_Q>zUOy(9`C&C79P#!-LO7vWEfAL~lJwCRa3`x4d z1B$X}N1Ch+UxsYNZBls@-x3D(-kByF<7n*G8_JRSZW$-vl(f6CRN;M_!=S`rb;ZZL z`0@C`s!Z;En&FSIR95e6_kmm9ul4Z|1)4v4`n><9g_jZp=x(Y*E@ytKEDw|}?Qiub z9kHs%6s4s(BfA&p@cfQBQVueHk}*VKe>v-Ybs2Dl`U)Eux5snKcEj}r`Yx+*3k~Bo zy3<22uB$(blx3Ta583RVu!ej~O|+~wlyj2K^(Vvm(}K}b6q7NzdR`>ru=xuk*oM7* z5TDl?q`e}bh4*>X9Xcnsp!E$_61zmRrJE2n93i1nyFE_5BVle@=$$cS)BW!{W}}S- zrIxaUUw_d|IJ37B;$#y?p}2lI+-(6pT-7RPgQXQ-U({^NcM$1n#DNR-yH7mdtvF|X z9}x!?nrunK-O-UuG}rlP3rx}^iinNduc{_R6UowdWV_%U)xt%aZvQ|~$mFams>6(5 zAIbH-29+^Dt_gX0H9425bJ0fDD@_%)f+w82^8skRvSm}to7Llzo}P}R>OdCU7#a?U94+8j}dk}9!S^X6O8PbN7(9;(}yXKp{ z2<@Ip%;8(J^$5(;jsCi0u7KZCHGnY_$Ol0s%dwm#r!rId#zYoR?Z5doV{MKnjP9T7 z#%kOsuU$u5;gH~ebwG4^^G< zj7M#}?eJv!>b11ue$$R%ynst*w8L0%dYvm*VM68;L&AjbN$B%di=X-Z#M|HE0tWS_ zYw*GAc-u}4wsP@%+s_r>dGDU@k4^L+7&rBr-dp1(uMW`px#t$!^Q$GIQn7+@eQ(F_ z!nGV?TDS1LZ?aykZNpib{ZfcFeA$|In7~%KxH6nk%IOx{v3EX^LY>CUR~?(ik`HEB zb{D1iyeSDoeh$;Rnu{k69BJJLMO2U%*aXWA0AX+e_hM)C$>? z*gHNzIx-O@@8OmARrx2SD#(DGsGje=;On{M2Pxmzo`*+MReP?g;$MdnGD@-N=m@vB ztCDlK)XO>6<}Z&o|FUR-7=bq&e;2+op9W1ZIu?a%%(5c?#B6@zeIJ+-WI`a%x5+FS zO4y&As$e(@R5n&|J!98!eEeIECn)vQ$(7d7nDR*Bw>?y|F^nAh7CkAFTiMzIklvVd;S7IPZ{<9Cf{C0$#qUoIe>23zo`hZ*a9bHOzo{s$>CI*vV zR?WI2fslJ!4lY}bkxICt*NVt?z4b=LgiY!GpNpc_!9bqJ-|>~6`)Hp-vqqc1U$6RN zZNbHv-P0M{JFUBwK$O{u6s%aah5F`0z~y)4R%!PkdiLxLDgn^QJg#1tH+bU?Ronws{MQ%U%xZ>o4%HC{e$jl+YTe) z;{zN0>XLNs%=|SjOH_9o?*hjVj)Sy*NL2MeRr+-1s-*qRJocyV8?x%2@04z@4ir>+ z-W*gYXzYz%D5amjnBBhUIL*COy8c2T=)4^D-ow)m;v(AB7V>zvSRWtsLxer&CpTI@ z>xW9|sC;pji&7EJ^y0fT!^X}lKHKgaKKeBzmj)uRm)qtquI@{l4V#+%Ke$rFY`FZY zB!OnH-qSW~_f4S)QVM_59DEMBEI)^)a-5Ioa$ObGtPSd$EgGhiP}}2ST~^^$Z*QdH z@tmfyRQle2sqXm8sE(mv^!S%7ZW-gSGEhZu{Oy92P8%4gyT08%chl|<>usGqf}FHqxip^odRUrPQG32hH)cMlO^VLL4snX3vIU|{QW2S@uoVCi>WWb(JLDro*8 z{wfPT{QZ&JjQ(?hU+zYb> z9*)C?^Arz0t@O7ZP0193g$9gks)@B0@bUtl;nk%3h(meKxw~)IS*{6IGAVj`QbYAN zgm1#lmC(agICr#8GbzKb1FGa^qj0#gf_kD;-!2Fl4yoIk$#3I!aFpVI=3mR7`ku2- zSRh8PE~;Q~Kr_1qi zFeB7OM3z-DQq6$vZ9DI%q}d|oS_v#=5#A-mQO{8_HrE-gOBUlh;#0+b8Yy;jukghY z8_Eq3XeF9F&Y|G3KzJuNH{t2RIC;oE`t5BrMXqP!w-{WVh^XAUSLgYz&@#CZh8i6A zA+`CM{U=v5L^oe>4LKP_(q|IiVc1fY@55*^vS{VZxpHLa!>jDM<^XEaYQ-A8ZUdj| zhAm6L2DZ&L_ScKN)*`NkiqT)y4kN;GJa5t&t>)(U0|k4PUa!h}LYI`?mx=lf_sK#R znN;7EXZy}sEA`srJh=>n=Uk5#Y|S1rs&)g=zpvuMX;Q)wgmVUqJ`m=Cq$ya6h*91J zfC@nfd!KOGBSg$#ua?FiHf3NcxTjKXWFkKDBt@8fs+rIgR<+77eREG_D8Y{o&%v1c z$Zy8}xu~=?jTr@CA2hbKxS{$cAL+nV1?oe#ej+@_Nt2G1_Xv(5&z+g>W11(pu2`gO zc@ZX1zTP(Qd6*R_PAIkJiLq)$Fud;uCyceGU0_6lYs@cq6L~f)z%@eYfoUu`pS}o1 z3;GiHX={nu2ld8Kgh^u$bI=@P4qxdwb_?G#>K%Lv$2P>X#DI}=?|eE*MB%?+({VWB zdp;Y-qhXAJMoJR>jO9Bi^e~mp$`@~3 zGnl{`p4L0sE*>N`g{Tx_aMK{PR}EhL6uEqFo3U@rG=vt1EFTd?x#8;uJ;%hI9;qmU zUg44W(6oBUCPJ!m`~c~4tEHz2$6=gY6`;)K!ZgK1L*&O-M2}39MmuVF&jGknQNyiI zl16D&a_-jvul7P^^mm=8_p=i2X zVqAvM2fgHroC@DD>Vz#)+hs!u8#k%OYsb8alN9I?t7wqZmNQaKM^*=kpDJ279$6qM zo!AiXr)ZQ#)=EomvF*X=MAn04%Gw=&j9LUmdv0(=-zg+jX{(Jtf}U55>MTJ$)UtyU7_Xt20aze zw>BC|Nz@1zu9!qPO(P`OKb*!5b~oEJ0iER64L+-8Xl<&$Ap$>@+|a0DAwl5;^V8_r z)m2Z6N(#ti1rJeD&!2#+HlH#kQ*7{)Mo`en>w$#As{W`BW9Zq_y~vN__Kp(>Chz?` zFc1A3y@N@VEOp6Leh=hPe9@{g2k^5$ZEv1BTc&o|zD=7P8Ghp4PjccpF3k(lOIRqB z>QOd74@NF#E4Pc9blMmce0HbA;Ho#7_gtu18A?kmR+?>&!2AF`zWIoqE;wxeGWpH^ zRD#R?hNjE?ZryD;BSuvRpUeoilvo_tTS|sYX}j;4qn^)|s5`AFv^Wiw0@4pZchV4o zEv3T6MVo7~2m6IQ0!9$I?8i-k>ee>}ENjH{*I+B|i_!~x0f_Y!zXy!M6}Y~hXQ+*9 zbBF?*GoHLSX`*rUc_X7CqYhhdvP6+T>q07b%zT;~i4TFLz#%_He8|W;_l0R8EMCX; z0T%1Uhfqey;3`9t-jb+aEmtVFp9+7h(TVZz^@Z|6Rn|DxmFk*3#5Buk)YRK-1y)ue zTN8+HLktQqG~%hzRE;nPm+$w9Jj~L!DJpK6v;02hsOKbG2Td!rSa1II>NNl?D3|kn;CW#&xPBsbQ!hL8p@(z zknh@8XXJfyTC*RQz~G=aoeF?pBx9gOE=gmehmgo>Vx+#O_jz#W_B$gwaCw~6aa(Y= z&wiGHBbZDUMGqeiU#fR9`=ep}E6CNeN4@T|t|``1vQs-R%bR-+X+rJ&bN?uCMc1p4WN!JTLN7 zX};|Hmn~_F=sT56YrUqqJgIVwg3;w+?^TX)4In>^#f>jIGC$*4e*fO*kTvzA#r|V0 zNT5VuQKq)>T?_~T6D0n`>M2cg4DJ~WX_TlgOOdRJI(mh^()WsVamfUz0Avzmv4&#~ zQmmDJQ7aTz8oHak%{DZ&Aw)wZLw80yeeWwvt@5WwWK6K1qf@aN3+APnt$#ytcD`WP zRm$akR&OwAm|ogZ+zmt9t8{-rBlPg(#O(+W8~QriYRwqQ)oeTlYUqr}dnvHa;)>Kf zt0Lz-LHGzctKV9mhi#Tf3ye7Wd!2kUp;w!;(5_~U70B)CjhVUMJy3znsUrAH{e}C~ zLh^%btqa6Ry;A2_0>{lYp$keb?=~p^3qd5fd%Tlkd<2WAudz%ON63iyw0UsCx5wYV z;Nve_t%=GzJdiKjVLn=v0h^-W-8WA2f{|MVdd;@%@=;ap(~U>{V zR2DSCSjMztGavN)zEEk>Fc-N~oIUc_IP}w~^;Q=JUD^7AxCWpYK1>+UVM>9FUDJ!h z86Po(`3$+~Q9Ta1uEq~+dv(dlYWgfzq}go3^HpJMV24U|r9Eu(`8t)p^zKl=8RI2d za>M6XKqW+617HqmDtw<8HS6Ka^;91HV+>dCcq^&}<4qrsvloS< zJDL(~r`)gfMZk)8n1Bt9vwQT6&?6$FHlVKHvBBm6AFE1HMV%+(?~EBxk|I?=YqoR{ zzpFlB_SY;|IrO8!&NEMCsbzlE{9fGXFl(ny-DUPS4?nLMlAU7v0RwBp3cs zAnH>rUr2mlq@`X%QW2ect+x+yAQk@jZhVn-+8t$P1GE*^WI?s=UM%~c_}b>4V+$vV zy)BZ6-ZSoojvYx7YDU^42T6B;8HoF9#a~+ZAl|_-ZH5~yy=T;k)npx*_eBPhf_!5t zcT}W57%nLr5rCh}2vUV{#dlcf%j{KXyxAm`>0!F%c#nx8+!xZXUp=NF+;^y;5C!%W zAujI3Dbf=h7VZ2b;Ri;|#o;gHwTL5yY(Zv0?tkElK>0hBV*2Q@Qt_hkj7k`u4cLV`&|MWdb{BD{8{D=odkRU)LxMJY2WQpX|9 z3~I8Af^M`}^wsnIQ(NcTTuo1k6kk_LmpFsUn<5*;eJDby>4@J0YfosnF?+PQ?xs9j1}?%9LD)2j#b}JJ_HOkLD`G#w`Jkd9~M#mAjibC5$8) z26M{Tr<1S$9+))xi zRN0h`#sl;th4>R1Sz$Eix}lZFBq)&<{2}KA^ZNvLJX1#2A^ple;X(4}dVz7G=; zkx;1YCd!mPTcFjCoBzD(79|cA|9L4_4n^~Um>lKm=@2MC2+4kO4T{rMhpBje;T;oR z48kLrPS@w0zj&H@nuBNsYm$*r1jOXJ(KL+tsUx?z(HAiC_r&SOh#gZ)y?XNHu<0{7 z(-YApi8w#`QvZntq0d}oTmt;AlMGkp-_W(ib#mGXLIFwvNN$ZxM87$aplRtF<=`Ak zIKP4RQ%SL_8QkV5T`Za08ap2L2G>`nPm9j-d^TpVT)m* z0h=#}&0?>7u;HERc{`_f$J)kAXD2gCJCM@v>M5-_`qLBPV~2qG^sk+%#iIp#AA@wa zZr!{-T{fZ9HPERFlQUwa7p?YU`BD=(a1gm7VzBu0sGud97NxyOY<*>Ioxd1-?AEh+ z`mQHu1sns_w5p&>)Keg^{%}jMc+|#ldaFZQczo!ut8cYv{NByh&JB`zVg}_2emV@% z;uAisa{K>}H$_DHPR*@>&HsO25aB%YAHV|k|9)%?3}~ZbB>kuKWilM{MvQ>a52B$6 ziJx}HgM$_swwYxJ<~v#*@;KhwWHHf{Ld#50)93SaG(@u^Xc5T9s3wAHaA-F z;XCudD9eVcJPhzSllLX6#Ag4?nRu#+C4fREIp>dc=Z^t<+;=Ooe+DWLw@m=E#2U#p z$B7&SpbyrZ9R(MdFmvomd3{&>^y7W(lT!vucRoOIE=oVJuUSci;Hbd=`gdQHK|c!% zVf&d80)EI-27)~>Ffk3KNDC8Wyf@EKEC4YlW!d;ivyKmp+9?t^A+6PN{DfznlMeV$ zK#4O8N;ue~P=!VS=ik9Mq(x92ioS?ES=Iq4!*+ZyO}(1SeNOvIJ2;rCBMJ(VCUJib z9~SxL0nUwOCoeWUzs}7quC}FvDl#?Iai5XkfBBYiy|1<5{=z{TPrVADU;mWO zOmOdEaN4d(1BGqK*z{Y`i+cU_5Zk8|m;LWR&qEUHYxYJ#qFe2Z!}wP;Q$6P}-??mx zySlRV-yXjK*rAX^HZzbb3cb+H?x)74CP1N1X!3x?c&XEpt)}~uwW_L0nns2k-TN>E z?|!PKM?p;u6Rd9!&BXu4A6QY54oD7_y)&K#8M_T0l$^HdMEzp6t zMDEH#$Fjn`j}7o!0+*Md%`Z4c0r>G?u+q>T|ci{#St3YH>z5f~YfW@N7T=3N8KB;-#~XbIEZ*hiLuaOVgLz`LmwO zOTFj4X~nCxhUwoGo%o9`>8viaiplvJqPn5yjc8bX_WbXlGNAJzydii2`9)Ii{ zubBRieE3BePWuyoM+zvf&ii``ZLI_%vFM>l(wGfNd)yBUGcq&M*sRpSBXvlav&^jz z&8zM&O^g4&se$s>?SXKBA=y4KRtC&+Rl{Vx|lV)3S~0;u6%Sy{PtWx2&|7j&ie4h6REXHIOPUV$n| zbL=#o=zgZ8%JxPNC2-)b_*T80{7rv<5Ujs-Nmso!QV^4G zwj;SvsW%ynts9h-z@+EGAp$yI?0LatN)X%PMbct#U>;RsCEN`~p0np`l(wdr@U`_;GBIu%`B7ka1BLF}ZMZlHvl~#f@o;Y!|)_sPQQa(f4 zRE>fd(K{C?IAwE~HAKN>4JsN>aqSfGk(HCv1bCKYQiP%a?Ag-Nk|c9-yPc83wCJBX z_jKQr&WnZEPx0g$9?lkJ>$`>+{=>~#SsfET7@s1XG647i)JIf>JVE(669605TJ5E@ zC~J~wFk%EH2;x4!bF zLJ)~4Tj1-iXf|N70HbNy+O;QZenl1O#xR32Iy%~f6<3!iCp0OcxtTLzbQgG@?+tN` zRyhIMh?n=^lwW|#_gL7Ta~7B_3-%$v&j3yvpcwzNYY&O{03Y1+Vq4<*p{%5py)~*o zmK%LFrq*#DiO^#gjj`tscm!bJv|H22(x8mzcU47G5qd?@P|5FI($(z#f+y@{j+&~R z9W_Y+a$1s90RAk|9Q%b&PXoUQ=p^VxVXGE?uXuZkMcR8~L8Dyv!62%LI{bx?9E?c0yy`=;pih%UvT}l3_&5 z)h`5;fh80rywi^Yz%}LwvcWs~U<4U1X%amdL!o$8GrW$-{Y+NryxGs>jtR6`bvor@w)UF=`0b;5yP#+BvGKQd z#`d|Jne5kz6HQSB;B;~h*r)-s0z!XWNC;dCgMJ8R9nT>U`Fp+JLB#Crac9Xx;<ot zD8$zDc%VptK*UMoZM9=>|7@zY@U27SZj?1VEBl_YnwdxL`U^?oRh;1lW)BFh5z^1N z;M$=2Sp{7|olNti-NK7fZvSbx3A{3odzgxduX7V*cp;J{=Ewx-go2y zfC|kviQ{{`u}Wc*1?#rQoy{HUk(AUL813utgXE-_9C685kJgZ@h|? zhybp3?-ig9hj%vyWZT2~V;c<9SEfbdvec4N)+JC=_{hm5N!9IO^~GV_DeMLsyr=z2 zPHR-CO@JlT6*Vc>VjwQK@l>Kj1vDREYPjmdFbF$~Z+}lMJ9wwM_-OI{`>fngzN3Qj z@(5t~tCaouzv0gcU|azQcejeW6{{etbhJYd_{U|prB-N+T|sAY&;h?NDb4Ja2Fc&< zub^&EE3#jZ=)0^l3}HiRmc&tKy-nsSf#z3jzFud`YggA=*DN5vt=5IWf)Tc;*m9fu zi@A~#PUXkIpHAv`fh&K3nDw4xZwzjaGw*s;_Tpw4JT`EnL?EU}r#~K_Xe#IjBecmq z2jfD>=TZh0+ICq9X_X2y8-n~r!=8<`zps|!EJW_qWKVPsV>Q@L!39yoiJkz^a$ip5 zJ|(nAB+Q@(ip3D|k<$j^3rmi0;~ za8Hdv8{`0Nr3Km&n$G^jt>lodZpX>(op=4{MMdm_D(VF;{b*(QGc09E{&6i2-)Zmf zfGjD!mknlKUR;QHFke>*(z_MX%|B<)DW9BSn^@nvuXTwmrR7%(r?(s*V-Ex8_Ema3 z`N~q*X_X$j=2VC$W9B1`q~7YU1gfr|wC6s;<*sQNDLlbqe6IS9iueqL85ATUO-}f$ zQNFTNrbWgedDAP&#tGr;4@2MQE~$U3aMxO30ox(Cr`Cnzx@NG@+2n7^R6WG(O5TXh zN>sKFUvjvFx;BGD$1E?x2&p^2xWnZz=9bdo_Seu%y35g*ow}*Db{dYV{J}iZauNK? z#t@@vzvO(wvzhvhh8F#0=qvzK-Kh96%!J0hGq75Q36_-<7v0)6YUhM2eS)u;f8% zysx5`o9LCLB8Aie2fP<~!qPJR?PeQqtE?&vmX)PV#}-$F^d-X6?Ekhd@NmJR(!bIY zC1C;-u9^+f(EC$lMRN5>38AH`GbBLA9PCSr)Ua zg|t2BAWIh+_H7^xLt>=a_fL|ALuQ!<05s91IJ0ws6l~Z*8|}nKo6WZ=>3Md@n8SizB1w(yo@rQjcq$8fl zH6B-FVT4ZBuyxYVKJra*okT^sS@W$pDA1haqwdjpep6FgWBT&2?oRturi1foUCrB9L6}|I*dIN(1E+wHf?A;djErCl1gYl(5jBx#R} ziIOj^;ALmdg2ZNQ%(F1*fF#>lKO}xC3H(PL^Eyn=JMd0u+V``a2%-e>N&Y*>iPtIL zTDJ0};WIh3Qj+ADF5QV$_E0r9hMh@2chq?$!^?eiHh5_DUy=yZhxRm*wVMk4%_qp{ z$cZrnE_*yS1~ozQUu>zd!8&SJ_*1eDRlB)?RvqBfW=xP0IrEQq#SAe1rpxpzoj*d< znofLOg@U7kzn5b3L(6^3$EB@?)SvBPN%GEBK5Qe69c^c=tjly#N)E1(o8CBJNEnzg zoCH9!L(ApSKU`D#PY9>vr09iwDZ)9z8JciUL_r!TH~PvzU-Dh#y(x!7c_uB(Hk_7p zK^$!y-c;h@bOO<6A|8$XcYP||Dgg>Heglj=1pGNl{7C!$yJ}J%CjaDy-=Zt6bt)g` zD_+p!@FrBHF@<}j&CA&j<%TJ;qCgC>?lukNL1QSSnDMjS_;bJNvKvbJu!T;(3>!8Hb;V zpi+&B7cjCz>6BfPzkbV5&2?t++rd9dou8ebRiOUNYZWkvI9SB+ zM{X6_vF|&?TRY~0f3NdwYd8hcLf=gV_CkG3%R#W$O|{TW!gOQIyIUg8e)5`nJm)9s zR7ZkQ!_2y`4FtMb?T)_SVLv#B)#O{mX@8dqoa3(4Wtt^2l-jqcTEE~Q^%?=UDKXpc z^h9Q0k8tdqi<#ug=_c!hnm^I9QCEbcRfQRde~4<0!I}x9>?6k;w^(6~l~|kv&%Zq> z%m(7t1dIQWQl3B|(RkaUrOin2e*Qy+F-s}=2Zw7JSGkKDIsk2vg><85xS~KWA_*1) zcu)sIEC&!hcR!}W$ywit%5YtK!JE#M_4c$1!i^Ctzc;LAn`G8cFu<1?YxOa@oTR4j zuu*&L^^Bi2NTP#(x}^-guygmyoA%upBLHM;WjZPv{QYSBS(of;K^ z_}|!|5p_phEplTZ*CQ|;X1d!-`>1t~K35Zwd4!hzx*#HWzs!g+=~b)cfj4}eM1g4| zFMFc-5gX7?PEu*w#g(kjNBpWl=V1V^81viD8{tqIW4*#=_5&sj2~SCsg1-Lv*lNb| z>v*h^G2CprVFN^y`%LDxk`Jn6t9E;#h<&p)%%?|ipiHq{0J;J3PLu|siGZP&giom@ zUBq61)iqkZ7xUI3(2nq_d`pKyV&moTM^FW!-uR_9R>RZY3(gEN68`zaPino~ufrGk zT;)eaaI!RVGZmwHSITNGYt7Zio@hjiH{a=#XL4$)kKyhfFr$=U)fb8%rh*45j8EK+ zxLkjAUq86P5o3&?Vsqb~BBrc0#-+5x&a5tgn-sJB?>epIuxu?!_3+5%a`)PWmc^~L zlLGA;x5kt|BsKg^{j^-Hpr-nMQLsLIca5CE4u5Chi7K8E?D`QwSQbMen>~#gOYK|4 z(6tGP(HTGf+$VmQh$}l1a~5+Ofz0l@kcVTiz4Md9wf*m5o~=1Vl&v3@q4@|C0SbFr zNGThQVN>q}t_cW$fCSmM{9umCtl*>^gM@tA(z@-yEAOoQqjb2X`@pC&Wp!b;ub^@+ zlGY8pwfy+ixmkc4u9wj~u`|+PbM1y=5ae7#Xhx~}byWSUspB9-aa9Lx>3wEf#b-^x z{&`)qw+EgG4h&yT;L3$?W6K^Kt=#~545Yuvd|V9qGd{JcF^cUP9_DPip`HhV z3!#KFaknTvw$N+tjx-+zU9$8F$A643<{+&7zvT_EM6wJGSU zsUr81bfA)5vfD@5hRl=Ofar>NCQ>cs9Md^3`7)-JvRNMl@kI{7O>xw;E`TUWHf8goNjtT?G z0j%q$tiLI2Wku)b4Th)`a^Zh;0^Ymm_;*r4PxyS9%HzHT745YD4>zt}qu1}krK8B3 zJX9~Q=LO+_gNOUwcyZ&ZS~PG(x2O1=Ugd4Jqa3xc>=C-xLQk21axuen=dRqQ8Ibhz z0O=tfhRW5s_9&-_oxWmcw_DX>=vtRmv6CtYQ0Zb?Z^uzn<$joF2SC& zVx``(<;JeXsiB%0!@a8UrK?L{aT90$0P_)QHXr%o*+X8Ro=NBibm*m&@ioj3^lgUF z!y7zb%BgldsF~z0X8xRA1}KWi29b_j?^$@A!OGbI>szAO887SoV%-Otlcvb>wu|X% z!}DDb5r18zv)|*hmIcOK0*9FSmDn)?g=ptn4IfP*!}V_p$(&+)ZcH^2uB@FcyV%;- z@d}Mv55cOl7LvwvjxyK@`5+Jm()-WgiN7&?XNC2h0aUr(0X2R*hWNw#=_NVaHE;Wy zvl|;zvqf_0d~Sq2o|lM(FQ<3Z7?+(d3)+CP1u%*hH|@!oY*&X`2p>4h%YUz~0%%L( z)bTpkb*=hk`X2WabMKwZ&0RTt&p5vau-|7l#_HoQ=}K|`V$`*OY)o?H2ScTW5pU3U=MQypKF9-+_)Y`6jXDxvey>B*et%)9)`?a?@ zn4K?*L|LlLM!Y++*b0l9K8udmVUjobvbJyIcpi5pddbz| zWJCgWsefb86t}*AF3OkT-}|lV)n2Br*Z;^=m49a;)Zf*gdsp^;0!C+1VvhZ^CuTo8zCQIGX$6LQ^8qkqo0HU*ycBd6`{tjZ7b~GZ5^T2**;0%QWVFd zu>8%ThIbqa=~x(?WU;SA(KL86e~I-^bT#`2XJ#GwJaq(@VxucMeiq|T3J9r4si}$PDH%ZR2vj7F zlDFyD{(m%Gby!s0*Hu77O1eY3OS-!|MnXEJTPX=8q=xR2mJ#XhPNhQ{9FQ34lKzhG z^Zn-GKOTmA?>Tqxeb!!ktpS!S3>t2AU|z_}aMMbUY@ib9ykt58EBK7aeG%`x1l(Us zc8irqFEz?wegm>2?%9b$;SX*RlQ`{%cZ4#@P+YeynyCECvYO0=bLCMf!G zq13Khg=YI3J8!vEjCA7IOFH~%UzyvYv$s}f%wBTBhsXzaZi^iK<~|Sq+OBspS``|o zUt8;gwWd|yZHor8;5@=W6T-M({<~Ef5LT!NtvM~EEPvfdMMGeOP5nFdwX4nf9>3Sy z+>*{pY6#@2hqz;iZp}BjW0h)g5sHrBA_Rb^zDE!`Y61?v$o`wk1--6TWT_K`4~ zq7)tVlxH?bc`ebr{ruzJm`4y?ict~I2W5&{U;YWO zE{5o#xea@HOstm_hGQsg{%BH|AfXZtjdTgqVD;sq;oBuk)Ym#^ z^_MBxc9|tlP`Uf-ZQaFe3m*Rwg6hAMc;eYDPu_Je3+@@6j(|36m|KV&4IOOC5xi%p z*EhqUgc#TJR>#WlB3o)0+{y=g8>nS#G2_Gy4wCd@L#=dmUn6E&g|WrO#aA!ig@^4U zkHr=+89z932tO+!dLMcK(rlAtHlT zgeZu>dbQFXz=8jlJ)L zJgw^Y{nlB-#q@~+xr}zn{w7N}2GtkxBX`aKJ|DwVKr-~4GN z|IxQ$`SdYyMCF*Vsp(kDIf9gm;#Uiz!fFc!acbXX1d+vet2oW)s^9tcrt?m+dG~h~ zPzK)qCb+g4$rL!;d1Lk<*zvau`S#a#oVK=hNQpfrCgw@p!_mXBYv36jaE%)dxqGT& z|3d~X@UnFt_^XsQ+m6l#JdEGRi2Gl1zZ8q9Kmd;jfh>j;74=BmU*wk^P4nyP>&XgY zc4YTHe{thGIy&63KSj%{fFT<87FcA;a}IzaOI$6`Ttm@9@lqTyn~PBk-c!IbJ->nP2z8@E-JVcX~Si)}4KDx_^d%;9S>%yZ=ke zO_L+X`1kO$L)>0&5)v}=)o*N7PFaPAIWMwC3arPSAoQV9CG;OnOjV@@FFFu1$4GcG z$Dt)f!s3g1IT)S47w)@?EWG*8^&!=lckQ887ND8|jpdNXm<&H6Dk>^oJh`^i?5bf0 zk+=yCZPn6~rcGF%U27XzH^fKx4M!ZvXIT08u$TG0}BGI9jci^hQkMK}r%>nEx(TZQjoH2cGTOVha*u zU)WWXCue8-xu}D(T_sM?8qVoA+?tXle71AmS!6KkJ2z)GSPc#pO~V^5>D+a@f4)hr zK(x;i5k0=FIdKCOU=1 zXD=?v!!!L{uA&jc*qcf-{p(^eEHqs_lnN~`i~MwoT6i(Nv3RGFdPWuivS4>8X!a*B zOLIH=+fcyA8Md@3Z|7{KlFBIdqMkNF&o>?JxHm!XLvOXH=$x|e{GeAO4*5Klnx_a! z(#(p4gHexPy}In3g&ARpEVrj7XYo+eiSe~Z7p{e*La2G+W&&_gPP@#l1a-6OZgvrIG-{8|! zIZ%uzYdCKyB0@qQn+9Gnc$wHOopQO&&c1$QR2AH~7LaszbKzcIpYpY0vFF1RjK9Fi zbD`Z)zuG2IRU&VD!7i7ED0*^gs>gF&47rP~vMt6wGXOZ^Sl1fkuLv4Vgtg=l4Z<9nCB9h@a?sBfiv#5LbeLlC0tCiHjD zf^3fOg-M##;Z`(Enu!VJu}{EA z&s7CNM_(T}x4@_B?da}*!GL*#7PFp8Z_C-;2R4SOL_H3U(yQDjKTCQOh~wbcQYNx1 z1lf06#RUbwPW~7KJ1YCV#jg6(EnnXHo}ZH1D?fZ}|69_*Uq2ceu@p;rNZa9edV5`6 zWs~|zvT1kR-Wt=DVhmO?_aZi$Bj-}5KteV9Z}*@NB0aXbN* zeU9op?eC?w!w&hrm6xNWyO+I@K?@HLCm|!tOg#w9HY|KZh=7|{Kyl5*YL+-v*T8l9T!(lnp{b|U1-ehfJ36Do z+EU{9S*lG;qlQ!Zt&r?HN9rh6m9d*(w&-Yq;5)Gs;5Kl4@4w``-u)C5u+;0vX1Cdu z0q0+redkx2Qv0RRh?P9|~x*Q%wpWoY}90G%t#hrL?msEU51w^yce{GQ};t6}?GYPB5Ph5I%d0q%wCO(zNsf5xH z06aGI7me{CgJAZcT9$Q$2f<4sfb&Gr{yaM}dKF-g>w4d~@6~u#Ag`y{W{nCjeJui& z_4mJt;mZxMa&IYMR#6T!)FuEH!BWawhG^~sjuIV25r`HgmPm!d;&DU^XTDpy61|RJ z=R-$uMnvCh!EY`8EDQM*gKLh8{B>nhDb2JAczP@2gcl7>eUxit?b zLrz!7XiyTOxT9d^a(s!WiG1v!1d#n|=vw*fZv-X9!wyQ>D*I^z-^(e{p5Zvn@h@-g zlWC+e%De)neq`65gYzP`u3;pCs6WY=WLG_D{C6ZnLGtWCWNJj@d|xC)G4fR?Pc2at z?Hk|1*t<$fOY0Z0xEc30j~^2cFqBE%m~N>Fe6%O}Qc=;yp9>zS7c-1UCH6F%P= z_@tM+%J~NK$s5fxMW!T`;}cb`>MIzB*!9ZciP`=Ap(-pc>X)x6uy)JQNK#@PHwZ;x znMg}3qj+xiMQYw#K&&d0@EyNrz>V_7UC9&sR_5#PCv_$rb!=7W*v^M})~AB=9;0T) zaTx+UwR48lJ$`ar7egF#R13KY{4>IRT7ziF`@Kbe(*z!2IxWeUG>id^3QscRsu{G) z>|4wt_-HcOi{(C7ZHCK-;?Y3Gs)<`hc6ay4 z&zEJ(=7@KQzH@uB++SPB5^CXHY+o`|#nAovOiy-ViKCH0JrfZtl z%rm-(x=XGLvyXS!bg4;e6Q1lMMGK6VTF6rZoREaqDqqpgF!_j=Yq2AOa=&hn=NN_Mb@~Iew3UQsekTLoTI9ql^;fii9po( zG>2%EyFma{J;$7!`Cc*T=c!4q_t?ruF#M+JJ znJY^d)5pq8GUpw#ICa+Tcuj9`*_Rp&YlPV!1uhUOzB z^}+g4zATA+H_S`qy@DK28J#L40A`JA9oOWUnVC&nsjqkT_xl?-rpovPfxVaK_IPPI z`xwT!xT-)F$3Cw)DovL+LwLUDk46C=*Hem%himS^Rr||O%9$5;BZ@7b$yt#HBmQpo z_tGQaXeKz7Q6VC}1gx`J1K-nghZ5;{3PiR}6ZN=Yef1_qD6Qs`O9E0QT=3#K6=lQs z8QUyx>17_%mGDlb*cOSZmoW8eX+{OGMj%uurt{-@+Am1LKfqX(x>%a}V7=@e?h=EN zLc>b>WV1rN=f1ZQVxxJLH~=Jxu`KfZ!;r;XMbrMEyj!0|a-en!U{bf)#yR)iJeQ4? zpjBzJhx7*4>#nAKfnwdtzyE9CnlDKAmUQn;^LK^Av%+spG3>LYnjqb?8~4Wq;4ksmNjVQm`N@??x?D`g=~Nl**v$*E7dJ0(ly$=p5d=`Jk1~$G3r9nZFli zbo{iJs^9zH@s&s3oar_MC&sjzjU)2n3YeR8EMLFcB}qeGYci`cmySpJvE#$X&u`*^ zUZ?NBS#^Cu=yZQg_Exen4qsFwKdg?Fg-=WOwXCb=p^YRIp)~IAn?U0J(OQ;tSoqlp zIlf$?MV%H;jw%{DDgG`2ISM+LN?8Pd%oivwKX-vh^M+VP2>ZwVUN5Qx+nqDC-CWEC zJ#2M-Dc0qy&SkY4ma&Vx0JR25g!&ZkpG*xx9Tf+J!!BaKb)5$%s%vbCp6r+L-AZga zOVa$qjX&lkW?_V#ZP9>cNJVAkHt=@mefh)>c7$-KZC)1MCT3w6w#Hkkv0X8^%3%b= zyW#CqY4W09KETq;+;C35UK&rvJWC*!jl2n7*;s?v)fBoq-sLH4=fwze)_JY(f3^5R z-SZZTRI%^3pgUFEP?74zs@|Rb!K0p3t?bUhn?FM@+-dR=stW2?=8o~h@jn=~ey+z9 z_kIRe!|E7@J}Ut_CiZW|8{={d+kZBwa;BVK~n=T#tTfC;*Wx7WH*MdNQz`U<#r;ZZFC0mMR&)rpUH;$}Z(psC-;!B9<9f0pO2 zdb~HdLkmS$54gm?*eWiT>V3m8@zd1iUe?z#`NFA($;#|6-wq_Y*exki-VN^O!J{g@0#Dj zd|)i!sziHM%iSprxsodXQi=NOfyn@t#9 zeOyMp((@8U)v?=}m$#hE{kfy~NK#WQbJgm2kn;v=bvm~FLCxwstOe z@n}D*kWfP`4*GGF&@l{cL++Ez0R40top}1d@yFDAn&=yEr!*g%7&&KB8-DGX_<`N` zE0~utKJDM4!-H5HrF{%z8E&eITJBhgT@wY-`tdok&`ao(_sGleiOq@YUxPFMbP;E>^$;>HaW zyXrKu{a-8Y^}`u>y*u;tg3y;ds!mr?O77RZ#9i`Jn3KvYDQTv8;g>LN&GHNNIi38K^d$+N>v#g0CF< z#n9LgPw2E>U&T4)k#VHni`krum|q(dvP_Cfmv$t7$P`34p}Xq@OXBE{OSua65z4MqsK^;;-oq&_9ym91~#ElR{&+{k4z3zk;a!NEN^#xf=nq}7vvMA8_~jD zEB}K`42$5~mnLzIc!+4@y0eXJbE&0iHk@$QQ{e0!w&{9UcJ^qRDg};~95WJ|U9EX< z0FhJf=;XUa5+~{bG{&{@m}-R&KoGYuH%R#S8GZZZ8yM2Sn@vNNNJte=D~CeM6vg|} z9Jij&4b7P&B_1axM79Gmlaj?3g6nPZ75ezg(^KPPOqW#`At}objg3ir%DfX2eO-|j z*P_?Tmo~##+u)~~14I6@5Bcr_3(5UAKPP3VYBXNEFMnN25}PA8~mUL>h* z`$Cq+rXUQG=)Fbn0M2YyeXND_;)p`bId4@L4Gbx;~pcbla{Ec{W@I7Iy3%+Y9Jq~WV4y=S)Nhv=F~`%mq4QZv3&`9!oMa+WiBJ+F@>8U zuKaU|5S8Vu$>V zbqzz!u3y?xTa7ELDASWM@-S%>(Z)zk%nL}tBx4I$?jzVrA~f$8Zau~Iy_UaXu$80` zI`-Ped{&m~B2gI`*Z)yk(wvFhX-|_Uz=O249UOc*P#Fhdg84)j$Oy)h55+2kMr&K_ zHtlFr&T2jD7b~3lL>7vNIyu5@!_NczN|#Pbz6*E-(bG4r+IrG7&N?J4KAuj;U&+co zC8H=X;3$WF(o-l)H|zd&!VPdCFEb*#lEN?Oy*Vr|-;?@5FFY;no-Tw?@X#rY?@FL& zAlZ)erQksz12f6~8yoTZJ*}^6Okc}AhL!ku zeq)UyRmYp>Ap6|In`@#7&FYbx)H*-jeksp!|K@|-ds5puTEXPOY6{~~mNA=LTU}Wz z@vsdfyhvQZfuM%!i*~l?vZW{Sg-C{eN{VsZoZpNw%9KXWlZRhF6*`uFS`A}XFlJiH z>*tIqacI>zHg>xFgX-pPEWYh|TuWEw>^b7Fgp4e-UbC@Mjt!pGtF_h`=t|C6!e~0Of zokYz8Xbqrpog6{i!cyKN(of1xqhG)o{BDp=ioc|0Y0F`(n0PR}Zp8Vv`*o2cZfsPu zvIDVxzM}WpCujl7W;g&7R_||?P@}XDDP(X#n!@`E<*|c^tGXC}qM9TX;*oO^Qfy=W zJ*%7IQ1VD0Pa_nKNv2p+Un!&bMucu$tcqIX@01S|L;@q4Yv?&zgL=*+a!ag2mE zL0h&DyYR-Niv3?kE@d+uqVd#(ORTmIEG;@ll0M!|NTnoYxqoVX$ceH30BzTIPAm*| zeBo)Ub~AJ$z~x0Ez`=T^gvctav#rIUCr_jX4kizN|G`+=p+G@(>)Mj`F(Y^qhGk|9 z>LkTgr`~Fl%5A6h_t|kzK;!`Do~Do7ozD$0Dbug+!;P z>sZ?F`K8C^k5}a}>PpY1mu91Nk7&QX8oSULjT$_THT|JAStz0~{Ai3AYh6sETlx!n z)GrP^jRUb*hcA6F+R!B2f)rd!QIV72OOvMnirv6BZl8!i>6(X zZ@c}CEd;F+>3vtQ{yc_}6*$Z-QaKZ!S4WJ}?UVH0q3$Y$%J7(|bhBKnZywQ=D&7}*nDAz9Zi)l^Z1%{Bpf!u zb7d{n9BRFcD`P+Ay(;m|J@pE57;%Uz!K>AJkv2|OOizK0j~$Vqlc-W$#U2f2tSo%& zU7&F8moI^7Hg%&xqE4blDFD(yt5YOLVcz>cPMTFFW3y9(!Ob_&Jyuol=sN4u4>a}o zA;U_nSA|%3pZ?#INE!9IG16>2&5$yG^wby?rDESw4r^|?FxAem0kgUL$0Pi zKCkhewtaiByj%aBwM1Nz7MJRJp!fDX?jh0tur==Y?_#mjnaT;T9MHpv6?7@?$Oog| zK~Ml$eOzqu%LN21JqYDHPTli>OWjJQ`|opCaG3!xc6^Rb^dIL}Nzg&BfH&_IVkn>QLQ})PzrEeA3yS+rFE?o+3i(kyEM(hG}mu_u&mohrs(A3e;cSU~qz6 zuS_oy^&#$o;wB6lpeAvBEz)s6G4Ff5TS`JfVt%mfJ2N+%j}WO~Yrfv&0KU{joW?_J zT%E>I_tyzTz1&}5u%3o}RZ|}fnkwfkIoc5WOy;PU8ljF2Ja5IEmM)Tw2j$jO4|P#A zRxMVsV)@n<`Xcu!ixyw%>7|&tpH@n(H(^&BVD74@jb^W)=1Edbk|<4Jy?-00c-%}B zJs@UdW0Exs-vEJ!EZQ=Y@j&h56Jfnn5i<8nhE`0#>;+A^wvMduk>3$>cp78@N7E*o z5g569r-TAm01HG3SpCPj$lE#|q%J*5$~$U&*f?wg+>V8d9t`7p{{TFK|8W6qFu)t> zB%$bP_WV(}MfS$|Wo%i3UX)$#HF3wqglt_y!_@S2tc{@%bbi`*se#z%WSHrv9~e10 z{^2^g=$Gsz5?f-=uu;CTvB6?!eT}QR)BwgSF|n~xam`12YS;mLrn2%d2CYg9#dfUV ztu*1OtMlhd@8I{9R6QI~EhhF>+oDfGaWJ1_S!LEPKe`x{MOVzxP>} z5lLc98~x$Hlo^+dC(nF#e*U8ktqRBBiRZ`#F{qt&KO~z{PkBEtFA9WUwdSr2O5Je# z3E_00qoJWSQT%N9k;(6vpaTZX6Wl(HjcKs5vF+~tDXMnIzB0}Q>;M(iX}KqFj);gh z+(P3d@6&w@N!J3;w5e`($5qOT^R+&CBLv(w-7v@noo(Gd=gH)AYdka3PF5~VB>or_ zm{|h$NK3`IA2JDZY=|VI)y?bH232|IMDmUxtf_=QzW2fNsOs}7zr6JE?$f^jo|U~c zR+S`GrXUsc_jmAD3WL4xW`4X4h$wx?$xue4aVxi7SK5n#gJ z{qB6;^YHHS@rUg^f9zTdjS<{Z)@B%b+;b_1*5eRa|%6Zyi z4;to7N+DuqJPZGTxL}MfVqRbOB>}G4=YTE)1V$Et|2G&Bmqbdwe$-kKUBE2%uv{xw zIO#t28${|_O-XfjDqs$Ny0q321%m;x7a@w@g_NQ`H{Tky2%m($0I0OZEouPtXVqRs zF?-uvi5oLvwY7x6I;Dw2!53gO#>qJGaXE|?D;BE{Zi9)9IkB=5i}L}@va_;+t+i+Q zxiim4OhZ=}9}reTx~2VET2;$*U@#qRZE2Atnyg_hU0vkh0-u2@gb2G)9f>@|jWPm% zf4>QePtra8h^DK7VgzQHy90rBF>(3vE9&dChW)$I-A~DhH8nL;0XvP26W%#dQRuwkfBCYWT-Bs+TBm%RF4A@^Sjyt(1^W^|Q8q{AVD?=mxX&%1{v7tdf zL&&C(;Q!qTg>mxZKNv3o((E>^-uRjMLsXQC>GA~<6n?Z-8JKc^A2^bv9C$IErvowh zQ9o}IVYTqx)!|o$UG8rKN4BVfcqK;Pe`$1E@{R?VjVg9Nyw43f5H6TxURhZIq64R^e<}Th4$NpgV_f@pm*c_1drd{j zb|lRb+o#uk@5lOb6KWq1da!(DwE~=JfDEaa5HlhUDRIDudqkcT*Z>VB;O6>X+XIf{ zvElHRH1Sd@}; zi-Nie%gjif zi`v@S&bAkz$$0A1O7wWeMl%_qI3G+))6_%+zixjF-VM=0dE0HffE8`%_QM$a_BYxc zQ@9HD&cC$|_Vk=Yt;$2C#E!UHlr^(ETotDf@%l6kQg$e&fb*i9L=uB5(S-L$!Bw9e zz}j7JU3-vdT=pYhGWA)-Ux?2g8erpU?`t_H|JfNN$pob?)fxgNx3W4sR?zexKgZ&r z&G+_i<$dPe&1ITjM;T1T)s?Hgy^D zG6Ircw?72ie*n`+TAf9?SZ1c`^ZBOG&LBLFPq+Uva<(9`2{Lex0pcUBfe2hEm3Yr< z*LHb%`6ms|r~l|479_V@W_SGKVs~s{eJ1#@ruioIuvprSXeC+WM{RxK=kvR}yBnf$ z5+gxZH=T;pZ5=@{7+&-MSW3B`4Q0omj|4lTgMD>&!EW^5`Qw~qAcPm~K4`li3NX@b z7%Xe=R@0TZso)?ai}7A(i1ca*uA|EbBa~vkjDZn}?vzC2%DNZDa=5fRE&pCFw_FhC z+H6-uUR4-9QL1RZc98hHjtq1w#-0cMmBn0HZ-2MCu^#eq`TeE16Zq}Pl^8PeUEA?B z;!WneH-inJlw?Ry`j?KTW@h@%=hwQ*bSXDag#mf8$!~NPlvlWs3clCFOrX^WH)44b zAb|GxZMVgWxat@C3d`*ze`RmG2|DS;e=i7lMYaa&8Q5IS6!xcB--=ad{gJ}vPy#1* zkmFx)jy!s6lBUJn*Lv92ivU4rm;^%b1Q4KIAI>{dSF_X@eYbNOeG%UTaeYR*TPo5V zx~6vdK(fd1GGWS4JNe3#CpTTE6C_jgFD2%~&?^w_B?v$B+rR;kb$>e%Ii(L~-lR@9 zwC9YyUwLD}pYHQ7?}E%rXF5ukb>&EI=>3lOAKvO8R5^dbZ1DUESa4&Uw3(*{({;Qt z+CU%8BIc%*dY|$pGY;p27ISDa9t*>U+usR}?VCMd>&L}}nii`#`5>^k+zw!p}UlG{Y zqNo3G3#H|*C1P_i87V;g_B^UD#B6vD8n9h|!D$4oNr{sMiNehWK2O21To>FgI9__b zf}g!Q9Y%$|_y1oCC|=A2DhzTR`I+O}K!?UnD5k{zXG5%g=1>9lpaG3C3qy&$B?&4> zH{;vd5W+$C32(~5Y8C>uM+EX2b$Tr3d&jYBPsq1xF1oicsYvtvU8!XVLy$%O0Svd;YRf?7D3vuJ5; zjZ{biI!p72i0pCCKR>Gzk8I38nNWq|fa$o!Fo^L@&et7B@}m8EY&|nw29k?^AuyUz zDFj;xnFgi7UiTH{_VuAjtYMYVMQf6hT^+I8e>;CW0tac(-Jr&cB}>5B^6nbzA9DrN zv-8HDSnt+6xg6&zwh(JR=|1cl>dqYG0xs59QG@~e`4Q=BGod!V+A9X^19=ZSTuPemkRTAn*CeJ$GM{v;jUW&{_0tw?@J%?Bl+Y_eEP6r_I=_)FVU-Z`N12g>-x#ZMx7CtK&qF>vTpsny#&DUeGx-JMpR zX}Cyy{5{8wi7Z}WMS=>f_u3!AyGn6nib^&b8|(4oMnlKBF{@~B&~Ty$u5Zz*EJ%K) zsxuP+#iydJFLv)#O=Yt(GpAfw!f-zL@Xa|_GU1y)Q(88Rd}5`REeJh6Rn+Y`XD^iy ziGZ<0CKVB1iqb15PJz>jkJwn9{j?!qR#mp$z6_BPmB@QOTI1ut{~D8-q^sO;UdPZ- zT3Y(C3|gB6HEiz%QEW++0YfQ8=hg45re&90pFNg;mAx@1F_;F=ROk$B-rU}XkB=#s zoAJ#G3vX44CeFlVb^K(*H-G#yQsCz&vY3uQ6kh6FnX3!W>;>(}-;+4mYaM$@4K;hJ zoP=>56!@fZ71m-BrY!@q8X>7IIa(-wdUiX>G9?m12`y8o(9YN4Fz<32p2`c+WN;HA zsv6=QSE;X$e3qBJFHL;fO@Z>D`U?{4jUIpZ)@&eVmKimoLi9avJTx-t6GFspJlhsN z5NmB?V_`@HWX&#?4Lv+O3~uILgHRvy44=_sZ-xcTya7TnD8phq|IV5HI`=bdzsBmx z$Kg!-I?60lsiwsxNUqftlSL5o-EOX?Y$kQaGr+`*npeXK1k5nBZIHBtGo8 z;dJf16Np#J&EX{Wiuo1ZYjKZ|yhm^pH*2Kj!G(s{=`HJl zkMl23C`{i|h!?;oL@sVpuqx@5b>fnjpZ@I`bXppKAA4Q3PVcIzQzP0|u;~-IE9bLR z-n5&#D!$zBYrub2Il9%dvlibH;k+2r`5v>G&hzKp?PHl(3%<(KxAG4E_E?xP?rH?X z97RS85|*Ej4iY{pp$Cy?swRT+iuu zFeq+zJ^k$Q@t3i9<&PO|^9Cc0*~J}oxIgHU%#;(0M$29+1Sj>gB*t2LU5e~NqtRhO>F6n*On>xYPKy5hOXg0J__tYdomd4vb?~H{EQIcOv5OhjDAU5D2X*z$6kp)< zHK_m7!+f|67|Rh=RQKjLtxWrM3_orXPft}IK8E*MMz&{DU_V8Eie1#)oFVn1Ya(Bk zM;*VIja> z{}7**H9Dn8Z37~0d`5<{i8IIis+;(O6Z%H|0w^&41g6?UyH537kumfIon+Q1%`Ld+*V zqqd?y9~QiRR_0Nnk)c9QVX--s5GrF~Wb~R;=*^3V?X0!FmmH@4-V9h+Sdj_V)AYP- z(O{6A9LRxwuo{GJT;7lx8z{Z@CvU0XJ#1Ja5eX3bh>GBl61n7Ef7CV{pzy<|Ml~ohw68^Pmrcx9^c$C zM}2TP%k>G+4(tTFo0S(!J3Q86W3|rYooifdzuzh`b^#?<;PMLl^3#51iCD(GpW{PF zRolICUfv@*+n2ugKG$)3gR5H+XaeDrVjg?7^vY8yXJOIizG756>3I7fRJM3yOd2Eu zp~Z>0MHN1xLa^j{chi!%n5b7H`W-#l!kz$N@rC)-^=_kf{B zRP)E86m6HQ{^bT8+1+7bw?`40o_~)lEUQUm=oJ_d2t@0*qzpCKV+RL^PnW6Z14!(> zw7h~zhK*_^_j33QS)<$>qCB7Pc{WsE!TK~}W-d#}k<5~vm-uD`jrSWTt%iKE)7q%` z1^7l#(xE5@o*>qod!O42BT--|Zq;jEFXJs?lMxU-x4z9O6mD%z%%nSi<05=+2o_o8 zrstHQme>oS#u((6y}66gSQ#J|-gh1>8N?IZx7VigmJ8 zO=OUc15O)1FTRUW{jF~d1O6LIOU1){v(6U<8;T_cF{k1KSN0z!p)h#by;px)x_1~{uq|yFq^N#*jr*wp~W14I)FVuou|8{?Q7=*hFf%hU? zTU#hd`{}8}$&Mvef2s%FnvL7edQJiGo1GJ7E`3SVvuA9~_D#f|T9z3}+>+aJjyv2; zFchCUyP#bpnDO!!)QBLds0QQmlL&wP)I<7F=x9;9AcO?F`Sb(WP>|y=y!FkqE$_p& zSO5>NxB%gV&f7+_N1yGWO!`M_Q&3A znRd}8bN7YwlFpiF&RBG!*)IH9g2820Mp`$2^uV;yg2GjUXJokkf6>++uzQ@B05%T7 zz#wL?5}}nY!IRM%kvSl!IU3O-@Q`X#Z2OlUVNb7kkW`r$u)oC-{6-bW#BanJ+O>W9cw146U%eOXnZUK0@|jCe-I=<= z+%NpviTTl88e8XMM@(4+1fp2ez4UoVeN7wg#;Md+ZhykYK_<}gQzCLe8T$U69u%Kc zV6LcV%VTnJG0Hx(jMh4{#DWC1+njrr2z1X{S3nn!ov3&FwAe)`qBpV!m`Q@vCq3^C zbO$y)(#Ms)Ac=qG-R-apxa0kKe*5_B3(bG)&=S?c@c1Bm!gTL2FSYLN{T)Ug#?VMj zUYd;M-f7H6Os05t;-$uSNomGEQeh2z-u~STb2oRMPh;898w|7#x7_}~Sy3}a1z}P{#)U0&^E!bvWY1y!Bk}E7-L&M0W zEn2c@_Z!N0C->Ly3>yz9@4-2k4n$lAUf!f1SpvsdYqyynK0NkmU!9qG1O6jin*6UtI{xk2gmccH} z`(EYirctmBV@XzAt2Y6&*2yZ17GB&#St9bYoX%?zP$2XpP((Dk|J+vYU*%zM4S+y>FetYbW3gngN(k3 zHaL^#(_cU@o{wAq#R}7N{>r2~+9x^mP|Czei)#gpET><(`!9b{x$cA4?0iyoap43- zocQWs;~TYsLNiX6xGbOYG^dR#VX#(e9|MK&j(o&Hhnj%qX)rhhn7MSyJ-Ox^eG*Fh z#`PHvx?IcFIjEj77+?|z0u3^`$=F_&B?AqxwiF(n(qLUp`#0R;6A~T=+bsFCnZ9bj z7kRf<8!!5MToFQu*=s!rr36yu<|L?^VpsoE-)(+m&VrK&LqmxU1WF~`bG92fX<`zG zkBKTe_x!8VVqF3_eDb=k?k{_da%6uMQWT@obao^}4+A-U#Unz(uy3@69duPdEn;8$ z&g)JyNSbFZbnjm^H-h zMl(&DPLJ(6+u`fxO(P7_K;V>+_t^b*~4DirnQ}}n2Xr#EJTQcHAE`- z;^Hk*$Rn3HQGTPZ%;*Vjp%3?!55FAE9z^2UO-7=&Mko2OKGSt$O>5~ZTFD{xUhFA-dm6MFb9Q|oN9(CUHmfiSAvcDMXaCy@CI@6a6oUq#hla>4le!#o< z$|Z%hRW+V9a)|O%wsA4}Pp~k8b6vo9o)0mK+y7V7S3pIzes7Bf3Q{7HgGh=fF|?E* zNT(<%Agy#aA_&sbAu!T{gmjm52vPzL-Hmj^x99%8|6Qzg7tWc(Ip=-Pj%Po6Kh9E% zS)y46a#hu>zH;x~$Y~om{o<`A0_KNLI@Z1zBi;7RQw#ZEKC}A|{VV^4ErqN>Qd}P!ehhTUlLJX_I`eFhFSz&-MaTB zw)$OSphxVvW<%3y#$DY8E_vt~wJZ<1a&lm|+aCU+C~=H#vsG3kaW1U3sALHTG^EmSL?%F+TbLu<^cKmlHu#hToOD@ za2}ygkVndn#C**8Cg)|%S@!`Czu*xAB9xBAdo4=}R*bsiS(#k~`iP^#S;%5>Mn35x zIm$hMZ1tcYaOhdmvzvBtrREg3<{u@Fx#uh1y$|PH$%ai$iNkI{I%`G!tDO=AvXfL2 zy0ok-sJHVWhJ+MR;j&y)^KyUOUi*uNg?255IKTLc>3e8>xO0RAW5conmTWn5f^hJ= zjiiS7^~G-^svlQg7!8@gccrT?n>QKHoEDM4hC@?h31sdZS&3t&HV9t`YASIO0MQbG{ z3-_wTiKX!BGV8$4Wn0d79WO}ar`xqM$ds?ZeDrx4%-0`73gIfpRNc_K>tB#kSQDW7 zrI+4VuR~HDVu7|Q)*Jsmw(tmV6!wV7l8#jj)_DcyB(pWx1S}qCFv{t=-?}z?VrM9S z!@G(wC$rP>)&HHc%qYc-Vt^*mcfrPbZe1lfZd93h?QOj{-0kwL{Rxkh7!G7cfQmsZ z+1@US1HPY*G$hqt@Q((he?DNdPiayEDLLQ>k52UaGG}$}{Cy4ZnD}>Vzm= zEmR7g;Wr>dmVYu>?ffR35NA6SI|~~P5>Veus^#%T-{NRhu?wA!^#6YMF-2gPG5og} z$+=p$^=PkF~9B|!4Sb5 za!`Aj-gm|+DUSsA;^M+@x6sqFY@5Pzb3RMKW}%xrQNaWyx9>~1q@48ghELqqKkgUQ zIrD=tNIvbis9^v9Hi)zsQM0&>WH(jMoLj5y+f|Jmg5eqKt~q#S52)?6F0;amEf3Xs z=cQ?eL?JgnB05RPeB1WJ%C>(u5fr-(TV^Vv6ddcRCUol&r3PE>avL&oD+Y^Lu6XQfz78I+a1ZA&P0XFFLr*Wjk?s<)?GHfp-7; znM~~S)TQXB3i>?ASRHXY3CawbL@{_u+TL)JQTSC3SJj6duH#b#53kP0n|S^je4u)y z>$PtZ6+^{@2r4!jV7$ts;pw%)FNOnigqBgUT~O{&FE+ka?YJ=&!*2&6@PeTRIrk8^ z`Q$zkBFuklmgzS?b*Tkq)__Y#X$uM}8BOfg0!a{7zffD}xbee6gZXiEylhrPUvF3P zBEbh*Yqzo3JUd3;h8J>Nn8?*p<@%-z5B>zDlL%;ZH9klQ@>y1cMOm5JTv__>(~L~4A;tDOoV)_0GOKRolijz{)B zf0YD}tf9y#EMmA`q_ngY#5KO=uw05U3WuE~DS189%u&|XIDzp$N5O%Y_MG0Gho?** zxzIy|mpFoo;ASfPd-d#iQTtDyu6IL;oO`^mIa-o;7eN%34B47D#~YIPPTbJ0XVZ))shdyg zV=T}*b(`-1drINln~m&n zC33Nd(jO(AJX@{T-~YwCv#(_}y>mn??IeB$1U<&L`6K8+4Cn2;cbLLNZ4=mW$h+LS zn6LM&^ak9TPdxlaHI(xAh+p60LN!@yw7y z?{++!7V99=1M*$L_qG&)A8TBH>D5F9-S^NRdRYqH+ny;Cx6o6EK2x>AR14E%_UU7Z zu$m1xN1lB}!}3tTkgUV_I*rK?-ki8spcAdMA>o=@^W1vXmhVY-|r z81H_W@-n>+`jOub_-i5!scxI|@B{z~Cy|&^QiHUjgSkg8*3xz~vY zOHXoB*`u1*Y_KCB1eklEN+|X@K3ZDq-MiQay*W9VlD_rd41G4O`K6>*P_Pv689;Lk?Q(A3bntx0a&0JO<_ERez>S(Q5{YHIqz1+Nvo^1{ z|Nh}`TZ^X-2T&%fn< zYlw~Sz^3HE40WUm&0n{4D!p~r(%-o>b^=F&FszweP@hm05hps|NTFc$#DBp)ePYw- z?7&T3(DgETBmn$X5nG4O%M@W7p7rC&Fm5jnQg{sJsiOH4PJ7Sy>%IE-Ed;{8W@I&G zwh}9roDCgCoZwwI5tthArJ~Rx3e;BcK;DMZjM3J)(B~B zJ$f9K#zgX#DDWO>K&0yDUk-}$X?Z;Dg$_(rHN6j^%K5`6JWV!yYe@#S5;?X@5B;v+ zef4EgH8st@7|yT1sk!Nt2+AWykGs%y&IS=d7L3d;gE zdkN4l;D0lOu`-b1{czfR&FL--UItV%3@;{xPEQvvx@%m@?Jlka@q4Z@rzm~vvQ6L6 z^otr({>hGKucP>gp@O07ajSr;MhS zRs|Gr4)>kT6F3aoez?pserUXxkckygMGnBq*qFB5z*LlG_UfIwpg=#0_YiTXub=8X25^WgKenyYlHq0QJZ2zy%1W_Z zHe3ziW5$PLY443wB)Uii85j|w<20tMz$Eq_fsD5|;h0?rnHzb+^o`BzCvA`g1n1Hy zQMI9FOWgY7J20iDcGC+L(NBy?`GH~vuZD=hGM#MYZ0HkyWz7W50?T+tz}37ArfUxh z*ZQ6xEh~EVE2q&Z|675vHs+Is7>47*xF@z}y-0MN*c-~`jhxFUx%YJ8I3O6O#BASk zd+OP|RZ<^sX!_DZN|c4L{Lr)}iT?-0pMsH-_s=+F#(xL6%>pvTMZ8{C#k936KR2Xc2dq_hQN7Rg^=Yy9 zH`DUV%DRMy$2DF^;CQDp-}TTbe9#uy@QEA`@#OP6nj{$yX*wfH#k6lSGj^}1o9~l{ z83spgEp=5jKjXjf&~@l=-rc&Yo{YRJ+mnAMP_%MP&AwX0>F!3v4Qqn?&rO#R4Y@nMBlk{DJnL0=jTpuWKxniI~n+5 z{MfFKMjFh4Kausv4Bq>WT=KIb$sRGm!FZl-MZutxYG;Rxj*kA8(+Ox4W)91m^J^p2LGh15+V?Ka|k)n^F;KyPX zabU5z`w}NnVNva|jKk3gb--Ef^UUyYRAcDmCS5;8^d2wVsw+|ar33m*z1D~aBqRqW zy+PQwB?`1{X_PTnRD4@3Jy6?WLRe~``p>Fr-ypC=-0H zDOdMBZo2p;f5{5jFB8q*f(ie{46fl5yj~A(-H;$8SPf1>MsU~#F}sup1C0M)1|K0D zE6b3LD#6~#68eKaup+%29M|}}?Lq$f5_p=4^?dRE%PB0skA*3oz)fT8@7c@ z0QN1QyFK;2y@<6&V}l;{|YiHYz#qqIt(yRwy8CGnVvLMwQhkbPi>vgsPGmhy{lS-7X6ryP4f-Q4odq^ z`PKa28lJ%?>8}+S8(zU|)^7y29_p)%pzo|CjDZ+$l(FBs-bl3r4TEPTRMW zLxw9|o0xdngL{;oLQAy0RPukf#$Y9h_A`GE0o?J;_)EG**=8?`*8hY2ANG`4j8ei; zE8=qux|pNW>*W6{$F3kAG1JWUK0z|aLdt6MQsSW zhe%xWFUZ9nfTs82GUF*Y6jt)fwq?qj#O$p^W^4(~yzm`FMtPTITGJl7^k~Znw57OB zo~X&bkL<7l)<_WPqZvS=47*1PSv{BZlaKZa+JF6GAPqz6|Gc`tG4Tt=iMRC|b*Wca z)7GD?Uwjqlsds>>OswCvDA+OSJhbRk~7qVlU(U3Il50>(y0pHZmZ?+;%p>FE3_uiJZ7QeIZ%dFBc@^U6hE zQSJ+sQwfF>>=+{mL?CGZvI3gJK!e9`x}l^5QF$i?GFYx3<`0P?L+!q?$+pt-Gt?Ix z$Ui0zD^YN)i^HAOkGmeo>Nc2R6=5l>arI?+-QstF%jOA;5F$c5R-s9+4Eb;ns-26TIu-R}_|zHthLSV_g2 zUfQd1F(2EO8*SB@#UzM4s*$MRfPv}zNV?kvXj$EjFX=NW3^UU!_vd_YDG3=I!Iwal0% zAh$FuCD!M>7cV2GknH3{Z)a_ zTH_*sO;S|Lj`DIi+45S4hu=d9(JyOX690$DFqCqY-6vrLaYn_WGuK>@7;3^avdZWm7Aw&HX1$ck?Z( z5?u|lz-46fF4t&1wbDlieCd||^$3a$s=B(07w0ZleY^*Y?)$HNElde7pU`X?wOc&= zQh$Wt9kZr8+23ec-OJ1rYJ0=+bK1JRt{&IQQTL@-{%v4ftz?^3E92i1q1?l463y(#^Nwsl}8jy>ij5Z)zhX=wpD)zEcxS;hvRQ@+fDNGULT z%eKT^y$V2pdZ^uXRgMOwoK`EWsYgvlrjznW@_)|f|#YppZBymsg@%6}S zS-|g#<2s`Fi$?P~Pl>Xm3^^pFrV^2ml3peZ@WqG|ynKEch*4m-o69VK$u=!DcN?va zE(2x)*?2|Gq6+b~tT(|_{DIRF5*SiIYU*YGCT~Efaylz)S`3ehl}Msja%2=@qEq;j zYPU1rgKHQJdT?`Oe@Om)WC8H`Qk0+u@D!4X*!2)7*K9f0%nCg;G&FQy7{*Y-(v#D( z7#ghW@f%xH5PD9#;B3teF09)v!`!(<#Fo=%qFM8hv*WkisU!c zdmwFSo0S&&Lw$`W0{wtFFhb@yvcN67a=v4--kR;_97XmsiGkg9JZ%<4nUVg^=xm=$ zP~gKtr|o(B4rhDUwaDaQ$}obd)Sag5Yl?g(5dv-LI~)-k?;fx@c)G{DliFTPDtJ;m zO;$O0w3XF7JBx_S$Af2MUs8J4k{7sUi#rZSXB(#eY!2@3{Tz{}qZxmvC#-8uv$uX& zJXB7T!vTFGbnPnDvFa>VdSIbW4mBxt(uDm`UsUD>9f?-poCQ2keyhzWkpiz%z6hD)t5Uel(+NQmd; z1T5O;UdU{uC58gtHUNAtZ0NmgVpjrZ9#0qh>A7uJ=-jm-F#i6WmHtLm2pZ{*9Xg7T zH)zM13-tQ+t9_>Q%6&R&Sd?7?lZ4rUfgwp1=mqQaoL0tzJ-JssxIJOG1+# z+J_MJ2+^_sYItvOQPI-{$@$4)*P?&#P-?>5p-LNE0~dt^BxDBEvP2P!;i=!Kd7rNh zs)%I`#0&UGo`IeN29rDR@xbcnwHzhwlj171FWPaCP>vepZU$V%q?{F3XIQoE8EojB z_qtX16WS~4mmX6SG;sb3n(5=0tevLVHgkP;*_{EJO<73|nm-YZdA3nIBRNeiEsf{5 z=kX|cg3$}(p{ig$@@Gn4>B`GvQwka5Id!8H{dliP;Jo%a~ zRy7Lp4!06iWxx0^6mC51`K{j+C*mbcE=&W}&s^=g#HT*jZBCM$W(K)dR#r7C-(2fc z4_4>u{1i?X563|ko|E(M_+3HYSpQGn^Ha)i_|bao&Bso|Hehos(O-^aqNo*XiF4g2*WHNUitQQ5BzKQE^xs0l5KxL39c*Oo2BrdSta2gYUBy=B7YGFth3x!>zQ5!!>Lkgp zJ$Fq^h%{&mE3{nKd{d~-l8t-aN84-G9HCtkhdMnpv)$P~+>Wi61L|;IUeeI_iLby0 zmu(swV3_65|M|=^*$$VSoE(bq+pX^kIL!L#E#rhJvlix>15_qlck*7e+3zlKxUaA3 zX65DG5<)Bz^qzTJGdKGX2)9qsikq2PE|k7{)&KWz!b@puP6Y5`4L0)1XMVMR%`c02 z^(x;`+lGU8+!j|seioQG-(Tg)jEN391O^5k9uHq6in6~qBRk!2s&84g-ySOPxYfjd z~g|!M|s}n7uf$I&etYZ~#HsVBRvKB;#Z-L&7nwa;civybT zaKI4E(F-kib-xzYA-F^GA9Og&p*VarhF|-m=jVWXShK(KUE6KkLECA))S0s;Dv8-( z{K+Qa&5xHb%$MI2JtgEwm(#7bqb#peY;e#T(1vjZ8}A-P&{lsCTmQtL7)NO^)i@IR zJ$dDvmTT%JWO(3W%U z+E}4aSeYdQjHQ5Wx19OK1!$NQ6f2-hK0OrukVx#uT=?_$&fS5L!3J5jx;RIh_stx( zvuj`CxLk-8es!FCoIox3$(ZM2l3XGmQCM5kM6KH>Ts1VU!rN2|lfTo#geAp2gx0+s zW9qkW!e!TAOou^|P29wOG)se`Bf-l&TMGzFGd?OtDQui3oAW_=H>qcE|9WhMLWrD- zt>MhS>89M*rGZd#o3gFsb3bJFC!hw6V}i<^o@;|cL*ZZ0Vc}`dRbh~X_t}X?r7b`2 z7#}5xH+ok8Pj~$3)X8L)*M+-rPaNiv_P^D6VY}iF#66fNfjKKQmtuLnBn8xD5ukZ< z=L-+Z`ZIh8$1q3>yzDPtzOeo`_4FmHZqF_`vZJ#*H6_Ix$UR%du#}kgLUuB_YQq9M zLCj8a9a@$9BJ6X1bhI{J?cX1v@~1$n&0=SvZ(<^}ZB`#NvZ1UOv$keV`bg8)NE5JV zVpMNi4SI75+v~uCE2|Y6u#Zf;jK`>*X8J=3*Cr~1fI-%DCFt(J$nS>iZQC*{bWh`z zlgg6~H$Inbdb;PK^z~og7n}63TdwgLPue3hQ;NMVJpJ&gC7H<<7B5bb`)i{Tg85G5 z<1iYP8FN)mfAWp`QYEfl#!`ri26~dh{>EsNj?-k*^1wVEjc{U3jUXo{=cks`)7ARZ zMCvDwK!R-jaLp+ks#PK&?~vu}4wJ2d^(H53%^Yq?tGtMY0q0xLpobagII~5@5LV4< zdFj%fSJKa(zG#yjDR*LfQf6fWhSF|hOyedlE+istvs-!^q!-6QUh`>TWyCi>lD*Q# zYY@3uLygpVP=-sAj4Hyt+fkV<|3@(HD8yFb)QAYurAR`7gIb#0_B1lke) z{Q5Flwgg#f|Ly=qU@nYyPS4pjnuxo3)aK}E&K-To8a-`ER~@3dw8`ttbgBA zKs`2rKVJg_A&T>pJ`OCWmX>}5#5wuAk6^UY4%;uEDbp{L9L{;mCyd9+xGUz7;MQF1 zNVydoh+)B#`~kKLbmht0!l(_}f2WY#qUnJ;h2Pzfsh(@NKg@$*YBh(=483G9S%;$6 zHkS|fT~6aQb)ZH2(%pI({SF`7!onigZIL=BC&w9>XCpN(w_UXt&kp|Bfu&4#-+sVT zbAhn4ti7YAmYthx*cwbe<~(Z-4L|(WQd$`_ z&+YXZEbg!4QL%qYNYIoU_?l9O4NFiy^Yt75!;^kFUQhGl;pHh?)XiYD6 zkMJHF8$o%SxPux)yfBStJ?c*kDWgJSb3w9;A5epTIIO;e2vCA95Sl*D?&zMayxKY$ zct~n3j99m@fU||i4x{7>!a4)(V5|wG6Yamcn!ZeD6s3NULxsT)>|S{B^;To@qN1q8 zhi{#pn^RCw_BkYFW-`LuAEqoC$+nVV`|x0CRq-p(YXiQI2@%i-;N{oaBvY<#Aj|;? z4U{e*W3lE>f})~jjSJ8BN#`WLfL&FdS zm<0$)%%a!mXZywmpJ{1nrE1n++bi9J76;(B!l{780mv%wHfzYq?_Hf`lA8j62hoM_ zASoaCn7;~c$b0jAZ}MOBZAJT0-IpsmK$|N}!O|Hf7qDlRmk)Sn1Y|Do9l!}3=pU2= zUIzc@D5O4wc-JdIF5qYgS>Kpq=DyqaJeV3813P-Et2=%7(8iB|I_Bk~q6v`Ef$Zmp zM=tZu4!Px=*xGs>E)1;I*V>Gbm>ANcV!MA1%?1nI8mGtclT$zfb?7pN2k&?H&X$wN zAy$~sxwGb^=v#ABaLpX{OGuZ zGOLOrOj#v?Vi;q@@%8Kb8@HLZx4YPg!dg{i3p0D;8pD!36WmB2$#uOmw~Y~n#{$RM zFA>5@fezJil>lJ=Y0twl%c^1kHCIr;M});n@%@wsg^mH};orH-3TpwN8be!H`1FoV zYDpAiu}f9SFFAeKW4545V*lp*js)N8`77< z70KqcEW3uTj-71_8vylEk+<$3ir1NQi@fhQlSO9Fq-`F7V7 zpIXq@a>@N87@y!kdnP>~T4vpBS5+I+tL2-i>L!}%G7Dc(@3RDjxtSKZOC zXIF*>+t)^U`z8>2P&NgX1>Yurd``PRtT%7oxGhFL9=VdS!} zUnBepXk@;xVkg#Ty4Jo%c+K2^ApxCr9&XCi_}jL28v9EIm>?%7hb22FC$%akP}J=S z#8WdsjF+o4zUhODJG$TZoJzJ39p|COl05 z?AF&UU;~Ano?pWw7x4W>$*ZEG(g~3t86<;glA&|ZzF=0)YsZN=Iyw^EZ@!(Yb3*D+ zS~^lD53gC6@ID_UuV;Iu-Qu_G?CHUdNH`>NZhqc&VV8WNr#2dtCBzj}{XK{i@6bs`v*wFe&T$O)2sO`by#|!z@o5n4H^WY3asrbd1 zNiY=E$&wN-fTEGc-RU_w5x1FSM!obzm=vRI)*XhYyV}2U{gM=KiyvN-uVrLqDgZMFm@Hx8E0`b{q2`BA zDe3764s~57?~DB1w{6@GkiU(&O?u?27bZdje1034tVY1fTxLUt&8)0un*(mc?&eNjmw4o=+Q4Ke)~^;RRY_%+h*@97@jbF zan{b#wZs?0(L`$vSjw1i9_iRmo?*yspsm4gZVogT0^u6WVs7pKEzM^6I?B%vh!r?* zKHEcf;D7}XA+7?ABazSM1Jb1D^Yl(eCP~(93@z}ZM9>CQI%6g;HpHC2`z#q?Il%Bq7W{x#WS6 z#P3<-`}eN^C$d<$@K8rFD%hV58F@nE{N4g!uZ09|*nFOz!a*CgVZ*OIqTwG?a#$cZ z0a^kOKoVBo%7jv9<>%uUF!}8N;jy(lfoT>GnA@31gNSZ^>|ePKu?ckWogmb^>uTMc z&-PwneHrsStMJ--^Cws1SEf8AOlHc>#k9)79|G5#(H5qvrXz3slJOD{r+{&nRa8Ww zYiz6;`bP@I%AdgPY)y;FC}AX1s1G(i1mO@x&26}m7TLDkgsB{%gdf&C`3;ZGzU9bx zSUdujJ;b=tDdK*%IW>uW2Ccux#+pAVr@q+}s$?e%JX^ z;-JWQkEP6fG&q$R(}}IvnpPJI^Pu_$$DRRsA8#PNsw%LkUxkIGXL@r)gKqo{ctjYj zvRHSrxd=(;XrX}{%Tet?-HFTMda3!y4_cd$y%T`v(+Y(P^LMIN3(|BgAE!x%q=Mie zFeLlO#^m}w5e@PVW~sSFM&%B3<~s}WPH=rOK^<|MD4<%|@gNXxdSth-J$bRG8W|CL;hEKiKg1ySeD4)_c|mXl^OWosdjKKgfEJSW?UU?p-wfxz zDVK2ZBHzqOfyEstLXn2Gkqg&+aw(texi~%WQguDOD0SNCRZBFYx>!s8#otlsn@GPD ze7Uc6Cis3J66a-S1|K17Y>bxj=YG=Xg+E`M;US~_Z5fgWEibz= z+Kno-ZbZhkmbA?tM+wkvk8<u;s)VoYBdY}$G4vvyr9jG9b zi2K8yw704|Gg6oalPO^T#LM}?ye0tw0X~Nn5*m+P+Myv0sUv@8!-*I6YC)x7sQ~GD%7;%aAT|G)1sb{}1d>uS%VKah>$-f9CVk@r@+{G* zrIq9ZMe0?%%B)@Y5n(%vLSBK|M64NYxueAz$4i|1>*KOKS}o^iCzVcfvGIIn#MlHi zzdqBdm6Wba4s&f#^GC$Kb|>n!TV7pV+-gHCc8?;d_-%b#WMlrdxUDQdA~i*ix5gv0 znw)6TKIEH`1WLRV+9NS&b&o8+*nb5oxYfFDA)|JPH%lgdw#ReQD$872AX>BXT{oq? zAPn+|PP{8=xxuJ>^+vx~UE}6Y0>-`Z;xNo|fEyD0)(qL$`=wfGf7=6@P;+%Bbxm4U z=O4k~|Ah?qIG7xdh9~hhQ)py|q3PSVZ=f#|hS~MGEMLjw=8Bqm@OyCZlUV|*yCD(8 z3gDXm^b`bM{0$C+D!K%0VuXDg!Ru7YpQ^(c}s!L1`C>Yk7H(Sk)^& zLBb)Vm&=2|uxCZc|KT{)L3}klO?w?8wZN_m#oH!5{i(u96`L`FQ14zdg1Hsi-SI zZ0A`hD9d4BjEnh^El9)EEe=!|A$4?g^c}{S2_M7RPc8S}3a_oC78l#ExeWItou1yU zPoK1@7&%v**}D2}I9GoXrXgWKXXxbCROiJo~nV*{5Jw>zFtE?PuxSqf*DCqRKEtID1 zKrmC7JzC6)2wh}qW(o6RV;*aP5*P^G6xkT4cOzN8y$VnEu;i|7xqKIm?&Auo$ziRnm9VYUhE4_$ShYWsxqv#B{4mD{&$$!fO6_Fz+;pE= zz7$43eP2@QPT4^&`M{5tC7*Fu#v@?sKKuF{hEy1&2Le+nvUgE@H2}D9M)QPnNW~OA z1G2uswEOY&q)S)~XkxBMHI5I35PHHx-g>z5}(e}w^Tjzw!8cLl8&jFZw#PQAb%Am zF2G|D4SSf?4d)Q$V*}5?yqvSDp83&2D_n2`H{+tTbHIGF{a8lh z8AGJ>C3s+lk0=~;n8*i*m3qJP=naktDiI6`t1R1(I1Y~GapAw}A0MZlxXYa%lFd6H z{!2n<_}aaso~!(d$!`j!IOFN#Wm1T`2QF9VRfjglT@vp&>BOx=-DEv~-@B3^MxAtB zjNlQo{7lqy^TAMIg65qXYybI|uHVXR=!67&aSO6ryc<*78uYR^RB@T|XQPN)a8|K0 zs3|WbJz5WB5#x1U(L=4JjB9V-*jiKv>Gh|B*R6moLItIS4dFC)tb4D|2b)-AytH6;XZ&kwmu#R%mY<2{TS>&op=6w49FhH|-7YR`{)U*K#Jw z{5Y6*mFkUQ=4^UHrf z|8(#D@1y^IV`u(+Rh{rEddct*Ay^}PvsgsPG z8m_s!cExXd%Q*vM0;4gfwic@MGIFmSBBPvG?1*E*oaF+S>z2F!#U&!+o1zPiR~jzC Of6@{P&+^6eKKwtr^-P`s diff --git a/docs/img/grafana-control-plane.png b/docs/img/grafana-control-plane.png deleted file mode 100644 index a4494fae37438553ffa144107b044a4c5abac2a7..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 265550 zcmYg&1yGe;)GipHNGRPPlG5EN(%s#M2I)>|5a}-I?(XgqknZjlX}J4*|9|IRW*ite z=Z*dD^{gk>F+@&A6bS(r0R{#JNnA`=0S4w}I}FT=;5TsKHw}Bz+%PcjL*10r92E4O ziEZq@8Jk%`h#g#QAjA+CGh-MSmxb^oD<>lgf$=H9C$w-(9zqmS} zFArKV^eW%gUA=iq+bJ(kC{M|`S`=FSw)M0qbbBfJ0QZnKw7Y8MMN8UpcPX^$y|+gj z&%d&_d+EM-{T-D|JmJ<$_G!7Q{qt$u2flAN61rc_+Mdh99{rM>r^NW%pZ?u^X7%>q z^U&^)*~Zu^KR~}MFMpTmC40-hDd#?A`0#e^(gq>h*-s z@-U2--}896bF%!na>*xWEMJkQOL@olVDgc#VJ?U$%P#h!Eufv>d-}RG_K%O*{au^w zId4`j`nMsE{0^cgGVz$)zSC&qqSM;3`!sK>w$uCgKXzkeyAF2imqD{*Ce)cO^8D9U zr(;3f*Dg15`jydR{P)PufBn3#Jj=7=HYa|`U5!%S53OLyY+7Asc>EwIz5i_P`C8_@ zylp7o;$moj?l>cC*w1Z5w{)y!^;X#Xb#VKyluh>(1!POX2AOvlDk2}{CEzG$mvnLs zB{8fAMSAT&P(2jnZpS&ve0GFIe(AA)Uzx=Mxt0yUbc@8q2Z5HoWHImzuoodspj%e?xS`v7NUaxNk)6%V@Rewh< z~NLVmgaCtXUmuL^0#^OM;j+$8Mt?*6^k`;r&Y#{o+yV~QMoC` zHv3vF@KVDgj+%y&E3YsQiZmlEDMCthEbzL`bwpw`19|FJn#@LXotujW2UjX-BqYm9 zQ!}aFX@A7ZYaR2$Gzx_gFWbQ?_-j_Pa^g7m;OzIr`{#jnELiO(3{q%22k$J9i# zybvdoOpXoCXRLeih*x|BZZCev^(VQ{5cK-D`LxEP4T3q>cPUs)rXf1qe zaf1DPt^Zv$V-ZD1Aqija$d5980u#etxt%z#TcI_4c>05-T%%lArxMpp*2bd#a*0t| zb_3I?`{y9^#SLOr0iC&@$H_xp#Hzv}=L_pFLEVFTLMI%UmJ;FcYJYK=NgMzC@d*>u zU~}UNi`d~%}oN*IZu%Z(!K$6D`YsM1IR(T%vjb$rzE6z~=l(}dkPerab| z&p9i^{%PoKV!psZt+iT`ypaOTKHFhHT8SNT^2>vnhP0wI9RC>Ds8LJ|g`-cpkq0l` znD=txRJaQyAczi-xvafQ4o-GwC1NApp${`7-7rYyYojup)a8@?(lV^A%wo-o>ah}= zV!vEiwK4XnVzF`VFNWGF*)(~SsK0-EgMT^6@ki~~0tcS^t_ri`dx6?5djjE$RcU z$Q8k#1V2L9q?=bvOKsaHlulg)jJB$;YngC}@dKoB$3+|@3RGjPDH>ZVzT`7NcBb+6 z2@{Eekrv-#e1Da4A1wD-5k!U!YYmCgS^iY$pZ_g2jf~gzfMHyf_nx|e z$v5S96^jK9?Pb!eVz8dR$W_sc?Mv*AKK(6HT%IA9@cB!(*e{vH6nAS?ED}0RHM+T0 zFi~*u1muz&D7Zd!8`sHSYQAWu4~;wuw`gMpI`oF;VhiwFJ_qX@vr5 z9(c_Tf!`u~4&7?#_dWp!9h);vn>p?k%f4kvMp=IhP#CU@7O>>J$HgUqd>AtwFplNLWF_tv5A&nHz-2%cxglc8ui}Y90fkMu{V_g2VYFJoxl#6e58L0BOcl=#m7`HQJ&5^LV`RdPOoe9AQ z#5a*$tpiC*DU&}N6SWRdB>iblx(l%EC}V?(JnjtGm-x*yAvbvh~X_IBjntnPZO z-!0?Im&o1hYTRn4TtQxm+4tWh5XQSm7!xqoZTztIgas5zoNwO42MNNt@9_v|O(-Ii z!Vx*#6Y~={HNYGPh;udBusOYFZTKe9!AK0j51P{xNq7VIRkvPsqwP2Sp%tB8eXWW) zl6!a96m_u?KenGKR>LCK&x-@Y*+m zr1C53BfZD8PDB=B*pBSBo`-Y_JkF8yTdVxD8B|g#L2Gg$X{%CvSnofEr7P{`pg5ZB zIaQTmI-2xlFIw?OXd*<63K#qRj)^ije1MjDirY}F&hcVl8ZnBYF;@q*-1k`MP38tM z+!9i4Xgh4`DVxs^NMzz920>S-O|{GS7j@A(Q+j;JuQYw*rK>0zF=P)TSK3I?Ud1fR z@!6wu*`QVREB&;DH=@Xi=2f}g7a;V^7om}QtuFqG{k?YJBb?Fqqv4G&g}fKr$j)aH zh}v}e@XMlKBDbj=duYF$d93;8o(8_KJ8{Rvr#2@2g3fC{o84O+RCZE=U5e;9N|93m zSL4KRYsBeZm@oBSx)@=AOHGhJXX>Q`;yLFnQ?dN{pHxg36C6T$v zUrreQBxYO-9Mz!Y@P{|AEX05HQu;*nE3Vb6Gdd)fQ4_WBwX7G_42@_Dhb11kQa$>w ziOFo6nR-yVnxv2dJ@u2tn0iF#GHpJG`kL5oxo4_n#J0$lU%g6n>EyysWKmBw5o{*= zS|wi4PfL33uOGSnotKx=YF7_GinIG_W37%~O8*7QK_~eK9wVJob*0N!7q4XCx=4*a zahha)mlr8xN-=DbV8v7$dC%YOhO^E*zv3{=`c2KL&Z$>3Oa=Q=8 z9Yx(1zhfo$gI99EiZtqsM+_I1MYQS>FEIC;DH$9x0> zvw%W(M!^cXesf~WZ2?=>XRf#$CPhkKe#=vh7&55sl*jGjFY|`@?OFbAaX!P_>=wP2 zNo~n5mz;itJosOaZxs(%}H!w#dv&b&}n*S6#cl!Bw5W)6kH{EBj^gF|7~yT))=gJ&)A;Oly%52|X1$m7<97HR*yMjp4VNXHe5q zxhHJJdLBMU?xUD%P$(EatThlYY`%hRIagLP+r?!x^Fw@l6s!~E|2fSv+`Q)G!#|r^ zV!7baf4ry#!LO+~^x(47u^sYE^d{a%B?uKd)3BpTB$_6^RS#%uZIU;^O`PSSeVgfZ zMIiPsR;8D-g@F&@>&q4o!P6E;qRHUJn(1iKdc-`Oh+x_u>Bvi1OCfZDend8?=JXu` zVX4>zB5aJoPm!@9#N;eeN|3DAoBfIF9UepC2)xGLRpCKPrA%T~sjLIMDNJbHC=gpr z|E?Vo{YoVlB9e^L&i$*;zu;dX{_+3oMqJBjBE2(}t^MzuEEreERw3fuePqI25i{C* zdb9~K;fl_@_=CwCgT_Fl?$`m!ZuPHdul%nv3PLzlf>oxKX>4C?V7KXj`w61VWy1^t3zh`<3-k`w~?y5wr|B+E>4)hwOqO>PjI* z=q`p@oh+W#1F}2Di$pI)$5BhJTuqM}F;3mWV?E)EMAascF+VE!a8ld`R5fG>Tn)hx zedUX!tVu0=v8?#W*T58Prx$VDi9qX%mUX*|P1?d^B-6H+AdJBokCr&$I{CTYueTEN z^!x1>|LX%vTE}BnZH=q+zNO$te@xdw+^wA_93q-OZx=HdUOgp>4a~f28h)}tovQe5 zVd;lp;br_=$77W;SVz=8^eHteeG_4QVsS|%@|Ue0cAaA8YunYARaA%qdP_z82%!!O z_ShB*W?j)%0->Ej?HjJPs1=9WHv?ZjlOLPNV6$s~F{ZFrKzG%W^A-pv`CANb;2|8NXfYfXT>)K{w?}qF@nQ^;8XW@NcbC;pO zj=~O%am^>6_XwK4JZtKOQB#Tbf!6PPG<$?pU!T86PatlX8Jd5Z!^aA87qCD@ey!c% z>eBS?>1Z;8=`&AXFo#)9gmZwl_-}{TcW?$Am}f{diJAtV-cZqeU*5EwYbBnaAF;RZ ziY)tFW}{W`>xWe(9U?MHcHVhFLx6M#b4>0p>0th-1T$-Ftjn-Pc<+bBoj!v#;Y6JJ zQ*Hthq`M#!4^pI5n=?eVDT6(tn5|-EBo+iQ=M5!yRh8#lAN7(z`0Si() zo0p%vbN)mI|0Mj)a4elPF3tF+lv33;Nz~j)W-Babt9nihIlLt&t&l3;Dz8G>X}T*e z#Kf~ zV&C_rUeJ?E4=^WQ*$as)y#ZeyZ;XOrV2EMFg$0#dQjc1lUGS${&z_T#tTe=n>C}5$ zCBW(@Q}wZ7YcXI;?a}%Q_$-mKzr9Xa`pf=t@KHPZ{R>ib3XZqaNH(zSOIv}q8qyzzID&qa@YN(Vv@2lhO+Yct%twR zM+fbG21Fxqp=d~|48`E!;M}njJ~BK^V8z`3obA7NuGz1(#OkWYzYXfX@!-S7!vjAQ z6iKk)9=oKI`jYC`FWPdY<>cqbL`Q$0+83U^x-9bf?^6Evit;yGVpsF>$iX|!%*;4D zL#f3p<#_5z(1H)d0tGqn>tZ1h{a=iY$-!TDm0|zqHvZ?mGX3iP4-(P-Wrr**EH*c1 z`pGFN+>~voziO!4nBDWv&$ooiMrHb7EThq@U*N%&mX_l4X^CrV6Eu~DspIj2TSkc8?>)7k4-~9~jXCN5h@lrmJ+H7Z1l&T_1Z6a-{Fbe)l@n8v z3|^7G+zO_#OK6HHym8ci4cgCx91g>XM5V}~obN1uf&)p!2iTJ0gU9UeEnw5Y+rcBo z!NI|j#xzwdU_*SV9?y**@vQy7=YoF2VmHYF%Pvfbp4HYysH|K$h?;DDHrHX9jysLk zr9smjhiNtLfU&qk`D@?O?63U*me4d(po`XzVW}#Q(zGgnK^nZ8A~W#JA`=sXhqv## z(Dk%uqaaD9ETP@&#Hb;9^@}0GB=IDc;JZlnK$T_O)Ppo-_lwSichy9?G-1~NyLG<5 z>&DUa(gf>s>(Oy8ur8-0ieypq+Mu!>b**L)|DD;xxd5#Cp-zw)X%wFF2Zv;Ey3&eB#Fy%LI8zD!y;EVEnpQmPwdFe% zjE9^*wF!(k!#l`1kcpFEBN|lCSIk>tX~ANK)#fl5KD+h!LCBk2SH~t2K`0G@(74x* zI&CBi?KUM2Iqm+OSLRhxCPPO66+T`_+eVXk{61V{!g0AOM;CGC?^G*%H^Xno^pnLn+Z zg7t5fa2pQ)EQixn)4G_VqE~)`LYeQQ#Cj$z+7POKtG>PTG)opCMg52x^2-d@=c=vA z%7j>)tM?$zPIi5mrTdj%g@CH0q({l2JH& zwpn6~&ie(hqJE!?f7YF~>Nm#TcO6+Fk-#?TNB)O$JmXDiX%`k6K7A2HvH-ZRuCD&> zP56a|B7iRrj~j-jGW)-K@_j!EB*n$wy1Ke5sj9|+{l9!4ymJ!8Sp0A56P>_`uZR1SKROtbhwfMULPDB*feU@ z)YVb5OiF4i-Tjn|n>%H4Qb8#5_t%LGXb-YkX9UjY z)ZlUb>d)2F{KCSwU**U4+}c+oR!=GLkh^nAFR;qcaVP9b?7BW;nIT1Z>#EPsYq|EuX*o_S|J|QT(PI!( zv`5>6d;9h+#dN~hg>SyIJ3zpU@mayQ6R_&v4fVrorTO4Ynr%KZmuH3C2Bp{zpng~; z?#(>w{?N#y5-aWYoq)IKa-Wf({-|_!G+M-|KJeyz%>0BfwX-=W#c-&+Y-(mkL_uNl z?at{9llB|*ubE|K$Qk3JUS534%F2IFmQ#v~KHvR29`5vm$Ki2d0KY^>N0(Do6$Ks8 ztDPvQTTn#>Q#69`t30~JdpYH}zkeOP&ishZ>f0Z~zXTzF2p7TObz?!{IfJ2ZIeP(t zKseIIkgY7sz6)o|D<~@N&z9>3ZRSB(o*v2;3W_7Pj9&gJJTZA93>TrIp%IRUIGwI| zD!hA4ilu$J79$mMa$>7C9?fz-U4a4Jzs_p0ulcmieP%qEUv$<{|t(A&EFi% zzlya2Q93!xMU2mgFN=$czzPPz6S*Z$7pqMCmXno{K~~jrB~w>d zABZ05==8&8F=bf2U-7=^=YN82UtOh*@ojWrH6Bqra>CP4+I_kw`jwO8Up1)!hloE= zmd0IJTKc$zDGcqhXSaqLwU%snDEt?W#-qu-1(Kqd&yTarCM>gJ(LD{en_}Yw0#hu1 zO=Bk~namH(EBgbEQq7V1;wFDLB4)}3fB*gh4jEUN1s8H%rp?02svcN4HqMCnGN@bM z^&|~hdlt|np0l+tQi%+{sj2unZu?(Zq`R%_Gpg}wsE7`m#=@m9WF$O?a_H?xrR2Qz>-*BC z-a~es@&p@>A5;6JR8;#pt3s+5)91Tz6N_=(Wn-Ke0BI3l*)(aHD6gLM{GrX0&KP@% z!XxJ9#sz4HzyB+T?zIg2Re^Y;i;GLwf9&mj+jeR7DGz33blS6T;rs{)Nnrtkz>#jyCClsC~MIYl`ZXX| zV99CGDmE?-fb*zv_qH7r#T7E=e^C9sfBb>X_&tcoE0KN0#m^5OWy7u!5GI0>W#hEb zxGn)=y~d^FP(2TLq@Zqs^g{=((#{#B*v(~HY3CQ!Uy@#`wjBBv6{(aIEnD|n*Q338 z)o^!EOhrYdKb0pv(;O^JqSCl=3TbP<8JXYtwsXlU{fU`5=G6Q79-q_Be^$pGseD#4 zf{>T1lx}*#L#oV;I<*$WG$4C%4}j$*ii1{_z7bIy$;U*b0qYN^c_T>&>Ir@bs*fINNDj zWF+7qdL2HnT3)x?B^tFu60fij0=RkxlV?*->n^VW4J&oNa{X{WrW_g?I#>UV=yIpE z-?{yo7j&SBHd905;mWtg=W~qYGRZx@e(*Qf?N8JCsGO=S(%GuzD-UN$CZOzD*w|1J z;edKrhv)YQ^nD3mWxg%pb=|xgicJbA_)Vw*ofvg!7km3H$CSy)FF>&(_%&t(*|>1> zXrj~81$C?m*ToW3zBMH^+V3jp=mbubH6&L3$f7h&mSAyPG6h#DKmDf3S}k4g+%Av` zFZr^5-&RdW$9GwOHpaH{ZHV@RkuuK){!q znHbrR;;7?KJF_$Q^2%|=U8(s>kZ_$c4q_n#km;Y^wXDRlnOOD>l4@pE%uP1l%P}rW2Ir%@`{! zE3ukpvz=Sk6~9xgh!6r2xg2ppt^0HxF`QR!Sy9o@439sh z*ljgx?ya*7ee?L_$b&53{mZ$^Z^;3kgY=>|&U{-#}34qw#opc%Iov8ux6dd@2(JVY5F% zG#(O9uRGS{cnCHlP|cvh^s~-t`DIaBF4l$TN+xVv-{WvQ(Euu+1+YJf*`&(z-ihz= zuOhirq5$CdVBIM=Ig>zhtu&n=k=&=Hq}1CPNiu>!0MvA}xLy_%6&c+9-Pdy6jzh#| z@UbXs47^@(k0{lwXR==6ux?m~Te4;+BO@Eh;9s>nOy%?B$`%QK0lg_2jhf)VK)55P zW(Yy^?M@2dZ4alHLAc3W=(ZI`C?yZqn&AmS%%UY3DC$WB|Y(=qZn-TzrG zEGz*2t`F)L)HEioMo;FKpa#$~aX9Tr!6uW+;7h~O^@xIf2^Sd^C1_~)iG9_B8MI7r zQc5AXOzdjM!b$HLN#=_$n225cFK9iZb^+zx5M*8c+9FsIF$59{7ulMG-W zF%+_SCgbUImTaM6Vd{-`WXXAqjt5Er^J!kIC@Vu3IzPi}fzP#Yrnzq3(sH>ej!ZIM zZ56u*u63o=T~S+mbfY(HaA+tCh+@0?Ca_LWmgBtah{bF?vNp=TngcM){j@bfr`2uU zfos0j0_zZ01-dTs-v#wrM8Y1EIxJ1fli$ul5d;u$n+?@6uELwoj(Q-39poOLG zsZiCwwy}0YQfEZf1{{Wm+wt+sb5SDoOl}jq2r@~`smaMM3{@>b4RhS#1aBz`2?Tc8 zw4q;ueg(zFVS}nySBQi>iG_;!2F@IFYFZ-))5S?Vw{RDimo{96`Yw=d2s$R_2L!kz zes4mZ@>#p?)~h=8mg|#dfL=cf@@!WhQ5eAOv|Joek0i5(jn`RiR*&-;{!4S7l+%^< z@`7bw<%w(aysy&h^Z-0xwsFm>Z^i53TomxZw#VaVy@MJvV)2%mHCn0yeJY@ z*0}oGp)K^P+7>YZl1qKgoCQ%AQU3w-7$iA zTk&Yp$;rtx7X{PH`iVbtri1paZdXF_5Dm>%VaGI0>!iVxRUKwkpxGp_D@T;-# zE$FL)g0dZ;SHMb70j1Hk?;zkFoJ%U6lW+1KIf%pQ2s`=}=r};=lA;0?&O?X-SPu{i zHuuNQYnOMXmgYT+3!zi{Ibbi}wCE%!C)1E|(bJ>&00jSj%y_u#*SIGqC)ZJ*?h$=) zVdr%lmZ7Mk61r^t3IFHsX6J9@P0Mq~$XeQJTjO!e7qy3`BJ97Bu#>k9PDWt=iq&Tu z+RR~*6cAvU6A{MsPF|4f%;}(18jZ7vu!5f=`b*EMLt0%erP#g^SQ@Ql8l&$wU@+-$ z%s8fr61w_Zl(%AbGt5;Q!nZ$c(@(IJYCisv?rmymNrfT{(2#D|C(=MR0%ZpQ-ow#K zYYJe;){WbkEjs13cgniDvcR`k>XGZx6#cvg!Un_djn3d4TnnNm$rNy&HJfk>Oa@Pt4$@cjZXQ0{qb zv1C8a%X7VCKa$AkXS>;V-iP-Mm_}kMDk?#zgANpgd>$NOn6yH(j?pnOz$*e8Mbb8j z#S9gQmW?#$m36R;QfXXq6zMMCSD&A*0gCD95C#MUwESDJpwX=R0llDd@5jT3zxn)- z-cXDmxU8X&;rL3EaYCS4umO|;?)WJ;Hl=vBYs$brA}bbksY){6JDCw19UjgV|IO9*&6F|vHO z{y>)jOD8$6(s&dXfOI#2)cI-?6mU^JJ;LBAffeamw>ca56aj=|e<%iZiSrZakek={ zV2QSNcVmE8aHR~SWVudjut@gb!`TR66)-xVXo1EHwpHyyX7Q7ozIG0oMfCAhK6@2S zwOy~oitD{JE?x?7G}o}&A26I!!dIXEsTUO&kAv~6f7c@;QhmCftc0>Ob>5(OACAlE zW75)kuf5+_KOkdsM1nS50an(!q#Ejc(ZNX#FHfyLp@8-%0=DmMH_=na0&B+FxyzUf zqgpg6c%FkJBU>lUr|aOsL3d*a8bx$eR42XHlfChZ8#NF=o4U`b!=?4heJ$53*==p< z-#j+ue<~4uq@W0TRiDEDw2tC&%HJ-lFaP2Bp8vV~3kq+~&H1J)XcA%)5+wPeC|UC1 zY240{X=&1=sQSb4qf73qo-WZb;8cJp$YvXo=F2om?@lEL4mk zKwo~CW=Vhg=MB&o-AH`rcK6vu=CUHs3>^alG%oc<1J0r%)Hkn!3qFHpoa4enquZ8x zPvjCAm7vvC6h}=<#{T||8llZjvprYZlG`E;ut~&a{#BT+zo@gE?*d{^8klqy-1wHu zRRO&2*9-Ck_RAG(13vicNpDyO0%;--OH_6kc`d789A3+hTOyE|QuH`))5jge4M zk(+ot;zbx#Qvrn$kMTGgGGSH??ihKbD3aUwi$K?%sF*@Z$Jw;V+X=zv~m+ zo;5=2?;nwZUJMfAh1zKaj}F&dKDDVBb(kfD)R?qSBKdJPN1X3%&;Q*|8W>m-{zvq# z7oGpHNA0!t?CoX7GqnE_Mg;AqyO;SD8$0WNM%v!qE>Wxe4oVcHEI*d5^NWfQ+E-=G z%&4TLrD@cw{Fj!rs6VNTh`^2WKl1=5kD~UjsA%X@1dk9PL}6I;CwgE1I4;{6GRP}W zhnCkY9{-u1J|E*gUAuCduQ5Xd2ndz*)5WUHW|O4xH0oUowH6bOX@CAGL4o!A>vwMg zgrJMsy!!YvIvUkTJx{mYOGU~YKzyLSoR-!IuwsGO{DH^*Q;xDHmD64hJpf3d3-71# z=gf+VNGKuzMGuS^aOHqAU(MeYc5 z)GCpwsUtvC4aW=~x@^a(np#_1JG(X}gR1em=_UdU z8*nC|V#sCFN2(yhGAU+b4j;rNB@GY$6apS1At6C?jbrbdTT=3a*Zm(j2=H-->DyBQ zWz>v+03}*wInM;V7iyIfy^Wp_C}4qdA(u|>0nn%3>_i)=9|=6`SSm%eCWk_0J^*d% z>Nr4&uQndnEj9qGpKq}J1eM&l;gf&<_ybk%?!rR|;vzuzk;TQu0po? zPas+Yzz7u5a4Bcob5Ao2Cq6Rh2b8q5ko$kfDkK$So}kYSx2`;lI>4UNNkRTF{L0Nu zO3?9$lA{EX2B@SEWolIh?-h4o2IOcirAgOU0v2|0vni{G4OdiDWx1~a`taFp3*g8B zpg{^%UAtG4 zb4yYh8Vk0ndx|51dG+ay1(wF!tj_VltJ)iu`^e5J-3$XGBU!n*0e}tx?j-ARD30*% z-R*8;ib{4@F9A>_GYdr2&TyJ_zsrwJnhsg;u?!KbTGay(L@h07QC7A z&1qGg^&)`pXPKe~yADHW=+lPQza9Rjrl#b+J`pId5+rE`rqm5_++jm2unFXmUZI0a2XnVJ8`7} ztOSc0ym(v}Tkn$KQa&3tYFxdMHrhFl|9hlCry6N!c-YLsqI>iF_uL%RhXmKY*JKSz zp`)ia8H%L>RZK0@#GLoq{b;UoYkPa$h68}lP5_n;sKPAS1}iA_34NWoO3} zO*#QMD{yH{;_4e4aX1KvEj|Z@Wz8#9skzq0^sHydw{LG2BWj&A3rSd%a_qe)L;gI8J1mL0!`M<;Ujs0tZf zx2f=c_TqcGF>HH&JXMsJ@9Zb?9sxdBladG6>A>`;SqaHAJ6fnM{*wB9tFI;`s_sf9 zT2x#rjJMMT<%H+yI{NT^XtLPYC98%{#4pL z&tb6JA_H4irCdwg*0xg417t31)MA!e2{%(2Pr*fnASNM)8sHl?95)TiFV7q{+X~-+ ztO!*+I+6RDDE^%VOHou()F6<)fC?i3AO*afPW1&_N;t>?6{Zx-IA#RCQ+F<$jHl|x zIVH(Ni1;S9sB5YQWzA92d^Q-)lLj6O68n-sN_qIs4kmC0-UNIB&d-|oxnBJ|ZPN7N z)rJF?Y)a2W>L~!gL=H1lS>7v_!a_5ToAvvL?EowQyu0Yv)Z(Z}NLM*|dC^DWnJp!$ zFYm9boyV=IJM&4!jZ94J&pKecG8ukup*e=$xGdv+Yxy86@fnN=m?npg?`~%24&RJ9 zw+-iKSVN`{ra97(!2|8cLknF%2%>-j%?6Q7- zSy_BGD=Rd|SeP72Z*{lfNEOcJ(r~w|DZ|^_i_hPBdb|h3G^=zpB5NStlA!!sVuZAm zRF8FiOfqhs~e*cNI4^anUtyp#lRAhs_cjB+h?9$_APS zV4Hv@cL8{`SrY(d2+ft?k@1Ovo%h+q1PmGAv0LZV)Wk~CHTr{4C;n-fky|OG|?> z7LBq`dA+=RAAF|vjkbs4JRZ*b1ls|AF5WNlFjvU2TaHj;GbQ>!ia+eIw^yPc-_-peTfZR(eewY?&Kw|2f(YFT zzIa&o{|?$54~__^BUk*Y513zxX8li~YA%4Kc?N_n;ClGOD_ACi38W!rQ+e`AO8)xT zP~8bm>8$p^sf%b325Z5m{-xdPrCh9krQQR0@OXD+ zxbSx$5sxnw=>4jyDww$zSLSz^BmAFW#s16B>7-m`W*v3B_P@dyk2C~)^E?!{3{8up zmObS&4;)>7?(n;5;j?akL}p@Qg0fh=u&~C*pWH4-DG^iU-ri2fbAk}aYxsYsyA%Cw zfXOk_C4`OhJ;v@cTW1y)PH4cyFPt7ba2e{z$}hXEWCgry07ML%%>mmfC@r1W{d}eQ z^!=L}d0o}vt65o6UfMK4KX^qA4GAkN+9dTPs0p+^oPdT9Ak*wqddS7Ti5xWF?#-X!`44B2f3YvZ>tWT%$Y{;ViVT>-Acu-|>JA5nmY?!8gTc?X zJRaO{d%^Ve>Jg?&7VPGxJ?!oJ*rr9s`|xxL@0Md7v*o+iBQZ>PzOA@}JAnuIrjb;z zJD$GMgwE1oj*GuLWzZ1)Z9;2xZk`=_O+ zYrnZ8R21zb@#hjXHm+_Tvrh_x`|CYKAvUwhNm6*}xQf3v)3M5rl7@P3jl8)Dc3`F# z7R(zN5*G<_^YS(>Z{NWA0nc>f$_@BynZPd>^n(M{gB*y6V?c$j?dT#sMZ-P|wKc$R znYW+mjwj{ zzp}Ev1FHqNS<@e-*kf;+oPK@{SIq(8zbGg&dGrAA1fkPjKMA;JIxh+-Lvz>SS zl!QSv3Ve1Y6_rVF(X|0tlu-K?xD@r^WK0fwau)nXDWf7F^a2I4Qd zUl63fw;n2~WqD?RWM5od3;>I|s_G(V|7z4N#ZeW#S7BiEF8SG+10Nh6IvNBxr%pK* z==bc__yFW{0hs_%<)pAhp#E=g^5gAA*X=2X)6v1f&SR%)z0Mav= zcN{wbD+jP5L*V_#W6W!H6qT3n-tK0Y0JaEZVp+Gd>E~deRC(@?s-YUZHw^0%_&=J2 zAC}h@J`E#Gix%NVM!EDMekk$a3d}4;LQ`_klViJ2M{2!LcNZm5whEsKfVbT$Uc@OazinMcHsazvq zsO&Wf+DkG=zXAqY%jLhty{0i>eE_cs39!uD%iYRvr#i_g$*zZ^>}HquRCFsHV`=u0ZRd4JSAxLMas$wl4_pTz8V54f_6Ssx{iZKAbJBMx@*wHd{`IsQ0bg^gN6& zE=$5bwbn*mJJ&Ll>5_~uPX~QyU7lk}VkaURXf2PeaLY)j3S?N#aK=v1CjWDC$Kxf( zwA#jrHHaKCh%8`mQK7^)H~pPd|JI`)F4QzAF8p11QcfPQbU~^kuc-J1>}xQ~l0PL5 z7&(ac)6zx(5dcis2()DAEZuUmvu6BE*%AfdAdwIp@IsKoJ>Si?y8!)G(yFq!u+RW3 zoH>vVfmoj$q(fCzOvE!Ewdt_xm+amF4zsnh1AU_D?kAi;+_>H!*Z&ux&yRbFfDrBW z_7;i^U=1-Y5@>{O9r7;@zon;d}27}gXSUnHTAc0Fv4Fevm z$z+Z=c;wJgm}oM|n-k~uVS2CgcR=6B0(r7-lL&R~rIMHeK7I+51NW+-kql90i(yWp zTL%V2j%)@W&*ca+v@C(T25Oko<%}4(NXEc?qeee0f1uxG+B2ts|6S(B&DFxymD^T!sl`bM#5t|`#U9DZM!J~4omg<^Ts#t z`;=M*a~-sZ!Z%G?bku!~)p7Hi=aC@zcXVWh0t7f2kQEU?QfnN4>zi`=X$DqfhWPs5 z;*Cm#_ol>mK4$!}`H1qcW{c=vxtnORvcv}d8M z(s{*|0<18UMFBqx>TL(25Xoq0;3g#{%{)y3!wsw-q4$HW)A7PrTU$nuNiMl;h8Glp zL}m*__7!)c(D}suycB2zT1BI%r1S^U#b7^?+1Qw*`*@%lKe?!_Ns^RK((HVkWg;ls zcD@I~b0l%L(jz!j)S^5l5R#gjn;+F_z1Z340$z^GldN~ozQg3wWY^{RGc?KtclN-Z zpzXw|S!Y>S7mSKf0bFShR>n90=`L4sss*LRhKKRACM+i@u7)$^VGtcK2-dpXJbdh= zsG~Es;^eGWzi2h6Qmj&1(*>L@EHIc7FXf%ngc}?%I9;4YSJffhaVC}emrE8k5KRGZ6bmn4hFsywJ{}BIi z=WM-OfabnR@F+R?lPKT@l~x5xaUi1t<^qvt%@1&HKa=!ZdVqQAwdQ0nARvYM;G%6$ zjyt375fS`%ZxW_@vmKp8EE5>32C#Skv340xk80ZMTB}ASt=JHGCdoU@x4dR)6Abxp;(k0oy^x^GX?P^XZRYn+ktJec*6BcF?E4Q$tvS2GO8(05cFe&fs{cVgZu@ zJP$B|TB6gM46wA1Y4InKESUNOfeRQxs&K!y22nH)AUQx=gO3D&u?tfwMQ!zL`qsY+ za43X7z`!;(VwKe*8&HU)4trqWftU}zNphlbPkzzkC^P(a{V0Cs$`D5XG-1$-6t zlLKHk_iUb5fSC#aSNrqTpv#w^Fo;ySMK6ARTWy1mc|dP~j&jG@ z9_$bO>UoeR_$jkrK2ngAYl!FnB;=I!N2Itzd$#>M@BR_yUz>Enn z2W0}2eq|Tmr@Rw7XB=G)Not|{^eM*uq%o-Iz&xH#-ST+RvuTbJ`UwUPRtCtf7^VND z^&qz*7llzM%G*PH1bEack1WlS+tZlG<;Idc5A=*+7QPXxr>FDXbgDcpbKKS@`wtxG z2#kO|&^15Uf!zKr+zV(^0&4-A`b(_Pyk`LqFw36=sIcc+pFt-!f#G&L{+mo?0`oIq;HzXyE>F38Q}8}U=ouIIys?s&V<-)M;{L6z^)5e(KwP82LSLNzI1F} z28BW0H+_2jNjw^o%X*q?mh9M?MG4{q>PAHld~!y{3ET<;x z0v!iA=a>rZ{B^j-8nN8WiHT_}?d4VXU4Y2W5BPGcWc_Q5tLf7@Z^e zDqr$#;(-sO!E7m`Mcxe9H7x{7*7b3bqcqj~&9dDK}ob zm$w!3N2KIU+)_f&&u83t2aQh^pBwB5)%j61C&tSu`Z8^|mBl`eNwm7QR9EKN>qL+B zZ;YRQ4ieuD@Q)dMaX#y4U3`Hf4^1xIdYSvEJ4Ug2|Fzg4=%;s!O@WIPF;e>GKv(&>dv#QHT#H^Aq_x~a5Eu-Rk zvajI;cXxLQZjD25hXBFd-Q9z`OXIG=p>YDi-3jhaaCdn7H}lN=XRWus^y=~Ff4-76ahW&YFE5OD03NvPWI^`_&ehQ|JR zjF-K}p@)6Z3 z_d@N@>s3=DUEPy*7ye&fVM+<^O-gir>a zeIybP`ymshrKL-3(4g|3StXrvD>bb+Y0%fQQuaZSb<&L%ih@ef5uq>(p|9-O*3FSufc zI_SzTHDmTS>y5d3D;)a|4-VXGPz^5wIB*Jqh=A}7i@NzufwT1T?P66*iRZ(z2%$WM z7};sP<>7J~WGlqKRW?$o#XN_9W;L+OapynN-y!aL?;yPV@gLgUKg@+I=Y`|;9c)9e z_XUQ66?_|h){83%8e}!z?xJy1_%ijr8*%4QHfMHL0hU|o{3m_)k9Yjf%TbWasG1#K z*gnMBmFGX`B>h_MH0D*--wjbD-I7Vh^o@1^ei@k&Q*-mAl$7~j|G(NB?DbFuSI1#lDpZ-C$z+@u?Ie}$a~NG=Wv4@pyTod2Wv_27!M(05;D zkN;%2-@*Fd&%u%dbj7&ANj7OBNyX8H0uX`uJ`opl2b+cQ|6#)^Mp79?i&GHKu?I-7 zLhndZQ9zl1QxgATFosdd{EI1w{l8Q;czSM@se~QC{uxoS&zFL0k;|~v9x5W#RxJ7BW9j~%ebC?(Lv8WAfeY)N(W4_+hKZKo7}0;{X#isyxhq>6 zQ2<7O|Hr638tBWPsQ9!!c2l<}hyMHVKiggdx!jK)r%DG4Zu9L0#{X+3h=sq|)2F85 zNH)o}7e_=-E13U-1`LE@w7{@nX@0CJ4tMt#Jeao(MTZFZ?E$Y@|CErH}rYJrR zTsx^uQi5rQBag2NDw2ehKZhBCaoO-_sy{lk(;Kwdy0edFiT-d>=*jn&Wu*^%B13Z( zPNFjM;8FC+bx-mK?-D`*NPS=V#3~bxIv{!m`ZDhI^cIzsOo+kjFb~ySSU}AIlo4>$ z9?0n7Y-axeOf!2b_5o8agQmhc&OFSsgOAF<)dK&i3`}l3?)Vod)AdampqFk3*e|B_ zNQKt^ADafTAOiH_lU9ph!QdP)F$D~a06Q;3n0ju3-@^XqroS}ng<(ethm=qTQ%{y` zo70^U4e`_k>LSc|xT&-xX>c%&`$?n63ahiMuBq|0DAITecJ&hW-8EVtZ&(nhOeF@V zi-s_Rh8By~?KX73uptrab!(0?KeAJfIFC!D+`QIM!NV81h`w24(^>q)d?X zv=UEMA!L#Vz$j37)aATu&S27`rylS28RSsa|uB9cMls zAAD5^Juwbr*@Fy|AEvbQ%OadaWHE^Gs^nNm`S@j%VVh?7h3LLrXh{r` zFjmNtb-b94yl1qwydG>6GWlUEa3_E@ctzvJ$goXZQFp2>=i!jI>X1K6RE(=ct?Te~ z*KLen;h@f>*(Et+HVMT!sBBSa70V#^LnbYMOh`1Sl4Bqg{7fm9RLBxy^SK62N7?eJOdb2x{g@8vPG&DRy_qF??*g{B%zT!&`y1(k%c1_*_ z`Rzk%^<+@WDF@lvStjL(w{SGCBEoEcCESh(`#c<#1Jy#d;B%LhnNJ}j^v@sc#ua|&B@ItF<2jjzCk@%=hjf1ZUoEL8^ktON9MriBJm_|j{62VdIFXH z{CDV9SK(MDeWd04)N@T-2=I=EyDZ-!a z#6bQclTOG^w;lX+LJmC@|JT^R*_Ur}2m4n;SkuAc0veYuxF23R zwU__F0uU-KQwU9XCj`9T0=_JEDMfC|1R*2w-DT7~CsGC0h&l_I48~FmFXz|SK=;{0 zG6fY4kuj5UCt=ZlHEASWHW!QQ2CDs98uie4yS;tWEnP=*47&p=mXBR^1eJ=JcPq!7 z9`9+Ia41tD2_TX3Cw*U*@{5+tLkDtro1zDYBT)&@1-8^j8bYODa;4TnfsgH(I!b>$ zaug3W+As83so=$sDSf=1k6WFPi06SZ;-Q8*gxhp!p|VMK(nc&Xtw5mvJ8KtH^wOL$ zZ4U_$|2E`lPJpps35%lZlb?fU_U8K{(!FnMOnZ5dSR4nQGSU0AeZ{`<22*1bh}lK`Uccbew!bSKCNPHwdN2gCW)T|9s( zTJZB-uL2?Rgg@2O8sl2fuHS=c1M927{UpEQ&%dvIQ@kzmtE00%uoCTtxLi2%nstaX zmQ{3-)RFX=c?3Qe81-U>iE#Jg^^zqOKW>|)FBo;q%huW+a&KOCCr_E-U@(*MeoY-V zo{l?SWbijW_-x$dJ_ud}T5`5W4zY3*Z=* zMOd-*&Hm^PGu$Fsk@T&@z>=^_xhN#;`W#zp?0O~~@%im}x>Cd8`Sx5gvBu=)p0U)5 z6ZEj}?(v}C(OMJM-7I+wB?xmq_M^-1=uabIwVeFlW(UeM{PaO)UuaGt9J4DPZ6CTf zwTqXtK~^N+l1tr(u|-Cx(uL^;Gdb;MTNyx>893EGzqrVa5tASJA12;&#Ch0^~5x z>CvGWaki4LZ75c#6l{dTz=1mvgji8yDGKTRBsagH6udeLfG2Yo&JPG9<5e8dANS7R z7E;ITf2Epn>-7q%sv8pEYZH4jcMF$9Y{tE!-|*{=UtGMhHAM@TE!c~7%-RWf&Bb`xFK!viO0lHA zWhhsaphCrVeH;d%z??;124ND!8M%bB;^z43yMhjrC=I15`<)s340TV#V_kGi4HM6A z}3Gs%D%ol2Z|>hYAaijjs|O?a#kTbSX=uoYUtmKXA=5z|II zjffRb95WX=@EBtype=7YNpR|XIX!zgh#m zGhIg0`HuxJ5GDgIRprSenuTIEH~o-2a$HE}23YMqFJ;;D{0EyY<@m7vT)o;YULR`d z)K#%Uy#A6T*qWzYCH_5S?;p~0PN?gC&Nc2~%l_={sQM;pq3XQtsii zfuJOehA|nH_wBSjdLBII^{GSB3+VSdI){T&hp50g=S9oG})$X?|^w z52&*iFGy)T1Fm-VS3Ql~E}L>U=y-eCjIvO2(8WeZhi3*0P9m})3zp*fQqIMV2n-lK zDO>vut)pZFO<%oC8}0MWsuN2U`mtK$f@_0E=#dbG&!$h7Zg`t)s2}2eAh}iIs72sV zX&1BSYPu44SBld+TnWvbfT*{RrD#ao$~+s^E??~fEk|LjW3y!a^{%#spIourpKKSvAP&)duc_#-N%P)*hw zkWU*gzICo}rSDUB`mg4UFM)E@!(+EU6z%&|xj#s4o?~`^xFixcaK3D3EU()mK$$zi zEE}cs{@!J)I9|U`0l#)7)!hUl5tT?}*QTp--zfq-?Gd>6;sU?g0Nejs&xV4S$uBwX zzAw@VFimSKw^%bzt@%^4_$v=mw+QvcD+)w-SSXQ|6hcMnkdyoT;QU=NaO8|+!u$gN z?!$eZJR}t6$u{7WwFk)CSqSF+m?dRvHNmf7iq3dk;@L3J9n&9rO;_&IZgi`ZivJ9m zkR^ODcN@Ju?~%vd*sf^_Xtn&;6E305MUDBpd-bh1vrpXJH*wi6PZYEM#KS!iIN%b

  • n;bXV_#*Ne2hQ&ubI*pP!Wx-M~soaX-joR@8QHem_Dp6sg z*8}fiiB#*(j1T zIunfZcBpWnLnV}ZVFU(PohraL!9v&(|1K-fNj%=UVHJ1-q$M#M?S&KWI=~nOOi7$_ z1f3JU_cO69vRmJ_-P+*yRaEay1$YKor8bEh!-!fs9~1mMnN=|VCHVJerHq+C2+q+O zptSoLYH{O^?<~GM=%rRT>y0L0_=F)jeWp$WZrLRSF70ayi8I!a>qA?>?Q&;B1@HPQ zFrns}O*}<}lG$Q@=8c~HZ-&qnNvf;79C>oI$71Ui)vS902&|Q{Z@+$}WtS;R98ALUn9Li=zuzi7@{Kn?3sHtDeZe%koDQbfeu3dQJs|S6Bt7T^q3sYXj<8b`4oemb}RWyKuET*rID9eTKF+Bcsat5tQmASgIhxR#j>t}eP zsxuEf%)A$-d^mCo6 zeZPjh1i0}ZY~!*WjEm7lnrc3_9OCnDD}0G-i*X4Tf3$VunB*xLY2@M$Wi>9KTXCe14e5?2 zEHK}cP8Jqhc$B_h0fa6UcN1syG90I>J8wb<*o5DSE$(&4?^*<;6sl+~V%s7pAC4mo zT4H44g+~nzOYmG@pJD{IKYgJ{ifyF5+VerdeKUoEWB`NX55Lg}-H_Ez$tb6x(P8oZ zijB|hjvu>t{*e@ATzsMI{#z>se;q> zUd>NwZ~5P}N#2g(?!8Y0-}To6Z)#doAbnCXOVS>Ogva~wg${My$-|N9iY(&6Dg0x2 z5+5(>A>C^mMgC&K+2wxzsW_9_TQ$6y4|D6M?l`CcFy!GM4i6ai307tBw=&pFs15!~ z7asE`=?zsN;dDGw;N@EMUg-AI3ebi~gHi#rfH#d_{Mc3U8koCTc4B9{ywXu^u+g=llWcS`gUo6m-wyh?Ux3# zh0XNad(kSui}RZQM0s^Q$LVk|y$N*c4w?Vm&D_%MqSVuw9VQ*`D+M#rMsT}r2zj64 z&V11d``f-uRn+Z%L*=CTs?Kh!&1qixn*??jii#3q-Xd$vAcQ_tC}z-9fe2nYMz*V< zm4=BH!jSS2Ssy2rX?&t+@&w*35o^f(AV#j=U%Q{qLd(wt0DE1frqxX4?q}$}Ob64~ zr83R@=bLB)eQ{tR5&joEzz*G)zvfrNeVD^{0y%7D=L659p;c=MMt7jY){96D%?^c` z(#`E`x6zr_=+etUSev7O150MondAxOO5W|V?t?x5l6ih11t1SRxl^On3W+lb zFmENed)j)UWFFnV5hC+Y&XvUS3SWNT(Rrq1p`+FYHX%?<5= z84bNYwrUOMMYrz#KU`<6U_NC5$K6s+nB()Zg-ZRsAdP=(##_KGVw^)`6s&zmmc~Yw zoZ5m^G|jQ`?C&L`H;sZkIF!WF$a|OhGNKgxaTT8qJwbeo1|N_*Ub8~OwM$y@DDS~( z;;qf+N?^62T#ncfCG#Hl;l&j0#e3wTz8}zxu+oV$h4VzbBMK=wFZXx*f|S#?l@!{8 zo#Oj3*X={ubw}Pu?y@nQ2{TXT>`q!t=ku3#bp?~&y$nrkJxjHnB)9x>FP(N*+rU6k z`sB1T{+VVkh%;CTMrCIEV#LD_Z_erk`Gdv9^`S6#+UrK;aLh3(D_&EmLlWVTSW<u;t8_WbU+>uVZ6-u>htYsXlvvX$13&vHlmh?|Zt%-I` z!8z)=%Glfs`?4}@yix|dIwFdp3Zw&OIOCGi!y+%|sOGd1M3|6<;1Ou>ktOCiwYAfa zxqRClDFwr0_xcO;$_=`+)d+18u*F(m%MP~`eX=*#9(j;RMxZHScK$rw3LThA5^ZE* zUv%|Xa-v28ShpE(c`cPTX%%7D61n7g1wJ zMyo_#GfKQQQi3c3FGPf^NV~0XKIk0Se6!g#dMn9woHeKt6a3TFV)PX;19gqHdm%Ne zUPtW)tWGk#hw(yLlk4}!^GBp60ercu=Xt-Y%(9l7j}dXI<&~Qr?MO&oI2$$i`%isA zc$XlBvYsL5k@T^)$Ie6~ONRs%Y9h}^I7NYC{JD>37({ET>J`-O*E}TJmRGTNJ`-Nz z_&5B;>_6dN$)tx*sNAftKW%v2q-{D=N>`N+*{^@AkNck};8e*Ot+11lm4XkNZj@TOXH=2;{mY}B1KHTOxy%)d_ zZeH6Ab~WXo0}TfK?V#y_0fmF3EwHLK1VlhlDviTv>Q@$@yAP%iD3%lD1$ze+yzhfM z{EMo;%$V=&SoVILSEsiy2OoN;oVl=j2=V+_0MjQY{Uf^-^^|tPar~1tj-`(-0oPF!6ohm#7j$aM>_(c!__T=1Fi>T}W2Y$x2G2iWZOzCtxG-T|L81>^c?jh0aEyw1?@ zcYV2=nqWizHA?I-_hv!wTYI=KN%sXxN$ViTkYNHuICz&LYDV0&PIZC<{6`i9m zn|{jWaGga(XCu#snRs-;gF7o{KWvI9;zk5TEU1XcTnWSBk^*517PkfL36u!bX1ABKF)QJ<-~G4-X78Ddk%e&D+_wlKYz*{?d~^o| zO_Le{i>4ewzvB6T9UZx%z%D^ew&Ty{HMIkA{k_JXWuZ)Xb6D=nhMl?y z4AtiLM<;KZB%5GHOM`{`p5hsiBgM%=QL=T&6Tisut>oO)bj2n|^UZbm@+)m#lJFXt zgp0LbFo}Mhys0>H`vmJDa|KjeU7`YF5;~Fj@vGN)w#6UvxeDwEoZCCG3qIc(_J!|y zSy-k0io@ZoQk)|wQP>$&rxA3f&6xA=hvr_J8a5+kzt3IOTJS;hX`(Yf)9>SH%*mIK zfR3zLrw6W1A1ide55aowRm++xd%z%*j=aO(75f$qY~(-(_!$CnV0C4}ag^K712FdW z=_jHXj@D}_S*ou`=4*|2a=5R6cQnQQtx{+zQ>?Le0*IbC%4a*Lk7I}-62ZT?um|XjX zyMBpPo{8#$^j0MraoWqUz&M+ZG1rq@lItD~dOG3RpH}7Rd}isqZ&6xGU^H`4z**E8 zD;$u!ES#|1DJuez>CZ+n>mgy7hmBH1mQlBx6W7LegjxtG7&R{w(kpa%KG55D-!SPm zLgN~AvYc2bn!n!EHdKH;YyZe`I zyg}s4qqVja?!on;UZyC`?*$YSyho4dRGC1r{sL*!C6GTuO;n@rv97wkeV>e&w`z*C z(-u7C%s`Ii#!p_o4(`JoF13gV`1cR2-sr1xA;k6{qeKv%F3sh~&R1)&fV`35NDcrBD7?n5%AzFNc8@AUP zHz4WCopC{;NIvReGbTBz2PtSRSqZ;!GY;Y=({qZ+3o zRmvHYDGGBQgs0}@Uc6ibO<;Yn#<@4M2Rw5w4C6JL%Nr@|3z%kcIL*3!U!>1$vWm>n z9Tn8j8%?yx8N4t zxIAvq@HD-GSWvfMwPVn|AiTbthEO$MR_P}0Hbpy&Y8YQV`i8-F*>E>a6IFl}w+o%9 zA}luv)O;F;8oFX-O!j1qGJ_K+Ni4;#BEAkD?bF87)E8W}v1W_PR3&O*3yDm2LJJG6 z(9O<{H(J3f_J}HjI&s`ZPrhA9^#%Cx**l?=c5~%YM=#F)70=52iM$s9*5L{%(94XW zkLX8UV9M=#{cFN|7qT1CtZ;RX7;WWYh(23AbmU6}7SF)e+4PDM?&^r!feceN{~=0d z76)QyJ)p_%a|CL_eU9Fhh$jm6yzNIaE@J^7u&;+QJk(4Uq%Lyyi97g2D#!L|F}qM0 zeU_=@XKl(ex-_8iz~Nka(}L-CRlMhQbjkPVbMeO2C3Rpgvz0^f&H}qI|-bB12 z>BL(&@%UMHTd17|*#9I{5-jkUfJ3*K3uIm@?4qHP*c(PvnNI+j+%<^r6ao{ zwps<9Drms3yTZ%!ihiKi57Y^m9@WtvjI1|l>dzD>p1eoa1kBZ> zd0XA_p*f3Wy4=2SmRgg$cB}!m$Y)YF-_OMh-axfe(EBwqseA4RUbP>{wnv#3r*Al;=<5rn!GJq(!a^ z#2=2$;#oJ%jvd6mbVUkPQB$RUy(>S7ny*LTSxvuPF#{*i=}&(DerL+yxY9rOGpEr5 zhXDX)fSSsLQkfci4!N|K*R2?IB0wwbuP~x1tv=WUZSMF{zTgNm3U;n>Ul$A(g^a-! zZCdFUl({<5E*foUl$|Io)ru;sA?oh-6tS1PLW@m5-p2$;%Vs;6JX$*x!T0c6%gxBQ z08b1%b_T6xf5au6q}MGr(m*VgA1B*33XO_BGu2`?#>$bk&}jwAD#i;A#xvI?AmLL} zl%w^?+;E~Pf4#ygP1ZaRagVJxsSV_uG)KUOH3pN8(ITVCUs^a@|vOJCyD`O3AK zOj)I85UQa?8D6E)bK38I>pXjaBXmhS=GFAEhyp_ettPOe>uQ~>_h2!H4`m^3_2_Ol zXheX0S)m>wfH}R#LT+J~07icW_5KHE(2+|Ro!-lt)N*siniF{4&Hch}_K+*i6N878 zJ6r;}qIlPu@Z2r+9c?sm59J2%2}W};YJqV2n(O2zo+akeIj^vV^cp@Z$s&n`#5oKv zO71Prr`@gOB`fg-iu~eJHEnHegR8ep^SS{{8*TNxZsq?vS}9_AySI*>^m<|}F?=n8 zP7e!J?M{b2VdCRfs&iiWE@aR~Y$ZLbhNzi(HmD4XTq4)vhv$Cz4vN~pOp)^Y620Dx z8rVd96s z-!h*a1h?QTM*G}nBr6-R%e4ZkSJBbzaz@UH#~VRKJym+}yWa)n5rsEdQQH6(3|EvX z-qlOH-Y{yr1XKU0F{hcK)f9XPQGVt1tpCM(Rj25CJxaOecyqSqJ~tuovce9N>xfjX zFg;sne&tG|;GKaVt)7UEQ);=^L}n>V_8q0R7*p!)lFFa8Soln~h-Vq8J_#!8fW?r1 z;CqVcqwO=2y1imiN$*t_GBQGk4y~@+}kZr`nzM`h>um4~Hh7M88kNM}y-K@15 z-FfLdFqv|Y<+I-Lv)D0?&8FdyjL7yUn>m)4huIq$GoCfh+pb_rk0D$CHY7(l=#O@C z+0a`rMy_BtW?VfXuqV>JXR1~GoS1!xFgadt=7J0(6?q7Wge`BkHa&^EJ#ORnxm%4l zf<051U*zFZ`_8;79$izt+SWVT34ETLnfYXRuHE_j*Bxfv*oxSQR9rPO@0AT!xkgDlUi%dPk^Lye2aS~~t z%|+YpGYaLlL3I=g?gQvbl>Je()*^rLxt00d@Ex!$OUpzn*0~GK9WYXtFp4u1^E`g^ zrK=!-$$qlz8T~h%VO-thA5awKYnexWRN|UofzU@>BUd6d3$z_f^=s>5vy8G{? z#KlIK@k28+a1g`$46R%#W4I6@gEttrq(VtW-dMZ52znh!v?Ttqsd7I(k1G6Q@VzkO zb9M~9veJ3(-PIYtBdcwGD&|zK!0r^bV6Z^8n#kM8lUSe=W6J97i*GTzEzk1YyG3eU z^?EDVtQ%H6dL8L~AX~^ZVQx+8nn9{Tdp%n&(`x=hZ7ne)YUrgV?|%tc(RIOJTytj4 zHA>KZCP5YA%vh0*ao#WDT&m}}YK6P!xbKSq-<47}p{%=jSwe?_I_EnLdHTZ__QZ;V z;hCWqj#EMWUe|%NZ3j_}tsbZJaJ5nJr~>`H)tR|CPrpiU?C42RvszoyP&d4?$Z_Mh za}Um*x_!zLCamPYJ|*p7#2>T7nHgtQwq2*e{+?zG*k_~h2L%>h`+Ms5pxO9t4n+-= zzpm~IESn)&=zq}B?hD)wYOVPk`l@tEo&icUw!MNZsV3&94pe*&(A39sCcm)6RqKWj zQ*u3U>1+F<9d0~g1s_hB?XJld!_g#xY6(>5zCHF*rLB*`)^`6!2cLFZ>tSz>ulW{g zoD$uUj`!)L#!~B`pJ!h4@1Gbo^YsSp&Ckuo3LW{5tXbP!X~@s%XxC=`v~6x!4!M_#T4(u z$GOyBzEZ{3$fq#@cSpEXQ6s~01^Lzs52z{eTdk+^o?#rLG-G|3sli00(#B}izCYTxIf8G7;xmN8+q<&iMXP0AmyHs8 z7!ymk%cg@ui)neUJ18hhCFqmpG;6Fm&|RX(>lV;YUA@B8~LTz<~fogn;p~_=fQ)_aE2CpeuB)-`olW%WN9Ba30)+qch*7r91Nqsww!toX} z_e}kz-J_XCMQyteXZZa0%=&Nr1>-1bbj2=7B;h$jXFho%R%^RCPK(#WlDKqGcsFB8 zsiO3;>S(cA<9V-iQ%!9_q??auWSS2x^{-bK46NQL5>Fj1d7~QA9qHG@?)rEybYd4B zL2_YA7T1lKJkQ*eZ6Ec4<66IRLQ|Ea#;Ee9UVc|1Kd{~}!)C1Ax4hnN zNR9AC&5y==-F7|MPD%s`od?X@N8c0kSyl((Lng+{)0Kc_P&Ct;P6U$>XfF=#F5E&i zZN#Y*dqZq#Vp)QEmAY6WsEy%L`_yRA`)E_$30jh835=d znvzrqrw@=|&ydVg#!ybyrHQTl1f^aTvEvDKRti=vmVyuV{e(!)nR`ec!$VyHk94MD zpdmNN5s?mIw?tJ)quBL}HRnA~3NFw}YkpzjT*W8!tt~n#%CTYj!j{XO{I{Dk-VBt4 z1U%z*vP_wiF6pLQ;uXedbW!VF(|oCmh;^eCW;7i8F;Z93Qav6H^RYY*X|lL7Jlk9m zb4E|hAew!e{G^z4q`RJPUujh*(wv;Cs%p5!Qu-_vPW+aIDvvl_lx(&ujFMMtmyC=T zzLQRp!XM?l5hHPoet8L+9Er1hC}hbH&v`1KV)7hJWO^)6Zuc=slQHf)?`#Ehhbx3N#;XSQ z=4&qu5gr7C>kseQ`@6MU?N^9aGY$9>5)zuN^R&sPdJ!}`(wpSTcV8_-j8t`XW9%$d zxc*SN2$d$*&N4zhlH_9a%gV|uCF!>3wm&1yqQI!R3j}WSn_lZk`2uIOF(V~qXp)iL z-8F{r!bF^$t~|6mJ`_PX;6D@a9NanP7pSj5q11x9#08)#JRQ&ZMe30eI0-L);7w0V ze8DDy{<+9h(=pQv3^@6XWUSt5DeZJx?`hsl9k2qtgu$; zk{JJ1+$dBpPSnY$HPrVYFfLASHsItqN0m&kYnLP~ewB@+Xz0jB$tD|x371+_)1FzC zl-|I0jdLr7GeAu(jx6!BCJwY~qKTuT-M>qt$OmfKgn~X>SX_kN&0e=`0fDfl+`5}g z463=*8AZgc*(*5t3=-xS!L|syWcOr`Dj}X#crmSJqi`xLrAeOTTw$t4S5FbAh@%%! z8R47N3MFP6Hv9G3${sdaQeLT46rvztzHL6Y*C~2~U_>pQp6Lo$a+b1=&t{QMNmC`_ z7%54nQq=~ zC2kVhH%9y+#4mDBB+LM_{Sr++IR!TiQpmaHNU3VWn6nE@MXn@GfwYTYU(h{U^r*}c zW78czMF}>RR@BO0f{6Y^{WM!eBMHnDEGb7-`oF65zLc_7TJ&s?Dt96QH+BGJZ5!Lw zK_@--<69tPyC~G;#bk|7@QqDo2ez-H|IzC?#{RAqI463$VJMKkX_4~U&TUdA9HIUw z3+oepe%TOT8{{f!g_j;S?obKQbqI~lWMr5LEu}+VTyH`RsJ=|GucA??qqT7Cl7x#i zA`odWo|V>G>i%Bj+r2GVwGS{Q!_jctJ3*PrlYXV^8bKP)VF^a`LT{`&Id zgA&L|C26z@*zCK(7R}UrlEsvTO`H)G0h9#YAmLjM-sq*I1vaN3%tn|ci~i8wZYR-} zmLD50!{eSIPyjImflQ_l0g<@uwOac{AWKGs#T1$qprwET`3=GS&C4Z07}Wl*w~%ON zx(e)>OyKp~pv?k-!Tucm=S`2Rl-~_ZAz9=aYq5y_PlWdjE2FbbT;iD?`E5EQA@%WW z77XGgN|Q{!wIV;YjaarsQ?;stKZ}2BMo`YnWc zNh?r=q^s;}CLX&qIKu>t^-D(W9o7*3X!xZ*I`rW8)mRf}kolAe`AzQ9OVsD(V!LDK zRdiR-9xvAFk|M7@gcT03^W>aYl`3l0W!eI|zM&uH-v}0k^PRE-1JZi#J^VLHx%@+W zw7rzz$9pwfd+ecn)^OaB(K_%64{NDv$?=b{Vh?>RV+>;hjBF8xUza$SHvZ58WsKZ- zQiVURbZOq*Ij7_+i@Zel9zFpGgp`eW2XTYek7Q#o^jyd1yNff_?!=31SRMnd3TIf7e^wYa zdHKFiXJA)j&m|!1Nqf>k1QaS5Aa?km6NdH#eQNM-AByI4n{!d~;~_Vj<(f8NU->DH z;q~Uj2QFt*uP)2SUWsdFFNQ_o@9Fig_e2Uo-Ld@OtGo3hyTOAyniHe{Ff987%4Wx+DD6 z;^CI_bl1e!66@6nFqSCPJRp%bJab~(3V7>So>>0a;nBIc^n6%++{f*bTUR)G4GReg z@y(kiK#DHJ=opb}pSeN?3r)krKM9dBVn_g<84#>PeILNTA(-`@a6wqmNC;Mf!?^w> zRrK8wvD+cUhaeQ(!hehJA^g8D{pa$3l?#D_n}5~$r`|v1I)~QPUkuE3`)}#G37K>N z)4l&&hCOnppZ-0Le+>paoJmZ+Zu0*vn_i-a+*JQl_V|_lJ)Qp<4BN*lE?R>{gx>3d zg1_Z;R!*+X*S$Qr8PBd^VN1`>Xm`-Na|gIppUK5P=abJiJW8ialB!G>zBQ8{nAY4> zdwIJCvrGjO$nZ}NGv5vPPizlK9~(CMH3aOR=Wf^5o3O9Xs9gQKOR)OY;3j;2pgtaL zdH8>`{alJR{8|q!0y8Bs9rnr2(ii#5$F9v~)1XE^ocwLqulp#W$>81B}`nNT5 z@{Y269g%z{mXB51T9MzM7|B`P`MR#0@~w=1V6=-gBCn{$dNboW$>ghw!4-c}g;@#x z1|IfDMCtP!=ZbDq;{>*0L(#-rlDmhE3*=cC06g-nq6*~DuN}O<=h{X3z}u+mkI>(y zLr%_pr)ie+*^=B#8WKf<51_%ZZs1I@Fe||dQk>mi!*JgYu&o_FcVwD5f5weXM&0Jb zOj03_L=<8BMu8Ls4St6}PA`R)oP-zD`a|=5sT_Pzx{>iv`)Me- zQ${0k+^RvS6@kbg`HF1+%U|a@>LN#MapVX*2oB-=1>}zq2-K%{a`!7;biDQKg;vW# zIN8vAy4oQOayrRxU#PI*GG$~U6Z%8QqR5z`;6Jl5MM^-KE5)~)XKG6+gF6x|cfo2x znLoe+cjZieD5hxDKw{eYb(bry^a~*!hc%onCF?vI-UCE|yI7aQsjUm*q&MdG9E1+G)GlKlA30OpE(x zj)SnZhAw&vqjU2IoND;7m@8`u2`u$4;u@GSKo?7*YILaG9d}U`Bk3qKO*o9N6)7ML z7~*1)HY_a4NC1Ke0d9@#|99!-G?PBra%35I6+JZsK7zQ>BWD6Guy?`^>} zU!VXGl_PEq(L>}P#G;1}-CyS-YLv*<49)2yE-Lqze*74LN0cW+@!~8@p)sMcK=i5>#yE z>6?qO<$n6%*gq^K!Iusce@cw!BTNcw=G?Y*8}>-fp%e>UHq43tJBaR@oK#aP<6$+8 zBom1SN`a{k5kVAFp`z-7p(RyLQZO%}&M*1;L}nXHr-p@XvqD9s0l$6lX7;k@f^v5AGHlR4KY0<~y{MI>B_Ler!ng zy{>ItK6$~*qD5lkpg9SqM3f>BNn~D%bEf*UrTBE|r$)}Ywu#h!%cNN4>6+oZTDm1& z>5k0yY-1Dc`g;9(-fAuNv&Qlj3pvRG(JaDi!QzSuJ0cctbEWjJuvXJ@nYXOtFu&YW zV3^F%I2&$e^n_V(fTH3&!-LZbDRZ1Y7*-S#yDp^Pv`rw;$wvw*)S{~Cz(C5?CgUvi zd8+}@DU-H)&D!q?8Po`Q)WeToO#QGa`pG}Ez~&N`CLidPN*3R>qQePdC%23>beJ5Fo`^}5cUgoHXM>SmEe6-b?+GWZ68hN!>Ur454?&yw+@+7hG4pe*O$ zG|DhCAuhxvfYawkq3l_(|dj`7vM z+~RSnURgGWZZ;9GhJ0A|!yl{)O<9!F%wH?eI5C4rdQA=Hj7jH8L(}?^gAvH-Fd&wx zoU`UKt*e#kI18PKg*x;x*HU0o0V5zwZ`*2h>EAd(alOUKtPn%$P;r0MH_H$tB>Rjv z_gyy460gf=GePQ9Zmi7G5oCP2>C&iAQ)DY1NBJ}DGe#H?C0H|-SYw=50;uHfem)U?WRpBC#{wbK6FlvJzs61;N2o`TK`y)$e+T<*dH{M4aXo zDXx_(ZBzm*YWnV0(iN*?zej!PW*$lD;gg^b7HJ-?m0CHlJaD*6*Vu1lE78olt*Ant z<&0*WsXMP@l_1_A{vZ;ClOF>=bHyFFGN)~lNdtCCd6rM@@9?kVt)3^^pio$#;_Q{I zru8yUEmm%mXwW~WB`k?T*}PcJ+ZR;Vmgs$cB-N4vv`L?k_BZGs6d{eoh)fYUOp4GoI6IeY1VeFxz}Mk8)fN#oV+BV^AE6w3;f8H zR^|*>0(y^?CFuy(>z9lCgsybTh-b@uyq=!0iO;19xK<6|8%q|i^{Y0++H0HDEef_i zq8(QWE9UyGmuliE<1?9>_0W8|XhCf6_ziI5D`%Gj`I*@NN7GffMZrB?6_Ao{Sfsn8 zVd?JfE)nTix}>|iySux4iA5TuSztjLln}&k-|u;T|G~X;=gv8E?%Ww$mo(Yu_(-(3 zG8ubOn0%9-n3|nyv;^tAqOr9ESM^{G10(rQ2uklEE?K4|w~nQ@zUirntEA#`7A{fN zOOs8cnzbd8DKe>Z^Ept=WHI%oGu^2rWBw33SHx@dJmGE19Nmz)*-{l~WL?;%8;%*=Fi5hTwQCidv7^WmH zLQ45tj}odJd%5`8*;bPt!C?MM zbK!^j+#~8s6{Vj}Yz*;HqnD(6RZ|Vf7s~4M7tX8n*4sVN2faK+y+ps+oL<_+VTS*q z7zQ5h2?R?eaeAhaA9-7rwOJ-)mTErvp8#hy*|YuMM~j;;WjT@G&f&@i4oaXcY$%J4 zY-$p>Wv1YK2@9=5VGEb&PV*L5w0HCGR@{d05SZ*YXa7q^Os5L{jX*8tvc^cIZU{wJ z7rcJ(bnSU63K<~@Z$~abTP0(vCwt$9G+OU|{VjYRsW*|-eml#KPFmfV!SX$kQThrT zN-OHXoS@Xn<2FkpzO4L&YzxC*YgBPBuYQbQzca`;wgjz8a;-NB3@0pV8MdY1Dn&)K z*%Xqq^k2?i#?{MM$;fL(y}x)r`LzutL*{WncAUr5EYUa|(s<+y!1rU855M=YD(l?) z@Lbc(<+$Izea`_;kddV4C<7LMTNugS$ZY>>eK=n5G||hxa^~$7yl}K`&-e_sW9^Q; zNXzB%1cWXLL@UUj^!m~ck>MA!p)ld5ym`yKufgQ4&5Te*t<5O^k?Jiod6@@b+|$9J z!+IlSYbvGAv{Pu#Y1Qy5vHhVRW@oZ?&vmhL|Ng-GmQ?W4j_f;}{E-jTTY5w}X%M&R(2 z)4IMY=eq`$7>q!kT89eLrKnzrle4nB39Pq(YCZY6*$4#$50a|SZfTG!!hi1=`r3d; z8(w)cSB|j{x@= zTz9B?tjHaq2sXw>hfU=qN~Fa_glLv*6++S$64GSF=9C$66Dc)>JW}6==HePDVdK%h zzA{obZ)f|_eN%ff0zWFC{p0l_pLx1nh!ZM=_#8T@(xffflNS}H6`}^*K3_;VZ;`$X zdt(o;01pkxuls=qlH~!DZSsBATq0&3T`vRH=NT+hfCPCIfXjp-=4M`Hy!Eu)-DRU2nY3NaS^Ha+LvGok*KsXQj{E=4-+Z-@|p`FhI_GkODxGT zS`t^mBoK<;a_t{PGl=!e^N-)2Y)2cc1Scz4SB8j|!ff6IQ2T%U1-|MeI%2TO&K?|8 zMl!BNaMHlZJbe?K1?fHtzS%f7HdDVBXccAd@5ZK`1z_id?xa%P?$mf>1k zHot+D$LGKAGTmSm5nwadcIPJsd)+)q-F2pz?lZt0WP(*ub?G8+6-k*^Tj?_E97+{} zK*t=qb93EC_SDj@WJ-AZ1OR8sS8 zjV3Not&BxdebU)czmkV0+{oROaU!KaNB*vVZKtr5_WW&NojN{Oa%KL}zcQ0cUJ|cE zRFghA^(T=(cf&Bb!k0kgh0Pj|91G^vIXLqIwEB^cgqk6+5EvYR2 zbiEg>(H?+L5e%(yquobk&2b7dwg4!E>yp*v6x=>6L7Su#9VM=fFDZ~lkOWkI&kkCj z4Gx}?Uay^eckHy)>g^}a9 zIMJOa@EC`o(1>jYH_=!9=^gHfErGOOvIc*hu3!E$Ah~r4t6sDMvVGX$ip4Aj^8r`6 zt^9ae>(caxEU}w=Pf#eeXM7OThwML6tyNRhT_ zLIaC7#!}5_BU0NVLd?9m4UgIDY6iT5ESE!wS331d;`Y;b_*MOwmS0kq zgv6J(cO)@32RuL{Xfk91LKZAMi7JTt1@)jHr_-#hk(52X9ASHDDxUw<0%X913#YX&MH`Zh%tK@_HoNKF`}@1tjf!0yonaL5LMBX|a|$+y7Ga$C(+Ce$%9PI>bP+9_caNaLf87G0?$6Ik zHfhSsmYJD!Y1nos&5bK^Qwr~+y``cRY|sxr;p~hRd4MnptvcFti_Jf}7hGr8jcVUq z5&YTSyDL@;9F$TQN1n>FPF?Pklq!0qK!S=14d;7XaO7&H62FHeY zw1pqsUAkoLoRZ8U(fQ7RRh~ISRR;Tm(lT1SesK~8DsOl+qAGfrOVF=DrTzMC0(0^a zZ{&^J!%N1gCfBanVpY<^8-U-cCB)ES2KOkz?lKCOn5)TJyanVj)IkuV$f&QAURNWZ z2P&IrRG3z_sNtT!(3s|_4Q5z}W_uO>?Ulp36tVWUTjaw2laONYqjLcy3%_Xgj70OD zc~;^m4Z3fgKQJ5DOXg@&hG%8SX4s?nYD6Rq5yc)H?Dx=-ykZk=hJ60#_+Qnx| zR1c%H-uq2hPO!PNx?WZtV>2YA<95V<^=w$IqtITK8vdE{Spb*#%o<_>?Hx-2S#=5n z(EZO9%~{*e1#KTACvh*utOYlkd89Z+_!c7n3l#Q@SR;;r>0vrekRS1#8#y;MT{5rEBJhA z0`c?hBVKvNIKA@URIEulF{Ky!J+jA1%$8Fs-43GxUhsvpb{wH_EAbMIRa>u*99MjX z)da=L@y^LNl~5ZP+_+|t4$=s;a?>b*G(5vuWG211K*D?$kCl^}m;OLlHIooq?j>w% z@aI%$fj4*hJl~jZIRx8AyGRP^lQySPEFHFma5Z^Lg(>Rr~Zj1Sm#=c?$VQt z@(&-;i3zZHSBEdN<>mwt*^LiWg|HpyzFCAB?GGsg4^eGma(>yzb;R*>+tjf8KX0ad4s;cqg-H~>?WV8Q5^el{g3~4^*nW@f z_vcBNUc#LfmavnzIi5)8-HVqjwoHdP6B&> z|D4AR>_k)7B-|lB=(BNoT)kWHmpi#N&`Gg^8gp76eu5-JiDj;@w5Q5}=B?_+^-{X1 zGRy$f0hS%1l7;7;nR%wMT=V?#+iceTSc6QCT>NXKr8XyMLujBO4hl~87h#Jay}n5$wot}YX?ps zh%xF6bD+2tQfm)d765juO@L~bAWtzMk?Yn;=fAH`56B&Sj9^R+bp1ev-D>ZBDtl>34mtCEDF92(Wq#*A%D zT+|^`@C#FZHfm&Iq_>3Vl9>8igw25$wNtDmq4l|95_zPkV^b=j7uTdnPKrqK(g{!& zzb=}NL%+WJsU!(QMZp+AjSH2ZeQ7qrJ(S(kVenHUdwn&Zw!yM)l0H25oi{9pPN~YwAnk}Ax%sZQ470yVxn2(yk z7*%XiRO(1nG#z02JXuvy)2AkH2iQKqE`gLDRxuWW3iDe0kE7EPKgz>n`jXqp(dGFL z{B~d*eU;o4l%(o14S)HpzfVezi#Ggup4j|~!2mLkEMA_~dIg+evVYVraH&)fOo9nJ zVnpL>htI3^bbV~7H0s4}81jb~I6Hfu8qT1EU`V#eGB>Q@a4f$)E3aUuwVm%e+;Z|( zXd?K;NUH^L(<`_YTjKLNEI?dC`<85-=k|Q~U?kJsP<6&?lzz-d?&kyQE4Rf4u?pKiM(~ zF6{O6m6cAhelsHg37@_y4lu}&&sXR1)uAA9Yyw=idy+gn#w#L<(xq5HcZzL_$g_sc zWRLorH~Rr5qF#(g@-v*oz6K|_-TejJe-0luieIWL(R}Vaxb~1onf1Yxj*2Bx~272IU&M*gL)~a@EK{6&@mNA`Nc8>}ld$|FxK916ryx_joHn{g3&}L3SqI zOWBig7L5u|#xrCS$~=hq7SV>qC>^q0@l+MmQ&zz%mh`Y61^lDJ-(H=}owm?iNEx*W zslBFVsSF8k^dkZoWdPu+1OwT!Jvt(!ttNK!0Ik+y_9=4Z@}nmw%u_};CtFjm7U<*S zaM-s%;iG6&rTWL`qaR3@|1GOUf#$JBw0(G&XL)JeM)}+CcdgZ5Th{ zug?U$8Fr#UW~75GnY@f34M%0~GzduxoxK0{7;qw>bO~s_n{Oc3yhG3kpV673KkYx9 z%+GH(D3N~ZPA^TJ9=2&=|5nhxk#BQSVl`k43x<18uJo0|o@}+Zmi|7k#H?X;Rt|TQ zX~}FhPM|#bTQkKTe-Wi8!<<`}FXsJ{7g6T<^t-s0HE5Xxq~)hn?+SJi?ph_Tj@Z;# zal>%s>5>@S4r!PU)Ma1Q(QGg_WeND}FuJU*rXU*O%}Iw&U)o0jJ{dzZ%*ZrgDH}J@ z9W-=E4?h>oa7x_k zH1Lz=;!%Sz^wMb>j5c*abz6en%S360XO3PXRDaeE+L@9Z?`u9oZFVcdw249>5iUp) z%fa(v72^2bcQ_K1yBg^LNXoK=e9z5Q@R(xl*<7!B^a+ zeHh$*GiN%AcL5UNSE!cnePeX)#_KcRNwU)6t-8qNG_fdctf=^Cl5SPJ>81;dfBO`H zO@^A6o$3r5$PSNsf`iX*kitfXDOQ ziV`O19}AoOU(81sD3kwUm%+JeG=-sAs8e?lJV8MH+PBmqeH}r0up7zCt{d$2leuA&)XhrRr|e9u*NMmp-LXCO+}Sc`~A~nXe)m;9J;whfg8ug!cZ6U)4HTF zy@@V1oDC#WBPO79Kkjk)HO|9}06TWno6nn*4ZyVI?WJ*Yut6iDUj1^Pr{5HvRE|FC z;zxyFb<^%Q-u7W3PM43<3vq+K0{i=|gy9_KY($`Z(I*kHX-N>#N=y`WEHt z%D(~kz8bG4CXdNs44?e#R=()*A0>6ftx#=|P;tag=_0LHjOAGVGQU*tZ!QH0tx~>= zo?`o%M&%8h+*zaBC~RI7@Wiw;?Pfdtvy#)3A%L8JAjqc|Y|T0rOo^TF3`ag+vk`JR zF0L~jgBb<1ZH7UZTsBAu(rmySNL6#QI5K7TB1|I#?;WukPU)J!;0TF?Q{id61jDQK zt!1>iTW(ioq0wrPvcA;Pu#DmzF2U_ziSTqqO`MeI65=YDf56jMvXqd~iR*>B8M#G1 zhWc{$7vo=Ur-%>WD^SqbE-@a&tsU)Yyb7WsF;#Xat9ul0P*P{Qf~}1x{23&g>Hf5N z!&mt;!Cy+eiEhM{JXHsY9@|tmy}1qL%AavYqeKAyDXN=>l>6|((?U(rh0)4U>!wGE z+DX5BL$uD;dZ)>%fu~lOc1=oxAM|nuWuoxmIobJiYr0A^OeuxRVEB5N6 zx+d_-Hw>{7+z3FuElzush9sc#bUHYNx-d;7;X4wb!e`~*t{&SXH^iUsR^%HF-cLhnES3X_p4g(Xyv8}!X7iD|JyVyazrWH>aEyE?WgQ2*-}QH`R$DFwTr|!DGD~- zGoSsrZ0`(Gzk6WIE0j|*&cy$Mgb(HZIuZb`Qs+wnCq3XG0jFy}ljKn*@`NH4(`5b< zOArWr1&OA$J!)rsCrkiZ8es2oXae%rxuL{oJLZeAyacLWljERwP4Pn_Lt#j;wO znYw+fhR!-C9jX)-C2=wCl$ax_hg0z~27P$;udpi0CT3bPd}ZKUv{5gx*6M3Lh}9l~ zTeo2u2lZ}-mOQ-peF&4qllJ0Kp%D2cr{F1le2?&iBL-NPIqx?z>=f@|@h{_iq5*g0 z08O3h;R&+l7(eh(?a+Zt*MGhuNY4b&%0>8jqER^a>QjBJ$?!9T`1LqBY#yKp-d^Ni zJx$%+$hsunUuVX5PO#gmu?oXp67q+UDOJ6Z@1jw8`dm4xw;0CFLi-bm(;;?h&AtxT zRA*(rn_G|{_k=0-M=tf%S(0d43o~w?gVd0tktI8crv4cL05S*ZZkXzBA0lcL4g7Ru zg{-OS1!(>P_H%(Nqz-A#LPvnLnxIbjxOsqd909#Mh_0Kk5M(b?`G9AeIe%<&$wShO ztsQR0sXs7~NRl=$5ji+%2%)Znej&g8Q#3MX5f^;wNN@)(w(<|XRPq^lULt&ek;F^N z4y4JX!il=G_`b<$UQlLLlv(r{JQZi(@%0}_(2(-ZzLdzkE7pw_N=?JVh-#=dZ%kB> zB~V5Ayr$x%{wFKAX%BxrMp}i0ITCbEw0VfktxNW}6N_N_O<^`%N5P5wVwNJuFPR85 zSd8oc1Y2-MX;|$?s zEmRcpcc>XH{gG6+7RFJN;(+?e1GPKL^^0i^#V&v3Y4ZEvrJ~6XH&Bu%J)tNm60)|v zl}$<0ZYE0|8P%C$SkEk}oP;NNp-|dQrk_?p7LWGtMUgkZ!B-z>MV;&=uO}#vCr6m= z(<+v@6U|D_33YDD9vg)BN43~Id-Iw#6ueE4Kt9sIx*=5QABmej8Xxq3$QK!AoXBIE zT_s(qbWw(8Heh*=CPd9)IJYc80CuVnUYQ$a&8BB*Na2;rx(6$RrqU*ME!Pxri8RJGW~RH zi^Y|lg^~0t+-ZGA%shnrzqG`Yt+@Cgay0lZ8bJttE5Iqpi+cJMqXH!`Z~yZA;NG3& zCvf0uF-y<+!#c1`V96E6dxDbgmH66<+~a?H^T9szQrM#K1tQR7zegZ3_K#q-OW*c~ z%x{p>Fq;4H&pKXzuCv?0${)h)ADlo@eI1e=++$&jLOdMEK8Kw#JIE!tQ(HIyWmmJS zp`Ebbif+#dSw?Hfbqz}NDWL6Q#|0BKeBtdD|Lb7(e}B@{MvD!7kxxaR@<~_}^P!~V zw;4^-za_$46WH&(*lJK%6F81h|J80E6wr>=txVNg&hl~^X#L6WJwkP|4fwok`s>)3 zE(tRPN^;P^?&Os@j5=ArDe1I{-_lAf!7NAkB>){})n>oFVV`Jpnd()45h@&VzW zL!vL1Ueg9n%Q%C8m2OF7<$L)&rL<0iZYLPGK_r&vCLFOWnWAG&`T`c8=cJ(VZ+jsJ zbF(>naCGY>mZ|$tZ1Jygrh!l6_koH=%BL7&9heOIVOmie`U(4 zHq}Ir4qc**GA27W5cie;`yENNC6OIp)h9jKsaTrZ<>oUolS|>eWX0j~(wrK?vWk=U zQI;L*kL)gGh#aP5Ek5G6*CeB0_;1JtpH<69?041p3MqN~_ho5e7EwMCB`cO;g`qc4 z)LQiyOKm8u6o??X$Zo17x-*?!Of12OXf}(tBg6}qOTkPvdUwcT_qlU zXLz7F?}pklleBHM%r*veNn*txQ$k17i&l}YEk>&A?YC^6i}~eZgM9_i&jtR=4*h$< z^xC#^5KU0zT%9~vBI|1#rtvP>k4{4jCW<~g*Nu4_F=9+e)e-ht(&b-4m{8$t)~5tV znwa75iInN%H7!xr07Tpd!|DyrIsq!#_TplE-)P+WE4~*JeQXp3TDrvb zy!k_u`>Z(hB!`u^6_fmLedCe#H3@DPDT~6PqN&AUl;)=i z@;Tr5YHkt~Q{hU!#X=P53&GEknXh8$3@;NKoUbw!UG`SXxjAj@iJvf#$eq_0#00*j zGN!bA5`XtaI{E5DwFy3#50A}SD3h63O0%1Ib9FUJT2b!2c3JrK8gpk;fIesmIG5e< zyP;)qhCNeJokQ=|=oMcJ;@XY&*smB$78-?_r`5!aHLtU$VB`JGK19r3Jf&xMo{;2Semqtj*C?`OePuhE^L9L(H#%*nKLF^N zt?|{Euf%&_aW1+8PQo%xARfggv-%B(dIxf`5#@(sG8ln8HN{~?qumt(DJ9?Ipp7Yn zSLB6<1-=RXJ{ZqGf!mIpe1uVDW?x@3)F%p&G(RDrwTe*w04xexJx%O{^%hK5>C)!r z#N-gV&z*O%fi5E=4(ZN#Aj6|JY}LQVYs*tW7EdW=(evH8We0=W$UYwW#X5|pdK~FO&#i1P}=YQ<3g5VEq*7|XM%?`_zbm>&^ z@$yUg$Hyt-XB6ct6$**hxh<*!&UXg%Bsycte2)3xlGLymWl#Mp=x{7mu&}>zrg^!! zfvUTy;Up(}1JXjBPwYHGU%!rYVp+dsK`x^-Y1GJ!jMBe8436Wt zP$A}T8elTexY;f%;6U)V@n+00?gUb%H9db$usR*PbfXzOYQGSeOxL`7c1qg5AK~z&ap&&@LjK+tbzYw%o>oP$Nd7__Kdr>@ zI#=*^SagvvX&ETa>SaPxwx(7b#R(oYnizGz;DkC9DCV|JW6Q;jB@AME(*J>b=WSSgwy1TMNa9-|^iw#eAG zVSgUCpE9aO2PJi=COO5c3%zrfar@VAGT{EH z+%VY~Sks(f08+3fT1V~j)EVla8V_IqK|K34CN00AaPZ&4Peqje8`bR@z9o}F)|`eTt^p#Iho zqGK!H7kfv;4#V0^eHquz#j~gV*#QKr@ZR?1i7fSci)DnzeiUrO3vri;F15eiONuVlkf1!t*>&THquULdWIXmN(Tna&@5M(#KnKWAB=( zuV@I-$nrzrj?kr%ytPifKFfC%&G3*=v{M|$J)>rS?`;N93sDo8scdCW+{`1_!XbObi?Q}!NzLRG#$>im7rNq=jcs?6)frtZ3b-x z0}0U}ki>yR)3{yoZaHm#fm2C`;}ZGSje77MN9GnS?;zQlj}Ea_xbWoM_p2Wf3=WX1 z4u+@ikN9edD2KH_%Q~0AvcGQE@4A!JPpD|Xkd`~Sl zxl7iIIavMWxykam=-`HFCN(v~J9+ZbjcO3L(79GVP_X6}q>&$!^jR_wuaf|g&F1Oq z6#q&@2rN}B(bpvz=ok7~>P>@9MQ7rb@U+0}kW7jLNWM*kmfXufz$;U3o|q5Aesz^L zIzPYwF^N>)B-OZ8lRFuBU(l>%>2J}UWM<~zsg3g@Q4UTV_W(jv{C3kmd57z)3BI;% z9++$^sJ;zVq&YeWROLVtpu|m4SNzCDQV8k!YF>+3MG4M|HW{8hSy$4Qo*Ncnry0g+ z>1sS`p&LXyIjEw1EN_AsL3EbO9@UV;pb0F5OYhW5=Se^rvSyP36j7Q?HM^}i44x~nivzr=B zx={}HyMf`ku+YJKy^|zQ*A?Gh*%1kqq0HN`2W3^4#A(+x-KNepg$`-c1Ba8zi@V@k zNy1*KCEmAiOb1OO^y1wYOl=615!X+hiWD!In7;_`=R_R-(RqtkZGDwJPTo5c{E~6^ z*{J;Z!Zm&~ggF~K>*ImZ@fPs9Ywl_L927BZZ5Z>5?_q1MAp{)7|N`*h3vS zDk{$#!=@x{u7Ty#@%B=<(5`>l3F4pDp8Pq@t5bJxXII8=@GQd3Vt&?X|IE{2e80>4 z6J^GX-f@RtaJ-q|-O~@?cIN67SKu(J12xC!w12xyh)?Fa30pAUx*1I^F9~|>tXXdj9b151Xu-#0^$2NUwR&G8 zXomk*x&faK=!TT?U+nhMKz*umFA^R>znRS)K4yd3vylJxw#_+vP6$R>i`yHU9rT%N zC>Gah-jso#EDM?ppMrN5q~zf?exP=Tpu>W@G)5im!FR zDIH+hc=xdY?pMmku+!_piOCHL!-FgP#h&^EogIx94p&q2lWZns4)v26niJsVlVN5l z+b97GLe6kEcpaz~W7bDsRDtwRFgrDOz`bNJeb`Am;0SvRubRj$B`wBY7jy%+*t<`W z>}(bsmTt2@9Z@}DVI?{Jii8blySp7ksnr%H96O@B{*q={3 z{Uka3yB47>NTd>@`;PrG7F8+xg?X>8wT~;x$7ajCeEDuV&#I61PpCqySa#0AFT3#P zV*31At32$T^QJ-&5uqZkLq7|TkW?A|Ih=S5Lj{Q%)s3-hZP)H^u`jMo_VP<};xWK~ z88Dz52ks%J(}#Kp40%VDBg}(%{HW**_uW>(9*9w-%G3PquH%Z|>$MlaW)x6Mgy zt}j}vbRHQ*HAzLEMwuyk-wkFswX1KDj-B%gCK^d~p~N?35In6s{_U?VKj2`Na?l7I z@Ve}iL>dljeG%KhzY^kB2($=#ET+Ghp|1G-<}YoZi-voOGUKOA*^~5~{s+&;AHi|F zQZcc);oR2qrVE-WzeCV@SeQeeRY?t>4iqPp(qHazcg+&d;O}MQ_(b)c_S&SJ{=@E1 z_0g`E<3)p&iVv%#74-Ale#?_1l4?Zj%d}25VYMvxTbtJNS|n$Y_-ACVl-p#zbPf8& zBvFWdsz>f|ySzR6mR{8e-F)Qg=-m zOvrxO+!7(gus3Oi_$x*AecHQzO=}cR`(B0CfytZ}g`!Qz>VOq4fU!)h=ZQAw=*?r$PB?+!_1u2T;vKkt9Ui1WmqCj8eHC`VM`BX z-i5)rsv^8(i4iy}96}dJ;UXpPv`AH%Z_zksQy*&bL7(qGw>v98XtWPCX5?l2g0~AG zmUJ7O^Y-)0Q)|bqS<^m!@lO4${X7j4gU1bg)YYRTC$uWZXkxLPnj9FQHrev4%(-e%aN+=p*7YR29NZz{t_z_RA_v49rwPFMD%?xas-Q_C41srARp>sWC zE5%|YtT|mS8#p7i*wxy{S#{me70sMR+K+0GsHC34M!!XO4yHkDgn)WyP zZ=A&p*mt}+=1THh7Egb+$tSob(RGoQe{~D;do?vsZE&&*3RnpKaErBj9?_&7L>pGH z_pvvI=BLw7)=PX6@*Hk_d6ECIFS4xwL77ZiT}OI#_zB$=rwR=mJM^bgTW{dWO|Mwz z0{(TFhreB$;)dllWgt7MHbtRaw@?nEad_-76MQouvxE!zTt+<>K{!-hlh!Zi>m&u5 z3DYcg$~(G~KL-x&79@k@DeV;x32@vNZ__#21;4kS19aoYVADn1$hq%nHzNtmL~^*3 z1xsX_GF_^Z&E66N1iWpw4&iT`S_0bh&wP)_O~bbh_lVy%?P%De!GnXU4+m9Zo4xuP z*ya|L6uMJNk}Y@{LY^Mpz)oW>T9PawwWuo;GvTWB^`vv)m#k^RN!5jNICJz`bxWYY zWPZ^tKmD70?Cy{B9XsS-a{J?GvjV*y^WlZZk2T%Cno_eS!#r$nhKHIl6I{ylyjqM5SN< z_E2v>_1aqTZulp5sUqk2hZ{B`S1hz)BIOYyNC6$WdQ{=p#etr86!3xKmfmaBYfb9G@@TKHJ#yF3 zwMc>rfleqHP1ewD$Gt$|C9O0^?NWeazPtYKbkbOI5rh!Fdzudx2i}i6VRH1WnF)-# z+Q$q-M11&0$aL@u(mWNyz;kwjTO-pJ#K}ZtfXi9s>A>tC9yhSH~;!K4T*hj${YgdI4Jzv4%S2niyVR6W=1H~b9j=C zmHgId;xY~NVtMZMSm1&=?%<&kBqlc4$_IyS$9q@C^$AJ(0`wd;R7BCqUHX?CH!p2VcYAwhBsrKUg+jN z-jIA~Vk*O{(cmtX0Zer8vEA#RhU}zKh`xn~^qB(F~Aar2X2es5a zG$G8q6D(kd!Lfpqob=0(Qy*U507qzp)VrWN+>`J)I;B9~0cfv@t*APQxwaj!%RtC~ z=I5Q6jpGhCXRsmJcb^&T4YcsDy&DL_0jz;(n$v)mxwkh}mgoC%eueVgU!_}%slZ#a zqo#XaZyW0s*gr0Bd9d0rdmy5>!Ncj4^An}c&}Io~BPxsg9>8Y3CMZ&6LdX!h#lm%H zpVQ`C_oqSQ8|;2js(42nrA6hfg^9}n+>!;54W2p!EqNwCEB;m&3#pclV^kL*udm&o zth|_#%%T|v#PG-Zc%6c>u<&24MSNi+kHE!N z_n02aeQsK(7d@O}an*zlk_E=*zhvO%j0x_L^LtC_ z=UzT$G>H=o>#0m%wbA>~@K#)kWMyUisBJm?myyUPisaSLJ;YxBqz( zhgJPu-#WN)jD_}}lr~IEn=~(9?k4~N>zK*;O7%8qhGelv%|DYYjFK=UfF2f@dlU1C zeRYX5V9*r~v&A0zyjsiPr z(z$3hcR`3@kWK5jav-nEv_T-S<~E@VTrEg$hOiuNo(O*YeTk?Bl+$7!v};PVMrJ@H zVA?2zZKX5x(Z^L&D`aiIZ9p0R4Lo$U5@-^q`>qu8qDbkHP?O|ewD>s^Gka)`tCN`p z|4NDesaH4hu_g;bmV2lZKJ{nJ%b+-FxsLt25yFqe&*y=(7TZmhN+8IGrl94a1Pc{5 z>@8OwshM!=vjKguv(LIM^zp$WoL`Q2ab&%w^N+F2wa?O#n3}4qo%D$PnL9^pvDj{} zumu%GS`|i4v>f+G(JZ(9ro)m5cqOd;WD_;^^>3)4A#2^bN!_8|J2n7^=G&{f4{8+}>JIhb z1y4jA+E4cVRAg^@wr)<@ZTb2eNPc&znCiMEDA@U4*v@TdHG3}!w4C;H{*fIdP()9M ztTK_=8fN4ntrKN{djhCJ0mpg_h5Ah3tzyT9ue>$oI_x|LC?H-%oox~8X8oiZfOpiS zkgs^8NOff9*S76%2#%(szQWtJM%}jwnhTP2!aBWFd75~QLysgdLMGVmn?y1cx7+X0 z$r&5UOgvte3O1p{>DLE0A_4_W>!|}p8*or4qFSRGvCmsjuSjzD4w<8Uq(UzZ?;(qj|wt_ZUYH7^@88gZE?AcMWlv#TdV9SK+=j^h#Mm2RE=yRJa z7^_u#aI#7zkg#u(3iYR%O77(SYGRfhVDU%a6q^#T9y8{M?mCHqwN;3#gQypG0$Qac zLC<~9Tg#j>|IyG}1~x7qIK#%MR~@uli!aoby?s;vMy_t3zXwnFCo9H^ne}pHB+^e! z*D2zG&$v!l%u9&ghTXvoVE&q<`6E=DCUnJj{hz0tv!ni9RZU>Y?$B0xIjkLcp}p#? z9;g|y9k5tcI60e3{*fh{A(ugh=O=#-f-ZOBdLJqsh&3E2enp;~8X1VM4Pn&s${A&d zRL+HL$aN8e8@;l^h?wWT#gWCyi?E<(=9dQWLAO)!0OUC9YpbVDs5WJg09G^nx>Y_p zw*YZ_=jp#{pB(@T##?D+BF#n(?Y+nE<`y!nuCl}NdSl~XG7`>9erGpPd~L1v-k%R*(45F?Isk?FF{ zX?wMkw@5NUL}X8FR$k>vFM;J?LxLo(LeBgnejwhKJ%&p8AMSyG*^taJr3C%a0%53M zXqG(0Ggvj4imF@LRy31t>24VPty0KK#%(+Wnq^@5Vv&(B;mkij%nvk=$V*4tP`nc* znjd1qK}>ojzoMNfWVj$XDK4e-+qo@g<`dzeJBYPjjU?VAM><4b?+ZAEd96{2f)IXL z(hQ=(8)zsn@8I!o6e0NX=M(oWKiR$jO9Oag{j^dqS6#kDI#UGTH`_PO4?R6O2-bq`EqkTg!x|JgPdR5J#z;M)H|#<-%{sC*$y(p=Pfr=|#diC?8lf z<8VL*g`s{6u#5APN$BfWu#sE7dc1dKM|+o-s2DxXB+x12r`IRw$&zW9mXX|H$;Wz2xM3BZ7Rs}T< z8Nq@4L_q*aZEj4x6xtGA=h2mr9Pjhw2Q}6t@}b!9myD}ADB!A|$6H6;!U6M!z#Ffw z(_`XEyWmqvsOZfFl*A%@7vaUFz#{0(U7&WtfElA<`K?X8RVKKKJ?LpyMkR0t;g^i! zP4F~jdO@-MHTf=j*x?!9zRkphx%xuD4E9sdul&-Z^_zeOuPryBI{fklhj41+;;$k} z*Hh7XMD^%Zr`g?Ei*M`$q*`j2pSH9}B!ve%mo2FVMbO-Ka8`V5v6mtYYNrlHIPdj< zQ2+MS60@ZukjZ%NB3q|li?o<$A;~+J5dUy`V(hpp6#@TMmy4XBnjG@p?C{?h<48K- zLD6)vtL9$)sYVJdmlu8YeVYSj@-HlwZz96>5?cu=fSdGP6_T*~h<_tTJ>PmMI{)zj zvU)<=NeiadpFdCB6XbY!2+s6D2xMS0a=c!EIf2~kKD@ng$w0dAZr31-^g{Tb{tK#s zJ?CC{ZGs_O7&t^_NxSR?KHQvBA_ik`S!uFkr`Wp-nBcQn1R1%HeN4TnnyVI`NAzBR z1IV;Pu$N3c2q;&s0FmN@R!+{R=;A*ay`R!$k+zAxy73Sh%>0%y&Jk|Lq4BzCdH-#u zVAoHLw|}>GOf!I;5K+K{9Gjn3?&{R7&#K!)D(W;_CT0BPs>7l%U@(xC!S`di{(tA} z*R@Nm{kYJjh}K#&j~hIElK>GYv}XuwtXk^s4v~)~Vq1B}gv4)iy{Bu1{_<~TfsHj- z`HVXSJG9fsAVpp&Wy!N~^nS|XTYUQuhwgpHhd)mvy=uaP&Ar4^jbmQ%o{DqM+N9EV zBui9}Gi~F@6P4dH$J=(bA3*s^{b$b~1lO2?y*mUuz2;}#Vm39qIjhub@h|$B5r4OkR3sF_@Gh?4f+hfvB}Qi-#+TOV&;_TNng7^E8%(WY4H46v>e1;2@A#y1`C znityi8j!*jjz}xoBow*QXvIXsa*Qkhosh%|UZ~r!LV_=S-Pa_JIZYBNziU3qx_n1Z zZ2xC6@M9%TNK*C ze#pWK@L-pp{%aKJR}`NS-|)e?AO8LWQnwj|7Y}XM+Ayc~d~UJF-n9_v^ljO9A8GGS zI;7@-Wi3j)Fm({$`lSN8V1i3 ze#}QIJTpQoWc_1|wQoA|v$wG6lV@6CDQ9X|`s{{I2NKt8`N=Zy)! zaA88n`h{wxvZ@s?B{^>R|nJ3oNW{pan;DB`^{F}JNI<>eQ9fA{HFN!ftK&E=pszFJG@-a;K%6Ifp+p}opzVg0e+*VIM?7N1!*W)rzy|T*rvkU@b|^By6m=scp-gXi`L1Kk35)iZcFjwyAslJ z&~_&i#lO9}$${y4rgPS8_eG!xbH4v7X>pY?bo+|Uj<6Qxq_g~ad!E_N3irMi<&+9% zYqgxIexGZqWK@UfEjnY-M@#$}idmbdFRBttx?eEx|9Xcvk;^)^YNFaC)OJ zX<9g+OA@JJ3yZEK9FQnQ$9sOGGsAx$n_{an3kuM^wU%!+3BQ)K_`mWtErvh;`(mhN z@>>#|?e8@v7!w(wJC}q2`X9;;A8xm4zOBjpTbdz3TMdbC6-;+AQ#%G0G{;s_?|UVv`E%J;Z}_ zEyu!D%7=(Q%TKmcM*3W%q>U%8i2>I=S~QI zQe)RE{P~7rPmJB5E8kw(?P~G40+K`btxd*`mV9Y%#zj+#pT97nU51YfAx?=j{QA`m ze(R1l(`oqswFGz(nqa#>9X0qs++yCJ&5*Qsez!Zrlg_fqghXYvVZEG+=_EqvLOMYLia6n5b4qMCrn|Aoa)+8l^g{vI+rMSgiWx=;yPK){#Txv&q z?#&&2ura}V;s&z}3gxs!c)}L^?(_`Bg%RCfn#6e-ed<;3`WwO7#V0N1a6#+q9pW#y ziSEzP`}5$8XI%(jdzIu!!kkfW7VxD&SWK9E6zWFBz3PITvFIZfeXt<9KO=r5BRXkA z7pob9Y+uo>7k=I-qM|?BafNI)?NPCE8z_{=yN-uWIuf(80d$>d z&po>HdrX7hNXDU04xird-t#A&Iqp&g{sWU_yQ9%xjnX>+8fXZr>oG5s#E<4QzuG4H zMvB>&p`R+z$1HSQI7~@pb5RxCzSP29&%5;I`=h>hKncuy|{zGGu&B_eWojqaB_wVx#AD!>wZyclZ)@Im&G0Nu#7K>F4L0K@TN}_`W&98Te zZ|)F1kf9G0=))G7b%B1>?{j-a=a-75KDYM{==t2!7JbyB50*rGGvY@xq8Dt4ZJG){ zrtnHuhusp24fZ^Pelq9Lg9X>^h}qHI#Hv@YX(%ZuK_>fiLt%Gp@1qN+Yynpgct7WGUvk#VNZyo_kAK^ z|L-@CluTzHy@D29*W|b7gtsWo|Cx+)j@Fb6(w#Sm;_r$MAMJMGolVkflhF;J2SyTt z*iC%2BziC-`BsPIYhCn(-U9sq03ZNKL_t*E44JOFHJ)Hg)m-kv&v-TMw%4NT*={vN zCn?A&i`rjc9?Xc26{xm@mc~t0f<}GauaSdPbA;O?VgtwTP00nW+ZrvbQRyKPc-zw>L;J1@;u8ZOUwnhm|iEy9G`NP?HI+sP{KQe|hRr_%i&r9D(f{%pw zj3wHak=)ZI`C6Cg+bMF~vanOJR~UXVpZjWD%Xjm+Imom_9V&y}K3$;a9HPK(Fo7A? z|My4qS(H->zeQ6#kaOsi<*nPJFy9;U2b4hR;p$C>gR_n&rtON)|JF764HQ$y?=>cQ zgNX+=DwKp3pkK{X{vZo;zFuS`kIDB!udC~w3;#U~L5CMZl?3;$Z#5>uQYk>+U*=qE zVz!7LAg2_-PNm7j^Q|(cvBKh4Ba{4w3bbd7X#SwNjtxqa!vKY2OoTtq=K1Vgm*giV zXkQrzh1gM}J6BH2CxQ||<(}ppRfqSk6#YbrF9=R|X;52RUjnkLQ^EIp%PQHiDZ9v;GV8%u#zT z0;=@*9TD-I(!)e7Ohm%%M+@`{27`4>el;-fJwF*W_@9#&uJ3+0 zAQ9n&xBTAh3{Q=F%AcK}+-Mld#J5Lnj)LZep0+e@?-1YICb~OCJzd~S77Is9?RJE?c_y{Bs^l?k`?kx`nS zvSc)iBd#hmd(h8Y8sF&9xT%ePJV)mI0xLu6?sH>7Qc%wq=qGZbrwWvK+@!`$sKE3X zHJ>Yiq2jh0qB+aM-$~iMNps1RStwjVu-mO;8YOW1(PBlrzjaG~d$IBbTJ!sju##-4 z?MA@A@6PgwcT8%%P#YO(EbXQa`Ep(=`p?V)C7V|_e;O7%&`^p4Wr4$UgIUyi>AgiZZT5YL_|&CSo215SEbFRy0FuMp}_lOp_1V}hN1viH5O zYv7N%v)ozc$d8PXU7Cd1xluR|u$c5lf*431$!UCXp7`bt>X;2@zqi+O6l}M5eWto@ z2Xd^$Jd|OMmw^s?j#>B`FdEOQ^0dXI!b~dWAILa9?|9pJ@xr6&*{)K;t_j7h&lRL@ zWo)-@$uES2;(twAyeDol4+9FZ>mo zdgrH?#LFt_1jnM-Tjsn@M_i(!h526rGGJH5Je}vxvcUY}I=UCcNWo~l+_7~6+iftxPJw}BJEg%KB&4y;j^W=Rg+tw>ye~zJ3R1``HYZ$gqC~^zmetu;t z(RE9Hd*XB-h#UM&)EX#S_2|yG*_;o}wXxSX>AY*~<%zNl8xDv<0IiRfG`}!Ud`lZS z<`(AwM;*d~Se`m!i5|}}&lR{Sh25;fqZ%RG#ev_XY0NvG+MBU`N^$i$hHjtjS`XXC zHE9X=9AB9oqIF7sLqHOCasOsAju$zLGWnb<`IEUhPHs2kKQn=y7@>W>j!?vj1(~xn zZ|#tLp^ZLJgaZ(d!iV@C==ZrskkgLni5&A(fhdB{HL|#oq3)=P&VEc0HH6dmr*y67 z#`9y+a?x&`gLMtX`iA26qs4+4<#MA=$*%w@o;T~5-)f8xY(nWZijF(Wzf0%nD1{mE z#L7Dt?t9It%`X?yW5`v?(Y-$#RcH}X@43Is`Qa#HT=YV3vL{D!nKs;O3ywLrBJ}6l zC%?Vfg`bKW{7-RA1YM#PWj96D0+jvWz`$s+w( z1TqJErW6mKEP1Z&mt8@&PRXy(Q<~I#FqvSinkniXX-#m_|9O6v=gt-E`^PEAM<{^y z;2MISb~L})rE$|d`cS!0cYZ|lc+PXr zm)x)|VzQt9Oz0x*UZ;5Il;cR-6RYKYq;*SvLHIwCF)lOFS?bOS@S&{3%__tE*aZ2` zXmrLDd>BoT+|zgJ-dNCWNAolDH2!HG zrd@ErJ*w4eL_e)kgE~|aJ(8isQ|^p{O64e2q&>Jq5sxcgxT}kf6mQ)Uk@pF4SP$zH z#dsuq^KgMIb62xY$*&?LxBn(?@#d&8@G|SISoqtl!%a4YAD^Ju8HMi{f#>cOh=zDy zM&ncSma-@Z66++ZF3E2TIJr0%?Ff&n#9XN8lN{M2N`e{)h}bO>e6BfC z(EQvy@on9(Pdk$E!dk6%TS2N_-s*_OJQk?X?z|vGITAh>06}fmWP3AC&RE{QJ**t; zsXtSNPrD|BJCBvj=b)BZ@U$+;uK~_@&j+Fgzt|W%YqiWI!W~7%A7}H7RqD>82xl#L zKN`pNRmMWXKZ?&Tg!~+Gmg`K!*1olQ3Md6GSBCp-!PDLnE%Oupe#!4pb&?g3;rKw* z;1Uy&_ATfx39}2I%2Mw2CHjUI*#=Yp;H&3^TSEMJ#`ynthWJ}4E>epfqgC#NfyK|+ z5@gnqe74Q_U(664w~PCP>wF&sC_UY)5?a?a`1HP%`%jgO$AYV_TMNu&o_C)g^S1LM z=5znD);^9ZdSW@(D#JgEn*-Hl2Q)-@%oTj1-N9dxkX;(%R${R72Zay8vr0Y9FV55Y z_#FCZ846at;JMCc_SJ$1M1n}r;gZJ3XKDV!9D3F+>Jro$Rg&JlGNJK?gwH&Z@!;u# zv4{cNoi9A^-VyVbEfMWv_1kWsOu}Xnej#eo5;b7E0(1rba&DfPt(xME4a#IR5?^2K zK}_!M(E8gM%)@y&t7Zgz&8vmpXKV;E?})!RPwTH{h!5sL0jciyp|aa_iy{7z7WW@5 zx#>_w%dns#D}?Uv`ICl!z9(V6^ed|TJS_BVcdJtT^Q3iFEk_7G`A_qCN~KUMsH*#F z!F!Ljt3sOV6+=XVlC`x9!d{~SPuT5y3ddh%U8E{t_`Yu_MEJF23{?ZCGPLAZ2r}5a zMe{3Bi+Rt0y7L(DoU{COyMx&s(S1`BNnO1iu?tRETTknY^ECfz2A;S0aT(mn_gaD@ zP@!W%9mr|@_Zb@BN%2hy4z14j#D`!xe|L+{nDE~o=!VWOi3>KTGMDh8&%e`d**Eh+x^o^?26&^4UOna@{t_0cdO zK;}K~Hx1sPV`lmqMV7#x{BYi8pDP(tY82etIyz=YC#tw{Ms zp3z!Tgvq_<3Kj7_lQ4%HYQcGE$*&?f;(r;BF{L#{-z^eCQt>xghl9F=8(X!1Ut__K z3DNVG@xPiO{!%A6nrJiv|7Arl2x$s_){*@EEMqs#qYc4H*Y)8_plD}1Vv={axbJAm zXAfkI$BP{#CD186Z`^En$5umT{fel^4hupMx%y$9@b)O7!{AD?7Q$_1&dr?+|E7d; zo31zIt~WxVpz+NPjStUa4wgtPi<$mw89m!f1occw^P{sg?(8B}r>*X`o1PyNy|cx4 zj+NYWFr&4^cDrS;-M4KsGgg(WdO9uM442(-h(7q9zllp>C@kily{K zuC7%PH%18^KRCP4K=_U=_{X^p>#l8)?ui*tE2$eC_;H0kSkU^=H2y#l_GLnSoy;)c zLU+waLiE)(V}CP4bjIRMjjPcLq~pln(1eR(KK*#kiSF6rR+)ugy}W@LE#LF9AoNxn zDSkd`5cfISssY2axBN|ep6+Iidvgm{=NnQfb9oQRm*z2FYD3$vUO}rL1idbB*OPp9 zp62K0(Q~%mfwTh2JjLr1?CyxqJ@MkU>%8zwdlHPQuZ#IX5tvq&kon zKJ9QoTg*FJloQLW-uHu`Dpn3#8) zgmz!TC)-9^Vdj4X5o^*pEzoCiu^$kWnKto6#${7EDreUt4 zT#d>bXr<8;0~=S!zMSS?&k#SC+7g|4hkkM@k|!Sc&dh&z|*~AKw{s zCiSZ}2`+%^wc%KV!N5NRC<3MM|CL(v6z;?FhE3SDEc97DtTU zWfJ`7&jlHfI&J>WW_CyO#96bU5cmW)4_@NE9E=Ivlel{gYCfu>1E{k`_}+F|5g& z6?gNYd*sdrFb;iAA?*CgGc!(NJ-+K3WpeY->RZ)QRZDXwu6 zR~LAC%1Cg+()#Ee@sl~eDV$Z-tOEoj_hfq_qRSILwztbn;W0|E)wzrgJ0`t4sc&560R{1ig4W;9VeU&Isg}*x z_M&To-u;OL*DV#-IVz$Me*hqwy^^HV_pfWB+hgl&3|eWe`y}^tkIlb3VJl? z2;`(A`P@8>`%`>OJ)d>Okt{Z7;pc!+!!CvJIoMvt&RfL65m${@9`s&06UhBAzBGt}*0;MvpPI)j z0=;O>3BPnWAQt?zqxF$l8jt4q@tWR44|Yr=Z%jCndp`YS##kb>3W&Yu>2cwM+ZF%% zhnp0;B2>AkP`MhQfmu91WD>41F?rt$IK)(*yKK%rUsBxA#E$FQK1VdfCrTQhnMdu< z7e%e=?^Egn4-$nr;MOPRh@UAIRov?a4|ZITHzYihdv1Cvrc?ZW@| zt|r-Tv#M-Y5nSjlW%$v!F;HdqKnxtTCEx63WLL*zyA1=`ah;$-;@kLY8*@iD_}e#+ zbsdG?R)q_c=DksaPG1km0}bFQTkz#FqoLGDe0JZP?_EoLUom?3u;xGS1g_6DA}h~m9bGLZ52#B%oef}7hZjdzb@n>7rkd}#3c2%2|x zFgLZs%<7sGer2engMG)*_{a>6hqKTDuA6OFgE!e#32wXLiwAN}%vc`YsQ6F24ZnCr z#78$9jypL0w&to6I_u5!zQ-hIDLiYSE1plxw8<||u$RQtQuf+|*I-VRB%hf>4%VEq zTX$4}G)GI4f0)HQori3yZue7wQpm4Luv;}>c`E1loMrDO#edvw_~pwZKDIgHxUFnA zt|>vY8HbVLRN`Wj6UKHKSXWW9nnw*uX{hQ($tAekfiN*jH58dez9eOo|X z(bGEUh6Fgds}oj5)$LwZDsA~TgIt|(&(V^Hj+LC46#o2t#s7Gj;SY92JZps$S0>mU zCUjP6cz$0goQLa8%tgx3VNe^VC5pc*x}3%nkHBgp=lw`@Ek^;PK!FZ2A&O((aa)lF z=3w8~Gdw%A;b)=-(|+ZZpI7uXp@C`d__?UTR&B@_q+eO_e6ro4yH!){iSf3E57&cA zgw{PN<_n!59I@tve=s<}2tJ48W3wbr=J-YplSdDwR;YKkIDE?T!H(s(E;ao1Ihw*j z!$MSgINxB;)AbfhtJWJ9-1B^>tTn% zo@YYTS-1213Sfb8QT&UzNr%D2w}tSSE%;`U;Vw0ln@s)Qxd0N@2R3i%qVLTr0VnEr z#zqMqL*mh=e00i2A`|eE9`|4UVQyN z!z)CvUEyCuO=kV#cwq0(_u8CWiY};vwUTCBH%#a^5bQcbWv(_l^n;1mTdk+}h4iS2eKbR67Ri6o6EF zKa&Rn)fvTF$M@AjHIGEDcz;$)pNog$90S)h zIP_e>BhQsgDM34My>?RWN#ESSH&s0gI4=`D+kK0U*rg0bA4{^Q%lP@M%MtHrR*6Gh z;j3BIHN3hIk=oAyd%!5gQSZ2^=(0|!1yME)%y`fBD&cAsGw)Y+_IJgSUj?Mz^JX3K zdL1#(V)w`b69RwNZgXa%f~ymJQL_hX!Ho;iV;PN)%!NQB7b|6vBM?tvjtW!s>cH0g>qMfJy$W*MP{!ON?;3`_nL&Pk9uW+NfGWY zbMCSw(Hk4shF(-Pr~^EqQ&f18&&;D9$?DnR7jRKduu{edYX@CfkN9?ffWmGlcv1T ztcsBZA&l4kbkwAwWWaVCz&=~>sS!|kUQ2wJ7ebf;rp5`dMzA9MQ<2eeZlNw-X;Ct% z6gQY?#p`Y>7M_Itpu*22ElOly)^|*V2W`Rad4|3yBHwP1awV_s`@n~eKytXG@u@kS zNC>Fwd)`JD9%uw5+2&{3;ojFR;3+)io)~*kOma&btNJQd0xI{o^CI%g6I9pL=N!KR zD1tD-PsRzG`u2nNVx}4I_(GP_*(4MfN8vBj?0JcGG;iskAI%}Isgmt#Ks8IK?oUZ> z>4XAd-OfS@<;5}OZbN)q2P=Kb&Jy;Ex-Aj;6$vUGUE9^by!X7-#9U({(j|(TT0CFO zx*Ule{_+^ts+m9YfDxJxq(rxOgAZ7vIxi0#;bs0@N#mwDWZu;)0j?NTc7vgKbA$Nq zE}XHUP~Jx$6r)J5ZwBVkh}ta#+nw{CH<^gno0zU2)W$I)d@=7bi?6l9bN?RSH7qY& zjSCDG&xF?OwFS2n8S6zY)TazE>m5H8HCbPEg;(lCugEK+m*HQiG?!~b%HU$-82DUP@?8<-#R7pY!@h$U6>XAB~qtyrxB!b#s(E5Bk zjGWaef%M>wkYCY2TZkXa&zg6X(qp%2@(UxRUr=V`AUHhlHZfb(;CdrX5U{(g%rT;B z{SntxomUoh?yxH$7x+qrSu=xtURS9MtY219XqJhEbw3P^G6R)%+)1~Yhc^+9?N*WAO`L)1Ln#PKf#g;w!t%~i8iExdH z2Pmw)O&L#n%M*pgUf04UHCi)Ki5E5Q>R@hZ*HeLbmBV=<{=z(Z&ecnd!$C2r@fXEJ z`wR5*C4R~EDG{pK06nVMO_|46)1Gy(ZD=cpr0;q{efB) zMA(aBeBI{hUIKdR?Aw&$dL5I=;GBL_ggerd=f*th$^@y?s6sU#n0%{)ex|4wZT_`_ zil(TZ$VtB5uAc~+61s0`VxBF~hswY#S!%l%CL?LPJ$3f?m>8DYEy8|VaBq>LuZi(V zjb`90!Nd5mjOc-s6<_?gh6US@Q+A()-X{i3^s1=;7_kn zuZ9K(v=sWGoaPN<7*9dq(29yuQ2UBU(Zv<`c`x)paZ-`$6TN^Da-G2m-Xi_bdS)y z!9;W!oIPp*U&}IH7ze#4s!!L~folozBPr(o43agv^RVF+m|HuT(-x^oC0a^gou;_9 zLHtAx9XIH39k`aFyf}ff1{eG?(5pbdUB~nnaXnS{$d%k@OU$J)HmXVY$OoYu)waan z>LSywe*0!$OQ`7Q()O4;Qp}vK$NL>BB{xlZO+tL2fVLZ?I6H6+p}ZssJj4+}@VTD1 z=$H+1w$Rm_cih~~@H;hjg9!q4b$BKMh$s0@7xhfBX16tfc)@%p#XM6WHA*BskX@f( zPCL}|<)AXPfOnqa(xg6xotKE7?Y>P#tm_*yf=Yh_&lk#!rqs0STl1Z>uPM}&{v3&7 z2G8v_rzL`~3{(Lf@3}%pVN_=Yu6_BFU+=k4Yp&5jvc12!u7Pi78FT9u_QE(+*oTcO zfHrfEAWMkwS%h>jn#Q5cSo>Z5$($dud7myEzeQ7je4YZ{yLzA z@D3AWkwJBKf&FE{L9zHt;(E=6DIwmQqW0$@U|Yj*QLU_zV+GN@X&4QtNg>ez#jY5c zR78)yi0xV^H!0kv(Xri%vCR!8W?TlvB@J-WSsus>?1eFIN+ET47M0J94i)Ho({N2A zaIaT2JfOjM9Pzh0ko$Vj(0hvQ=NSC12(v#&=H0-xi38UHyTc%LDek-k1YubCA(PM) z8L-_1IP5I<*`n56-x0ahuL;yK`E__2O7XZYd9oZ7zJgGbzFI{qGO4z#XSbmBkuZk7%KJYa8`!xh8T|7gU36>Vb@8{}Q2#VL^eUo?=gg z-DXJkW???6hiV4NZ;L2*MMzc?YVh|#Pc8NuZP=tVwz|<(Jj9H5+?r;nZ3eeq*TZfr z+GPA}PJCx~F@PTS8CgSk4h5L|QfqcEHL2z@L(q1RXwrKdz0II}S&}~O@TN{~K8!@U4e4w7o{+wiAiuC4Qh77Ub z*K4wC5|Vv+u+4soyaSqw^2!8?`r9om&OPo>iVKV(_k*@80w2rU)qmIT1J#jc`&x&0 zu6EDG0b@$B*Ou(F1rxHUSS~;z@SZ3kS&|el2bMMY4MY0y-l##=_m6p?0X$?&4k`=R zG^mAMP9;BWUMSJ`WcWntQ5j!DfCkld#NX_O!&QefLpV0>f*3vT(8tO_%ANxf!EMp? zsGPq9^eWIdnuK*rC{_`063;zpfxS4!Zq&6=XIDNXSaQk@219EYE<8vis23d3y=kCJ z)Wvp{z^(}0cIaaT^auDJhy}OBjD+nfpzA%a*AbVRh^)^5DiLMw%yQ<>)!5xpJ?z$l z7>J(8(FfO5*=q+C6qKX8Q)ISMZ67uSoCCkrKts?^=PW%mB0_nt!8b?aJoVNJUaMj* zRfaBuT~zIbhi!@L6R@iby!S!#V-2-}h}8DEh!*a*`2ZVO5mw@EQ<`(Cq#2-zTNmb6QEJOm%Ls0}&7e#)nF zqW#r2;<`l$YrrdyzQ0TKXjUIF8Ykqhi&3dZKb8;f>n7}*%`S~`>*{a2j!;y-R7L2j zaxQ&;es_`ctXT98gYAtf4BJkOpw3w02Xp+0bm?AGs5TSEdV)G`YxZXq*vKTl7@`>miqD2M%7 zd?<)S2XZ)=*OJb;1}ecr!90?MefBlb+z<=Jgr=C(M8`_#I(+{+f)f0cqSzK8b!n`8 z7xcyz&Q+RsnuPhjZkPjNFWgt=obZlVsd^NgD|qjReH7O4J<#j^)@lB?(!u1ocRb>X z0m&E#G(>p2iOD32T|iH;QYpr-pk!SZ{eupaAJdA99(D0l1H+pfhkC5j7F zm>KLjcMTMHp0y?UCWYNl6Loe4s_lvHO+yi_CNrgLSc0C|!x=~PR9>HC(gC|8RL2oL zl7*zahiazZ?$G#gsRzaXd%)t^D8l(lld)vQkqA#aOB-K@TK@pNb9H|$u7UcAUx^3? z_(Imj_7!Ru3PD=4Q)x1m&+RTt@>>{dN|QGzOfjt>>Y z57%fP|AD+m)t7 z2D(Qv@IaAsEOE#MH4)4akj&AzKMks%YXNk9;Q=M+IfuC~!`H=j4dhoQn4=bb@g!T^iVI@wqzVap&BMCOl_Bc0-JU}GQ?}pOcx zlpA%}mpbGZlL$mlWblHkhm!Cbz9)Pnm}g7k0|h>*M^Kt?(x3$ORE{bKovrfVH|lV< zYK#(?62;Xj8en92AW@3_Wy$df-1dl}8*Rm(iB6WN19e#vzJ~7yPY70{XG?r7-nBxp z(I8q-PZg-*Y}=KMs`j=kz?*bL{31-N7w|l3OWcll__WoEnrgZh{dmEE8s-}62@hU8 z>ZBuiG!HwmhYYb$Od0%`MjtLwr|qCpxCNP1q4PWRcg**O0B=D{47u(sN$?x;$lGVM@%Q^fn>JGT;;RQQVw@xcN~-N4!uoX2S> zH|u)L$$l3Uc&0?SSVv@iB6S+boa0$rVy85!7eEjBb2tO~(L9VQtaW|s4^IeCPn4*~ zvwFP$EwE1EF3?0zzliO6k26rz*mirSc%_cc64O?|GetpXT+m}``2NikgvFjMQHRU5 z#CJp>N+9*fLn$g7_8|%o4gPY2J{cHSOL@WGLV0feM-~KlosL*q!tIH}o)V9tUdiuT z*LSo~x91fr#WZlAEe7`b_O^}ei6W*%mQ4=oRd+if&i6D$*sc_w#df|ml&GmMTF@ci1Ug^_^;*AkLPa?Ft-i*N{mRyC<8E{st}?b(XM4*Z11ZjWF| zSGX>a>eqIqh{O-5#l{GSoaHfViFX?8u)FOZyaGM%QO}kPD08i$_VAIS@koY#p&YJi zdu6+RouarP#(X#19n;ty^|aldjc!J&)ntn|Qvq%7xI4|!TMc$f;fK8@P6B1sJXaud zcCBbNDyT$3YBV>#ppg~>w#E81^QrFAUh*m9jKGB5FN}hhf4+|1J_V{ z2;fgS)X5qMrg(6XU?(*y^Qcqh;QU}^yLM|mZC8LC&lM_Ss@lulyF>+K0{gNYcdo&% z*TdaiUnNlLlp}gH?@0--VMHJb+`LDQ*_yB*gkqgW5cJ8gX`%m`#0z#?R9kjI5cu(2 zuA-`NHE6p9aDSOohpmnS@P5s9*YE?u;}KCDc9yPp3mU_UFbT6=>CpLICYQS`o!>3j@rPRP$!5HDWvSeC& z{FK7hV2gp^1JmkcIsE6ohL;Zw(SbY^i44O%Pu3$73V(sYJee~f4=X{b-NoR>>u0x6 z1^OPY(GlawfbB-WDQ7u>L)VP*yoM1)jk@)oFx&g2D|y;k5(ewd3gH}4qoelht6%3S zMYFGx)uT>6@Spmk1qnba9?n{wV1@$Yht>JsZ16q)$OQNbEh6}{4JQn>Z179wx`=-KE-5jHHLnkZo2JMK$!^m!3}orYmn{VTiG z$1UoSY;C>KC_+L)DbJPY)DO2aqd*Fe-Kp{G4eC%(jvcVwGT5$NuWHr(?b+=Z!d9gQ z=DLEor01S2r#qog=S9P{TL|yT)S(~fA_Z%d=MjX6!1ES;+6`CmNI*((lZtXrggR8> z$|XwF&Ld4lx!u%)Ww;c9ZROdfHT}%&;NpZkY(YT~iLDFjLY-a1D}%4e^Xu)OKja<9 zoE;Eq6oMM`#mX$(M#N+@qW;DZLnQ;j^Qj-njxbg z%nY7G5SQ$Si11udl1*vK?FKdEeWzZ)=N_hAEjw{*_&(^dHcC(YL(Y(Y-VSlcz~94ob5_6r3d#3hFC&4iNmc|II~QpxR(kE zUH0U;0x2uIt=aut!}o_Mfv!g#C_+*;3Ff}XaR z|DU}xkF%?)^8I)1ea^X4jj5zE0T~Hl3Ny-#%qoI{C@Mq-(N+d+MX?{WXlr-Be$Q50 z``Mx>gP_>m&Web(C>REnQIMIC5Fm3>!yV4F*Zbq#n;L@|D&8xf{5~H(6>e4C+V`BZ z_Fltp{no}btNu1qIW~c0VGgPeyCNPl4^dJ`yQD?tn+~C#%x9*tcsew5En}7o>ywl) z54O|-lx-oaI4n;xD%(yc`mYwMVAm(uffU_Q{H85|fJuaSVF01SERVDG4F8c7T#MOm z!)|+w(1jtchrf-$cQAGIi^FzXWb_QAW~^6$7ZYSqv*bsMT*sZ*3TQU-m%#*oUD9;B zwK2F|JY2WMdny`Pm~5~%3W94h&8Pj3LaZ=VEIlH`b$y&hzi+>Dlr=klTM0X7`~G!9 zprxw!7FeT>N}7%wvY<*1p=Xp3+(coARepQLjL@<52+H@1VTX zf&Qz85A-YMrDV*UzrQHzeZ6t7MIyy)&`c0y6tv4@hcz;6BeLE>Wr_&?sPSYzuq;W4 za^N~!U}mC#sVUOGlREj=4zYr4aLDQuY%_V!ne8T1ED|ebqZ@0xLe}rzVz!$C?bJu5 zz^33gKv<=Y4WdZ5857Jm111H#(Y0jT$4+cH#0V0CS&=k%6*f zgBDtgan(3;J4GmoA*sKfu_T36s`RUCt>CwXf~K9=cC=}%#U z%gP;1rqjEeAUVnUkF1PH@(AEMP4>*SP~hee@(M)^PGp2xD$28QLR>V^Wzi8cixD;3xX9sZ$IWs%4mC)?=d8-pl)ch*G8i6(gWe1TP z%0vrjsw9(4)&SokX!H$s8B4nul5q~W7m}1HFSs_rG}D1laGs#65-6vwvy1PDHWPaW*d&3 z=KAVED=Bu1Dv@S1e>*CHYgCwOEGatSuEB%{zX{6px)VKIwp!}{Se>S5o53|x-LoEz zORUI-7ML-)6Wa#`0>N7mV>hPFRgP_xTsjTXpJIp8`y^A9)pk2n5VRuM-01g#%VT&N^%NM4Nnq{g$PM{xUz&ToAH1|%t${->wQ^lA1bdeR#ZOSWC|7wiT zEwV#ulM|k2PZf4EmK-udDATJu&`BUj-k{AMm7vKqz5e~g*Hk!eCG`Cl37#c~d9x+U=XW52}<_tK;fH+K{*(G&Q3wCva zU7I$a#<>#{10%31MJqJlc16%BHnZItBW>4#2Btp0yZN@esV@p)SFxnSE}aD;b*xo~ zlq8!I+RPL5><-z=YOJC zJE7jJAsd&nn^^3qXx$Ux(Kse42&N!SpA#wy@?4ZjWwyxPvJ=|@0Yc5Flg+-LBBVVY z3^=Haq~s<={XDXb-$(XYBiXoxvy8E9cyny}U=UT2*))=Y?C)+ET<6xm6geAPf| zirEp3JzKpHNPBE%yQ66JDu8on3)yZ)dxjatY@$6kMC$rbo=Xz^PTO2FY4kk}IAlfK zG6Bd=OaKy)aN@;iwieR%N>Q=|{s@!MjHQ?upvR8+ zEPqkdOVKU`sx6S@*og^)0%oAunUxHg?K)h{7wyo2oxnpmZTa@VhUBMIgm*P zmus@qFM_kOlk)s3g9@0kYkm>IWp*p6!l2?tnW&^dNm@#4qak?JV+@H5Rku8#3aru; zpVkq(+KHXm5y)5&!b+EpA&@9g0NR-NcKrztehVV_j4cp3pprT?Ul%55ruc?{O%zF# zCJWc@#6&{CTjQEvx^A+fJG>1mZTa1ftorJ0qyi?cH&nQ zY|=W}`<>WM)a}JMum?Z!|PGvQ`CVZm>s)@u=pGG3R0 zQgQW)YhIeS0X2G6Y6{U#v8z`V1Z17d6rY=XUtN$r&i*%FIl@J20G zRH-V&7PLf@t4BbpNG07#zk0R7fviuF2IsyRs0ZRBY>&nfWOmzZ zHC_;;;x_eF@5HMFB1m6~RJ6IbRY3zm5p0z)Wg=0sGq+WwlgS+ZDS1P#L+LCk9XaR> zgf1(DqO28%*CLf)-P!N^>O(8t&rKoW$W0_wXh-N2*`!4?QJC-vx0cOlABa%ZOoCoL zM$BRROG5!W;4nKxEmja2Z?llCIvg6#efA!jJ*V}g!pQIa**B{9ztk%Os_3S*G`{L9 z+QEpwH|+PfW*rS`?a+^I-SyaFtWPsmL6e1Z)Cgh(%Hvb(wXL-rK%qV&d05Lr_Q(mL zDhbL6DZXYvz(k52RMcr!&6>dVNd2>qsiJM2;f}Dk%r>^p-s*2O>SwdBjkRo%?bX*Z zd=JTfZ)lI&f)50^5$oL{i3g|1Mu(_2k6K@U^j+O{wXEYlUs`G{+YOL00i}@j?uV+a zLG?@(JK%QEoYjdqb4S@*-F~;q-Wu;T>Sssn0i)03SA36bEt(nbu(N+l_K&*djx^jI zlUM3@NaQkFL=)iv@j+{DY+*xzVx9Kf7>C$o-(OVlJE8C37#D(d0OW9*s*iyz&cK4y!1Jc4WdyLlYYO zW@8t!2IVmjs5-@2!FkfWF(r}(-rj)A`amY~0TqK4N39lNJRi@RVIl!rYCwW95QjAq zHTa%~F@mZ4`RW{T663@ppYt;Tmd(AB7|?_;j1lYMS&IzY44jLJ6NeY%@y$4tS26=m zNge#Tl{R3mbRAn-7C0I0Xc|$A9iH!F&5(1SCNYTHaEKpP)%_dtO3PjLWVAx*c@z~-=t;56L>gE z001BWNklCW?#nl8Qbc*v$bK83aOsv?l8@ikm zWV<2;AtelBj33~cVQ2222{4Df2?j@4i%{d?c_y>Bx6uW6&fd zj1}v7SUY?VaJVEPNg>GRvAA)%zXT&n6o#m=^)ow*)YPTKaf0|ceEVniB~g+@v7QCh z?Cd2anP{?`02Rflpd!tyQ)d6yn$zV>5!NoPA1U%ovn349D#tEqkxm-4XW~|8wMjT-ngZ1#N8DVcxoMM9(c6<^B`Zn<5GmmiJZ|~%{zk8gGv5@oa$jlW&64l5}-J4V2 z^ahSTU@xXl?!+TvaQ!MCyZ6`pLnRLlLC(r&!|oQrtp8xE$ZF-anndJOk|<8yrOy8j{W?7*by zxPMYaNTU$vcd+*nM{)GAN3zeH8BCtkhH)VS8`rb!i3hmnuDiJBfu|_r6IdDdFcwLv zRF!G-58&`)PvEut@4@uRofPwiT7N%lUV4@X@BR&UE_sk;>na5K93y7OiJ)nfIGD~6 zZ#;#Qk3N*y)2ER4m2hx9&p-A6Kfm?I-211M*qlG^H54(VQ3Y?;Be?RDALoE>Mf{MT zf92DByA;hRV>LiLnrE|h zD4-Rc(9wcoEF?7sYliu+c_SCR_gwa!mM4x9OumGB@d3VY^~bquZHYF|F@Dim5KXAW zhF$kMj3bUaom-^Z%uPx6N)zu~UCALgaOm|PGr-mFG3LRzgt$8-)n z=17h|;!t*HGCo9YVCdocVQyFZ- z(2A62F+yCeVJGd%;fEc?v4jNL* zf>U2ZyGe-R6sH-npnK9(X79ca$GzdLJa)%_@wtDymM1ql0#7#4Bh$D>aqbCR_34js z4HHrlp=?@!0!z$j?gIK)yt<0NT01uX2 z`$O(}ay@>&G_Dh|J@*3DrjOAA&_FT=IvM`~Q`;JcS_6U*j9QWcR)m=BV@NX@SdIO{ zDW*0tZz*EH#S9KOW*vGe=Us3<$M3%@8ihC!G!aUZgtmhJinOmR(G($_yK~-$F6SM` z?@P`_#EHW>hl((%XF78i?$1f5y^TNo zc6rpI7U0s%a~f- zT~pX~&K?|o!fCwqf!q1QXTQkZ%LX&X#qdP2F(lP0g;}rRy&t)pHyykyCW(mCG_x(~ zqNjHkc3<>b-gxHQ_|45<;&V6M&IXs?6aX8eR4T&WC!NPtA3Kj(g_Ot%zVCrcm@<79 z2Of4j#~k(zKJkw?up-hif=FU4adm*+gU;l_w;oPckm9*D{PaiP<4+qr+Dl40!RMo@ zB5n(->^l%@>MaQl#yFQ+w_B4{r<=Xj2CNL*{~PQ+lhtK(f_ogd2Td$jjSFLhs9K|I z&l9-hBOm1Wy{Dj2MB-AMQ$!{)W!i2mJm5%9JL_%y?CYQ9OW(VPPz!kVD=WrPsrtO; ztc&^h#b?rMBVx7qzD3iR-kEbb^r#a#{J<~q51;!k>ohl}Wn7F9RU+md|8~xO<9@W0 zVyn+_!w-MLvPzDQ;>6qAs}&ANoFg!1#A`)?yb*MQ<(?;8?b$=J3<*Q)R3K2HraHlY zaT}mn;NhbBOxW$N9;H;vA-(&a%>VlA=lR%0r!m#mC|4uWG$oGX?7w9Eictebgg6YD zdf;36`>Wp1UQ-Nx0|V5;1o3hdi+OUskc2e`H}*l#{#qh*(XX>Y0#uvpDCn z%Q$jYj=ugXY1GFbm)y>;e)|v`YK~g}2J-Vy;*!642eV66YN;KOs1%fn7C82t3pjRe zfod$QdFWQI`_Erd^NLuRSY_r0rOBofSJB@lIK}ybD}al~smepkWOQMui0X@sNCqij z6;kJjRWqt_GhskgvYxK(eR4!d!YXlY3a7sJDn9#9S90>AUMd5FM5!Zq;1~OeDsr7aO7@n)ZzpYgRx4b zlH=&Nzn^!Wum{x*eZ($i#bZnO`R#YIY+a2s9H5e$!#ghi5GTy@C`UTdZmlJ(R+)YL z+d1c?-HFO6$%e&)Do)emvi^+x3T2m6+}r&Uw?&p$DYr{?|dy1MI-y;#UN>wxU?tly6|jv z^J_SxJo2M&@a^9`Pod;Ze5(P_YJ%Rb4auip;| zYp58+7}9cu-UDCH2QE5`$*!Mr>=<12GRKNC{k`0K@9%k`FD94I;YmvW#tO5Kcq{LE^8uuT zRmAgfVU@{;zLD4MHHpE&m_jb*uIsMmLm&GjSAYB?{L43fMO-RT>lxp)C10B@f-I?!;~O-neSeIJ&*Ny6#WThO%CfNQ%utxXb@kJyxHtHtyXYI zheNb3oHP72vdmjm6xXoZCSO`{Al2=pjfmiqnC?AJNPG8JfN6w?#*GDRrd_G6g|1uBVcQ^O_?g3U-lzcA0lCZHqX3;6< zaOMfS6Ajc5n+YtrN{!;&Lpb-mGnkz8Q%=D1bL32o=N^54dzakJse-p)2^zr#{X{ufCRLk&lVW*eMG+ z{^Zw^uh!5;w$)e&2chQ>-hIyL^k_toSAPH9Yx(J;>u4)^J0_D|cLHU&)7lIuh$l!v zkj>35Z86&os5{Zapg zED?BF@9Cy9_m$qt8j^CANsEqW@jFkVP~AY)ffwWnbbzNG`aSn7xtC{F_T%NUq*G;} zkG2ITa^bsAAXj&yp`K)Lc@970eVn>a2OIh-U@ZAUKzY@3+<)&9?t9>I*2f-&oQG&c z-+<+qbKb)V3#Sl;F-B4<+RlMToWj0S6Z*q|Lb#N#fBK{R-PNDuPLc< z%;rC*jQG+Lu%=EK9|a)`23iHJvl2mEL}({*(%J9j*f}2S211PS$QL~NUU-t<-@Sz2 zKk^gvLl9R9i!nA2WmFv;U>c$hC-`C+d5hfndb z4`0EL{ITH;EXp8a0Oi3iO?;wnxoHmy-D&Ew?bk7HUcBy@u3dqis= z<(jK6=dw#L;j&9Uz=yB+7w&$(jOPa!jj3omM;v!5`}8`hsUwN=?6=@}A*FC({$4)+ zcYn*JmtMl9mt4%pKJg_US(D)T9+HMs{i&RI{7KC2im4?AOG<@2iw-}HLw2zYhKjKs zfv)4H|M5?J_|l8H?2->~>7`e2%NZB5<#Gx=}&hE^f)=n51 za&`^BxpfJv22yf?$I2&u$sLbwpcn|D7MZi~KxTJ_glQHqSYpa)8z;Que2$)5A~pei zfBGrc|JOYv{>1kAj({qXN;c}K1>l@uH?w!QfJwJco1#BVs^MRNM#>G_ZBwl;WsqEe3sK#Ke zj|o@soqxTW57zDNvJd?|Ke>MelFMUIs*$qS;iq%-{2b-TVU3W+G5M*7aQZ3x;>uMp z7OxPn`hg$tPanI4%PzT?%RX>1fB%IWdAjUTum;j9VSWK`KK(V6OeW0Zb#${}(cTnl zHN5s5FFtT5_dWa!F5iw+JbDGjB5!~=kLkBd+&y;hF~m#g$Hl_8sAy+G+tX5lO}br z=8r$)D>wg&DjwDfNiAmb?t8M|UUQLZnZ)$6OK&H>jX^BQ>gRYNGI#XjO(N2O==nRj_FKQ89OSdg+Kzx`hlaD#tZ=1l<~p{=nQ#Z8`ztz=5Pg^3}?MAaIJ2`G3q9{A~3 z_}Jfml&}Bz5o&5NTfpJV`h8M%-FH70&gmr$BQPGmR{8A@zQT|0f01I>B)U6`Jb&-^ z_}YKphx&fjuT)8xx!?ZmHETL9jB(D9YwP8pqxUCQ3sF)0T#3~W{*ddw|4aI$#H1-x zkoAvp-F4sLiPZ`DoWUgtUVASGAAJzTT9sHzOzxgWcUwdv0bbvWEbkk{86P8p)0h>@ zSA%CrVxhaUmq|SZk|cBFqiV#A15f9i(+|Q#3Atzm|8@P%{HYJ}{*KjhRjCUiHWR{- z?OGYXiqv#|=ScYV)`Z~@fwp>YEU|mVp7k9L9oCPzy{0RmX-F+{YQW&*fRRpet42a~2uCVb&uzIy!+ zcySN{Pe|j0!jxS(=+J%fDj`@)n#9aF>}dAv_J~r4N{&Kp8Q;D6ChmN4fVQ3~OlnJc z@W;3Cy7fh0XF&H8uJDoS5e*yb;D=zR^`kVjare8dXDVC5O*?1_5vrW$CHV9{x zjI}df^#~e|UaPqL;8`IUXU{V-qbo?7VdvBq;qmF zUHK|^-Et#8yYCrHF^@P$s+L)YoWvf5lxhNew`2bPSIHqT>gP4`ip)MtK1$NnWZ)W!@VFF@2T>K&r-o2Edt%SIgI8h4i zMbAp*@GG&pdPG3Ipo0`zL}6L#km7H5PR##qKtY zm&pOd)f(MhqTOiCMj%x&QgIwA;-Y8_woNec{x}7@9X*Fs)%B#h~2{4 zfYF4Sb}(=5LZ)>&q6Dn9goDd?=yxk9b`{YiCQYGG40!2xOL(qYcXrZ{q-!qw&g~^9 zF^N+O-TSa;ZhuKAgpdfNN?0zF8bO^S*OntF<#DNi3rX4*aP~!KGdl<&@Ob2B zH}k`LR#0fmXR~-^$aW_hYU@IA&3wj0GI{e?vfZQwj9?gI)iF6-B})YQn9OdQyd`Q1 z=JaHsZLeSAcuteuxL>iwa}6C+vzR{sb+giC9F|vbaRWM zg9B94Occ%;k7A*nf+|=;wVbl&sf#)5u$hFBMb|yVH*feJYo$P5GSR&qWw!tt?2xzJ zI;09^hdQz`mct?SGq=fgm(dVm%v^H;!bI!rxYk&0>#$u?U=oY=odan$*W1tgta{>M zo?W?-LaB(0LsDC$EnUUKe|(G$Nea0&&p+wGQ+9%bj<&d{t$Q$?RB1{8nFZ`ApzWsB`K?xtXsb#8lu3Kxv zsp10##?JF~?3hH*gayBi>QNP?Hcs^JM97I?Qq2y+CMF8SDdY|6Z@Z(Vsu<&uR-fgD zPhZYe|K}S#u~sRT3RvS_aTef+wHP6Z6S{Yq%H&BMXp(|4NF1{Ig=gp!50g5SYzC&O zR9L%wIUB{tlWgXv*Ex&%(=17nqCt*Hv*ys1#-xBT9=dTU&#sB;RX(lvA{dg|^DJEz zV~qhxiQ^J;=IuhEWN^;l4ZBG(37amKNT#Sf>@#{;1cPWy9L(g5^Uq`Nt`zk>p8L%$ zeEar4l6r<(wM-a>_5UIg9ow(AJy5S+%x|WDYCs~OMkby!&RWJC&P{{?5}+S9#w;Cz zHIj`lo50avTa9{&TCAW2vt6`yb&BWvJoS@n_~=JJ&bRJ(9Lbf)2R2h@-lFvgxD;1v zqj%R`X-{KP5sWcJ>z-xldd-4pZK!rC)ynfMTkG&`mR3mO4t8HK13!Q?Nh!{m$8^u) z)PV$e)m1$I@;YjXht!V%=PX|OGD}x(Aaw>~lsLBZPT7@dGYUj8c;aPAtGX(|h)<*p zsiCT3#9%NO>!?;OuQ}ra-ndT>HT8*>-^uki{*<-OBdJ!XZL-^tIC0rr(@L#dR$=d>SKmd)~{SQN?FukO__oIbyO;r zqJdN$p6O!d)FSEAHEhwRcg7TKT7O)uAzHtVRY_bQ-aUjQ8DQ<&LF~dwfRH9B-BWtW z=X2=BkT8z3@c^m>ej)2EYxI#>pTPPU9EhP7*1#y$a>9OZU(8!yJA*J*Wc~eId(ADZ zE6iZ8J?Arb?reGs0hRJvo_g{LmOl3a{jreu?TFD4+XPjlAed1;!(SX(Y?BEyb=fga zfkGyx(?Up9DXTJk5JV$c;%TP(>>`I7BliCngpxUj6HwXrJmJ~(^B*oJ97ar@*-OC) zE&;5C!2wp6`-!cgxM^tyF*Vk#=_5_09)qW}b#&0#)s2Q_l4?j|GK!1!0|M(oCO4Q> z1HAx?f-!Y9>x8sgq33|pIRC9j;6*XM8Q`X`ewlk$dCc2$59TeH&n{hUNL*&=(kFT5 z$>&(r9}?s?6CrNZg#-jlP9}5?>^4EjdXK9PIxRZNr&mB)gYkfp001BWNkljx-Ts>mdRsKa?VI(v4d!~K>-YeXTKV#eC=auzFf0)m40L6$fZ#kiC(Dido$ zEWMm@&iNcN!y|PbYkz+WU;o~H6sON+!S1^=XSeBe7L>Iwt>Eb=o?!V)Yl-miC!{Y? zkep=I=w??{1!T~{Za(;?KIA(4)+W2v76~Wo*Q!5Ml5}L7qXJ!dq0OgW9B8qay;%@} zB!y|y=*j0Ht=pRm=^N;y91oi|(y+I6{ZzuJE{46oJ%{>&VgG)rN}`YycZpuOr&nM`7-pIscs} zQgk6+5b?w7zRVxnj^)#zzLX>Pn6gP|OcAU`Jbm}~`G5a@9S^LEDfrua7)wTZu2Z#} zf=1FX1wzdNH5r)+G6-g%$^Em#AY_%_3W^<_v;uOH1;BC0P{p-LYj-)xoapU@Dg?z6 zqMG$f%GOsgRly4a3LT|7(<7r}hm|tPD1`<=NRpIlxrQ;hjFzfOv8$7UHLP>c*42(k zJ^3I)8r6uxaMaoq#84TiX6zG@dJtZqR4n1SHS`UvVIcJBaAnetNp$B+SRDi`))zXa zbzu^R@r1#ReGIPMKvG=D8H>+l&+e2|e4e`XI==HqpYuNT@0|0xJt&*GgY?m-eJ1kvMs{cGPRf5iLvr@uX$ zc|8SE&59AK0^vID`O&p}>Dt>^lUO`&J55)C0K_zZMz<$Sg z&SpIg4nmv6d@LGiJs z`7E4PgwsqcxhT-DBjd+JAeMSTegYrA38p;@Rdw~IcN^b7=)soK#1oiou*__>oI8H}6HQ6;*qNGObTz%8b4%K`|9_KPDb}3)T zsFQ$*A&P7Dit@<6?3_}o#Y41X=OAAskO1Fq;E`uuAc{a;LT>UN95k=UV0n+FD8@T*i*v5a)09pH(< zI*_VsZbe-|1A{A?2?@5;xou_}2$}79HlUQt;WVSJi3m|#9YNip{yvLP?Oyp*R`BpmeSb2#&`T?k_jR^G$4-+GV}FaBpPJ$+wH z9FnA&-KGk0Eu?+kQGDp?t2lXHfm)nyf9rw)f*mWUwM7uHZpd~=e<}qSY%MX9a=p!H zC`C#=5o@v;z!kS$Io|Dt0#-=d%ur%uJ1fB3d**_nR4CB6YDNrc91^*Py^VbLS~VK> z>kbVHC7}YVZnqayg;e-^bHWFNeHG zSh37qu#joJ9)o=YR4U5+1NNfJwQa)#p=nHLx;S?6$NAWWN8@XR)6t&VHeAb~o+N6fS1({3h$Cb485s)3 zv)hf9HLL!Py}iwWK;JmyHdUcLFtqv7{Azj~Ez!R!piar>a`>k11BgLfN|J=6VumI8 zvhNXvM6TZRj>8#CkPpb^h5ohccy{San7k#d=Gb%Jg>-emK>r}s$g;;ld(nm>76$rP z^V}m#DK0pk^A?{-TNIJ^2l?^WzQ%?<-pM~*bsjVFjxf%A+cb4(8WIK5dDHvZo`}gh(?+@kBEqCRp%Fi6v{98m)6IK+b@Kiqv%bg)g=T z4qQPp(W|C!g7vh4@#&QXNhE0MIEs#X8k{)i1^r1!bCb+$!OCz6itVxLbEik zuY#BNS$5a2d3=41ym2J{bl!T&CA{(AJ=p80H*x-Z-@$HDCds!EE_;+;-Tg2#j=O+& zyly^e1peUTeB*0BVcuI7bJT3h#{LAGzRZ9B>qj`}o#*n8*Z&V?&!@a$fa!;v%;_gA zLd(^md869^QE)w$w3(tg4d1S&S(UO;Sr`RxpmiMRhO-z(!1)fTlW*_akUh@4?^1B(->I@d|l1 z{^?=v|I_pMr99yv9Cr45dG~2YGJoMgyyMbKIDDE6DY!&{p1nEiT_@9-M5Nov zXhj4!$>RJb-&X~x@rt(Vw^;C7gE2k7kkPt40b9`tNb+t!T+txS6uTS&56>HR)(B{# zn-w|JWcVx}Me*^mSueiljqqm#B~eGpI5wdp&vixc@UYe+sBYkozx^#Mk`h7Ok3a8d z-us^OdF{eE9D4diy#MrrFi91!P+;AoKjXH?EpJ?W0Y}X)5!(X&4}YJp|IbR^aKX9E z*7cOtCwlQdK7YmeykqgD+<50wj0gRFDF>c*F2~R7Ac~Xi+)D-RG^E8Qk9HBVqKbu# ztfnK|r44u$m-Xwnz|qk_H5uiEOGY0rdWO<=GvQv1P{jA@Y^dzwH0*8MEp8OeUDaT* z7%w2`dyzjZ`915T9UJ!3zUWjgIqz)t-+dm(zU>2Cc;W)$T0}nQ@xuK-<<~2Ec>9It zv44*x%@uh0t{b`eVd3o;oJmi04KC4XHocDZ`??1Q(-^BEleU3V7F|VEO z6D6*d9>11>0HwOx^YH8A0UL3wQ^lL$D>vcwS0T$pk9gq7W{A$V9wt<%)GNQuGHlHR zxU|vQ)9j~FGD^u7UycSpW(3@(uj%MfLYw=1B3WvsBsPV%I5 zE;xlx{QLEM{og*p2@5)j$!^7bsF8-S{=t)CtUb*+%U*^`A=J5K%=HbFB zem=+ZcYKp;e{dgbRxRh}w_MAQALyea=P1*|frlQ>ZtWpavh51cs*vvDHGdwhf+;&F zHZtO)zMW<^(Ddi7js|NZRES1ZEESkxEo~~efhKb&>+8wvwpF}>^v~PdCaQSd-gMXx zNuz?Lq?(TXy>Xur1~}=Uw>)Zn*BtTy*k6 zNTIXC=TEm?%eR)qoc%ZN;eZ}Xn#;3l$+!9H&)0IuQAaVw3=#(&gwOnnYj6BMPrS5& z$9{PeU;pu=NRT569W(bio2xovAUFBensQpVUB(2B>J-YYW4l5Y zuT^!V$fQG=gwSg&$$++mz@FA0*{0uMV6*4s=I*N<-61w?oTHB27hmbEMngqO9pp+L z&;Q|8{`1yH@Vh(F$guY*7xBgGZ{qV;UBKK9!4x_$>mTIWFa0;(p>O8hryT%MN)RvO z#%sRCK=)xBxpx=+6`{lLyU9W0$aX1Ma7Xiq0k18-SJ(1aOYBr-EAasz@cyZTfTDRjr{Y6&th7jc(AxvW9O1Yf;aFe*L>xNAo0TLRbr_)j|?g z{y%%~9c9^7o%{aQT05Okv8!`b&JtWuAjk-8z{bWH0~Z;A1Q;+EKjV0f0sGoq`(8iW zU}GC&Fg75NL`DQ7l7uCcb5^T^I;m52b$7)Rc35k^Kh{1KI;ho2Rrc#SMvd-KbXMpUX*Xif0cqie`2F}5WvTD6@1guY1p-2agX&d{InLsdZaTSFM_hA>5 zJF#lapY;m zOdRZYULMm5qdQl(T3+V;!57~b7x?4g&tkaXaDsYcK6KV zw9}TO+BsnbHb1_Gz2h0xYK0^=Y<}=jMigL$TF(G0&RC31izCBrCm7O-ywA*jSR`P^ zEH+W7P|tbncE$@f-AwAU1*%H1$QX~?VwMAT=LJioOEWhz2Oe)76DR1RLSc)vR9Vju6nh@MiNF5fhxy*l#a#8Ct2jByFcCa(H2 zL)&?3(*|6vi%PW%y>l&3?i>Lf0-7bKU%-G9ilbz0E7i>{Q%aF#9&a70y(mqfgZ)Pomcp!N zvW-EZaTM(wTL?}kBm6@k zzRn1voIAhzIexIN&GNG^V7|0)QH6BtMxNTS4O6RAsn&5%KgOCpEk8{qh*)vzdGr*8 zr)SU3<(re%@p-!S%)Ox``q21$_jRR@AzpNpUQc!Zk-}TEDP}Qba+AAOZN$T%E{qOb$0D^$SjW_OjRU+X;%nRYO!13C z$1N;?UahI4kcPuJVOCRDqLBFk%rh`CyEA8$Qsh}qYpjjZe(M#1D6C*mGqqI&OrWXO zLplW~g{EkX(iA}}M{{&yit;Ri2_nKE0%$@-c#Xo4vymq%94JXQ;T5K}HMc7Kcd^=Jn1euseXx;Pk`3k{-KemfKKDAIWsE zTR8xB0o|NS!LP^Y1j0h5l2#6gm1aO|T-y=mjx3%frS|f=O5Gq!1K3_L`*QpR&Cv!O zgbG3s#)LX#|EZEftvj0hKT0F*2FEcm6)VbbcT0 zTu}`EgsZRlKI37X#fy6g3a?(>YE01BJB~5kI8h+O3=L<1!YN4l7SbOGdqr_XX(z#p z!X?K-JLTw14CrD19f~eLGHtRFU4lZlosTw?*=Lu4e_X7^N%!^h7H6s~1 z+oqH{?1>1+)C5qPbi75oaFZ8CbVv|YF*4N#G&#Pgk`NhF_5v%Ev$UISGCbZ0E2xom zeEDzwkQ=W%nX}G5g(dynWMjM8w0;o6R4Uc)|26M^)U5YzD#W^FH4geGTibC`)#n6Duf z(8iGT_aJTzCNsqj=VI#B9;&hR%Fs$-ZG*kL#s~sKk>%_e+KVxL{(Cw{T<@b|%Bav! zRmX`Cn2?^{dbywp#srM+*+U~sQK|~gA=(f{hGJ}xNAKLuLuH_>O+cU=;lO!Zan8=`Bi#WXtBESmsEDMp=T3}HT+@=jra9<8Z4g&mh^ z0oN>?r~)a3Vu7I;Fa#U&eMW&jx2ClsO;d*Vj9|PGdF_5`MaT(!5XzV9;) zKvCve3T9pPe7Y$GSt0CgIL_-cC@sWFk&Dn5K@uz69B5aVwAmIllg>!8-3;d$Cusvt za6w3|t46FGxu{7xt2j&C)lF9zB9ej%FnN>VvE59lfEswA${-;U%igW4`RV4zJ>HbU zgb}pcthn@@yyJD}BUwRIY~?HedJT__h19wru2(#3Byi+;n=J3N|0_gNnpsg+^EFXY zp%M$$sUuDkA1f0P^qk>edl!DO)}`R;G`|TL7Pp@e>#_RGV`hRE*mznTO>r|0-l25# zIqg;VW|6Y!t8?Oo5$2g$rBAxDA~dvfh~eP|CNN+vIt&LY}>ZYj@hwo8z;7Hn>X+G-RJ(<@L7ckQ2ItkbVE7+O+Rq#QjKA4w)iasouK|umiLiGg(W%wD^(?;NXsjB)v7xJ>OK8Aj6US5 z4$9dp$xDSf$eC9a!PM%UIMLT+JbA%-FW84)?W8LRSfkIfHh%uw^!>qeh>6{_C4Hf^ zTi#c;BfkgtchKm>=}ewe3nZ3O-!pc1>gOQqIErug>~)M1XjhZ+Tp*5g;02(0rP~3| zi|LkSO;jrvR#L-3LY<*HHe1u5mbpeUkTys}>Q{*>afLI5Va>PVDVz-<`woe!aK|4# z1knKrV`y8d;fJh#R9jzS>Rk|IYP{De)0Zjjz1%wVmKd%TYSzJ?zZnkmpmYbcvUO=@T8aU4@UlgA{dmETK1Gdq9R4$qEApj zm8!bOs`>maEQ}fMtj8;qYU}}qHKbKy-lSYjJtm^XR$|H7ILA@pK)D@t^eh$$xw;oy z@dW&6RySAt`C)Cb3&SRx58lcI^&w?!k(Jj>TJG8T(sc~GcEFS)lFLmN&^P2Qv{0FH z!hE0dFRd6S7@-BeG;A zbAi}ajmaeZ5fHuD`tw>Bz9JWL20rL8GMwJ92Z@4}TW6H)wkCO@53@vwx4oO~=?(O) zfD%6neEqdU)^iC%!HXNjr>gCVgP+zAqSQ&VkSSwzj_M24c(zm|b!+Zs8+3JjdQty6 zKM)tMT!J+HQH)IROtZOTDe55cnpc}BJyadWzBju29I&+#D{bDO86hk;KM!3Z!~u&? ze-;Z%at5nAKPG9aQNZG_uU+}ZAD%7jO+PJb6?eqAW8cPMdK z+p~-N>y;csWv|wSUl6BHgPAoZiY;CAX$C6_W>cckY4-L6aEj1<3yevcp{uCU@caeu z98(L6BC@{RDIIo2687+wES0`!-8N&zJ?cEA?YJ2nr>w|de)*xMdv|s20;MnEX2G_a z7*1)3^bUC$BZQ|A9x(c*ADPtPz2BfoDnG%#%*(^BoHUF}0=?8O2l6M-0;-I3SYY>I zL({eZ3V8E)5 zPb~f^rCOC=Ox}xzXDbTO6k#u}RF{zZ9RZR;VU++kMe2v?Xp1Lt;A z{bv-IvnEGJIQhVSw^F?+&0O)^IRjNG8ZnD!NEQYaQ2Ik1Q3;A`kd+X~Tz9_BJLG8L zxJ%OUk}M&^1jlu#&7)sH3mLPxz~zT~x4rbEeITCtFXP+HPgI5bP%IaXiJ+{7Np+2p zw%-?tg33N98uo}*3i9W7^K3j_D^b9}+Q zS2!jcdJbuau%YN*gH!8S*iaVpcZ0mvJn*!eLtK;vF3N)d?mH}FO=fDTcZaUpy?-|d z6h;!izvrtr>%vqt*R$*UY2SnLNNZCzQppdDLXS`FYeM=&CaCg1I1 z&wbkXy@(>RfOU@No4E6)VTf3&1XXS$tBJXF4_?F$_g!v9((XPEH#>3^i|hT%67Ph{)M%=<*8>M_V+Bf;G*2%$IWCl(?f@#l#4-h^~M1=UOf@6olK9 zsjhA;Hn1r`b~gG*|z)zdcTOKw?*{kv9hcZpJSRMfXOdIdv7==EWb#%s~&%f zIAPn3ZLd-a^?Qy;n<}sAr`;|Pd;DgiPdj)-{#SG6x9VS0&cHP`aDmfX)%PUyo3N;6 zJPB&L7zMRlU(Da7K2i&8mXz?t@8;^}(l@DDaf6e7%JLUwgDE6O4 zl0Wu%soG9-|5ydfYYL+oj{)IQbe@PjOalIWX>m8Su6A38Q54fS_m33OF;&}+sRX!=ZWKN)g zw?jM;h#JBipPv3(P4-j50M!H;v4;#a{_N4S7$rkhiz8QA!`r@qQc3L&-p-8|r{qTN25MOC1kQJ_e z?~c`$qH-irCR#M|VUs=3wKuYsT;?{k$)UY;#nch;I!^eOSyGT8utwL!uU0zl;CTnI z>A5o8KVSe)v*RxXd~WcjB;e-y3JSk};U9)@sU{-k*(0dcYN|`{`aDEE$$(^g$QsAES~`8pHL_}Qy@^i519CAa=sF|)9o<6*^b8G zeM&AMMaw3*gDj^KPH0nOQYVVi$jt9UB zl5r+5Ay%uq?Th9*8_NumMUG0w6OPz*FJi4a=-OYY?2xItV^_Y%Pi}*EYK+s}K~g$j zI=t7LOUzphqc~{{-?VZc>rQJXyOLpCX$9ZR{u}vk_iA^K?Anr<#=iU&>eSSli`fNOn;0bXZMI0v;bH`Osdj*D3?98eRY>G188!E(X0cnt|Oe5aSn|*Qf zVM+2vGHt+j5#DRDtG$H!6lPmJ0Dbqa>vJHrZ%SJO1&*+NPt}8~{aPnisVYcjeHwv* z(}!HkOE`}*GClh8#M%4#ns4lr?x&FsQ68-1}u z4i0QJePx01hZ`xiu0R-ctOy2!HSMkKyDe{o1r=~h#3F_&y{pyX6`de3cS)8SKcy7< zA`EPOFC(z#Yb#mZc?dSKpKFWUeL5_Mc5A4T|B}lo&&_u#6tlS+OqeZZkkQ)Ra`bAI z;xS#16}Zkcncx2YVkSIJ$@rrryl2|+gkAXW0CK;80B?340D~)T3y_D?N z5ws>#Pj$x?XvbS4y~)kHk1F?7a>tiY_hb<{dBZB2vN(lj>s^Qz=AsFo%UTj;gOSmW z;^uu!IY;|AVZ8o@jPC9@do}sFB*Ga#w)Cor-GzlqyxuI@0_ixwe8)?(fH*2T3Xz5{ zN9o8t8a^Vkc!_(g1ge8!$!e{FoUw!Q-D)c_DNjl!n)CErAD{!=b?bRdgVAyJ0LyU# zr&qjD{=)Rx001E45dUh66|;9hC?u0ggW(P8qho(Yi0IxY*%s77c9cBdIoSs9$GeQqzV7 zh32wXrSu0+U$>;{-G9cC5mF1B1!*Lv$h{?b%C9!JULRS zU{TG`OiDYu?C)skouA&>`T8tuK+1}K5@(}h!m6*=r)A?)r$`?0obVyz-wDZl{cF{3 zs1MvC8EjkKmv`%*%92IxyXe381E4*I``_<-C;xPu;@qkZjnaEktMPF5(qaz-PWp5Y z(WkDAx$B)W5PjUPc&soZ#XW-iEgTd`Kgzb`=UFWb&`$0Tgzze+;^g&H!Aedh69S4` zQytlldXPBGL4u$cxuih<5;Da7r0$QCgzKLyI78+lQ>}`sN^?tGSiM>{squG#( z?|Y!tar=GUPAAobXq-|SpU-mv!fjAXYFL>%g1cW+O#LLU^MtJHWkU-iw*A-v4n~#SsMKI3xO$J6-ivl>M+|=@fMy&KSBah2>u!AMc7%dtRN!5{ zXr_>*uc+f8%XY7#Z7@A-lCLACdb1(CMKx=nk`*!e2AA7zsS~#I)~fB6F@NqGe{QQj ztY+((sdppt=Vh$ERzws0u zT5FQ)3(~{DMqlk3Yc0@Vg)k*iS=?Xx5d_Ho6BJ{;MMwfLWXjfi)s{9^l9>W0UogQ7e(QYz z-*F>Q#^bCHx!EfgTh|*iRd*S1;O4 zV;7Tm3~U-}KZITzNImr-gO&0!k@R5>e|Y%Qqiu6LSaOazDJoSyQe+mju)@~ zv@9U7^AkE7>yye-w5;QP(ssYWKaK`D!2p!~%L(^*dnD(ejqr-Oi&YOTQ8fs<{vZEU z?S46ta;_;!D!4rDv~_PXv)@rrGK~(am4YV?6JJ_>ESDW?bopg9$Jdu7a?Z8Plg;*; z?gt+I%QdRg3F2%+^a)>Lv-fpD#~>i-@xnzU#UM-mthDZ62zdM8*@P$x1)rx=u{w zZj+GPEGlzEDRej9gJr*;Y83oI$-&Oe^RrOKGrlpQE)o(AOxWUqc%MFP6j4C9hen+h zl(&0?iqc2fE{%J((&a?1pa%rnOz}IEI{LJ>+;QnEFP8vfXo^${muh6imQUQIy3j8T-yH^R;OYWoESB{%}J?TNF? ztzRfj6PJv#Aiy!5!~7Osw>2hJ_tvz-&7QwMb8L&Az9YYOaq0FlD4g1em zznHu>)L3hOq%n>-SSGT}3>nwgR9^@U^scS)0x=tp+rLyHhF~DnVLw(wtH)%vBPfmmyD_kx^>EahA>edGhAHIZ1xM+VZA` zNla2zjca1|-ID6~OF&TN+`BpaJip}|(B1_7kS;%z#eth>wDxk?{fS##^$c0fTac>C zMQYnDg^0sX*}w#*5bP1>IIQ;R4}907Rrn(SC%z<)K^p~+p7|C+?RFs|HR2Gx>P`RU z^5BZ8%82@pfgCNtqbFUfyT5zOp7P4;MKT>ZWW4HCo6hY5Qs0%2buX?4y8r2>i2lE|QqaDTgvBa{JtC z1ETRLG8Q0(J9Z5e+6C(f87=+cM zXtW+_!i-I~+fkA_*i#jz4?B?)PH~cyBCsyr4$6HHm!o;dNu{Wm<3rl6A`G!24ZZAq zn`3FCE6_r*H`xst!XByap{n`@UM*)dr{fHDJ`nAKrZ|hM+ch=7nGCj66?rwo_FxbH z>zkS^o#dyhACDSkv?aUU$C1N}md>!(f~ZJeWzH8jpO-;eq3Y9}>`f&mr?j3M@z&?- zlKUv&%A_{RaLR|Y`zsy>a}wm$!%~lSA={6O&zF|H8lM!l972OV`$Z?>7l8O49CZAA zcM-B~DL-AFhB#**2VsX~cSI2v>avhmQ_RLfVe3@i206A#>49XL~-M{-|UNgWKCyi+o`+)td+;mm)GVlAdJt| zZ>J(dnvQF=mut*aM%m!cDzKy6_Jo@c;GceNE{9zbfL8~u6txJ0X?wsHW3cbOnt5O^ zJ#iM5Z7vE|q`|H0@M?_I!O~ByK_o_{rnVm`=93QAZdT+&IYjpxo@K zyqy?3;i6n=X*D_c4GmAx>+$<9R~?>KJ;=r2dnZ&9{o-MjcmB&qQQkXre50%Ot?}y5 z3Gj)t%lzyCFwvbqva421jCILezIL=ZZI2LEw!tKM6M8rijzJ4)m3$va_%<Ts=C|uII!cy-B~a6BUk?tzmCWX z2qP}F9iN=!;TFxY1%kpTI@M4wBD%QFRFNglQz8P)oj>rS*m` zfkq@e1-7*%78I8TzxW>~vj`N*WJ|m&=u_=ii`Jw zTJ5mLknq@M$zO6;S(F48rgR}V5BE#9jVW`ASS^+k#>iC-H`8tLRdV5JeL*!>_zeS7 zuu)Sr|By9`ISi+<)Fk zb%~7LFo~m^g_t?Y1t)QOhsMa@Hs)coCz%{q zF?PEZ8A;(ZtYfjfAH^(B@4sw)Kcv(*w?sOr38AA*#Tvi0Ibj{QXt(y5bJ{`{guv}l za0Z4|^c=0cQEffHzV;W6==uQTZCASesXq4%WTkI_^|{R*oO!D2Wmn!oF;4@Zy=Z!4 zADOvMGr%2%8B01!W#{|0)K?#$&mOOGCJx7`51mY@;=7`kvlm+6Hk?!o_OT%FnCX-u zc!?`gC`Jl+WncPEr6(PPc*7%(%SR@n9&|{8Z1>d^K!Ln|Q`joA<~8#?k%KgGh*M81 zW;+J8w0|6e6F*zB`12%t_+=`!s=y+06p#uW$}~%8870;do6bu$d${W}P(ax*nMGWROVa*!PPm<_M!gQ;{l~by*t4iQ@@4IFH!27bC1Jgwj`vx<)9h1LFhG+xa0Mkv?kJ{>_W3V zCU+{L@*(@!;J@~{z|Pt!9o{Om^ah~F!ZCHf5DW(rWoyESs?4(64kF)YKG<(Nr090m z6I18rvJfPv21XFD)%Zp>o_Ko25fw{?Ay7ULk05uUSEl>DUgdl;u5vsH(fO`u&?e6E zu%O)yqk1dPTCS=*)%li*B$4RV9jC9ZYN!l0J8Ztox8(k78PO}&`Z*+)tH^V0axSIO zvJrbz`t_i-?s8Mqah<7jEMbso&D3H{>$>T~l@a1lws4&bRWFndvVcUjWr48qHz5aA zkp{|+Tx5hB^zSp(!Oakz+v)x*EYJD&`Z2+>bDF|mrDR>(F3pB=Clr&qO0RAoZbBAU zloBPOm^G1g^yXEy2?_9ewtg!h)Z!5sb|t=uV~b}uhZ6>K*NcAFT{695ad@(6b@nNP za8s`#mty|t4|T{y=;%q;>t^3)ge~vS!H(iH~1YCsjD2VP4d!MJpE4v#J`u&DP^cn5%^V7(iPA>JQ z$STCuVU-x$c0P2T`#t^V@{Bv$Q@soMs+$&?wHn2f#|* zk49U^T#yA9u=exyb;t2eYK%sdw;H(QKm(EECG2<&DCL1SARI8Gd83v%J|q+kcDre=!gd|gYIG$(6D4A2K+e7uzI4ZeqZ9XA=Pl^USgE} zK!NC_fOzesH-zz~;MKU;sUR>A1PhddrkhXi#=zgpIkV1xdP*b>#dqQ@kSRw(q@eFh zfMUU!>8I>H#ObW>Ij5qOs^xT;!6}OC7(G@WR}z4`6Bc1fR>ZL!qEj}hvCf{gXr_cd z^Nf_l68lr?W)JUbXS`G@>PEIe0+%(_Inw7O6fG3ZM-zP}qTbQjXZasCl8M8`fS60$cD zw)co)BB$AG*ICja(}>TVAQRzO=)Pw!gjF z4A3EtZC^GRhODQ6G+N$KyL2ld`w3M0LeLB<(4{Dsa+wq@-36K(G;-90k<@qii(RTh zGiJ8Mo+E3P1&2zv!}lsah>S*_LztWZ7J=;>)z>FS!fS8QGX~S_0{-QrN*a0v!Y}m; z0cuu5t;(Oa7Uu_yYVtz3s%aEN!V1FaM-AyR4V$Lb@~6+rKA?JF`q8u;^l4=X&LDfn z{&|ZC>vQ0lh`0+aNO~sQSRN6{c#X%naGk8=UKE3{9>p5Q0YT8EDfq{a?qB-eRk?jw zpfGJQ&+}Q0Z?;CH^3rM3xN0WcVTsIKImUnIpU^Q9EYOo*?~4}H(zO9!cN#cQz4B+c zx#iqp97oCL*^_`{KTTJ7?YVBL7oM2#ul2malzQF%3vh021m5f&8#6sQIEftmg4f?J z0stkJ44|y8-kSr~j z35|enl@0qN&SNITWGD{`v8&pka^`?WfJtB8>cKx8%EEsps-Us^h9XL5S?n(~_@6&< z9tjNNt>#4zk;FKw{j{b ztzB(()A5P;E5m;^8pmNI^)B6JmQ04CQ!Lj|0R^77fPy` zMaw0ch8(E!h5bUMm$7GQs0WHX2RGhugYWDLuD>z4K z-SA(6ge<`f3N^?CCrmA`QQnTYG&lHe&)aAwoL3(5JQcFSnXJicDz5I!iVicPvV5Me zkM*qrU|Z8Z&XGYO__C0#NWiWohi`2WYI?j;-j%A)?oT!-_`WO$D-*0)zpOo2lC_Iu zZPUmKam+P`eD)5^M1#PtWFDnAZU0!&IyxqMb|j< zs?+(9MZ##FSn&R4(sLCGyKZ6*o_;D;gp#3={`N_|-TpE-12m5T)S{^od1#5ZslSC@ zAEJ%)5s3?UHy9gt;(u6xcw2{*5J(t#Cq-6d*i^Cb5M`9YmJa#D4u53a*Q;P_s6iqU4d}N?n2?+8o|l~wc+wRo*u|??s zFBAXKDwGcUQezN#iQ+i=D{PEV@mQIQ7mUi!9#QWk&RMBFOf;W-*hOTR;pkNU7aC@G z>^l&mHV|pYZBFYsfVj6r@|#73gBntUBpp`>EfJC^V!(v1*3*I^+g$Y>v zBE{jAp_c#t>fJOO-BJ{wviJt0`>7t!>o={mwnEL>=c5V{RXC^A2gIpLcu)C(ui7<1 zz08Ow>h0-`wY$)tSTQA;laH~90I zd#$qCmsQ+kqC>TNDakTBys}_FePM_u%s^SKDEBpfbBLR4Pqvv0UJ3tlQv&qbk!5?n3HIr%zVlWGd14cy%m!sN3l zh_R5o$sJ$bhn`{~{QWoL9C0T&Ic)fsl98pXr#Xj;_uA4w(@ddYDHIbm|4l2XrhPE@ zqAp2gH_qtq)$C*{JagGiF0FZLsN!bcjQ^rZI55RdTlTg9ZE(zQHvof6&rV9PkS$sV z@B!c++1bkiO!;XUy_@h(j-rsJS)(O0lYfiVNaO@*8?+9aaMOpG5hRMNqU@YA7};wv zcCZzt>GC`;Ef=5C)&?}?Hk+goN?n^t)-Z}S--L&ZQq3^tie}e*X)e0s!HRFF@!iLf z;x&vU0wiV$_$c!^*!w2gTL$$pd&VX<*G+qDjMZW=oaN7dX~D#9?A00KVIyxJ5*<9~ zGre$u7lK44?^IA2J*>hysz&iFOTT$jZ6dvpByGkcpVxv}GSPD`U4K0L%*ZD`9e~-$ zEf>mEL#ufUmk$M**~VCnNcgPgt6Pl#BWtBV*ZcG$lI5=_JgfX*2(YN+OZ#bIy>82y zBc!iSM}-#4DS?ac*ZnoeGg0p^3x9jtQLVc3)UCpPc?X=-RKRNXsJTDEsLFzMlAmj= z-am%JY2_Bs*Xqh$h*16&y-xG|=ida0UGd;;IOns4w9`jn^3tMjCk2S>%6S0@;f0A^FJaq@fm39%iaqC(Jep5hs6G++AJ#d-ew>yu}PD3;E&kKH| zYu*f;6TMGN@iw{Wu2b$oTwXZ#-$Cf)#u$D6vL1{k5k@8XO$Jc~LkM~ZA-uKh$5RF! z3xKwD8ncAr*L@gi8=sJMBN#Swz=e+XhT5)Fgyqo(`QtkBqcoc~3N1}eCrF=6ik&y8 znhHv};JHJfHl|`GlF)WJKlAZSR#}bTY{YRmEIQSBSq;kOEq(#mpAyI=qhp0LK1aCS zCPN~Ga|)Gd1tnfPIGZj3Rj?9TN)jAJE8DuhHGrM@SE>0Vxs{FPD9JI2dXgeA0*lry zN(V_)XT8Bo8aIw&&_c^DcCz%6IKY|aqUE3#S6(4P>n>?Lz-=gQ_fOE^P)!fzFISYV zrM$a;39}VU0_-^FjlxgOj9Sq5HFt(aeF?LF9#emi;~XKoJhOecQ&P4H=dKC9^G9$e z?Y^>9ZN=qaDr6!HY&^RcZU?h+V%LGQp`aXzphSX53#B0f4rBDLM?9Ldr5yQH%(ELX zv8|_wkQ0&@wm?d#2PF&K-KvW)8uy*|MEYU7K){YeL|B8iZ`gVhF-@xb&z*G;xd49G zlTho~HT{j=)JdWOf0|g=Q;7s3&K6pHWPC)W?PY`Jf6qIII;uDX2sj8>Q`H;|FN^$a zBV`USgbrQNS736*U#R;rUc+%Y&a8t*F1@XP(*4gQ8LEU$bH5$>TS7^*UGRVv1`912 z+Cxb8-VVEdJU=u`O)Zv15#Yau?2^Rx@uKj&@3>#gAY9Ju2J*)>;=)?KB7!12b^Dcd zo<%vy6e?Lo|C^&3Tye&zSd3Ztw2-?ClSZC$Q;mD`11M$zf)Q-NIq@2`J7GkItVkUJ z{*zAHyZTWyCi4eB<eF7qDS$izrV%b6b&5%e7{yAy&) z&<}kW1Ogh3(JtGp!l$yfH9$9?jY@HS0$9Rk_S0GNQu&;U^ILYA=h@~_<`MU>HtNpA zjXgUyjf9HMS!tnxUTd+&l>Lp(X*yOGb&+F(MrI0vt1+0|et1axKxlDTXv+XGk94Vn zg@rR%Eoqsnf{PXsDB3m3W&V!^9V)Y#z~nhuG>v{LON`P4j=Tx=aj3s6bJ%ODixY?K2a{~nagSM+&rb!12f493$wW`3IIws{HZN{h zQS0ea>HTUG4s{nU;Fb;yEp_b)n8Vv^!V2hxb1Uj;am?mDo@AVAgXH%3?97KLo7}Ak zgM>HLn*7Nhj1cp{ph9($Mvc0E#u?fyR@qDa+DU>;S(>o*7fBl~MB*2JXf8$xZqwNj zp@z-f3I#zq+&Sry->+`C(f3|w`w_P#V%ugO)!4%%>GUld3dL4#KaBNeNlW*tn1rN5 zr&;3Y)4G4Aau=P*-eOCe=Ij!lG+$F%h`Y4VjLFcTs53iwV@aI4 zzefchvWXg5_?w58&)d~%Z}=&4n%ylm&_XlM|J68YtBBe=aLPO!x^Sihx1d6QsQR_D ziB`keYhE<%avT_l9Q=OTc8TOQU_Yv=+opxL--2dP!D3z|#W_X7PZ`S49xy`PnO9b1 zfJty%3Xq`AG*)o9R6WMS{Xx}3G0udnz+lquj0|cC_+EpCEK;f#p69F?-COvZx zc7lsoZVwW(Do7@jIMM#`$cnGhu-wpR&N8sV zLHT1ZDT4dumuvmcIph0!2&qHu9!L9;&`QDr7LB4BwDc%_0UUj$?an_WJnfJdY*5d0 z39~dFaoXAK(71v11OwU#2D&m++KDP0(TrpTiHQ1Qc@by_*z2_R4wsL1jBz5&d~g~@ zME^KSr%GF878&?rMm;0u(-EH3tmPSslH?n$q4)u|BCcyU%Z|MsSjG~ZmMJ33OQ1=h z1KTW^yOm^s5JxN(@v_;r48!XaT;lFuR+4*F;dYC|1N|LS#)HaQQT3x%TKFrE+b`&X z9ls3|i)@=Q`qMf7iCUf}lj%YypbryPF5+BcIYY2K%Y-!VBy$iL>xYM{BZN-G2xUWi zv+0jg1yimT#)vOrn*v%zKG#2m>xAy#LS8k0IGy>s-v76|FcsaPi8x(ZE0I`v|9-o zH@D3EHj>=+^dy?A#x(VmwiMEF6%!-1^aNszu?b2INsJpAv>Pillu8|J9__Lbpa;ic zGa|dv(!z=J?^<>K?7FU)!cD4|BH;BJWrrfxEhw_o_2%$?!nxIT6hudHFnb7rYiuy% zYR@{#9u2HzyuUDTrO35GENp{;A3Wm3zGvN%TV#4HJB8|x3_@{h+`INOrbP_#9IXJUwTomnI&VtEVtbD|+N9fU5 zRnSVMu@o23`ww+S9iFCzx__#b^Xq5s)v}T_Yjk>XQ50vmoyQe-IIpbi6gR8A2&7h8 zKD~E!Cz`{cAbONS#H5{7IqpvN{&$2?dVp=OdVV~fiQkYGTGyUz)@|$B=4RKLr@kdE zD=Vw8e#-||+gq730=dQXG8TlZNSF<4kqV`U&tW*iTfP?*mAX`~rYIMj7zf_nUYMxm zrb1sxV~R3WFn$Qi?Mqgwf#PXT%zKIjM0|6~aK2wsK>BQI=zKYncgV)4>DiBk(RY4Z z^R#!)jrvs0WmNDE6!KP8R7u7fmQcHPllwY0sM@UW-r*$VM|9q=)BYx?9T}KlX8g_} zev);j@{sTddDAquf>YEOv=NP10iJPU{L`>hKDC)QWWk;(T6__R=B@k^H$^FuJjic%R#t|gP$*Y+8y2lQ=t~!36=9>Cd^BTqgJJO@zW5(B>R!jc|MWd5H zvMOO+)g7~PrF3GpT(&>QP!GB08a)lMgO(vAhARfaEs{CSKnoy9pTVI#MEDlQ>2*52 zo`RQsJD;~xV@zgjA0FdpH@m(B&qD*fCpo9V8-ox8Aql@9VK(Hw)&KqGl?%}QU*iAX zcxQuk|NmJa317Ljb(*CqA7YlkQU6QxAo`7T4LvhQS+p;Wn!ltVo>Pa=84LRhY6kma zgHm@2 zIYVp9tGC}FCo59WDg^`|+pLZ=)l{KH&^DkF?bB(7Wy|^pN3;6KmlRcT3U#?NcO^rrhh+l;WEHqdVwBefv^)`)KQ?Y zt!Y_RG?MDn_YG)-6!3;>X*91h{CeG0lL6G$ZG_gdS}4O2KTr z7qpdbKGHMr{FV#}t%Sm{MzpKrihtRjaVs0xuvgb$)U!@6d5{Iq{RfNksiaU|8yXuB zhAP1rOGtiNrmFpMEsj1XQ>Kgkyu7q09B3az!Y(N(H8c}K4L)S+u=-`q-0+60UIEr4 zc+bCxl{CdbJFT39EgA$s(vSrO9JchFfMDAdoTQ>yXJ;Renx>7$$x~C7J;^o4D#fOOf<(;ihJKqxWM?YGfY&2YFCHV`&o(I*o*u|C^& zTS1SsQV$3iCc>E>aJ0k(?zy|^KpTQar4mI)HnF)+*|6=nc-sxn=q||^7>ZDCwBEl> z9o3)x9!K{>cpDy^Z8PR^Hx4(DveaU>x0=4VpfHe!Yv;Q|ib*UC5Knd?gu_E_sP&P- z!xmFt5PsY(Xe8BN#h32omH z9^gkZ{SU%Te+~kw4~%3>>w!_QU&s*clU}Nl`cZl6G-M|;T-o!2VW3no!|tI18Y-DB~H+()t&maVeT1@&GHo695L%}(Dw3!YnSmKv+zZZ8kBRG z!{EjG9PRCpuB6}A=K-!|wI|$u z&%XM#H{+rv`tL}C!_v%V74{0_^TthM`&&1{?)FMXh83YOkofx#oxn{eNf<&8xK6`Z zl@}7aFIf$d}4`J`~Mv2 z_v5(o&v=tFge$Ducrv-G%Aq#24%GiG@>t?gIc#c>toPKVmuA~chAeUA0!Qz=FaOu| z-lIrT1`rTj)`hsL(Mgid*es!IM4q)3DwSH((}1^^odMf18B@0ZtLl3?(S9r1R+7T@ zYts=P*G2L-zGqNGgyoyH4}TxlNM2L_*T{Bfx`p=8qn4uD^$W$EalUfJhKj~bR}8c7V~l*hmy|H?yarKfhT4+^?ZTP1aVn4@y$bLmyM{iD zdZq9PBQA@#BUjKO+@_1Rje{;cE6GxAot)%FGXgqV;}-U{pKBKvh~JIa{9VRX^`A~=q78xagDL?}b0MZiyzBB3H1_Jx>Q zp}EDJf9XpKsGE`N!U}K|&SElAK5aLPX}rg%MhFCfH)IfLDfkz+v+G3_Kpt?c8QUgr z)f=40m5=0ZUP@M@kj2!NTyT{X=diDk$z)Gq4(xrVrTNPji+YwmvM9G5l?YMyKG|wz7J2>WHyc?L zN+5Yw&-~3pnimYV8lCaVD_PX6M2~6SyAYG#`hjLszhQ2I%9kYZDuSpy-$|!*g@wuEx^a-(8pt;8s`XFo!NRgKI~o2w1QOtP^x3s5 zc&(dnEy6VA(RlbG1)sjT_P)g^{(CZWU}T=2p00(!S?djiA?`Z>R@c^Uy!j-87d^BX zqP|H_APFCH`Vd4H{=6r*KVLo+*WI>~r`0#1?)ovJd;P`ty&?LgylD6Q7u|m5ZK&q? zXS6#u2-Pgs7JC%;P2#%G8Y>RFK5S@~6PuVSE7M(a>prmv3Ya>L z0)~N|9cz%jI7KG+yiFd|y?9z_je7k&XQIuSl!RP7=OZ~?0_;Q^$dfh5Xc8ZKs* z0yh!3bW^(4HTZzf8*zGynCUz^%;6(GfL+}sS$|U_5I>D!g@Y*iYuH|40IW+6MN1k7 zNfc2PVD?)dK`%)bT0T9FAWC6XgjxszOnAs(FOH#Zl%!kuI~LM-t?pf6USIVJ)eOf!rGjV@L;In6|`kL{R}{p z4lI{SzhwkPE%{!| z9kZh|?Rpm*s9EW4U(*1;L;34`j?_ZrGJ9NG9HD$$pR77ja=TJr&roEG6l^qJ={XUL1#{>nr#oIs4!2heI7!B525a#M|MD2cqWcNd zGJpA4dOlGa$8>vT^}4FFfA6;SjV51Qau_0Iw;OyBi-%iFdzj`~f156!$Kj2(tbS+s zKQvusTvXlH6-A{(K#?x#?j8l{?gnX)9J&!HY3c6nkgg$w0clWBx`z;k9)=j+>+}Bp zANT-wxaZE<`|h*XT6>>kdEj<^XzLDtlEAx9lyAMrqC>&6i(z^_h-in+AosM%N@@1& zY#fxRve_>`(UlcAAp6SeHa6#mlN9=!_VG_bAAGYcId9nY0``=blgnLyRwkU9fjVR2 zExBOS_2LPIBKnsHgLexnkW$}kU!eYR&fcgqzECc&kESaJbobYp%HN4&e?#sp^8wD6 zBV3o~bFPe9yP$iel09M;S#WZB~WA3hPQ?QQ`jbOC1EH37oqvDeG|4OB@_P+mtUVg_-9oWPj2psCUC3DzHez# zSc%dW5D_XPAd7W#rL;9NtC=r}ohnMVC8T!aOgFoEcW@V1W84z&B@q@=_wZOQQ}22q z+j%Fdm1WuEzEu10Q`LaPp_# zM9F!z9ytJ78NDHVsDFaIc@=h_%fpdNJhzQ;C0jV4nd8xMUSe)i*lHn8d*M)Yl#5oY2&rr;l$w%l|i zL?W*bBIB$+gvZ3ud&p4I%i;`Ti;)JLku1BOGN5j9gLkjYdPhbdTnooV#H z#Cvb!mXP=E*!HXKpG9KQXPUhTA@}{1cjoh=DcJ|gF1m~T;7(453Uxu7`+-;O56{=u zvk~Hhh56m_?!&rg^M~K`SAPCD#9y6AfZr04SFJ){Ik$(E?Sr<{>R00=u9A&?`!&!f zA_i4~lg6Go?Y>$=z2v%;10U_!CXHH&AQtP-5@+J>u2#Ez_x2YI_+hUF@kGK9)ZV13 zr0#nnuej~DEOecn`%jHaHopKyeKM+p)jU5+0H%1d^t>(i-0$D&bzexK{qr6Fo~x>u z2Tm>5UD9QC^`94luEVb_eJ*-^Gy2Lc{zd_NdiGdFs95(M$RTfHJHAWf$^CIkjf&GL z(TXljxPSHF%(W980~s+$uiZj`+CwAL7I-fkG^`dV2B3A01oL26i z=++5wBm@NW4-QB^xbd6K8~%+^T0lCGp#_>y3kK5;LVukP+%~<8bO)x zokXNYZtd=G`*c15iLY7jaAV0$rp95m#i$K!PRU?weAGVIxXHKhsO;>MC;BZ`oDQLA z+M-JbJ`DMhz|J(SWTnLMm^4~Y4W@t;}9p@E@FL zGY=?2s87=)U!B&McSoKE+i(7q-Wc`Ivy`S)v92BZ{`}#DSPEk%xV@mFy4rHRF9iMF zC;07XKu%JwgPVMP4Pfk^a{lY12p+gdL{zZ>#f-`SY5|^UBz7d`=T*G^t*)|h&G#Bv zt%pgOP+?8!B7W=rvhIPzWy0)7+HOY3;N_K0a_!U7Q=^5`i-(|m*n3o{$>2VG#62?I7A zV7nH8CY>2&KFn%$H&5=Q!cdbpJ2Ce2k%^p+HcFBOMK&}fEMJOd$7!(Ri|tKrSowCX zkblje=-&-HI5PWmm-DKapDUUMoMC2Z{SrSvsW)iy;3&^e8g>~icJ0^9TBv^e=2yd3 zfqdcgyI3jv=MDmyDN5$&nl_S1^za{wdDrJXHKtuzEe_7G-CwZtQ8klcAxJPQ=-~{v zo5FztM^KRT2|&tPZ+*3B{Gt{uMam6cQ~~xRmpg-X1@~Rl4w8B=CfYg8)J4uS8YKN7 z&GC3<4=zqHX>^mG9j&TIfpMSMCSrkD7M=1oXE{= zZ^R?jj4uhB9Bg^rHNKdDvY!UUYe>7X2L%dNFIpH+4WU0f5^Z0!*d(3X(X>OG3faAM zaec^LTd!jZkn}x|qtGz2nv8K*mzLEy;^)0>2`6^Ma{O-eEuC*O3;`Y|OcvR_CK9>& zO8fquvOW$=7FJn~D~=^26*O{6A|xwT_VBU0shOJJYo8DOhb7fh#!ff3G6g8oFg4O<4L_=cOh4 zJZpD~bfKrjxgPW6&Bz~$z!8RcNtFIDZoTB zRJS^=Z*=uejbzvDt_5A7MJqPmxPaFGq-K8^_*$VI_gj=ob?K#Pu(@D-; zD!*OB{DUUF%W$^&gn}aRJxG^ZJMJ#JK<2+4R{LTd%1`HLoHUJK9fU^E8r3b zDh18w`|Ke-(RVS+>V@-yuYa_?ei(B(DF#i+m1{EBMqzW5Fy>5xf-coJ9Z769LIW;? zgNC%2mIomBdz_|zo`l%8TtnL08_p&J7yDE0TToQyQpA0lOYk9V8N8s6+OG=UWrCr% zo6`q(!?M}UUK6FSQa{ZQrsL)`ogSeMzOiHs-canlI&#;jb-zAsOKT_ye9F~*BbqO4 zEAAo=O1|>QBD4G&YYRa5=UM-lh?ZdwPWuv1O?#f?(RE)cbArFERp-Y+Tp>+B&{chz zvXwY$7ajhnAjOHexEo6|EKGJwgy34TY%Gkt)MVB^avBje-dZU<&pJy&Qeq;bZdcwz zu;I(eh}Wo^c5pxukGpRNr)kR47LAL3@_8gh=+i8osi?|dSd>%Kjxf&jAJ&g<29Ej# z^kxjYoSDsX6xU&m*0&*cGOWo!hw~_v;};bjjU7W#EJ}1*ZS7y`iC!4dl*IQ0mN>Gu zr0P4x$p1tKId=S8TEm+O6I8IRd;V++%~uD(54nM!V&?< zK%?>_3Nw_Tn6imPrvzi4nfT;B!=oG!O_*^2Fg-*2cA}vYna ztpni1WiE9b`p?r_HRSO11C}x1=y7<%%miI&-9Lb%HE*a{NgPC~IRf*hn%2R3_0}PL z-Whf#vvXm~cJA`uN3a|zrdMrdN-vF_?3ovcm-Y*q)0`hExw>-S9FxEzPmf=VgRyS&;V zJasH6SY5Danvn6fbuK9+t>7y=Wl789qry8c!)V}|LMeNY9ACJXx?(Y9oR={%VXaZN zJ>sCEhVm_EDfb>j2HdXDjwJq&;AKpKfTDj%bLsC@}lD!(EQ}gO}4n?6Q+%~MWQTKSQ zFBg+q{3zp^XpsS*G)HfVB4y6rX)4cMir28rcDG;*L&-g#nLYe~33ryoJx86R%^M_m z?DtCli(Jz`za-Lgh#7mt@{EquAHI-DjJ?AGd>zgjK0r{(FD)0(xYC%F8rQW~dd!yc zn5ekvQaMi;CwNdZt;S^tEMYYcuuswQ9E*sU)qgkX-m zdYqD8NO$z({xk4`cu$y?OQh%L+^Onb{`)xIvlzP6Q8XK9(4OQ7z}g$6K6Bs+g!BWK z{74Z;`E>vC)+a-YC~Y%(13U%bk@xW6!qBQi`P2B~;VwT%vwTQ3?0*Cs!R;pHbBG4% zV}(y-X+fM1py;B>Hz`HTig{~Du@z)MDe*Ar0QdKdl1&eFQ;X5X$%TL46&Ndw;MNi{ zmp6uXv^|s@LwcK&HpMdYSxDVgj)>0k56i3|!KR}CQVot=> zb{Vn%f7th2KUc!>)=qQQVnBYF2;iRl0s`PI6(Js%Rq-Nbx@fwxTyHAu-Kni8ru=p6 zw&fpZrAof=D5kT;zY7_cvoXfrgShM_ohdRdJwv7GCbELrwDI&+Rn^G#X8XHsIS_-9 zM!$7>VC)VEHRI)?O99O-{^38H|Jm9>S#zL^y1C`<58|@*s(h;r5>Q6tbQE)=Bv$y) z{L3(=rPZvZWs0-qmrSs|L$E!QypcbV14sOp$D#K@&{K#)mYcGt3T`8h#qbt94I14|EfUU=`%I#cp4|qoX6?_&JW^zFU$zMzN}f;SZanwE~6?7ReXyQ(${p!j&H? zQxr{$`!Yz`h2t5qq=(f=r!si9e6|xIuG@)ng(*cHrI|Dst{$^|AN~lQ=^P)~+1nS^ z);hJ_>gR=uSM=t2unv_0P5CSDn%aIiQHnZ}f+@arG&ZWW2}IKYA(N7ll3z#&G1{88 z^k;hd6Hmmjd5$sE8AyJcmPE4rUH1|#Pd7Zr_+jPjivkc{qo(}jR9ULx^E z%m*VpP18FMKL-_f-YaKkXOF@^;toj;7%GY8+MKNSJN{`Z^%oOJQA8b0DY2}4JKFGJ z_s;98Q?j(Q%oq0>i{;VQ(<@G~*cy%tyt$sR`R0=c1)jIl-y4`a+XG=dazV#F&)egh zjbT!NRs>7{_>_6qenF1{v5Z}9m(v(RxVyfHQy<18B82D}Al~SF9kqU*kzfm&wQBbe`k;qUR z1@TR<8Ugys>im;Z2QepVP!KIr3K5v!hqw(eLc7xxyLB_FZTVgbFYaB{{f*)~8tZon zMk(!8D+c@c*FMN6-!3+u( zUHX%Lbe%a2Ubaoxa5GBP9F>6Z^wiZ#n6r+GWuI5CC(2Q6%;}ki3q-tD12*`K2)Oyd z)&XDj_uY)zH@;4c5k9(@Gq{+S+MUc*0-w}1lc?ptza=bv;wuMjRmNigxAOJlN@&Kl}~OW8GTAAuiM{72wXbwY5Nu&}BE9oPzgeYqQEIDPfb4khIq+ zIjP{f^dg@<3Vn`mGaOs)bJVi%Wh>+A>Pj+qc={D{Ee1C2rR8C!&kud&?t~tVXF0Lb z+Fvpr5N#u{7`v5wCV)4=0K2Bv1%gKkwpPd1&pC@cuh^s6(nDd#JmJ1ejKqRCqbhld zM=67LzDL%NAzx{^I(|c&E?%r2QR*XTLUCx=X3OHQ()vPDAX)85{{m?#LX0 zL}aj;a-;rxk&h2r6@W(c#dP>FE7og|3owo(LaJ5#Uf9W{S69(w)py&gZHXt1d9A;I zZ<3MEf_Mn&?#jlx9i)gcQ{S_--X>mOaK|?-5@}(?a(txq!CMGw0WCbCY-97wb6Xd) zGM11F@?ovEt+%}IR{DNRnaCKMq(mHE9Jx;nsh4Jg6D`YWVO!Er`?XpM~bw4k#ck^BZ7Te986Odhr{9)CE(Zdl^KSIYk#e z{A)y1sK};tWYgJ>fTQtmdDg>?PQ3Uw{0HN>bT19X#_N5OcKeB+?u^NkWyw_=NoX8l z1V#AYn7_~;lqr`6Ey~&D2_(;p)EDv@aN9uUQnAO<)p^|5_d_m!GcUq_%m9?gEBO<) zD<$9W-oLkLDX#jZ^?4iw;$qr1OL8Wkwj++rl8V9=H;V%JL*K9R)+K-3Obt;JXq<8n%n_iS>@V|FE*n!X8P3O_I2$BvdAcK@c$A*rv;Z(Khj z2SF&h3sG45n?AFDAz3a8@3s%ed>mb^tIh+)T1@<`C-nuT^4Y9W?MLM#eHothk4z$c z)BtOiu0XnMz*;HDA{p`|?Ts1-bV68J!hg1v*bc}gin!Djs@#8_B=YA3^H3_ioH_du zQ{lFL%jvjACJnwP>o$IqN)IK7OuAtY+k-G%llZH!$b7|r?fadZF<&rmhfi;&yA>De zzIy*zyY$iT$4WNr(6M&l{=Auvva_u~j&u>EXj%Ma`Rg;X$p_|e9X7j60Y1!tHIDfr zQl6G<){U6qL|_=_ubj-P1fT5X&$dNxjl0H zFrzhg+Qb=|ClTc#nv34&b_B50GvxMhm;t;kPe}dx-w+Gclp=?0H7d2;R04&>d{YO) zXu~}nD(pxV{oYc_$_9rM+^3*JHl0$na-kG_NnoZ}n^E^Mdk053|C_#q0mG!y_)kM> z989U_8Gct*0C$mPFR$b;6@9Ws_$v5u@R&`Otz_t#=JQ>+0d~AJctSBHyU1V2r%XYF3Wh6&h z)+YXEiJr*$3}yx`t+A09km~P6jec?IYbP4!KG-8<5AqWkO55zm4H)|&(}2Ss(kgB^rQZM1<%!7%}0)OgPjI(u|6*UJUa1d0@pAi zuzB^NHwSf5$^vjvpd+|^rDg^VWt_waPWTbWkLd*Sxyv?oW)PgNQ`vB7d;Em~zw zs6XY?b<5A9^DKk;2%9zAG>!HS5kz@FOY(E*2-=JSTic8Gh!T4FFM-$n@- z2V{j2W6MrI5-FT_e6@pC+cJvL5H6#}X?Za^D17l(Vcg#XsQ9@H3M6E%brp}3QXl{n zc#AeiQj5U?;)dg$7-D@(`GudnX9G%&%&p=QloV|q5@Hw&ln%*>wh(#n3RxAtL{4pe zx4ad@rNIS`1~Fmw{fKNs8i$Gse_Jb{@@Y^&Zp9X~^vBAEWm7cYgO!chy%1EF;`5ur zlT>VE@+V7j29}jh71rMgac@&z)+mO!{Q9b>NkZi!(HmPqy3xKueVyj{tnb-NP-Bh) zoO&veS~P{^n;s!?XkhnOwj!FMK&|JAabPXg{RE1yCa(ZXDzOwCH_%t)crXh_BkYIJ zo^CTzNdXgSMCC8sp=uxK`go1<>ib$lT681PNIaSJxzYJd<@meR=d`V_(9txyA1Hai z1RrW;9O4>V)zaP010ie8p27MKqpz@-OTGNX6;76(vpi+wjTQY!+VM^#YoWOyx9705 zm4Qh+$RSaOtNsmpm~h_Qmh5|=l1a`y)_Qb~jnscuo-t26ek9?_UTjgr#~>lVJtVch zrVzZNTO%Nxr)6kUY1?81fWCKSfHFRcOQpOrYFps0bM2tVMTC5O2>Wta>?z~Z^ zC^kl{KkuC{qKIs;XES#v(n-cEb($HNoegor=D?FIB{sMJj>xs5@xb-Cxk+5kU+e^> zXl7_W3qgn6hkq7s)y5yukZ4W{4>pr()1MJoJa6I)}P1}K@ zl;KewFSYe0Pc;i^@DXbjo=DEmf_KDZaHpg0YyQd7JvAC}&$n zu&E~dTjKDL5vjN@4UsC@V1}h!!)EF)N7LR^<}a?AqkB!U=-9OH9|{^u4xcB#mQ3Q9 znt0SFdn-6pCa1y563}8p(h{j$YQIXZpw1aaaQ~_1d?}sz%iR~W!Rk{@DyQtl89!=P z%`3gu3$>7RaN~T_RUB^j<#$cP`DR{Z^C`vc3_WSko_e$9(k_WoL_Dx+I0<$LYscrl z{kbapv-#;IUfQoiLPg5V>C0|QZ7j$a2WsrWP#jNzr)w^iO#Geo`IJ9$DlPcm zBJ#~3MlX?G%#R{mfGsKzR5`mm#!?%#xP;N_Sm-uen8EzDNUvy(wR|KEjwXh!=s9|E zc+*9x^>J)!X+?alZV7&Kou}#UeVKK(#{2-(h)Wb7215NM1btQ#emS(DBDKT&lbhLesPWSvDPOc{2 zU63On?L7>o={xV9^a3y+0Xk3x6wC7+91HUWaBs)T+4&E+5)z)?7TO~0Je6y38;=+# zl?HP5c~aKuD$%v+rnIz|1KcG|O-TU0rhqm#HxtOvb{M~@)Q-No@*3cNM4XzMn(JZ* zzyaVibDdn5&ffy1ec#iv3PrkW7{mp>q z8}l3$+Ep&b_)2IEz{^Oo#gs@cl<4#8?0A)p7O%?fk!aAJPhHQkD5Gl5_Xu3_YZRh) z?DJFX_RYEc-B~3r%ln(lWnY-Le)|WseeEnG=Z;jO)mzivtCo6Rmt*5ev4;T8hwYt` zS$ovY$>7~?8zg*ow(uPRx95Sj^ZjNzDffp*02Q<#e!Wh85|}@yCYEc|do6M z-d0pp?BQ^8bzWO=c5G`_RaN=iokFkg;XzZ70OXi*26xC+t6-JG#|#m7Mze=Sz0n;v zfGu(EMB1oIB8in9uaQ&m-AvoOjWi?Yo!?vWGAtADAcPU>Vm-9EwYh*Y(nE9tWbFHf z8g&xRFDQ7nA_f`E5^%U47IL<|3&VV`F>;Ssn<@^zB}WG2apMS|eoyYW>O8Edr~VG0 z`s?Fd?SCgQ!w{|vVZ7XHen>En)~s^ku_Y%E21 z6n$Rsv9F+ub4hD!8h+s3<5+=lpEHF;F;Oq9nzgzU^jymY+@Wm40dea$mZTe-y@NByk+r;6bymfcr z($~+zv6bW|V_!!v9SvJwLyF}!HLY(E-Y!~}H=j{I=j(3VjJ#7{&Qo9PcwLoGX|sgK z(QAetfDa?5dEpbstEJw#ZWKlHHbwI;HR|YHxMMN@m^Z` z34m(v)}ODz4yr+esd^?8fr$I0sh}(OyK6bnW=XQ@CbbJn4wzKg-m}#Dv%IYYrg-|4 z&&Hfkjy49gzq6%E*KNTM_@+TS*1Hqg0Ja|jQ#f~A#{4tjUU9-164;GwLTK&7_K3!^ z$N7Y|wzkK!9&1Gw*;vy4WHrfmwD~aVy6)&wrL?n{hu+9{mzP$229m+|mo+Be%G{ef zdVBMJ7{7ts?Qn?z&7ju<9BqIri>GlI4J&c=jL=3>3|ualO^toC;c@%Pd`3y>;o;%> zOGoegJpVqHNKG{Au&yhQ>tPJk=!oQ{^4oi8U;m8nzT%k=>#)t=`8lU*Idrx|93x{h z8Ibq%{FUorZQI%C7#Hl1ZT{9k4Ckg9YG0}Q;fd|(5Z67?g6Vn3Wa56F4J35!EIV-Y zP5SK5<@Z2DNe}vKdH1)j%atPXD$R?tcl*V`$L9@>e|RS)5XMyP4?%$UXh-0@9%!;w zLQX$@<@!(`3`Csq+;hfaxwy?A)%BZKOU_%v*)BG#IUU-+wB3(=VW+oi_s(5UwvIFl zQ#FB&H2?~A@JgZ)v9U6pX_Kg{jZ^jEbUb{!j4FzX`^zsxKze&i2XXiD#PFjh?RcS+ zJsBY|lY!mHU@gVS*^98zs^N?ylqWQ*ltM-y2NU}??F>iZ$9bnO`D+l(;5s-|U(kO? zbxg)f0-%@NR&@-O%`{N#WBcy|J8IuT>>QA{t)OO-Ru@~d3GD;5{K}`zEk3w|-0{-T zAJ)vZjlf}Ty#r*@OG#}NpbduQkWdW0`e##z-*Q)WEEh=U+7iqh+Ngc-4*pjQ(DiUs z+39as%o7nEjb-Yy@iZ^+${;H{8#yA!X)%_@0kA`4rTmcVAqgNXpKzI>c!5|feuI$n z-AUDJzv0u{62=4QMe*}-96X>YiHwXa19^8|WBQFOoGyP#OG|6;dsMQ|I?C*PS~__5 z=CIy&wnSk(S1cVK024C~LhiJe63%kth}F}2uX(9areVVJ5#X7Qg{9 z0+ix9u;Cah_Dgmvz_tF7)NA+0ZT!y8+wKd-(kf0I{{|e4z%ABiKieg*2N?d2&1f|~ z_ucb=a}X+2r&^cS@tS*aanT34qc>tR+g>*a)D8S;2%?1LxC?d|$Is8tPii0KSwTY0 z9B}_cV-3erj@3g>*|3n}a@2wRxVX4AW{|vtgO9Rtq@!C2py7-CIpnSoe7&ML9-zuG z)Rgllb41)TxPu;Rl__t8XHL0aZ^&tyXpId##eOaDkw1P`IUgnbj=&X4`Je#2(1`oV znN?%Xh0KKBu63qpx{n_}o*1X2J$ZKVhfAcCB*?$70Q1Rd_IU~0Gw!__e2t9z$0Ymw ze&6>3Z`?FWJ#p_}GPQ3FK`2WY>F;{YnX!-`@4va?Ok9yE{A=_D2w zg7PcZlv_$uK?weO1Pp_fJv}{PIkCNdcT#7Wy1KevK?+wptx~gEj= zQiV^Zls4ii)sB9UR8WcQ!E;&79Doj^+l#=$mNaEXY0h-ZzE5Z?Jz`zjC`tQ|AL-Vt zRUN(m%CV*v>9e~X)|zckOXKgfE}>HSro@w_Xw^JDabIRDbMZ&WJzNR>d+o)io&Y&Z z<6f;yj@yq)Wf3bZ2y)_KvxDjZS?dBg=)@Cz@wbv|AzlATCAV1<$|L_<9f?JoUSA>w00SxBs8<1RP!hl10TLnWX0siTL>i4jGUs zj9N9!1H$l-MWX#&8TiO^0SwDsGY858ay({7mw~6odCeR4&KAICJ|cyIrK?_g!o+~Ju{mo*<2c`^HEaj)eM4`fln zSF2)^b=BPnRNabdK3^ceO7TYlM3IG2nsH;C6MMErAKDYzuL#`jUQEp8si&L@7z|;-`G&l7F(P39N< z^LrM)<-w2=0@yAxvJs8`o!s5Mrz-(fSe>$)skWyaJuwQ1;Ednr3G^f_6&Sw9&UX9q z9wt!kSZrN>MZ_V)jpk3~I*Zv0S*cw5aP1I{K$aC)tJBu059dfw9&3|+{-Re7=9=E4 zL2lO$W0;ftbf%OBF>r$&w_J5vcRFyRiB{W^L}PvB*1Cg(9Qx-Bu{QJem8bVHQ4s_m zmbU#mPgL`wp|I@fswquHV@^F^91(N48P*g7aKo+@SSJfp3BLhp0(8u>!+qnaE^)X% z`*0+c-xo1>XUtUavD)u%{z5{lu1`$s6M$Z}F;}#w=IoHQgd*nOt6;kRa>2c+49%)-VXAj)L!?%E7&sBd&s9sn(XcOj2PtsW;pwR~*#y^uXKD=Tq@ zmjbw z+~6CzAY_ag0S5Z-PEYK7)Sz0U-3%QHX@*J-yuCU*sIIrz6Q!TQFm>rqJG;H#cJ~Fm zB@&NvbCCLmhT3nB76k#_@?9M*L4lti$=$#+2Lo9L%B2_8wzh7xh7yGL9IB!&)IbT= z2I6N?YSXieigby>E>9mrt?$xo99~4Bo7Sfra{~HPSy@?{k2scQk}`uv*k=N))Ax_l zmsD6&2lzQWbm$Y6*H3+c{1*oo_w>jmFeYtY7j;Fxw*CC!H1JLmv~P95+|o3QFRejL zmpG#E{Pmm5R%6M8rJbVsdKW1&pz*DyTgFp~Acq381x*jiFQs-TWW(y~*U=en&$+3F z%32glr>!@Vy6%8tJ^AEL`uIo~?}dVBE@S$HNd^}en%AW4TK#(E<%?*XWEB`zp64o- zKW8{PE*_rh$ZgyE{k9y{@~HQ8M4k2OK7Zn4W53EXFgltkJt3z>mzI{c;?2Zd(qq1} zDV|}DdT${WDUn{ce{#aT@8P#;ss%Ma+gbP!8;0pkDY^gt_Guh}{Mf(*)q8hxaN!HM zXc(_XSWi6g(gRU#)aKj8)s^b+TRp|7g+m}Qyt%oV^mJE^-VtVwc2mvEdux$AiHLZHTzI$*2J|!5YO}y_W6tNq+WDdV9}J>Ia#g>1 zj7|WPYi@uyI5>E>`3lmvbxJ%?Wfq(p3A%dD-eb)&7#jL_rK`N)$6sOe$00=s-Lq;l z)IX$GP!$aigx|88c4x>1-?R0e{i<&Q56b2RScv4`-ogH~I+2TcANL(zekQP*wdc$f zh=D(>KQGZ@DO1fJHhY*@Yxww!^|%WmDT?pO>pe+vbNl?ukj32~C?LCV!#wCVTMk*Q z7M%5P-%`)xe?>80Zzkv9z?Of%1uCm=OOT1FV>n z&Cf2)P4xEokM1-r2f}-<#ZA2&*HjskMOdAU{UaN4W|b5dusH)v7MQlBRWU+bi4AR3 zrRG+y>-Ci#4-IoX9#6Q2&_m7Y^c2+vN-T*2PcaXU{XlWvAo=ZZrnfqif8G`lb5_}t zwR&yf$!jP4kk`wJ*!SdDu`hY(`+zh|6=!{u*cRpYkVYD)!sKTb;s#=JGgFGttgI&z zf#AjWmJzL7=k6iuQ}9*~o>i4iyvMmL{H@Eg?Q20t@^qHP7)gP9`L z@X%QuOB}i4y|@{n)cUqt&X`y&v2QD2r_DxEt#aiz#CYJ7^>?V_?6viK>gz}3LeAUe zAS4+1T3B3s{6{KR^2E*;+2Vj5HnZJTe|T|OYXs!}X05^U)~if8^E1I`N5L`B!@0N5 zHc@^_Wo4pE&^h_vH3)UvQk;EOH|pl_?-B%GW$xtm9zFD3fF$*k`}-OnlyBANU+9m~ zy#D@vr^7vRazZc#dJyk0rZVBWQ!P#Lk)Q(Re0XF8BYBC5p7@<)Iz2HS4}fiEGjYsr zHqBgt2bL|OkBxo52ZzHPf-m*NPx?adPzZe*h0^T7cRbIYJtHM0 zr6G%Z5hDZcFdon4ODAE|Jptmfh;v)^sFo;(?o;F~q@Ne@w_Vv6Pq) zzBUwGQCr>9AOTT7LG3za!2~v{&ztT$Rx)hfy{+Cx=JMOW&{T|wh+y2lHyQxa4I?9k z^10a9SmnLm$ISpt?+GJnsV{ve1rRwvv>;kNUz(kjVt?7iEDo8NmF5kczv}$mKHGsr3lGWuLeHp==8(! z3z!u}n)q{7)5pH;x{1v0^FMNKyKPr@hl1r9lYh6yEgW+Azb{Q;NXFz>0V$puC@P3k zamfvd#K%FmhPF-#;^Nm&eG<;9(iT&`KrXw0R#^;Uw^*e_Hq-i7>0>$sMViESL-4tM z{;H}?#BGd|V%k(F;4$1DMKk&aFxp&I+BeprCW;DliOfq0&?>!rJq)m4-rDSF&{@>Z zPBMEKK!lhAA+X4vjH~D3MVj@-ITFYj^wWl7xVk@C8|-B7M(u@4ek093h;4h{)(Az+Hgo12KOc#V%*mRfTfP>##O z6hXrSX#jvAU4YzWa);yfXsKnwYFTS*>vC|yg3+|UM86=@!2@#GzUw)O3=7+6s;asU z_ZO{h+N!G0*5S>XQvTee;ves46g^6qF@jDT&<`e1sMoi~0jHYi4qvNep4yFB&->IL9z9xhx8zVoi+ zUk+fRUusRHJ|Nf0K$!$tDnw1=*6629&q2x#7RfO=4L@t~e1ZZ~u8rX-TamRrXm$u% zP^tR&9b@o9zOPBX-Z=Yu2sN&F-sqoCKH_1`a%AAhWdtytm51CTrbRBc~UkiHf3frWes5|x{&1$)WD?Y z^HSQ2F=s`n|7r{*PK|*&nVJ`IYy5(+QB1t9iPYL~q;R2D^B=&_$%J(5bsn#10uRfTG?>)t$(*x{)D-+_87M}q6yyi0VEl@1_Sn7v+Nz<+{Gp7); zpGry%6`kcz_5J9|626LE#Vfd_hjI4I%6-&ZxFnRUMn*_nIM4Wx5qq>X>u?x>a3+oy zqfSL!K)mU!=I5$vhDfb#O-9_gk5LygtR4TE%Jn6IeD|CUjYp`XnoyOwae!<#I)QMj zOj&tMXSTED52&QHBq2vPwc1X&X-;433myw2!LLf?>L$)%MX~qy#)SiCnd|9GW#NCtmvYzilrw$;yb8}vZylq45YYM<5k@wk3r_P(&l43 z=|=5xy*YlzCC;|CpRzsIv26DeURd{IHrRyuru7KnE&fzR^DJL&ox#J)wA;KX*qA_= z(n4iJodIXI5nbY!VrJHeF`7hqieJyu$2=92Nud8;H=3}b52ae#QysBcB7BWq^GP@` zb>e;z9a3)!;`B%l7p4$^lg&F=QqSH^(%WHqE}?+7nnSbrp}|l0rSA*1nv`{~u9FPL z)+6GwKVh28PU30l)#!!x6_TJ0k}-v&)re`6@Xb-!BLfQkFfsaO<5lH?Ni$AY(|@su zWc6v-+eJZ6vqvsVR&Js9fY}CI!HukvELpNa1^L%k6YbfCuE@Yo!Y#};m{JQT$|h1< zDs8+c%S7+JU7(8P>JqD`Dw=xjdm1TiY8uJ$T6yKO{Bi!r9g!V{zFsd00Tg8nJsQf92S6)&+OP=8cfUkofHn>2B_eLp= z*M62EtcZJhEn(rXu(1-e9;(EeYXN{|?Wa}p3`FhGGwjh-XSctqXQJ=VeZ(z!X-H&% z>evRK(-)FACvupPg8>izYsFn%T14~X*@hhjv=GSjJw!t@r{ea7<%A4BS(`UEYX#0x z%Fi*hxV(xU^K`bkd&Ar*WkD0}snCC$z0bqLac(SUt@kfG348xYQ?#9&Y{ubGp~e6+ zOE&;v+>LJvZTjN!=l%E}?z(20A?}l@oQVH;a`7i(w0|fbNGP0uE z5kfLD@)^(E;WEa7LT-Q)2DFeAU@`XV>;kS^49C7E<$N`pSWzYA;=)B$pYA9aLr)1* zxVCS_i}@Vvmn)QOR#YjnzqABAieK&Ht`NkR_|xI;6zpmVzWxew&Bd>>H7N$SSnCax zXTf|H`a535MGb$>cKieI4**Na1|+ipg&8xpBc-8j>VG8AnLzPw&NFG0K`D)M$huv> z_YMp=e44enA>)zL(RdzTq^H9IFMzcq5UX;G*?^ya^PBhxs5lYo#LLb?e72KTQ+X2B zTo+;*+yUQ|b9qy_g7%gEaiM@li@KYF^qJiay#nMF0EpGLtqj{r-WpK(|K&+rfy5@d zhApi6*{P_D_CypiV*-?5~A5LS~w zGE`bcPh<<3i{T|_F`o8?!flafP&V@^A~Ko&NgKP4Eh_C_5gXDa7P=cxKM!;S#R$ve zRL&Iq;*rlON$wAytehH@mzC#zA2x4HB8FL9oI8D;8Xr0i``W>-%IWQwr$IYH@f$Zy zW`e_eBg3~T@``>LCcmTka=<+TA!nDP1$DJT=d*F+!5{cxJ66bC+Kulx6iapk<~1I!W&5(R=T2&kamS0Wb$~Ck zc|biG&yvVS^-9+DT!{h()HCrE04m2Rgd6hZnUw+0JJqK^Z&u42uddcDJbV_E5228C zBz-34|=gKt0> z=v8}%hxCc^u7hzxO=qR)SjhU;xVl5f#rDMEVOdc|HO4&o4@Tj>Dvn%qG4RPb# z1%Uw9*p38UqXA2jJOhE$*D07nsV1QGZ$_cVZ#DQMRC)JtI_tK63+{W6Ju+FacXG0Z z##53;fEA~DG=;sPKP#29_@nCY;q^^+8+52su{Bx(3~pZD&qd7ahRt7&mRe5{!3bRA zh<%Qlpkw8RGLZw#uwC6Q9=yqSt_aPMj@z)8POhDj9EW304 z(|fY&c)0od5`r$Ii)KiJulLpF0=F@$b?ew8D-yz^PyJ~Nn1%na|4JVp2;ToSlFRks zS5=7|FY`f%P&E2uq8QPt69u8JZF3I~W@1(0p+M&C5hD6gtClFrrkG^1ODUFZP(E5C z>(C+noaZC$)XpV1{oUP`RXl|hxN9m3jh+7;Z%T>atCApD$|zi>sgSz=QtPd^7_Joj z>EwuQ`;sg8JHwZo7UKvB!g6=KFlIT8xEd2_3t0A3`fK`xuLt5ICkN^Bp57rFQZtu| z3Z+;~vY#wP#(Q>NWZ4QNPG!juwn#j23t zBSTUOGK<*~Mj%4@-d`^0`<+i1yHB(L;oA$oI;5p0G1TpQF8@ZWM*mbCSZQw4IV^X_ zGyh3rKoLPxh_%5>dU|?ma2XQ&Rg=?_V7E+&kc&vHL*o)c-+1t9<_KWH7joXpEXEm6ZZwai=Q0s;TJMe(9Xl zwOX=jnH>Ko<2Ck5V34uYbA{xJcXUxnen$02qw4CUbc*yXKM83RU+g$)Z@EzrC!~+} z_h9IhUQHqhgRXeodgQQt&1`)M7yw`?7V81c^wN1=r8ER)sA8OK35ER2ZW!^J2o#8O zz?kw1&IR?{g@zmB>YA;WK8@dA4^*2m_5#U1w(|!d(^PSLFny-i^Hz{Qj5*qZWEGtH z$|U@;rXs(|PD{Q^pBYSA)MbTS3T>?UwWu&obV+MneA@iWR>wb|>ka{`cXn}6mS&xa zEN*soIkxO@8RGc*|9JY!uqeAOTtuX$Q@R`J2I=k)kdp51erf6Ml1@QDx?4ax1{}IW zq`S`cJJ&gXK}ML_&)RF3XH)Oq^gx_#S9M%o@@(|_{vIs_j#=! zjm@UIc;F-exUH?NZIr)1QZl%-l=k`r8ete?@trx^k2QYsgf4%q!1`UQK3XkA`DyuCQw&~o0vgD&z3VVk5E`lW{x*P3>7#M2d3Pmr3>v=+EVlWI%sNvf`V%B zfu_F}hoYjA5wJ*_(?=__vl=*k?;hMo%<5ZKNC9@`FJDd4R=k(G_p`A(B&_YoGIcV-p;XA`$uN^nPU0@$6d_CNgu}YIHES!=T z7Iw)9Mq63@O_OM46>3iWiYxLRGx5L@@{Kz&PHp;SS1vVAZPDKe1gFLyR2O#*A_Ph5 zg@?#>U2wqMRQZwYH~oAcm~#O(KW8nJz zap~gl6@Pf1M;UNdxjfkjWJ#fhS^vGLDh9YFbBfsb#vHsDSH zAmVGFP&@dE7^!hB4AnFLhW;)On=N?xe_DXrs$n+N;Q4vUA~nOnC-*%4uAH)bpD4Pt z_W)39C#GaB*Bkv7Sa#C z16s=6wmV3waI-jli$1Gi;IaEB=^4?2I1nE7SpywY@JpPeJ zpL_elz;&0!RMQgt;qkJp^{NTF)apvdq}`=BoPs8DSE1tkQg2TD=Z~3_6NZ)+CgL)y zw%D0K=JwrQUQy$>+_pA>;9hcoNtIWQk_Ft)RvSJsBA}Df`_2!K-fi$yJBj0H^^O)N zCev!Tj>CrM`9_H*;|snYk>s=*x--U*qbAkoQX6)NBgfdvYS#LX84nYS3uAyA?c~gQ zgOA>j>_8Zb8704BfR5pF&$}IduXfo$1Wy;*GqN=Bpw2Z>Zuxl)W5f%&;A6-Cokf2m z1|{>J{rFL&4Z#R>Xya3>AMYgvu>NtL?_;uAP-dA!Wo|An6J%+@6K&A$ zQGdCcohj%^nyCm_kOhqB!8&FXrCDH&BNmt&QQ~ExAgJC{wWRaC`CHP|bX{I|z5lKq zRFl7d9ag=bxJkjl@za8n8e1;l4U*^1wyk< zZWe%E!-*eO(@F)$rmfMo)#2HXBNLT1J-Z!no{3f|;Anw9=j<^GciDZ< z!0*pi{5l^R;AB3}4#h6UNeJYM6vToCU|RDVIjNpTe-e$t-BoR%`{+i0+!h!=`Mplb zU-qCcJ#->xa!DgNfiE|K5!PA1GC3oTp{v_KRJ1drKxV#FkFeO{{CX_?!^HYJ`P#$H z$$8g9`ZNSG90&#|a|vN^DS7W>fIk*As14voq)bFKdLGpVi5pu$KW8z01xZ#e^EMv5 z=qSmQf@hej>P37pp#I*m0GK>P5n2<1}FrwX5U9U4S{cw zZh4=lhJ8spvX9l|PJpbS5nW%pm{akGU>tpTFWxyCb7hq+n^LlyKp2K)SuLy>Y4VZ@ z$E>w^?sC|8MLM5=hY=o+dtp>nMz4Ykr`*hNmKcpRGb4WjkIDUg5hkM77TSZh_y9bgq+B*;f`py9u<)_) ze}rS-wU#_U9e{O^Q%J~f`PZdnQAr6Pr=!T^HVm0G2IVdbdAPuM+5|TJ zm-+2r8KH0+>!lWrs+JUG9^Zk^zPQY2@CG1@0kdPdx5|>IkU2>a7VJ3cR*AogI72ZF z<-C4kfMp`4thZ*d5PEKc|Gvcs_#4Y4BvjeOPpXS{=tKq&K`S8!R&yn`CjT%Er%uBB1y}L@S(`>2x^_ujpAJj=Km1akw zv4j+?pC@ZE%y4?#5%;eV2&Yu#NhnT1 zq=@6Ri04L=Nq^wi>c3e$mIq(?WXPdfk}`bk`(x&OpC>(L2b-|I#MG)SV7W7(I#H;t z8(=cm~k(|~eZ?MEcPhQYu0&k{unQ7HM-w9U~^yH9#ZslwVR zvFm;001+4}bdn^e{#fIb=f~3;F8P zeyr>sIQ-7f^!G5q?`nGFc8_Cm$Q3)`jbq?Gb_$iSoYAVolFq^h>_i6qoHM4-;MDOc zw|=F?1JuNCM3^-VwLg7nF$xn%Yz32?IEJgQ;YIx1nY4erS(?gSm&p$Qu1KVWYo{XX zQX^0rM!7r-)6B)#!KcF(%>71lqRS^!G=XYTaXLK2EEMk0`dxQP)#CepeJa-8u<%I# z`&bffQ|c(FgkAr;skiS=9@r$t!21%P%l%rXJlKy>#e_}ncVew}+4nd?r07lo#pQWStbF)M#p{L5Sk9lwyuSQas1@CH!PobO)xre8E2ZVu zRofkzr(rMTBs*ekZv*OxdYYYU%yqENj#F$syND;6j`rTe;x3(aktD)5TdRvh!8pz6 zGk-$87-rby(pQ|)W$LQP5o-Ql?3&m9zM43jWJ7}QXC&v-v3p}(E} z{)ejk8D^$w0_;qvtO!MyI)l@nYlPtRc{6VP_}9e|Aiy$^2)SRiWa*7~kcnmOzTpke z_|AtmQ#br5CZIFIcXx`RmHw;3>-@dRkYFNB2{Vd$Cw7V7$}~-=JjLiJ{dn$7{me~? z`EXNnhcY|F^U|Z_ac#z@7@Al{YuMwa6h>*4T;3-7ui_wqi z_+H@M0%4d^D-~_@?Y;yj)z=)EBKxMP7ZSm{2f-Am?vqHN$V5LKv`qj?zOvE;%Hz-s zaLlljBr}dkd#HK;{0`$>i+ij7j=yf2lt3;kQB7M7=oy>r@$FzJ*rJXsIWkRj$(?+K zbkg47#5i*1%Mh7%zyGNG5c%O#WK~Lv(7j-l&JW$$ALp^^pGuE>bjPdRT=%mKum5J< z`H7_-5#Vwb$9~dMa+y^A@tK7r{~lZ5W1DG{2@A`4L&s(YrFt2kloHge-)w9YmP+1AF*h^=FjeJIu1U*;R z)Z5)cevXz^v|qe>NA^e{(f!2WA)t|AOzPS#wNc`Sd{dc%C8dq%DV^aVfzUeyBM~9N zhh{3WCeB}n_;2=P)W=BC47Mj4{)L}!S@ubqo2t_mVKDg%ppyC9G_w$a_)s#2aFhcSOYNsSm4-q}|uMv_t|4IHn^O2%bU^bJ)+07ZET-M)Us zJWGugiDSX0m>fO?QtbJS53mpT%7YlC7iz?=5cY2e$Y$>xUn1=8|4dxsuJ#xHnI?JK z`eBbS)>KCl$$%m~+%a#+S z5}Ce`j0BH1zzErFd|-m75xfw9z%!FNG!smqPs>`x)nPXSJ^*>u1;hIq) zAS&Cx6ZWUZBFQx-G)=QA3nGso6qOP7PtoF)j^A(Zoe@#Ypd0FesK{$jH z!3?;*87|DMi3)wtlHL`|*^&2dGwI0=bLw0ZD7=~WOkg;T=KaGfvJ*Lf{EIJYZgDQ10L;aFs zYqheL7D$8v@f-lDnV%QiJmUif697#WZs@c1VMoGoS{jPe# zmzTK_TItS<<>s?q!I*uZ@Dm}dr$q3M8g8@#-s(3h6vpy(PDF^ zIvDM>N`h!K4Uu&_Y~=9fke~A_kO_(IUC&cqC%s8XXIrT{mH)YsDHdrY>?}t^l8UD` zN!$7oLl<8~Pd$=AD?>F;Y1+8v^1scOtnh9tP`s-)yN;k6EAsFv@poP3 z?QUaOZ#o9u4J-;qrTF#~qPs$w%hl`jjm5IO2PXSSx_#OOY|ER`*VJ1KI9e4kcmI%+ zRSm2HXBS;AiL&R??20ZAOdll@501!`iHZn5IfTI-3cwBBZBPs#jfQHrh$g}W+d1;C zj%b|y@J!%ygPAutdp0abe3izc%QY;c5gWwGn6HBPGdpPkqB^m+ zKU(PQotk1JjgJ^frb`|tHt3sevIm80gvhZYsNm@KF@SlD5-W0Qd1e%x-LddLrAkCg zEyk}!^5X$Ufkw%>iO^str5uTh7IBm1i%i=#f3wo(#JH zFd)l}w-xArH&$)bT!Vq~9><||2M-MG?jNhY@@k*vjll1f?s~-y0ci}WAh9Ry{o{@i z2Y6}>bsM9mrZ7k}O3yPV1Jd3eL4d$l{GV)365=j$b2r)GIvlqZ^*P;E!dnPc_y#Wa zCh$uWy3nr;qh}mllX-FbXaeouqUC#PhQ?)5j z0S0AA*;PDbIE_5POG|cleNb#<=!;pUO3X+XLnx;N_50RuVi*$&3o^Z&qsmzSMJfrt z&Bz##%WY9B6YG7bnxTrwpv`ht2qC5&ZC$W(C04~L-ofzRK)P%jVpsVgJ1kaJLDk~K zpe~P++vxD*Rq>sTMjB`O;=uGB4$>$ z9#~Sm)4G?p@wffN`1$dTPMVYz-vA(Y6#xSeONo=``&ZYbj^O6K-q3QZtM^72{y*7) zs|^8sfhr;r0M9|@P7CVm<}oQC+pVc#-JQ&jHzojf!P9~W5k-#7uSZV!!otFD)T&%w zkg`E&vVu*1-iU4CKgig)=ZV7;B}vagOH4VE_!0OV`3 zN0k`%umf;3z?pW|dqZ}Xw~%ExS+|;T%1BY+fsPP}(s86HxQGku>LP*4TwOyW#CkV! zun>_vS$TGOIT!;2`E8N)yF}FtIi=)(YDoawgaIPM{K5kG9`=uqrRFsOtPJ@6el|9x z2Cqb#-p{zbL$r2KhGn6+*S6XkPpjs?ofQUElw@5_M^y8P?9)1%y2?d-c-cM1yV{;8YjhC%IWDECIlB0_Z(ky z%#-mk;*^<9W%Hv*X`J%jo;G_=tuQXB<|R@g0elDfWDk3#zi>V18Ua=(EfSh6p#_%~ zTH}oP8f85J>EK5{U0ErTWJgEl+-dn6OqMrd(j)Beg+p=@Zetl!JSoVuDMw(h4n7Gi z+lT&utJHKV)}6qT2=w`Tzak<0D0BoN)< z#ScsHWam!#YvF_eZxc|P=1;2ZcP-4#tzX;$s&QwzK2r7|D=k!SVxOBux(88huIs0T zn&z9E6vCET-B2u4lanPmi#1mLOw?q7(io#Taexl_Ce>;w_3mA4xU|-NV=v9r7nNn0 zb|cA$`1*yegoOiX`+)2K;zVA=>U`v{;)3#L=8}P}v-K+<6|By#$w!1)3s^x?yhuql4=^Y`w!tbOAfanbG5kThBZ5Jayx9rn-<2a=SrBl;+DAyOGkWE4DB zfLP|W(vR}QraKdMmtf~k?y2Ft8D>StjUzjP1NgbcJq}a3$9$bB~J@)fS{?z$t znbWS}NTOxB#@g?21;_yaD;v&GK=<&2OY=e|y3AFoGdg#JFIP7=(Jil_T!|xfuzrhA zee-9qvlW$m1>(+(1T5QJ-&wASufn3mM$X(1go<3l*o}F@15%0^-2OUZ94#?=y4J|S z@>5$*720H$(ZcvOvMLxyNeU8tzjAIdzQgY<%xC4Wdh$3ps0B`{*I1O7H9K3s`P`z# z{P<%#IYPsuj&*7e2TiNUZ_U^3;7qH0VMJN8xeo>NQ zl@)Fe1#;wiH9oaE`)n^EAguM0DVah^orZ7`qmh+(`3zwIoif50C7&3@q88V}fqAknYuVcB0NsTpX1e0Y}Y;NXBCGVZMxiecv3+MLUlf`zJddXc)} z5_dAq^j~T4+FogUd&ES@8;ONuj|S&{_`ltOgauPaAV)n916BVh?ygPw)qt5%n0`woc6J)I3filfJ>0_Aj!@x1}k2Fk_ z%>aKqW2(u+^)V_s`j$(pm80WueRW~Mrzv=G#5gw+i=#9aK*K?fq=ab;zo*y$6PU>A z@;z;kC`|sOb{-oj@?fsQ`0m^vUjGv!>2HzCT9$f|El)GD=khNsH&z-vTE?G0{|EF7 zF+d`hFt%%DYx~b6dixXr#ye^qT(jL@3ST5Cxwv@FM=Q0Keb*jZE@u?(Ci9=+w6qS6 zj{z%@_L=Qx@va3J7+N>uO}SiBl9E%fG;!(jEl=x=X0Gcd6JlkYumAK+$7-NCVitbo z&S599gZqajq8m>4HL1qccJusH6^f-mE|{t;#KGAQ6~zogZl-G=mtQq3KqS`N)Zr1w z1jpIBqZmy~Bl2qKKm@CQ0$Lva38RJa$}uKJX1{XzV^3MnLdF2bsSHW*iHE!gjhGtq zZ)@>almlw2tGNBzFlcBY9KV>^Gg*jx_$yE$OD3B1)7Ge1p<_y2&C&aZ_~nLvg32N) zJ}-IV^m%S^ZM&8xYyj@g=ZU-);sikajEPiT5W#&wex|{}mEFc^?p*t612gkJ#M4-s}2j`mNv-4{^WR!wo93JH)8M8UNyFFQJJ| z8U^cXtTK?olkJgV)Krp6M3xc~Aam7tfE!HORI z=g0*8@MFOEJN?w0wJpBmoVA@Xvc0aGm3T@x6_Gk(8F`1UtB=5^;q|xG()$HLUW7Q z5^f3wjt^(WS-(9eVg}4`F%c|_b#UGu*JWg_-;D?hs>r?iqz2byjyM|TX9^*s>Xrs9#q+ChJ zo}HVVVY-ozim;@5EK4gNp%f7$a&pn1WWt4zI0*JnRhY%rUe#a_OZ} zO3}#*9S)9f-(|~&0jadS@E%I4#@$j?r92;TX=i6{SCV-$liz5OO9^PC;-CI{wyZdaCXAT_F%o!%TD2%?agNLZHdIgwCO_OTORHg9 z^oPg(fi~B!%QA&)rVH(20pW-|KTBh6ZT!ZD@umFu8aCzc-_V_XT_p-2WdiKsH1{!I zVnq=TYu@%ui~<97WpThqWOL#t0|v=Y49L1xZOW|a5?b>v6~TskYoF;&7GJ44lCl1|-`O+RaW!f8?r9;?Cx} z=$ouJrc(=h1y?!WdUUNHe9Ql+url|K%YequD%47a_&_$4aVVYXBdTu=GJHI;uZRY{ z=u9}#AsjD6F2G5LP1u(^#3Vz)1?KIs!7mbA9Uc8ieQDYHw3Yohd3XD9epFwPe!vCN z#(Lh{!9rFUVmYOFi=MuL1P{3JC6mPZV_|*S@ExGU9V*=1)C7j!_fg;XfUgutUFrN5T@@4*_*47~BZPT*d9jhhWGy+u5OekM ztaY~-Iq6}HT$!$-9S1+S;w^QNZW2L~;zKdpp3}ar>x$%ubydPuR>}@w(~+PU?1OgC zJy(=mlDjUM5$4hlkx)ELYcD|8~wI^nBG$GD($GXU3AM)JQF+(RI-< zV`|cPEbTb4xRBc%?;=X~;&ta${zJXWr(CylK6K=un|fH-#*m03_y$Xo0;bRuUc*uF zgv`?dF#%)b)hn#x)BV?&Eu4F+t`FW5CpW%%g0dbzrZ1M02@>g*SA#E}lCJd)43xCB zAz^^ud zI~VXLd7%5+oTG>;ukvrpmn>R1-LWAtm$A{Y+s3PLGyU6PUY`SEW$Y_CR0)@wzLrrn z&G*&C4!Yc0gw0{jApUjE)ZsN5MA)B+PA zbNB|+-y7mHf5#loJO5hE{~Dei?YcF5y7eX5WF1aeb4KN}0%ZF`Z2rptHU4!sFAdF69+#kr8**Qn2x zA6{c{d%4>XB=g^!T2v+*msDm371hiigrjn(?5*ghZoS}SznvC;n@TDFUV^iw5e2R! zp%)`_;Yqeat@R)vXE-1~R=Mn7>Wy10Bw zrc@@a`_Y1i`wGkL;XB!8(g8x!Q5l=bRZseKd5-%-1+#WROmzwLut{ANV>DM$hJ<%Z zf0g?b(l8A=QW85)$RC3^n{)f6qQwUu+y<`fhL%g>##C;e4z`8L9t`=2TZ@kt%dpek zUWK|Cn6>0=*iUY?|Ha#VIkdETS%B0jVb9_&s_*q)vG})|og1lZ1<4uZ63Y%x z0a_|9!y%4^FU7rc$>DY|_Y@a#MEROR9l?`|E4EVUW~<{4cEmJyw#uSX)QWAHr|;2y zT+)^@<}!|Qd)dns4ZJbb^u=($c{~^+yUrO|+iy%&KZ?1$kjA|{Mi~BX;0ePIQb$yh zxDQ#)pRIOhi@13onxa!gwP@-*nr0~W*wcl?lkD2U<;=Quc$T<1Yy9XrB=Z^N1#jqv zGXJliOa72d(OMN*Nna<-ZyQwbB$ga zrGQtetUi3d);Z&6FW1X5U#1+t-w2|+n!6HqyFU5(z50OOy)w4?j4D=vc+V9Nqu75j zt#E;-IZ;KvDj1&XKK1k)uKz0f4Qx<|!W(l*w~>WJxFfDX36>;BPX=|KeUnne?{t3d zif^arEBPepSdpC>)IaY~hEOCucoY^F-#4|(y=)>(xuT+eC_nISL1Gw*v5<=`wr(km zo8CI(-xQ1hPblTNheM5Zuqn2Erpow`Qp$UM$;WjO>Ac4`Ku&^(>MgRPF{oeciN^Vg zLR4P;gtXZ1N+3puW~rc;ux==;|IdH_Ji5`P)-=8!@h`)Ww~yQiPV|gqO(%f;vc$TU zdhX=LG|OTdujzXLw22O5SNsDC^^*Gv(!qEG?wgge;2i^c9DyCzv3dX{+&(>TiYzsO zS#O9frjS6VSA-X2c%wxCmyLW$)UZ6ny>EjpQxSfat5Z(n;O@H$jqTYCT{p;YpHx|+ z`cQZsq;j^TvNrh1GXWBBxzyH)GVAi?9qb^^K&f)Gfi~usdnE0!eXj--Ys9%$Jai*+ zL^xX7aT_G(RLK!yD7RY%J+08Om2;`sOM!J~Ie+-qSR>$cQ$kfM$6R?NE z!uk9A1MxSmwtmpsiH!Wn=}#d3$OVcX4L!XG>)oj5E)eIl-xhQv&{AXa`DDo)YR=TLo&!UaC3L3EGh2j$hjio zVdLWa3GPF7)DR%?2YFT~_^f6(e^>rxNwgg@nt=Z%OInE3a&ID+S+^}E{N5)t#f+2i z9{{tJVmS8FMcys$y>XGU;{v%pX=oc-uZl46flro%6`+vVe)FyL1eR&;eu1@ul0RYS z*P^UxxJ$_7Ya!T5jgnD9yrRWLIPToN2VBqjK8f+*6VlhSxtWOHMH46$#7nFS!wrW@ zy-^xa<-J9E9z2cVBsABa)Zob;C>Hile_(tr!O1FMU&t>Nuvcp!mwxmjmf=sgko5mo zKQa^JFBzfIy1K!s^O|cdiRU|2KQBFIUlvuqhr@rF*DQJ?!B@&JL;b1F%Wkrn<^1`VINC1$=#`(lX5@v|;oP{;5$bKSv| z->?N-sINhKa5*==Y_7Sk)J^7@{I4*pJtf|iUp+lA;>ZG|{weJzUD={+vXpt`ILMof zg9RZ7y4}XFcXoD&1>6vETI4Y!C0oq5z7g{;%d4oo7w_2|RQS1De|dSSYP2S*t4j!N zb{yWjP?BZIdrRK}8`e+pd>St@2AJF!bKOU1_cPN!ZxZmH&2m{gSZ_E!?~bLUrkXC{ z89hGk@i_rO-H~ILqQaO^ip$+YtzTIF)1&1->G%gA$JF^@#ZIP?!?#&g~T8XShh-2JE?wgMdYK zgnTQ!$G~BnI9SuD_Nvb_7aJ8K?U$IAdF3VP1Fe5%eu!?rWHV|$caMCyoPjC%Ao@6N zf4gA!^?udJcs=pNG@d3QTupU9UjPn~BwPTCtEQUjx4^Ic&%_mDaxIPP`mCSbG4EWX z7?`MihGlc5M&}*~rBoIV^tP;Dgp4q<;-)S>WO+>MpU{!3#>FPfV^vp|o@c!sSXldy ze@bOdBR=U!IDE?Ohw4ua-;Z~`K6r0OK`j_5`03%P)@p^6xO>+!a$8M_y@$Pb|JX@> z;!-j-HaZ3c>;Tobx3~Xaq|ve$6F4q>aUnHFe-_Vp zBA{^za+criqotzp)U=Hzo4f9m0Qm#}AhdsvG59bIea@u*aq(bq`}A?>bP5AVGYtei zXBIqw3kt{p=>f)gU38547R?33fx$X~U#r3Lg!Dd+>^*<1Zz*{RHOMUXxw84&_23|K zc81Vuu_6#~)Ep2^m+u$sCIFyUU@bxdJgDR2ec9b+en6z}-_G>kzdzyqGU_>boNzo_ z4^ckSH| zdeweemzsY@S-sDh-o$*0ifv3HvZa1;SsZfdtyRb&Z2JNS3w5*7SCit&qvB0t`b20@ z?c`ih3>)@2hWcIJNu4Uj)Otn0V4A?K7q(wBN;tFQOu-zL9ipRuvKTo5wKBGrCdLk5 z2tKnk1I8lX_V)}%hD+10MY&y#>azLy(-6b2E?X8he|;04^ZgK9onR;w)Uzc6 z*Y-EmOx)i!vIIyAB%9G?OS!I?XqHpc@7x`{>Hkt#wrUq9%78D3d4p(&^uYL&>6fj* zCo3Cn`V$oTuv}$t?qUso>12_H`GGN;K2K#oo>}d@;~V3+xFm&vX)Y3OrK`I=1lXN6 zI1~1FJgZVWZAYSr@cI71ik1lC$Ywp*#U`vXf)UY3@)X74{2Vd-LXv}(jWM0Kmk+Kp z(ujW)-|?nPA@Y}47Lp=Uxjl7&&FwTEj3f7f_h@xm+NPGXs}J0@yT`lre;zl<*dGsh z>>$9+iWw=*@3jtdwD7H{x;jRdD@r{oEzM#mi2-~*fo2zJEFcdq&5BE&4`&yO{y`Do zkLJ@Hd0vmsfRFK9b;nOP3gB1y(23i<_Mx|nK2LUC0Ig}u2`oW5AL9*-oZQ z%g1URIbTab;K2H`+K@eO3sDD;Q{denGUp4H=F`?me^S7nz`>fi@&msC8jtI&4Hr0R z?5VWinjEgbryjardSUNM?du#B*x5xo@{8^i(vP! zXt(Y3Tx{fInZZ@XTP^}*%(1euk;i9ck}!7_a~6%s8E`8>xFSz{qU)a8!31;fw|cQD zOgp(6--1Ke9NIGm-&TI3r1bm~7Z&m>@%S%g$F0k+QO0G4VtK>41gz9?Ro8L&!m64f zxw1~nE!{o^Tn|FO4Qusb@O<`oXCi9cjYNa_XI+QZ!T2%B=&xA=m;y55lw38iLBq9m z@J)qXKp@ImH&mSB-Qf|TLaCKiIS3Cr+lcoI1=ddiK>=CXVL-4Bxf_6xGhkc;reGLp z8EN|3^3TV=8@af7fj&*$hFHMsKVZ>&4Cqe(bd0iX`WiOhm^^eHq_A$RNJOaC^rhqb zGIk)44EK(s=;0QQ9B5XyOd1#%RO{3wr7~%|Z2wF&IlQn7=}`bN$6tr8-lvZh9QgzU zYMuFwh~GCA@&!>`Fw8wY`LDvgrjs0dzOQ{ohSs%gT?=}j_9A%i9$4N!0LQW_-$2yP z*j>nnTdIHcy~!T>QPJ^(o__td_UojYb@}{32_<-RGM9a23E6zHO$8BQh(|>Wb9h2h zT^CLkd%K05d>Cx-ZAD>lk=ssDmrdUuEE~0u;C?*^-Iv1>T`7Li##f$mJ*%N~dG6;J z$`VPmGWx~k&Q(*>-}$<^E3w4TvTa6NrK)KQl2DWBBVpQGG}R_(>*C7h^)A`IIF}-_ z!BSA4@f!7UH;@LgPzY4$?;bUd>+dq1eurJ!@AWM37g^begF0~l#kBn|>!>V?HzLOmz zq((wwy5WkQ7{W4o*L^|(H(HNq(3Eg^{(8Ks=owe(-%)sz zqK>$;+i8;dm0F0KF*7hIf98{-Q1b>`X=Xo{N?uqVV~}uq6HU_?x^#|RxS}V%18X#D zXb1XkN(P-MLzrzljdm{mDLk3DB27?fhLi@Qa+7f;YK^)qb5vl5 zSFq%*1d&DOSKs6#U7X(*nJoGx#nW>pU5`A)Q^2S0vzt}PrM19LPAmt zFRO&lY?5pQ+sDmo7X9~ci`C^;z4u5et17L8uUSDREy&gZ+PaprAtFLjQig?B5&g#c zJ4xeapg{nY)YsRSeTFF%WC(a6PkL`ItoX;w+b;a8(jaZAcX5#b=w-kwY)0}qKA$TA zCz1>sL2-R0GK0CAVpR*Bw~3W5UY$J1O>eUZP74^=c=56K`xyl56{s4K7BwS%&@bQ z9TCJG7^cBXO+le!n^Mv2O3|O`Wdj3&W|0K$>6ME`O)*l=Sqt?@ElfSy9KxIKZq0ME z=8=L>g*iHBi)Rrjo21bt4m5=I#d0e0Ji>yMCl7*Km5bMQ zYUNj*-D=+p)ax6KFcZoGWMuaEsJM0xzwiZwOX2lu+Hxs`lzG`>Q;p{8D`g)VlO5A^ zFQYKbYC98V4{!@r1dKR`Pm2C0yn$w(Ibh^nL*W1ZO&C(N#&0AiMT8}7F)0Q9k(JyG z>ujVTj#RS4)E{#3#4!$mAz3T{S$qj`d)fXV8y}~57sP#xkq6rzUT8h`1mK_jyANx< z??}cW&v`qY+4x9Npn7JFlNCE`dz4kSq7Fiqrxy%mpXzIIX1{ zc07Ua8s|_V4E&5>SZ|hT`*sp-9JgFxSdVh7G1@h@{`Z8rI*(e1ryNdk3;s~iOn#H} z_hrv{@$9F+wu;gIO5DGuNwYu%tJ~Gxp|q?I<1Z0KFX!mR5eUNKKBTLTqjhz&3){kD zP(*VM!tn5jvpZBbgR+xbXAn}$zY0;Xw4|H1BCPso5Vpl9#YED7T5yzAhMYI1W~kPC z+`JNmq5ART&_u2vArTRkR?II(E~T5FR;$5#Qy}Qeus$?ix;`-y}KL70N}g2Tj1@E#0Y-5eSX=nHF}TheH7!`;{jC|;m~4ZOAt+&HTMOK*Lzb>nU>&%41y zr>U)ljrwM>-4b1JHGn9;)(wGRdU>W0xnajie2@tP@$zFCIy$s7onrnoZd;fXGU(O~M}WBYXz4bfH*v3_y1U#KxphmIQ0 zhkP{oeWpo%G0?0Zy4VLVp!LDmNN0)!zb49{sP4Keg^heo*|4hi& zii*hTX|8rd=vrLTO9P7MXlM~;GRWCd-u*B~z#3-H?4nPbDU%R?qLEM4S4dL@@ zhxgpV!oTh1K0jhr=_Ns!ubmxgKU;Zqb#WmjU~Wr`ibA%AX{!!$h5zG zidAQgJL2HtllAo#jK*UrEGS5{V+}EF|IyT?0CInU2kn2NgsjiQDI;b&?H(-i^V~vL zbIH~E-ngB^zH$@wb>d>Uf_Tw&7RR6JGnpMbcStU{Z6P!Me+YnL{g)BjTGfTUc2qUMv#cX#@6$9r9DyIL>DhMR*AXYkFd7;`kg!jh(VRNIXWaDeYczI>9N6w1|qTl znc(Wok~}?*fkL?G$j7=vTcvPGBTo-t3Q|sge>fX%{YiXnx}QNBSGwgOcLwKWl3w*qt3a3 zG-;-1Sq+}j_Z)RbuteExVt+A-_RTYLD{+sBsU~258aGHY8&kLo)dY;NMN+;BP&pUG z##ql{t$Ty3$(1^LZG*62P5Gwen;wqF4_ltK=uwM^J{nfA@sw4 zc!+HG_ha~Nww%EV;g5N*PPET$cbNO;DY7739Lqe4Hhc!}?!q=NYUbFJGECYV9I9~I zj4>i5zYK4c70aiu8qu3Qr~KX$H!!OR)#Q)5XZp8Yv8dLw;DMG1g9+o^TfM)7c8*0? zBQjI3dlgb1#YG_*&AT?+N}dT`4@yHGC!*nfdpYw3KwSdmG1b(XJ{zx1Rw@%hh+?R2 zrat#A4cHO3aBM&30V^68y8{EAbMw?WGAwGWe78XiAH$&fXyH!c)*@eA^xDY>rm!7Y zNBOeZ!ESd0;89H;GiO&qVd!vs{LJ>{7nH*~izn#zcastTTC|P)xof+h-vM0!FcTUR zG`Qrb?z;x{$WUU!sE?S{P4K6Wfu|h|hHxQyR)zLZ;7JFJHV!f_#5{`@ER(e2LKgk! zf?#$tpd&`ZPtPokq@^j4g<}a^mH)%{N!R->U3oRL@2k|Yy8$~bn}EMfsHZ3ZBWt2t7Z8?Vv)M@Q|Jw^d^a z+9nf@E9R?ar_e)a{4d!ym_B0V@#fDZe$*enmW69o)`s=luS9L$QFN1a zwx#`y;x4{EzK*+KmRrkG2Hm9{G_Psz#4Ze@kp6O$??hOn1xJ;DJi9c~wtXOL6jAc= zvKei+=ri(8-b(xtz5ei^`*MSWhMnyml)t1gd+Hi>4Nz12@R@F-kCfPI{U zQt;HxRcOD0@RW-?kX$My?#Dwf#DsjrYNr8t3*KvyZbxw+2QiGpZFkUTppgiGIto;Y zF&+Oc*oBo^gP{~SWu0|{*75%>E}sgO3l%5IpXA-RJ#15^-(J$iUF8HC-WR?aZ5=#B zxN}w_jh2uwKJ>f7JtXy>jJN14Q|0GKT@Z&Y!T8m-Fl^PvuSs*NFOMG;|6eWuJg2?E zWeN2gmc36};~BC^RJ^8YMyECbPgrrqIB{6|n7J_#^u<`z=I|&H@6=x9_&;efBp}-9 z!xH$Z>QB;}c9fvu+UJ$NMh-*IB{?+knZxjms$!rDjrBDQ>wu}fhPV(qkQG?S(PUAS zojj|JQnfaX_bkXvGcsB)`n|_`e+d4a46w?Zm9Kfx)y+fLZXY!ki>MGyvu4)x@eeMH z-=ec!E^DOJ8mK&7@lwgFjdq2S|F?f$^?j2HP$`rC!GP`Li^uTlQ^_)-(kG!5&S8yG zL1Ks!=eB%IEWG5IUAk6W%d~-_Os~lWa2X_#j(;Uea?<))MVKdfi`;YgK^b6v9iQ2i z9S#fm8?gGY@tJKiRW61D-(LJ|r(O-fCd?WV2%x~o8k%y|bFR1E_1}a{zwN((>FSYG zLS&~E<8VnKHmfvUX45i zoY$X?q0s-xwe#OfS9+5^K=M9Jo6IM$!OT2tThIQZ^W9yWoj#OCpOBguh0TRIZmu+* z+<=~sPZ9~6vInCmr?Oj!Qx(}`5Er8PpHW_$a8QprPl!#*SoyHUeV8sXrqSzI-@&)k zau?yhKTvHOy@D5ToE{~2cEGnq!t5zV6?(PmP z?(XhTtXOe(C=SKl-5m-PcXxMpxbywVO$Z?zNC-JIv){eedKSo7=6Ep$-|gnvQ)OJT zIcw_!tB>0Qhu^jzN0E|hnQ#VJ@O`-BhI|W}ZbwevQBFS|(HQ!c^`@pjh-+0ZoFRN-gUI1np-?YjNRmhHbP2;2$uHq6%7*c`n7M=XL&E? zVTVLLP=|3cgOJc(tLehE!kdVef+s$10mIw{=A5$$Sd7N%X>9F^U-o(wUS;UhvwN}( zcHt!{9fOUof#a@Q`qOqve6{!H=0VBT=Dg$N3W|C5MevhKucV?M{NU3f-)J@>A2M-M zEe6i6o-funkxzq&i3*a`#=nEES0CG5G9)NGJZj3Z0h1^NR+f!>ydE?nsZJb4wHSKp zR4mk)y^^dlfpB9md$}5~^dQjKh%w&#Zf1XPBUkwg60|^Ztn4(yWa4+Z+MtAm)^SN5 zOM;To=O0D0H6wDo zY@JOqf(;Tff0R@ZX$f95uboPUncuNT-br0k5#N2*?odZjjIwcW6V>HEA{_;vYlr~R z>XZfd=5p(NclxC6+%w{Duzb~DFZOZ${D2XgscGU$hNH&gu=l1!$7FL$A5aLnq$cZU zpV#LRw;H`KkU(+dW55H)(Id6c3AmsT3UF0vdUMQ`JiGll9WcoUoolCb@v1{D)!mE` zG?HP-@Jf_K2xCYwNBZ(}=m@f`PqIPuFwpueoEe*4JS8tC{JC7;a=FpOq;J2u#WPT~ zL@wZnWp0-V%qhv`y>LkJN#BHhNZ9nhH11n(giHT&)s6$1GL+dX_KqLRZc8mjXhAOizs%O;HOd__~ozsk3`XrdW!e}>jA2|}S+BURHy zM!dt}Z!MuPJOz~YMG@Lys_=QKztI-X3`Z)tE$uvcKz=oes`&(^3bDuXv|-+@ek1$O zSu&%}sbD6P1q^7cd?C>LrufCRvgr_-vlYn@?lrN^UmedTbUn{}t*OCw6<|0talixv z%7jwt#G{9yLML*_$z7OSGXH{gsya?jx9oXVl~&A~N7X_SuKr9rUw{{?TsbP(=C{Ct zz#%_S&j2Xu0Jv98L&F3I3h31JQa^zM0(ig?VQ;YnK(EZW`3B-F*#TuuG@8m98d5-z z5>16id7Kdg@X&L})B9bvGXcN$?9-i}6JxHOoSF)l!fF|qy}2>qP@{(aOQcj*GSv8+ zq9H)h)|Ao|izq^3S8bhdp=rV zu8^`x+=-pJ6Q&mc57dO4QuQd(k;2bT+kIc8w?PY)*V3l?+d7^3U>DcBw&XD?M=UD~H;72l9mWqwfC))%dO>M=}nAm&a<15D7nR7+Zk&6j@cThwYewsoP% z;#@u^U>7b9!;pt+XJv9f{m;UEf+P71rMl8&v&0|T)#_IOHIe8mxD4%o;sjfI`i02m z9hciP3a0HMs)Kg#@nv?ntVFiL@Cuy4K|>|=*QkG=8v#(bvH}#9-^?ig^!C1a@qrm5zL`;bmRMF7RMO}}h*SW*-)&Ov?i7F@TjWDitb&2G+2UcFyl-M&ou z2XtJug#~>2{TI38?!IAu-}L|Vz3llC*Yg`X5V#pc_c>($!p{Bb$M)JbMV_9W?SH)J z$>KQm37s^|+SdeWJ}jU^+iuW&^=x0-i6KqBmPPHyrS5q_(<+nA(Bx!U=aq9f2P>42 zXMcH6$F<@k^D22vgbKmV8q;fLv)5MDBx9mSrADJ0i%| z;GwBG)Ohw%zvvs44w$0uynt<=2VpG#4cws$PM^IfcYs2QWZ8p2y7~x)wtWP>r4Se= zF7Zn(>>1k{=@Y2YW(_ri!Tl`KF14f6rnAzuAIJkW-7?Juh|Zf`gk8Q955&^iRG8fB zau(Y>>3f1Q7qhn=m}1*HTzMHfHV(_-#}_7*6?6P0*zKurC!%r@dH51|V#lB{ z3Gl3KzBx{`@rKu@4Ivm3XJi!6s>{64UbDP&ZiE9z}Nns%%$(+^nV2Lcf zx(Wd2->;#CjjTM=0KDC>ifJ_E@N%6gbSiU4)@6ze7A%IUF3)X za7K}s^v`q5z7fp+FOC2hFAiub6L>r>0FIH?8S5TjAM4&%UGGm{pOg-3p7emYz1IBF zwe{xvd1UxS34drfe2Yo{pFI@2`&#nzW9ush04HsDYz_Pe^#clwdI1&ZxuWO&+osZw5L0JcS?cECRkaT0TVzjELFG3(hb^o@if%{ppdB=IyPg#tQ}UA6qA9VVBU zU1)51d`j8t%))NG(0|R9#w*qx2pyTts#WQTfBb&K@$fZk!lI9i=cem~k7V9BbN zYQ!No2cFwMT6pY#UXr8u5z@;e9riimC->n?Ty(M-H~SGq>~*pv_WgvA8nTcf)qhPO zuHtOK?Dn??+;lLBX@`%4F0v!h)Ha{{4E7=i<8LN?IiT1fbV}M@pRu~4w6c5`$bK0Q zxB&^=Q8X+DEfGLH*UjCG1UWrBT>wrCuOkkA}ZFVmp(YhQuF2pQUy4iDzoJ;3O-V}!DHV`CyM zEmeDKz1(oJ+Q<+Ea1S`-kLsj}i-5Jc(EAQDFaiLnFzGpt-^_-6mmZrt07}akkT`)7 z7YLxX%BL{JGRG^_iK73%hRESe?*A5R4O0RW0M~K3)>x2UgJHy-pt@%V3V>C5?kD8B zK3^LX`91626ADj4Uv_5#SL62fcJgfhRQfw#tTHIY(2Dt8_zq<%pDoKcudPdSKoJ4G(<*hOz~ww%IKUH)?ynXM^Qhpw&ljqu5R5 zRi4CPsV&OT*=N|vuP|2*MaF47{Hg+D5A`?dEdS-B^qB_0{mGvH=0}m6X}R!9p6`_T zRz@ubvkqVbF)OTIx51;kZ`f8mkV$-nm@2mu#h?Py-k$#a8b%flA2j#;GKd3FR9DuY zK*Oaab+V5#a^u_?{~a%M%d~1{!j?(~jYa*TSDA*1J$wG^tQBG${Fw|0CnJz>Iva4} z9I`;d00Yj7M68qg^qf)pepW;$rP84mcEW+?PMp-8Hhz-7f3s*CQ=e0KF|t44Be@s; zWDlkNRuv6(B^fSwolfq>6Xfl)46X6C2AU#-!c|A~C! zKwdwoZXajKpo3_R&C6?zYZxuUM5*TbYJ1zLuFG7zHTJ*yQm==U)EPw6#Uep)ob~`~ z3)m1bAm;}lO^0EbO?#%oE|#Bt{Rq~EkJw=#fZ}O@+CBit`DN$`AwWb${wWYd<)>K1 z*6ZSb4QT5>=H%aR{9jCwTABe_9<^G#CHCu)I(P3znt1PeXF!#YOO>i8MD4#OFKVJ! z^>`*!a0W?yb&^L#BWBr8oQNMtLpzP%uSYM^Eh8 z=dBVCiQQGEnOB&A+PF*oE}|Pr{j`9%wC4_y?OhX-lzHNS-a(Tb8{ktk#VX{y{R?AI6Mi=n)VLXknE@1u3BU~%D+i^ zLlrj6+~ewcalqkdzgXhljpIinyCFJUFw z^CW8<^6qEo^LFjnBhJUZcanDM*8)Ooccem)wTrb(YAp*8zg~21HC! z6e3&rjwoY%$s2FZiH}dXLO+b;5|lw9I4xpmLP5q-t3+u30%urQ2&!l2_E`QdES(03 z>XY3xCWT4fr}^bTu+0~{3K1)?vQhT=&Fs^v>)>zwre!=639NUSa@u50RKSwMUO}r2wWW zDw6u%=Q!M-Hit+Il`pjAVhMQv z0mAr83N-P4xq8LQWB!QiFuh7H%68&Q4`(69W?gy+Kya3!fC&*$o)=*WPfk>$i#X$< z`P0*rOP|UtO)Kdpj~L+@2;}@19hZM4Cj!&`cXmMR=hqdw2K5OQ)%N18K4U|RySt@l z1_+2J=f~U3U#B&I5<0&)fBnMj6JWn)uE5dh5KPg^Xcz1m|-wzFurA3d*1kI?!QY z$pxhbYNJj|FdCC$?2Yj3JHN5LXz;Apm|}li!xFodk@dkj z+Hg6N@HZL^Tnrvxbs6n2g&TKP@=D=0r+*z}86N`Es0&w4%&JO{6frVpSDaTq(zvc) zmBd3wjF@=Ae9l0qKHF_YYBl2LV(z$dO491HH{lFZR~+4A4lt6Sz*U!>(mDQzs3gZa z%_TFwAU|(lApes>^|jY8izFoeB_Qmc=vE9aev?9-0*z`Hie-*=@TNuEXNsyiTHVy0 z0MhHx67zse%shF^58dJnzcBXVmrjr>=K!^|u}0gz5j(o%+$K6)K59~>L%gkTx>kzO zgCPFN((XQ5fx(70EOKHNeAphBY1uzW;w1W|A#5ac19wQ%DrnXuS%MyaJ;usGV!+w- zJlR$lLeh*W|1j!;adS8PQ>$wJ**-Og-;>DD-<+-Avxgtg^%6P zRQ9gICIWz%uOs-5#^Sal^q_>xPyTT7Zf-L_Mx<2DwF=_ezw^yuc!umu*&M% zqrin^ZEITqIAJ^ufwhLo?+O(-d7H+@f2KSAl7QOlT3Jz13PuR*HpH09jh4>hT``J6#GiSb(|&2iy?)0UP<(b<@`b@FYCu z-vkOF=DrTwEOQP6@T|jafy&1AEcxI4<7zs0vu?)1D;W{3k$LgSOxi= zS(drJp{XrhgV`SIqqhh2H1za)z;L<)7$WRiFZnbo_4?4`gkzKO?vAF&fetdWQ8XZ= z36P0NlSZA6vOUmXAR=w-@u5FFRx z)L(MCMUMw_RDn)YF$_5@daa;sR(G)Wx;B7~yPW9nagRP%YF-XgU1Wxwk--=^8epsX zmht{N?i)N7RQPjNZP26yhFYKQx?UX!*C=%dn>2SqPJzPxm9@cM+>S2aEv!<3%iWS@ zNo(m>hTH0OHqm4#dF1|dbh5MXurdso|4Vdcb{tZrmYhodV}rbxT6B&hf95!Z{JTm% z1NvJNw)IQzNBEgT#x;X_Y)}u`@MMjDNS&#^RSBtwpL|yBI}KT`NmlMW3j%n03v0bS ze$UpFEfDga{0FoeSlj!i%5d}pPPPA#1i;Z2uoYz2)j@n^Rm5z>%>fW|9tOj_zvB`3!nr<{L?LNKKeoq>xssh$T85tQ`&33reDnQ2A#pufq zVW9Wlft9_zpt&&oqadVxAwaqXViVG_1Ua06njxke02kQs{U!IN@9mO0{rgQ2v5&An zg5jT@nm7O(c2^|)LDsTfpHSm>K>r`=9B9bMeQ)_nozNBP1(pxPfIk>GB0aw!eFnc> zexaPGMG}8KO9Sy3zn5|DbZ$qY{~mvFK-Pk0J7&qya_Vx!sy%e-K4H|N_#P>P?Zr+1 z8dr$zghMD7nU?ZIEdvmz?bkL(o8&?INfmd-uf5nQt5qPtLG8ww%Hdyqe%4hl9o;1; zl)yrOOJcT~Ow=&!&(RZ%Jj|+Yn)bjaC^<^a8MjBK2V;g09N_qNR&L%hNXcpGlU9t!AtNG&!Jh?C2N=Wi%|=#?>8;Lgw{EsE zXKnd5)&+^H>h(~BBS-v;(wjiSAb+Wjx>%JV9gW{}eu|+j@nCkxZ!@Ugb~^&ezBm4p z?JO@QHnq9;RxKJ$#0*J zQ<=q(#hvDwqm}csLYX{LXUow))t#L{m>>#>Gp#U{&Z=oZZqt;)9XvOwtNXZQ$6a+T zh%+ULM&9YwW7sVo3A4gxIDVhDw~*Qx3wKu4T(AnCHvWRLV{SXxXlF8fs>Tt1B^D0j~?7vDOkp!OZ$J2_T@6#eAy+B#4iX31yNPYWS z@xCMJsD7Kga-G!Q{A_=Fh#i(=pa4Yy0Y11Bk_RR8%|@Acfq2(gl*R5pv)%n%3U!5{ zN#e3G3e}uJx$|^9O?k4Z<(WYu?PeM|D#^%X8D~(u!EVm*BK$69NB=8_qy4zIO>u4* zlrHd%UhD-QpYg7vEbl47BcG|Ox3>=hV7IJHM*=gC&~{ff1}qnV3#8|zL3oHIMu#gA zM~?RU-2VJNg>)PdWdB6CmOPPOl?JcNGfRSu+3-tYGSD#39v!E+&7(eLx_XNaXKHs-K*Q7ui)aW34d!Dhsz6~&nTFaB6QWGgM z0L9*qu}DpeR!ORgj-qTvQOiDp8_|caU@a z?Dqm7l1e+j%rdAo*plIfGI|0B-8NQtDfI@`=*(Tb_EbNJ&NPfkAwtBQXBc|u zlq3?lRyCq?a|`t<$x_KsrtL^Y-ZFaUIe1|oQ<_~DTmO#S(z&nXG>kZOQ6f$1H-J}i zb{@g{`F4>UKw^wjIWz_CwQ_zD2Jnu^Rw&$Zf|`$gQ@INHle4}4yG*d!#s&hkq7^CBW6t5h}nxV)z(5Ld=deIq9sZG#EAf9F*Q z$wrCOAt2hOl5^y#=Qu5pt&@7qv*h9A1HDuOSjDaMv0cvf;^5VB_B5%EeM+-?A4icf z`;A1o?@0>|i6{2&- zg8obEd9{rzuMWDC$}=E555Io@3X$*Q>?{yN^Yzu&QqTT1g;jk7Rn^+=uWpp z9qRoD*TwP5Tw$yz-@s`(S}$NGvauDCr9hi)?1*??ROZ%j{}CM7)L!GNtE)TUlq5re zcM5R6-An`ox0fa+=VoVTaa_;vQGiqm`>7JMWwll`7=5!1}MC zpg<3wU_Z0xgI|Iib@OegL#<)~H=)pc;^f`EVz*uV)nqg$1mzfHmDq~X31gu$$PJoY zAj~f`F;cx~&DON%XYk%3Zgr$6nS{t>ZZ{m3m?suC7!8C|HH~Nc#&(!w1nEFe=z|l{ zQI&TBvj<(tO8%zf*+(mq@4%VeAK0oFUj!4_q@y7}gC%L4`Mrju=BYQd4Lu>Rk2Mya z%;XGaN%t9a3C{(7NrSfK*|dfx4REN$-PVD!FkuUETV%y(;o+%!P*HfTw>u) zprM5y0SGju-!)hYoi}8EDro(6<=Zl~^w1**Nv?!WYeRR-KC06pkq=1nxRxWb$7ffg zs^CtE{HC3$hzLHQmHpeWievNSJ-b( zYpU1cYG^t(GgCy~g0JdE_bm%~UFsgUKo4o>2HOfwm8p%C(?vn{xV0cl)oF2y1h2F1&q&EH>&SA%(MESll z>H988!T>D-QEaldgiw82cQVwbzN*jax%@Z?xBt;3ih7PirzeF-kSSdB*JOUN z-@{%2aMi&;}Tw~SfOGOQi17vkiT*?sAi$)G-}SjtIOtnbDzi-aw9j9eT>@Hj3U1+hkoDy|N@ z3QIQTWQ`!YS<|U3@tripd+HCMrr(WC7;L9dVa7atcyCeSBy?Zi#$=1ZpmqO34q-1@ z1qaVsr4I%8>>k;;eaTIDs3fbLKQjbD>Kl7}Cu{=dLXfC1^n{QgMX^uY7MHsPr z|L9z<5hx8NG@pk5XwCV1?7iuJIJ_QlHy3<Ha}~& zXn~($;Wbj*yRITvCLNpoQ-j7*Lljtx<@L{#zd*i;I3n>$gt!lkELBAGGZ2e`+1(~| z{~fUjKN=EZD@^6wP8Cs_Wikx6Gp*U^!@ou2Ju$9h*@d46Gl46ogf2Nwy!`H@meVic z(z{G3RaE6L;6^{=}u|z1tXa1>`TK@=$H)Z+6Y-cjWMGC0her~m20FGb8h@$ zx#zV$(oaHijaf7bH@b+UG2eP-UiehUVnukk-jzV1Fl8#ZQLjU-s$&Zr^*f%<`|dh% z6Q55;!ZRuo&)Pk6mShk$%G9m}keA%>XZ1)W_8Que>yEsD4Duci@G6PZ!=1cv$&nZP zs=B{$*ZJ$&eRHM8{M;fI9OUiGdmu&^e4^l1r0*63#q*DUzpJ)RJv_u78G?H~z*o7N z1bhw(8 zaA<<$UcPp|l^%cLtM0qW2N%R3nC%Ph_fGrik`~d?&l8?~H)z{J&BH;z$ z`PLT>F!zp*%8nhdO$j~`0Aj}d!$Y0@T9ClodV|2*c>{rfXE;zVsZyo}#BO~)UyfE< z{DlEn99g=IQnlK0lP&guxcGB7z5p9 z<`t!heu|Hj$z{Kb{JHJF4fr%N)yB-3?)7^)4wpAW#&R?{Pxtbw6X>gEKpX9C@>ujJ zuE7xA9zk2MTI2#jAgyv1-(*g493zln8{;C{P75cwgBFe8c+{I->>5Vi_L$m4NZO9( zy`sPnm(vJ#z~1h;gy@M6INgHD4k$@`n|#&eRu4jsEplp?ZX;booz?Tz8^Ciw!EnXXf2#{j$((0nRh82*YsZoN>e46fwi%1=0q3)2j1YKebw^~9)2DW^i zK-XHk$GenZAja-w1RKHWQ}Uh8WpI9(>p(ct+n6+InL*b>XItSvAZ#Td;86` zTg6I;ju}j=`9V^_nbu6Pl0xCZ=szU`tH~aW(n^V4vzv4-qcpkMC2=j}fCT#eMJOPn z%&1&iGZr6;b>NQ{-OVyk)z$5jURTHZ-}dnNnMcQ#%Wk9!%G)v4TRNDLewtle980Ot#3jPp?Sm!;R_uS&a8}xSzUYe0Ns9ukLx=lq z!9RSSI=-0s-OvBCP3X1%hyb7;s;cZ~+;-c5{)M22zug~lC-87Tx7<9D>DUR{Tk}3v z<7Dp?p9^#~0u%wcwtwwzXLymT?V|NXlyf8(8WduATKoGE=m1VVX>TMJcw=!1FjIWY zfkV{b!So4#)%fLpry4j>EqQkQ>iEiWJ(^@^nHIL}_s8a$-GoHhA!|#35S+Hs48le; zFti2}pKEI&Bn+wxpe?i}dAOAnM~ZO#1(Kr-&BEd-eUG-yO_B8k#$lXu$EwJ-3xm`M z)A>eM3^_2k9WFGL%ygZ#EFKMG6Jge8E|3%Fuq$gTSi@t9t4eA2`>%&V&~(PGRfNne z)k^Z9Dp6Pvx|sy}TQuSEYR^Kji1@$6drSS#p19DL{znA^Kf7@^*n2T}6tE)je^$wd zgdL@^2zv_h^X`{!@>jxILcpm4+}tT75kCO2#(u1RdZI zwf*^Xu^#B?0)p{jNQ7ibqrukHh~VO5QjKo_Ki1`C%v`AZ2kcriZOS;}S$+BFpC!j- z5vJU|WohXX?GCIhH)s78Qg0@#D1Z!OoPjkGM@S9>mSxjK>RsKls0s!tYE>lUbxM&c z^qB_KEThtdoV6S4J(k??HV6b&lX$pO3jpG#H-ey{4_z z{70=yPsInC)A0ir;B)Xnr9;vGn1Lg}ld#rqA_pR9JP zUQQWypFvS%hnpG?B5ZMKf{c8p>~zQ^cizOKglAH`y(b&iBgi{eBQP64Y~{E9#)>)` z_yO5?l;r0`Z=nnQlG}}yQSb(S?26PV(7_PhN*RR;gj1h#n z0Suz!g1FFtn&53q#~#4hWT({=(EEf-NW3#jowLvUxy#HV&;A{h1~HK2PeVo2XxS*P z-*9O1bN3YR#yq~fEY_)--CTqU78(Tq_&{{gQ;cZz`+x$*UBJ530Jq#U`zIN=ADQ4` z<=`98kCe?OJD3R3e|aaq+ZdwLgur1GGuG)AELSO$Stq4g*Gd&@Wu6cINE(C!TVx+te*r1_8R&XA#Bh+2i4nC{Am~6h(QU9!B*wS zvf#{Xc(vbl6IaTO$UA-iRSX6$NOG0SGQ6lj%|)HyEUHzOJ}6!>viCYN!k8+Z(9Je79jChpIcI8LC4Iz&6r(+DEfq z5Cvu#cjs}U2|?@ODTNKq{7MunL{{48Ao`EWI{Uy37g=zXU+7jpznL=>SxuT9pPv@C zJi!31*Pj)d+AR)*H=bFeqqOZE9jt}>Ma({+Yt6iV=QtZ18$hW>qP0Cx?FskdL+(^_ ze5w+~?7|-5ODuGU4 zZ<+uH2NM^Q{KaaeF#%TAzUC@jR(bHH2NrpB^|#Bx2=vaoyWG8YCLqQKm;y`dNP!AP z4*7ajRY2Zrc39~@KK>gm(q93qB3HgZmqTnozV!s$_e|<~eN58VRmnS0P#`j7IwZ>( zWnC>ZtOUfy3(ZWM(k(XRSg=TiG$c~KMJJj5t#nzTNx+62jsZgx<;G>pkD>m7UA*}h z1(cWduQTf)jX4+?2OfBXluFH7a1HaP4Vo=X?uaTht8;BI>=#IxN!|mFAVdoGjMMVZR)HBexN}^7RcJ zS(&sPWZIl1&YPLre`p}KI_+WDx%u&fF}I=lrr0CaR+KB5k9kCWngl_V(s0PlM@1i& zE`*K>3UI|$DqN@4;a6dfEO_m+E&qa?m(zZ&ZwyTX9SNgbmiq1ASOIQG>-Y1WP8mHo zZBLaPV$4~_ekF8cb$H%eSR5M8IK(tt1&-cgWv~#4fQbzhms`bcZNw-IUyKJy>^;|lI7~EJA=Dy(e zo)3Q|o*#{f7pOwDTt275;$#TETje^}CcVBYcrgkAQ}T`53Hla1s4{7|G@0JT)zSNk zs~KAr5Z(Vz#p!;i^6#IX=KODI3m6-_uWly!uTbJfZUo|hQbtppKoXdsp=#ZhSb(e% zIYc%wF_9(gpJO@_i+XZ_Ncs1d{TjQEO$Z8(6ppOkA{gYA&pv9b-~2d04pu;c@VRsa zJUi4m+a3N7np(vn$!KfM$2~3~mbqS`+58^P_g2is*1KK?ds%bfUz?>8&V*s|ncS*$ zsm~|6!jyBXa5}0?saLW0wNVg77^rK-fPVr2hTPa1e(~|UUs3?FC&29Rdg0zu-Cugb z*g!U4ZZW5}owR22W$h%9Y&sY73N&_0uK+st4z`B`%XvKcM64nL0wf_25#kTv_Oh_B zV0QtNH+Xvr;!k!eN~S1Qg-BSJ{j|bhF*M5Wd;91pDD!_Ee#M9e!m#E`gp<_Cp3&fk z&nP6fDxj30OK0_=Yo1J_MvAGu3cx(9VXXL@>jWbMvY_a#6f%bYTK#VzQzb#FBis@;W&BGA%4di zO!96CHIs2E$v<#`wM0%CTA2&saBj{Bezxe+mv0NczH=M22x}z5$<##h|MDciK^J{^ z5%G4Q%I_8Yc&2t}EpcV8v-p`k!gct7Vc;sLF>&657axyLAAc@8+sHJuJ<8Uf$o^Vm zb>IJ5=UU;2<8+BMQdN4uqtG~{+?}6TW0K~v2h-(Kx;6Sz$h)ZLZ*$m4QC#v`U08Yw>c7+ zehiUjd)Jp+O1L2~Ia;7vXo7u8X#C3WOgH(P`G0ADR?ZZAEj{`axTC4uy$I@@!m_rA zC|_T_R;PgPRIYOjTo3bNIY$W-7Wn&Ww32{X^a>CSIXF3YyogLq_bAY0fm>>b3Fvq| zNC08Pb<5V55P&)bLl$~R2{^i$ktSy4<&?}*0;w)}1~h*5{Z~2Ktnwgr_{T=yH}|aa zfduj}z~NRg2Bt@>_&MeH1^acM`@#NrzB17Bx+i?7GI8{?+8z_Qug`s7(=I*AkCZCV z;+(s^P$BfqPyFW7m!iVKT{7BODEu(0DVHEuf+EJk`Mb!~GPH(*h}^Y3f_@i{GTZW( zEL9Ahm|WvD3}QakU%AEI4X*yut=cqY&Zw$#MqzsjU2F1!j$^8b!GmZT)?pmYK^CD5 z_ryZCKpI$vT@KT^B1%|<-okKMq>$op`Y{aJJsW7vLo3llv1tSe2v0m85)hv%7BW;g zIrhu8eMpSPVd?FEUN;a4I^PH_&@_lVX$1F$9D;#9@J9~=c#SH;cCPrk=>20!ql8ejeB~^hyr!@2p zozMiMwOo-4xqlJucZhB4+%`IKFbs43AuFrjzln8%Fxj!SA7JW%s!%k(1S-C0}JW~_)3U({Usq?k zC`O33Pd}8iU1;x9X))lJda|gKILuq2($4BN;ll9(>xG}hLVjhH+@z3@Id=I0tQw%k z9MJ2I^oHw2YzwW~I0Yoy;>Ga}*O|=_-OyGW+!=wcxq_CK)PR5hDB?PXiUh8D2TV?c zf`kkOIo^RduF9Mob&Q>#k$LTqDlh>@?KqrY2Bc!+^(t`V^>|)hhy#Vyzu^M^C_8+p z(=L$AbAK}}Y&s0VHY7{U-mPlcBm?B2%Bc#}2l-z=`M3sH!X<A0{_SfztLa zWL()u;_9PE`~MV_2L)eXdAj0Wf|cuAKde0|^sTP7O^OwkKCI;ASi9cAc(;-;9er6c zF8c!tFRM$#aHm%+xaE!6(6^y)PZrSm{WW z7`cr>5Dz%;D5sR&LaC#>>=N*uoQ~!-@RQQQRpt-YURl?=B}u@OF&(mpBr(b;CFxv! zG&pHdN9KeUwC6QZ9(#xXMsf!q$ptUR_QK6|Nq?0GLhlA{ zX;Igi$(S6uI2Qg&hLm)?%p52V4iBCc2*%TpFY~+O<<;Ak{GcVv=A!(zqb7ek*>l&F zuY(50`Goj`mc&8CIrgHmqfRa5aEciUA(_M3^>WFpDX%+#eR?a-8mF}yUa6sWmZYfJ zeu`(9V*+=$=VWWi-<2PR)I!FwX;EvgGC{|-Fc}foHM@7Qr^esIG@X0#Vweq(p|1KN zp`cy?Sp^ZTM@QN5?%tbk%lYR);#T|U<=iSGNhwW+ds8iUL(aq%g;xssciGg&*n*M| zU6^8G!aj<~kf>ccQ$6$GxLTPg3OO2ri-p1|A%h9-RtMAHOBDSRdpiD;It1{eMAk_G(9}b?dT1D21cJ-O0~ z0BPF)_X51RAEhhkIeNTbOlr3T=J2{^$WR2PIbM%phK8VQo@ll0KXIXm{VIrXh`U){ zMLd5EvoM$=X0^izx3L{nshA&agdpQYtC;?NlHuONRrL)aKezS$AzqWar;gi?4#x2h ztd@w1t==Cwc{txB(@Tx{B|JylA$GCrdX8srOb#&bjcUS3C8Yd(pE(QEn%}JaObKZyJ?$BK)j#zx)0*xgGQj(?!%)C5Y2e5ybry?9k+I7# z2n$ga2%yk%jV2e{p)9h`v=E- z#8rFRp{6&4bVG*W_YnN13b+6=KJ_6*$#fUa(9hs~N6GCHa?tKTM|-XpZfr2A@{BZI zVd9Ggh$=U6IR0EqPs#)t(!uh(3w@?a1puHbI#goq%?{1wXtsop>zA)M>u45XW2>!F zacuonHC`b6k{PljuIQ~kw>d>UJ(w*O1VTUIDs~NNd<{$7TV+g{{LS6s}Zvq$nOnl#D4ejR-?YFdmD(Qe`QYqt$vY$vs@9+A&rvaYT$6}q@< zcf7uD|L&jtxrTq{L+mjoPS z+F2$}_mv-yXwn0ARH7?}uJdyo^!=jNLK;tK4vR_TQjPy?5F63(?6(Bs^P6!@NqKoTE{_8})>$WrW(WlFO%t5~Nl?$(47ficp^p`D9Ji ziXwvrD`$KMc|gD^#Xo8z=C$Lz8033gIi9q?m{hO`geDvr#iOcL2=y1<(_(?$;01X| zM~l0ju%r-WlN5AHQDpK9zam66usyfYi|g#ao~arumTC7t2kAYk}%*PUE0>8_o9Z# zfOKeC8LhySef+vykr>e5l{&u5g9ON5K5!BTDe0DO5CrM&ZfQZfyCtQ&ySp3S;(qV``JTVSKw6Vu> zp!dt-X>KpZ_|rNv>NY|7Ajxt?_F~g5J5TWvD#+%8v;{p$KMZo2iu_q--p$$#%%y6X zU(X#x$Q3r6J^ zbNEy%uJZ?UTaIdpRr~|;HxV5R8;mRs=c)pAK2fV`@6luPIcN}aDCj4?Mh8UmKtAtD z{$!ASzF?sA`reVVs_T-Y$$@Hb@-y^Qv{Mf^r7@=-0V>1m*$tRQ?;RXiNd7t)i#v#p zf7fNUxjm(+T&QAZVY5?Mkh%Tx?D7%;jKSm3X((DxybNTxZC8oorQui9UVBh3ZC8=F zL`1#pvI4iOKPV|FffE|&ba(^pV5F|iE)>Dd{V6)?yxA6aE__Z)A<6!RD$~9FebS}L zPox>`c12%XzQ3oT$Wsai91ALRvy!{tb=bV<9XC&ITXijwsDFCS>Bym{e%8XWTFx*0 zji_s)iKqH#kS=Dxl4zEHV`r*J)sB7}{Ds_q^+E)_{9ZC?oCI>X@zhtJdlo8CQ)Im1 z>+g~13?C*J_<5__wV>?vGP!lcHG(1!F}vzdMf~11v?Xm&g?t13ggrdhF#|f}I(VN3 z|00Uy!6bRNk0dqE@OOWS-*$7LTdw3z{O1=rVokUTIW`n{PvAzY{#hz50S1o&`uETL zU}b2NG$50t$^L6AbHdN>Q4&o|sFH!uIY+Fq9go?sBPe5?{{*wCWQgyZ_1|>E`!WsD zp9q*0GDuyhaa0467t6ut^l%dtwezQ`7BD%bo2G85K^5PFnf$UuYsR@@63tkBKClUr zP-xTq7WAz3n+fnJRmE(pnJ&p|4F-Si)Hu2~(;ii)p}ide3RS zDEav_Vu1du50Vz*)sIEIKb_}v(5_QIufrJ}3Cwn~T(p8W`E}|dFvj}dZy|msBH4%L zhR{(9!S-AINNzJs@}pPUn^NY8`PfY7gLh&RH00?s^k)J0DftuXTM^je@D7v!b_zim zvKmcnVNG9lioe0={ShEKV#X#Gd>Q*jLf#thif#BIGJ05>RAX0nF{lH|56TK|z9|*g zMcSjy)`v;V<(KwQWE$e9#91$%#T%3h_LEJ>+MpVb-y<wU%`h!Cf6_+1tNqp;@2?as9-ejq;E?dph02wK81$bK$eC> z_9t-bO!YFj_kAx-Q;py`v#MQ9Js^*b*q*YJRhCw6nTXeV3kRPU%o<#yxIV;UPX>cD zs6KYHoN&6Y*RD2$`?!o1`*P46Sdc*Z?fP8g=XqiALa21DH-eeEM_2d7r$BkKbSQ77 zY9r0vCf8_5_5mQs=XOw1H>wGkzazHym*&n#%i&+}x|rRz;@dU=gW$6GDC;Yu!{(`e zV`Jm(p{bw*J64_w?OBh!XJ-O~mif1DB83oL%pcv(p$p{FL9fAM1w;a%=-InBQ4pmx zI#HD{J^cmrGVZSH+JF{4Xl|kR0}aCyJr4sU(Ae;L<`YJZk}JR^LP1rkbUOI*$~-Pq zS^qc3h8ZfBiy9Ul5i_z?L58*-3=OC+YY$!zb-7A~?j~K33~V3MsOI3d{bO@&D1LQ+qR39{CTJZ9 zY!yk7LmS7#R(b zj5pBbo+p0mc2WFHa{9h7g8a)r@`c0#oVdFzhk!&flpKZTV{J`)sQtAn-*7(qX$SR8 z(~$Lj(pC;LeLZN^6e>C9xz^OeTJ7pEB3)0Yu=f7O+-WM14D=yyM~5aciesR!LUQyl zHOKN<9}yF;#(BY~iiIqY2*IXEE|Mk$$9(8pVQX!LI_1cfy)AKyi6LsNSD&^*(f;ro zhQfZ3u+}iDue(1IOYLtkas>5(48**0#EMH7S29e zS<#^~IIFxyI!~DuOiu)ULI_f%garVmd3v}8I(r}xm>Sr}93OKaU80=k5VZ_o19ep3 zk?prp|7sk!bmEzKv{0AB@pu|f6vc#jyU+B(iX!;PY}0&(v3}+%N&-wcHZU-LaR5=a ziHR=nH?RPs1LI%2V>)0u1Q&;Z96cDQ7J^d)*jea!-hFPqS!N0n&wkvt@r->nyLdc( zlN$%Tl*!Ra@WKI4aAzbPu_#Rms8%UWE=U{RY z%0_6x_&o82#e*8H1F5wPYikR1rtVrjWD0OY!Ks zR(K6d^EP~pzvp;Es~A;l(EJq}g{(GB5e#g>A!G~}2}?!2v2Ga@ zp#Vyice(+Y#1#x>Uy1ZOzDSOL_vK)9Sf*a(lVW8_%J$jDt}&7Z9g?@QJ##)e;3`mu zC15R`E8IY~7KyX)^$Q>9dw2}ry_r=Hb@s{`QbZd3#iyaOHu7|+Aa9t+yjsTv=K-^V zKPg!e;<0K1Px6ad6Hpp(H?qAEE<~<>sm@x5> zLNCYKdYUc?sWT4}bc0U`4ud1@poZo-_$}ue{U-|Kv*HZwzlF$9ist9}+c*5n5671x zkF{K^_x{A%e`eSV`1*s1fj~M0C!illMVFoct5cf9!P#iI5rGH#gXyx~n;%GBt2sFm zcefjn0Pa|urGi!~`?_Xk!xW&;3>@FK8MK0W4eCojgG1_q56~n^53*~(5MjVi%jsdbfcXhy3k#jJ~i&cE!}lc3J`2AkmQ-y!vn zb7Ej~f!TN3IpO!K92nMSt$ zNg-<~@-8gO#7XAPSb|FDb&J1$^=#Lm)dMYpR3u`gYo%99JXacc&cdZ{kFKDCSIbKG zdR+cI6gJ>O173N96uL!-QVMn1+i>O>W}G#bvfIaLlYW~ryIb8}(KnqTVQX0&^UP9d z?qrv0jr|Pcpv(p~=8iAQ3Yg?T zg@$Bmy(PuKzyMhX>aC-bv%v!AUy*1d(he{#1E#%5><(vrXe!U4=a1LOM4rf{Rm|Gt8a+9^b9$;dnE&t|V=w$1aSx_6$Q>d>34)?9Tz z@0NSdNvffL`fk5l-74xbl)gYDIX;%-L!s`3i!cpM>GlJ)J$WgSZ9eM#Tc$V !H~ z{ayR(R9DU@=Ih>+7fVF@}k^(qdkDUC|{sPrXROKn;?EG+fE z-$rMw+PPD;;@eW-XrMw)OB?nwtTQN5mTiuDix;m$mdklq!M8QY=}C2X?aoktK+k?p z7px882eG0Ymru*?6sVXfD_TCjM0IecBo zu;A1qGhzEb92kL2D!kC8Y&X1cpy@1-X$sU(UcF2`PTW_>;-!}o1xDV8Hx}+i=6fl~ zBU`0ViZjsPmRR2^oy2Wle3Z^(O)%yNvi;GnS(O)SHDV=_@%!{&JE5G!ef=+3(_3hN zwY*h}x?r>GAq?dPZBTc7 zE`jt#4&y<*fJ}g*g(Udb-htH-DZzmuS5MtIfnh+7UU<~XX!8<}?!na_mXte>f!Twl z+v7$?TB8CJIx!t}d+!{xmQfI_O4%oq2Jg_x=+7wvS6wDqJIn6mFOccUaoZi>O@oA> zQrHbGKNwE!!3d)f1_MVgx{iMevBXE>Vk_eHPgT6ax(SA|lX~X!Y#IFmB^=bG{gtl1 z-n}uYT*TLHebg8;WBN;9tpNjnV)TkLKBBt)vC3B5x>ZF6o*;@4Mss4^nUCx#Ez@5p zRWs5UhrPN)g^PH_ia$zUYa(y`Nws#BqNzBNlr3^Ss2N>C5|*@PfNyL4GHL|>{7qkY^vjC$6nfL$O74iNRW)&#YgoHEekF8kV1YegH6kjSKM;OH3l z{Ok$-A}Z>;Y2P3&ox^523{HtpFH?U&yEZGDbngo@k}u#oZVe`Oon=i2=f=s%0UF&n z#oIFN9N<8!OeLRfF;);55C8%rq zKqUpvC%#CjsS9=en7r-kT=%{c`<2E8hRZNR|KdSnQ8WEA)WRPtUj0iK$&C47GpIF4 z)^G3gtyb)c|1IRm&FuG%X2Cx4ZNHzwqICPeQgP>(xKNgXjLQ+uVe~B&}{)aiWzZ{M^I7 zOKem2+$R8^`)!_NyPwu7jUk)n?bmCEaG2Szq1SM;lGyTt65M%=7c8Px|^`t^};NXF{L-D@()?<5wotXk;+aLq*!OQICpCz zNe|NWf7MBtqr!^lp=PbDfrmxk9(us<-s>a^c4GFURJPwju^;*Z5l8Tc(KI!oYd>_@ zeTdU>YcSmJo{2lH;fO)Te_dCTP&CuMKf=ZF&ZwNLXYs3if~1P(}x@ zeI(qn-PG zjMkdH`2l`m%RX5;&qp68{G8t=KdW@OpU`y`642>-Mn_SVq`~UUZ7Q{VCG>RDEskr@OU8jG?8)$k!!_lZy#pkOhihQyotRJ#mSi|$@BZ_s!o&3 z*@)dBBh-!M#L!6;m2<_=#^GyAN%-eq@cafly1GeBd&zRmJ37Zl1CyGD7D=NaGrm8u zbG>2HCae>Y}7gkl?vFnN{HSWZY+)5B#D z=JbIWU8PSZ9lq`nf2Vn}!DdOAcd13l7~vR;~KhS?$|QX^f8>1=lt$1BkVQ)!%36Su(l5n8!q zA=+Ca+x9&RiuZ=)HwK6}>t?LpkZPO$D9d2wBxd(NjN85uRev!49YN50eO;(USqts7yC}T)c$Jj=1YxjBn z{%ec6g$usxS+TV{0JOMvebqE}xe({4`NM-TJ2JnI-Y@Dg92so3ZPAcfJ&o$G&1 zQn_aJ3s?EMcT;E|>k1zvwqG}Kj+?)SBKv00En^=dddVRcb3r))OlF6jc8L|pg)?DZ zzFM5soYA60U|Cv~pBncuf6iP>J4adthssWZ`HY_dIS3lkr>7?M>g?0)Y5l~oZqWA{to)`p4M7nz(Fg#hnZl9 zZN7Lgf7^|yeLLcHrF)zCa(YwE&UL}%^`Pr^x0U$2tgQFs0%MwtkNXNTKTc*RPBD}= z?oS9*qPa@DRE{VyU4y#G3PjsKVRTPa0n0y`kXDEvIf3X&g!d6S&Tj*9IyZHbqyu;( zoAD-?7ArqnR1vY5%JSk?$kqwrAieX3GswF5PpAtldZ_TdehRh(7xy-{1Rg~w3zee; zsUCXSWN2u}+dmH=+>1tpuz6ugJh&r<^mac%w~yJ}>PXpJpv*`3?{Q<^s(4hQ9M1m4 zV)PCd=Y-TioiMo6f>K>f^U~txToa-hJGAVEc<2|_oAp$YYRXdXy0C3rInd*wttHEm zp8vT_V>_A-fzDaU{Pwm*5B*a-67;leRVxI&R;R7XW94x8B*M(!4RVYVZk$>aSZ~^Q z%pjjiN*5cIVjllSG^*l6;B5sHg=y8kfxznA*(2v?b1NsMw5Y%&BeIDe6~f{lvA(*!7OOS*>1I|e6ws1Vbkk@6z zv@r18ctwWGj#}Qn!6IMJF)9tYo&?IQM9xXzVLnG15-;XTsF7&GB~D*n_y(YVs+3iE z=rN2%hV`bni$201e3Gwzj~<~Abtvqoi%8Q@;q`R)ouVX$Ug6QYXUDtKj@+-_U}I}r z0Bb=py1RX^I(n%_(QIamtRv59iht$km$awMU^i75ByaLVIx$H!PSBN{6R_{hj1BK-}`*Zj{`0c<(7rbVv;meZ37*+j0e|+2bT+`uaX$wc#8(0Y9Aj zGYejX7-rZ~Xs}zye}%!~Q{4G^k8llsiSl#mIvt`z!Qzf+7k6IJ@twB#AZ`m*gwXq$ z=jA_kMUu5cE{-;dIB3*GMCaM>{4QQw(}A* z#7J{vsr5JL6&ZGvQcz;KGkR^~6hE%cSU&GxBtapdbj$HR0Vc@V@I3BLQs9m|06 zBKr}WF;gKw3XQvOSavwMx*$nJxJShP?RATDWrGc;z$S4%R7m4&Fn#DYRBpJ7m1EfF-87`sv+)1{H-Gp?1AP zsgoFXUg;O6JKJB;5NzYf_@stSTjYQNl1t?kEObg%fgO9(>Tpq^>j|tQq4>Bk=0nSc z*5l#wz{?@QUOPsM*o^o014b=swO8}H@AhT_V>GpW$fao9?aKLL6?@9+;)#$0;VDD; zZmzBs6x1cT(VgFs5nxK?W&J@u^kxt!ywRf))q1-;9{{bP;S`3@dOMQg4Cw*O9#Dx4R7 zNAbU0fCO)mT<2z3E;T{iau0|4jGR4_Wxi2vlDhbZ){enE@4Bbz6U@x)qt)eumWdw& zLJQw*h`8U*U2kUxm0b*vXrbVE5Cz9U6Iw2eL_|W@n?upB;k9KNs7mrH!vP1=N_wws zg0Z3MtB0pdNp2?&vy=oriNgS?o12>gq*XoU-RbQqb`JsF{kg_cGKeK+#s|g|-WC_l z1pkqpCK|HG8?E(#^)9@bh9L3DnWIop0=G&iMbVH+E;Lk z)7*~RnfR{FChgQbVQ}Q(Zvx7^ab^Oe2_`>3>%-8)Zd8s*0t=&?1-oaCX zeI`7ZYLLkkNC!m(XiO-*&8EV(m7p4Z)Do;YwR({Ga3Pm&bGLTMq%y3e-^1kyAFr$Tw%U znrBy0&1)EX$FD~4?ULH1gb}mLmQDSwYrNmM7Ur-s z0su+vHySbApr*3{lX9_?at23@PT=bsjO`llcsOYnG&QAm1)=~kbvS5Wg4@+M2#AO@ zc${Ibq{0#su;Nx9F1Ck6C!Db26JtLBi80=jsMy#GceiUbIB)Y5%y;-b`9rB-!V`P8 z${j`Ub`3?NT#p1y`;g!uA@_;@liZjQ_4E{IcS7NCCyc&C4LERQZzPhK3wcT>bzu45vjE=UA*;y`=M2F;(KhV414sADO z$;ahSkop>4>OH>ykq34hymnx~<-oY5LHUvk$JGSa=RZ z_tDT#h|Duj*mzC(2UuBc zdY8Lnej*BosWC?`rntLz^tYDYu}ja@js;Zrxh*Z}EQaFB zy1J41=6!0G^KLu74h{|=u-v|Z1dka_M8R$wDFU_IkfkRAKQQ0ochSIHl_{DCD$-jo zrgysan8P|MDxcbkel-@Qgkm4;?V9b>!q-)}-!#41UJ2v$WH`8|32QzVNHEUp#DGvy z1rs%v3*w4#lLm)V(?Adl49v&2xsf32yseM{Gt|^Y`&i?gNJdgnBnqnUUxnH)iX3?& zeDlqzLt?VWPHqZ9zMImP5-3L!k|mS3HzmmZacLuO*g^zCx(UYmIB$!JOwtr?um~d6 zT2Kk3Vxo+3wO!a8b*27rMd(mRNcqV-NpoUR5p~G=xiDoJ zM~KfG9P?@&npl2Ks|C5IB`q~RO@Hz%s+mlgZl^2QiSAYdZP`R&YboJ_q93v5lHukt zY7@Pe-|oBO9rMfhmt9RsPa1DGlMo<7?48m_4^l5{rY6XmcZ}x0BP1T6EaT|Ndz~6t z@RTfbWRAGt5*gvOApe33##p1hFEQJtnryIn}v-DJgi_V{l*|rLfh)+8Q{Wz4_U?tOFJ&5U({= z(a^9@RLjp^57YWSbs{mt{)shB6T+#+=@p7xT>N@yMC5MYQSrf#n4=7|z2yd?T}jvS zYu@|Z&uf~TsUDkDc#{hQy>Usq=T#)b90aVnU6Fl%}-bl zHz!ndbaZ54qVn)MPB%+3shsb;h}$!knm*b+a;F|QMOG^2M3E;SE$C$B=fCY@?i(0b zU*@n#V%8@u$_8UEXwD~|kul>+wU%yC3e??WWJHxVCpsRP)DX{qMc_K_-{Ir1nIOf` zU(83S0fEJ>X9g{5dg^XwM;+H^H{jYe${wDbO*(C}*cTL|L@u^{RJXhi6l*M(J%~1_ z76Yt!50{<@spQh?r7JF&BE$zP!37vhY!6|g5Jl+YG+(ZVN{W>4wLXBeEW4mC5j)c> z1F#c~Up;)$&$tbHB6k+NF0DMTC%o1>{ShtQPK7DulcinMsI+uFlaG&&!&t8QpBWe! zjF$r5ZEkJn@8WTM!@OB^XYy>zY;5GIwOJ4cEg(CB%Bm`$qfJlCw9~aA0I&K+hAJ&1BeUu5!!B)YR_!P{XGMgIc)V`so)s+EwELeG~ zSK7SFa~3a`yk6o?#uU8HmQ<*4M$qYm1iku8_g7)~f`qDyb2S$nz;Cqv#Typqi;AC> z#;clhbxmz!qY`u9Ovd+=ij5v1NBPP?0{K919F6i^^_>8wm;o58C*Zdy&oR5=WZ|@G zjf{y|Q%@KgWNPy>$9we)@T(NP4D8~2S=a&n=-tNQ@=pE7wYpX@W*p+^QJ@owZDwh< zniWWxZ}bU1K>w_#R2v}D0)~zhD9Fe~3T~ywRF(CM4dk{Ko~aRH2~3iJ?s9ZwWn>!o zx{-6pBHMNkc>daVnqlJc=n%V#h*ot3!V;tOA z8EkHr*pWg)LZ!^$0lNbBr#=oYnOZ9x&Ca0HtD=I8Oq%~z#5 zHW!Eg+s)Q1vGkwvMm6^@lrPa~V9)l`%X2 zQy&nI%+G5`KK~mEUR>|IyRiImN8!^!3@jZDPcoK(o^q|_oCtNCtf=T4Y$lz5I2({L z(!u#aruD@B_)vq;{Sh9S;PxjFMb=rW@NHiFnlGCvEIR~~K(8UfO^{z(oA3c_qG0j4 zayyKZmK$VUuYGg5Oo6{6p63|zygaPdq;AXGt*xHUM~lG;lI8jv;@8J>q9AEosB7sw zQno25w*`Z=+Vo$mgCee-Uui0znOgf!W7hs9v9RqyxJ9)+%D=$(R;t(Ozp-h3=E*|h zp<6rR2I}hN8}Vw?AXFVUyLkYoZ+1@ha*G|g<3HaOy#m22taIgHN3N-e3MRz>J`#q0t9dH90lKqQ4FegkNEhfn^X7!24ZVv^}L(S9PD^UTzgwWjca< zo7Q`hG8_>4&eGkOE|yZ(U_4g}jP3?C&TB4vT*w}`K3IQ$c<3JLIGkK!m zquV+(5uw1p`z!q4e@cuGlkDQ+;uLgsM~)`)9d8hE`N`46smkzyQ~I+ssJcksy~76i1r=U6 ztKIsi46oylI<9MOz@Q%N>g>9*NNrvh^x#hoFV7FJ;&ZGblpPet^UN~NZ8wbXO@~jr zV|6!1vwXo@p3VO$1A4tl{O-uu!wm$7HKlAy_i%dI*~rVIg{EeTa+4N5rhc*p0of25uuWoO4C_}GEs3eE!{Bwzye!oOs;#1qe*5gc9< zc88rI!;y@oLi*lJ?C~4~YjQGr`on6k#kPhfuNZQ)P%+$(_T(w~V{>y-?0ClD=rd~^ z26P}Wy!fNBDO!cSI0$Hv_e6a73`}dQsxW{o?c?1=|4_gFeD$jXqrm+x86bmHn8g9n zSoQ{zH8BOi3i1?&#GsP-1=58nuK;~iB$Eo;F;awcMD=o8G$6tPwhXD?uj1l=+Dv$O zjDwO##{R?bTei2SBqk#nvE!kvLo8=*B8r&z-`y2eC<#9eq zcS*y=0Zl5y(2#n<-&q#J!Zvkz(&%4RId3!)aCI~XdWncc6O2PbLx7MK<~zFevX^@s z>JJ}d5_Mb&MY%wTZ~EP!AlmgeR6QZqF0>leCm{g~oc_oOLE<@KxNHFhnVu=x>js_|OfMqf z6WYQ3JVdN-jTmkq}<>y zf^+LVEpWOL|C}J#hgok}1gz;p^>F}XK{m+Bz)lB~o8n29}%T|QA2~6(8_LA~Dd?P(zL_ePn^V%4ueCbKUxJql+PCp_?$1^t03GOv? zba9zjSO7d)!|ouoPo#)aL!?O*9^oH^zgz0T%~chfgQ*$ zoxlbdbK=kGnLsqOwzlr9wdx7O?fQEkD)p50K}_*Avj9d<+0+RBFfedZ1^{~Ucq%N| zJKoz4*KPt4pr|-(d>rlZEYmY^>7-Q@2w#bh?Th7h8w~yS6r`ZY1IJ)C1v?KfZ{NT$ zB@rxOY=dglsgBTw z9%};-oXNo_9v;%mW$;OH6HubUeN`H`WqX7h3K$7cvI zRGzmjB1`+Vr8fVqR6=yAE$PPyQEFaZCmBj~_*ltqHtAXqv{O<_UC%p;q9WS!671K3RY9%tyDK==!yd)*E=_~>Jbt)fQk7bf6i#w6U1jKE1ZF7)*dZtJ^syqfW8u zTBrZyj4nBoAd2#gva}Q>kVer@v1~o@w70jPZMDMp^n&bc-1YSjZCU6(@Bp+e3#1mcwO>H>ar;U=ULpF`(h}YTti=oJ0>NofSy}0}-cR+> z<(UAqODRDeZiw|p-dZ!^uP6$C*qc|3REYRzcF}8r=;3jQqpBJYLZRLLrc~?oRTU&c2V2A{meKGr-Y65SuZrx7NzK?FXL4bg4c#`r!h*(s*4Wo?8W4FP2V})U+g+ zaeg&7r|s|4g01-3nK~87P>DwnNU0jXZ!9&QtpC0<4wBB11Xv=Y1-gaZqa)$ZBu2!r^$q%fI;kw#a`Eif~xA@!dNOak@EfHro!L9 zZSUOX?6?1=cY(%ws^S67SG)-t<$_B6E^$zqx*3d7pMCNH`609OqPp|ptT$M#UsQgX z;{m4O#%6zPcQ(s$t2+m{N{QBU)rb!dL3!LH(C2YV==PwIW|F0fA* ztJ;Jm(ZSo5d0zeDuKf$r!Oy^Pp|TRai}^QzGQpBU{fAKLxEG@`*@HMD7zoOT`(+M- zms|CoVm89&51p^O=BvuT=@O=VF3hZb^7lbtx1yPBJS?^;o;$2@oYW#p`ev^wRs@B^|o%0WRCR5QjJwZKRrENUS2-xoZRP&#*hR1NWi#%G`_QW z3r*5T(9Z$kx)Zz2q~6}_c5!=#;w1#q=LGXg^9vJ|%t+nm^&N6_JK&B@K|yi*=!rD3 z`PkqvIW?)k=fN!!b9>v}3+fYJG%^WhgUz6@`HvU7X`EJggKEXAJAyAXS65f+`ThIu z$61cuaGv+_FQ0UEi2$$z@T!>$Bdy7=_3d|6rZm*_pJK>C^$rf2f@>fA(#?yz{@m>B z#NjjssyXHybHIYmVw zC0cc)&>+7QkHi(Vu%HWnhn>u3iu_mXZy27cczs(YsOckfmVVaT9QFD=TpJY#$k!&@p`N@zFy(3{TA17^bn5#p0U&wXZ=(2aH$3L2LkxuHC&o3f{Ok zQ7qNpO&bsq0?$^vP+t2R<)$On0C54;J{!s!^SRW+o?9^kuSHA)J2gr%K3UY;^KYe^+w!bM1mPk2VAF%HqY^FrgDZF zSfPoG#`j^#j}!8qB3h~Vc8W@x(!@BQK9kq0dqsod)`yJQ@eM)XVj2P18m_J^;I89h z)S@I#r_4EQE3~?@2;QAVnps%ov}h}8gg@c{B%||v4W^w06QQB86!r4@+7`?_0EhshqgbP= z-439YUjKl#WpBF6vNy5A=1D$ir|^V${{9l_v*~7$h;T zac&P~Mxy!@WZb=tu&OXq{D?s*mz|fFe04B0F+V@hh2rIJOqWol*C7(&;hMD38=13* zr&Y`M$?eX7mTuVg@%Ak2pGvU+oIUz!ep;IPj))!(kZ5&k%|1Fk*zoYviy^k0fK|N7ir zz~8BpP3u$ba-;n(7oh%Z!)gw9``1O4^Y_!{; zJoZ|JTtCCX=d%7Z!47s{`{mvQs1{!hD*=GQVzDg);`A_`R-OIhtsO`tv+C=UxO7(j z{tbAGMgO%sUp5sUdNEg2lhuy~W7QbT>pt1096aw3dH{p*kVeCShXo&ijSWa_>;Ob8 z*cN$5r&juj3(47XWjJ_)2s8=ht*Hambml^eeiv*0;QI5Wl$-#$%4TIbEGfsX9x@a27iKC{a*pUJ-@uuE0H zVef_JPy9hYOF!{Q)b9#lG5Kc(ddnaoH!(Tc2Uv}%B6e) zyo5->GxC7a3Q%!)SZTCeH=mrP&*7bsoDL}nIW#GGiads*i$?R}# z8R2n^fpmg&FxQQWk-TKfp5iI-s zDWZ*`&v7@Tg@$Y+>V=g3tf#?QN&mw@hTq$Bt43yC|Lj*0tq2@geNakq8fs znSg&I^|U6PK6--ki`rfqw1ic_+X{sgu)WK8UOghrKakFN{$;LN1RZ0Z0`1!occC<~ zNQNDf9Tgw%uaDDhP{HK)g#F6TQq&T=jf|C-H3s z7ee<>$(*{dtogQ1JY^m_`D42sc9sM?^Q`?%+iTxmZy{A3YrK~YxmSw?VX2r!{!DCO zYR(kfIg^$JaeO^R`m~H=k6V4hbClv6JUsw-c@J;TyoG~r(IaoRs7YCRBjnlG*w0LeppyIdyFlq^r; z-Yg|T?X5z>mI}YFs;V*?c84D7e|+tWi;Lw6K7a*wd2^$zqT;uIL1LSO(tIN6Li?Tf zUMHK43B8>z%O!7PXApwvI~fxbO6G_J=wf|)e0%##>*U}6xiI|9sbt-eQwDIXF1w%P z)fJ?IG2=-)x)Gcy_HcFjg!wh6$nibDE`1RHlGx4B1Ca@3#L%fjMED4Su`+NWdu4oV zgQN1-uU}ChG9`U~m{e&PV9+AW1z2%OCyPGR1vdSv!4Lj3*DxE;3lvatfIj~#IDd)D z2F^_iknDm^>-gB1Fz_#XEi%Vr(0d{?g-q3`*z)D6z&}k-2Uux^ec+7%ESa(8<(~bi z8|G#sZVe5Me<>dkC)kzYAKx0|m{5~Tkzz$wecO_Le(-uB^4R*LOTokxNhX0z)(t^s z-OZr6^M4pqM736sK6F+eVPU<6Vldfd4d*X$J#Nz)^Koeg6D8Sez7)-uo1D6jc3fxj~=B zlsVRzu{Yv_1UnvCwB+dGq6}&D?IMQj6XU8sZzLgq%40L%_2F_?hcD4704JCYw~;{0 zgxXt3ukZH1M8f~w+P0xxkR(RXN}AMV4-#$AvWlwM1p7@`g%ykGFacm7$ZE_{gQ~NCWMl`}mlxR1GJ{OVVRuwNATjPW@8i{V ze`h;_vJi2Uv2yh=`fdZm)X8eWWc*W|W1^Kt7|zESTkbF`4X|GH7pxzqr?HOfH;{Y6 ziAI2Lj)JPLIOQ5XQnQf`sZ?b4Z}o)Wm7PH0$)MX7c29qD65H}4tp$90I#D=ysh?Lc zGs@4b20?8$5F}9)z?248?F*Fyanx3p!2KcfWTvmLPz(jZ!aCMU^X&(N-;n{QTmGrO zl3QI9OdHCJta5~@h-(3&RF}YbxvTP@5jw3fGtMR;&-@PnO+m80%$P9~+cK%CsimZ( zguebhY}@ACxpUOl*E4(eYFevGzqgN> zvu1MQ#0eT39j4EiPGLbIy}iB6oH>j3_ICRFdTDBGWc29K_MlJ zs8L9gL?jYHQIu3QkN0-J$C=Zo0fz|_CbDQ@IYO2%Gj3%?!2wLu{M^Rp^MKQ$dLP)9e1zu?n z*UKc_`H^H=2a%iz$$M=(nWkJI%Mx~KQF6=lIJOOvjD(GqoJc~F5RQZGfFw)a{KIkV zq!E+;4k!67pDc}%_7|7f-=PsCgoGq9V1-Tyfn~&q=#Z5iL|l()g>X~$gWKP@hKq;v zzE1Bq92?7V)BQX4rSB7xbm@6qy=;S9o7FYWQ}0c`&v8IV;Ml2j^q(~b&EFE}@dySg zSwSCMXY6mt-*|PeT|V~UBzfzyUUQr@9&k_L4Cec98bTx!@(obo69P<~{)9mwD;q^P z*X7+y3rlitUWK1Tl0PI#PR&o-c9KqrOK_5XBwTjIaTwJ6xoHfxj}Qo3qd#g9$OXKEmTR$XkkVyka$5cl z`_p~z!21(Nr@zm@AIe7MdOYX$r^hI^?I6kWRn~*cW!crQC;2dKo+ockzCSr?sgfD? z0O`6}xMPiNT~8cc0~XQ#2y!rlU%H98nw(pG6XDf!&r zk;G5-(LmP}fpAQs{RxynHa>;xj-lrmO=dei5m_+znU_h+;9l@j#Mqhq+#UMhczvUvgzeaFSTfFiK6=ClrrzN zNu@s5V#IY5OV_bbP!$PNi>G4h6NZJN`Vbhzq7h>81c^if3kg+G5VlT#ID#Yl@yP%1~D3VRIzmHf#LyIRc9oMsjtrLw#h{qEo5(bWA5|71* z$K%8k2@;x)ZJX$ZiDeoXmW`<=&9a`9wi)&xyb``9m_U}N8Nr*B;14oNHSI;LOAL^OS2p#$0VT}ScZY= zIOy>hCK8JB+1FI_95gY*#^?8?@wbE!>AXtP1)Ah1R;tk6+>7wEp{wz? z9wV+Bn7WRQjG{;wTI>>kYX(B`p*RNp;V|)phL%WR+73oMN-Pm49#5cY8oK3R8M=E0 z#;|Zqji_e+PIwPoz6eL+2*rmgi_4u$p2C(i;vFvWp0Ev^{4w0MXgZ1RW}>Eooa9~4 zXh?-sITrC)9MiPWO&bM^SVBiuRlJ}pBM~KGO88WWdon;XafCpRMTo`{Boc8FnucjR z*oKZ_Ss1#B?U=-)8bVcXU|@cVJ_=u_k34$Q`bbDJvZG^Wk7xPPiA35u&>RWr8hs>e zlUO`~ZJ8tt2cKlo9}bg9Xe1IEwvbU|m+x#nMkH>c_!txigXz3#Sv1YpC0#&|7AK{RQMKLS~L)S444a1i4sS=hEBOEo5 zRX>Vk5s$_&g^a4mn287x&BEvNxq3(>Lfn)62@TyA_*4Zm5hW6hlZeMLEP*0B#A6yl zRa|F`sk`!1Q7{rQ4AVk4l6@O<`}Q+uzaeBiZZ_MqBqStBzUrU!=ZwGYHcZo`si_H{ z&qpj4qphut{{DXY`g+OB&*%2rZznG=pWfac;_(FGa6gGe0>^Ra>FIW}IG2s0va*u8 zx;k=la!^$jpU;QS=leuWde3I6t*xQGy**ho8P1$JjbqygA<^2}iqG#$-EWvCU7ely z{Q>&=dI^WaD2hT=RTYybPo|`_l-%50y1Kg1G>zWgUZRl*LXgT{-PzfZGTal11jC09 zr=XyK`uaL1Po7L)Umx*!{7X^$*TJoMeDdT85(#(Ae)>;8p|P=X;0e8My*j@FNm#_S zAWI*4lBK!l`O)`ZMa#>>C)wx;4O>$2D-O|c1XK73g|d+`&~+O@g7!WiTmJOFvdnmi z|LgDf5E@xTw%>=N$B4xBK;r>UTWbve2oJ2ndc09`)2hX65S;e1xb0x=r^*?#1)+Bpa8S6H#qI&z! z$a(0mxih+#AO2=PF_FcBO`90kb&Q>r22cITpD`mlistiE`_8lcddGgu(uF+m@D}Eb zFF=mhvggBY<}8>=VV)06P~;vy+;<*9ly*%f5Nkt}&7(MSkY~3Y!SDx>oX^IyDkOms z(;0ck13b2@h;9G)eJacxa(n_q*U)W=fKMVG4r2(F>`({=6J56u81!^&%v}2)`1hL~ z{`Q-HNAvI!3j6_NbRv-i$((-R=wwg4m&cZkq^%d!zLOko7{`D3&PL=&6r30zzx@KQ z?Kwqn&SakW<{z?RtW4Mr(*5CY`PYLnR;`@KzUQ8$k$glV!jw&a#5eDrhGpr*dzv|Y zqMPZfW|M7!kZmNZkJtYBU(n}1&SMMm=rw}Gt9J47ag7;exkQlA8`>Ck=Ms9~|1GZ` zX+jD3ah%U)htqY02oJnP<9Yu>llUu z#OY}B@$g^#4R@ID@>hSq6?;SpIRPJ-2_jJ~6%An7G86CG!pcz&eg15sC-?C7@m40T zcz_4jmNO!#5RWwS`mesv9%BK2@$bIM=zxLlIMg26hjsg1OwIR$WML!Ia^_tw#6#%) zJ@^%q25TH^id*@~vv1IAsVI{3+3YF_fny|4%I@Hg9$U=e7k|v#RT_mkJ{(IYu2}?v z0n9{{Xu`yw6(Z=9FmxS92(<2Ylm(CSzrVGZpa0EwInWfKs4#?2a)?JG=oSd!hBo<& zZsY#@SE7e|$uV1b=d57!qA|R^?L}IQY%I%W+Um!cW9{P?Z$)_c+ke2ysl{k*)$G}G zh*7J)#=Ig60~yINIseffDkC|BViD8=oA!%77A_yg&Y%36voRT;G?ma_VOu7eX5z~VV8tRtH4A@s4p}~dt{ET%dbk~J!V~=Td(-%T zzw_TX*{_hF^Z&DV-tkqI=l_4-&pI<3vY25n7lK<50rx^%w^ggQ+P!OQ?bKE+wT_|| zCnC5IMN!0wvSjZal0X)jCpjl)J?sAcagq?iQ2W(iYkNMt!YesBInVQ4_jQkJeu8*a zmuNJal92+!wva_fbM=K|X=<|Rua@)73c*i)bPc`qB&HK(*~?G!)`opZ$02{rWn6ut z&2taWqPgH$uDkg*j_v7S3k~TC!aLt)>Z5Ps8$OvUFP%)ko?e{#^*r{}B1TM{K+BqW zEZz}D*YddJrVFWlc^=}7vng(CM9a{q3n`2~rZ?}*oX*x(9j_|?g-GKepkf{$$V&)A$ zVzKm*=MQ3NF0r;4rjt4xDKd$bbu8C?{Ujn)Ve;}@ng85o&i~!77@O;2*(N)d&f&%R zYe{&EnQ-|vOd3^;Cfiu|{v77Lvw=1vk5ewZjwz=NM33*`t=HF(d+c~l9G;7`U4%!c zVa-BT*Xv}cVO)PUl{FDgKW+#sUVE0Ml?nX1^2KDd2!U`CxLL!v>Kj*)SoAlZT@fZL z(+7@CB59#{yeM{zNX$g@1_<~J95ac7f)#DW${fkB?)x#@@4cC4cj^>m2k;mQiD(-M ztDU1$aj~6VOu6ZN^qM-d@(n(GZ9YS9yo*Z*=|mlu>h*8)%nOS+=Bn?JwecC|Z;9iG zEGAxhKE8?_1W!1RlLzM@q>ZZ>wCwnR%@H4VQyoUoOIaw$`9n2k&w82iHU*FRMXhHF z!bxCdjo_O%TtvgiPcnN+J$czaq+^jt*yvs_SSd%d<_!?=8rWt6M=03gX3X9vbN6p< zV)?WmGJCB|mmW#LIL!N!FoFOM)$@OO& z%zEr8wuJg{`7IY>uUW`j%XTvKqT86#ua<|OoR3>Po^Rheh5kVuMYu>qqh{$VJo@}1 zdYy3%S57*Xte}^wHLo%I-A2xyGK`O&noE^w;q5zyuZ z+nH9ZX&SDZ>ZI&A4zA@~<5O)KAq1vrBBkt%LdIe-JRT2sSK~M-hgHfD z|L>VlQcA+%Fpg~#%*goMU7i8>kDxm0jMg3jvfZFo6l5ZdHfRD<#^f;nna$)6A3|$m zC9Q2b`TYjtjnz>V))_nbOfnNytY5d4HZ_mpejYTV7sm%%2N1_7q12sIia3w$f^<|7+y_yfIA=5a_rXuJ}lRNA% zNpMo33KTa%)H8^SE;y0u_ht~wA4>07J$20qg8BU@@x|G*FTpVr&LE$9Hmuo5v&^Qr zpAXbxPC70^qaC6tUck|3T)^mj$<|eCX`+aeCk-MXBvSPt!{uym4N%l4pQNJTrJPK& z7oNyE9{%Ht$Warx^_HKIW&W1?)(06@tgztuM_5+wpv4-Hy(ckjSbx??3s+IVcF8Xp zLSnsTOX#%^x!$1;nF>w?oh(VP1TVI1)MhA!pTfIEuVL%y+Q7f z;ZR>oB<7IQcQ~2JDyrl8j2k}^rEwSQHkKiS`4kly1hV>Y%z!$Qp*o^@#f&@mVl=&x z)oXU4^%%?vO$MDu$kiyV0fal&ksKH%9Z`jDIqtJV~#CeTcW3 zC`ElK@z(IjKb|FI={Si5&Y~p*hJ20NFVExQyC0`U&tb}SSK-!HQ{=1Tz1i<`>K#8| z^2m+MUQvY>IDAurBYXQ(dMV}ph({tRIb+Hc0(<9BnkZy=?+~?(VO(Dih7Ag^ciSHF z29IT8|19=wUCXYzI6Vsc;`bO7op=;A;RMwUQTiQ!20f4UQ?+#!dm=i+Pnkqm#PUg)1L8>;bW?M~? z{Nh4{n!&)Lqo}bI>gwb4JK;=9bitmDtJxI|a@2_@QIw@%ISRVl$l3~>o_W1cMUrA) z4R%I80|RwDeBWavi!bB{H{M9a?SE!nL}SpYXEXVjYCakA9*awBc>VD|v#9t?zI)q9 zUis~VY&YYGk~8_~53itd{v-T#RV)4b`q^juIe+j_YDx`U0O6x#^nCwe)}%!$N!kG_1?jQe}9@##&O*8y>D?@>tA@Y`6Rx6T98?P`6DfbC3xfY ztbTSCAC8#FFRnh3x1V@~#Ty$Kef>`uK^e*HJ~S+Oh>wd-<+6)TXZ^jeVdrI`IL;ww ztDqxo_HcUGo<0pIilnVY<&;Y=Vz9G?C#wStAL6HaPd$!5lR^D^(pl6|%j;Cmc(=Iva3(n_YiTMj<&r^+vlG`d4d*;y;QaN$BA6)Eg6Dm&(! z-|z3taCKd`Gb-9q{^9fafc=5M&th10q~&(}4L~3e=)A7q@9%uQ!I zUgKAQv}`O#9^nuokwQf`QZ*vdu@Hs_t7R`+ntWXUqi?hGx!>^Q>K@#7^UX9Ze3sh& z7jS03B+c%4jw*hEM_$#q?N8rB*;UT=^&hZH%VzZPr_zw9VDQL`>7VlmuWS#}cR&f4 z7OK~vIax(j*&3FGlf>0hY8q8ezid2BA57=TH#d{fGm|A1ZDbF-m@&N@c{32i^JFT0&2j-r zw7HIovQkQ0vhiy!1IAy+DAscKLoZY1%cHij7FE|tG&QrYp^=cRr>3rnL}WY5K53x7 z*=3JiMMKy_bHqe5z;gB%?{^;q*OJ6-2YCc5%|+n%r0O_vZ7j_oU^TLIOC!f!{v!s} z&f@p~SkL%x-N?zw?tp!wA3VMC=?buB;s-04()1BWv{C)r?j+`%s>E-&qE@Xq@gK9b5k8P zwROmXeC$|=$~|S2?hKKe<)w$diKV(AV*5!{*RwCHnuIfeR4u54L-`H_$Yd1TIk>KP zzj`~Mcu>7Mpnzi|y?#6`Wo)SMaLe7_V&{yznYm1G{U2^d`FIY^!zVM`(~9jsjlO~D zyuM{Pzxdmggja88!_v2@v;rJ6elV@kASI)RGbC#|bGI1uE-ArCBoLkeE%rXv?OwwY z%Y)@8>W`F!%4we0LWaL5;n8c3}tyy_ZY4xt6lsr8Joyy#5SyyM=I8BcTxN ztF5Cc;IL--65LRXd~XviwGC9ahViHZ*K#?O>ZP7v5;5fw9RyN>p(_~e^|X$q;5A^+ zh7Dw%{3CAdwTavR@(kJIZspb!nt6m0uDIrOT57|bK6V7NpPoVgYwzT`qCG6%x{3Fe z*OJkH3=_>3b}56orz>P(yscR$@@=#l|k2XVR$I9inW-A$u0czsSAD z1$`(f)JRwgx(HBFzLigwe}arV?DLeNsQE-1YuH^;hNFaVjUlK`60PtIMmx)y_J+&w zqmN!O4LBe0TDB5iHNqET$U zjc{uWVOW%|SVAZfLfKbORdWHM@{YZ@Pd`SsfF`9nGvi-cR718@c)Do>c8# z&cenJSp`EmeNu=@dUMffgL!c3%LEJiQqt3dX$y3jP34Z|tX#aDXe3TmZH&0rAlX_^ zW%+LQ>b;0rCZ3+7nJ~iP#e1J-L!FA+_8QN;GLY|_b}GvjHWL#8v}7G?m)DS$nSpM$ z60fNz6l$ifrkbWUm-0=^G3^j|YG|m9U{_X??4yHeBM+m80*UFyagMZGONpu&80~4c zwkgqllG?JJSOur?{i}<3;0JfoI`SgEbYj*GC-G!_0?BF^TSck$o~lgod3 z9yRMf;NiPw(D&=V;`T|CS+wLqUU+FIUSX#qD6WfbJFKXY2*)9+8CiIB z6{&m3$jrk{))S8S$jH`F2+&;DfIc{v48=iXAG2RBarEA~BrHHG1oL!uygdu| zykBtTnIE$;ZlUWMt{EjTbOLuyeTs-G`S9sS*_`PkR##3)W)k#j2uDSzDz>hpYZ|Ji zqU(MfS8Urit(EHe2?l?D#99(R~K}Q>*DTMcg z_|r=*)Jh3brO$v8GKGU3-OJK7J2>%#AruT~WMH<%y2S}jxS}tM@4cNR{tNi)?Nb>2 z#&lwq%bu6+;l6ij7=6x`iBpw*Vw{QO|XSG1n%2#;e#Sd8C5=^D?i^^ktFkypHnjNXs{Bg$nF994MG>REqjLDvzMXx6kOHEpn<(1 z5yo!W!26qHoIg09U}GW0ZM*o;>c<8B<2?SOdr>dElOLWlnbl=Gv0Cd{@aS)OahsQ` zem;eL8{Xxy`yR)8(Jz>K@i-QK@&vEW-ix8Mr@p%mw(U^YAP{z(c+N-)f+kyQ8nCka zGVz96x$gA7ta;)0+`A=8_Q=b*{~yPZbWN5$`!KJpZpHBEsH%qU@!%CEJ60^ml?uC6 ziPQy+jS0pM9!$@5l|35AouLUi|5)J2+*)`@FE=WwrJ61rFKI0*dG;@P?e-iSMRB;*tgqF0tpD4e8h%Ob)P zCU8`69z`dWu;uOP3_SlP#OBwS_SjZ#ee~~Kc)|y)K_R*MP44^4KWQ9x9p6wInDgi~ zUarjMxBs}GbBE05$#-63n@>q)fk=tt*i<%Dr_y$lt;oR+!C~j~w}(a$b!}G8dY1J~ zNrql9o?gi%yi{4pkIoxO;XBJIkGL3yhN@}knu_AdAUj)S-;M@cPbPjtM^$|2m?VRP zx%KvYm{^-Y(}R@s7tB&EO4qKWMpu!PlQewvY;HyyG?+m>SJU98X61f)Dj*XGy)VsU zuOXV}LE4gTosIw%K{Hvao+Gol1qa9}>PzoTm3`4LJ1dvaJn1Y(3<#1nxP;xy9w%?y zZDj12!=I*a;)bb@a`Bmcm>1PZ?p?@Tf0|Cb=cP;$O}zEgRAyHfa>u=Y;;iG}VA>n6 zvm;BzP6-spvT;{6ryOTkxY{6$oZ)=?o~O9fR;c>uSr+Uo;tzI+s;ExDr=wud+R}i? z=}ltKt33B+U#|PpKbW**8PC5okBWUd9#4u#QdKoP0gdKOpU@~%heO-+ksqr^2KzC% zB!f*+H&xf#DL8?GLsVvPR{r@+2=>5o58f#xpbLl94KMLVG=$=SudpBe^ZX>MS}EPS znwrFQ^dHiP){k_-uxmOZm}5^BDije{k{n$8+C~7|GZU{{EZ0sq`PqXp!KP zxesvPdkVMSb1xIe58}SruTr-@fa|0j#*Sm-RMEWUTo`PWaA9CSM8sVyve?Q#CLAj$Xteh=fu z4CK*Q=Tf~k2<`Xb*fvhp3P3?vQIb9V#Kaf|k2!_@`W9YzBgr|(k7Pu44vE@Q${R8n zcj9oo&Ib0DTbz2q8I;W5LZ#~^(7vu2x9Mp-Fq+v*=4kt5>=!P;57aXrGWr$J`Gnw+TAMgejq3f7|FdODo5w=W>yknVi?r7>?pN8eO zv2kN5Lm0mj#qYrrU(cSIi*f7(O71W&yznHNKE9v0F4?hiGv}W-h6#?3NKGlN@j~(n zhH&Li?nE!>!HzY{a05O<)qAOJHptJnM}3R zcRT9dI-Xjf2ni?{MwFdf_HyZU7jWA86}-CDBxr~OYA_`dkKfCA{WAzC^3bOpML^2k$8plx<0$lm`LL>q{wLUR?IAAM@=?Te;(jrKx5}U1Wslkd#1E6>7F@M!oDZ z&OKoZ&%M6`J>c!^&@D&>;Jm8JDVbS}HC(Ji7{4|GL$TNx3h`##!c6L%IC%=k1R9yP zJBnlNVr$hzj+t-{d71URzb(d?vjP;>FBm=CW$8?BiD~>#*xC0Yg^c=jwiN<+5dc; zH`azQ^dfHh^*2#<11a~%A$3)ZNMt); zi*8;S39bU(1fK2l$ZBdNX$y{@JcUu&%`C2LCfZa@S+nG*lO~W+6lK%Pwd9|iP0y@z z`PJ{vCMQ|S(kdHYL7d$?Do`?dkQ>m6G}qD6mO*}=N_|t9zFC7f`7q6E3>l z!n?B{=7TbcUl+IzC>{;FWj8NP|1%%dX(+KeqIu^aeLgaj1g523v4VU zo1wv@85_)?&2bLsn%_B}6)$GwZQh7AVw%ur(gmDRBAD^&KFn6h&V5maA9pg=AP;44 zuSJ}4IX(L2^5Z*?A}_O!m8+Wx_*802w-WK?P>}5<+R{WzRFKg#gHS^wwqC%vDJO95 z_(7O)3!l=);x}gU`ieRNJ`HKRNL@#b)bZxrdw70h5|0}r?C5wEfn{1aQs6iiw%cwm zx67jI!BHUE=c|HY*zDQ96W=*ka@LrScw$Kdnx`vUgaBzH)j|sMMpEDtSni?H>IJGW zX|0@3RV+c$bSXLGLdN&Cc ztLpF!wAs763^ymA+<-x}sg4F)Av5TwCDclZH=C2cb}Q$d)Q3dUA)v*0|Akq+x1kc< z??o{T5_OI2t(P2s_W8JLA7RA?jngl=f^m5+dzLRCn%AGEaJ4ByAf-*NIjxXU1KRQF6RYkSG|2 zq-^sxa=v>O6Sr;Rxkc4v2J}uQj5N|nBaJT_|2E@SN>B_z>-K7Peq>|F!_*k5f_OzW z2`SMGFJ{fh-2cb`etYRD^jW%+ZL5}3cgbyBGF0V*C!V1;b{y4J6}QQqUU*qGC-$%nqMWhTb0lyzrx=2MuHzTb2 zWIL0u{3hqFy^BTr1V#C|G}W$Ucl122ykb0mow*9p>vXO;r#~BKK0-n}0|kv>Fu_MJ z&SmWVzhRuai#4`OrYg|=0V1Eg!|NMH@S7iwCLQ3Sm@nx^2oHc89J;Nu2S&_gAZKZJhS4okKzV$Sle8QoJqN1q+ z*S4?}8^x8VnuZe()7;#Q<@cfLsVShST|1pBXez|oT4-zxA+tRx8XHoSD{eeSaO620 zpHssl_ddfuB`Z~@EV~Fz3WR18UNVnrwj*rwkd6hRDa2N;z|mBM;m566!-LaaW$I0* z((CQbRPKDAO;hgS+v97PIpbBzTgS4mavLu^^gGribI9{4WcN8AzgI^U0@pFI-8@F0 zGL&bY5R?oZL8h~pinVWW*Rq8v<5Wr%-ABM5Na?#~ptaPnd~H29Uw#u?_dLR?y)DEo z7gurHI{;g>G&j?18R&*LWq=6^4qD)36nKpi6ImM9*ad!-B0fUcQfIrd^RnrWZ6f1xZ|f^W6xXj*}bch z+?FbKuKIwx?|%~`zX#Hr#WiPT;MEjxTx`oCZ|HFxmF;0wUN1)U^|EH(E?$52W9A!b zYWPs_2Lc!#t&`Uep`a-O$F@=pCtQi5=?KRqyQmMnd{K7p-NTOh1jpw25$#kwx~?AL zxdfKwa@52LSj(Q_v9~spo0r)h$?8-O6d_1DHVd1Zi5rrbJVXQ?fmd2AO<1@n`28mD zJpFg_esvcUPVK|;SL@lbVF%aT{2lmk8awN&DQakK6 z*TJ+Mat4g$_})*lsbV-s4+^kl{tn)lyPAax?J3v_-e3TauC$X|0g8$yY$D-s%28+9 z6rFG~eG{wr?ROq#d!x%y*Zh(1pEH(u8(%{zsgCIq6+dCtuieOn-@2UhS8eCb9SI71 zWa2edR27o3Hp1aBhMs|qg4)e!O*$$gPdo#$dIk^7+e&6p9>TWw?<-P*&>>z=#?A#O zxn6{ma*iD^DN%tCftD7e(C~RBAI^A?l3)IulTR4V^5-_QW?L0EocApPduQ-Ob%YbE z>)5kv9)Fzn0a{@%^l%%ep5{TJ`rrrVoSr zLFx7yDk?%u7@LJhIJg}Q1P7({NRn|Fa@y(SH80@aCzjw86ks@A)YXzy|^(kz7Wg5@*`!y$>bUdG|d7UL|_VW8HE<-UlaL@W?JOgW} zEZxrEe?OC0W+54F3;iy;0*`K>;E+t(WEKx(*q~xWsmsXWy@-AM5^v3UjSpVcaNU&V zd3*u<-qg>cU2dAH5Q~Kgw@LCd40MsCqAJ1H$DKju@|ST2Phd3$ zRJP8W#>^pqVchY9SXsK5*5CV8+}q%`L5%5;Q|6TCs!5em(g9 zLyyr{)bYX-Z_(r{B;S|XJW~;Z6z8NQ6+HC%HTEo;&C_`|a`~^Pa+wQ>UGFmW@i{y) z&R-q?03ZNKL_t*l_%qyeW9=t%J znU%cs_|v?8EAoES~tF4hpa57k_^WC!n!x)-<-%)-&RYU-8(P zZ75bXa~^n*_p2cp3DFv}Qf>2{DD}xauKNC6jK9IaDSMxPJpB?iaW5VfY}>+3TG+0Q zVi>52!|1o65c_BbyO!y6A@VqcewzqGM>;Nw$3S(!wq2Zff~3>P=B?FSa(XM}<+U_L zma|~hK)(0zbLcY3%9kJKt!*KiTY6v%fub8EYqzo^I+*X=Go4rUDz>PQX_>0%#wr*qk^p|OEYJy~nUs4lsHf`9= zuuFcxBZd8_`FIvf%38_E&qo#exlU93xT$(%q3Sr1dfa|j@|(X7#xx98z4$1bTk2>w zW8{sT#P_c~9o0o$0hbf8%T$`8c z_c8C#zwK;xWlAsvxKhF6Rd5^=*KrX}8@qPyARcd{YGWy`ZnI?0OANa08$34qo54P`A}t}pQ8Q)0$|SotY@y`xpE51GfPIT*^G>-#US4jO42KRGwGWg~5LS%l79FAL z=&noD&0tLb9@K1KL)gioPd}Z|nonpLGnOG)f-*BfYp4}V3XGtS`c<#;cuoda-}YzD zRRwPCN8J6`N?Jm#B<+*|r)kg@3K6sS*NO`vuo7`B+s1Vzs>h2e-EO6;DdSf<^<-{( zii{)CKe#rE1gWSXEp$W2F&&(wiK1KVShtO~3n#H}-9{|G#`-zWGUV2qczotfIPn^u zyZ2tgEg>Rt6NH8)qwLyVM)5hn;E|#n>Q>ER!4`{x{2WxdU!L*+E?fc9qBYcliH4>K zqyrwKjRn&lCF{oPx%+PyfROB2`vyUnR6i7#9NR5nGLNWS~C+MR1DRme8+Bv54(X! zE-E0t{eAwqcrQ7H`FNH6PSFGSqyd3Lq$NbmQPH(j(;7n;tef`~18%&Dd#0U@lu_0$ zn8n9Cq6B;z2|G@xx%D6?oIqkF6PUJ(Wvh5}?GVqOa+)Y^BV}dtspy{NbHH1jP7Zok zs(3sS>A1MANy1TacdenU;at3B>!>je)TMJ-J>X^@m~|E+S<8!0JVvCkfpFYLpkQbY zJ2q}-)Y!{;92^(?UOk<*nMN9EeCZH<`u3C3 zJ@AO||DEbc?oT^9#IN>m)9&TAlm6x zbi+WjZ*poozp7)II33)uu7eb*D5jJSmg%H;?nu#dXu6IfQr+mJE3qvTSJCl$ba0){ zlpyK4xCqp)-UA`A5-MZ9b~9f;PT}dP53r@pkH1RSuFT@R@BNq)oR!@F*j!pg5RD_yNPXIT=;#hA1j0=cH%9W~ zKi-V};S6THwSk19q(rM~D8j|E98|-Du1Fjwb)a!wiPA;CEl1$>8n~8;Ed`p|J?-%j z1MJ$EmV@r`qN^!=wOs{Ok*RL9dPm1WH{~RfE{+Romk?a44ZlUsF<;|X*NtNSwEOvB zR|K#3$UJOE%5W26|JooFg=Dmiqp!Y`n@2V*ooo`@vQcyoy5f>d+Nhs}k6606q$m*-A=~=`*qwTl%1Qt$nP}$)Z`VVY&avr` zkz?9q_Z!1aKe~#V7w_SjWeo(p+7T&cy2mIxZ>#-vrOWpT)8Y8be#{NWwerx^S(Hai zQq9%0E;3T_wnuthnR*_H=$r!`>!?R@Nqi0&9jFtj_05%_D(yPh@g7ZScQSV9Zi>U# zG%Xa}gGX0z9T!)+?HrJ5=U(7C4k+#8DFm*S#5dp+zH`gv)W1EI$KI_aJE$Ewb+T(Z z^g#mM|DqzGEe2!0_Fb+$!Q`3yrn5CH5z;{_8oH|BcF{S}zCL$fQ*6sdXdXP8OCpg< z3q7=3=)jvfVBPGHsbgZf3SO@VAybq%YWq5eNM*d(W)j;~(Dc;W>~vWpIv?CIv2#ap z>vwL%yLA>1&RT_*nQ}lL?#|>9?^$%2tAb=C!GOu%=0jt&JY92ON{6<)Z0&`@SP_oP%ZL zI!TO@N&NEGv)MZ5AzoNgg*RyI&j{)E{Iq}8kz~~QouXp5wNQNaclqf#dA#`Oqb#q8 zqKQ;Wv8t)<_uL^PH|3N+K)1lLETm@O(FA7FM5x+lkWtDBYTGV~uBG~TyB3ZsQhTwq zaa;vW6G$mK9W!eC`DOn+Po;dDmW%H7qKj0+9x3I4I@j&zlvCXyt7JSs`pMaBcx)Y!qIcN3&q}2x9^n}kk@^y1|My+Lx*Ek26+N*C3eO%Gp~j~0H!zL<)xeu~MiDD4*9VZOTlwW6WK z_pz(I0aX_?RM(PJJ*kxJl!A0Qe8^6HQM&85CQ^rHMM>%RXMs>tgz0X_+)1Teru2A! zTlr3L2+?`?>ih^KHX1$h^JrTCK8ttMqv<-IHnxY-xvsj`2`N!Eh1$~Xw3s?pq>hHL zor=bE%n3#9oR6|o4-TAf9s23IDS3V-33cdRhe1s@y=j-5P&CvIuF7s>)BZ6ywj`^d zkmTN#y#Gl#LjUv%l=frhL0=uOca3{#!lkxs7fsT@3DwgWG0_dZn~YLjq1)*MJLgMx z89AR*MhEFc_cf~Pdk&IM$6DC2#-_}v1II7r1lfP90_ive^9!*nSM%QTT?oTKd={ru z`W zQlRP`>7=6FxzMqNQFR2)m!?g*C%6QMV8Lw|2<{dn5ZobnfZ*=# z?vlY_Ah_K*=e+OzzPr}#A2YL7cUSGHuC85G``OP^+s@OkAox+pz2>oFx%P!IW1%u~ z8T(925MkVCP@uW&S!%ZBOgKDAV}vtp`sZII4n*fwKq+GKWHsIiOt;*_mqQ$F;{u;a z)f*WE>Sbvat@1DvQVm&Sk7&>4p-EPnzAtWu5uRBRLF$;_Mt1;Lp0Dil6VGi|YxBks z%#{+_M%E$Lp%TYMS1}_)#IP44hz#-T2f$k|D^;4<$>0ewmebc!wrYx<@wakdVS`}5 zb`y8wL9TIj@yka+s*o;K=}3d1k72Le4$`lFfF3$xCn;~ZJ5jl~mgi`K_bVtX`Scqx z=rNN=;&&rbfjoB6v>b3m-t}r&C*U*Y`eUy#xB!m88({eKD-Uv4jEp=oJJCyTqSEVT z6=k7{p&|M_NtX@ja_v=4gm#8ga#Imt?83h#@8-TbAxVQa>IiFR_O&(QtW^s63t2qQ zPsHxrS0#b>C3{4%(%3JQFKRCIU@J<5Z=5|a-BUu_cmZ%8!TT+MvwiP$FhaYjsHpOx zPDp9NSL};MVZ+WX`z;*%=$#{QP)-uv)g}T}SB#v305Orod0LHbA@su+EBz(1wiY{4 z`CT_^yfI}#zfb^SOB`*H?%U;Z;O?*LLNo?#uiZtb*Ip^sfmgO&0!p|r1m2Q&VZR6FDQggi5Xz@Ae&)%phd#5`xVuR?z6WwX?DVJ-!-owRa zM4ZpDfa!QUTP!=KlicsF*CkT<-B%=_Hb@LBE?xRS43rV-_=DCV+MC6b<2nUkt2r{c zNB63>8=vHKYfW$G9)1jN#z~s#zt15J(XPBm^9`+T0tX!4vhPbsUzqZO4zJ|E_r4D~ zA@w4!GN8lbyMQlih0#|<lL;HZ6{H z0cU0-a$MzhnoEVJb#O1Q*231672)6E=zsPq{^#SRQ!?-vnag)?mRpHohFAsWnH``)7j!(t^%ih25h04$@ugI%te>B95B9~p^D|JL(l-0lp ziJ%ZWZ7$gU5Z=Dh{nKWZuh@=!3X!-)IwJHvc^JIF|EHk--tM|p$C;2Urb`b8TDX3f zT9)H2SSHw%bXuuX_%u^dU!&s-g&%~>MpxtZ^>)nem*eo6wH9AHza%4^1OftXAtQ{0 zTrZLOgU0(e+Yk#s1nsWNySu>ne5`!EF2zKXr1vkG z-G5zy+mqUFv)y3S;@l7rgnRRR3_)V*r^Rb+2RrZ0VuWdM_eTywbfmw(+BUQ^o8hOq zw?8!4c;yDi-0}YB1njGq1kccZoqOF)=0bHiTeH98pe0j z^^McwLD?1YBd8;cpB&Ckf5<=g5*1im2*N{w@f_{sV0Hj-Mw8edNtH$=6(x#JlxxEO z(*w50!&nPX(j#_rpeMx6HQNpE<6vt`b>M_g>EhN=|2cb!d&WP4}RZ)#rT^x7l_h zBz17M11EL5f&~V(ZK(QQYaOGo!pf|>7k=$KJDfN|O10{zwN$IT0>E|D3_yNbcr);C z^EL>PM|*H!dB*YM2ll$$^om-)YyhlWA zx!FU;My{NiDQM&i{+X6kA*MrF_#W|%3O9tSp|~q4baZ%?Ep@?`3Ry|8t{D#XOTDq? zp}YQCDq;%&@K8$iPVnZ!%)#IW-Ov@MBK6YCI@s^1HYfKe>4>pY<1u!bAf@dGdohLD0XR)ma*R+}g{W&H9Y zX5gq;+AK4)XF@3l&*B0ti}5FmkDkdA&rb@s!YOH;sAZTAsh0eJ6C*yi`6Z%LzY6>= zb}b&Y9OV$O?>ROyDiFw5?`O;kzZvn5D z8hYG5B;DJuD3oaBD(>|PS&x0PUcF3lXAHOOktq!{ zAC@!Q_90I_F}@Z01AjC?Iw43Q{KX7G8pcEl(F8yI+^n-si{CXUonnaTg5n6^)B}*o zu#+Y0jMquMXKe0v{-_vTM%o;>;w(vrfV*m*)wZ7ALAd&&Y&V;d-@bZ^v)4;WBH^Nn zDGDu_;d9P-<3K9Xk^0*Bciu`hAT~SE-?5M4tuZId~RO)d9>|2B;!x}&xZc^#vtvfMekY*$$ z+^A=QkhB8HVk5TXJ48A-I;hd(rr|Jbbl+A91O(**YBt}BPd*N&H60U(TYM;q-nAim zjN`uB;(VKsET%Eloh0`%;vdgGm;r**&F0CLJAV<3_CnQ+dtVkIzu+>roU=nV8~$#; zRsZLf1iAMoBDzFM)XNW^6A}l2&G;I}?~E6bBh<5=YnPu(5;Qrbg~+Qgz_o!CpGeui z9Lv{7A%jBeS+&(^aWJ!wKVnZ+5)0gnxe5B-aJU0BY&VC31KPs-zt`2Cs#En#qJ;Kc z;pk}=U<2-P6{%)CYeT7V@Z;!qibfp>R+A1GyRh&YkIhjcb>8&n$K&9eP&=*W;z^Xr z1MCX-xAZdJ^Kn=#0Qgw+WhQRqlO6p5O!TB)Y!srOL=?X;omzMM2W-{Seeb@W%2NX( zYZ!d=LlH658e)Z40R8*op*i$Hw4F za##)lV3L6YQ+ZNQk?Qk`$q&Nc^>Z+)1tkH9RJ)jhsRh=*rkY*2SG~^6)%@`H=Qtd^ zxzjCK-&vD))kDVxs4?YGdy$#v$C6=OFYDiOEfz6_qM=DtDcK7e;hAGHX+}fPx0BXm z5x&Tm2s*60#bcHO$?Owi1BVN@$jP(@gHRcI(Y>0!H9F~dL~!=PLRRvLQ1L`Ye+V{)p3r;j+YNe>K`#5(XTrry z3B)&vL5x9R(^}s+9Ys*)`>Twz5EAJ{Ed?y(_|5VnlT~JiIAd&;gHeY)>!AR(RmwKwFF^ee&2*#sI_b+%(dHtXL3IT zj-f+_zN%Tewkr`HDMEyL6sI^`MYwNPQ!72nE_ zk*%U!v}`!-j|4PzA}HVzv4*XSm2)IUrwlZu!noVmz6_Z4EkSvTNwo>hZHNg)J#m3R zbqQ>%MGo)-cr24EgO*AHcZd0VF{jp&^9>7q%qRN8t|c0=E;@poQO>s$mUC0Y(ztJe zIJD7o+(^&Gym5^U`KiJwM)pk~apY79CChLVOP#HI=F(y8rJ=u*X_4WQEH)=f*xNI) z+K(NT$%RSfaG@zuqkTkU*}Rc02q9p3TwAW*0Z#kQ=Wkqjp(};4P%oklmL&uGAWBMf zDbOYZQ}r=gTgBpHgjuis7bNX(`~8rYiFl{B(1`~sZ=pUCH@LJ-ca(ID$=S2Gk53`= zsjm<8P7n9J5iWAj`{SzL2~=b(hKmyi8VWk$l&mNk?~@(2jY~4S>RqHtn=e3NYyQIG zkA@j)#%`j6YO0!DG{1JRKv%+EbKB&8~ zJ~1c`<>mV=>vlNL&$-=ict6*mRBMcdmczjv&JNnu%znBlxf?!`YT6HnC_*)^D^z)x1o^sR7CTe~Ed-Dfv7d z<6-1^*uZjGaye}O676;NB9?P^WIAO++|5iR<@EQe;sqIq+!7TNa627?g%5oe#yuBs zV)GJzfiwJxgNMJ==u7_Ucl=A=>^ZbA2RtG1(;Aw_u(CXxP8Y#6ffuV8gz|Qb^4E## z%Wl;7C2mWB*mQ&~F7t^p(00&m5|!Ev+TH57=`Ms>bfV+oc^Of6&_etfN55`8Ag& zPHp#?jq~IC<;PWfWW7o({$_8lwdjBD5-eUcyj282JHcZs&e^?Q=lXpSyOWSGrHz3~ z3W4bGuUZ1Pb)!BIX|zq1Y3RxM{m=~g*?+MB11gvt-AF`y7b@O>vKf58{OG(5rW`m4LH zHT?Qgs^x}+YU5Xnqj=#&sggLl?`?q(W}W+wzn)X1#N9A*1l$Ggr({lilbi#t!rhfc zgMY9KBwDJ|S`pk`sde_VQ`i^rIlg=Ld;6XQy_y(^P~Xr{>v?MMw_xV^<4{_9)`KdD7{n72+A(@x!EFOiWEtgWGo?P_e!1`uql3VAK zDK34^81ySCj#k`WZ^_u+3kmy9(vN_lu!=rUmiif6JTp2?clM5W@hbJqtJ56l{FQ1R z-LB@?e)>(vMPHB8R-LXy8;oj}@v;_lfd5k2^%`blbBd0o;{Yo<-v`Y$p7{d2t zd7;5#vG4ad$*!H#nU7h85sROyuhoXUwq73_J>Romeie7z&7uH{M`kH`??DM{H$md2 zcWY@)eiHl?Z3iHFck#ZIoc56OC+l|%pogy~i=@-|*0vFzuICs8uG1c`u>uFUm?Rof z&vzCAkTH0lyHxw8yPnCbbyM-4ubqz=!gAzYU<&5?30U8P9WoAHD93w#{DJLqvDr|y`g?6p>_){0f( ztTHL<*aE3Lk*3cXJ!7-5+#=_zyykbc`8au^Ve$Sq%6(T}pNDPOAZe2=pg%utIA%`G z<9Wk^!@Z!L{3hcVrtK;B$F0k+2f^6--+z(ue-~AHnb2~4bQxyY>__wR&x&Si4Ppp- zzTeY7&ueT=ksP-zMt}Eo^e}mOy2;Y7&|I4(8<;T1gJp3t_bj-5An3hKuhTx&{pa1} zPJOV0)9Z|t5wu~oO!#3p)Ad5Hm?D7h9LCCi%znqi|9a$dl6QTFNHLWhg$ReYDoqxe ztVrj&j{TVPneY62%5EyV=>XoV3l{5OK!_IwPpsg%KvI|}mdkoS#BKanz(7*U518C6 zwy>eD>nvgp1QK8MMH?=eDPAwtyFn!;lP4V}tCk4d(8*q%i)B?}c^aL(ADGwue9!+9 zzHXtq2mjL{`{B{Q!g;+Hp5JsL&AFMRVRKO6wh8~`&-!Td-6(&Z-$)r~Z%JQSY%gm} z!D;8Mu5n+#X-lG9s#n5AMZherB~ouEK1ptIaTttNJF1^r*z1foDnFZyY4!~vdNqN8 zJ&R$&dPh|FtG76qzB|r?Y-xjV=-hg^2k~SmJZ7aPTh*S?ldHFBSV~{`oWEkVE|`@p zTCc7u#Vk9xs;%^cJRy=Tt)vVi*H|WBOsvvNPyW2)NxA+IyqcAu&5_>B88>bUqb`_a zaI?2fkHMQai(-)RnHw z!Al)Ogd`flgWh*3W?|D$fYAyn^UiS7IG6>tCTWyeCel`PO$1vZD=8uG^ZPXKN!p zGLvtLkXE0=E^N%#^(MPik@qLorz`tEw~ zx56_7-8D0evG!{mN^sWK7+8UHskq6>g+3={pt}mMS<0xTSPSG{KGHnoUVx)rJ!Zm$ z0DX*nk}Ac-2Y%9s-G!NEHw@}@szZFeh9@5#=YwMHpP{VlR8<#*eCAvYSyy+*Jp@Z% zy%?9G93IWd4XFi=PJepmoe~oBiH<)t2q#2lcwg2n)?mW+B748%Q6Xq>=6e32#T0U9 zxq;sm^j3x2Bi!(~^FGtBOCQ2gz!DFd^uIE^s~WA_Tpw7+>WuK~Qc&luSGA8lrMLC_ zy=dgo|4H>>D*SHGV{?0|wPXDD46h#j+E!#sy92te^?BBA#mkFk9e3D0&d2LDu~WQ* zm;va$mpHKt z?pAj;f0tqt&N_d!6hFJaKV55}Rx8}$u*u`eFW=QsRM2MRvV52Zy@WN#kbT|hpfDYE zCUB(Tbr%WG_NE^kFD)oI;G98Oatg-uPsXv$(@4y>deECuB|CqbYZ&X#<8!czz0mVI zpY(RPDKdHtrhxuAcis(nZGtZhO~+iku?Xr--rFCe z8a}mhxqVyktoce+=r}x~zMBk)0*^Q|(7?*FD#;wGgWVrQopqybhGNxBOUAIVKy!xB zQ}>?nG8B$Zys))YVX@G~J@LIl7`lS`l0NlO?I>|K~fbvGQn?z zC@7crC(_*{gWBZH@*=d6xg>hDni{2nGKBaE2rO;4a-j2Ug0ixPVGqyk#>}YCnzV&| z>@rPGIEzv0s3Ay&^ap2!!h~`vE5%8ImD)ZFa_g-pNmJ5^J#`MGFor2QY~mkt{dFb3 zr#-f(N&O-$pPf6qb0|QmQwrNNt?0=*RQ;fA2e$Ai9*Iw+B>`t>V@ntC6OWh1%3@cq z(D|Pt>d-^U`=;WcGz-zUQ{`t*-7)$>kO4Hm`uk^E!5Ce04XrHO3El(83n3ONA zT^SHhN$(KBDKCYA!Z(>_@`=MHjo9a9nxdx6IKdWBdeKnT4RdW5zon67#Ol5Y9orTQ zpE_iJMxY@5Q5#Hc5%0}l0lowzvJN}xcVShxCgQEH*L$Vb6v{&Z_zXy^CX{I5_ME?)d1qqrbpB)Q}vV6-K5s3H9K9Pc2E z5>T?1*gZHb+^}hP7~=qle`hRmw`saPF|Q7z&{32@-SB1+m&=t*Iz+ExJ%HZZhVBbE zcwY<}z4FJin?9l(eWDAkd&|Gn@QuRw%7ag+LC@^VjcPy9xd${PR+}!_Q@+{hyc|6h z5u7+NL5;!?x{KnjO*^N)BogmzdM)J|iCeDy9z|j?JLmKEOt|3hIW_1qJBY*&^46OL zfeP;)l$~V1EyM?{7$ZIuJkh`TYqR;DO10&Iq<*es*n=S5a|* z@98!)pb6izeBoAy&ah~lR#jubyn4Z%d*mFTQ=5x+jU)cCjm}>QxGmV1} zR*vAKvX1XtPOk@KJ~sGDx=8o1Wc$2Y1skp=KHKeti9#2ah-BF%luz7I960D?!m-$q zp5%IU-<-Cm=JeziZ=C8qM{d=&isfAT9#v_+eW1)@gsue@;XVgSTO)*Lf97(F&ru2n z;jf||o*YmL-Uv(x`*CB5L0JNto$*ZXrT2wTA^?SVlQ0vPHnH78pkxh_a z)chV4YoB8+^Pmd{t7yFP!;>5^V>eYUdwVdOvT+gEJ2}! z5l^V=a>#N9dW4LO5Jg$B?@!?WN?5o@uDcRt(@4>7%rBA~SJ=q`86%{B^`$-hw~hQh{K61JIpl$i;1^*u-GG zV6+DP{c`B1@3EqLH`JW-UaT#}#Ui}!(}ur$tVpa^ z+|f>YO6!s_>gRO?O`>UgZ?}*0^NYqgllw54#{$^Fu(#ak$TBJk4pMK0dt z;_~*=bGX{Vk6XxcyUgfSoY3QoZw{0KF&vCU;t3k>=lB{p+UN&&?xVe3u9-X|6G>hA z92D*Gf8O;w);sA>T*ewhgRUD@ZYQ$#%B&{LeqswRUd#mtSR6XVFT2HmG|(E7@_LHc z6s^Tim#3U&L^Ns1{>{nC!_UX<`$S96hrj~z6htSIQW-?eHfhA(bhHF1!MEzLejJV> z&}-!#3<=(tNN({Y_jd>0SDmuxwVKw{WxxVdde{@4eWzAlg{#8sgmj#$ku+hcDjOi^j+G=}dSzplRoj zA;(h5p5RzlbQ10RN(~9U%g{@&t#Nxu_<2A`)|ot}&tu*g;i~d&suhb-ZF&n-E736Z z@}v&qUYJ#Y!2S3nbSx^i-s#w~XU&UCq%&|e0LoMN z?i-L9qnf|barf~>Nfj4Z)ORJ4FCmnR5#|p^HJuo5EF4{Mb-labE<7 z^g&1Q=4gUj+~qnD%JPH4n7^L`q_NsyXeM?&=9@tFgw>T^qYqXpBsU0W-8g7^mlkW& zho9NN-)OyiN9Mb+kK>AGZ% z2Z|DZeVzX`@!rxiQO?NEtuNxWw$bJz65(!J;IHzjo;4@;Ru%fSLB_rU&f@O$<@Vgo zW409@SKjU~ihSp5QoAuy0=6-~%LaefTC6%PRO$`4c#Qg(84mqv4tJZXhc<<>vV7~z zoa>Vkd^ji&{H#gFOnwLK1lh8ZwHlzT(PIi&B86U0{scX%hV14h=R5UDAGsduw|%04 zNk!(y;$vn}`CP=`{+!Josg)b2{&h)&U$s1RmIdRlb--|>`>x=IX+pOOQdhf>%gzB?V+_KGM7-(&V3HI7CS}jlcp^lZi>Xb=e@?Q~L4|PHF$1b~Y)SdVcE}sj!swrT zs2Ry{hw-8q$;8DQ(`XNy<#EBBYTa1V$ZOIW$z;=6S1z+eIQR(tr1w+QS*m7sIiQaB zfPR`QM|K6K?Y=`uG^5$>5>6412~+H>mMrS1=dM+B1;SynOb2tcr)Ht3N@Ht8)>uLf zT+2_oHOq_;W|U{RtSq0fNx|^dfIBhpRngV`ejDZh*}col)hL7O~N}bzm003?$$VrDkLcS zA~dtXjnZT(z9&qW+)o_lQ5dBDgSLl8SC6z^Y)ih;bN3t%NkUWITK>+y?H7Mm$Y*Bp z`uj&f2G0kLQDHP5&)o}%B?-ID;`s3XODn(mzh1x6CqXPG?{7*A&G)~TB z{p^xVSSmsRrMZCB3S|rhxqR#cj>fOhZ5;MQ>TSs?;qcE|g-rHSw|nZaXO3)@DD0>C2+K2}P6LxWcQt@4O#MmJ<683s(!( zj$dg4Nl_#?H)me}sZvZSluW;YNKiu4QK&qeA3RZX7)0_E<&@XbI59@1|nhNfKU;4O#HS)!pL#0b@Su?Kk z^k0`mZ%02ac}*Mawy+gZ<|a&@i$=Q1JfZqtR|DXRaI+!XOEN5|CWhh{R$jLD7TruPI=c|sC{~0;d zrk^d3Sv^(gd@@3tN|e++-PDrpgQTY7rKQ1*CPAAUy6Va&SZOt-5p-zIUzU6_L0$6N zfcHz|J8#c^>joIcuxELbt%Dbr{6wL9x+!(o`=Dc2qNovYi>ZF{sJuLyL{S?On-^8C zHtILVWRUlOA{A}b z@tIqeiBOb!;Aso8;{cH8U(iS4%i8~fr2MYR`>O##l6ip(8J1kjHs6zz zIM7jg0gE0I$lACXO!(7%S{g8gnED0WOCJD?C;hEh;y<7!XP3fc*fOy&LC@2nN2y42 z@`<$S*nG7S`|lMk?02KmQ~f)$(9r`(0%Wkm zDZm0bPje-SnswgO)G*DdyFLang{vuRVq%g*+w`G$hZ-w= zX+>K%)7Lx;|WwcksWK=Z_xi zpGDhNCTha;DuA=jQQ`gtm?jByj09q~Jioq)3%AB;EoNzXz*5}GvXk}IK;hROS14y`4NZ?OR z^O)e)Uke>gsWQ>{(;L%Yc41jqN`8LA7_ll2>n(_j9vX@;4X>y2R84G2Kp2%&7&=uH zHOzI~n%o|;%J#RK!X(4&?T-yRM3_8U%E}_Es@Q_Q_(pG-YiECl?JFx5;oN^&Q1Xk` z&=+U>I$Yy->-5e*v5oBshTb?lIPf0!*uM1$0qkA+c>Zd5ZvDx{sHuq0YA{q|HVzAd zSm^6K85!V1vc1UTPA|5CZ+|*0UuNI)0yVL+(oq}Z8#V}Np-^~f!~5&H3aaY|KtqMp|vO zb9UyxE`|XjiHV6}>lVM>MG`6mFkq_1e(`uF?yjWIx#(?+IwLGWMgpMDlyZTC64jwr zwr8Qw7q^=rPp_ddN7G2&^a_6YGbo;33!WpxiA?rlJ&{+#-ekws z#`f}PT**Is)3^RGaqie%0q4W;(wPXXpg;@~zP5c+rWB$4hT?{X6v@1cROdGB5n6Pq zx@8uaG_$kMxkeH8;?mO4KrG4T<$0oF0mVm{w1V@UDh*YiSk9{@SEQ9bV-XAu1bYY_ zZo9F#sEBc54~(iJhnA_Vh8tS!%#FU!57*TXYbgV^{{Euh`5b*m7hh3RyoxlV;^Hvt zmQM^fz}oL*usu9HEV+mp9hS|HRML+;oxXIkO^OiRL4WXyrEPf2SlwbMSXP@zuE?i zXTE5lg!}zSgQe~6jbC!z+Yp^J`!H(((XQ6|w7oQS(4ie{^HoGA(+4 z&dKo&Z8haP9rox&cpzP6Wu?2<;D*egi}x**fieUJ*}ZISJsu=zUm`e?y ziEcs)HGzMTgSOth+$?s&Z3$q)hLczl{>v;9n4l)??V+##$8Cgxwc9$P@mVG9RhUxp zTU%QPk)rD`c*$}8aYsnMKU81Y>*Mn_G$8Ll-y7I-r7KU`P)SdEMGP{3=-FX=MV-E7`W#HGt z>}zuEF!@i}gy@e<`A@gkCej)am(7T_WUpGN$qNyeVIwz3Ak^t|=WE$@2-ZQsNlTvz z1(L#TPN7h>^}J$mw|Xq$p4F3YC)0l|sZpb5jFzYJB~h&gRC=%ojIuH-4IG}dn}YHK zIPNz@P`~KF83s9`kQ(NGp`1g=8^G$WLr~C7_=8Zzs)5H@;h_(>1;^tW_&gB&7Ync- zus@l2ra!x!N=c`unfw}r3kS5IsyuAPO|8Oj5S7#~82zCti&}VKIFQ1XCfSxRYgd%% zA^6NVBQG)U@v?h+W*S;y^p#LA=KT6mTo22AG63FTHlv*{?bu*{O%u5=;=I7I|F%{$ z&UXF010x}TBMUZHBB$E%(gA#WpLXoHAN#?_>w_q1HK**;zhLwwhJ)QxgyflINfkg4 z1?ENzOwT$+nT4{5?z1DB5^9nv%T9}H6lKhmV!wGU2nWcNpGcmU*KmeCUbjn<*uF0N#z)`oh7CelZAVc}-e z9FEEDDG`fn-{nyvQ`dzC=2s8l<1Xh@o6hV-MHwl4Wd&aLb96YpNllgqnr}07<0N<9 zMp>^$h3X1XG2Ri2PlcfNsK1jIdLapGzcml!D_d|Oj-#=dc1B@A;RQx(<}No54!R*| zGHM zWCQ38MB-yWzqjRaR8I2q#VkGY7KU4bNrep9Gt}r*`zS2HA$&YQBC|BEze;yJo!<1K z9=-s+Kx$VbVjQlw(@7#@s7p3AqS^905~kP>H%vmQ?~%t7e6jHOa(mECdL$?%Y#o$l zvSXL#i0f<}<8CS>3!tP#IkTxvm`zsaH4u5tY|5`S^1eyRJ^gN)v_=xTVq%TI{26PC zD3RfNg~w zuUtX3TU1k6SjlXurZb%wdoaW-R2OxYxjEaMH#!*wS!t>YrJ+hmaIj~_0mjnp3Yi(U zXP>^f=BwScp1oS^?{3sW=d%w?TxcFnv=rO}E#Jz!QM%`dHpCTBO5@(DVid>v3X4g9 zAUhI0?$p$5*RGFP@E2Bo@FH*5HrY{4F2D2xR(Ix>xXE;A*WL)}J+3cqsOEw7Pg>O` zL$964E;e*hcO?(+h|SUZcFi!;P-xz}>%MWcBUQ zV$LgX#NbsJ&aR+Fr$oj}q#N3o07?vF!FmI%L)sq4AxnNfZA!FBGb|5HVnUlMWjbkI zJT^iJk3+qhUE&_0*-o`NpEsX)^yu?)1gF?UWB@5<2)?uznoO##zpfYF`7d5vSHU57hu%nu|4)urbJI zWf{^rSIoSFd(WIyfa>)gK&I|DaK<+er~^E!C$?C?P~tBU?d?K0L9JB?mB0`o{64Gt zFm1*BDAd(3Wyz2FwePWL#=qt9Np_t%$&ADvXIKPVDBw65O=S7V0H^k%Q3rq8ox5T) zQnL;f8d=jLpy(z*S-602ggjF`2t1mDr71&bHPz8^J?x^dSCv>ybj=SRSSrb=PW3}) zh$rLPpWC6*%2Hf!h5k>0t=wRFZxlY~jj{F4dtQo%3nFw1vCOy|&s34l5C-x9 z(nSwHm3QY*+B|;Wbm+BZS`+xcvwB}zn zEer5?PpxV>fj;nW`Soh-3YCq-wtT`i#7682#0V&Lvegy4Rn`6=$cHW&hN+Zr%9c0^ zsbyWJp|!`U8wB0#nQAp`$eIQyt38OC%EVJH5WC8#cu8DGEJaPhxP1oObc?264a!%k zpQSo}jMikljDJ&{&7}tWhSr`z50*5wbiWtpu}Q@4b2rl|82+L^in@Sb7v>X!Gwg{# z`VM48+2CaehV%52sU0-5Z@ENOLJh&)|9sY@A?}zQO^>g5?1>=K%J%W=CZ9)8O-Xv(Rk&ZLQg%O}7sp|cZ0BQjtV-Em? z_sWV$JcvI>PiN@)cF77+U(l=2Y>nf)W2=;8!!JMs%k-=u!coZLgbS*aB)rswIngitNzF0Y-$>_E)Dh;8%A#r{;oT4+Oi(e7;?EbRL7l>f{Ew;xAd?*pyuBz_UYoz8$V^ryn9~QgCv2$2N zGw&wae8-8i!AC&NGF$>vUm_Yqg9Ocs;PcIqVighvdac6Q!;W`+A>EWn&I!NV#9`dk zlo7Ki@Y-7iNN8D8!R}Yu6a1y+%+AcZ2^(Lzy~#~g*KXspyy*rpJeAeRca`lbr*%Yb z6y&foBup;ZSKn~zr6=#+@QRA28hpsw&na7H_i$wQS!d5F1_Uu8PZd#@>(MgpP0u_) z%HIS#EVexD5VQ-Q;E{0urHDm3tLd|fJZ$21PG{k2veUFt1>JzqBTs#aXG#G~P*%ru zPw3W;`0bnaUGG?3-C>dQ>*-$?4vTvlb;9(0o&#<+L`ISdL@u6Xq^&pF%Y8L|&?&Lt zL|gturbTn$3)Z7G>$83{aJ8{zT6j2Nl>T*L01@DyJ_*#Jsl$H2D3}i}DloScix9hjFD*}y@EH5Km+SwEjF^0Sq z$<)%F;JLEP4ejXMDf2ic=nbTP?0Qf4^Maun@K4)=_&x{T$_)ue8e$CsStG;5r?M({ zd_sH`1K{sZBNP+%N1x?imb9X1r?gQ~abyBzt$uD@q?g#32Uv{cSU@me6GqHx;!42Z zl-t8l(0}vawR}$=G^m@P;N&x0ZCknkJ$i?;ixb|8mi3yn@x;Fk09bIxKO22;dlBnJ zd@D7vN}Dn=pWH@3Xm#rs3CwCAY+i}1FB zb23tYUE89E_^mF+D6n zSD;9#SoU;ieAk=ITmB28>Fh^X+VKU`&M!_<=Ew1a{UO!$)WQf7M1+}D=?A2YpEXOU z-eNE^l<)P=`?URH_lnE=+EE!gi?b>-c1_Kyl0t+xL+?;3 ztB(C7uNI}8GH;I0bkA$A3c0a`oJ#$2u!`$jT|8ewM-9A(b2LnmTB{k$fTHvH}mUaz-4$1*rsnWl)T`rMK`pa{#1Azu-x3MQKDp-`pkOKc=)vF`#cRhQbZZ!6SWrxeV-N6V{?AHg;^KQS%EU%4Wf4R6nqMZ|h**BRG zO_mfFr;ic+ey~JWo(*~$vh?l2bj?HD(&lEtiiQ(so~eNf&kLkWZHYiHrzgGaY0GHA z(?G%h!_`|yRrS2_!l;xq(hY)icS=cj2y!GuO1kq9($Y$I2nR&E^Uz&Kxn>gVvuE!!Gkf+t&nIRk38L~b_j2|*S;=VNLJ)CnuK+;Yuvc! zSfy;ym*?gEUG4<~!r!k^)rb*?hWDlAa3zUXY`BGn-;r#+)`(?vsitj(}nnS52 z5(chrxQPR+5@#EPE(Aztq@HbfgAA9#`m0z8k26M+Ssk$67Y5Yq9RI*-%1r650Rd+x z1OKFuXyREcGZzouk-%q#7J$tR^$6dWThajCQd|&CadEM1sQ=^%GoN=|_Yvo>P--dI@Jgol=lbZW zoZoy7gv+E!T(Y?VsLPhvt$H>AOMW=Z@KrLF?9jCr+YR^iklHGdb%4#vqJk|bNbC#`bxahI0hVr@rshW6wY;Zhot_l&DeDGxiF zv(r`*uE?u~Zp+A9`M;|l7&1R;Q-!^bpDMC7|AkWWXOd)LV9cShCGy3>YDe+|d=8^@ zF}?Irv7~mt*#a-9n(Cf1#(S^RL*PM~u$HnUn+SXX!@t^qkE~D~*g4p=JB=uRb@lz# z`SJzDxEAMrJfB~d(@}i)MuvIojU(k5$zEI3lzUMG>9Bpt-83z{*eJsjo4`2R5$^`m zC!pH?2b9_4%A>k3)@o_)xwt{Xe+`ygGb zk#tX07U$pCjWjhX9p=dm%{)ePCGr6c;W#>@xlcZggns7y>-~7H&ANhj4nn@~TZJpA zK&jt**@-EjqVhV{+(D#c!Gsjk|nHqL!NhC}77 zRm!G2F%XNnCBN42-!CEt84Ip=^YxThR(6~k9z8d_s7yNc$G)SGC0n{IIP4bc4JI;d zgFP$&%Mkn?ELH_o^F}2$;`Vp~dD7?Q>>s)u0h-ZuA&M4(BdZ5c_xbNnQciZVWBb-hCwX$Q+SJ8<&_AXiHxJzLo$ zU%bX+w52}PzN(j7xkEozP(JKu(1BzBrGOL|yxW?|HR;K9kIvTY7KyN~leDg$>Vj~u z3~(nzWGTVH*qD}Cb%~$sqO*mSwkqU81_%ChbC-^jUdgQ3N>$s7OWG!m;Ki5#hZ*Di zMAz75+U>4)W7+={?fOvV*JBt-X~4eLB-X4%Y?G>J?UEB@E6{tI7A|KQM;eo%!~Ah; zJD!Cu0}KDBi;7I$y!}}bQAux9SaI}kb$JxYSjiV}azT)~@~?;0j!-POhG^@$xyKri$4TEa2-MM|A%F?w~Xotarcq_lF4@awg`)Id&(21=;;)~H4h zd(t?Z5-MQb=TSWjK6JRnRyWdDMv+{lEFGs*C_vrZ921V7%Ke~hHfa;Nt&?cy7Hw+D zX}}!AXC|7|@3Gy8GG-aIPj=xcCxcs;3ik^_O_M70USzr(BfE57&*67pS~ky*FTKTg zV+aiCKa10eVHt~d;Y!_uQjmmZ@YXh13nn=J^|7jdUf8ZJkMe`kNFKSld#6=8Zp4Z9 zqnUM5KULZ_2zEYg|A!duM7K@-X6Q3%YQuL3tYu3!p=QTxm-?$}eCf1NO-jLgt9NP> zyQzQMosgY{_6t7FiUo0&*mHm{N_7+3i*9JMYbKfl3HH zzq5R`s~aTH)o*`a<|iiL{r5$>sAsHHjzppM&(SZRN@x$di`iRLMPN!s(6sDWudr>F z1UZyv1Bf%K4Cm@tk#D!D6{A=zBrR=XIqU4B=~TbC2_pteS^e6ieM~_yx$e_-aKyIX zm%vlxQl;haWZEAxmn9I+xX24YxL-Up4H^DqeOx^hxFaC)WvC>vAhQ32 z+`~#p?soJLvNUh#-sb#c%2`_x1E1u0Wie z^OspXq)00be>HtR4flFz!8CI^8WQUCpI> z;Ea-eu|?jSB^Z5~DV5V^ZS;tIv7ffKFMh6tk1k_Zzao8iHE1-_5IXWk^_5-YvFzOb zg;g61P5U=|Xl?K3Si|x5E`~pfE^Ee#aU~6Mrm*k2ee!24 zl@xg1&Ew7Sj<&V7q{?;X0~C0*Ox07rIHhJM(0>>7nQ|_!znU&x+mZJgb}2|D;^!N$ zF>StC+!_z=M9olHn}*8^c-u&u43otkdpTuz+asU-)z%YC+Lb5HNgz4q#4vAsOA?di zaI-TNJ6Y`lJUc39Ha%H&MmRFWYyPx0X@BbtjOjG%!PozDp1=$@qU47NEzjBD{XtNO z)$+J6A-wA2O!;xzK^WoD{h?phvgaCd3^1FW8TmEQQ8%Rd z7rknHRi8e;zx%D*eS17MwR?*lzXg>GrTMM?j%X0kejO*)n=iw@`|%+EZb?qP=Y5b7Bwc>FHsRaqG|YJfw} ztjmYNPjuYoZ-<2hyy7+S(SjeAgQar(O8fIyR0mAfLowaB3)U;l4L{l|+tmLm%oWRx zdxsoT!qeKEn|gn^Slz%8*q`rFP!Rv-o>&qGkw%Vb`+^+B{I&1O?!EUFI*X`qmaH3L zr!Eenq#&8i+!Y0nHp3l%we6_z2oWN1gR%PxR1PI|*?Zdfz>js41=}sTXxo+yUk12g z#>a+w_d8HeYSfM!E>cno4;UQl6>|8;mMPPGR!Sq__LaTYJx?v6Mzz4mB^YGujWGai zAi)?1Cy6SqE9hT4WiQ8L{_QDzuy z*czke4QcOhq{T>>Y59K&6&|BMUPPgIu7@RaEG*}PGzC)*Sb^I`yKTabi{ho2Y7R`U z?E_oO5<>s=1k?CZ*8B>E6)&p*#k65Vj1mW<(^?X#wo>KD(`%Tav{|}(AMVzqQZ!2r z)3+|JaS_h&A1BCA4K=R{tEqjgw5tJCgGy9%<$%&SZY&_{#zl5V@)tWIlOUz4S%7R3 zg&>ztg(uXm1rsHB!mPKhB4ASr>7U7F)}PviVB>!bFWwka@=GQ%ym%e2d1ce<1;H=T ziO?+Uk|8o$cob83#ol=Jn!WW7O%bAb^IZ4nZc99COZ*Y({`9HyB#HB9ZJgSJiJbnn zJ!`(CQ;@3ouzoUF_E!()Md(N_ZiZa$oaxklbZS2X>unud|FOmeL0zw5f9g}BfGB!x z#ZmaVM<-e7=BkBQ1f4?t`k2yHU#Pfpj(Ya|zSBxD(TUb{Yalv%^Vj988l%n^9G?A3 z4JQx|$fV6tz##jY{nF#KzVmra`lIyL3kQG{0{kFUx%%vDl}J)cw+|M%7@HNd&2BH^ z)@itL^&(#jb4OOEIld#bKzW*Vj|OyG!t0v%OBT+9IUG7M1ss58EL~jG0c(~aiYXj^ zJCp~F@Ca?@ga;A)nOJaD;sY6G1m+;j$eDeL#SU85Z=+{YvWzELy57J^;c2HTXUa`6 znpQJw%Po{PU17XNBx+22a-!QD@OYB-hBmlchg;jaR}2cvaW;`O zsLvOWk>X>(Lp+%e7$f43p;rk;4{u5vcWd;0h9?(%E)AqD6Er{mEE*9Cpq{+t z#JKe&&Hds^u4|~d*~?yXRFL&siyfIU!_KpshwtT#Z9WRO#8fex8j7)e1 z1mt|okRbRQPAS)Tr{tm@-@Cix-f8&f%EeSRHxIqng<@@yLo{E0l{E>rs_7Y9eN7~G znvyQarA8C}?dP}dU>PiO!M;%wxv-z8hhkM#Rrp0b(z!DQ5zf)Gvm`d3U*^gSAJ0;^ z4Jk);&53bf_qmwbVc{WK*zRWQ;C%ezwlGoXpV^-wmvvv2FzNEP1kW_sDgxW$xHbY^ zdII^(lo9qU-(b4%NUtM|dp28i>i1f9B=qu%vm^AqH~v~wipTO)qX1dS8fYhf(l6Ju z^vzm@b4E8x)zzPscp1Qd6$0hFD()YI1%X3sJYk&tiS1^<$$n_e8v7%icnOydrgLM-1xnRmwjlwo3Lmq^X2wi6q? z|IImC5hC>o1H`gZ5S<2HPtux+Nz!_a>e95Bj}vm{ zCFTXCglLAT)+yDInJCK>hXA{@rB{82lG|*K?z=Sw8TWN`ybgtlC8^d$@P~Z1JnvrN zDVyo&c%DBxcl68ktwHfL+&C`C3z`{4y>HNbiD)c87U6l)zea3wQH{bfjhk-LU|28> z"r&^F90zyoKxMXz9rYrg&yciD!aVnvO;D)vtDrtOQn7zi^m)Qb~2J&jX#mq#t~ zx%JLR)bQ$D5EHYoSd8?>WG+jR ziJ9MR0xns9mlwT?xM!O0(>xaOtPgX(L&%spnzDj9OCEoTPV8EmTw7vFW^Z5UjF-su z2scd^O|KVD5nA$nB;IiJJ*iQNb6mQFG$ zx-Ts&HDP+t=gzO$Dd1({^5^{=TNctADKyuy0H?mDGrTGqZFO9;`?(VMqr zI6hl?rhKmGi|dU!txSSO+WXzk1DqW~xMo8nSIEYOS*9o#Y(rPye=5}(AX9$%OylxpQ(63c zMp`0YTrhh~K4;-E9ut36p@eofxW=W%4(gDIBGG~bA>zT~< z1!SQ1`To1uifB9hI4 zSFsfu)a+mQ(5{#Sg5cCH&qmr_Q5<3!CiMw9`1aV12<$^9!p6_n2a)iK7Xv@&k23|2 zP52YpnO1RgXFVBrTyB#M_cUKn9idgf@6y4wl7H1Rh9qn3l4^Ih1Tf@X!mmWwl_2ye zC{C0EGEp_rrtf-;zb_SOczTSrBcPZ}r{`r53_>zt2zmg|hFWP`M4 zf-1m``H{2p*T*vjl-^=;`N-yvKkCjU zS{qho4Q7dT9{RcXTaD&#r!ikJ?#ZnGLjZPb!!&3M$69_FeZ2<2v;!7v|aUp zl#cnEqPep|$VI-zO{^u5b(&ZvAB^_ZxXm3Gv9lbgq zR&MOC^(;o=?ZGXn(aqQ*u?b^>qQ#PQY*@c%H-|nni{suo?Uo9z!4kg0@{|GE^NV2* z5O?nFv3~bLXuh1Vs(EmS;7?lBh@+L%@>^P8uPh`7{oDSNjZ~Rr3 z$Q2WP!z;%Wb6J$}hnEp?ugr>c(Pqv{96_V8#XdhzXqGsOUnok{pyrpJjc}ymJ;=kM>7_sasgv?lTXR%$njAIa8H zq96^~kTefRii7fiWy`e`pJ2pGnASfU-|{F=E=#$5M2kHS1tul_9w*44XB**o);p9L zDWi02m5TXfGimB-EiI8#*k>yI_}{B==cjtM#`?*{>X$M%{#2DVa2Aj7ywZE=?8$dA zYhI@xBU(^Q@jUkkPtL0VIQ53;-5q>d$Mf%KvYHP>fw;J$8!eKg0?otQ_yn||zZrM1 zlXpiC(NZ+ARaR?P>>S;{xM{bi-xT6j5-Q6*Et(c%GWaynm8}DykogSOze19quH3~v z*#W^Yw0G0QT#1wg#`}ahQOftDj3A$KR@5na>EfJ9&XbBf8CM0{gDKNI5)`>9u=e2W1?M;- zk1gWsEfj0ZA?=H_s%}U-A)u88xpaqm$jFKj%zI+^n z7xB(1dEaFs^Kp42N#X=ibk*|vp0df7`soEbV`-03zmx9O4P?ZG3_g&Y-9XZcLj^6i z-7Lm`)!<=hizJ1Be&U)1^nImbHfW8 zSJ@(un}$Z`osO(&H~N`#Trk2(=FQyNK9Cj3_2jh&xcJSSJ!XLyG1P#?!2Tk-9!~w5 zEN~a2>6+xJAHs{vYTet~+wCxrXJ(kmQ~weJqb`IhLf8fpR zBUx)h%_p|{m+U+;jU2hIoz`L*q15U#HV+kYD?AZX;4BF3;tN+g#vfD6EY{8ZJ1qrk zDCwb6zv{1=zbj=?a%qhrl2VHOaM;U(ES{zpe~dI7=EsbVPfZ&h@o>m%WBmQ5dsPCb zK=eq94Lx>Z4+F*Ig!1A&-yoZSd9x7qq|M=oi{BtG$h~AU1zj3z2$7X9bX*9#TU`Uy zxM3%BH%luLKVzw_EajOYD#5g&A-!W}>z#6j>RuuKL?PxAuV6fHTvDKob|o-PxgLFM zLP(l8>%@7S!3fMRsPhT7ws)JdfA@{-H5kkI)ZA?2HR&8XTJCYD=!=VaYrQ#)A=$)M zibyg{nnUrtC>EG%;`k27o-}IgQDm(UMn2yrF>(HV!C|%0-BL;|iPcY~v8p^DEgM`Y zoI_Bo-l$aZ*xh+$srI{h#L^8LSCVqcfFCUJ+8Mcr{%D)#%ktYrAP)$Bhmt~<8%MS@; zj@sP0TR(5g-4Hr_8TfU!!6#j?B2v?90fDNxH=b!Zt`Gh#)Jd}Bfeq-UJ;v$~fFa=H zNZkmxNV<+wD^+LK{USU@6#qn0>`~W3kI@TPcx@>Ty!xU#+F>amze0j9xi|3W)5rX7 zmCsSXZRi%ga^K(DSb5MW7qF9w)vmf5xTec4J0v2g0k1*i6@*$<)S003oFi+^#C>|F zD`YN6PXf3SP>wf)+jFSeb@@2!k)9uw_t8Eyul0@AMapK%;?ED7K#>kCzo=GNmlsejkIRVX>xKgvs zB;!pQv2vk)kC0r`qAZgPO*mtsBAMax({zd2(KCuBEY{0|MO6PlP<1z1;_4N~z+0gD z@8W!l3Q6k{xO(cYtH?+NpP>G-$Y_VGf>Ay;i0XcnMQAJC`RZo+(8Sj_Q@45d{lss+ zS7`_ik993FHK2`Im@of`C#BEXt?1R?KtxGul4v?qKRg0X0F~p5h?W#H}2}pN!u>M&G$F(Yk2Do3$WqU;JI08!zL)x^|(ri%P&2v z`9Sz#g_a7)v${XyLI}!y&#eW8#9fC1ZqMv^7(zq7U8h zO3L@L?vXaiJZ||R1K3w7M|ubb(_+%CKZ{jr*suL~ea2RIblduPtM7f>?Y)X8kdv?( z3lLJyFtH&ZqwrXq)IB{WvtOSXHDTpo1H|$WVHyp54Mu!^a#QIarA*NP*~_Pq;~-lr zJg4As_%h%tWB5DDJmJp1UJoN>@rDx7{I8FAcRka7E- z*0c=E25MHw*Udx5choada%OE}>0S<-<3V`8BSXks>I2R69< zP9HAE?P>%cLr1p6A4U#Ou3>7fpM13P!O6vnO8-k`%5B(0o_&>i1w4RsT zjc2&O>gDPoHa!2WJGy@^=Tvs8c8%xWXBLhva$dk_jo88WVvv1B7e zv#Ijfl0x0>=IoSlK5>h2Nq!|cHhNz-QT&$qACB|=Ph1ER3z+(|Cun%Ns+Z}e z+9lUAlZ|hA{I7oC=Ul!(9*Jm-olO9suay@vOQ((ZPYGtl`A)5@n(M~D*0ECmr<&~H zdWCIS7>V=}QGGehFcBL-3U*d=E%{ew@Bfr>iVWFE1de|8`Hqf7cr~G$7LN|X_?IN9 z(fsc*pYi`32xa;*uJUmdpZJTw88c*ko*Etm4fITh za{fV@-kEAZ#poaoG1UDp)+)+5|}ks0X<(X@RC2x-#HrKd7eTpUjTI*HUD1C!5q&ZrGSIAE#O zDf(co2Le6M)+vgDHb3=~WU}%L(o+tp}|3M|2m4N7W0gWMfK#3Nm! z?S0^B3-0UkRtO@Yqv5kwZ49k*roP=8EKJ27K2In?Y#;={FZhn+r_t`sad8Z4mG$*) zBFpFOOP5QRpCfZB7XB729b;{pZ+UFDqcq&4T3@$U^m>j|WdXDQYE|8W6ut~`iRX0F zQa5dkXuwped-Gbx-R$j>5|EgfomQqh3vPvo4itQx7LB>9nm%5N5JgVTqXqk*wE!}Q z?0nQ*m$PME`Nz2^sr`EJPC8)oH@xZ~I_bjB{JRoLmA`0d64?%pL3#cntHC`*`Zhcs zJ=JhefP8ykZt~WqMA~}PPd+Olw-Q%}#QW{HrDE>i0wUMfp7h_hh6`s^yqx)Olgapf zq3-d6W|7Dv;-6k%C6-+a~0v3a!831ky%LqkIjE7%5Y7UH)gcF=PjcJQc~=vG70 zr{~f|vw1|9Pkp}O6nr`&lJ&%3WP?hM5B>J1g@CE|JVCgJn3}CXv*fP;Gclw*45H6& zQle52xbFz7yv(FV)tuE~1{+c`23HkE3m$&!bkG%5=I@qhdKoFn6g36@NWTQ`VEn1i zK3<|d>Tg^ohv@{M;-s)R;xsyu%0-Vc*D7eO&+j59kl+?DrU2c3Zl6D@2dy`n0#4St zBgCm2C3u-WJH$ADb$`61!HUnMJ1{il!b$&qgs}lP4w(Pc83U+&4AGcIAoum3D!+v6E9tqKg*b@M92fAQB$C!Lzb@H8Ym$=VL)(3t4Jk7%V>q$89Z4o9BR-JbOzh_Lmx_~U!YWhsF0|7!H zNxbiA!olZ@DT(t<$>2Ksa#L5_eZlhK{r$ghK+q;Iu zEfp;Fa-wP|9uB#!iP8Vr`C4SbbFre=O)OnEy#_aW6b>B`>#DK$rj0<4@IK3>B0;6z zIRZLH%DS#_KPvq+NFx~(a{5%{4$WrSa$ z`VFd72dARnd>9PI6I2t;h`|NO5EC@?UU9kc^r+H5zgUP^H>+V@Q)_%5ZWA7#uA*6+ z2FGxPBqRtYIHlOtNf%9ietfu}fLs3kEt~7FLt4Uno%&5zSDU3nN0~DJy}B3Xhi5aUU_wO^3LFh zjesIc>hM%Fm$!4Xv-i!oKvj8tpL5nJ`NLw_12YOX<$6~bUeGK?qF{7NR@SnMUPD2H z=8n_JdVd8%5;);buaJkrst!2WMNbG&T~rH{Dn8&qNvNmEOfUc`G4o6$qqjjX`C|XP&e9 zsHts$HK~XSPu6tX`8C3bU1Ug#gaItR#D` zdV&q4*(&PCJ=got_Zd|2gR0{Yp0Mp4uU_c#b|A zzhfWAg{DOJnF{|Du&%l_b-E}u(UNYzcKNNg;DX2galR;iwH82sn6Y?>3%>KVfS6tb zf_z|gI)q+2T^hLORR7d3RcBAK1^P~r`&5Q^quLy9PBAPF7hcU`R~8+H0}Rdg7Kf9&Mb zT|iX#3K%xejt7|F5Vw&PD3!2XB#AKpjyEvsE!)}EIARr}O z0s~;hnD_$o)1lb;hS@7?CSa;ZL-JZ=&rn6d29!wPbT9zh7?wpy;S!($`_yvLg8onK zBysrvi{Mj27l3{6AnTe0E%>=^~H$B2p%{t`&>E#uM<~h^!u~o_|L#81;D0(AFj`; zru)sPPY{sJvft5vtxmJVs3y~Xgh9jSF!y_SisAqDfg=CLb&W!YTI;Pv{b393L=s&8 zam4+E!SAm3O9?PCn%5`}=?=QRZ#LDOhij&}#l_VC2c(iwW13UpkhRgJS2Ts#fDDmA z*Os`u2(Uke44E_zqI5`^b2Oa-{~o4};t#r)Bf{x5Gj#26D0E>K9P>J%q>q+ts6v4m zJMB}PDa<|OcE3lVssDJ%q~@S5?XC_Q>1?N4O}3T5+VF4S8H@lq&mjdMGWy(JIs+{o z1(4YA`?8g0fYGezr8HS?@fjK(afFv-ZA6Q_g#i)7E7NVRdbQ{OcmY<%uK=4#D4+rY zlJK89}O0%#cO>J#5n+>8L#@$d1ridxzrtO>Z>#p$}RH0s}|w)log);c1IFH;R-`^gAwA5k>> zQ=2)C?xivDn35NQHZwtS6{5c+Q#T-K{L_BctuHIf%iEr?8UXy!+3UXoVrJyyA?(5C z*NX&j=7OwPD`R?<4|Z_m!8kT6{fT@^9<4|vfQKUoyDFz&bYa(#rLpzSz( zxaVS8!zGK+#-`fxP>ytHJv_q5)Y3AnbBkhA{BgNe4wz?Vt<09>WdNXSBCyb}t!QDG zieQaL8kHb+xT7rAN+U}~%5n}^A0WI_J@`Lg14)u&Srd=Q@kjY3y8zCg)BAs{_%53> zJr2WuKBxRL0Z-=ax`oq^f{CHYdzpY1f zI%^jSgf63%ry(>sZKzcE;irC~!=pfJzY{bQxp^ScpoDKOS*7H95zwg*4M8(thRO6&0s|c8cdu~hrFPL|PW4CwO`z`c=)Gq;FFfs48cRkv#Pg>F`AdqmHiRy=_{p|O z>iHqs7bJHr{x_v>%Jw@fCmglu!5OabKK|*zJK_JPVklr?6R|4CdVo$A4wH$?gbxBSsa`7fIF{^)@n37k_yNM`Jt4q!q+PD zTRy4OWx1J}a^l0L!@GU)jq5jXh3TqW{mavV^tVe@5Q>ZTyAvPZxY9QV^j#Sq($>B( zQ;J^f%ADh%^QlpAj8iI|OIHRlQStF!s!kH-lw*mD#rqyw!7cg5==!)~k|V1kc(VCG zViXlqiM$v074y@pY)WC{@`W*?c@0y}!7<5D6`5A)ax6XJjjql@srx(8#8r@#5ntYP6>ZG-|NAd(*we`Yr7eW z^swdSuZ5RcWU~L6$&|4So~ER*($0SAnw)9g%Vq#^^_zRxJ`c(1Y2tl;@I@FeUvj>B zlsI=(@2ZxgXin$yk7`t{zoJ*nOU#kRmJyJ3 z;gq@j>Z=+ygp;g{jL_oOBvt-g<9>1&vM?M#DvTrMBzbzISRzR}MT#D{(&SGB^ZS6mK zpJz)=@tu4$kk3?9hvzUkrSKKG_I=+ppy|<(T$btjr~WGQsfZyIe&1ugyQbd$qVccK z9AOMwMKj&)KksIT-IJ^Yti6PtEEKVuFMB%UMWuxSOQzoDQ8fQjZxfMAr_;K1hWFc( zEro=1ONEF&ld+3!ed3 z@K)g-$Ef95dBsPSO-YpSRplR>Qr1IoUO@yd&2CxiNSV}^U?FU>Z$I4;@+u$nsRAJC|Ex5WLv!^!L#SrOw{uDYMz4}% zwrs$J$BeY9Dk}QnuOv$9uUDgnRx#5Rv}id z|038fjkMPVa-Mg{Hggz61{Ed9yn8I}#=3;!a_lV22M3Pjs~Ulj7ocHflXG7ToRd^8w#l#wJ!d-cg;G zjzUDcPP5_N40q55PLg1(A$LB5mKiw`W68JqW7GRG(;IOQ%ajL3F})tVb1o$!{6H58 zmu%5PF?!Aq_MUt5WRw>FIc(FaN`Q<&Y@n@QE=eB5zTX#<&7Eu2;Xg?ksafBpS)x-h zGHO|;-lM#?qC9NR3wzm%U$lf@CK2aF*XbltM}j@;xg@R5`IlyAmdixnQU(Vp1(%OC zuAKO<7+zFJIr%I1RjwpCkl|N%O0t3tV2$+DpF;O`2<62nFa1Qagc|QAf{Uqxm`E%; z-=Bmr7czr%hI5zMiTQ}jowL8=$Bf`)SY%9&%j_lJ;e7nvSX#o876g9^w7I$Yg+J{> z8p)>Jg1WF-rk1+m%BxUk%Qk^6@7;dqS(hUXZ2e5tGSvb~R-4AH35$q{&!V3ZG|N?Wq>)l?pP**mGT#2c55>B)`4sSDDNeS9vYs{T3p~8Wv2}zUDJ6=2p9pm!nfu z`O{QmR?LT?>g_P%c$5yfO4&>@R4}i?F3mAX)QbHw?M!aOHK?4gc{DTCP-v4>ryO%f zy#7nqF`qQX>KD!4Hc0ahg*25p%44d{^GT{}Hl@5H3%(GOjQok-BvOkPtypuT<6FYf z7G>LHp1GdU_zh3A3g;*DDQ>@2I?z`B?YejV+&$g^NK8QF&!LIk>+B)RzAw_V7CrU~ zNtR_U-@uNoS}g}o)6RD06X&fK+`7!?ZmKHgvPcEUEh=yGC$4_vxM-b_>z@`um8J*h zwF%?qRZ=C&9GwfMj8uL_QoupYZ8ne`DbU|DP#iHj>$r3+h zGhlgwwaEY%pJtlnj;c|Qm%5JCd&-d}<-`2Nv zkA+?fWIuXA)3q9Ff=UJP9V{{2mJEiW!hD(nUW;k<0{a|3NfWe*o2Cy_X=CMKtETz{;o& zJt7+R%5i|9Ug6AHJv_P2s$4NDk(X5*J2m77jAKH)pk-YMqJGL6ahDCMaE6UO#W z5`XTRsNNPl%BNMcELE4#t5p;Ygp@ww?@Tp5)Vx2@jwNGqrr?UalW#nZdN$n>le-n| zn^qksYq+^9c6z^+-yM>q;<2^yC|H zn*}=VG3RS+NkN~yMbS5mkZrdneW!j%^TV#d-{Q5+H1U3E?6>c7)a93R(3;O($e=m* ze@-6U&QqWd5QGl+6o&p#h?Lau#25%!u9If$Q{X77;khkZ)YRslU0 zb>!vx`2CcwNR)?owZ-L!Nn~~>Wp;R_EPt<+53);N0S|7vGh#Oa9%+D-1WqLONF zR$nAr!lu9ucQxsU?gC^Np~tte*Gx7)eJO4jRD}6AF7*lVYDK6Ni_~%cBt>Z1$s3K0 zBcJ(ZtL{`hi)$@gF53U>I@rG>qFd~)i!Oc=I`zVf-zmyVGgTqwjOE9D!K77&T;?;z z;3+f#+pnKe?JR6Z!1U~A6SL=PMRV16=cGN+PL)&a(U_$_dkS-UShWKP(M7Q7_lxw{>@!oVrj^M1R~SU=9I;I_e|pr9dqbVx-P9H zL$-JA`@9a8KmPfX<=*$vp_XOUT>fu!-Hx=Z?()IIAk-GcJZr&$6J`2^xTU6$87VA@E%H@NTW5_lgyl6ICo5kTzYd z5_5L1Wp!JHD#r%-++H{i^G!RgsZizWlsN6z*srTS_C6wvss4zf2yXaRVB|(@VK8N^ zX(_Jmsf_>k)wBS+lWy*1bUE?SW7Xx7qo%%I%a*`v?lB8bBN=0KHRFg5GMZ=4Uor$L z*Xb8QYR7rTp{Eza2Q5BTRXnG&i(Qi)p>Pz`i z7|xe5eM7%H`l~^&H6GqlrG)p@JHn!zW1yFm3W=k+jRw85uj=(x7ubm&%JGb%lfKxq zp5MQ*f$pFErdHU-rBKlLu1T0Ap=0VD{8q}A#xyjR)9Tr5tvYnp&SkibTZZ`~TfZBd zW$2(k<)l`R%|M?3YxA{Xb+)<4MrF{x)_hOSPiFbAr75-N7PFhFX2~AU*{i>0dKiuA zCEz5sZMeVge+6Q+t4qm#W)lc<6#ip^{OS{k-Y_P{gEeHYQfUJ?m9~<3K03uF?%V6! zUuZfDBO{|K5^X+@M7BzinPm1Kr96fZGyOo{EVQqLQ=z5lIL`}UVU**zFFI}OEK!?< z-2O7n5y`dulx--M7lbm8`VPBYu%c(dlvq=PS2ZP2SGhVV?ZLlrzy2JkuUZHap)LHH z5F|kdY)(x_8D^4_xX3_*L)eG)GHKaN++D^fh^deL9B(&VU!@gCFdYl$QS#+XNQRJhB@rz{GuLX6eb)lucs zMclDk#lM!q@d=XtjE?&M{tcWDKDZ)Fa8N{i8zJ!n(OjE4C3z`NJgT1NXf9#0Q;3>;pPw{6B{xdX+Q$)w=l+De-`NNV_^VUY1h_VmauPzoekuAF!=l8J|V(Akl_ zc^UY^m~23b^HbMri5a~m(~B{e6a*$|Sb5vN)9ck&zju-MY~uEdUq;GzXmDqs`cR8} zcS$T3?2;aNr^C6kt(P1k)5;T$E5PMy{iW$^abL0{-YeBuIt6)dYC3XC3LZO=;SNQ_ z->f)^`$tM)Lp8;HFJQn*tTM}1n6j4}KW5^a?`B9eFImNm|3AjQGAybudRGx7r5gl+ zp}V9(xbglI|{PL2{^}rDF(T2)mUw z*B4{NMUJ5|)&>N2msr0f&Ws4&uH)wPQBu7YlP<@WljsFkiOc7TFXV<5+6$cn1IUG5 zz|>n#0vtRd_i1k*K#0-Yg4q3uDbGK7jO%)Xf`#^d`SqR0AkOYLvMBhxZ z&IICm5A{E?CJ_rlDa^!s?-lqVn3@BHqr_$61!S9Q{-5VKdwg9?2ZCtTUnk6X@qbt> zEtXA{`VkiVtb-s@RV>@XC9Y%njgj;38E4H8DuHodGS-{`+n{EAxJuvd%iRwvyjyYy z*zSOB?AKbD5keIEGXa8asK8U6vH=*mJ{%h4&NFq2oo zefB0KQ9%m5_ZcA60V+IKU1H8_rUg1go$RO@#*c7}gUXX2#T#Lj>d{wWp#6EIh%!1Y z1`MAG+)U2LjEG%wQl7+Q6>u!}=$~8NkVGn)urLzO@?0fm>NmLQP9`$6|GtwQWXoO~ z=Jnzu#D0&ZNd7}p)9-}^7gvoA7<%Q1QzwWJHBTJ#B ze*e~IjMI8RnQ1c&2Pe~cXSA@vqJ-LHzj7t?OWjodkzPRrMMJj*xcH;0LVqZ(&llOF z1e^qtH!9wn#fG-%@t_eYv$tiIi?=Vs9{qf~Pjm;#1X*7C4Y!N}Up6oJD-V_4FH%9d zSc#U7XYMF3xuRw+t7=^}u9f5Zi4|RT|D2U*(p7DPi!l<+2e$=@g&ox^5*+y)kSOFW z&!$nE<5RKIqgbE};1C-<22A8$@bOqAO#F7oaP_(THiFpX_Xo4UM3bC5$|VNH<+Qgi zcIS=*O&LqH4T$r4M;T6yw9=$5Vh2?V3nQ=l9EY#oFir<|r?jxV2D=S#&u_|dZaZ`G zH@H^Mzat4^&cT&@NEyYYjqMNiz%%;@$s(Ib6Zcn|X6SDkP%eVovLbgSji(lIURT#l z#fY!H_}I&9!>J4|)`O_na8>lDTRR8IkV$b(_l1!7yd3^?Qu%T3D;h?A$L}`yHO}0O z;zuxqGo?Kr*!N|o5I-k$i~Zn_9P z`OvGY`Osrdl3Gg|i@E$4uW1k>{|7lUUv&MTpm|=}qvp-4jI`*jZs$sEH&8V9NKJR-9ZNB$j^`7;7 zBImx4L3>byy^z!@vqj&9C_{}jxK7Ddu9%@*Q`V^fXuB*Ae0Ig?TkpdX{XmYnb?tLv zAr8U>_`A}%>JP8GB}$U2Z<{`FQ_PE7>PLL&qnAt~NvI`~K*?31KdH7S7jupkd%P;Z z14LN2_tk*O?HBicjPKF(g_h2T2IDd}xru@26$_31<)FydcE8Z_oT%1rNxh5JAD4Y! zrR}Fn`a*1WReC||eW-<_YLp!>X3Q4Qx>r-s19v%5;Dg*+EIpCmcAb9go)HooYJX<< z7Z|_9chH~06!8vk#KO;5wu=?od*V?nNKPE#ydrmJ#d?1yEce`8RD?i+Pk*y`inB%)m_WVqa1PZ=16*kI`0aGJ}M;4agD!J4&P2+6340fZ3 z#iBjnH~#xICvGYd;yJfGYmiHmXljXL*My_`$o%~Ln{|)&?!+rU3n8ZyIHC5e@KfxX z(NUILUmDMH_@DjdZw5lgAv)KeJ_!|Ws z@R-D9{&I4cE7@hhwj3Q&IV@|gDS{O$0pkg)ONiFSM>3d?8amNSaA+aZO^^JOAuKY( zqnxYq=+?bW>?552E1CV0D_GBR>BTlf!hjAr=NHU;lb0q1^_u|Q* zO+E6JKWg~ExBSP~Xb>?qmB%aOKnOuYTlaLscYsD1N$7nK2gc^bU7|M+1o0~msWM3~ zMFO%SN~1MC6^prU&yGv!EX6c4RoEvOj}lBuQ!`g(FOgH;kgUqi-9Q$`ldEDiiqn3n zCkRc-!Hg<=s&xBV^w$p^d!EOw?~_{UF9g^{Nf&vEYO+0u=B%>v*`#p=BzK5LC8a<_ zxA$6N{l=>%doLMel1_uC4tfy=HT9@U%~WVdvw~CR*@VQf*7}_iivl zB#;1G;$VA%E~h8xYE*6^Sy2Cm_cjnpQ`%I zH0R_qCUx(LoPu4H7)G`afG~?pt}Y2hZALmY z;{edyIB1LmgsIe3l)b%?PEPEJBYNsaP43{YF;;%G-Xm-{@v{y|p@@5Ud<1S1(2+1O zcn{!8eRTc_WM2=U^N|ROQk~3G{5p{0 zw#~%EbnfG?-{ltt2nV>hxG=CXORC`mQ-e{ELizL9SfCznfm68gumcG`w~4?9#>z^r zDll#aRGNjfxT4}Bg6p4UBnvb(EiEZ6PZ?0B$Y3*|W9Q3APR0cmrn5S3uDLn2p~Qyt z9tr6`u$i7tC@3Vzp`6m&E5XDR;e3BdySTKZPX7={c0Ocy^Q3uQU-$Dn@I&8IzK_&U zE@^4$ZTf9-Ny#^uku(-G_yh#NnugLM#=UzdMUA7NsL05~B$G@F49!2Vc2rfo=O>mx z@e2^!+1VLRmhzW;Q*S+!USMG%mnI*Azl}-D+6Og?d<$) z;gJv1pTlb(l8<)8l`#1_vEhN3@%}AmCT8Z5kr5z5#XIrmrh6P1gRbqprIADn5)_Hp zr)Os^ma2DlygCip4BDi$wdWI!59&r05CX&=K0MlvWil;0@TFU*7i2~rOz_Dsn3B?6 zU!yKnK{7i!{1D&xrKDE-2v3~j(2R^u6cu*me`W|{fqv1_!bLGx)Aq~XjsG}gTVC9qbr}OAorjc=T~`9IPmOZrz7rSN+}t480?NR`V!pcU z?dIX}XK^vQ%*Ncpg41_`9PJW4Q^1ujxa!rrxH#eK85ll3eqvspWMH7!`}gl{T3-If zciAfC>yOXaX#BSfK-b{z@j}BFZEbFkWqvx9TsdcFPAipyW}5lGYwGD1`Cga4V+DK# z$~mhh8%5GAh8>xJfea=T43FT^*R(DlS;}xd;t7AAifKJ@!`|4&f8Bb>n5RQK9#?l;s_V|}Bje+RrKRD<9GM=% z$vncl6FEcsQ^h1g0m6Va%*@Z$J55mow(_$gD^8BMDZ=DKm@z!!dMsA}z5Gz>6>x5WfT5m6es2u#FdQAgrio^&O3<>NYo!?9P+3)U%)` z4E1%~Uooecu9PsggyDds?$5lU%_-VCYY(R3^n<6(E*@i;_!5rnlIvZ{_1Lv-59tfv&zb! zRpb)~u<}3dd0kziTGfW*V2c(Ss`E#<-@`$7$j zip{VC<+ssO!b=jQObv+XW@a??IxP10Z;vms1Z)X4HO)x)JXmOC;@5|hXvU1{f`B|E z6HDBaS}{w0e`n@qKu;P6Hr}94XVFyvf`740+~(cI)>ys)bQQqA3Po z;^^)OeZlrlhT&!@k&!t(!WZIqug*l4uOi1(1cWEFJ-s zQawmwem=#2D=g$lAuINrd03%QQBkpRSqjy+w(H%y-MU4-o2?BP0S@4xMD`i^Bs}oF z(@nRrx83R#@#XhF#K#goe|2`UAPMC1uOA%;cU(A~H>F6)a&)>)L;z>`Z$$XY*{Lb4 z%+vbYH;|ZEK^&Iw@EZJN?Q(=(w)J-Pk>6!|tI_SoT%NkjhFruYj5UX^!Fq~14EuPk zyMTt0a`(JYvP!2u*>!J%%W6U&4ISO7O&{?wwBxQ9om|K#)tX1)Bv(M+y6v}Ow{H?& zl(AZy%-58`laUnqh06)L!MK7?y1I-@E%sY}E8e7Jq?rQk>3Bm4fD{-{#5*#X*}wJu zrku~vm+L?-mu)$)s|zu&4~C5X-r4i1V-~+Nf`!Enxmdul;+e9t?DE(I``@*;`oiGx z9KNm3_(1e9EVNmm_kwjg%uxY(q8)m%dq%)9w*e(9;DHbUe?K2pP{|V*pE<1P zxVl-gTWySL7H3A7CIPd+U@PO1XWf%MZ;&A`!N>w`=N|xF9KZADM{+^0#7Ih?%$4>v z3qW!Rvg|OCCjwn;IvSpsDC;cFYp|Md+N+!;7j_D=3bx{&zep6?JcMOgWI=n-}Vm= zH;=QpAtDq&9K5>?gll6`2m(?$^6eiE(-JMRCT=`#PFJ8&WqARZh|IZ8n)8+bE>%F*RL(U?q}Y;qlw~5CfoTMv-$S3%U<+J;O%Zo zeQu-3U>32*svDAt8Q}N$$8gpG{|MEt`cd1R0K92u;HZ4F!$R)~G<|AzRvLh6O|HAs zbBm+SuebD4-db2 z*sTqo8TrO7rL)qIM!8&=UTJ>)P^h6WAbL^JdAu7LXkfm_iiPz-Q0)2%_| zPa)PyGyrl=$x-zEO4i!yjpF=FR(2XgW7B`sHW7x3m&p-8{k7Tdzf#nOw83hA6YEaE z^OPJ338`NtkGlIu5D=oT#@GW#9&C&yOzsdtg9~PNxSP20b z<2(-pvdfF1iq3IbTH2YpIhXVG*Ah_zc$neeWz4Gdo1%rz^v09=A_bf`L7o2oKQWOV zT5|GF9WJlXtsQ`g2otp+Xg_bx8WEfRcj!)Y!{15;7DBUw=~Bc{xPF^k6oBp@9v+6F zP_ylyYEKK3fhsJdJ6T5X=(%)4LKgkuzxA47AykIGCz`z*PK$BK7*hLN7yPg?)pD1= z9ESea1W&KUn6;|RH;M*wMSST97>-Yl%$6Y+JRd(M78I!3Dl!8(Kq?v3=Gz9~g8hTj z!jh8huf;*I*YfuaVLv*mau*t`t}ZtE6939-0r@2a34NjER^*H34ghVeIpeC1?Ydt6 z!7yscCxCX&SQBDIN%6UwuS0F`juu{~WP#Zf$Tp6#A)T~xTKEp#TbKZtMF~GX!2+|m z&Kr2TK8)E=CoupTW59qSZvR1ySB8Gi-g8w3p=CA@1Gcud_TforaeHw}?^Ev#3C#i8 zV=l^9z}>l#3nroWu6v*Ue*LgJQK0&pa(#P5gFlanUWFVuE>my-v$|Yvm~;X9Nbz8% z{k-pI54MMZ;1eymaa-FtSgM#Y?K2yMji5a5ERe@y)SBF2HHjS~lk`ecv(L`o$mp-E>q{_<6Q7AheM#T9&{OMTr z1z$eh`+6tCM%AuKP8)>^Al1D@4Aa?-CMxjgzsnQ@nD}D*wxFo0Di=rr054hK=fZR0 z8>hn(F|U&$~Y2!;X^ruS4(i zdrb1mQBO+mFsz%CEg+wxPyxB8cgJOFYO1Hq^W)Ug5_DAa7jUKy8+gu<{FQdV`~iiF z2!MMkWEfQz-y;Hw&M*n8GNlU3)u;Q8yG`^kNh%)YhK$MW-OK5-fJoRuNks*!rsn;X z(Nl;HkAnO47t4vt!q$6gnn7puH|H>%p7Z5du8ZYZOswC0oz9hzo6m*^8aRj z4TF{BZ5WUE?AFsVjLI4sQk?jefCQQ#a)D?z6yJA-#o_tvcZqhn(BRAG>v3i|baR41 zDG&k#Cd}itf^-rVR3I5IH`|ev(v|{u7t<^;?G6k`q0-Z1Yj7RH1k6^#j3(XU0!Xn7 zt**%LuHBbAEm@W0;*@eZf<=5iXhyiMh{pB+Ok~rl&+2U~Wn;r=J6E%n*Roay)XWh! z9R7n!-@aL19PQ-~@UQ|pgDBw=0B&yZy=_1Cxizqyo%yV<-}==m!VIGC01WM!5OVAusM6)dCn5q&kphrrtjID@ z1TfD~!h5sL{wRKfk1M^nfHyskN+}-kdwKk-;4jl#gtA>D% zA9BJ{Xd&6M?;70A!>!K!NK2RkYi}ZNps+Hm&&1orxS=(kS`<~7#OG*Xser!8ZI$bnKk~81m=C3Wp&(lN{s_#^~pO_yt@;}z|bNF zAcP>CZxuY@w+xN%s}*N)x)Sz02e@2aLjKH1T@A#HWpai>DE7t=)qvN|Vh9c!HAxeGd2&?KOal}r zUuBb$E!6u9wR@hKjxRQ$D6W6e1aPsOq9S_3w{IwKLvzJE*-q~1NaNJ?^~XeRSBH*j zU-A_**cM(xF40cA9vxP_w%h??AWDk5v8mZKi|%WSoyla4!S?pH%h2Uc$K|LZ`QPpW zgR|3;pLJo`*`zmTFAUY8zvLhN7fxCn4Kr#ns+ zP_v3sfoL9Zmxo73y3NNP|CtgZ0uK)lu#wkuytv@z=J99dk1_qfX*atr6o>n<+RicU z-HP%8KsMNGXD5S!nYjkxhljg0wrJ?-$$-dZ z(m`~p6=k(8k(-;iK0ZF`ktrpxu3Ym$-=5y^D!tC^wa#;d%D&*CEt5WxTQ18n;(*5- zBaKA&NmW%W05bkrnH}Kn zexRjj$ua3;BGKTDc!+r0VMxi_V0&{$5qZf)*1EcoqN^HFO1Xfh{8d%(eR6;|>harlw`k*X;b6TTP$4 ze0!Xdf7TsHgGZ;byNNXRg|4vV<|^!mG36evs@&y_^SX z{8~q%&TfIrFQD7s>31y2KjcuQiW(U}$ZeVbKR3C*a#zabiF$qxP$3sn0b-nph{%51 zM7>#o)4{Yfkc3TktFr8tz-j12+0$&eIbZ&0nhkNe?kc&7O!S7|1WY6+t4Bvi4^NDm zPUH#zCIysn|IsUjp~yBi_VOo5{C;;F0J+?BGzV0&AOMyA?Q%zV$*#tbd+Vz2zz3kr zejW@urQF{@_KX_ZQp)Ec!B#5Z3@s|6&Suv)e)xCrAE3?Rw#@`eYJhS+{#&0HGd>Pv zPuIuyo}bm!dXBmSS?#f2!QAr}1zdktRrLeafN>9kL?QOpVBCK!D)5=uEbdmP4FrOn zoeC~|z=@4FOXO!}^Q>-bJpUsP078X3U{gvZPjCd7H!;%JW-v0atWN-# zDDYHc)z?u~Pj)o#`0rnckM*NXj-n;doradtT zhenpm;?i7C?`xK>MyR4D38yIl=#{tr9l*Y*R?y+bI3K4JH#Ro@Sy~D;$plV-a028$ zkO|XUuXdfCUIOC%U0CcVV_yk1;6_RP%oEB0D2mN5@5l1^5}ZI1Y9>@>gRwmuPIUkq z3HTX6k$s6H8+v@anF2PP!_kS(eMHCwon8T)UEVJ(?$S0=*0TD8JWp)7k_5|6QZP#c zaFZO$d6kt+g9((XUu~Izr%g1Es@K$1$`;Oom8){P&at$$wJ|WW0%w=fHFzCZBC!I; zrf6gVmdbK^dt@wIvk$nO^TSNV*APdbh>`O4CNWHi1S-Qn^YgL;btXVH5kGWd3NW>e z&CQGSmKkX-_{Y%9IMbYYPBiT%HH#Bp%VI(_V8Y z$}Zz{?nNB@qUl`UnguM&f2Bo)rWV*6MwgwF(`vtP0x-@_$A9US3It*RKJa2Se+4Ly z(ogdp%2W#mXDVRF@V^Z+H=?mSgwXa1pNfrPLCbR=#_E>>z^J;1W5W`Ct*wkoD zm}-fVn>|5~l=zzgT8^?k_`VH}+OEi@-Z<_zt)KCKRXZP(&i~;IvVAx&lNVy-KwRdlLGSY-UbpeGMp!J+ z&H=MSDr5D)H?Ay%ifRAkCK?>k9d6WRfnB)Fw}sCi6ilB5$uVd8r+0gNW=}luqNL8f zge!b^YtmliAw7ofqlR-GgXG02ATyNC<9CWpqVB5EkKt)mUn^?bEwRnml z-KIz|uO*D?o7G^kRB?T3tlZmX;L?YB>e`2GzYB}D)9wUCkye_qX3YA5TXB@{sH>fme3lKo_c zxFEkbajs1^f8NB5&0#dlmi+!^38of4ZK>n*dmx!IwksuH$b|WlU0?LLHno0RjVRo8JbY1s`~Qj^ci$;v zDc>H9b33YL;7hqH@_VDNQ4sL!P9|A?&ZRBcv8J)ROWDs=#9K&5%Lu4biat2oj%-eb zI>ue){Lg;8_**`j>7M)X51|-b!yiJwzAhNLSDtk5{68D-|9jK^zfYbcB{~Oi10VUH z&;6f&AUjF_-`{jr;T?FP8RTd-zsHSOVk|7cx2@h!&;- z!ta5|h|1ra%&hF8+s78lM&AVg39!3{aLE=2*bPmyzDXcYsr&G|9M;Vo~o#Vf!CIF32Fx&&A9{%5`I!;;hT;04G= za&x9`{NxOw-|7s|*K)b33K@WE{|ybPs3`UVj24X|+UJ z)YoznK}AJ!2vi{Rf#9NHIh3$E+ojRu*rH(+;176o3d*ZFftGMge^6qQD07o^TGQB; zDUgdS#4jC@|5T%cY6~MLAAd?x#-Eg;2=Ea|N=TTl;8(*z`J|?~*@J>NCpvLrVWKkE zL?yJsk58hDpok*WS3Vr|j8;?0K|8`vjs|9-3bX$`%qE6}bPpj2wdTPD$L2oefoN9) zEQCJhMVnfUg<*df0s5#^BBdEJ``Rdlm2^Gv;yOtyD?d~P3z{hu&1J%K!w$cE4>z~0 zSr{%tOX0R+qw!rf&I$^l*6k->sJcHjOO;ajOnI5KhhzQrBODC3$MJAH=_W8=nBbyY zV;$j=;psoONN_gdB^P!_Er*3C6ZiGk9-r-h{*9~c&&=3^tlgzS%F~RizIjXdVTpIK z<_-vM#r*jA-a(U~gE@$DdDH;4ZN&GXyOW8c-)*mZx-2J|k-?yiOUsQq({0V# zy~2Ul@#GE?7Z5dE|N5nR4i;o!f0e(A_-#BsC7rF50$2ikSO8)MiT3;u7cEyLfJR z1vWKIR)2J?^Sue~JD{cKOUOND@uQ%()R&Ztx~E%mRaV1wd2RBNS4Z`BJI`)1P+ELY zAxci(IVsI-l0>oQ>{=S~ydz?pYygnxA7wrQq=5`Tm3vS<}CNNJF}s)R3H8k~d>ljO$c zR^EbttKEmW?hEUXX$SXyp9KTN(wUnO>rE>~}ZV@%>3(_$m)g`%Ap>Nszs7Q;2tv z93(p5hw%o}Cx)+8W52sLGsTHP^?OZ!@iUlyT^7-IKcPQ~Tm?aQ0b(BdobOQNjIMaW zS#6F_^R%}Q@gmVyV+$z2%AhVBl5j(!k;BB&)eRu$+D{UFc<}vvNylyaB&QTbxH^Jx|CRnt zy67cUkKgD%VW%}5>C}JX5F#|=Ha6q2Su?((Y;WV7x2u7G?rXln@9~FFYUrK}c=Mm< z?eyZVvC1=t%r1|KOfMqsxlbp*Qk*e=SdzwY-IS)jAVS~XvV3)G&YGR=CoK?$NIVt( zlkV%~N-36QMgGdAiK0zv=Ane_{dSIb9nv(!DNC^yq*4#g-@}fwur?9$MCMc-zhZj3@r=@DV}(HOAQn(SwlIzAaXUTt7UnWaLSh{Wk%6$Q@ZidN+z;Of zqC^xyz+mTM*!)7cB_reu3D6lcYThBrt}d)p@K&^dOk`S?gZ#~a^ati3CX%qn-hBz< zeL4rnSGvgzWI~Z#&fi-w`&%CieHjjxN9a%02V=ieD+L!K9h{(23HWcDKXMe&;mti| z(p$<4zq~xJ9d{G=eGjiEcgO|D(&oxe|KDB!=Jf%zYZHFqyJaSs#)6+apmlfb(<3@u zSW=#$3%E1$0;S%X9LB||hI*Hqc`9M4{9gR(u@;ut9(z_xI)vPQlVqeD&R=VsNw=|T z_)tve5cpBoFzd4;dp!gR97fGU-9_@WNq>tYwx)8RlV=1Ux1D)92sGXyri_>yC40Ga zJhn&Ma+g%ih-ngoqkm*$`SFWe2Kr2Nx8s{<_jIZ!u_EHNmBC-Ax)VIA_%t3En1s0! zOaq@^HBXGc%N5j42AOh(risD7=4To^cYT#T6op>B{7WF;cUkA6=?}x}o)511nyH5f zO;c_r^Jy2FQbJYzkP|c{E=(u`Ly91jzSv1VNkhFsw_JI|3%K}AOghn+l2_Vr-x_Fn zG^NEuqrm^|2Lyg+Kv+|xXPL%#v6~-*Ph^RQ?hd1zQWJR%hM)dIh}TYf60jze8wSgR za-p}+hcCuzDnvw*D#C?kPfKFL(AMu^C<`7zhwl=1u0U>jM4a4IiYuba>bauh{YO_D z*rafOXMIsICc)0UJnOO$F(I-9w4^S!Ud}fICxcX=K{mtKvws1YjRV7zlXR;v`Grx+ zOYMXeaZi}{%rQ$m{JFr(LG^tbNx2YyPqG1G2UA&h?BWQ}VcwWWXhEzSc5sYg!o&s! zBkRSG1ku&iw7gI;wh1|r{$0$ic zHjGdvUXN3e?@3}Q@0tXTSKYo}cnbG}7w~>ubFiP;T5P3=nMNF8>`aP4)6M)@{O=eE zwJXRMZ@9imBp%zf(8{glBKP6p=H;nCMt@>F6@-#21< zJP~q=hyshUe;77ye827AuZyU-_h)(Ui40RZQj^D<>d{RRD?zrzPYnf=x{AMgUyQ@O zWRX7vGGV5vEig`F@?rQKXV8|P1tG|8nYV%nVzv@x- zseCr(Z@~K6=ONa;Dfbqf{fDr%?zZqj4#<0BR8;wXPLZ9eE-PLQ+1+VpJNjC?=U1b9 z=l@jCe(1OM+jcKXdOC)wwTSe;faxOl2X4LL)o`~X!*6rK zuwGI(w8ReO=nW_%P%v^vPh6n_V0-3j-4xxXk`Psu+(I$@0el2l-S;qwZ6)2&Xw+87 zr2^yU(Bpt3Wzkl14(AJ*^fBW2`HCX9|#(ZN61GT2rd;BY{Z&ls?Xlj?vnK!A_w#n(}w78l%6T!=eDvyUW8c4wLiVI z-g@tJYMuHR3tFvtMd1}CUyJEadIciyvB}zqbwpae@K?I_b=c1 zUZ$H;-6$+3qv)c`YKlJo3g#0%G@cT{`YhBMA_tEYei?l<*Bm!loKbknN4Q*hZx?WI zyU%^Pp&9_mEF&3s^00c-Qx|WeyRV0VfhT#Vdzn|BB{Vud@}B)-6j}V#YhP%LJWEl( z!HF5Cxw9+hvnqSN7qOA7SWSgzgP#<7;B|5Lp!e=FctUFlZg8K-syc~2 zFpl_NAW&66q{Eb7?lPXPcQK<8FlWFdRfgG-KQl)W4}`yBx^b9#Vd`o&Tom|4WvDO+ zpZ|SJxBZR$?}gwJW)qj`sKlTagppiB`Q>r4{y>Udf*%#pP1NV)WrO`CfIBAeIpN9c zURYH4LU3#Tg(QZ~9OmxB&h8)r9hL2#^rVX*Har)0JblhogLX|mAf@q8`b28C1cV&8 z7|b`|rGILvv+0`fOCcCB)D3y1J7Cg$-u`l;dY6Hx`xF^?XEn#NI`Z=LBu%(><$s!~ z+fxcZJ3ox?>whYS992+q)3!&tX3)3=f@5g(NLo~fch*-uZBmBnefo4Jj#p(GctVgV zIuqmyXk^FbV$QLM-|OH{*N70vfuPv{wLryUJ-0vm!4h6g2&Zm(SSAlL%e0Jem!D<25yMu!2 z59McwwcQVx?*8jq_oU5P|I@X;+;L}lg(>l&0#n%A&~N^cel2F{*DG8!I;xe)Y2*S^ zXI=JokK--V?ug?rR1~xvJ|CgqhNzFIhVy<;d*d!RnoMVx-<@va#vj8I*=Rr6$YTo75T57x(m5Yj z&;Rw@Md%c#gi`n=OP5l4oIC^#PM}1z2XR_R>kHSV$QZGbX;0EEiR7P&f@9mp*_}o? zb7ZwtWf(Ij6DV|*j*x|Z-llTXiJ>b`TAoIi#-9J`=ZU#M*XFRzM3)Tu-p`QfgG{FV z1$+qy{%^emTQuK+(n#~r=agO z#I{}UOX%YLqNS3B=k2w^Biu)HzFx_s(s7GZTG(#&Z^1{^=z5d7OR@ZC$?PwH%6hbiA1F;Q9`p1tScs$WMSHqq79u&LR z<7cjg+%i)Tp}UF0`eXkc3ke!6m&vCMFPW;%NGz4&XLuetzoF_}YMS5Ks_snZBW7He zUj8ZsrMzz>`b>BN|C%~y&Cz@}?(#%^rQG_zDm(Pc5K`(bx*R#PSfUw=r zC$3M~Ku4W$XGefojZcxnFIL*S+HPSL!if;5KoD7a{Z#fXET0D*dX zaDopQnrDWKwaiO4J8C+{*{O2t(%oGOu`FLRuC(+{5vy zx#)XxxUfgyyYkfsRx9w{#ox9lL|>mMMY=LVS;p#}$Anrs7jq%{5>&slin!E4cbOSB zvx%R6n@Jo3npds$AuLKVrUc$BbOp92BsT)Aht(Jv&DL47tjY0<4`sxJ{%0aug4 zt)`bRldIm5^X{qlFWlv8l@7WtDXc^bV2iJ(ox`-9f{7HkD6}k!K~|m%78ywCMnML~ zebe$m%D}PCFl^kEe14DR?NxjEX7jI|YXN5!-C{N7LhpK~Hv%Y(#JcA9#&fZ`pzI*rLxbLc|+_wJua1$;3 zQoe~rujBgyOok>Em#uTFxQdbp=!|@$Q>=BFgtaDTa;su!OQ?U5kHxVfG7J+E)bWSD zXZiVgir#2q<;*~F^G++?BlD-iTwzZ$;(!@Fx8ov0rQ*AZ#^#hHjLp1A??a-=qRGy9 zsI7apiYF}K@WT=o4^B;MJXw0U_ij=F&8{IdqLe18oP4_ld;aRxY$7AGI*H~_uFC!E zf=lW&PHD9JY<@A%8-3voGyQAwZ+r7x5@g%v8q7P;+P#PSfRCabN$X%+UG9q5d+n9} z#U!|JOGQ*Rv*>RNUh$4zg~Rko8Wvno(^QpSJ4&~j#ELr~mxB|t*sp{-%RYiJgD*px zM}y$2i=n>@%aTRx3Pz4Z?~retHSfufTy>HhKY`k#2#T;58^eL6JCJ}}pQYO3R?ZFbw&nPoe7_ftH%z;oGsO7#<( z{5psQO?$#}R&z`3J@Zkm3srt(2_q<;V&~qm;J2Id%%(zFLi)nLZmi209F$*lSK1?q zg!~8mQ&MlK7>&!n)51YMlTLzNv4Izz$c zrTmOmLU}`HxVE8pP`QT`1FPMh(3%&LKp^3JgU=$aTTFmg|EgGm2T!aEaXwmz728;sO|=7eF6lz!zx=@jy3B@jkc%uj6(3dXT}Y~ ziXi=O%`(~Ka7#=>hs!cn1no$aYVzf6K1pO~#)df0OyrMkbfT)*`)3VNdWTFrxm;?Z z5}{V%kEXVcv0nz9Q_6GepNDhBsi_X6Q)Sf%q-fnTAtV{2bG3$^(0`;R@Mpp)+WI*u`;=3TPIZ& zKP~NGGWjA46OB2dyb_SrHI1QbbJbm;Qmo#Q#uIIBw%5?Qo8qayT#`^* z#ppQAV=yWB90)Zfyjf~KU+hxXs;Vfodjg>``5k-hQhn@71UYV?5hA@SsxIGGn&q(l zhB|4#ARqX}=_SyiblwPR#mr;qbFQS={jZ;qG187E&qrf4^ZEi^-L=p&3T=0+UT#0IwxHU%TezJ;~aFo-^DvoD|UgE1QMhO=wB~6ZBO3a zePgZKCE~vtewFmRCHw-H^)6-XovEiANB($~@<>6?dyVG6sCu!CMX_;EmXtc4DKQ?) zsvU>8S^z~2s~RzG zuFXl=l4oX~tX}KxB2w&^1nNsP)UeTte7-p3eBamX*z3Ke2B>n75awF)aiaOV@1?@4 z{9+ykC~u9S>_T^cCyPoX5H?zW>~i|TPgrs}vbYnTkd2m;*RvBzhSig*KuKN*+6=?_1ICV&>Lb?QehnGV`!e`^M}?JhdpxEz zx-pDZkxZi3+7}gSS{eVyIh8~fa4su&>cvI~KGNCNp4f^H%$h{p;wgYf(v#rhDj20E zPrqWLcpVtzRuWz_@e%X*Z)m1BA8ZiOOZoM<_HgWiD+BAY+)GeQ(PcC9x`4~}@WuQ@ zf!of!D^j+R>{MD6apJ>nLdo#H$yV-*xfYM)ItHpPs+5-W?De@h!Nwv8&v}~7T84Py zbhq88Gs7V1qT}rBB|c3f`dmMW;B_)mJ}7jvT-d1YD(b~ztILpwRvvVUJAq#4g-LWGlp&HQnBOKeSCLEuQ+M4%#-7{d2ggcHQA9}*aVty3*GQchSz=`k{6QTGAMzt-G znBTgMkMbK$7iV%JUh6;2*J`KmhpiGO_B}kx&S2+o1gOiWVFt;itz{)#?rNL~^kUS= z*zzAJ=XP;LmwxeLFEjx+dfXDTbrD}&S7O7ryiw%Ell0XuKxJEn`)Ckd_mz#Wq+Q%X zW`)R-h;FE=*HhiV_RXNa=xr%GSlB6@HByDfw)XZVIZTnslFRv^Y1#S26zz#Lg2`lw zt@YrN%j<{FXS*8UW||cq#>8{FYgB2okU(Ii+^gYFYSsH zm^bKeL`dixBIO_uOWs>=!wm?G)(*UUcN)0Mp;7b9LAhFoQh+xkMcv&P)D=2;UOd!7 z72WGqh6Ij{9j=UPqey0<66Rypp+(J&MFmH*lS{dX*zvV%PLkrLIZqQxXhh}uMfrAL zN-jq4InNJ0$1nC`INMq)7H5#)#yLAXM%)ZBe0jI7c-$n1 z8r}SNk%P!FJ}9BfG7v}js5DyWj3G@9ID>_Rg;lXxSfmBu2y1>$+}nG_zL?Eh zRH(rR5HE0u`H~kK3J71br(qhjX#7j3CS0Gc#TC;N_gCPJ$*C;3AiN3Tl*9CJpV>g_x5NusN{8>IXjNiCC52Fmn`3L z4^iS^h5av@z9}%$F4%S^nAo;$+qN^w#I`%u#7W1tjfu^PZ6_1k=$JSEIp@|>zw}G( zz3VHiwTgb`7ND*zs+!F8*anO;wLB&aeioO6D$~4(5)iV_K@xjJE zQizjEMXlqE=C0cn_q$?e+00hT@Cjf_f}HS8qgSk?l{&ZDF7`SodUz%X_fqHLC>dhM z2T}vmvO?H@PYR_hRNhZo5{obfN*Y$bBuMJ&8QJ~wF*3MR9i^{dgB%$bl43B+^I3$@ z=LTMIW6*7*>N@qvbPy>daNpB3!AZ3`jC6EFbeovu6_#4f+v7H0YNh#6J?dgXhL`DVw#=09|Q+0h{W;-d3eY3GM6q)n(co9WeM&K$3VlRqV zR7FlFP!K5)-V~GAUSpg<8OgfswE^Abt`qcdx6sx>kOg$dVx}!Z5yFsS z(CZD9nD@L`5Xs;Rx@*GUapWcP@}eQML5BSWJ^VwvG>>@TZ@R?^JL68pnP%MqyjE8j z_s5tGgu{!)t@}rU6kZ{OsDyNN5WOy|yofMlLm1Qoj@Rj|)^Ol>99m+zdI^I?5C1R%IRq zcZz(q_5=3eA=}VugA~?aRc12{w?tlZ_0?LSp@J#jR35>-(;IQU`;q%~UE_4=PsymY z#m$6O_#wAm7kIFyVxiZREfA9_&v@Yb8%1&y9tr5JTOetoHRgbXfCL>(tnX(t)ece! zOYMnLn|y_~{C#DnijUu0J6L^eU;*-<*o%C#3)&YY!hLaC`TXUHrBwfkER0;khgMWV z9uMT#F`HJ~QNfsyR27Y^=}7zFm=E|f8w=KZh$?N~#LmKX9G1Ea(JSm%B%;#WP_VxC z;Hqbs9{~RK9~xBnh=!D?t@*PPIX}lo5pfKNf0G6gj!3CMt?K({WdO0I&K~nHu;OI_ z&gaikt42H?+Rrvo%7Wh&wp$S^^5@$vh?ao7OH5}ZwdO1lzDRRF>jGnIzUBy2`uVnF z@{PUoD`-nBGFSTt&GzIP5M9FI|?Kzv#?<!$(7@Te-@@|>gXwzZ(3f84 zWTvwwN8X4fsC3G?%0$4`X4#S5FJZ8u62{Is{O<0HHE;AV!2=<8$l|i5MQA6~A3xNf zF8R9BQ<~hD7nvv&KWL&J)7hS$4!SbcRSE4ai(Q1XSzT&Q3ctRFWvGF(k z2~+B}0gs-hOjjmtI2id}G>b<~%{DTC=8<1Vuf>{bki{!2HcnnZ5Pg2R03*jjKQ0L! z{m-Ta|I7Idpi%@^a+rS}f!&E+cUh;Uj|_qWRIk{_O>w5*MMXfUS^Q2w$3@sB#&Xj6 z*}vu0@b>kI;P+uN!9xwEH}p{-nlT(-pGKUpHk+EAgk-hjO%QyK*FEt@5Y5np_NaPW zNNMgC@K;-g4U>V4M#RJ@8Und8j;&Rmz@gz+j*Wj%c%Y7?QCWQm#9#|Q_Xn18678&v zvh6GAO=M80$+U3H%q0#0-Dd#pL;~2t^*?yp9jJz(Zna$&lj(EWP#kKL$vVz>Oj2vD zCvj!xqR+V9ksGF})f?V!jGq46`RuvNR5NK@AgM zeD5322RAzUj`Lqj^ktHfjkTYSXtPNYl!=rm(R+9tL2{hB{q+%oZv6KUI*(j4b4IMb zixLx=X;SO6Vt3wwA1Rc)#yC;R*p;8;e9c$G4B3MiKXpYF%uwFL9&0Zg$t32NYpuOt z?1tUzA4jYdvu%rFzlJ#3N+XeTtqx3IQ2gDznR(~H@8iVti+|I@qfi|Vo=)`9$-=WP zDM(?{xx86LG|U(74yvj+S03gNz0@$%)e}s4gD|N@<#KkD|45DIwr4ao4K{M%Z?DX< zv2!-X!D`lfY(lJY3G_DS@Gt3mGSD3<4xI_gxH!Nv&41C~@$8wSb|@{m)u;0@Ggn=) z*xY^M9`?d4kI#+%b@J{P3hkA@gBwzYQ_P#r$rLz+IW^g|K8!`C=QUp6G3D$D?^(TI zPf{%FPxYNT?_QL0H&*5_TH(8%{ZxrhH6E_XIS}s*UrhaD(dZi-Mz|?0wWgF~?^048 zznLVeFKtwN0Db=&gIe)x@Kn3-*#4i)d-o5oDt=8pUxrGH)cL!)SkIR<+4}`nuh`{= z3ro6U4jV1MNIaYWj|E`WCZl<+TaUT!OY!9nQPXO!#kI)`IxKu$?Y1Cd`2>6w>LK%O zxf|Y@R#yZNtA$N<0pFLUau7EdhIy72S9f*iGeKVH)pcZkqpDqcq`O=Iol|Z!xHhlLma&pzJK(jcD{Ssd0G6{cV)?4OUFXEZK@@=#4sd z3!Pug0@0T1FmB64gK6zwk4bMZNj3VgP$~jQ{Jitc&;PgZ-&jh7ngkDM?S4Fd88w5r z_VH!4m5yvEzZY1|S%{J-Rc@yzcruGcr`b~pbqLb8TdG*cqJHsZ3VtQghdJ>ouoMIK z+^61+EMQTC(Sq~aBIpS9R2rK3wGhB{Kezs->1$6UKm{c|KtN7`H-s=P0+Y<_`3qsb zRH3k5{^%}#uZTa9O@U!R9%=EKteDrz@-Z!{9<-zH*Uh|$C* zred>Xe3)sNz*k2cA)(8H$GbWEApnAc;u+ujfw49NRl!#+gG~iX zVwBd{bL0seHCrJXApT-`2Et}jHF+F5I>F5^}%mxg=kj) z+S(Wn$8-U_Dd@GD<47xGheG?mQna+7VLt4ZYt(A3mNb>IZ&K5yY3?^1x{a2o+3WsO zLl+8wJ8gOG#-svyDyj@#M-9kD=v{6Xwg)D~A>XzZr z*N4R{J#1o{WjoQYJQH@VJy5_h*BmBK|;B! zOpCy-Q~x0|I)r>55-4pRXi18o>NcK!id3it!zxO(uO>rfOsYlVLnr42 zb7t?!BscUtx)S(1{AP@aaR21R4PH#_9uoS6Jxe}Ni#~z@UDT80pE|116N@?|8_Hn9 zJA`Y5DZebMT|g1)2%HIYrWQFcBAAq>EHq$Cd32zVoD~sBzcWhvU2IsED*lc*0b@7` z9J@%G7nYbfEKE$uxU zbx=galo=QTq=Ylw2s22P9wa4ah*0(;tWU;PO;5QF5k2msv-EZ@HY1MLT02Pb? zDm(~Z*b|T7B!+xi*!mLVGS(AQTn=PWEX)XTPTfRGdD0-3V5VS_^Qw2V#9$gUSlpFz z>^gB=C>R6gs8i9&(2h1UFCbJm2;reAzz!h}71Vart+R5!>h0nAcSzw2VM0^+Uho6j z+J1=%A44HU+2=>SiE%q?KpqlTHnEim^LD?iczChEn}|@gS^l#R_X$J}=0Ht5d4Wno zNl=j`gcwQVRxO4$bU=0t8UdnEGLa~2;(#h*ZyXX^{GVY_RhZyFY->)<)?l%|MQy;w zaI2FK+vjsr?p%7jOMcA9k86IU4d(DcWA3TUciXVoc(Zh6YH`Z!kN%EeV6C+=zf{*g5^y%Ng6H=#XU-&@xdfOWcjq!OGW@t?d$0ii@Hm4(*`d)qw7be9 zlR7wNH5q!PK_ceA@~CpF*n-Ibe!9*wYgnN1_B=5sc5j03e%RYeIdokSWI~`2`cNsb zWn=19ane&zHBXK;Mf?`>x*S!LRao2K_^sKT(TbamX$H{u@cII4kBbK*G%j_0H@bvp~kwATCbGMu{qs z_+SLWN040JcozK^@z5ml7F>bG%sye7ghM8qCIx=>0Z(yiQBIpD1^(NYh%%@+k>Klg)hsm_x{5XM)41$ZO}| z5#bHnSw&l0r#$)0#@Tc}w25?i@Rr8MyBk}tv4@@N(aQ7N&%9P&cLM7tmz0>i9gZ$j zU0n52id`OyEosf!d+*K!M<#kxpKj!+l;6J>k-To}t(1h?9=5eIuJUsuGJ<(h3lScz zCuiOdO&{8{PB51BtiG+R9HV0#nhbhLuh3=~8}XvGo{-UT1GbP>v*VztS;Keu+r<1H zYQ8d1_!a>!9YrvF_Z7cYROKY5Ut(f8i1B+(r8SX`g^2_%=G26Wz@59OBLE^U>zd zo$hm2E4fv_0RsDu{`U(*$b1P(X0_w zP9^&9euqUZw|hs3B(H(1Gcx_YYj{KO9$)MHY#vbVH22W1vZBdDFEV`(5g+5{t~om4 zhGDkX5HFSe-Hdql|L2wy?c2?A1i|gpNAIxc^4xHvb_tVhI$Ts^OISZ;Xmmf=&7QgP7@0@Yk!QbV$cys&uZla%gQ1zU2CQ|3Tb8 z{3)mR?i;sppO>vuH}zjoLCFa#IHBIh{Ps18h~JA1f~pr$Q9-6lCeD@(%A5+7BZxu0 z!?buGyy`#`HKZsMPjuK>czdAgG9k)y&q1nHbt-@XwRqSt{bh?l9ICCKBXy8gGo&mS zP+Q`2{&gElY0y)>haoDOP@mhPH0Ld4m8~>Sff5l{zu|8ClO!NL3QZ~9!P5aORh(UP z?AFg$ziau()gyAX?Ml?Jv_RZXq3!Lml77{0sHS1@?Dme2@>9ogs#+$|y?x=$dwdV_ zLKZrJ9Ohg;P?{g~aQ_oJ*u!Px(N76&3uHzYst;>=rZMN`(mCCtAeJz-p~N`}w>Fc> z^c4C)F_rCq39@K|B>p>Hk)k|M!zH#*XYnu_D>S5#A!Dt;^3<_&{oHU76lwx4cXWHk!^y}Bd<*?N@}W19 z2`4dFtb(P=alhoYvW&EQz+DZfHfu}LU^3~2h)8KbQP*oTqEs`OlB;CD##^Hap>L@4xy!L`>O2n((hC zdWH|6o(K}NmGj5tjpZ3%*c(l^5a1fc1b*?1TRWDK<;YP?2eX5ioh9|>c6LK0SJu|f z14uFxUg4OWg+G6S@T!foI@RuHvt0tG9k7(%Orj}Pt<)X<8QE15{3boPM%lx#WMA63MnA<E~}7rdM5A7 z+I5VSkO_zd*v18)?7*30kWH$?B89214PwnW%0o?Fg{enr&52Xveh4G$xycL}7cpSr z5j9wA*73g4E4>+WNm`Nk^$REet(zY(hEetn2J5c+osdq7dnU!0HF!$*6UG0A0vC;w zL{BUQQj=1K-<-|-12^Se9z6$Lr@wDFRiLtYs?R{NQ{v>u}sx= zK5|a6;H>jKQ;{c`;XqhzkfA_J+d-k%@`kYk|AJb0aDmR3pHAp3lr;jrh?Qce=@s;~ z^DwA1Y<%g-=wKv5tiO?4%`BJenCnKt zR!r#CLRBZ)$wP9tL_OlhY|iW|DS_5qC+Ec;#-z=tARdR!s3!5m91V^zewuP1zyiZh zv?LEWfQ9!*?#bUZMTH+~kR28I5~x68Yv{?x4hDg>iVf$zK!?1IR4Y&>4>c3euSPMlxb{ z)6=SjmtBLiDUAY*EX5uaf~wHqf5bVBkk7AFN1C-^e-NWG?j%H@k5QF7Tx)qxcFrFl+g_JaVYa+}d;4c4j2 z>3f)5FW2{@(Km-$S4uCl54N1^g${)|jSH6;(b9IJA7vYTDs^7JIII2jN<(;dz2A~+ zI^jxmc(|yMcG2On{V~jq1Da6}NN2`jIWpdC*fExsfWjkc&rO{UZM{h`8-OA!dmt7F zSgle=yj~3BijQy99f|*XZ~x^BQZeqpZ_6noL@a^vkhFEuuo7I?=#C4-?MLzQrqVk8 zQ*=G6A;Si*JT13D!XB60T5)~6{`1JV!M(N#uOzEOGvf)&HT$5%vuL9xbbNvc11 zUukmqQx_{!R5YsRQ(+vR?fi!i*x-(@PV;PtB?ixVzk0Efz{YuwV--Os{=twvtxii= zO>yRKYy9+0MxQ;b+G=|gwJ|v9yXM){r8G_l3fFOqopE^KvOpLzvwxj56Pf5GEe(-r zR)ttm7WXer@{EBGG4}N1B+8nT745}QOw^{PAw%)0vB(uU!SUh93LQ}u8H#=-NG5Z- zV`|a18~&khFfQ5+_B;pHSq&H95I`w5V^bQn3M|%gY6&?hc!~H>>(B_3q@d;detwoO zoFS}CUwyc*0(Oj6(y)cgxoXnLc+rxzE4wHPwSqFLk5z$EeHmHk2DfyB1-&HGvM|dE zO_HVD9FYSJ7~$%UBDaPfl{Hp)LVlkmQ0v0EufHl zbzsY{jP$Z%{?Q)JiZFu!|K9gKePH{WRdKl0JRJ$Iu(k9(-nw|OWP{^@HRe=S@D_ei zmqpx6T6&dEN1!vAgtrAz%o!@KUVnxwT^j3L-n%;9OW8@4c1r>c8G8G#_0i+!s6S;z z&c+l?O>8jUEMBia>L`TNFssaETq(>H9%0?!kt`OVZ#Q^aGRS%4v~iwiab1Vc6Vu z)hRXQOWf=^Br7r%D}(U_0bHq*6Qv!6P&;rW33hjMXz40*)0LUAd8yNR_Id1r+KMO1?PDsk5fae{5e|9D(%4{Fvuoiq*5ly&0v}&E6Vp-h=6+9k|bzeC<4kU z-I3d35H@~y6Qqcp&JFQ3kR!@*MKCGk zbeD>*pa3u-J&5vZ93_`w!J(<{B$;QSeH)>x!H0sH0i=mzd| zUN3y?FK178M=7&+{}dlC_?$i<0~|-VLv%fdq0)DeIZVIFi3u9geV#e~ao5C|P`yIO;`avZ2~>Td zb?L{>^6EB}Wbz;zXX*7b_n%+Tu?(6LX0Se^8M+t$d9L#-<{q1T{7J0sxe=IHZMp7l zH~wN`mFe+^Ed?p<5%q+xJLC)@2|mXHlB)L)GgEJ#f*y^$SaXh8jV@u$8*MGF%Fdq7 zJH|bCZ;qfB{*839CQb`gI<9-y6ns8eSu=K_AN*C~?ldFXgbY*@RgI%S`WalFr5>P! zP=B{n96ZT1Uxxc7kfVk?N79pkXWx_k)}0v6v#dr>Xqu`wTq>k7K6K?k&i-m`9t-1m z^?A8l|LB|K*^7usBp9K?pPY=Q*ZuI>9nN#_CFr=0-TjeXGjnELRxh?Nc66F)U;OK` zfWerBb-%RSgR-UP(`%w+@5!Po7WKx=Z&olH$^Q;_;!v`EUthHF$m#vHCV)*!<)|y{ zBn0VtO9%Detx|63_Fu8i1)&U`6vdQlQ@7!p1Q6|rhQ_XHEXFW)|IEZLHO>P=@C8XQ!A@{AOXVqnY0nYq*Vwmwz@pMv#rGSjFIqHo=09=wa;QH z?52YdvBsd_fnQl&WI4i5@LoFjJ5^As!RVal-pWm({|;ZK33{+8=CmrwkC=4woQdCY zC^0hhsQoG&fZW&zF{azfuIvGVXkcg}Id1o~$vt9jyF1TeyuHHfg^&@@})H2V8 ze4za9{@h2V5QVQcwcPW@>gM=86$jB9(ROFhBpNW<i%^DxUk(jQAx>x!6taj)6`B&;xlZC5T6?46hs{2mkJcBR&<7AaO zL4hB)A0QC?)t*f%Iw65cHB~iwDL9C7#(VC2^c%=h@(FRRw6WUACULT6LV9}?$WoSK zB`w?bLe+9wxJj)qk=>)p^-c<%L-4t|98Z4&ZDbJ#QR|H$xF)W?%=8~8yVB!ZfM$-m z6?87Bp56{BJuer%G7`6!8(7pSIDQul@I38`o!dpI@n2YDMqM57xxeGRLKpC1&@av} zeD0ETKU|A=$q|{J9?uq)GKo5DO%*=&424enM?^B}OZt1_@cF;sbl;7m6IE21TPqKb z7KINcawkMNGVI=XXqMZR5w4iy&OacN_OF^`K7YJF<};1%x|4@k^xpZ1@(l4%CGC|&?HL2E|>(wtY`WeKhZS`-@mqYVo4#nfvz8Og}dL_CKw2} zGq;`)A6}+!>s;lm))Ml~J`o9Dr`)Cg+&tYKB+F-~82!>F+WHvjah?I?KUpZ>6LGP0 zJMtIWNY2;Uz%hX1B>n!r;!!QE;d*Sd^DLEfwD5cUE%PIL)E+$dtyd}ib!WQ4?_PC^ zFjM||f$8bkpI_wHe#52Mr>N1i`N#d&5^>`bsHe@>6cwj^GgJYE1)+E5$17^%fi8D7 zo+e;t8yfc$X!b2E(f&K7V!;u3%zQkx;?c6e!xY4Kko~Xmh4D7Sjl0K8@h2Frs zB(`Jhh*>73P7P+a96qP$cQPd^R0R6)sX>WV<>CE<%ru%az1 zU0dq^JKkk;P~9n9$@kf%5z|0PW75o{LW&oK%!n77WUuH8gw2_-+TWb-3>bc6QyWU2 zVJPL+?uiJy`o#2%2NUG602>up=kf4hpBB8MhLY3|v(Z^a*6JGsfYa-=BrSY6+$0(x z-tI8@1dOK&3}WSON&4r99(Y0XPnbrTItbO1CgL8ODB2;j<%p=b?#l-@qLLCr0F}s= zHf81Yhv&>(Oy|8NHgr|y6j5z%o5y{Hjb+y_ z1tuQzg{<{HYCF0XtQej@Wq%GW3*a@s`~?ar*t^%73pMi8}T@nM|ht zRa>!S=|O-YJy}Wgoi)Zj;5^$(7$$YGvRq<4!@eWRhapd@|F*qo2rXX={t%ZjjV}RJ zJWz5w@@_U$+M@pk8fp3F8b1L{@8$nv0kEbM8`ir8u%za$Z3J)SsIGH5;AY@nRvQX& zD3}9dQV^zzLLC{5{J|NIdS2z>9K+r1enM!0Ree% z6lr~+ShYY=8b3YkLAK7g`7zQ>!1u;JR9J*sQ=*%~`^M@Xpwhp0Cs|sG7(znvl&0ln z4wBk=%#cJZm*HsFl?{O;_5In7;UO$|W9%RK{6c;`p{o1=bX%V={J9gGFF$O{fyajU zlUMy%nj=MecaGmL4XIeTCc@)GOucYDFSLMzsnwLfik$Zk(2A~6P#I%Nkst0sPc7ZO z83msiS%f+nPUt@@7}O-qzgSBz`%&6yC%0n^yGwGl^YvIUxA<}Ska<6 zaQL-kh<8EOco9`m3T55j7OT*MVW$))F?eAeUVl?v;KYtyzX7t~g%YM>)Ts6nz z6z_{i$3ZMJ^P!19MX6Vn9Oy&h)X`;rp9LOmkD^vvsd1aJxc^fQ+t`q^l1O9Iu|p^8 z|0|sFd;00~Jli#maB=akzx&;yYwC!-w6rL6T=4cKCVlG_+Uf%OWWHTdI z?5DcAM%;v!B_wW+4cR?&NhXPjnT#>=6yssr(Wc=Wi&TTZP3ZpEV51g~GP_Viqw=gc zmRyRP^NJr7>m6u1ty2?qv1K9_^SUQuk9+-B@bCYzPmhne$bCEWH(44Mgx*dbJ${0g zhz5*lro8e%gz*k-I^ksw)C&e%$4H#6aYT7v7aU1;AS1iH8cFhqvbwgS`f(EcM$Q^( zvZD-*{@Kyn?i!@7_p9qU3jnopp7Cq_kH@IFCZmCf$eGT+mt%F07 zXw?P+DSY+*f@zCH8UJ3U>JQ+!zbh8hPX!}wcb3=7Mgjv4CQ-O_%ZyMfYw2Z4THgFP z@(yD`Ivt?Ldq6H59abTGmIQ%d)chmi;NNQ&yC0u{ZqMLpj zY&?V6Tk)u3I-MS)ty&Eb>wi?F=NC$;-At_)ox^(>!R(~TNZh-@+tKfuNf~2PR$SXW zp8|!Uv*WCY_3(av6`MwLlXd`YlwG4(8YY4MMji#rS#H|7Y4bm{$0ebO^D8&M#ZFTF zazPHBsV+nx4C}+{^)>`i9xOLyh6ig+>@~`sH@&%RY-+L-AZ)i0Y0`Z9Iaf!Oh+qWX zWu`$!q2IIpupSO#;dAxq4Ili`iI;*l#05f`#i$`DM8lP~r`dhWPmM5PQyBWWy2lOGAvG8mV@<>y&fxTh0>R5q^> zz{^i~(~nn>b|Qo6`L*?CdMT^%u`=v((eW7fz_^@BG;~lip1j!(?bwg2{u~4uKftBJ zg8nVU)*p`eoWJlP${|GUm3SoT)4h^~t$E)T^}=Rc0UEz&!o?Am7_~=V5E72Ay0RhL zjIH4ss;sqz`@vJ|EW~JP=zqzahqKrkJfVWd-3%=yXo&b=P-7&mT=A6{j7qa{XMQS~ z417ST<$8s|*+2gV5qbu`ga@MYlBf;4;1|s>D-aAf&yqrllTprrZ(JwV(s)5@oEXo} zHL;iIAEZ#3BbQf^$YxyDT<%_Mwfa3+L*!e2M-kg@Cu}dTv%X|CKDz4t9at{@Kp5S2 zYr>g?xYwFZm(J^ndl4yg7;|P|a?O%DBeXeHzi4ME8DNn3tFG2m=wYKWK}hJ@MNsQd zMKy2NbN!eE|N0Nv^~(>4|Lo0XwFIUHU1Me=<6wT3W-~W?_g!1(!~3z4JTc|h3Q$-B zfL4S1hYD{(3c83@Cl)J`8`yMrM(AXEb~fjWuEH%z#te zgmZW%*Y~zTJ##@p1$0$xde{1bp8TaOa1*)S9||2CFZPU3MgyWfV)FA`y7LFi`8h&TozKP&5_Jz{5=|Lq>rJ-0OdX7D;9BPJL7kqCGG+`G zq}W$96G=Gj(&_s;Al`cuOg|sxj2gYh5OaXk1oaojj!h^XcLl{*OzGszC-*?BAeEQs z!zUA@0s~hXTdlr3&vBY6`#kRxc+?Ov8q3!}2{(c0A+HjjCe((;_+AWcbEeU{PZg-S z8t$RL+3qG`yX1+K{xTLrq?w9|(k0aP*n`5ZFlbaYMq%JeCT{`jB>(I)4ks;!vKG-} zxuejK)&nrma?P4K6F^Kah*Z&hzYD61tH#93fEG{AYxrgMrqUs!skxoKeE=cFG)Km; zXdLO~Ii~b3XNG8R0twrb@lNJ)m}S|yZ281-a?3N+C_!_upZc5ub&DH5*Dp3cx2vQ{ z-=VIvBc~72g4{){491dTOA0v&2UabjEtW-iiMp;dbHRBl2b3d)$f-pBoCCHm@72K6tbFh*d+vl^H&o2K7~e zDRO%XwU=~{LXEqedrt5~t#{-|8?l^FsCk-&ITTpMLM z^$Ds6cW4VZ&_ph|sS4zUmN1pn4$+ydxZMNjbEc{^0>zYs5{--rDleENmr+vy7;I(y zqf}^H#SK14Q$2=YRQTOo#j6xP&C2!5D1}zJH!Qi62C~(r*rhpsv}BdWmQIn7L_Q-? zt~`@h))xQgOsQQ#NK8yKekl6G_?S3n!6VC+&e_O6|;#bDN$cG;BK7hw?ZjP)d z-{$33H9Y=R^Zlxg5m*v%wCw?=_vqhDfTS@mEQ>DD>td^n(1TceX2`>sQBC&RNXAYu z*igqmgAPf89>-X%&J4hqZgSF{K|r9gAi==U_>B&m0hVG$rjD?~!%c`y0zO@T9Sp3H zwq2K4{E~ef!uVq9)zL%czub$26c86ho_z-@-p2{7Sd`M9X9W>zYQHX;Qcmjv|Aft= zM{I@CENAlh|U37Cl?9h55Aci;O5c>PD2< zcr+W>OKZG&)mHB<^DnAHE>G>i5{TW5Xjj+hkOM17^`M3!6+l;X>}YzB9a2Pdy>5^t z`8&Lvq8!LMA#?zCeA{xvV_aaZrCzDyoKm&L_36*ngPmn&_pY>8|9UDl%BjDz7|6Ws zixX5tej~STGFquC)kF$k=orf%?WDdAO%;XZwhJLOrhf-rA=~o<&I}!v{1w!$>o z`J68hj@Yj9ZPHuwg`^1BQcwr`H59~+5pwp9&N1(rnk;V5|3})Ghn^Z{C~$wov(aWpjhc9~mhz(PSY^r5kF~xcI{oi-01N0PDP6L!LQ* zed!2gbs_#|Cc#MZX0xEL6IotnL*$Wqp;#&+YQN%T6ulQ4^$n6raDfa$Y7HMpT=INM zfl6CHFtR3v_WWRyl8wNDjh7~WYo8Ce!9)JfYVDplO}#g@k{>d&sVJy30#;wO1MQz= zOvuwpPILccSUzvrWNayDXuN|!%df9;uJR4jzD43}@?Hke)D=Wy^b`e3e9{pM@S?s4 z$X@!05y}AgMWJd3FK}|hJ3=+CF{`vLUxe}xN#XK`JB7P8^SyWYMI`?w7n!eVjn5us zVcZ0FnfVCUL-EAcPYU!)9B0T|-uY;KJJMj{%;TbdW+z?dkpvf?LeWKZbl1YFUqsq~ z{^P~;f=thJj)}PDhzZeo7EqYF##wUjRVIWydspe?8+Y0B!!e)z56BQixbC#!GO=No zy$h&Vt`nCn?`IA-?;+!P zmxbtI_@11GB;&%(Rqoi=Q;m7=`jH18m}SoD?;5>}0Co)#ujh~S&|G49g#4rB%o$0Cq`Z&j5OU7M`PbBq!5V+W!eS=QN5qx?V10d9ZFA8%Z05o=Kab^AfKr0NZ|4l|+ zgnzaxc&PT*xIbz^Z?DOJh@n&c-V-WrdoJZ?Cj$^QpUzNMMxV8S zH&xopOY``{_B<&DN?SfLJYL(nD_|!6KieiU$@KOE|8RmZJW?%UMeLi_4knC2g?Sf9 z2ava^Csge{%akcYOBDpeALtzAqk61Vi>g@h%*K5UiW@mw5~R%>i-d{eMagiZ^FAZ_ zZRZlPXGMW~v;=s+MqmgeeiflwW%;zy0~OhEL1LDl&!lf+(loOZ%o&ZN+EYDk1HiB0 z^(X*Q0DoBB`>xP9)b*b&rq0JbmdPg#H2Eq3RtgS{4ZQ!+Kjbgyf3NN}!_GSL!`rkl zG-SZQ`j7#oM0d`@7G;>QiVvB!%}iNd31I?)NMNuqtkuSbTa$ zJ70d1B)v6^!NG5RVu0;&{g1P^b^2OJ&Yy`Sq4DX0f+R1!5g$ z)QMOq+T?%4PmjG#R^OaGwp(I{Hiv}$A|5rHgPt52rKf$WaGtoe-uk|Dl{yNEWkvBTvibXAgF)#H1jF6{1#9^>j&NZZTTJ<*t1M< zveTZ~wSWv;k-_Dthoy9AAaCa^Z~HHB2QDHGl}q{+|9?UeGV(Q(S*E%7M;P)m(`2<+ zXG#2vsEm#~y*vb+yVUaUZQNCD5LL6I@_loSoC$tZ3i~Nw})cML?ie z)WFzoHDh98yIRALT=(4dZbKepqTI6~T%ehebWvc-`a|bb6WY;EL`Kh_wlGlKeQhgg z%{ycOif|C54 znHA9XRb}Q~T9%f3>yzMyIl3Kkm+a*c3bdPlVwRJw3b<}?Lmpduq3;OJf z3qUEo`WICPou5z4!kaImF~mA}CRl+qe#ps=etl4uYGgmL6Y|X+)zlji`agTZBp&cD zxuJvbXb!m$Wz04mR;xZ}?6vD;F~e!ECK>TI<3F*ezQ?B@A4>l4TXAfR@>nTg+cCt( z%`v$*SJlM$LdDbQePuS(U`$sRy$$bfoj>kh;TtZuoQBqcMAohe-rz@^2uYh1v*G zbO^8jDMpK(UGESdxPKI>3fm!4p!ZySxpAGVx_=tX!QXbg)#mlIC39Ucq%`df=ThlV zGJwe-Hc^|3z$8NHh_jW}`WhCbZX>F?D*R@u8Ft82XtGDXl!ht(h_2AoI-^llUIt~1 zh~)7dkG^T24h-Lp<`6SJ4``urPSo}K>so2RPp3^K#(cnydP$AtW50j$o9?Zxa|!|w zSV7%ww0mG_ePIv0-^#o z#%ynYSQ7<|A8WkcQE!Xy9~vVaItJgyWS&WlY2OIEgQ9*EC9%b7HzEd>oHz{=Yn|%G zN}o6eqvlj!ylhQO@!3u*XUv{Om8Lk+MmwHb^L_Bv>t=8Frn*1kN=nE@YuBboQOBsq z!vq|i#g6Q)b~q^4dwA%Pwxc>|kNX)yPWGc%G3LQMtgh5A%xRrZ{=-Aiek7NT(a)pq zs6CIG8ccV*&zaxg3@q+r={rS_)4%&7BH63M<=7){dhgD&>a&-1Qgnm*~i9nnXt6%o@N zF~Hje|6fsO8P-O(w(Yi1v}l3i?(R-;2=3Ydh2YX+!3v>*6e(`S-KDq_+#$G2u>^P5 zk7qyo-QW5#bIiY)S-$S`zAmd%-jtv4sF527GadRmC(t5xIby9*gg5)UANES{Z8odN zTv0@VNEkCY_|3aCe-sdEA-hRKOzF8qGjC!~;qtR_FR;5of^^5;crY2xe*(!7KKPC) zY@te*HiW3`*n!am|H>-PO_#0>e6#cb=#ASlY;i@>p44gsi*+GAYW@E~TnZw#QYWv4 zmkdvAoL`VslRZDlzEwz6X~<2G{(+)xO8iWoDOs`W@+phs?~dqy(8QHr5cDYx|MsH} zBn9T_WVgJ^fx+W`C~6W2L`4J|8LEL~B`u1BH_)DaSQH9>LLMAKNX;=YFg5djk+OjP zW=fxzlv9@~u@5mAYjK0#6;vdM0Dww6 z2U61B62E?s7IOZ|0iX`)u#`Dux=2yHHV1fkt_3OE&6R(vYKYiS1}-Z6ikslJrS745 zJ?r93u9RDvE{)_cet>M^{n=2QR63qhdpxAqf;dz~^mn(b6f0x0v*s}OI3RBD_VK)g zQ#v}`EMbs67xMSTu6;>SIVT}sbw+Ex_z)vLMyWtJu z6)4(U)D&ESn@U$lR6QZ|KFq?xLk;MHHb435_L=VB9+~a7fvlZ*sx#n9MI#H3jk#S@ z&J^Wa6gD;^;RC6yaUDFHjC%Y z(=HQ1sHU#5($nJu^P3y1K=M91pmASjkf^GVyyrsi8U0$eJt?Yywt?x-{2c;Np)3fHp~t;sd4Tumr#|G9m&k} zBzQE%tnHyG5x%leNRJX zeB!Hkso`ywlQ~&A7ey z#knBsu|uSf&K)R^f!}7a7IxCas>Zl6OjM3z7XR~CfM@-pqzroCH;%-Br$;1Pa?M~n z0v?NDVicjn_POz2#0p6+ehKl8FO~kZ>5?t<;QZ&30}^Zsy!sRs0v&8W3;Njf0&GA_v~=P z*F6NItdh+SazJmf8V^R{#npq}+utHZw$D%a34-y`&b;9VJ>e@(Z^TmaN=@bmgE_!J z?lrl=TJ|(G9k0zS&$+WNLY2=ZCIcMscu!H0%eb|c`~I^xr2g;4jOSYV1Gi3b)#UE{KDj)N&#PD zE4OsliXkfa`t9`|!0mLl9KxUn*YHhR^%KuXd}@d) z2eYwU2375KEdU;W`FVO69Wm{&XqDzeWdY8lG$ zzmCG&;M}Ttum{vvY--@N$>}FlJ&_df{^3wH@g~!FTE}j~rMo$_D$u4F*IKkb`75Cj zz5Hm{<~j42t6!e&KJ}64=@v$SE%Nm!y-oslzpr3D8@_>R6S-9@DK0y6AyEOARpfCV zu~fd42wv=+=5u&dRz~5UI7i$yJkd4$U3V**%zD=rLm(A1a zF_%FXyU1ob7sI7VczEoxPiZv+}ZhMG;j@Lu1>&iTXqirk4#PLDFSVz*sM%ikwP zBjxiRK=eD%HD8@TJ}F zN2iE5?gM#!(C581PJaY5>2QBU+hKxc%aV2MU|~a>Pkr(kI&@G^9^PJ>@346~AAL|k zlV1-P%_DZ0MlAXDm`tU+@Fj-N2TD6sk@E|Wj37+HUC+514!4n#)B>Jm=a0KoKG9b% z#xMEn(P2Q=Q&&v-Z>R{?qIWKd+g9=3=t2LYGD-A*5)vP7=u$YlGZLL%<2-EGGq%pf zU=G{yN`D~~sO`qgbZ9cDV_@3-P5^OojHA988~GXZOH)fZ$Za3e_4{sIqCdz_#6mH? zuR=V85LB@-x)J$tVc)E>7aLEI-lI3idRsltsJdaNx6d{!>a_Pk*k|<>#B;>+ni`LG zU$@$gz57%HGK6!OF%WQQ&r&nPOn~B{bjdwq(?3^6>hP()A=K^p+ATxba#O&3X9^opJOs#AB~hG<08f{{%-uEJ5ij${ z>xq!@**p5i(@NxOj-_F*_#eKf{1CGqm$BMo+7;^Ub=$03VLJRnXjKLN)!_yr%T#ab zOLJLS?{e^LbMWAfRN=&<@pNyjjcVEvi9lWViyz^q&G03&)(qG2t$s}Ma= zOVpKnw5q~z{n@j*8>FWd3EW z>Z76pYi#4L7&J?T;Ljq2u`H6IS z5X)FrwA%UIA$*^nbp#%_W#0%<=g;#%t5P!n^# z;^MLdoWoj9)>Mr@Q2xy_Xm0r1w80IIHVak?^%M#Hv1j16+#{6v3mW{P?`R~&Y=kbC zbxuEFA}wMoDhM0mJuSdqlF^=1PZdz!a3heGL885mPPlA`Xhj@ujWe*QH*a}d5^MGK z`4D>^s8T)-lo6!6yfuhe3BfT-?cZdY5SV^+$5A-kifPju$k6hY*a4p|@DF{&Oj?c=hpjHzzg{oPKVLaaN z?y(*;M65X*nd~KjR9~`H0<)o|6C3x}sJsqaO5W=4OFb;+s=YKXSH_A zFjt*X`BnWJwtS9U`eH4PUh7H0>DNpm^Hz-%v#pz>Evg^#wP}aBXMjK8vjx#$aTzQ5 z?li&tQrn!uVhoAeZ*3yl6tlA9y5fBb0hj>D4Z8+WJog#`y7hC%e=~j6KeQ-C&Y~wy ze>+ZBm$&6ohH{BRSyn4KL?4;1GWoX|XE$?2r)n8!rj*>cA-*r2!@83q(rw8Gc6=6M zhhfRarw`UP)(-+175?KPS<#LWkMFh!@S%)euvZ?cL#!`S=H=#pFBehS7RpQ(z=TJ! zl*@dm4cMZXW-bnOPrEnObxv*Ji44wBU=AVF;qxMp^!vyhh~!`>#v_~C4jOppW^1p* zQiMUn@thdtOfhw8AG%Q+4t;#D9!Uf$NWc{&7p>-rX#T4lB&O4N@xw~@(2Y7!;Ky(0 z$p;Gx>boPeKZFdxy@NGOg&w z3wEQZtQvMz#%OArI87r>DJUM3%N1R)HjvO_93w4hVPyScnLt5pW+X$S(ms>ssf{FM9>e5_RJ_Q@Z1pUDxxnflE z4RU9C=;Jhq9+Rz*VyZuqidbQ!-8BKaH_lBxI9LHUC)C4-6~!eIml{=(YZ=PeFY^la z^Ta*tX91z17bL*|aRtV-uAlLmnI6@JML)iGGLykaa%f2N>~_!YKON@*uQuPZ-=Fb` zUkpAQg#%)W=OZmD(v-?Lw|K$`hlq4+Q*4X7&-#z{E}TtJhh=&xeNny$5q;FZROAGH zr_|autdAJ2$;{+d&8Pcit$0&YBTsr(P%+<*?;KOb&pu;Hw`L?2sA$!0X8zvyqz6?HW|iv zmXb!aK1XMcOjQ^UKK^8Sbr zwwx2ynT<=Emj_OlwEeBFceozagMJLwTb@fhu&gE_pY5vUkW}&WV2HJ@b)F9hvoi%F zWz;Cktg>r){&z_r)xO$Z#|L3RQ;Yg=%TN}JIle^TddAVOP5b8UmldF-d1P_a;UObP zpS!T`>nyx9udtm9k3Py!u$xwN0InWx!pa;HfKC!4|5>74q@r~Ic6dXh_{(S@s5ERh z*UsTEyR|jY$Sr+6sGu$TPd2VR$3}P;SX^FqL4=vrraMz+A$jJ{byu(mV#MHj+^^nk zF#FD~PI*2{w71zU0gsDap+lnviIWntY|?DSK{n&x0}^`tp2w>W9MP_Ox)L#WNAl_8 zv%JHyl+q%Ajb;aC6RV4H{Fd2eRRN&m;1Es^js4!W7}>vSya)=mjWn=W;NEYi80TNq zctOw5gbS$yj{(D;DBPj7xGP+j?}Wr|Pd@b1r5r49GgZ4}vONCdnTWe53y&F1X9 zY?iMv=WA6bcThH{5F%(_l++#PvC^=iY)S!Py?R>RDfRxu+xXqDFpTNOGEGH}4XG&8 z0V86AMwenlZ9HM$Vt#YBamS!yosfuz77TR1&^oU`8=sz5v+BYPeeWNf8kdCkL(UUp z9GbL{%v^WiwDuB3Qx9uR)C-@VPtIcJmA*9&D6-YVVunoP^zdB1a83ACg^+A*v3G8h zdHu$yb=)P*JK%+&&B_(BtSm+Z+n4~rlgah~Fj;xeM|rz5GCUHG8;v1G>G3`&Sz1|g zKBincJBKjGm0h3`IfI`>x6jGCMIcPJ{C)anXI_i2KcR9i?(1299*}~jsI}|f*5}Ge zZ%@|&rQqWU15CrJ_VrN*Dn>l69s;5+jfKRnx>fA0E1eyEW!q09ni5r#rZ!?(<4A?N z+V1q)g-vD}a~I;edR@u|03RNdDT1-(#^i}_vdqSkWNP(ECrfHAR||#^?>;N?8e6=K zEmA{h)vr%^U#0UXVIC|~vpCc%aWsf$NBZw|b?otZTXMhDi#^4IHu^d*-f|Y~eh
    !#2{5snM>SdKR#$M)=EM0Ag8JWM+)TbUObR)i= zjGe1(`x-f0qtoz2Z83o5$SfA{El=Ke-P(Bymxit{NHEL~HDSY}I9%eA+(T<+qmylQ zV;_|=&kWE3mY&p=G&0d}<-&O*OEVo$oz?$-zL=1nWv14n`s3~f@Tpm(TUjN9^wrHg zetzrJ6&^kM4n{h1m#$H3%kLOaP?jfD*%q2b04~j)X9jth@HnLC^B@is8#jCSFHqB9 zS6z8Eb@kkyD|Eo?s*z0DQFyj&v0-KDTRtE%KvzhGaiZOnj?Me22=&Icj2<8F>C$-T z<$uc)rlZ)#>=~|k>~Mibwe!pnv&9evG>YpccDo&qwYWH|VCJ~DTSMe1Amg;#rA!sF zMWUb2aqbnKAjDV+??(sv4{`1Al4_=U_s5v0dd}}mDYzDf1p5B56+mxHo#eV^5wbj7 z-+E^Tzjv$n`&g-DiJ*mCHP`ohx2^3Oi-Z@Oh}d+ZnR=ZQMl|g!@-qw43MCrAVW3)p zUq_M9QU^=IP!lYD11IeEtaWI#`}d9D?CKX}>np_47lcEL zhz;)BPd-tTz=2{GuHzI*KV@aPF@i|P39J5>OfEY(idibseJumSX(H$aE(W8G59aEQ zMgv2UZR(?EeAGX>V@Xd8Ba|vyd*|3-n{}wzUGWqxkfV*3i{3t<%jJl;xV3;47HLVo zy?goZo~2E5;N{Q6I9zRx6Lwr@J#bX+q9&yn^!lP*=VlPE<-+)h~gP z5b=!CY#M?{m5R$Q~J%5CyS>< z4O3Z(^>O!P#fkg|GUU3y0=ZMC9S0(CjSp zWVkg8@gJr{GY|gg@E^cS&Wt=o=e79Yvt9wo^_~#E#@i_}Z`;T`f~&A`QLF@5vclp^ z#PNn0>ozO7y6_d}h=W08(Amv6*XR74n&(XsnVNx*!|knXmbsS*eAu|(JMp= zU%P23hY+!3`rWd%G{XddZhL-~j=_Xfk^_ER^-<0e!NI1F9ny(iSED0kS!NLgLPY+w%=k^21xpEu^g5Fc*5K2DAq~EIj^Vit&Sz2h zJGxT(*vE6^Fg~qLJXpmQn~Cd$>{V5bXPyi6wFny0qdaoR@T>onI+5)1Hd9)wiUMi0 zZwT)I85(z!zRCk=lABT3%Oh~X2k$E8P7~QIRxmY93HY; zo0G)n;Z>O|Tc(lkq-HjhrhfmE*$hT1OS2o3iPP7^t6MOt%?fjN5tz2(X;Bp-GZxkC z)aqv(5XG0rd>dA1IU#sbg@Mt9F;EBQ_q2A&%ly+46e^Q8nt(;`M5qG>Lz}JohMkT0tyR@soDA5897o_@n?U2w0P#| zv}ELU4H7|7d0;<25<6TThLA({@xhn;x;@S&A;p+Yvz#{bP5fII7X*RG%T8q5RRrrN zGTGy}O|-}R1bYS*3`$A6$@c3(iZ%{n#&#TpuY;&w?QPjQIrNQ#x^+_$F7~!Zg);*S z3yT=?;kHJz;SNDGDKrkv#N)zEi8Rr}7TrUIUL+l#3evkqSAdYG)4uI4 z4fp~&R?GFVN)6@+mw*+M?ZTJ>X9`X933E|t9soA6^1Hpxgx3hOIx{uYB5gE(+I{fZ zH(pPUKX?>wjUS;52zKXvt&sGyC|bz_gqL4c7u1jddZ-F4?bko^uU`S2MoMl9RnzVB zR#DN=YiphGF4uT4a_0mx>!a??)JKD*un0AEbRuY~YwBwz;sH4dDn&({KOC9%l9OGA zY3&_`XqIIrZ`H@>x^qyTB`9XJLV9VVRHBDP33R8li16G!Yhy>BNwR{{s??vm6Z1=Q zD!rMx%mMM+wC2)wU`I*TO(iltHkdmIF@*?`|CinU> zE3n&7zRxY+dabyeS23w#bcc#6*gDnY+$Q4h&gRQJ&!mGhi>qQfx!u8e$NT&yB#i^Y za92)S(Wdnnb!*%#W-W@C0Rmc}4A?us@&jAlSYf{|%$;tgNQ`trrIHqP1Xr)UK45%vy z$$lXn;lD2=qB-6t@5#}C2XCOBG=KSQvtX|@tb0nc@bYOK5Z}3LtgMIi-|55hGJG{r^O1IVzTXI zv&rd7c1MHk9ns=r&$q?)&ukrH)}C;JA+h>6S85W>ul9PD<~ikLx}u=8kQIaTS&wOL zHi)cudii+on=*?_o&q!tXKP#H*4)Y|ym0VT>FBs4T;3A*S)Z(!%vNSy znA7wKqWZdvU(2P^Rge;&WqlvA@6|gc-!FnoGZ5!#wiVl0mXr^#&p9tr4C4wrZZS<$ z`kXVI4g$?XB#cM6?;nOGYJ>HrE2c}B0kgeT4-ZWe7hkN})XY#>S9(WUv)hX)&(jhNANu5?NPViyO6uW!d{*fb9pEJ0?^azj1-ZtGy@DxWz| z<7T^&d~UmCBC0J(Y6>mi^Og|VbiFy5y0|A^+${AKq0tGx7bz6Y&!Wji961>160FS< zTskmcZ4>hA4FxkG9C%c|Sto6W20MLWZi2(~X7j5B2ENU%^nIg%xeQfjbK?if@X0(i3NY)N7&8JQ(Lc^MGIFPOSEgP}3#-C6JRpWe>6|JRF8E zI#5@yc&CeSh!)SvCy(FnuT3WTF8^v-WZ*Wz96}O+GV;rRTuh0`iNcS(hgiW5S_>pA z#Q9B7clOF`{7ORSEDx^2hs+>A^keeU_aIsmXx*O#n7}jsS#14r+rACE-b|wrVR#g# zz)~5|@u@Goz~X?Zg$K+cBZPM&arpA7@fs$THjykEDvND$KvC+nbLvoy`X3M;*^w+j zYHDU$iDrwD#zf^biQX@olBo{ByTJK?B~z)LM$ay>>x5e?xUPVcPC0Kh)rlDsP#rd zkn*bE-7~t*>V3B%4Sv{L9zF2^MkBAfeI2uqx_v~#)h(&td4!IJ7}?_9U2J==J_gBq zS0+|W{uW0vx3;8#VmR`rM^zU0^%LQIl=OW%x7By7DZkccuNg09Hr@K*D5}Xa;@VeAqT_)%@*#@h)I-z$ zuQA56@%foT@(48@o9OY;jYV|;ur;!l1FZkRFKE{We*`uR;If&rS-3&vper889mM7E zrbXbpCC3xQr^Q{Oza{bXsN*X6DjC_Ud}wuk3QGK~ohD7$TnwKe4Zt2+^x3|>G5wCK zn&qB3;7w7DEHfoj?)uTzlwfQJ7#hDKPJ^t@`Gf9_s_erRnVHN<>rhH~v?3=cCrI_Y zV&L`hYCv`|+RnrBp1|M5@7-<^0Yq6oA0;~*>Fs*x^PAmk>#ME(#J*`r{)adFNJblw zalE@%w}^6&pq6-t$RN-{G#(P6`?sY5=t;-D#(U`|y1=Ra8OZp$bTV4ZRJ)vhjgM)( z_jLatdT64qi~D@>l;zDg((%r^vOX(M0Y!hEioSk*5~fG{1&~G)N@19v{^bOY#n+09 zQWUhg8^A?^eg2oR7L{Tf&V(Hj9Z0F5MN|4f1r40&>ANq#CBX0Y5cea?op-gds<#dSt?Qlp4-444m z9qFGz|Ir{ydwr8P7hfsW9ivY#+9Nc4Jxc_C5iE`8t8x|jZx_0UWh(7lkhVyytWx6q z$#~G++$(|eHJzWJpgwbt|Cu|a2FVXNM!tS^XR~|R&u%28Cm`9eG(%hZ--+eX={kM& zo@%pAp~PvC9zSG-Kfl$#;l?MYKDhn*-==XKJ)uEP{J-}lQp==13pBr_&FdauI*3jo z>H9zM_IL#S&Fa5#Y^O-Z=vD!Bp{I={(D5Z!;GqcRe}TAvZ^s$mCuYMBnvbqhrq2G` z0{?A?XxAE2(rD1p&maH(@8tQPkG)UK9{(_7oAYw1eo6~x)gAh8UHtFpsvav35j^6w zKuPE61`+~T6aF87_jIrGC9j+$Nm!gpkPK-Fhv?$wJI z$BM9wRnoJ=`pF{F&$nlyc%!C^KGrGYFJ8PL&v8YPvHxy+yf=*LKEqs_a0CLR6sN|>Ai-~LoXf>kzNy| zi*!PVP(r&add~a3-}k%U8266f9rvHR$2pjU?7i1sYpyw;IiF|pR!v2YoQ#%?goK0~ zBL7T-gye!P3Ca0CFI@zmm@tkKz(40*pF*@Qfy3{T`5W;5mYa;8o2H|so2RLZ1&Nh| zqrC->tC@?1g@dcLquctqCP@;K+a!=@Pqe&}Rwg{XsI*VoHhaerj8C6F4P-eJ{< zHa1jyQc_rZ7s;hJY;*-{RQ1N+-u6rPALrz;XMevAq`3XBFH3&5Ovo`$aAzR>`+G+|@s_Jy zg$ixDZ#1|-{QZyNc1twJ z{m&*L(_&mq2{{He-UlwVNh$V?)K>ZrbYhF~+0|FjCV58qY1-Ijxuk_!)?VAP%KX$F zwJR;#u7}U1HK8XHUw8ZAcj@YeE*$<+Y0zjm-HRqf=H;1_jxNclqz&|=&Ud9{ zeki_HqrBs%s|6IpC0bHH#fJsGjib<>UuntEw^%BTs4UbK8V%lxs|iTi zv|nC|qrYDB>jqjerh0y?9O_>lPGno651>gFt$O;6*tnm(L-2GDOsK710h4nIE}_t) zOy~=z;I^OO*P!VAiB_gnN4D7 zpzp&5LjT7XJI(GRZ3o1zomJcwiFK~UUkl+hEfC(}&sS`VXX)vv{y@+e4<6+k3McSO zE-NM>%%f}+ay-lRDHar^n^D=7RTR8+GojncES8F5VLZ-#buZJ6$rWNLN`06VwZDA% zVm*#At_ zgPa6HoJ5pRC(A_Y;y|tDf~24clK>w7Aa3mqda*|@%mD`L|9QhQ`#q|_l?BGa!G2#J z(G#B+lO-FIM?t)N;HZnF!vVvC7u6o?lPLeg-C@0C{1SH7bu))g!pBWFSB1f%)a&#I z-(d9a`c&~;r>>X3owlKT)4iek-?OSAh(P015^4*aM;K6c8I{#l6t(V;riP?O_3)8G z)JCQK+L%5y2%q>~VRK81;L>|BoG2yn){hw;T$N51b=NAl>^4V3qZ1QzSrlVqId3vB zpys=0D%_S_RwoIfIRq5`;wtSf|YBF;2kDYVa_9!;A?XW|3b~eV{YvYWmDLEw- z6-(VjQDLDSgQ%N4Z`}IaIcaHWDmuE%i{!N5Ry}aD?V1WIDmo4hYu+}+9qpq_91cd9 z!thlHRqs1;bV4or_dWu@ab%;UCFjuEfHaCc!iL*F8H!ot6`v-jgY7J;L!0119;~cb z_eaDBCSjWI<}PdWKV%mY(h>7m(K0qJ+Fczhe)Gmj(19M+*Y|wH&O@J@dSq1y{3NG+ z_*f`W`K{o_L;xRTrKH+2~~VIA1=gLrhkWwx+? zDHNrkDpl0g)x}w5g@)Q%R6f@py+lDVyfH0JMNO^q!q?ESP}FlRYj)P0N54$#;NXHF z73;FR_E)(~**}+ui&GOGzU3`#NK1QcP~|Y#E9Hn7Gb&fi(qZWd3?#v?PwAqdXioR~ z&ba};rj>zZjO|d4GM)QkpRwz_pV7Ti9T9zBb`vPV6y`J(Zy&myRTb3%e^8$plm0+i zN$|=ef!0&Clzdf0c1@_VMJ<`war+%j>5S|<5#(U%m~=knF-%WS*Digfi%w2fE~6$+ zo0{2ZrG2LNHe3q^D*!>^A#7+4Hdj|#7!*?rn9hBKD(skzL9e)`-q(o zuowN((g!L2Ms*(Absjaasgt{_6Lq>_)I9l^@AVcmvJJI@b!B(&-E(iH$~3PUi~0C5 z-F~c+i-*SuOuKTTkb2vD2FjZ^b03%$4d|M5Yd<-2dCVm0(ck2Br#SV2&V=fY@VM84j-~mn77x66GW1v`1o<)*7o-7`g$?4s8%CA{mz2;*ZWsx zWM#2W-B|F;7|kb7f_pzLO&tEbewZK}xqTFX!=hq7RV#9EaG+_YijyZC21`t(iSzJj zKP;Blh!s)><$?8Q1Y?Vga#03VM00)@9zOK><++OIe>H31_YV%1VhtRyu{;K7AAACB zSTQh45&COj&*pX=N6-oD)Opy0J*NF4);x|k8I#+~!qVX|QLB9UG7?EC2L}8#f(|KY zi-q;3@FQ$u+=@>26M%`4Gl=Se1&EK2hgRAJBch_1Z;@Tv7;e33QrMF$#=)8s@3XgN zl-cC#NE4Di-4e?6@S&DUuA=R55vJ09Y}SpMxBq!5&GLASlM*G1tL@fDOQCqn>E$6~ z7`1hQeh^}enjywK_6XUBY*20$MlfrduqomOvefE^B$a4g3GNTS@*d|l<&Om*JW}}G z*ma?ZHJx%2=};HkMzPXlQXFjDxWC#t8ZPCW0has@5>o)I26s$h$tOoQ2QK)QomDo=d#5gn_crJf04H|c-@>m(qzs1z0we?p}ijdFFVv#|B zlzMlf@bKnL8y-LI`0i}6&dRT)otjw$|Ph5=t@n@*ISd-N|~gp2>Z}eK^melltA8H+^q!uytG> zDeNC<+uU4@NlY}beC2kuj|2{dHFq%k=&1hs>ctBeIK;#ZkUA`0mC54X1s$bSl(V+I zDGj3Vqi1M>u6jUEBFwM_+&Q+XscCC#>!IgbRYy-x?1b^7D?>|eOS!2!?Q<^q231AG zNfL6L%&4!ghenng)m6R=w_}Dt>%B^M*QIW^+y_=FVUeV=pkK_V{fWZ_D`=BYrY)#E z%&mtw?&^=f#=%lG8}@u%Uh}5#gP5fcCna5dJy9${Ctj)f0mgQml7;tQ9=Zf95iV%t z4EAMY2HR&Pg`$02U0kUsX}o$|VkCq`YHGB5Qsrc2m$TzevR}PA>wnT3SEk2Z`(lT^ zVZYh6b|Ergpwz5wr#Xy|gOyd*b3#Oasy8tgV~gcH`0*qj0Y$^|7g7Us;`Q|tjxob7 zqV(WY>tmb=-(uw6vA&$9CMI2)^mw(siIa=Vz_pP81)hXfV4hP(2$q#87m0no#zb6y zH4_<`Wc>xF?RlQjG7Dr`89xgZJg#Xj_A=Skt9`qxVihl16;$u`&|wZvZqd->m6ViR zzx>f1hqVP~0M9g(_9SSi!sZ4fSU~*V-DdaYA&z3l>_)Ywd&9C#?-+M0oo9MkuIslB zQwUv^_V)HxP&$$UOQfDCghD$^_!N)6jfm)HiQCZ$x=2n8K48CciqtqYL&4_8pbR`O zkdiKcd3%+Gxljif6RT3SJ{%t0UYY3F2w@61vUw{SaN-?xjb47>^UJuHYU-?JcZ2}H z3T&$Jn1!J3$1N)(W7GAC3+;{*YHe-heDp}~aCdbW*x>SDZq?bdXQQK|3$qpDIM~_c ziK9|cd4F_#>^2vpYd!Nd6;q(C2xiUZ^yD10(Jweb$iARIGa!>GAgOqEHi$Ss%EA+Q zuU})9N6M}8PafLQztnFIa*y_882-H@~Rpbq;L?L@-2n5nT+I0FjUy5*^l+^ZfV5h>-NwUg1 zlY^ddxHj~nsU24pG`KGLYQN&!jqA4C1fC4jZ}a^qGm^Y$GRHTy@F*;Z1n^RH<3+_3JvF zFG{~ziU~w@t$NrD=5#|Fe3WCZqcQF+p_O-FTgbueZWwBn5>uq;d#q(TP;MSYFP z9BS|tW_a|$t#&;avXb%4V|(5iTIJ9Hj3`4TS=45#(LaR{)H*i$iIqIrAG}^1Bv`XsHrzg1iZV*@vALycULM2P^}sa8 zTv*V`O~iWdSc zGtUQJQzO4@a;tm{(-a_uQqbdx*YIiyL1fjB8)j$GYb>ZFbBasezpn;P?|&5>CjJ%* z5%pO4Gzs66h<70FxS<+NHkLx;8OV@=hU`~Jc2*Xl}F-=JIs6%PdwRWa9^iDf10tccEZZDFovN|50 z2S8VFu;2#0S*hNvn<#!J@(Sf2Klobi1B`gDYK$d!Fn?twNdr&aYFtsX4~&L_9XJf% z-4$&utt`UsYGzUrJ<6o1)N2Tl#bZ!O4Ai%7u?;o)``QCM0xU+Ed5gFJ zG_0x=ek==|1BW2~lc@W07C@ERxlDUQMn{Ixm{eduoXpJ6!fx{Q_h;sd`t82~@DxeS z16};~@!niY-11h=wa^x#0|_Jvu%E04=rg$iTa*t5ad1Gz6Y-<4^ue04MC*-W3kEA1 z+4%O6gm)OEx&1n;gM`F4{rN^qaz$OM@8kS-Is6P%q;*1=52jpx>5=yA!K1zYO71&L zuE3ky7T=OmDWC7^N3bXgdTpeUSz(O@;7O7PE9IO#KI82C8Yd?wwBpuj7b$4CU`V+>z9X7tuHoV>i9d-7rE}=PWvKMH8xbw1kZD0Ao_wp z0*lB&8ZeNwq@)%8y}F4U*>IX7kR{r#PeKjvZC6Cl1?ZTYXZ`&7f`zIlh5YJOEeD5b z!dtMkcxIwaxbJV6GCXoh!_E{If^DU3BhkZ&YB5lJcirC9)D$3tYGMLp)<$MEl9K#` zFF`yuK7LM{?pmlc0FYP^yTHa~4$Ee5hp9uM39;Pz1pqh3Bf4S!TN(4cfvj0TbrPy zbhA9r3T??JrxS{$X^~lI0>Sp(`}Z19XsyR;!6<^9$}7MHJ&TVX2YvK43f zkFMwXQ84xON4YPajw`ftp0=7VI4i#OnYpKolJGl0pl5nY3tuuvG!bCH8}1}2-j*Mz zE;U0&7xF+qc>&)6)%a?EYa!vO(Zlj>W@FtO3jJ~UY;YRq@h{Vr_e-r ztSJp82--@#6)+hpH4Cl1mj0DN+)E8SqSoQ=O&d(Xc@g^e*FDePOuaPI$GKo z{pXDvnebx*Y-wOlK}ksqsly}q;)@d`xbyz(?Cj!pZMLCjQ>f5P`SrbTPuzW8PRp9>WgZuZ~RL?1AL zCj;gE3GL|Ijc!h`}(T)KhuQ4k1gK6&M9@^&MAv`8O}j z%p}IZ)QeMKVBbq#f=HLmYb5IX-e!pXsdVVIez5o;g}+lXQFPrMIPTz3VXe+so*ofo z1$ziUzrasuDk%woH1G+rZ~_W602@V%7*tu;yxM~E8duJq-i#61-p<#|1@l13G-VuO zsyRj#pD~JPtd=RlDDok5?0hU@G$`8^hGR(e6{*+6c4>R4pdAP^a2aw26p}qiq z$@MK5OV07(9+#Aq$)A7zx!iZh_+_rE2bhAwesn2vv=@x0J8>^aX^>kzbi!d6oJvf4 z!KWZn=mmA&{*Fw`Zs07>xUITnqx#QJ0g>IqH*dHI zTKSsZ6&C73u{G}P(GS72eyHf7Lq+*hG;V= zcDlRAIgxONxT!b%L$5LD^rlRpoo867C?63MAX{1L;X-{)P(n!Bo>r$Sm~RQUs}yKY zJUa3_I81HlsW#*V@prJ4t_+HNbfb`(vAV)$&=Mp7rCsitVQhI|{iYEza4n8Irf6aeHE!?$E3XMHd7f^yI;fmj zY5n7K=TMbnssi7c=QJVkebYwZDv&SF@kO;v_sw@35a;pM*MnsHPpPTJ+Q{oeNL^Jy z*ihcHNqLR8Cr4AK$Es8C!(_tVdhU~;i{Iv4Z1&b9M4f-W9vB=1hGm@p^$|$e5~Kps zLCngr>`vg|<;9j4LOy0lnLl)#G(30FFb9+~wNRzBwCivgbS;n> z7M-y^o52g_G$DSpv2YRWb9|m#?QYqLm^cF%2AoJ5gJ}z(i5kjr9tTuD{~V)c%@e=7u5Q?69nb1 z$9z94w^9#Kx|mq%1|N3TOHAFr3MzcP2Lj>EXxM~9RiO@& zJGS}Mv(AXWTL9Na-Kt)pim`uJqnL;ColY{=PgQ zmW=HuRjX!2afz~SO&e8YO?@LZJ~!hLmndJ1_KjgD^>6?KH2nO0K^d*Xp^AfzO^);$ zvkgeBm$4`$Flu(lXoGL$@a3`PuiiM+4e*>rpoHRU=@17VlNdEwSWh%oU6)KtJFM3% z$||cBp`!Tj-~3wT;{0C5qepVs8Vkm|zFU2rhZznJu-SE=L9)X2&~q1hTH_FC#E>|z z$_>`>LI8O>dL4*aG2sQaP$S5B_RTcm_{&>Rpd;X>PKr$6BPbYUT*u?@ zAbpxleV_?}Se+;gH5dp_c#N%MGQ3?uwO!%LOTXQEchZU4lfNshO|L@*xt$ZY zYbaGWw6ZqN$YVK+@7R>;xo7CPG><(Bt2PX7(?jXFFAeB7_~4^$5)veQ->al>*W1@~ zzRfFpQCL{8n_d0k!*e6wRl)2^K7-0}*I|2Yj7Jmj7kBT`<%5(BWQBO~ayDUMT~M{^ z2X&{IxVVLK+~rI=L3eURnRYYJQqp27*0?Z4-!TD?FT~Y@_*sm7jdDo3X*8q)PWsp~ ztp?C0K9z3tSEQdUnYX0iR{X!AG(KRyy?g7NojW?9Sr zjDM|)$A%YnnxkwIKv3n2*>C^$jop>HNO(EJB*&&>)uVk?2+%g{xr6#IEN$!ytHKi8 zC5xuFfb&&t-u?WNna8+3YoJX>SOughEhS!SLVSdC>Y(;{Rl1|Qd%H`eX!%hQyS~Ay z`$Y{Ey`2?`6QAADr0ZjchO^jaIr;?jWRalns)C3!>;80d676vLWAd|#tX@oqcje26 zV~lV9W*D`M{UNWlg;f%}=e;*njrM9-8Hb`jepH8rxEk%D3hZyPlrwwBK4iD$F&}bV zu(cbt69NSrEbub!Kc9i^f2kWL5yi`6OMkPwM8X7SuKs-lceFDeF=DY+m*e=dXvsh= zUlhir?t8TFEc5J{mL99H@bI={f7Bvp!7mnxWT>%LWdNI1tJ0Oick@N6(}_Mxhd~Pd z6yX@2KKd?Q7J-hdaFvH^2VBB*WuN%Why6&nl&vx8yY3&sBvsVR4=A8m=7tkOVhr!! zlxv#h{!>I{<%oI0fC%-UQo@=c4KQ48Qpbaxu7xLv0y^nAB0@bq*~1AM|6&P9On#JK ze$Vf)xhJCC^>N?a!E;Lsy6VhOQPGFwY+C9UF;iuv9 z;N%?$><{w->1_lBr9s`oQPDQh8xjxvOBCyTIipfJ>@Ul@N|q5f_U?=W+>twHuG8kk z0i8muKYWYBNZ@|&_BQF2b1-L>-z4!($-!tpo4fj^;+FaAr;U_=uJiHqiwM!W_g{F> zhd-fcfjJKhOzob}!cxK$Z&9zVf_wpKTeSVJ`}3N^J)|y>Ug)^b?auHn)TTBjISP5U zEuQql;s2al)$vk>7P?0j{$h{8B`D2GJA^eleq9W}2rjwb`yIUZ3slb0uP*zC)lPHx z#47N64*<^b=7)b>f$6!~ne%_2pXb~4AD=(|k4usKPCdZ4HqHpe=a`qd{Bzl%TZ877 z|Ge2h`!728$2mLhz`w6b;vO0Qn^Esvy8q!HEdU_D|D9bgX@AHP)%q(;fz#4@{=3os zccWtDAu1Gz$G`4Iyn`LTa=SVFp8@{&qy2Aq{;!#>zb{IXw)UX>w;BGPT8{Xx;=dOm zP}D5pKjzKk-$odS%Hbrtbm>DkEXI^ZP@6>!m!y&ll7d;G2~1I#Z3euc{lV|+|A?T4 zU@x9O-vN?^B_Wp56MO9uR0!~WIMHwuK_n*U&6`W@bFp=_(g(v%%7%F}=$x;=&kU@B zncq5}h`D=HpWKkdkT;iDD+)Lbgkf3))dMg?e~TS9Vo|IEkl-rss87wof6cjcSe9$s zK66zbauj^<_srE>0Q#Jj<8m5h(UtqZKYMKdpgTeT;xE?;Zl@-}{4WzC{WlZ(w@3a@ zrjp`*Ejq&XT=?(v)e?!T!OU_dy4n5A-=ojt$!X{R#e;*(|G2U&u8$MmA`1>UXi!3; zi=>~BzkBacNKPlr{_tT%k#v3)w=)mb$=qI<#~O4veZS(CfYtISt_X#@|9dhDP(4xT z$qgpo2)pDuj5`f`+gzMUQHq2w7QD`BG!zK`RrVkW^ocDbrx7s5^16$4_B{+2wV#TM zsP|*OEX*{sDyXSDlURE0*A>;|+M$O(Oko?lp|Jb@AuAJ#tkx@WJGwztmh3B0`eg7O z2uIe25TQiXvVGwG(HJKIsr`gM4%MXo{0 z-FEKaqhk^Mgs+rX3^KC0>Bv}C%WWDY9J=4x%9$aRMG}`(W5SxL|-g@PFwRDOHMF0123h#x%NM@u^>?zr3SPzoK|1Q_+Lp=Zhf1Y9j(z6DXzb58=m)Nr&$Qr}oeuhaLs#=;ac5oF9 z0mI(GCOTTI!(YjzY&X*WBK$a}*cs^LBXTijMYNeQT)N9ads2O=r#oz_ z4Ip3Z!w;Y8Xy%9)LZa9JCzjLf!+9N)<*ibEReB}2lP+GP$-^LZaZ6SyyyYt`nmLQ0 zNX1eK%B(iaL%9Vm1-)GNGSuT|^2<8t)~!qg7HCstW(SEP{D^@y~X*_T*%a zZSxSNs=Dzq>bhgM^+K2R(fqE%b;Q{82a_p9mO3L5$0k+*HaU^_+@T*My5{zSl^UfB z_1ih)?WD87RmD?>l})9xwoHGBPi3)rL}aN-9cbz6>jUN^H97@uXae8N26_X40n>qyAs>*qwfjHAsAy>mJg5A1L6M=2 zDPU+d%0ag{%-80iY|i5A-d9{P7f5h$!zfK3f-~B>|EaUH^BlH3VCux>aM?|C&P|l+ z=FLnV10mdm=OiX+#I=Jv5l98r8gaZ0g7yhO+Y=KVoh8PqiUn2i7|AM29;2JPVTZ9I zDV)gq&7$xdEJe4PrsLv7B(KEsBHS$fQYj|GlO_EL-=cRm<(MFd>({UEG+*KSwziPV zDB<(1x3|c15-!IZr>Gdqt^VRCpGBp9>OVnENi!x<0~!>5k$kpCltq9o*G)V=*L;0a z`ma6Zk1XPbWh1lSYxBE&I5~0SjoKnijk$rP!|c{SnMJTDon3?Cj)9W&asBv|$_cqh zc?7q5zlQN-O^|0DS*P4HdHr~P`uEMdl@1739P0(cfT*i2YTw0}sPQ@>6TAnc@&_?_ zO$h^o1DJXl!~9jiFo{OH=<&vNmh&7uFC*aBO)MqAsNWCzM=@6|A-Z9jQLM_8jEqIv zmyzO!+|*$i85tcB`Cet6s~(-O@&e~)GcLG%cKt+9=crq+mAk<&xa?GYoIlLLI- zU0RvN#r{l|%I@UD2n(@x7LhaMhEJeZG-HDz4;zl{0*_x3O zx2a}*I}$G(6hyQM0Y+JC$)-)wUdnC1gjP}C53{GK~GOll8y-EAGoR8-5@y}Z3*Y&fq1BPJF*PF zg8KZns?&YvP10MjInkuMkS!(&*^)Wt2`iK9!m=Ll{KA zAeC`C3li{O$SQDD4c878SpDQO=_dm}7s|2!v8i22ac_l7uTlO2S1iZkUeuh_i)h2T z>lSaVXGc=^x^>GcZMd5L%;e-YDDir^HQ%Grm*q>jH9P3CI)WPKlZ^6S&MHeiNGwz~ zquT1bfw*cRap>7T7wx_C#zpE>>m(?I<44G4m(@{yIMcvjH2$a7h^U*+H)+*x?kjmN zlU|Vd{Hqp{$63H#ka_PHm+dtu9fU2U7He`qVB^)L^YHD66#q4m2K>CiD5#+3^Q+`) z?oGND*SfD41&2{D0}0d6&e9;-Yx8>XC}cEyyM$z|s-MlsuulxY_?5(~iwMd0w=Z}g(*e!ndNZ1mVM33n zeR*<@ltVyZTU>4zcLT`g!lT%ben-XA2^*I6SUC*C-^j|iRL?fxUnybQ!+%c+-5E#< z<`?_5@GJf0MJ>PYO`<0K=|7a!|64NbG*EWPvB5qQ3{)+F+5_eVQxF7_D&*ULk}4EP zszm8k^m$~JwJr>C{7$0)&99AQEP6{o{znQG{#AKKsm2IX#0~_s9(BqgPhOGX;|7d(*GYIEIaDbg9MteDYg)22)o&0$H^4HTqty0>Hz~=6t$O_4H(LFR zAcy3*efy2D4oO~M>x-DCfHXs1tL#YUnxy*1vmn809qSeMn><5u1l5|CC=066^J+%J}iJBnbhY3evr{3_M&dx!gGJNQZrRUKrDH(zTvBA)g8MOL& z`pMzcDJTJgfMw)4;a+Cw?GKw10*QQ>NpA^B;u5fs=LAz13ErWXffpzthnbFWd4II%_b z%L+g$sXCb3!~;YxUT=wPCZO_T;tP3-QQ$X7-D2a~)k*^@x|>KqZsMfYkL7^cbug~H zQ4Qn~Lr~(%F4fu=>jII|TFruk(a_0VBJFZZz|z{id3p2^XqVUzKXZ;()5)m>&IW zQ;T*uK1Lz3rY4;lSd~7d8L%p8IXP{JOa;umFr%Wzs-?oW@~QcZxgkJBQUc$rOQZxt z&eu%k?B6vx-O4hC&#?f#iqb@_JErRv6AXwlIsJdJ-30T4e7Z)`R?7j0iXay zi{5z>#X3>ccagk8)D&$DKcK2H4_TETaHKo*1V5nG3~?J}Tu#?CEM+hTv} zrUU9MfiR@ykH^p(`A(o|MeVGOLeGCpWiJ(?RzfA`39GaidK1^jVA& zz6JW_IYiwHZ|&k;Q0P6bch-DMNbx%2BW$Ox?G>P8BWRb3RZnCNDd_uJG#S->d}=|D zgZhQ@=NCb-04UFyf$l~-;AAkBs71(ERn=B=Dgjyeh1F4f7^ClsI*`M=!&N}FPZBme zcMt72W$X`98&E3i`*9~o;t+@!1l^Z4K_=w0*&4v{VDqK~;d|!0w-FrpN$B{1MJF*N z0qzFDj9>GATj?;7la!RC&~Umo)nr0Sk+WDOJf5bWDaQen!(yImLnfz$EDyUWb%!ya zmzzV zMjBUAWw9{^JOy+V2N##(cEFK3V9eA&6}`f9-57|l=N9I;Smr)lyh@XnmnVOWYYngg zeE#UpHY-pjX!#R%G2_Jx*!_bupw>$S;6}A>{ax__?$1xp08w1l$9uAib&gF@%tkH{ zM?vep$AdYl!!M6XE0z|NJ36{FoYX$R4=i-w?qR=ulUHp(-q0b+@&MZlR64akXU1$t z4F*+vwIucqivgJ%9h$S4Gc}P#>}&y!G8ZVykwo#ci%XftwVY^cG0Dc4%u6lt0zggS z0x8&NfcG~C&%rI(`K@~BQHG@&hkL7oZ$91wf?-4ATimc?pZl;$E@%>oMj-AI^}zZ> z^?tsp1PbV1NB1#jBww2IEDLH{9vpU+?GZIS;^Ies!RJXHN9Y5UT;uUP9XW%LR-6gE z1`e$$=sE@h@};FEz^_IzTP&C>B0!e`aze+$WwRNL%FG2cIA59F#6UmD9PvIHsB%^5ZFRvWZ>t=EX>S{RZR`XL?IDy*+cAQADKB; zzdwKkQ|$Qx>c3at(bzZzTd1F@NHEL*WGT?+P;u#4f8S^pljGVx@dd70+1EB<(sxe+ zzcQj&rIeHwXM7U3_Q@|-CFzm6W!>6~DHNAa@!q^ZP|Gyu(eP3sJc(#q?d=f4MoXmo#*e>TQ||qE3aYXA(aoJ1#H@FD6mvj8KyXONvX_rf{#J-- z9wdr&6m6@EYQ04ASQ|9U*p5}Mg660G4|k;9W|V~ihBf9!pW4*{%CTw;1vDhK2U~L~ ztG!Jd_027Gu&pdH&Iui$?{NpuxQNz0MP0Z;oo8|i_c_{Z>j>Y|gPE4?2ZL*D-i|J} z+CNL;ln%5ow32?+%uC%>lkBDITb*(+!cJ9;N8v^tEK~MlM~@1k+j0=J9e{2HO+Z0& z76yR5=<}0L)%we~kDv`C3aBjfm-eQJ@vnocc&NNr90?TEF#PiQnhBNTnbcFYg&r0u zf#}?f%KC#YwU4aI3G9OSytUeGUKu$#97WaX6HvCw0+JSXKH*^;A&?MqK}xL{@K=Vw z5nwR7CMY2FULNS=#oUXOyc^b<0VK&j9e}w~0XwOGwmpjS)~)ZXYlEcHJ z2y1og#C-!q&B^6QKs~?HLLp^6w5VQ-$4&vo@R{1}9 zDvWOp1^P`8%=+9R#H}p{>S}=GI<9~p`+(J00<}(EC8fgAN?Rl;V?EHvM0D6wkOW?V ztWP$8){^vI$zvr=#)O{-d=DO|ONU(h2#^C+%fHBy*??i zV0UO}pNtcbwm{pA$@YpLhJ#Ly+_bdYB4lLsqKtX#^4N??-`#kt3J|#gA?ziw@%fmH z!Z9y5C7}SsGoKE65f?4bN$=c?6t}W^PxoqoqQo%tPcU~4d`(NxBF{H+EkW|w9%vcbkd<)gK@?FiVdRa`uEnj1p*?$zAsSjUUlCa& z<@2C^lvB6Xs3;;L;&W8W=C(Wwk$5=wL>bu1Sk*evsAyosu8};Ja5aTN{X|PIhOfDF65)kL_fux^{0d{icbRy6+x!V;gmVqeLR`ZmLo! z(PPJH=B$p!Q&u3o`xwfk?cnep7(gVk)IPUhHLQ7>v>e}Wpgy($!x^UD;{}8!`YHzC2=;YNo>j7yKvJDEPAK!v3(rlsx$SP}2UsSHn?lSbq7XwDDv0%lDQF5Jx1a9BqA6cOn|F5L!KlQE}7?F}BsS=~! zqPp&o``;DP&m#ZEUj1LsO};OSf^NX>gh!jyc-)jKb(l7YRsC&D{+tk6|5j@wcZ@dw zRh#KoGK8b;Ufd(lbR1Z~f!pqh5T49CbB@%j<8`oLUs-nN&hFT{H4);9xV)xU$z>)& zGd$RLL)!Z!B++4@gV*peVP^>mCUe5Km&u}hWan$5BRf#83=Br%YH>vv3C}4mfbWN= zyg--IWuTxb!hHZ)N4>`6LUOOzrs}aKfb+4;!E&!bK`yBPV9sIf?<6Ez zWPk{*4$NXw9RzV4D3w9cG$z4E>&eR}upwRzqTPWG24W91f3@u zs0($9_90CEJ0FS7K*VwkP%#m+34ln926L6;m?zx#j6f0B7({=(W>H$uS{$*C}F6F0HGuEUmhEVC-;yK4u8M@v0*YeThY+)fMcc^G zFdPx3xhxtk+A8vcUOwsETi4sp*O+@=G6Oy3;drp0o_DHeb8|Cj1JETFY*Oq8bDAO7G^MHMM=l2=r})p2UIesV6OVEZ_-$ui^#DUH+Dr)nGn8Z+Q;sG{)X9S8u{>-Uw;3nu)rIw zA6fs*x;(nITeo$z(fdV-HEMTZz91vxDPneXUc=DPuwjgcoqe{Nq>Y5xnf|s{*I&FO zBz-R6dzNfHv7KmTfTjdHrH*%Q0FkzdN1YMqics@AdcN9=I{piINx6HnhWQ&CPN2^< zf>A)$*JJ%7v-{GhHiNwoNGL#qrH1^azTtng0K_~3)MEwhN41Csp3jj_3pC|QteVK( zntw>5KvRURSP)!x6|%Qsw$wngvfv(4fa5(g)BOsA)zxcP+z9DZ2XdQ&sZ0u4pq zyC)5jS8ebh+S=Mnkrds!aYg+d9i!QW$@&1t)UK4hgjRCvGuSiW2g3L|aCodP5CuFA zvsv-6>iYD6u$bAv$#GDW1FU=mP~ob3c&L+cID+Nbj zPBk8D;MN+W`^~QM+nkSTaiKmJBtg_87WR8OPSasHc zZf^hkounjbPkzr=ppe=3l4;NzhDr7u&;Ib?4;DT}9ZSpDEy>QPmL)Zffv2DYvnr>d zWoZb6sx&dnNsh?2%-)>PLU> z!=r1rLD}@cf%5i0GsE&sO5a6C>l<&S?T=3Hr5WUrvATCDF-J`#;cxLY^D`>WH; zbXpfQh^Y=0n*@xkdH@;wlg-T_wA9|?k`Ay|>f^tGkhQ>nx9vNqxGEOk%9QCTiMp|y z&tL(^ngXvjW z!J~Daj>usQx=J$$j>*OB9lEBxUz#1f7j}o^!X{hac=4~ZKx;*V3dRUN@BP%FVw45Z3`zFFVv4T$yY*5-$7VU3&8T^0P> zaATci&@4-05WEC{4ZRDHN_M9t=PfSs*3OUG!WtHw9%*Z63@j36ZhW=fn#E}u8KJ^K z`>tg-ZEG6^HhL^44^(PEiwm~VzeYhB62sj6NkqD7zf}$Rdz5~I?5$g3%M)wD-mL>U zy?)sB6-EwZcT*s1^N9MNDyLGnL)|L!Y*R;}iV<9&~rl5fdh7>qyOlnYO2R58Hx z?H~DFy?ESly6VsLJbC}lrMEB#?~onklz=<#dEIxd_D0Q|=Dkw8H%AgOv`!CZ?0T-8 zY*^X(Cl5mj)pV3M#!WIVNIzRBflkAu+n%IR-elCB%P)X-CQaCe=J7nxJwp;$8CF6~ zGI}{TGwS*Rb4Ndif^V5?bp1nphpM{yV@2MYXHjgbSc&qre&Y6e{saN4gxLcFQDm1( zmp=wa7Sz^4Bc)D0@iKYTfP_9j%ojgkubIOsS{r6L;p!P=3YxSzd$*0uyeHqrE;}&X z+{tgEa0b-27Sxt0WXq`z z*S~*Pf!n`4bG+oEZGrB1pS(L!^0v7bcU#eVwQW09v~q+)}Qc& zji*#>obCj3?V$D41AjtIhejl)N;S@ZeW4>VC3e9LZGzuf>^Hh_AsCLfAO8p~zH0`X zIzvB#+Qq%u7W+mI+qCVRljEAr-zk=AjAjr+6$n&PG59yWrx$_duEs9}mydX0mN;V= zP~vqj{4YsKX~kYUiAV>%g$u~LSrwCePoH*^As4J37aVRo&wDS(v5|G+j84DkDj$Xg zcicERzPgVz%hFE&I2CXnw)n{UU2Gno2|D2N{&p&hiL16ht!VC$XFYK5Gj|Fv1bW&Y0DBkrC6%Kt1V#aP@k5U z7BPFiN^`f3A^ z9_cy(Tv>4rgg|J`i(qPrQhDkyNa(dW+nKKs=uyDU{Qtp_sKfE6zi#t`sToT0h?GHO z3zah|gvRp!hI^Sf`_X8igBn=omh#xio9_+C!}5dL9E)O#J-qbSnUAkWKKbjU zmn~MP){n4bLo4i1IL-)Ue3*_n&>=xxK{!KVed+f-(z${?r-1koPB=xp7Y%QZXP!~4`pS|x#pZ>jCZ`_ z9k0n+`b~IIcl`@cwh170W>!|#z(~kXM@N;R*qc{{rOcI_gF+(NfR9t`HVvtt8+W$` zW#)jW!wmM7YR`WS(Y|htZd6r?B;M1H2gwGf%^eN`4~K4Ae|wwMjJQGr7Nn1FuIxVj z@RUSDW8*^q1qrYN$md(jf>g(7OG3JnC4K^6%s_6TD0xCcO!mu{`(;8bW8{EmMY1VQ zOuUrA+1Fcp*b1yZmfLqkJqcbC<(@!)ZYm;9z5 zx0(ddU%dfUC{V8#Pr$1<8 z*?D<+A#XTn`R&(_CW0amRGpdx3?1P=*ebbWs*%J-o@%Qkj^BA`@3i44oEcwU^@cE) zJ9lZ%_35ag;b9#AiWXpurBcPe_XUc7mrM7DUyx4-dviqxg70H*(NIXE5F@58${oOsrH7nyv?ytQu zvCRvepbwz;=!dVTttDa6obugA59lY$Jz>8Eg;4w_?sFYLesVcV-V=ugG=H(0xRLeY z!$<0(iLUI(jn&c3d7`2o2xt-W6LNqIx}x(txI(oevF9^>t8aEl_N~R)rdTIfWIiKa zdu{BNkG2d#e0hTL1yu%|fk1!~eg3OMo(X@ZwEOy9DoF51MJ3uaEuypZ1Mr5>B_ z?;o4u(0?X3bFbz7D^R|q0voc$d2_8!yxENzo^l6NdV3TRJIcvgvwv6|X8U256^m_- zFI%aCN>^XsdJ{ourb;)#y3BE!kn`t?sExBZzCXTAzZUBSp{Pb8NzhUp$xt)nZt`1h z)%+qGcv+KtDHb5Y#u0k~LRwZs$JBaQcsRwX18QDPX8&je{@MfcoeowxJUo2cU~rSk zLQfjD(292~Ob0rz|m={T2O=+I>E*8v6M71v$9 zR;V%sOUs2uUf|bEewHI8@g9E}E z*@U?zC6C2=J=WgMdfnJfUk`dl0`AVq$tmUI(U$p{S_8#O1|y?0ngD>XuwPr1{Ji9! zyKu(X>bSO-si#U>1YuiMTHr6{|=|5EmMB zh$Kk4VGX69H8eD^$ixU$=bn)q8nG{RoEle#h(JbVv{_z{Kvn?*1U1?B%h0uuJg?zy za<}vJS)J@Jclj^dm4G+-oc)|PX&lj03`}F1g-^T3-#QI&mqhE=P28~y_ z-}9ng45Mn|&h4{on~|rGZYo6cx0#lQb82932uKjGc&*(n0!mz+#`{B zjm^cH&sKu$pDc0PPZ+V-Nt^tP=TouhrIaT zNaJ41HXGP+(ZWtIYlBV_3#YqtXW2Q4J{lSO%@`9rDKAV0cqIXFgnjffm^sI5!_Y8S z5RyP;n;WZ(1WPYfumHD$Po%gMRh*NVY{z8_6Rfi4AAuKmyOJ`Q%^>QArI7r)99mM% zKm$g*(gX(vxMF}Zs|II|**>;VnZl!{jU$A##l^+w5kMZpA|qES4pQ0_s(G`$@Nc>& z_aFFTlowVUZJ1a&eC55u3qPMC+Yj2Yp>Q1paISBYK~ol$*S^=3O-^QaLj8UMkqJuK z$Gb4e?c?E>0bTIFFlJ~`0%h}09j(-^^;osyfP?=~m(1U#WbcIR#dwE~%#7GxXd z_8n!BVIKWCzA_dm^r&)}C_FB$C+ynEx#pNn+J*Awi%RAh|Do=*-a?+!bPCsTc2B3{ z*!Rl-&Z6~~2voR$lpNsazPW-H&H7e>vS4f}(>^uYEzKHqN`U_*? zw4K7y2v22tLn(@WN3Fzz11!J4Bnm9MHyJ)9#xd);D;T(D2k$CBu(J)yh0{}{ukI#X z3))dpDDbxQj*kPT=R+LseK6!Oa;5y^?*K3OtsG(i4(&|vQqHXqvFS5waU(n1AMhSV zU6+C_J`0rvZlrKc?8D)cHQc){VWy@|8JmvnEdR~FYj%B#Gs9P{j9QA?ZHgdX1^m)) z8!;Vw_1i8~O(%}H%gxgjs!S=&pd@rJ^0!&=b`E$;HysaQL5UZ0`beXyfO-4Vzkm4h zE#30 z@~;F2hBL`BX5YTyk=#6xbZfym{+3ilN%+8$PXpl03?Qd!!a@30gUvbS&7>ci2gp6n zA3p69el-3{uo9P9%DM&OK?d*#?_M*ZrRhd>mGTalff;8|;04h3ZWcEf?WcMku)W8|outjwJ?Cw7xPiprM=mb6Dcbo`D)P=_AD z3>J4+l{K5jD8?o8+uhUURQoeWOF?j9K+o?VqxG05>`#3cS#(3-_chICVJLa%wK=f_ zYtYW?0XwwyN~}Bw=4nZ<-bJkU+w_t~#-D=2X+}6AaqaDw6RfR@-u^G&-W+9mwqVzu zKzpQpIjNK~B0%{WsuW$x0|l^!fxXi-p`4qgv^~A83Z80lMgf zpw-<%^01?@Q$6ePRV(t!m;t}L!S8E%^E|~}iUq3Fs@2h>9eSFZ^OGjiW8#|&x1yS= zympEaR+S5v{l<~sv;ry9xKW{L|2I+i)u!i4Gh`GfPnfnhLih%`NCiIE{)4jdDeJ$(Hy_w~`g7~BJJSTAn z;Jp*`jXdNYL7;*aL%o>iYVRO8rk?2eiSvluf}*AxhsarZMN?A{2Q@fu(ASu6+PQH- zfK?rUpR4_UCVA}^cxs43RXlp>gwsr4PyP}d_ze7yUhSr4>YPGXuK;QK-SE1JE#eSK zG!na~w+Au`Del<4h>!iW$|57w?-+VZParEz0T66Jkli4f`HzVG#><1Kd$-{cB`2B+vcxxo$f`h*Xri!(C3c2$<|mn!e3MMDA^!+$6*)Oj)DrU22Zg0+oqKH8`N)z=rFez5<0P{EvPayV>+O z2sEnvb8K4%$kz3B?}LWl>AgQa^!|P(M=A#RhS%I?E3GLb;|3N}-`>6SKY2Q|zVqia z__ZM6IFTzf2=z2IkPk;bhMr9^I+4q^T4%wGFSH)BURs)!k@1S9w5SxE=Hp+F032`# z_W3Eujd0o8wnY>dfB7+vsqq=^Dz;@NRb-?CYtiHCsW@5&1`hBnV4gq)5&S|TSQrpi zt+FSNa0t0nLs0*u{}_BAitCF74sPpVnQ3=UG=N45jqy5ZP+6k`ELAn2eD0^J-v0oO z9)38@t+i$9T?l1P*}xKHm;!16=$H4r;Wb=rlXh2Enb&J7n4h-?(%B9F_`59rpD~ht z^OmA5;BlS{aO?kL=81gV)F+PcL^h@x zf`I1E@~CTkPUeGh@SD4RVIwY`D7M7!Dd(T9S)rZ*HHm@8)3MAzw_ut&@+64&-d}zR z;-Rqf@bG+Nh2V|v-%-h9AuM>oNWfhzemlMHmijSGY`%AMor*x5*+?)(O&J)9;Oa&vO@8VFIWAZXXfeG(L5L%Q0!m3bCYg$SlNBh+;fL8M6=jlX&kE-O z3223VN00EEx3fgaw_np4=mKRTA9Z0AD)3)88SIiW2%mB8WTvZeah8zY5r*^qjyri~ zz2|beO~u^qe%=K|2t~OR^pl~0^B65H)_DT3#ZQ06ZC(2(G#+pr>ldE=Z6G?eyZ?@l z0c8x2TTgnh`ngC0d_n$w@F0;9H1AWCKF``L6PBVHwq!JR!PUF~^rM`zDk{FbdXW_f|%#qr7@eKA z#eX_*LEq&T6pCv9$Ee89M-Lz;(qGTy?}Pa3h-_qIxPJOH(}4Pf&XO|@^9`}vp}QYKY>f^ zvxOp8V-K>ja&-<=y8GO{ib7Rw8JyO?_sG96q@&$I043Ik=Bqk6;Vu`=wt|WrEwlJt zRht;_a?XOssRfWR2Y96+{Dz|k)`yNujyRX&Oh^pIMpI^_D#%|hjV#X%M~hxlZfnzX z1h#AEMIH{ZH1%wa`r)}R!*ihSX?x6s>MvhzXSNXvg?P27X$gc=v3>{yQ-5zi3tZ{X z5IKI=d82+lA``N5spweDp=OeUOEmM6tUd$gQ z#*6ZVef$B6rr;_6r_%vyVc*Jv3yaWJS`}SKa!GDqUY#MlJ0}UweFzBB4R*geGY8ca z!p@6kO?#rh#zmE3XH%hu(FXtv$JKyd9MMNFn*4ojH+jVF3ak3=90acxN95qq4r3Pe zd~#~)>Z~%d|7fJ$@UwtMSw>Lp7(zVA)`7p7BCQ8cfkx2brGy!)z_-%lMBA#ou3DIE zopn`jOC;Hc>+Xs8{NOc^+04x|O`6K2LA#+nBl{M@%L!pp%yQXO%46>gwbaf4mI8`; zFEw=+RL^6f;1xtiB|?n;$|nBHZcplozhQ3PhI+-~fF!PV4nXjm3{sJeHpNJ(5aI%@ zm+rB^)z{azJwL3;AtIs+47!xMfVTL4cNQ54($%G76u?rRyA{y6v11R2W0*mOJxIJx zt60cp0qlMuw4OF#2H=S=MwiWliZoelQO z5TxVG_Q=Z0!d?G6iBHO)^<(yAmWef{tbwb%Qk5!vCu$5-;HpD= zRLFDNNxUCc1mNt3)Q4LWumbrAJhB9*ido-^g9RP$5-+oUXCZ%jN~csxhJTmrQdao z5%{I)M~;r&4mrZc0xB9Zm!mKM4`u^(u~^oiS>tqVB0G`c`sv#Dc>9!npXjlX4nr+* zS9e=a$38FH&a0{TpU*<=V#8l2@%vwGcQ|FoEs==us|c~+v2sG^50`<5kw`wM;-Xem zl-A^wNdCYGLfp!6Um5vG+s~@aXOqAVMFwLk5$vL(k_H|dz8k;%CXn66ykS?)p%}OS z0S%h!6aAw;o4{4_!?JZ6K7>tT8X=MrzIL81z>bfAhI#u|Lh86QZ z;JVajbJxl)U{e-6eF}RhGY5O|i>UQohgyjcsQYs2?ab6JwO4N{s6tbaun)j&buBH> zf2KLs%OkkAazP0mSCUg82Mp<({ki?{xY8Yf(n62LIh>u`|4El!2H<#59jTdOSo?$w zL5~x8L`Vuhv1w(+w)}6zUKMch%EoL+N-Wx`wVkJFY8W>ST^45xXc zOp}%kugUJp80VS3)MGrAofc?|EQx@}ER`gaXnp-k*{&}Ab<5#fl7p=6eY_GTWJ_07uj?M9{jGu;!k(N8UR-F{rI@m<$j;s=uFk2waFDXXK+ zT_L6son_yI9ZK;;b%=Oz-3s!tDTA9c!QTGpNf2T~`gFIcHwaYo;{z=~-=JZagd|x#sYYu#hs{1*X_^^Nx)~1Vnr| z^hslyCOQa)z~!~IZSH=UR`v=hc0Es*U#txbVymPO)b~COEv+-5Jwuv0&H2<+aN>WY z*_82epSC3wng+Lwh?k8g`*)T)m;&CC0fgnsRo&$Ic~uT>ZqT4T4%GWV$lXNrGi3sm zV;qVDtSS`+l;wi22|R$Z?bs8r>QP(B=%Zs(s&#y6D${GT>u%sF z0a>tTs*oHAi*c8TtX=9_nm!SNfireIJHHCJynfobN(x==hTrW|S9S4b+HME9;ee@m z$D0+w1X>}JXwIr{u7{5397O*&fI41jLHCd2+Lc+-T4AHkYB z@ru&sq01>@u`+yG6n$-MJHH%2;`LvVIKny@Xq8i+dhAP!;Hc=&ABj74Fr4;yHmDwh zOv4~_@mN6w7dq1O^>sFd(CZ{TlA!`lny}mA^WC3F`K5b>k|hGJLlV@n|0fZ;FhMrD zE8oitj?agisIGTcqv4TG33&!oj8uuR z8zITsR?I!CTAP=XX%8`tZ>w0L;o$Gg3SRQWSkGEhX)~kub<$h8x1|0wZjlXfHmb;n zXSpZc$J4Bto*(lMnHN8uRB_XDTgPL9m#6FZYmRhqL|QAr=zuMPj81HQvGA{Lsrhf5 zZbHfH#O90=4R6{v!-*9Yu@2j|*V3Y|)8uExkWUp%auiQ3DRQYDLB;ke#;+-w_;=+O z#TslwN$8Bs3ee!Mfk&EFbG6;-2lF3+G?e#Y#E@*gytn5ON8pR_$ffO6)3%ctr@jiEJw6$$Za`U`h2p3$9!_q)7CztEKyF1hQgYpTo47rifnW5Vd&-Ik$? z^(z2SV7U~kM}N5OLA8qU3xjLIwL()Xe_3bQ%NW)Arf_w+=0E(>a`ZXZ3xB~A|1Cb$ ze~&xmZ(UCe$Lp}><9i>(K+&}sqzL6r{8>~xG>qbR0;B9}D8&6=6I=_VK4^zlgvb>1 zoP{7v`vH8LUd72N96S&BW6t~p=9^LHx1pwYEwF$aykHt{^}6mf_~e;}b(h#Qz}ITA zD$0=D6O)xC;+e0iK5`$qI@Q0f?i;T~4Db0mYzvq8*yHhL>%))#{xM#pKHx7d^M4q;T(#`p{MrugzXoxA zghwPFT1-H%3$tPI&0_&awqY*;r(Rq6R_#Gn(TY1>o2%lGAU@_c*`ic$QrKm_kS;Eo zliI=^-ec5Vv9j>f&M~ViEu|?fHB8Cl9sGv10oqQG z&L%$FESX!iCXFXun|YI26}i+^Jjm1ro*kCY!8FltJL;Z9V3R7MxErN|^mv%Y;i06y zAJZHchg9g{JXWX(3FXz@OJ7Y2N5%OK0Q`v#y^`L|SXeb$Ft|R0$Ox-fmy=vVFyTVSq|-y5W5*af0HGtU zllfx$1|LW2rAhfSbtG&ozJj1 zysj|zYEAiIdQd#TyG?wK#6sEY`ZNh&Me~>^qPu!Yv#A7alKXB7I8MO3Do479Kr!(I z32%T;D;6|=@4ew>KNm{9bPUz%c?#O!K#nic?T7DiC*X<54Eh!OQSGPgNRd`3ezW8SQ%>^k;ONEl}BQir7yRqzxHK{Z?2ewqbYDjXuTvZq2)0| zn|UtH@%(f68hVbzrr~Do)3?P|eyW17h#JJdS>*L!zb@&yem(|il-4oRKGy|EniGHh zxUrrvxS;{`atLOjFUF2&@y#S*<}ep9a-cY3K<;;`)_0p=m1zg}^L7Y`TV?nLUqsSD zbX;yXpt3P@rrIknIi-b0B6u6hTE=ie4Jd>&rDUe*Z*x^MF%Ag;NRnD57mNk*RoK|8 z4$lz1?>Z(lwmF5`qsh3pqXXeJyOAfjWEg(F4`%#sA3@x7#rE( zAc2^+<&O`$AY;=Ws>I!WKEWfal-#0DelycFX)6is-BO%@@)aybNJsQbHY67oSB@K) zDGs4II5^q?|5rz-dF1}hlqXMkNRUE6wsICO5K`DsOL%Vb5JQ$!uy_)U*q8Qxm%&tE z?sQ=@p-9$yiPg~k*$v5D1_28U3v6WbpBb+q_oJh5(o1Ym&&@nX-_sQbxkZL+OWMc- z;;F*kuw5fkLy4NUroL;cfkQ{j*!yRrRwz@yw{&}xLqZ;aEbGC z_@>RxwaWtB&rXYB=ko9?*OY;}&u`o?OPVBT@^2=&PRBl3g6XT+;k@^Z~LEs-$0e(Ct@8XD+UkKjkM7C;Oulo|#;?c@V*Fu1ILDnnn!$)684GHT#)b{l?=pxht!vu8x!=S(#sS<^OWUQ^V zkY=kX6H0nSA}1;loq0ts=_{E}V98JVj4qm_emSOzFt-sDEx6%PwF;U#c>&WWC`5zm zO=5H4Ng)n+X5H(v1}rki^}l`0FP#{3NX)-_QlbOW_NSl6HaElV>00(cedr>Ydh->~ zxVx$=fUK>Aw8kK{Ni*XuT%Z^uRU|SQsoj=*cA$%^cpxWzqvEdD8&CJj)v|@WYmiP% z^<$!>Kp`&6#M<}*iwqUB)}4-KD?v_*2U1?uDXt`g`1XrmJLLFRDD{*kr)uTvHIdqt zlw*uyYpT7EcMP-0Xk_DRriBC};o*-D2a9#XTsTdt`NBfSo-_1wTU-{)Sahol!2oAR zhO#-;@9Lp8(!aUs5)ySLDo6eX-QF0MZ3-->HHvQRdaNy>lUoS|WaC`-?&`kWU-HfGS5H|&edQwtnP1;N@Yvx78=s7R zOWjAm`mauL3sJF8`Nl-?rQBLMp8JFMMzX|-Imw0~4M;!x8-(8Xcd|yh;$h12a5#xc+#&6@9}RN^3uH{jp_o${7E{C?$mb+VJsm0z$ruqS=%_MySbncEg}QrDV@+=hDh+h4xQVzd$+|9rW|WH z7UVX@JkdT@04verV{KsBY?b>6^Y?r#Ru2xNGP9<1r4gBs4fMWU)8DG}&O6xin-AU8b2Vn>(ks9Y?0Iy&yvjc)h_@m?EjH#WGKWFt zhrH>4D49vjTYc^6+M7%H$eeITqkCWOGFh6MZ~%(95!f^=QibO4nX64a~| znzSG-*nk3XdVRjA7wWGamwzR>S(n4E4r6B*w)^f4^$}rj9)%w{nh$lVZ7@9u9dfuE=U&FMLUJ_Hq#n(;y?Hjp3htOVKjo+*g^hmh07hlT8 zDnd_`J3fcgZZ#Xnhz(+(s_^qC`trDWh?`X)>#UO2m%DAp?hC9VHloX|J0@Fe1{aB< z56{X0mya?LDBkt{`VIk7xr4pocAqCKGHq?jWlIF+iP0F_5SA4R<#tGHvx*aOKo7Qf z9)7kl*8qG-P>E9NaKf+O-bCZ$kUDaWd@YB1JOr2G7HaNd(9Iwa2q+kkzeQTUkqG>F zlAda_f=Npcib47=6*lqyN6(z|o|{iS=N1wyJH}#2(9kmzs_U-DoKJ{Omu5oW?JxU0 znd-}9S+_JzA{d#NO#bKz&dAA8BM*8dFh#GtvJ%XWk0q=5yp{y_<2CfY1Ot~lHY%=3 zHUN1k(3~NCO9{@-czcJ>R1_L3(N6c013`gubSYsM>isJHRV5kRaUcnznnrYe(LhaKLnA+~W1{oPE}jg?XH!V|w)y1P*A7}Rnb1DZc-grDI&-aWycld8 z6qg8Yag|@Ekns*JVwx^>FQf48qw-1mIu&>K?W=U!Tqkpq%QKHL6uNep07Gljd`&^f zoUjP}%`%)5;?AC9m0|Y$wP{qS#cxf)x~&5%m49sRD&nP2`I!*1_D`c zw(m=d087`o9zl86@>2f^tWl4EU=M$eynb!}UG52Qzwi2+ra|?0=zaV&*oNOPp1%tt z^nqLV=ereuM}&{I{qu#_w);qKCPqg$5^gIf;DjB233b&ycxTrB^97}U)-ydzHnP2g zM$nAUx$iD)3!k!yw7LCz-&(?QeMY%;^X547=2|(lQw@9lFk|K+V0bPAXq_`>k{U$A zv#&ZZ;Rynkh%yX$z)Mr8&(6oDY=sSQV0vRl@g!g_3;sw4{tXQWN5jZ4(KN$awKubzcL9PAhgo z)oG!{E3ftKJt)%G2?0D$uWW3{#3hiJ#FI#hn>>n>LmS@l2${j1zkI0jO~>sq}Te{lg2 z>40pF)NWBhqc8VBH$X>M#(u!JwPZW9maNp6ek3P6RcD7~$Nj2A5H&XHJfe z(Ct&n900#~g!Y+#1Z0u-FninqS&Yb!hxN(J?;rbgLevTxA!tC)w3pj&Soh4Q0%Y$5 zjYlme!!q|DJlMWg-He9@8c@@zVq1=ks^YOUbVcpYyZ_P8y#13*;IrI)bNdLzvnQeG z2|$*q+o}nis~329W<$`d?9c>7Fh(5~Wd|q=vB)3^rcs3$R+)Zi=(dAKB=JObOk1KY z)G0ub5D(dSJF3-)@tQCwi?xm~0KKr|=`}VuVj#eDX*&*rcNrzO&q)V`1qbJkij!lZ z-6|eC(cw6a+W}`fb}e^=u-NZ<2uxqrTjZ2!RxvUPv!QpA>>qYrNhd*~Hsg2UK|_-$ zySuj@v<2w^>2MH~&V2HO4?3NCgHxX}-CNYv%Z-19LfL+Q{k8}Ia1L%~i_x!souzW7 z*8LP*1ecz6zJi(A^M)!a3)aj}TFI@i?nt6j3BBsj?<-R|McT5bJr(ro)QpV4fUn$$ zJDYM6O9T6S=yU!Ti$VzB%tj>(qc?qbpzgg&F??gv6rUP&KZ#Ifm`+z!a)&!ITR#ST z-zHa&JR}q$gy6%D#TR=p+4s@Lr-EIMyIal=I|?hv>>r z5kyB1ggGkAsu{ZIXDo5rq@Hx*;?RfVB#>9P0N(_xdf-L6-d)9dkP8kAgF=N*pRyOn z4QVXrF|gS!(U<~%hl7-ZPDWNc)lTlV_6&SJv@#)dC3*f)`2xkx{>LFue}u)xgI5dAoeXoms0yHhYPv!fC^zO_4Q{b^ z^os2z@N9@$4PM!p*0dB?(wB$6Z%m#)4~tfRcSTO)x}dn1!Ah-9#*UAFYfIq}aY&n4 za}~=w^RRST2#5ZDrI;pOM9GaMhYNzNx#6gWrrbCWG_jkftI)K??Xl(k`>#QkZnBDI z|3MXJ#iFDtL__{OsO5kJy~X#$$Dodffv!Bcug~RI8^QNxLi?GZ9q}bVXoMb#@$qFV zRF_`o3VPUE$NQ{!er~Q!2rBU^Ds)4z_t$qrJ|?RFIj9Jme0?MV$ycPW01_;^dd0fb zjKau-g3znAbJ98w3VT2X?o{Q%(ltW0mskHV$U{{Pcx|FrmY2;Y3h<6ljJq3M`s}6y zGN4nYC87D}o!ypy9hSOvi~aItSurs&S@|+q>FSuHfMnkefuj|q^7B=|62lpn2#(rB z40@G^K)wO#f)sXD6{OzcnC^Fs>z149?jA}Io=!{$ z(DUAK85Gqr@LI>|Yc0OatD;!Ijq$qxf9KIIW@hH5SOL|Of;o_DrJf}$l1N>QF7mDQ zvAS_rUiXhXmpt2vdK4mktGi5>TeSM#o6+O=rjp$-y* zAdarHut~|Wu)uU>T84!?>4tEBE9M0PVpfJ7yAx^Bznd0Xu(8TT-VSMwh$Qxy#w#l* zC@fT(p-=}nxVXBAjS&-3rU|bT5`qshizS2l+?#=B=vkdth0gDm%NthtqYT6nHExEpsRaIa-jFh)*7fC;oufKXCR)PRFwNeq<-+`>gRX1uy*Yti=5*t zbk}lmG^fUv9uEhMwvM~D<6HR7A=zR=sOQo_Z9A#33D3{!>Se3oJ`?-&&N~1jTd^DlLI(U>%4j*jIQZy2&P0-)98w@e`Sz3I|+~3^Z18 z%gpOf#lN|9H9LUkZeeDI{N@)PXG1CEx_!QgHGfZaH9Lz{W^Nm!Jc-EsQ)$b`z^eTI z%W9@)ZN&f1GV^!~>NPE6JnT2>v1Rmt(y`8z!lT|h~?>xw`-b_3=d3-!yLOczNd}j1X zWS|n)B8Rg&W779i7adfumhdX>MtIm*%>zJ){{8wNx=*Ls6IH{c$+mOjz&zM`z4y7{ zwx}}oyzS+QD{9^EPv=J4`PW(eVe7)ApA6{zT3kg!mgDh>K-*S*0gPUWv^vSVquExnK=!3z(Owr zbJd>H)Rk}z(-)(AY05J8*Q=U& z5~<@EHF2?c#1>CX2j>Uf~wyDWqh#Lq^tEe2^ ztb}>&vD`seaJ`rlcbQcR)Z?Qs8)r2%HZX_`w*6?$_jHPxiSo=xtohJ~F&y-%0KaXb zW)CwGiI3Pl6)IZ?mk&msHt@pv%d46KZAI&}N~cn|-u3m?NG zH8r)mvhw|uUj4b$>dTjdPI^|Fp@)Wq+0UJ;>o9&Fc%>HBey-~M`#*N&?^~ExB7)%1 z9q^wK4rz?L$`Y!43-4_IS?{(^5?Wk<_S)Z_`(5GLS;u{5%7E2*4Bda|^MO-tTD|t+ zSHg?WE2NWWRGPaLIIcMrL}ghqiX7=)8gG;Q{22-2UkSTRj>l69x;TK; zBOh~g*LV>Ss2-FY#44T|E(-qb7CkD11E}>$b6{dP1|027LutRInSPLIy7Z%1yWAvK zF+sTB%nNZ}t~5{Ys4J~X{A9H2JR9c}p(YX+SCAtg&Zww4SEa`x8pbU%AKJ;ve|JB% z%`@4ixt}D=|JKPH@PjHZ(}cVZe=37c9l)2seKnsZ2o&74I&Nj~UpaX(QBhKRTMya= z^oj+6$h<`FQLw;%%&s?EcpmA44+dJ^njSZvL~!e+iJ>X{gka0t{Ck2NR$B*x3(xUt z=}8CJTuPXl5H)XDv%=XI*;p*79xDljn*|CZqj-E~UmxNASDj+dOmNS;sVu3e{Ap>| z*0=p}$$p@hS61pL94!V1DjBYrCEg04&(tb3vHbak$kPM(0J@-e$1%|FSX-lNF-1;y z)GuzRj)3yowrjT4DG*o4g&+c0Kz;D#sXQIiWbot1!KSFyl6bdyB@)FRR!bn&M4Q~Z z!Cly;g&DyMA|Xj%Te`806Z`u`$2N#g(xb7NFZmuB@!c zDv2jfkmV6YZ2-=Y>htwg15Wu_OVXr(o7t!jxadjSea<%l)ib-S>=qOrr;!oUjWTlu zpDp^TWg667$*i#w6K7MtzlCUy2l2k7O$Je*BhF}UuM%+N-R87^; zg8)Vqj?A$G|AmvJVC@HY7}@yxO%BvQ-?^z$kS=E~n0F-ovbci!sHvse;`AA%$d0s? zo3)O)%1&Lz2=OlU`}emd9QCUpV24{>ks%2GgWEJQQ~hFFn;;hxv9&06f2v2{h7C<+ zp=!p=%l%!%^Hii!c5*IV6?bL~%LGNV?f`DqbT)C9=1p@`nNpKe^#YygsrppfyT>F; z2q&%6Oh$t_ilLYI(hAV-QpB}FWcyJIWtuTQ6&n+)dVypw_dGu zYnL!i;a8lNQ)t3A2JC4xpLIV+yd@szzpX@3`c}23U3~2g!beWAJI3ZO{IVr25tPO| z64Jsfr0TTl`31rt7O~*o#b+_T*MR4X`tA!E1~!HG9K3;-q&O6a?QS8@4k{g;d6NhQ zK)~pgGO!xRNgg^2gg+HAv8w3JJ_ljD(|Q2?ci`{`Up0=j&lg8ncBC~iJ%aK9lsEYW zBk!#x=--LXW5jT}v)+bJa!0V7@4jsE%emcoTG<`xib4cLX3>EuAsg_?%A%hFsd=*M z4gIS8pw-!LPvoLNz7m9&mpU;oLV;bCdV;2J(Clt6&hofRC9RB#)!lH+Wl%c5z|Gy9 zyE=dCU^Rz~I*4|@q?-O&E2@4wOUl9 zAas*=hhGB&eJ95HQ2U2C#0&g&d2Vj*rPA%z_S8l|CK7J-#(fb^0{t5q6=&z7_I794 zkRVege8x$eJEQO9p@>8wPEg&zi-a2$5i4%9hC=;P(g#s~0vpK*K@Cov(6Z?bE^e5m z<&g5_fHvDF()aflr(kSC0{kvrUpm-+Oc_@<@oK%PwG-oK8OFXLr(RKZ3U777KBy=$DV7(HME zal>`4J}9q+g;n^!Zhrpbot~zjo*yooZxC}^8Ib}hvit$eWQj@S|Kz_9Lg z?uk>CM}CeU8lO-hRE`sO3+*+Nb8rYNbPByJu9K2%OXK=h_#rJ1kR@vP4@fI{CvKf1oYF*V4%9I|Xr?cK^*Mr=% zwTQclrnMXJcozA9`atKg zVOxPpX3ZC>2I32QI@VGAJn(eShESp>4d3k%e6!lAK9oacwy=bG*GlVc0x&o<5_ON0 z^qF0c$`o6vqr%x&t=jMqM+R%GdpDBkW7LTuWl@t0!5vcCZyahBjwOlu!%kbDjp@Qv z#f&yvMLxEkujC{8vfsMNBEWUXcE9uk`*IO(Zy`GtwrukBQEsB`#+rA84slz_RZ-_VV_NEzm7HLn+atU=h|EgHAVPx}z`Z zS+7RKm42r8^wdt<_!GPpm1SeED8LyZn?-F-19yEef7zRz3aW&WQ0+aoi^nrF=-ruT zy)=PTA;RVF5*zJ4JLz>2rS%RJ2iuontX=mnEvRmub~s}tqmJvDU+nT5yda+aDo@uJVCt^_Y~VdWv1Cw7KcT047VtiygR#a z>P0;`1za`>KF1}9CJ?|q9K!BCGyRwxkv$bwTAH!#ease`-{a1D;r{!3#nyxAhJkAD zG`5KelnD~Peb6=KTvtHVx1}#hdw_Q#vjW(Qn^lfH|`N%O_@Mf<;p8HRlg5A4uUt$aTZ*5=)#)ZzoL&1DF zS4)SY+By?XcW5-9uuQ0p~*tM@W=G$RISSZawY|RAO%BYEb%drfSrA=$sbX!KvV9Skh>f0B!-h8#s#j zwu5|~=Y^axFS+8d_UYJBTi<7tox5)DdsjQwp)hXI7SQ{Imf5mRj_(FFITS+{<5GWD z^SEH{U4??rRjEM+D#4bXxgm#1*QVewgL@tu<}zSDdhB|?Drs_JrFm4(6tIMkjE;Yi zW~R6CT1}9yHz}62po|-&2v%0g`Bi_t+?7rQ$l&&*Wy$)4X!!x22s$gi6LXK&l9-GwF9Mn5JV4A5@?U5iO4vYH^k0Pfolh#!A9rz$v_O1Luir93n)t7K8wOyGM))QRkDT3kvcyf5>r zc=uj7GewFcoGlnOPhV?xoh zRI6sCwBgGayI!#d#kfw*mNht+#nS~d!tLcRdYEav`e<LKT6k7 zDEX|GV14zN?8-NEK67p`1Rs^Q35m}!1oqpq4i1jrD@TMkdMs!(ccV%``T(G#9d%r8 z)~TibYXQUI+6fx>OV;^RZ_rvjc=z^Eq`HQTuK*+V-TZz@^t{DX2$f;cz<|4Tiw}yz;tT(OJ`}iaZ%2?RiPHv!^0i7+3k# z0ZQWS6r_~ZOm=g&OC;!e%)hx)6h1gaaMS zr?H<-eK3nb&fJGX{1BRRWC~(L*V^_7R}Ae?j}Rm|)1xHh&LM!Fer8sI-~p~H$~gkr z2TxA&JsQZTZBs7rt!sE~CZfhGEM#OkH-j?tult5Zr(VvSvIfszRyf^vI0nf{t!_j?K7r2xzJ8yXNvaWn~3T~RQ8(b2iY%3h$P zv_2W=T3a-DMk74r#@VJ_eOzbW8hUS6P~7pI!zJ@{<8tgbt}|lYN{!5X<@8!FF#Ruy zrfvG->i6)0edm%b<_7vNtT= zxHlGzVa4q7?ncx_q1s;NXnK^zEC;W+I7V}S*6UcYmtvMSHhn7orOF?;Cj8HvFjyQ1 zPi=h$oDcMr$K2K8Toxt<{x z0nO+7b9G*^kbP_HXuyZM;l89Ik$xLyO%?2S)8%!k(ezA0q1E9tK#zW~WiPXL^M=4$1Djd&xPd?Y(S+=E}S@?&GHi54n<9h#A}Wi>W@mjys4BH_lK} z=%w8ArB^L4CZxDZz9qJdbt7Ul43MX5il2YH{2PJ9ZX!52s6R#kCTFdl# zn6oxxmK^C(v+aRdQ{NvEW(V(!Ghk|5)l_$6%)IqByptvrMmrzB| z7rvzC`FRXhhjnTSfiwTIzYB z;*?qYifX4=!wlEpN!zfDd^L5wlZ{VgKwJ zMKhKw)Kz5|c`q=5WVp+>mpb2`8Ox<9tetFFu4J(?Z~crCP~~!O`gR-JB?hrwy=X9+ zUHQ`ANW{aVPpbBLHREfB)fL!VxXRQ=I6tKur%8+t4XXv!qeLBw=U%(B?lRUcb}Y}F z-gGna-c1q^3imrAUsQbnm+~}lz4CVZakgSxuBJOHS_U6GByOs81%1oQWXB&D4GgT! zC3&%NejK>~{dOl3Fj*JLmNek&2dApsGOui0`il#&^7zu{r!mYgC||=4bu7GKFsAzT`eT`adC);^!?#LOTB(?W zZq~*xCVYFoew*)K%Tv-fw%S(hFd(XxwLWuU{QslwJ;S2Px~)+#p(4-*L{Y#%5D-vu z4kBQI1j!kdR8Rz!oKaM40g)m}&N)d=LJ0^+&N+*uDl!yEzO%6VJLf&8-}Bw?KKIAH z_1KTLQhTqx_F8kzFyJ&STPyEM-BB>3&(+j*yj?&+0Sz z@vR*6^OU04IQmwnJh}j-NRI%Rn1lST_1>@gnst8+CI>2Sy*MhsXcUKF(x{?6Br??9 zF!F%p!0au(fNcIFb1cgUlsKIoH*UU%#6QmZc2`4SX?j{35XF~X#FX7PGqXe)LtRgp ztzIpSPcbY)rNm_RuV0J06`Ql##fy??D-Y)& z3u1Mt`YSt0XPBSM)+#rIyu%j$VQGoRFEh|t&kwF%&Av`dG+J%= z#8O5^rX^xC2+&M+RWmu4?}!s{xXB7(>Eq+qS`Kiopyr_i<=624TwP$_J@o?k1)iZh?GjTZ)zGDn`D?f!#TRjSh`Ve*PSJKI^q6r zr~P!w@>H*MwzBYlb^VA`J=niqw&T2MdGJDb*E|4dLC$#N`@ZSmSC2j^W2{o=W9ie2 zH15&HtYzk{jd1-Kh;Uo3x5HjSxw3>-J`pX;%28_--t?>9-~}oq0%*U`a>!ob>O`pP z?$f@InMvw%MB`+_RK51Lsj;!K9W&()xMYxdBY-uIA2%)V02JS1m&^`kOSo`u_@@vHRgqx(Kq?+o_%14AcVU`}jBrT8XydAVZOpqb>ejd2g zwy1?^2x(-k49)E9CVH1KN>N*l#U;pfK3{v|6t_OfZakyLW0wA>w*g!CHupsgXSJHX znfdj~SI7t>$Fp@sasL7}sX2VFFLsKy*Q{(HkGy<>mJ~%pQ^SqgxNUcav}XhICd;*N zUX)g1?B<57{bvDD)Lp(*$pnC!*RPLRjupv^IIk-`&sE_v6-{gf*)d#i4%8MpPYglj z3mQU`r2S~KjO9Whx2zlx0I2RMQoHvSj^&2`wbn4z+XqcfgM`!it*8J-YC|nNB0(`( z;m+{r^vTehciW%2WK`I%WOu5R${tvFO}K+6S@UeDY%tR0;pX#V$TBpT@QIq5z01j! ziq}4B8un0;huG1pE}PBc9VB@@`Ia>`%V&dY1%m?O0K5ajgf|JmRngt$nz9*(`F?1b zJI3bd{t}qA)2q2m34}5K4sMcJ>;^`lO;j(1imEcTc+^!TLE66)bsckGK}`TE z@4@s8jQUA3bS|2KOi8I_WfNH%!0u&M2UWeZ^(700TBOPJ__1NO8)?=)3r+4X#pCW? z7jaJOyQ!EK`saeT`|3er!p_4ULi#odQ#`C1bBfg6B;8F;fT}y0eb_v8uSYM*V7F0` zrcsW^EOJCC2K8g4+%ve?=hSKseq@)jyJ~p-EY3fKRb=ef)ncR=b$QY5_5Uc#xGf)+ zTeJcE;FfP4104Yjh%3F;1OYBD`~ME!cyyeE1ovy@;S>QF3#g(b0#DqvO#Ip!(D-&& zR=DsyYZoWz!$7{R_j^E}K_2|wW`h-ls;1_r)wL@hXumf%-#CUO;jgo#tH1Ph3lady zgH$vPkaliJx)MDDky98sTMM|kJjsJPN4)3}hyKZ>2)Uwl)diQ&@i*DltdWqzcv#pr z?52ENYmu(Se4Q8XwpH1gQ{5vltmxp+CHksVBtqcU{XT~^zL6!&xrMTQ>r$)Kla^I) zey@YWs?OKvZ-t!bGs!@>NG2; ze1vRfrSmk!B6IQ^N?L(Lj_6Njsh0Orl$cpmv(MQ`*r^;wNEHck?|RurtWp;!`|LW} z$4VT!ZS36(qg^u|bjW|2!B;2_wY_>YLt51HAyqHnQTqAXx{H@0I>l16>DoWh>Fe;m zB`k6#cr3PmgpdF&owQBNr;NF-WH~f&#wG^^HNc83j&|SXK+^*?(+t(^_Q+EICX;;g z&Fm3vjFnr$4q#waq~6G-WoQ@_B!i9$L7p0ITl7V@O?1uI+}u2TjuPMk)(*~!;c@oA zqmqm(!KXJL=#{8-Jg=jYKmFoZYM)(h-ib>w_pW_E+aA>Uvzd=7K705@5n{(|Nj)(X zrSl;IH z9*;`k5H3a<3X;6JCHe*Sp+9*D%aEDHH;!RIsvA|HDttsn`wOQ84q<9<>JLF9+&Ph1 z5fJgFZn+y+*-P?>H*MibQhJ1j9hHUz`ZYW*T}*D3xG2A1Z58B-kSZJ<9TH6e_$1Nd zVN$pK7*ssVCR;JrA6C)$F5T=Oe*6^Bu~USE716`g+s|du^8p=Lw;dYV>=WWDSwiYh zUQ`ZVBfBHea!SzDeIq@ThUT^wskLamMwp_N5> zMNMsxKIj&>a3Hk_*C|@XzC>(d-0N3G&=# z>j$v$LX2M4^W$AMwtS zJ+Fs_2eXgU@}hdI(!nbr5_h>zh*Aa%pK27o%Bl=E=gi!Dh08Q zbIG18zHBuH^Uha0FrrBy()8T;UA35;oJ=TRiq1rx(7kCuM`zhzBH^L4UzE~hp5IH8 z(Ozaq3HjQ0MTER4T$0p45g2-iuprL-^1bi-ZooD0xNTQZi|#UEM-$3Znht>_C6@`1 zsAW!o4s-x_Aw7ba*gh>;I*EY+Ke+z?tWegXJn8r3<+hn-?8JtIyJ$<^A+IH}iSTxp zmD@x%Gk=`_fuFu&J=dMqMHWo~-S>deP6vl1hs{YoEkil@#pVS+Y7yDuVP~Ixe2xe1 zeY6OV*$*Q?-DAtDxUmjAA(5vx;SD+x%G4Rqt180hQCYc-Wd-TY zFQaRKeM~JVP(>5i*Z+6uM*}$Y?yBj0Hdwftv&Pz75&S*>7%MybH&r_8^H#EG(ut2_FeTe;N_)m!__=_2s;cD+TLG)wvj=g!fCfnNy*lZ0P^#np>|?jc11PaK zet%YGEmMQ?dTcEO$3m>}AHBR1IahA*{2Fy60FnPf(*g)k zVCc38`zNU3Jtd!}jkYg0y4>7*?<#fh`n=k89Z`Ce^;oB?7?S+IiN$inHHVIQi#DiX zGg|V+Nx!H-b+9&(D%C*m-C=N zt`$pN;gsI48nZ7!bHNf7Rvb0-f+cA6d1YJ?^PU%XlE=pgR>BtZwHaF`SOCJu^U0yD zA^h?GMlV1+XVdE8CWVTO(+|TvZsAg|lqTfkIgpR}n}`i8i_fLezV!W5ynFNm$*mKk zm*qj`k8lStVm7aiBtmdS9X%`|hFFY!ytA2~XELK^=HmQD;irbn=X*;|&3D1+I2N}n` z^O@A6#a`J<8Xnf{w7aUA93a?i3+ghp#U)~bqFRCd-pYqP& zH@MKcdem`GJ}Ei57sUll3n1&A3V>M20e2L&^59HHYeO@4->I!$| zE0%XjwF|L3mfE0!zTgqc%46rJtfDcuzq8a;lTY{P{{7eb9)@8G(amb|@_x`_QTVHe z0el^sNkw?fuOdZLmndq(v14;n+Fxq=Q~QpK-0b0Kj}Hpo8q{(IarmHpi89{0VO!E# z${uGI|Cx(k;**uF*>`q#j$G>K=b1>#rUaA)t=-Znv~sm%Z}I~;Kiu4LY_L=_2l2U= z6dCn9_~e~WS9+edBIGmpet`s<9^!gdH8qEzeQIr5Bnj-M-trde#2MJ|`e|vPxWv!@ zVC_KA02nfXd#l5XLNpabHJc_*THEdbbrN=%S%>>NH}{&%e!2qpWh=U|>todq zH!n;X|3>BY_fO8~y4j((4Y~}roTw^yJi*nJKDMoanvW$fV;ps^s?FW5dfwNWK`@0UfQ^rG1s-@#Z5f)o<5ghkLP+Fv30{dMT{xlv|BEb z!*#M0Twl2DV;raDcstY1_DomiKz{e#+p0qj^WD`{RE<;H`zqen7?F4&=vG;(+=-sI zFF$wgT!GsVZ_WP{@%mb|i$Xh%4;AJ{MhhbVxxy$~%tQKs?$>jJ6^wMV#PsxW(RvwW zXHgXr$B2j@dpuviU7)Eo&EQ!J_=f zCE(MSC0BCY%iQjZdz~#9>S8T;qH)~ExQb2qVlTV1aW1*B5rK2ZUv%SLoj+X;h1l-r z#Vr~kdlpeq3DRhTSm0$%Uv9n}96}xA)ofmf%~$N3t9`?P1;StgSg^o;fm-V}mAfHO zO#VX`1>bH0Zm`g&LBpr_Enozn`wmt5`kqk?C5jEc^oL>B$Z5{uQKus9rFpK6O734@ zjt}fd>070?9lEW&d zNy33fIPzxN##$b~4Js}^_`e3dsIkvhnivTLf^2Ec33~1DU?ZC9>t*CZE0Td;fO^y$ z90_QD*no!cmtH5#zr70pd~K2RhcEkRH|SeoDi=`#s;tH~YyOx=CMFlJiwk|dYqdjw zeIlSvos5!yn;=~?XrpjJBSP~r>pR%}6&1y*2BsR_$Css>uU<$z<}J@xe}xXeZH`b$ zuc`gFSRHI3a$wLOb)=)XOR=?w5L!< zj~rR=Y~lcux2W;_c`h1_hDHPsiG9Y8jT&}!bK`kZZEeTNq`{FL>Y2-tyR-d>)hDsB>1%&m;5>1$cl zUX<|_s}D4FAj{Gh%GSlzCVEp$jE+Xs^**b5rV{WQOpD&tIMi1Lj)sTK%z=fdJ}W|R zWq85=2KA?6(;-SYrz;|}2RqOvfAm)ST9_8jjoq!*Zi5*_>cD8;lW%tMoKmmlcMbjn z_2`}k%FCX_zkGc$#h8#qZRMo6aN4#0cg*<++KVzbHSO)~^^{l<1QAL=VtVwep@C#d zIgm-da@+=hU%eiC)5UgUF=LA{2ftz$aW)x_(LgWB=Z9H-9}A;txz6}qYYE~Z$tKH6D1ukMO2TKa=z=%vHMN3@9-+NB?gwa zuAtKRo^H4fIH@(ZW;~6%<4EKGhHxY=)XJ~yZ`XYO%!sEsaL>*oV}A(v(P7+h*4m0* z*88QWk7k~Ytx21OJTM>JqFtQ4ouTe*lGK^cVs|HBdBCY>(EX!g&BSk~Vt9V|-QYZ0 zHtH7d#V?GnbuL)o)n=~w9#^N^ZToEd5v0wb*95k92=8Ca!Y}@Q z$y`mz)BoI4w)dwt)n=BPd>Bt6i3=G1@+4yizVW;A*nh?lmddG3m3x^|RU1D3;UdM) zLNCKRI95P3c=fOLitmhi=PkLqc^SVCv!xT5Ea280Z70hg^Si)#Qk%p>@ZZpR)Jdl5 zy>AP)dT3KG5PdmMD7}aa^$k7gLK)H!qT}{Pf~zTT{uQOgX3xg@>8xSp?D;R|N3@9U zyd)H&Vxt5XjNJA+E!E7N^0-O4y0N3h7~yHfCE0q)Qe+ZSn#4(M z$`YkWjsyh+OwB1jBUafaF_S&GR3P20=j87t{kW;b8? z$sgBkE1Ex@hS_o_nEk6pkBYkb?+G1^x1=6PxaU+=ck~Hjd-gH&%gPeC4ifF?>TLZFP_~-^xYDyC;ly$`VbT+8I;Hn5P&u@%X2D z?|$5~rraq!nH7KS!N7$#ImKB;QmhfFw}~&jxh4_v@#OU*#OKa&A|5b?jnQ2Pg3C?fK=MaKVDnlSMUS($aL(A6ozioS@(Pv zoSx$-etYXXMhUB4{1`+`_jo47!t4f%H7IQqB|(T%?%#X1vr zNYc+C(bHqr9iMM`OihO(M~6c3Qt>FUi^ObFp%Q1Qd5k~rwn$S>Lm{-x%RGWE^QX}! zzU)3uhOmxiztqJ36n3sADs-~IG1#5X++ka%DBWQEE?;wOE3~zHf7-pTE_nXJ&jdt( z-5J#)M!js-Jq-#58L?nRd+nR1C!fWL2d>xMy#5|*u4?7}=MVkT#K{z&ktnvWAkA>wkD<@}vOQ|I4@cu;4I>Ls}g?e8`GKZ93%B1P78Y}lI zG*)&%q?PN*39a`jBa1s%j9#k8D^eL$tUP%4s##Rfz3hSF?WT^fhoeOjxU*-?dQzFa zE?hEEf8ri+j*k)doocLHNN4%RF+!?~=&M@M!%%OUTg;8tKbnbJaAl&>Ef}%>%c*`Y zQ|K34;R%$NH=5k3pUEjAluewAL=AEcif&i_MC$+kdMtBo>%~3tj!yZKT)*j2mF?}# z`tmg%>nv2o>dTO81vVe06)w}GsC?*|pHDug8U9{U@QmUW!4?y>U*P>N<9(h*9KXd*q2R+O$tYI}cbEKG>( z*xO`}x12nTG{w&^Bep|>);SzZk7j9Z{aG1a^QG8keT9~h-qP}(5J_6!ax3=Yi2lnb zqe8q}4hD%VD}hFol#Zi`vCHhgpM(hcWZQP7XLrY|hq+m?UXf}pm~nBzSpah%{>(Rb zx83!~J$bC)t$Taca}zzpAOP3W+?8!N#&T1jYeb8f{wx7PdvkBmVRJS)fUlvxUSp`( zBio@j+r}XTOsV&5SFMJ%@VV;y2C0F0M3zevQBj8ZvdZvMIx9s|RK}va=B&AfKS>gu zs{$(^-1_b;U%COt-Fdc33&^RuelYkHnzn_?inh{2JAq^Q>dQgv50uzH`SMS7$7>%$ zz6|kx411V%`swuXEUkQLXsd-^moLv4@v6CsfLQ7EaOXBvXkyJ!NogI7`WPG8Bmk1% z!Byi4Z%8lB&UP5Sce$p}XCU?v+FZ1EHnbN8gU6HeA&e)p)1yn69;3^=mhUe*8kOUO zvKfhDa7^`73Y6YA`6#%4%7*=>IBCHd!aOeDnR00dEjIbMICZFJs>zz>YOc?7*%%iO z;fpCfx|01OxDB`W%IxN|wx^;n-I#s6H6A;lvN<9ofA;JG{(Az2#+A&LQ*@c=YzOWE z0RiKysBOc3_6J2i*(rSCiS1VGOo|k%J$W|*PfK16|J8IZw5W96x;^g(og0mMD*ELc zcFX)YKLjvh+3kE^E2~|6W)|VbkaWH|s35-^y(%Lxaa6U_#ItD*XK0A@+}PP+xt99i zZf$myoSR)q_qs?s$3;rkuEp&VKL+W*)!TaEcVs%H$9T6prxQkR%67LD<|Ks-y3!-g zO6uXzq1LNC$`P3N56SmlpizAt)28iomT}ufa-W-@MDz3}VXCfq(0Mh@*$in1vzk2+ zze1K(GuHpnlNbM8ughiK

    X3%j+vc$VHJV^!?b2JXoGDlFYwi|HcC3Ta_eWz&IT# z5GTD)Fh>-WSFdI=%`|t!Yfu5z%8QSa-s^P0bjE}bw3+rjcb?(_S|J4lV8r&)Iu(){ zEO=2S!w(mB%wwY6Osop)Io~*h9NFe+`7?;B<4isBsn`SO{M(~g18*`?hBtb9Gp{@l z+Zr$FCwn}yZ~2tgmPKdfXES9LA8^;)b?463@}Fkj*WVDc#N|E@LN&C?3epFQN_69y zt6lL1tk23lUe*;mh-i{>pEy$Z>`iV~3}JPYku6?B$nI=J+E=(JNgcQ6n>iz{_XlCC z9NpU4K`joO7P^l(10`7EqR^d7K@kxdGPF*rAYe)DwubbJxuee^d~mO4PUU=}FB@5t zttGbKt8nyQ*@JyM-_yoP2&rs}UsNWXseU;)k=4fCkGQ9yGKl?3^01onjDBQ#*WK(3 zLaLhHtHt_D1SFT;FTybuTfTcSq#d!#|FTllZj^ELh57lPKV?o_ z{<);%Pv+)om-HR)?U>*%VK?i`g5Vq?y%4=4NyF8E|<5ceepY1|89vh$lz#={ETG?uJXKixg+b zy=!A1B3!9A$G-7(*FTZzS-`JCPerNVFT2L13c=Dre zQm`ao!osunn`PDDWv*n@a7ivpUMM*hV}jQiEHG1f!>y9MTU__8uJQ35WZU8hl;(qz zABM$u@cG*(=W4zx-1AwW)F&eY?#Ch2SSQLyzpKtqY_SW;jg4~XGw(`fgR3Oax;utM zyFUF;T~kx&+`@GEpSt?dMk%d&^vR zU#~!1h(2>-t}AP39l~sleghRT$!OuFj9u5h`adcnm)hU^TOKpl$$dAft|dg$FgEY( zk4k7z4B)d3zzn1gHw!OlZ)S0#xgzd_5)D_ugDRm?^&KaqXoNNKw&#I!qqAhF&TM+` zISVs$M~07nPxk9$Sq2z=Twjhuas=Pfam%3%t)vCg)2F%Zhf7YAEqsS_4S753WS3$J z?#;}MK;P8QpU6SCE_#II+iyrPHMK3is34oeSHVpV)Zai3^$<7&{4#q#Yli}#a2XKz zt&bn2XaHtEe63G`X&Y@)Qj%J~wTh@@8L3}KACw*&M+{SCk`=PBW^9)Rk)F=}K5;9c0~u6Nq6vGeNLS|# zH!$J`Hdyq6JPE61OhCYO503;w&P*jm0f7cihLqQOr;>&?X0v7J+*JK{Cjw-Lv~ORW z`f0@XbN%q8$X^o@(#bciXT1?$;qHu6MCsA?Fv)sTc%*#!B8=$V-}@6N9}SWoTgth= zJ+p9?-cYB{1sQMldgDy{`1kLflfYf4?HhoDmCHsZN*b@=yx+63?vqh>L;Mjd5aEyj#D<=_7Pfa zhf;LdTJ0vUx1~{iQ!LM{^e#5Ce0^h(@x25;01#G_hK@++v2*NXS6* ziJQ*x3`5P&7t~wG+i!69jByFey~x{PT(pt<-23>>YU11=yh zr4bAQ?Zr+FL?!e+!r|2Vc;Or3e1TmP@4$kZf5#6TE@OCDmqtOh^W7C0Wb-G@+Iq3L z{{Dk>9uEY<+nASxzzq}@7EFR;pTCl@*f;ib`&5Dyc5(4cY{PRu9X^6XERF^(^9z?Q z%q-;U;3Ra4}Z7(&(YCgO9JTEy|Om#T)0F z{4?Zn7BFa|`9fwVv?y?^Q%&zbd{B4Zra;pJ@&;skV3*B(Fc`g_&;i=var;M4%ZWzjnMrUTSP`9M7FAQlgFnlnt?`9MBP^ThS6KP`7 zCJAg%O&QScSbgRP^Z%tL*H8xk)$94`)2Ef6Jc%Q4X?h%FU?ujs4sB7~-=M*14Vg??>UL?{M_#Gy}5Yg6j_-+SlT2M2;L{4yK3y>)D%GuUOr)#+%VJmlfr83ceV6 zUOvCemn zn$Ifp{k%P*GrY&ztSecbvh!is>KOI;E395su_B-lGu5buP6T>j+7a$mMqte`-`Qxx zw+`A-7LoqH;4P$Trzp_y*bnoQUF8>7{3bTHI8VqKrK6bYK{yBz$k_1sqv%MPJOPrH zI2a@5MBw5%4M&Y+R@@=K@A7yROP|ByFk;f`N=n#feHyUE=de4gL0!h^6!{~J``&uH zS3YnerPRt@M72e81iK2CR&PKfJGGFN?whyvKfgy9aZ@uO=2E)w)oW{!I3QQ#UZ^xf zCtjC)%FSh!{KKSz(UeN-HsKm-7E z9_qNYly6-0L|?f|I5e-A6Xd)KasT~rkANx?XlvgrA7O{D`}*2&Ml=T(wSE8o5rcU| zAR|jtNuftTYpzOl_A!L=ChZ6w)M~K-V#3O1Ixwcl$a!f&6#%9%{_6>!@sP`Q_7C|A z({zZq!6h?YPd(Z*DO*)L~!w0Q$gtGE5am9G1AH?icjk^cQ^yp=P_e*6pfB40UQSy-v1# z$YUSET4IO>ICC_?lT2y~!QFq6cbb$G-`!;c6nXdEeoc6{;-N;?t!VAnQ-Q%KLi&MY zZ;Juo3o6Xa%xKvwb^j5$mi&I2p^Z{~gWa-GcO2V@;D);8NSTzo`@Y&V6BCmRmq~LF z{ox`P*-K0w;gck!6O#RXeJzk@zt$WYtmA>hPNiqwNZ^Xi0KJp_qrPuw?x6 zXHgYL`%3XPrb2UYpjJI@ckrgjh>aCT7l5}9Cu;Ub6X4Ajw7rEG7)Z)NyF#lONR9-d z(KkFiJX)f5GJA7$av+3Jbvk2U0P^tD*k{UU>0$nRhIQXqME6hC*6nL#ZOO?B-p&^g z2ND?&6U!_Z)v>Ywt$x2#Cb7%14~kez2ld4v3H10)6Y{GbdU;#AGcDVcMb3 z+gzk~x<$wKV2{fd`x9ZT)`3({vL35sVSe5%m<-sjXbXr<2@16v3qvV@Oa>lu3C>Gr z7F!`7A^!tTZ|sds<2lYHEH z-Cxl&uY4u0*qUBt`4=BR7;v_z+D1xrBJ1(v&QALvu=+t-7zKPyeZaZR1C+3AT6vn5 z@|FaVs6k#L@eCkgO~T`P%_a==Lu#&DjuuGG&80A;HO{nQmtmJaW0JgIpG_XDO1oCb zF8dycNiK9g&^iM2RC<-J2+>OE(9TW;U`ZrfpeG{g$snBUc5Qb~{&fs`zW+Zu`*w$; zh!vO1$3_w&q(^}%$-r`KzjRcqJiBhn_{<5fWW|_d+*>^+IxmSK2MdSfu)Bt3P)^IZ z0peQ)e+8;*(U&5zE`CievvDLn6)q0od0t0Qi#_?mC?8JmC9!Rbcli9D~A3(k>^tDF(Hc#3QgYNTCU{FQS@2?&pW?%ZZ_NeipL3yIbJc@N{8d zEvY(lLXBNfG zXP!FrGWRWQ$($?c%|2o1M5PQhhcUNMTG#RGK~*~&oAzmUi*Ym6_~V8!*^ko-?}*&+01BD*yI9gHX8bS8T4jLqax1FM`NI33ukXpkxuM=hCz1Nt^I*$y%uKr#CQwK>pcq5+PVp{uW~ z91kroan+(1Z#!za#c$7M@mS+_w@A_SDCD~zG4C{x@1(}&#bRr7IMhLOb1m`$3v!kx zS`c46SZ18Tu3J%rHp)E)bWLqr1BBqLzw?{vD=c>{PslGJ=_aY6=?di9mn#d`+l7&) zFpzYI*oD_jtJ~3q#;?4*@INC3wAZomr#*uD*d6$2J{kJ;i)nr5V9iDT6Tbwc zasuetAc6xZa%Qq3#^Cz(i}I2Dxms-`!96q71QHsz_;^qYrt2y5@V+V^;!u*Dn{(Hr z9yx~``a)$H?T{dHESzK^(;-$KPhJqUOkYVX5i;`=sW~MM z6o3UxY`q1*5M^H;0%iLJAnSqmKzMO^iq-(WAt;k-X=&vRSU|wIieGsYBkEQF%s$h? zY(XkxDAzU-m7XjYhY|Z7%{`-aBygl49zAjcR_C+$1*#;-Tmb52&nU?K*8~ZK5!^!p zaq)GPbwc*@KbJ1?u-v}=v8ui=sbCSwWcK&iU#Bdu+vmmo$J`024k=K(&T66VT;djT zFDkM=xCh^VX6BUI!-BxJ#}PIkqnMajp;?!-8=UJPZpaz-P`k_mxI_<0n>%C`fdgOh zi~?C{T~1}$1Fqz@c1a4+Ow9gNj;ugHJk`t|0sO&l>PdBB=xZZN1<) zaX%X;Vso0b6S&$*$O;@LIMj2rleRe8LUVE?z;_@>AP%*54DtM=)?i{~mBaOAbyZ`& z-1e3PjxL|hE=9hkt=PQek~#>{c4-o7<1N?p3!Fy%HvT};7UYk<}W_D#Ir3W{S0g}_^u(ist^^00hR!#`q9NvOJA|lnCLHo}7&?kSZ&wM@q zSBee{C51R4mwosVCL$C&4nNC8gyiGl=j+kK3^BNzTa8Wmr9xLAWS}2J5D}$ogxYuJ z{JC}n?D%qEjBgPA!cg&8ivTg0=XS8lN)=@Vf$P6>fpDdNxlzqEkn(v0M5r?vpe7i5 zb{t;kZ9(AMcOUdi~mb&M$1nWrdFw9>CDzZWV$$B#ShmYpU-0tx*>*$W0mclT@& zPg%meEIo>`@9gXIw}+l6N?kLiD@mr7%-(R$FbGgT0hF>MyMckBh_(>n!VaB0h4 zpCm*po1u{%xrg1ngdof``R|!7vPVn2{5^A!R{uYq=ilG_m$~}qm-35eF#0OFz|~ry z0rD%7BSOl<8Ffaq5`D&Q=;Jmek-2Klc5oT|{;h(Qh}&-v+b4ahm-^?&Gw6m303{x) z11ahPK?a6em=XgZ*deE~c1|-Ojb--pYOhyUdx9p>_z2Y}}Oa6fQq@r{UEnPj?kBR-1!D*yPRMf@99f|05Xvr99mIB&zWva&ExL!W z)5#9@Q|}2vK+{KsAqc$U*4jjKHdpE(5PY?J1kjFI)vHcx`t6@_-fE>S(6fneG%!e8 z%Oy|-%ZGD6TH3>G+RnBY1L3+ymcbr2bo-}R%q@tCz}3bJdVB2c#0=)Td0%ATm8&Qs zY2Q`~?QKhpQ3Pel^8_kxUz?eNi3ER^U@g0J)}usO2IAtLqX#<&f?`7p_Qpn$eJMI( zJuXWXeX-7IjBIHirraq#SyL7DYD8G6Up;2ST0k_ty#=^!vV+msn*gbUawoj@KqyH< zZno7HV2CjuART}a8;G6sqw^@wj06p#?QuFfBzUuM zPnE~wAA55Q3~(Dg!Nf#Mxe#_)j^wFTy1Ror-P`Tr&Pqq;HR|S|7kD*?p(mF}iK;uL zXEK_SOzQJRyL0EBc0f9iAn(kkvyn_c2d@D{o`>w+de%9mbD*JfXz(@ zO*=x5jf?)k%xoHk4uEriGr9$qN;i(18g9hJt1F`q9iq*!9?RpZA&;}#4 zGh*pr&%Ge3ya2mQRuOSoWt+RnDlEp}~^p zfrQp2m!-XmGTViCI(PmA&`vi8#&u_GngEE~d;(}2p)i3!gU}0omVUl$KCQFn!DiAy z#!-@Uh2~ST2xX&~2P0yJZ=E-=X+BgO$QEpIfy-tzb<^kMwd3P7Fs1sTCypJP)o|F8 z1@WEI{q4}s=<@*lQr1z?8L`{US)cHxOIK74nr`zzKwb+ZR42_QPkYXkJ*?BXJEB1GskHlda^UJKiuw8#`dG0V@}8NcBc&I z$In6T2}=)>S(TNQf}h25t5;2<-PIwF-(g;if|ako+vUSaatyh@8)~=v>#hjC`znOD zHp!QtPd)?JBC5!}OrAjA_4!H7`f`njlJ+1ou){wf6at*51UP}=GRH%4%snCzsGnQ* zCiA!VeFY?vWquw0${B*hv<|%m(wE^0UjWQU&d>i6EuUDkLORRdsP;%s}v+V{}XM z5A?a2nbo%x7zsl1w#*_-!QDgoPm*)NE&>|FCi!w(>5Hz!1@7HRVlN8W$;ipk^m=7M zdQY4NX1;!XJT*0=7ICoNHAe;zFt#y{#b0mSpVxCl#!%4hE3obt#n)})m3x!S`)I`M zb(L!c^F)XB>ean>)QEcGM5m%D#6@G!1a(t81)#=a=zWJGs@>H6Z8H z8v4Os1(>X-nhvH{H{|UqiG=Cw=KOWDK8~90M%=?0mQ#JMl;3pOZiv+2TZZiIj1sKjbl1h$e-poV_ zx=JcB)%(+EAP|3^Aba2Yfkte9kmq1(>ogfzN0*|A%9Q;Kmn`dIsDZxH6LjS8a1(ZT zps?>VbxnJmB)84<9z+T#kc-N&qIL>|e#SKC1zh!(4;ohbvjK|~Lgo)>RkeT^e}d`` z5l=k(UO|7@~%a{UD=DN_X>%L=^Aa(XnF%WX{RGfg``yMEX;Gy zlBRfiazLvk+g{I`oaTy%KMWpuDpj%6RH|zt;-z3iV`FEse8hVGidYxyArFH3bW5ZY zhVV3XilS-u=n<)932pgIl(@j^_X?|U z-OAhFnh>Mh>CzP0_11dt$F{cw$rl`fuK4uo8oq{N4nmZ&g^eNZA9ZKL1=jXY?;x8w z0p#1Qis4OM%%Xq|S70=%AM>K^eH}Qqb{lo*r0ucIeU0MXK99MT0e`da?bo%7=H=a% ztHd%!7(vTLQ4H_bDEx&pXJ;jxZ3bRc?6e9Zk_=z&S#G_MlEQ=TtXQURMowC= z*?)hoLsL6(<}6LQpsMn4Y*f$%D&||a5(Dy%5oHa*D1J_&Y?;~ctVAWPJ%T z1*bu0;QV(JkSEjs_sRfY+h|C5E}bDHwz99-N4cIkVL6tox0I=;!93!7xr@hYSd(&K z0iDQ`F-BOckvI}E+Drq;U`)5}CD>>=I+lO}Pm;|HCuAI;XFGPvI$H^nfp6R}WwUB6 z?yg1i8k=N9Sye&ID%{)C9j$FpdM)zE)w8ZFjTEFLqOOvJ01o(oSw;8%6ikVWKJM8F zCNnj6vhJ`1dSf3kv$}G=4n>ghzLcm~Y}K{1gfVy`gjFTV3Wj1@jkmwj{hgDwEw18l z0y-Pcor6TH9QaCF*Y#3)R5`vZ$CJ|vYo_eY{P!BYO*Wmd5uM<+JSv$#Kr@Q$QX4z6Aq+fNkZeJAH=r<&&HyRbb75MiR z{hl&K7$;kl&RdE8atD9?$W11JtK?x1LE2C$x<#E?519(8yISlEHl1xu*ftM$1Uv&} zh|XdCT|<-CemD!VB7hlOe951sTX?T&R5u;u?|RpKAmyAttug*Uu=~tW9|BemY)-D=Uh(xVi6>US#jqFmOwPt_}@NjP1aSs0Rw_P=1Ye zcK*Pq5SdEoSh*Jt{1S%-Xk#+pIDe|ah?Di-J<>mgK6AhEhZ4Kq`4U^ zbV1!9i#NxxZaBW6U|pJ>c&VolW!jj3_EEGStAN13h!*k=L?72BPvHC*tocJ9r4(6p zANBL+v~*=FbYB47=k5DaW>61s2S^#G4(WUECWi`9ZcRRYiv# z`?5|Zj91hFBy&z3@^cfsy5~P$=jE|OMTH(eKahX#AItNDp#xWALDOvRU}wP^ziZQ` zp&b)st7&56Nsy)o;l2ZEE@=;Ukc(7?wkcYtpN~L@6uI7#z~f%+NN9Xhn-*YW8$nT5 zP2Y3kWp6w`B_GKxp;K&O^PyHM3QKdV3kG%%&G_&V2UOB)<@5tF9y)K{XyNCFP1k>X z5$j?0yDu_?h6(Xmy)XY?a`dXQGEswhkCF*9LNm;1VhN`qH}GXck82tfVz6CIvP_C0 z3hX9@+6=*!(VuMxfDq+40v(@_5FEWPndC#mA@17w;+WRRc@

    TtU9-}seL!UH@K{P%%F|9wo(-zq=PwiCBsVMMZ`35huG6_*<~$}}z-!~^d- zPF*A8fJ={0_<0}I1_&9jPriMNES{u$2be_EUYtRelsI+S2txvSJ_El?KIk|otz`kQ zfJ`S9jsY66?etc*F9EFH57l*9lyZXTraId)(cVY}MyI&A7jgdG3!=F3KXV&*!;N;91@Br6R#0O7Dnqv*H(NXVGX9M2gU$S5^twz!SJvL-{ z!=TM6vS$Uo`_uMtX=baexsaS-mmI6{AiMTNnSeRFIk-(Feqg3(Xq+n>Zc6|hsst)k z0d&v8Tp_((V_SB#=EUvKxpQ}5W=IXL71>?S5R zMVBIhE9i7V6&6;QWJBrUL4jW?F`emH1xgYu^|?|)wx{r%{;#Ob}fMrHrr9?RW zILG$g2ivRtE6(%J8@ojkZ1GX9{SfPxpO0*8nI?EFz+_B!n;(f-7-Bo%eeV;H%z}2) zy5(~dat_5wxAxIZ{`j&zXsCejFNL31UEu@oA-p_%;C}iuJjGE|=9ioR`amYOzw)iZ zh5$Iwy29bb<@Qhoo79DY@tQvsoYGaJtse{imqsw1e7?Duv5EQ1T+Bh!fATIzh_c0z zhrgQizyw2%|2_A9DKZrDZk1GwbyJjgGzOS*K1cK=c75SVym4`6=DJ`3x{jL~U-RqKYyF zGi3P)0deEJKAI-Qvi*`#LgqxaS_p52xJ~t!Jxlu2o)vABQD!@_4#O7;?C=V%efFh@ z{$I>}p#{XK`1Pa6V@v*VZ~Uwq#??A``BtL}fW^prT0eQQpFW|itW0)8*FoLmq8<9b zm#HHvK!%vcZV;=6Qkic&{i_jCH<%^BZZYMjI9lG8?qm_?|0ff3yrGy82L951DV?@9 zH6^9fC~d5DF+d&J9U1uZpj9t3u-45y$Rs{gXb1Ai8u6>$#p=_&7~-v-QrFmB>t2ug zy*Ia0#=508b}!(As^l*CW8$fYm0A9i(Rzh38?yc4+Z7Jc$6vyb!FrP(uS3U=e|Vo~ z3OKMrP-jC+m)zMdww}-(H;-9?(ka$ja~4*{M*(R&c~8=3~c>z9ZRn?B5t4zkRl! z?Y(gm3!7AR1KalCf^%U@lWv~5JQW2a!Yn5SCWWm`^!4+kn@6r?e7q!OfG#j#6tY&b zvB}c+CJA!8i_6#+<&T%^hJ-L_0P`wHo^^|GAq%LhsV$#kttBs(l!$P@t|iM?8;vkm)#h^0KvUF{6|it?Esgn(teJ4MDUZSj!0fa6w;w za`N$L{cCxBT}r$!69e#8`F#P?He)UEdW$EAl8QDd^t_jmJ5 zOwR!^8O`G4(#w~H=vE?p!4*LliVG`+>4g1^0soLnHiBYf>j;he53$A(0fe=bTK-WX6M?Ack>lcZeuR&J;}N_0Kgw?Y|9w%P870NIl#=--QA)7==EX?vJ1Z-Z z6fy>T4plN5lLk<*T6sM?N_%aNrN9Dr4r5efAfXRH$~(i`-pw8cc4f0Kh@_mHoQSKF zXP&=&X+@ZvT&SWG%|s`{I1*?Ikbtfxj-LKE(7mh6*4-uuZ+IcUS5G=e3opokW&#ng{@21}2#^13Wr?bOYOLmnR$p3jh;d{*&R8!V@P{ z8bF7#fO4G{*4j%%Xp)sg=pshw`YR(dA~I zAmdjMI-V7z=9nODpJRX7c)1Vjwc#!w?Ovq9TYR&)*e&8gBQ6;}=?7p_sCh87_t#=Jz}^TF@Ng+~5VmP!4QNpJ$EB(BW~g6sd;*?#a2MLidBF zzMDEks~sVldKPn~;F89eQxQ)rvJ0LL971JkdNc7)I|b=vZuX>*h@mwN50> zCncOsyy!IwAE8J4YpM?Lkq(^<<5PeVqt>|VPX(x}dxDZky@eJX=^gHFjSOOu^O;}2 z?gvE83kb*S8*5AWvYJtgo^qU;vT{7$z(d|Qra7>e+s2(&)5PQ@`~S*rUIZHTQTCJn zhjw#`&*`|aHP4(NSVovJ9qMyg+2TApS~n$)&>gm8S4-*y2_axmzvFd>tYFq?YStF1 zlT5oFZfUO&YbW?`ZtV`+j?`kHR2;mpB=n{ZN`fvDcj1WH;0cp6F)?umVMabs0Q@P4 zS`3cU-|pB*_d`AmCIke_tswi(hw5_B__{5=eE!@52tJ^EJM(^#O5s0hN8Ic>0>I;)@!$U)VJ*AfH=La-i=G{Dj2R}Y* ztO42XGToQ9ZTl`QKH!43jx%j%T%|=$9fZwCih68MzQBjy%`>$a`1*WxvDdD=T2M&H zT?~sT#^8us8-Hs$gj1Br8XFsPsR1h7%}Zhk)R*|fynp_R@7UX#{OuMug?s}7G(jl~ z0^Z)g{OiXdS81x!jT=wSDMX!o9R}_hIS!pYVC*?=5_JymYQ`^u{BwiXf;5ODgY_Q&r#sf1ojCiK0!_HgE2 z1{%^v264|%2(!qQ{O23IDKgaby)f#Un~E({ZsV=0s?#?tP1-(l>0&0@t2^FM-;EV8X&P8GuR2KIGkryeVBgJ<(aS`4f*zx- zlJd=T}$6`ib^ttbNQ`*r#ipC5w#e^yNLL7!A9eE?dUEA$9dn zyBYC*x;@a!r{9~tNDi*Qd$irN*W&={wdJ+=7bUMBHMOprJ;7b?zBBp;j_=1(wj%z7}Y|n}p(BIw))r7Ask?~lU-!zYwA+WEeIYB-;f^N-Kh^(efW;{f4x ze2~DZY3e(FfI4LD==!OBhvEE`L>KTOVjAl>K*p@5VI*Rf+LI24q1E)}uT#E;PvsQ!c$ZIVxOF3Y~MS6`D`Kr38_fz-9m6qLBefG*U=36 zTBASljQ>fDJrEy5bLhhMT??8}*kRtD>Kh_v<&v?%Z`z0X@^m-IXQnjPrpx9PHjMiu z>P`2ldQnX`)-o`tpz8k;qK=aj`9<-52P#{T<+73M)V*J7g~jW3wfGLl18a;@ShF3$ zWU-|?5S}DY=<%T1dcrnSfwv4K(=OL{8F+E8)Cc)B|9;NEyC#VDhEGqUXS?l4f(t)g zLF7s*p`jz#QaB=T4C=am3OJ73K|>LV?CoW7^U+eUx?_kIKsjS0EiLRkx0rm*8)mBl zyN2)_SrmQRS~3wcGxH!ztN6zA=Q@ZY;=cyJ>e||tRXP(+PimLv>YI!t*4$@M|h&5FjoPQ!z35VA>S*0XPhq40K>% zV9DfkDP^wSN~{-w0@!&khO#k;9TC}>Y=i%YZG*V;0#AQ-ASWxg zhhnt+C3-u8cIr+=s=jTJPPs=fdy!641jA+9GvS^a{%45%Pa1{A#jivCzJ5(0rl|1p zVu(S#--ZdQ8eR>i!l;%m{wB#(+)~)_*9{{tV8Wt+(N+34i3b4Da`6f)4$IVu^&(Y9 zD~do=k$Ov;^UyOtjH_xm%xm5-t#&tiB?l{{m>D$sFjQ4l-xZ$aNhFDm6-Q~t0I(st-kH;_3CZryawK% zUbNnYTMPI#mM}M13L=Y3j7Zhdfs*?wUmU=F9H=a?AC$(B;Q3+slox`eZepO=5y+t< z;2oE*#p>ANMtanv<*V~dO|0gkg9Qg(>|qcvf7=ZCaFOI3MYx)7j&fp*QaV@F;l2yY{__0YrC=kC`16$K1*uycspJTq<*bS0p3K?Pml$~ZRoYSz!^ zSIC}Q?0WpMvFDeabYls=JV9*gYNs?&wFeK-Uw-Dd{m^YJwfe2y+HbY^OZ!O|IhEDb zL!b)o!+PTi!?i2#J+}|>R)ns@OC{}NGw}XS7k&8G0=l(zi^yEeDDAWrV10bz1h0NV zDY-;vNj^6IRx@{-rh(T7Wb(W)JTHFcdfT|&uD>J$r%KJ5Z9gRGUWDRk*stru2mj8d zAia2Zu4s+Z^h=X?Mb(u4ELdjD7N(_Ig&rUS@2E9_{1#B)Qc5$$+0FBF&zD5cJR| zp%Uo!AVvbcMqRY&&A05ZET{HZX)k!b>9e)Kc@$sZ5cg<)sG%EWQS|C)WvSZf~3qBl|`)9h*OQu_ZXFtf$Rt821^abG+emxDzD z8XB#X|J$j<**?GE9+C?Lgq&hxdOkirP^So1M8xU#@bECAQi9zLq7+i-d6CbBBkt%# zi)j+URueE%@V&-Gz3eC$29PL)0_()i3H8;$fg^G?&@jQC9le37zfPoSLH&Bxr{cgA z;iIlPzKe>w2ue@T79NLfU&_k*taL%Dclq+=ckkbG0?hy;js)Y$eeCp1Osrcr$#0d`?~k)co*>P`E8xb_Ru@O>rLaTi()n*;4vN?WLHv zeWRXjBcf+PQCncz$WXpEU7C);*e!ipU3CWqz~oqKA}BumR8&<9fZ%Y9dC3dLnW@Lm zsg&{f@s9e*U#Xl{O7`@-{W@A(86e!+M}>@@#!F8&1xO%Z0uBbsUL}DZ%3ACtxj;;P zgH_sZW}Ue)8`u+0(_-dGf4+`94PHy3$%!*Ju)&Ugq#kmJ$xnN%YE z`*-h>pzvC{b+bwn+goa*>FzEmF>Isw*a0o(eFXtw;T@nht>MB zRY+i9JaOIhwb>e&-VCT%)`Vhl@zpV-ENzOnHWTCb@Nry<#Z*^4vXKO6o-AsEAp7Za zpRAaG>X0xt8zPmcoj>LV=6;8hSDsV##Yln&SxwJvIHZJBXs{bS#pK$Toq?gIK}F31 zlz6VL&-2NdHFOaRx`HYi)}x8UsBJW3ew!y$N|j4jaP<7iF54ev^NuOv4t_%f{vd<) zKXp~b&CSg#GVqFrf%gW?qKKn255X|H%~R!56gno>=bPuP$xuL=i)B9F=cs02K&Cg))625JO{;>c!i$oDiXpMeylDYfr>3sXj#^zF+)Qb2mxH{F zlv!;}ST*P=p;WM;`1o2usv2hInFpYtpil|2&k&%Bfb*+sYVLNT7A`1$Oxq0j`kRl6 zhpb-u)#1gS{_fm&Nw8l6D9ScFq5ekDsCwf%i_YDQtSx|xSjpQ&3Es=0ad=Gbga zteT!)^ofh%n91x?jY+(n`BEPUQ`6I~s(MC28m*pq^(|i`NbC||Q<9BgUOV&Vm~#C! z5ML)FrSYo)o6C8YR#pWja*xH=GH>H`gOAUwpxr2~53f!=CQL=H)pCk0IygrfiAaSa zJBSF`H7%`{9&N}0(>;4J)|=`Es_@pIRdPLO#l~#yGVQ~yeB1oFd*>d%j)CNrFTUxI z`3`zjx^~K5nz4vfLMmb1KM?+ihlD5pm>PdBFmPc0kwOE5%~j2cLBL$b=cTN);U_au zc^rH4!$Xo%X{wpU+}r0>KXdEsb?P=5mz878LP~ug#Sai))%uJXbt$DR4+glsf!+1E zNpCE~H~r{oFM#@yx9p=9B0FysCJxc(S$FTdO<30;loI&}epW&=0#)8*037 z-dQt}iq5I9nvJl^)owiVwxManeJ72Re7kMCEwx$+R`)iV-;Tm#f_@fp9N+_?bZOX0 zM8_WmTNP}ly>@zh!H9sJev{uzg2WSGY``BO%OG^S8ZfJVjq6(6T|y9X__2I9S``H8 zzT>CWiP*FxO_qkgsJpzC3H>jsHnI`h$z#+Y#V=yFZ6zMFyGroT;Ws@Vxr^F$f$dA0dDOmtrB9 z42~UlZ(-^DQW7c{!UYNx8LV2HyOc<|1sN<8Bas=WluUiCkp&tj7)(mxj*a4 zlP4NRurf!MmLqLTP|!yk?4ixPvfYkkK#&y0SDvjdkyC<#qTR{m^4pkP*O%?H^f!X^ z9~$Ps-Y(a_$n*d$dpBTZEHMBOtn*oVYrBgHwi|;ik?Lwg8)F*?Am+8Pt9+sD0!l?$ zIfptgK~yy|GAbP{Qc2C*KeA3WHul3Qdap`2Lx1Fni406A8;MvU5f^J7d8VCyt@MPB zC0AaPQ~7T+CCN&gTXx+==tlG}yZ&N-By21cr*OZhHzYWi6#_4)JwSpAg6JvOfaRQg zL=tQx2Ed_-nVIXdULb5?`%9-NZ*ZXUZTyI{8@F&)n#HXYy5@_*LhbbfV}le1?xmMK z5EW5+i3*+DX`QANB|XmO3b4>`r)j$7OtJ6&`rBH$Ki0CS>6!OF(SvsZBHuPcu!<^;bnAd&=qQ~N~g)VFUp{TYQ70A+?67Z~IY{^n~&O@rOhsmdUo zMvet1CI>TPIi%tcwa1ea-G(tL# zO?wjPy9XL?k1w!x!Dm8`Q&oV6U>iKP@r4u%7AV{V$QR<_%Nv!JJ$BMD7UFGN=dEv{ z^iE8nf~=yxxW@0vqNjC)*mQ)T&(+&4j#ETzi^|k{i7`#$cEaTxIUx1xPEWVH-~N-@ zluUKrG)-!?yneaolV(ul`9HW(Rj18A$ho{8%*R&tuw2yBv?wPDe#dqA`N@@*N!t11qsVXx~6Y`;`TM9R`Z`^zGe*vK3 B4c!0$ literal 0 HcmV?d00001 diff --git a/docs/img/grafana-usage-node.png b/docs/img/grafana-usage-node.png new file mode 100644 index 0000000000000000000000000000000000000000..36efb030a84db417de0b2d2503893b054d37ffc5 GIT binary patch literal 105880 zcmd43by$>Z*FFp?pn#yFfTRk7N`rv3D1x*S(p}QsF$w}ocZW(zH$x-c3`jFDbjQ#& zGw{3Ey7zvb-ZG-SXfxsC0~mvU}51{ zV_{t(zlsa~V|&W<8a!RGe<7)K6&xN{4L^hTcO71-Iw)EjJ2>mv8DW`NSz8*h*c;dx z8CluCw{}2YZV<-8x`!nxCaUC;v@zxEp`?7;z5`2+pm}lcei{9TeS}iLe&z>neKq$+ z#As?=ta4*L4|8tKlkYX8EmDfrw^zBpk?>!6anF)-^QfIZQigN&qs$BnF>1kil;Xa{ z2PIsJw@)?rFuU}0xX6}-u=(fDpX`L}goKv~cfIhCCAAp~b>C)Gn&o>>)}^%>AN+kN z+#!X94nFl-Kiu!Oti-99%+Qvj^*>} z2Eq!hQtw%H7Hhn+|K}SPhDwJVF3Yhx7x(A#Cgux>9Oa*HMjgHT=LTlKOhtBMBD8Tb z@aqc49%(=$<^n$dIXWcm-0-Xe#Vn?LBj|5;yCxouEHfsQ$^3h98)zE~r-T~f`-__s zFI;Uf8LBsK{(WRW&T{P2fWmR&?ja@A2=On~b-6J8Jx&j}SJOjk>vCS*);Y9(p%ivv zxRpuhIy-i27jfF1aPObf6b9wgw-0kj-_}m&-b_mn-`xv4rnok9+9&&Npg*sv`ri}n z3E3xysgq=`LQ7`6jfJt{g%w%$?Gc#a>eP~4tc*+K$!EK$Lt|-;4IDI+cs#B^IiXe-98H7}$BCD`3U%`-cq?kW^`4}c+VZ=!O!)TuEOM-0_Q?oK zEPe@pn_;w2w51fE%caVb0d?N32Ol+jZGT%vi%2s&P7m$D$)uG^dGIZQma|0x#nZog zm6CgXlX;>IAIF-cDZh3y4I7O6DYH&&%*z**<&&RCe&h$4J^M!UB%2bt-$Z~WNA{bOLje;by{xK-BUol z7+>`yz_XfqMHZ7U8Ofk&w=<^D?ICi2cepjB@vY?d3@g}Qa;hhBKIqXSDZ+F+XYMPe zS1{QX;%*43P)@{;wnawl9p-MX)}|&c@rG`Ga!g58DgQiIEXUE}ud#ZYy6F>*tzh-s zOy_%|tp}zga+!(k(_(dQ2U0BdM^nzbD+cm5uqo%86cp-a<2&=#b5+YaKl&JcuY?ZM z?`~PN#F{2jBCpJ9ChQ7F{G67c#Z8Uysd%Tk=6tVFYG_TR*%=KfP0y3kl*jPF_!6OH z34zv&>9DdU>4mw=MVZ5;1aPu%bm324`sC{-P7`~l66c3b$m#F3lzw_}h@EJuAbwiv zk&I+SM3^pPw1Zt5ROtBS;qL|*qB}O7Vy(wYci~bW+#Khwd{0ZOIvIh>1!9tz%j+-~ zqk8N3$?ofXJ#vSY{*<<0N&}w6j=nyC=RW)U^LG%Tore08m4Rwic6Z|ARan@68|`rA?OVss!NIL&Yev4ThGwA6b=@mgZ(y-XTKN*9u27X^_F(1@wBu6)r>HiZbwv@7>*~j<-{q_C!g!}=x*zN8EJhX9c2n|LbSOh=yosnMNJvPU2g;ttu<7(g6`o0} zt0zD;&vqVl@=kknH;ABW$9B=1hnsL6GhO!k_fv=w5aJJbXUO{z5s}i&>+%j$)j?FQ z*i7&8^0q6uzm@I9L3#pGay@)-i59-$(avPVGM3m3d{-RMyl}MPiffbhu=~QoDPZiv zBO{v+Hw8Y26i91oQh`rZ^WTBUWmc}s8BhMKNQ5*nK0W>Cxdzrx3!caXw@)ODa!S=9 zT|pE1j|T<@2q}3!RXeVes*%WN$x)A&Gx=88FYA?#o27d`$VxmtSRcrer={RFTPYRf z<&AY*A1O5(Hq&HPxn5rI{PpXro~Qdud8O*N3C{SF$cuY z*VAl9m&Ni?jMq!eyW!#C`y1mvquP)(^)>UUoI51iUN>u|`jV>lp1JKmvbD7(yek-2 z?|HiW(}&o0<|lTl0hsAlmEqLWu2ABCr-a*Zo_f5Jd(>DdrIcyG; zRU+c!?`}83cT?KC*=nM22*@HLN>I1U{|b)_3)vk%^1#yBfm?Gub6B~EieOfSgDH7g zi)%X#zw0#Y0c*3Eh*F~N%`XZF;pd8%QO5sC;rEt~QsguQISQ@!KuZ>r2GVMbO~ ztW{N2pL~69-o7msN;CB(Fwmea=x(LvaBNk@_syWsS51a~gb!sa;$bDS#Fmvk)np|* zSRZA4_UtpT&6~8-QJ+6wwX?H`KI_RH`ec+$iK}Mb(l>2%l}UARzMpws(6FUW%BI$ZN9(nq zCG0Ww28+gPfdlz|Sgq7#Cskf#77T3;i;IBGa|q~>TD5~(`H=IJg@tnqu+?l(V@ULK{K>6v@C*9J;8&p(1ugJ83!xD>!llzIv$n-pZ zpBxoCY^Jx;-v|~^L_|b9ueC6iap@P}5zkJ|`PN4ZO%^%`fE&wzoO|`8VBD`mt3BrC zC~O8^k#X8D-4{a>-Ov$HqgHL$FeLrb`7y$4P)2S+{p&Un4%Zr53r z3kL>uK*#yu`&gQ&Cu%K0&%aQtm~_Xmde@=avGKUG%5*Vy+N| ze2{ev(ag*1xW~I^#qzp2-qN{LV5((7Srnb5=`^;sRzQTyFOE-7f<%pr$JgMBV%8ADyE59Ai|~UH1+$z_PbPDL;nIWAN=l;2 zltq2%>5*Y3l}Bwj;RTXnz6`FWn~$|W|(_SMqyVrMX8yJ!$ zm)n0m=K1_6w=t^Nb|2%xw#q{$g6yE@$T{_fzENkcI3<}hHZmb0rBFAn+SPubHa@N# z@UZ5iac|-|L8V>-5HslajC!#s~-6Q1(dn<}b9ty=wQA)028$zu^7 zE0J|xMr&gbVt`R5-(gZYje?xfvc3CpR??Gz5qOD9Y`i@qie~RUqOHG4=VuR29I+nz zxjozLad;B|3QFr*IpEXJQA-*!3?yk1!3*UojCRXYM|Pmy4E4Jd!1>Si30TcTR|nNf z9PZW}ZS0J^Ez}mGao^YTIyWE+jAho_{`zs!wS98~QI|OEO80S|;^|B;uXZy#q1VX) zFTh%NIbGP<1kb71>I6T&g0SL#1y(PJ>KQaNG~8`2qb(k{2)b*3ljC6BetV{oGgQ_U zM)wms4Uvz`l&##VHb1fNyhTKNNWXLB%$i$+S>)UL3%uWXJJ7Cmy_w{BH`KUS9Rx*r z@wRKb=R|d9r|Va*Uk4T)%&b{?wRBu_)=63Abv7p^S8Bp1z)FV%V==zVjPpS7IQ}{glYZ$AC;AE|nD3KGRAV>3y!YgE<#63pvw4&vFuF^Wg}AOS?eo% zPIDgcFvOscHF&whO^Pr@hay9{T(<8acX<&Wfxx17>S={7$Wceg zW#1Pc2S8M^+2mFktgr;avPqL3L*l1-3TP zg$ng8^=e?naXGW5Vo-{12WyMh5eU0Im!_ts%JnVx@j4S?ED;Qv!1{bocB^_%?jm+q zISKG;LRhefh0#U4!z}<&bfq65n99#jp4ib)JK>#spr{cpo61}D>#QlRhgG+_-#MeY z?I-`ro8>BIX6|9}SWUJ>Je1~$ml1~WR%*%t^KAZ=(f#C3WppIdlPCBeUiwSXqV>pe z@rY@zYzr%^X#;3Jn`Ga|@Z?Fre0ylA#kkG}V)G>NhjMO=!{$Wv_BMM6jjTauLw6wH@#m9F7ATKE;mH9SJJcz;(o6ESz@Mufulb@f>!LUlP%g!w# zYW|)$4zdzYloNpTExU_MujLS$TL#69Q@RK_m{952S05ego($=ic-=?p?`bv{2F|J+ zKHC?afBND~ipK%Tigp$^8DtzFNmF-}Cv}z~xZiEHJ)SY%GohbpqR5D3C7Sb>6@55@ z&0ZaxoanxEi*ieU{Vi7^8=>*$M{oDtq)fT3)#O(eAcfRmK^s`Hzw*g`IDCGJGhP<# z$gyS*n|#ab1Y7L0w+uovCwrzB5ndt&9rUM)dL7xwYrdfhwNvW8u?Jtc0 z`xq-R0x7-IevQ}E=>g(eDaftcC(75mlq*VT&?0j^_b49x?Tkir!DL0ihjQ_4+vS zLKur|6h1!w+~~Bi4fyVX0!s~^hpr4apFvu#Cdn~!ne<&GEgsNf_0_>FUJy-ic*T}` zWfCHjdl7Z0m9cLt{$UV*BI;dSDn#l>iE5Vvb2-H3efnUdB7_ zP1T&*EG(F3y@)9a%I%@<3Iza-^;+8;t{0yB53aVDmE*Sbtj;l=CoP`W!E=^^cjwA_5O*r=QI zbbqyXd-mnYZg=7FRt=JzL;uqgFb;mHXaSu@$r<#Hz%uCJRojfg*pwwAhq8=smT}Y!4O#UT= z>g{e513V+064JeSV;XMM_l)cpAZa2_M&w}7_CEkjK><9>nztN##-H_dy)ok zvm!Ti4|%wxoya_HDym}!c@xRUnk3gx<#=d01{>m&qfs(RMU4>Fg9h2NV&+M_K)tzQ zQKE@f{&9yfMOeuc!>^L-_&N~{^mvQO5Jum5=j%;E?fbf!d~7rC=WH@Xn_rXlu2oA3 z;<6rPQ`%ZU;T(y*SMczPjeGekEFSacjK~@rGl+_cB6hjIZGUZy5U)T{s=6pJGOv#c z#u2jxXWFE|J+5(hQJC85scNM!N1cVjyrZZHbF*ah^S=DJp%@RpQf&_t${GrX#)i~< zLT1Vt0j7$O>e<&n|LIiBRC>%N2bc(X;R8Zmgp1I;b#=I`q+~1WnT8cjXsiCxHwl`D zbj>6-Iz=}3Y31Bf-M6X(K|*@v0|c{9lHzqLiSU@`2CQRyKr|CrC!+Cb>@^OnGQf@@ zJ@Fc=UQzU1T!EcBa(cV5itpi>*DvGln;aK?OB#i;$qSy`?o9M-Sae5XB&nY4c6glq z82xnx2ZzICOVf%5LPsY~Xrica3*>qCU*eWJtaAPep>8(2zB``jB}3TQ*hom#pZf9g zDOv8Ij!jjqrW|8T?RnVy867d>HO1e;&h zk;#-tYEL?Kv*f83myDgRSwS4BjzoQM0kJ$++OufK<=Ga*3B7Z{77Z%44&svo{0A25 zL}YaIqO&^`4s5-7tk{4N4kd%2F-ke_&-tA;RI@cBPtu0aA_%WtWy91H(eEIn*zK>p zb=sPIWtPK8oz%U!o1LpI1p8SQ6hxT1h!Z+RhAf=2TFy%KS{f^Mpd!MPl3NJb3uf7c3+In=W7A z2WIA={{A;!C`(+45Gp+o!Y=N|W2s*3=6FVr@RLu)#|w;Q(b($6R!^Bv)>&G!k}KF+ zx<&*~-abqvUD_(*%!~H%3*T<(x%X}pTH;X{_|~$7d~C9F)ZC||$8pA?F(nB7rAx5! z@FJ+WCk}H}DM`D3am<@AIb2}kzMBru7u?SEUAZy6hZ;~|vUgTjgY(>|W}zB-|ZG zp)`xY5H#~D+Fy0|auoxTGpn0~nYf}-S`b(K&S3Dng%oL7*$mUc%(C|b-oV2}rf3U3 zqa;yGt#*wYEA@PN1iU^H_1y36DvoVE5z}YePLLi~0>RGHJ85n<9-X zyN^}251X03-|v9<#kb%Ze}5sHZbJ6d#w#FrPaQ9u+|pP@y_1e}bPI|af>#wsy$ht5XkCub=8!WjNiFo-(uAlK!-|7$t|@5Yf6UHw{AkGyQXPf z`m=hf57wEuBQrZK9xyWcSM4ovx%595K0VfPSRaXY8yGk|~=w!mM3;rDkeNwi~zp<2W$b%SF(RPZJr%@Xe?1`#)e`#s!RENM{tS16O!L zLRS)>{pzm_f{Uz~Ws@8%1KrVr+YODgU7SJa57lrDo%aYqVL&$`e$MR7uMUAA;_djZ zXWx)Sr;yuTPk@mO>da}9h!fcBBus}1UAXR!~PM_8QUn zDBeFO2x@7$0zAnZW+4;cl`rEFVci401Bpf9X}WH7f2j#^>XAQjR*BtWj{YLx!Bz&d z$bctoRWN)f7T*Z}@rGo;@n=yCI<(W5`ehtiMU;KJ&iC#l*P|ff!o2P~=W=`3Hb_Sb zjf$-6vd&yoYIbGbz590B)YQeQ6p&owgq;F;j_^c!(s-?iZZ#ZfYkRnQLxWoNA9HNu zr@YN_s!0|HFpF6zxgn--rm*(P2Jf^3-F%x?)5ds1sh#pQGBYH?16?TJZLz*X(*7kN z=jH3yuZyj1<|$gRUF;7l%!VTvX&^z_iV)q2P?3|UsRoD$DB*kqklJXg^t%_vlK+Af zz{a2eac&Kk%?flHOCltF&aSFRJ)c?Ut)QN#BOl;~<+36H7pcE~Yi=I-#Vh~jLMy_1U#q)IXQx=|00LWHc81>(z-xJc1yO; zsN!T&ZrbiGu{?6rCW`C~YJR)>HS76;1AlH(+UP1FI?IaJYEle!xM>QnxbA+qv9t7@ zCC3CtGnA_mI^Ql|WIkG8ho_YI)bvHFa9zBBvv6l;=LIPScs1;l$y7kBdhGQ=#l$`h z);L?^UZ?0-=v1_MmSgSh43SIPPDcT9Yd_O2C?Mbg6H~W9Nxxyv*;m9YGl@AdZn3}= z;E%>9oeGi}O*Htz$ew#xR7YD%5FENcfTBgyf33ZzSX|Q0jJMTMH>Fx#_~=EXO=V@U z&A2H4@vS<>PmkR`*LPHWqQkKX0FsdL9;yzTm`R4AYtFC!)&hvAUBBL}yF^MLPgKFx zXSQ4@#PG|~xQnu38q%a#^p={JlE`P=hJ#&(q@Lfm1-BO;=aM?OK?8>zrp5rdCQ_yDisc@l59qi^i>A9>s_YFtJca62_|5E<3v+Cab4T#=foeDh8l&v-L z_VKP+26@M+xH zVCG1KCkR5pD=Y6eoCMyP-vC7V<3lsB4jk9S0F5?_Ka2!rq3iqB2W#A87CfL3;i>&9 zH50Jktz8RCOC=y9v3~O9`Id>eI74>~#uvb_mHzwSmdQm$Z;+?pzPQ9RV#|HWps0!7 zX{VB5h@3u@5EJV!GgU+~@$j?>tg22`JH8+I&IhWFyA{P=7&pKxaw2y&*fk5h4a~`L zfmd=n_-;DFTs$;yO#bkQuaGA-3U5a8C_Fy?)pUKG>Xs?*?c z#AV`t$A3_E@KIdd@Z6Cb(<&PI%vcZgmVCNdyuq003^6J$#@;h&_IQt6=~O#^=Fqq_ ze!LWNlqoIq`g0a2mJrek54oQ&Li(PX$Gkp2!#si1DSu1$#Q``Ol;hOP%_K3W2cN%u z`DV~gq-ocw3b=_YKbw35i{RAJRt8n|;;J?6eV-vTvs?FPC%Nc@_`lU(w;btjVTzTo zft$2@R+t{`>lJL0NWJjnT2*7EIb|KJ5q{D$ld$)~Lmc5`YBf6{{LN#>#K|?`VKn=y z4>5pQ*LkM}LG?6ZtlE(&6~YlucD{4Ty6%LZ%eb#e56WfOaf70*bw>elbqe5IuaiX; z*6NiPfK|N-cy@{8AN&`qr9FgtPe71Lw_`?>S*`X{P>@W+Izb|bJ*f{K@!W)YL8fKm zh{ae@cv%0N)BUYiFJ4@VY)G2(G!y|41_DZ#p~AcDw2=M#d&_M14mxhGjpTR##Kv3@ z|8+-&$^Oo*PDq&mH-{R+tm_#axgl5 z5k!EUWmr!O*Pfrz1BwXyC-C2Zz6I>tGlpi}Gc*oWM`C`NAizxqAn9?`c0agt(KXJ} z5^_*J+u7Nv71aBFlsJCwad@||o-_OliS}F#GE?7qtjcABoZ{*GpY@&(MG%VNd#Vv& zScH$aUxFbCk3^j9(j5J&oU+n}%*zrse=Q4Ny@*T?Ua_bag<)Sl44nt05*M+TYfVM!aYtb+6 zPm^%2cG`R~TA&5g3T=j*H>ox{i_J>MMYmjHC0M0tT6NhVczP@sC$ zxZ4O^@YG@~3{*9pHpUv~TBTRUis^*h5AO*HC4+o@-p1w9b5P|q`1!$We`_jNAsbtI z-hm$sr_Kf60np1m#`ZRF*@c<->1{8k27(G59;nk=OsQ8t3qk=)`Z*@JkI5| zZZSO(Z;gE?@_~fQz9PQrYp{>2fKS(Cre)rFgXo~8@!5pIQj&{_uL1*>Zk)^Z=nWd$ zxbDE_621u2?ItAY2yIEp*4LUA1DhO?1mGGP(7TkqiM&~W9B*lB(=S)==;<*8xbnhm zV_B0|$BHlKtCwF9Os&eAm~GD9m~KezOBMhngQpnO5!1W2Rfk>BNkEoJM2lae)UC|c z^-4ao%&&Dbx{^HSY0#j<+%;PvFBg>?651wVCp#Uc-%Umjzspg)aV@=zF2|~jQ9t=e z_ERTiV;_pN8AT(1X{mOap#JU7qfOd@@9PLxRK8vLW{Nwq@XdHY^;xxf)vmRQ4V>D! zn2F8tMg8${W-VqUF+C&0_NeX%HGdEuWg?zzX(*@+Vwy+|7xA(P4x2|YV; zstPdxxK@z?(znKK?sDc!R%gW4fabKSo^{)T%$&cLk~(CS{~WqpyW8y7&g)Z}GSdwK1=@COOUV%tr`bC^f9HR%J8E&pdb-7`ASAj*rn=fqRco^L7|VBEkL|u-05(~@@)rs( z7pbE0c9P`qteg=fDX)P3$NH2YvU9=GWcKI;P;E1^mlYhQec=}5e)80d(GAEZQH-|{ zu5oFGsykuYSBn8J(oN}EOq7y@WP5eqyiytM!(pUp*0QDsGpRLmcDkh$97f>7l})*} zw3XT{9jtO9a9)qvgUFuLAi8o_>eQoQr1I{X2-pDnX7VlWeZ8Vp=F({Fpqq+{4o%Ln|F-2*SKdj;SDY`hgo;3(DNwfk*%B*G>eJdHqQ-B z+Y_T8YA!$Xf*J(UP)R{xMess|))CLjpHl!cSpV2$$f=_7rZfiwK#7=y1Q~$>m4H*8 zDr-jo{wJ^MDJxA@M%LIwz?lI0#i?eBawbQXE-S;y?8cU%uD!s~LB-AnQnY76yUw0f z+5h{|S|@tLfj=2Iyg)z*mD(?}E&--%%QAODF1C9$m}LpCrIBk4#(qygpq=u4!G_68 z(7)Xb*=>_iJw{^L&Y3iUCeN57qy@w3)T8=1bAvN+fB73n-%EWeou0G5oyC2#J?pui zE>j<;VRy+nJ2@mXV+VmkGR>%Tum07u&#%=8oA0r((k3Zc(6C6s^txDs?EXr)B%r zQs_mKd=}&01g$~YFte|Fk1isX!YN67me1OXj^#6PApKk1=Q2->UQMJklkHpaVBGU6 zR%Hg-JLMyij@0YQ_=rQK;m5CC@(X)gp*%F`mHmXIjrmX!y~#@Hi&C8P^yTeCI5MUt z@1pvDaU{j(M9`ufoESXf6;_j=SQr6~xPzSYaHx?a%wcdQVprDcof;wudLkLTxBJ$G z7XE18eLxc$-Rt^CYnc_hQaw;PrQ8j7<0SX{Qf6SPT)_AS6^oQv&3bLAvafx8$i0B4 ztn_QJa)&EH(Rvs`^BU!`=W>naQoil4MP{zc)oy;aqrvPe9x+JhWda^0Si0dw$HBn} zUlq1lc0lUXb7iDH`1rkI(*EOQUk;8napkdTcJi=><_7=FT>FV!h>?qTl?bcDc9RK? zk%;SqICHi~V^i?y<#$G-(FzV8#j_^yzx(^z(l!$<>;9K;GneJa%<{d77iF#>X1uc9 zu1nbp9mwOQxiV8dK5K9T7;<_D z^sr8_VLvBQOM_@T1a9?rKI`BIeAb?O2yZ*YZ{gtaBz zd>!PseBoO1fZVt1&Y9LflInB@(}n!sBL*H{V<~L^tJS8*di);(>RTV=){~$(lfojk z9ubpI=KsCw3eo@cyqhqq86T_ZIHYj)YB2mWq};Oq)tG)$ zq1}pS&f)jVDGdJA7=M)fuao|}#jhLxpWDQr&&Zq`lHKt%z6@RY@6pOS=+Kz- zYU`o!9bIN&pTudHAF=uOj@h;pe=oobgZOa!zh*egdE&~o*WovBa!-BYN@Gv4-Ip8DSy#d&PwDfNCHa)EO{P7*2cHK&_YP#ENQSk<-Hf*YG zlsOvg2i(6`43v{s=IrtRv83$a+qL;0!ouG47;_QDCZg~5nwi$-h^zJTzvu2p zJ;l?{4+R*rejOawFq)BQ1%Ax0E>QeEPvH}5K?Dd=BDT9q%#(&3iM!rP56V*qSAXgk zD?MJwaepCKu~sSfTO=-KmOCimvVK&E!S}<$f8`<+>dW-$Ue;ab#X{3561y&?*?-!U zFT*{B^^Yaod@T1r%)$R;(ahXaA0j)aCl|zVVgr-59Hyi#zy335Vp)#cBgj;gRoaYm zbFxbM`;~X!H2Hig9rLAG{ypotRN}w82QKDE=#{r$=z~+#YGvnT0T4g0}o_bwC zM@L73RoC5@M-hJvI_dH4+Uw674chcp$1*Y#?>_`}zbjKU&V{SS9gj(G-);%I%LP*R zSH4j0#NPdNXBrZQ`X1opw@M2@{@}5@$_P{}AF>r2OvZ|4AC4%7mjJ3{H%hB4%v3~f z9!Gjw@O;DOLP1&n(!oSNElB70ho2R2_ zU}-~%>t`o($~}j^X)#$89?hgO;-I*2yQ{Z%ai^V@@!`XaCb$=B3QEOicO|gZyVv+g z?P&MYHRd>xRJe6bGOJeg?v^{U2v7}&$Z3_llz=GD#o=o`DF*`3DK|=x04?MLdL8J$ zCCAIVvotj|FC$#$9l;(L*R$h!Vz>R3Mge5~ONCzyKnN*4WnzlC^~aQQ20P%99sD*| z9feQ5`DwlU$~B0m9hdjN^`*l;CvZk@x2Jr(3Ntu*tC*>uqY+xvUGQL|-d~!9;D+EK z=Xc@c7emX_x13-;l|J5B?9Qul3mUXHxQ(9NgvquH{rHkqK(l-cgK-ENdj>ls(~%pB z%EnUVccudq`3Zx$`8^AbrKhJ_?LfDiFDd4EA~Y*6NbHys!6QKY|!e}-(tJJy>Z2H4%lys)OP#%g#?S#W0|l3BAX4CuBZ!{R#pX+3{_7a`lh z3?a$+>~3QPQ*axDvnE{zgTZCVK;dfxw}f&SuH5iSJfkEaV*&CRou;ItM_z-OvL`dm zlFX;}H+9aB%t3yN|5IM%Tx`1Dv&s}Li{9IW)@@R8854q>jLUwh50pf#_E_1nQ;v^a z-sNk*dV@H*6vyq(vz+42iUlYG{WW4mwf*G>!YDM?M1|%2(J}gXFcesA&e%G@68Qy+ zZ!C8U>%zJV=+|^~6qeK2iX2v#v~+%P6J$(FXRWC}P3%RlM?%8K^3<|NGeqXCDPX>h$NPiwpd7)hb4He{QuzJ+gIDX?kJsZfGrB-f*jQ#dX!DGUS*_%g zC+hH5rsKN45y`EDi()2#Nxp-thCs5nF;Nu>Do!y~u!vJ-CClo+{l=v)-cZIbHz_5K zwTNeACSD&dr|MeWLTZ@=$lpWHGIl0&MfQAw8JLcK3A2S$4UK$E1cz zv6!0;`xULXMPd$O4QtPCjc9vbt$R-;UUw=cy!*5u>Nckij^~AOI zumdD9dlTJfmwmM)NlC*xj+^$);-5krE~m+9J)m(XT}Dh9)^lLn&u4x@*q9&CwxAE= zWH!d?eTog*>_(HF?bh?5{b?rh$WsGT(;y1I0KlqotYVH-p;kgZc-NQWO;YB+`P1&> ziZ$-7pfcuJ7sLH&!5mr}fV#it36!a7ZkK6gpbqb)|Mh15;%%_rO6*#n;O^7@%u4m> zqO`6TRaGS?3#ku#6-c_X%9v_5U3cWRE^9Fo)XiFxgh%V~S;M*}nn5-pN?qQxsku3x z-Sz=n-7zPoZ@nEzja*h2W6?35D~))P($afsF%c0>1Z;vj@h?54;xsB4*3*ve&0qO8 zI(hZY8n?zookLDxYg1P>D}jL9{*UQJplG=#gdykS<3kt$NfHAqYk`!JB@d#bjuf!a7-H57jRX%C6nzg#q=EPyZ18v$k7CZ&c{SYjw=1TeqEtu&7gkv z5)n7zqa6o}>vLDDPW|v{(*okS&3Ulk6_y4gd1{0-Pd~#%&Ivz&ha{kH>o1&k{p9D% zxr&i|mMk}tuhq?aj`FjO)$U5PuD$j%iY+TFJiIX#a@HA6CvkF^XwB!eK@P|WgWd%0 z3my}eB6PvjflvT;jDh7XV^wuP>bAep=zT(fu0SS3rx}=-0s+BgztHg(NcX>lhO!l> zBnvpjjH&oHe{7z;CtQCGtjNXrP~j#2Y>{J@#H8AI0jq{&Nm`HR4h~NrKYl#v*3CjC z=<;K-eZ(_h(7^C~S*P+4*_sshy5rS6R#D1eO^f-6-8-hA=XFXXqxyPh1>N8aW8G0< zVY*3j)AL9N{;MUp%Gu06frtXOz{besg?7PWkV{fsas)dBT3Wu4oE$;Unt{^hwTw(& zzp%6dW#QkT;kgai2WTGlmJwg2&2~~vt#zbIo(hPU1ZIzff*$izFq3XPT!6CV=NZhq zl?bEa!ir;WIW^~x#j-uztK;Pu4nDjB**o0icfrC^w4l08A*%8qc2Kci+q2`@N4=wk z5{En;9|H`<3)TjTX7(@ARQ$3#$%Q$*Mi16g(EM!FWl>u$-uc5fcL=4w`a4pLX3EH+gdVM(*7dgGHxj zw#ymWk2VunHnlm21G5Lri;K|fyBNvH3z7&+UU=t<{ZbE$EhUh0x9T6N8S`j#6ffS+ zWn|e@t?Fh~D~UQ{tb<()@IkDO=~Okhb^QTpVkEEq9t-r?GPPBIc6d~q&gh6n_+*sP zsG{(@*|g)6-+)+Q58xL`O;gjjrlv-H9Dk&kuxMQ$-9kz3vz=p~XA!(Nb~?CjP|QJ7 zesv{Ka5sk6en~Q&)-xSX*p^7SzaL<3lezKpHVyVSy|858#|8JXw#w*k+$Q{>#}Uhy zGUiA>9H}#EhFx=6(c?kprz@HginapDRFHjSnE76e^YR4y6W7`&-z6j4(rJVhEKce% zDAOBc{(9sHT{DkC7X?K{CZ+Snmw4k1*0gEj!37Ee(z#hcaflViLfX-Lphk8_Id;B% z@B}q|9s+u44M9;N>yK!FM{QfQJDF0k+XeeK=0l#8Lm8?Rh@)M4tDESgYGch#)tQ;k zt34KGCA#Z+8-4sggJL2Og5=fCI5hiRAFI?8a;~3GB(8Xy*0#tBFE>{Kg3n{yiBK$y z$;$AD(lPA1Ggu9yU-Zb)^Q~k>i-voXrLdS>&*-=Ce=J&2md{%L7OKlgH}2{QoI4I? z_NxD9Bf?Q!l^n2DQlA-Jwr!ZH`Sr3=wju*d)0 zyaJ^}l{~Dpx4%|1ugyFKW7zl8u%QwBK*x7&`K%7>N+^X-xi!SwKlDjCu^(J^Mb=q& z)r;pQ1oCNS4_8m8;1H8&(-CeoJheAGA^JXy4X^+^~+WUrDx`QSZ>!Ug=IwnAt4Pkq2B^dNKQt^6RV`O zE@Lo~-Iy?R%ZpM}h&I3720WoW}(|Q}N{n4`xft;gZX zG63SAhHCAPK~d&c;kj!qYD*xx0QjlhP_nbeDXirZG-(^|rZlZ* zc;p%TfTSok*8*e;nXov~b1w{^?Ocnb)p$H3*oJ7Zx*Cl@ObfkDR9aAHmKj~lRv!Kv zH>5jpKHdEfKI6uJ^BExf!wUZeqTFXIt(sv6E-{al>f3*k4MoxaPbt5ZG==%!S^z*E z{Vynq|4!s%ax9X-uAVp!gR2AtOvN$^3bz2pzpM-J9ze#KEhRt)A(O}x92FIn^K?Dk zwq}ZZ>Pq;B@9toQ{dIs1fC}4(Ad9-sz>qn_2>|6{M>ri8NFg(O_(^Wva#{K2pYtm( zvdzj_w^-e_`gT#o1k9ph21qHC{I{<0jpxk&n_cJJu4Q4e(HOA1!{VM}WMpzvocvub zPBS}JE!dGvQab(u>|_Rnd$xkm+19o2H3hBF?B5CHYfI&fna%h|OacTY1PCTwWn0Iz z@4zpBls0JqFvNTnR8(>%(d$p#`AKf6+z%6Zw*(u+78ASEacK5=l@S$r<7Qb5Af}awfRsRyHbT^F>AM ze#EqL0I6XLL_Ch!fwawVI7b-^h`WNfrfNtiDc7MMU^ajpnVyL$S7Gz<6G6dC&Ft)i zmv_A={631FxqEoQ+$9i{f(;Z8hqUVt{%z|W6WwgHJ62FiXpMg3bVQd*8O=YOMfjj{ zv^YZebS3R#etYZyEW9!kEwXwPO7`EF7@USk4+< zp`JS&fKuQsB8{ycD#?EI2wU{|VQUe+#m9}#9>bAo+VG!YGMpbJjqjx%t+xgO-5uXD z|FTSk!04p?6!@_WEOp#Yjf-b+ynaD%js5%Oy=&pMIKN+hpcmUE{rwUv;>LfR+#X#k ziO=PE{=mz+%s&jxJEuEsJc`{{MC6RyI~uQ!K%dl`CX=KvbIzj=As%T#RK-2!&` z_chP=?hgIaK*RrNV888f?W=B3GOAumNL@ki!Bl`?Nm*OFfKB-P1*~D)yQ{l<{j~0I zjcIM5)MRS@&7<^RzZ6)ss+C|ON3~K?(ndUq2c3+x-OGEXla&MyLiU$^;>qy4kZ4$TL=9m{*K zEJ{62?5d_{;(Po~k0`%hBVl+ywDPukOJ%fs(}{6K=FdP-5raD}C72hy1pxI+Xceau-Q4y_wiI;zLhzt*q+C! z=dUAhmxwIJb^q8v8M!fnAslS?_~GeK^n{gl1c>z?q-Zqg`LqSsJfF3x?8J<15=ll( zFI$H{`}-O}&;xopx|b13nwpa{dlIA7LJ6IQ3JqSE{drpO4g14gg>#hm{%RW5hj%Y` zdE^M2`?J#|Ob-&ccTtyb!oY6@XhoEntB9tdz&>Y3lv&8AwtKlYeC}c3!E}{6VKNlxKc9aj(X$+!F8@g=vIQm z!SLy567MiTj9Mo?Oort^Z*s6EP6CZ^09)^(qhg-aXbCuOjJXjpGafxWCC!VVu z)R|@LZS7i|T|7jfHu z?l24pSR?w#xx~q|UA_URm$M0n=9<7#fRrQ=pMc=tbiI&QpkP_YMKqL%G6xiy)s~ao z3%a_bxD7kr7?~KE?nJbz*SX35$JfL^oJWY^4JcUfBoa3N(8AjIujQOhM@ndJwo{ar zdgN9&QN2mINJRDZfRg7alP!~^SAz}#8EaIpv2vky9avT(Cm65ZR1qC6WAxkT=x9ez z$hHrXiH=Lrx&b4X_V8{%KtLH-Twwo&cGYFP92J=MlgF$h4QB^zLTAS^zAwHRw_g3z z`;1{@*5+TGRl3pL(GzQ(@Ru7-HDEU8oLK)@)Wj{H1SpGd3(N2KALd2>@1E~}F!Py1rZ5J>28J|h8_`V=@JG8k?tH~VBp($-@o;I?^@6E#ry9&YjlkxT-UyK zoaeER^El6~WF_BEOGz(S4lJ@1a$=}HcF|Gb9)V)$qDMim1^;B-A7@}Q`Yb!k&h808 zSUW-2m#>aCMqpaKu7hT~vRIO(Fy15TotJy{niza1#u}&Z)uh?>^8T{p+cum91#qM1 zHa14``*r%?_>=1`z`s9uaLem;CV)>O)58l2FFt`!SFf0Na;WS7e3ZKLz?bl!k2vxl z7XKX;9A?H?_@8fo+-3IM^I*bohN!KDKXZBzPrA?|27GiW(*4Yt3T7uF8VNxITwK{JCYop3LpD zG%SXD)_<<$wqw;HUGSPux;M%;PHN03>%n3f0nl^bk7()(`?H`XPY`pLJ;5{>33nM{{d;fT2E-J!$5$$BL`b}gXl5_E9~m^m4o`Ms6?tQ4w}+Z8gM^*V9Lm`;{WK)}Jam_v~48`lA)SGIMAuMUl{Lp*aNy@Ubce2H%i>Msm> zUiS(r=#=RVuT}wpTsaV(+V98rP(REof^)R1A8yk3|szE;SYD}?d~2co3vVsQ-yqH-kNRFoisWi^V~owgPVWy z{p2+lc)Y??IV<6e=-XINfV#@!zjn3&;<(RZA#l@A7jLQv+xn);Dhy$PnP5A==6 zv##KkXqv)&(|~NL`+twZK6)u}U_^9cUqsSFdxT9Yew1{a%Gg-|MNy>$Hc*67Z8x>L)N zi!yg<2Y1*gM-rC|lWe3$60hiVleejn`6;@>S^a4nFVqmO0!O*oDGtEH>WPgfbfV+?4oAMVI2vt0IP#s zb0(6mrs@F1`i6|Xm36f}Vf2BAk3`d=K461@nwXQ1&w2L?SPPu&>_e5W+RZ-RmO_5} zVF*ApuWku*aS7-Cf<5qM;LV$)80&FLk#q$LEBu2%&br_B)XG@7aZ`Oqb(0JHJP2Xe zEI8fbA3Y6s_LDJ6?PqO&XaW18t)o+LcKJ}6ii#?hxneSR&0A%1AiYUTTRZ>sVA&Sb zU|Ao@kJvvr$fe%^{lG3DAbpH-j|PIUoaW{y>2AGp#sNJtDH8T|%(kG9&>fVDFpxa0 zoaU1Z60!D+fCK1L{-HL66?9h6q+ypV_nx9iSs7bOz(petCX6n1oa zPm%Sp*N@HtCN2Rkv!>;!+!jGW*b)=YFZOt4oY%~?o_#v+pD_|M(4P($Bc2>mN*tfp zct(@bb5CHpp4=%K#X z_R~ro3@mf%1nog&)5|pavorJcvEn!|$ca3CqhL(cL4g!%F%O%XPMfeh*;bkk9Pgh; zT>`wxEMII}+!Y8OEjKUk$j{o!s*@J^`Kop5>aI%2>mm)04({T_s@d8WgFNCw#ZJ=) z(OtP9`?9t_z(#7W}p#4Bo$OsJ)~U zogsmq-3gKfi7$@(CxK_^aj-5`1|bJ&AT6Hf@^a<5fPrF%9@9m%}GHJRYe08noJMyL-?hFT>+`WjuREZfi(qKuM`ya7_oGG z=Gk$K;6uz+oS!?7HK{VlS49qkngm%=_p>7S8uYSV^j0#xO>$jX-OhD2G;%(EBmg*} z0p$+Tsy-EcHdI$1UVTI(9u4Y705Q-;;pLfTe}L%#boK+1!4|V`a%?f5U~p3 zGT#RdRxs@?#th6`-@d)B+_V7S_nB!#PZ&N%M-^93m0{WgkL^LeBv|qQZ*NRlXAA?o z?+F5(C97ZGyf9ZfJaJy?t#}-8vg@~Hof(*Db-s7e7%X|&F-R?8Wxk@Wub)kIJg7F9 zF^e8DWzbMrq1D#a?IJe?!LIo9bm&rFlJG_|PAze$#k+C2mUgn_z64HS@a}~kZe4=S z|4??<#}f=v7IXB*4)NsNlYLd!z+h4KepCQGnA48BmX^s%R`yDPG}OA+aSUE;XBkPH z{JH^Dz#FjOVD+5#V(VUI%`uPOI3^PeI7qQhppz?LxWI<$VtxMapSrEGQ zjQGS7@v1#+W5&H}9c+zR#mI5wKmH3eMT#X(7d595!I{-HNEWg9zuLn{M;WZo<;A@= zs;@XW_&w{(lCs|GyCL|NO(I-odsF&>||s; z$h+~MKW8JI5yl|>P|M4!>BCBOb&barYT2kjIJqFpHXW2xJYWS%t@3ZAqy#cgn)dhi zwX}_AU!H-6!dBx{VSZ@4S&`FQP*cQHZxw_lDP%Y6ZQD!xw5y$&fgB(?Jcc_c*Fra( zwb;o=%!7lK8Zv{+v1bv~v@}3pGv-L+Xw-9Z8}PTM>O$GLSJwOpS-74X4Jki1($T3? zP*luy63@ECar{}S4z~XQdX(xw)wi=|inf4<_6jrhYBC02lJTZyS^D}CH3KF<<+_P; ze(hFZ^DN9Ra~?bKfb>QTp;}%UU&yuwVBKoZ0&@u0V1c9m}U{}$0DD}i_roRw}@3u6O-(* z+Ofti;8vzAt{r=yb(a`9P zFFrFN-xuhw+IaQb-Z?WjP~{3Zg`*(6IM~8Rth0A)6;q6r>iY%KsI6- zC}cOa<#pv?g`9qp!;E7AdBIB+T!jJ#S|6$d@{4_E=%*ZyzNfKYE%uzOEGZC0i5a&` z=r9}p)fNI6i1HU*p~{h5ES=HdCwiii;rPG2n$W)q`BwQzu`*UQ+2`k5ewWzegt)le zrkw`EIOnu7z&-+BqDU&aAM@>%Eo$a;x}`}iHs@+^CC(>h$^W`}*mlOSIx#nsSr6#xJ8 z_Wgf0X8+Az|Npcfu4Wed(}~J8dfb=}zTonj0Wm}O3v?(d_);=9Z!alo66gXf?~Q!J zWw0Gc6GhiK7ODFAH9BSbr2~y4*}%rv2rMm z{&jiN`R??uR=jW83;fC_mL&2{B!eWObdE^0E8tI0& z^n%$r7rl3etrp*`MsDq&2Bf%5-Q?-(V{!#7$en=`R%H?q6hf$Bek~PHW$slDG;oRcGIjMrnpdB zQ&ZEqDF}<9n+ul$bgi<-UyfG>GHQkXT{|PyE&B=vzSgTHGAGi`?75nrKhC%F!G^)r zQI)uRt3oYntx82R>X_aES+`Yh{(<7(r`FceDypoMl94%Mk@xlO1AvABZ1(km8TG_! zjB3EFcQQUW_Vsa5Z-Ej`RMS=Fg`FAVIu*_PC85#x%H2CNa&ZqR$0BkCFV#eoUFpc8 z{bT+$jKHjl=_X2VS3C6%&e0D;y3=!wo4h-r9^b;WayLu)(BV&W#BKFWHWLiX@ulsLHRe9;$etTs&djNgdX5!>wwRkae_Z_ ziry6R|29B@gH!7`K-AQnfR?jyx-Qsu zQ|9JlM=XXIgYHL^PeQ!6pY*4T^W`{#{J)fdg~bN*QjPeFeUg-dwQ+9m-^C*Rn zWtP`j8$0qB#92zm8mVYm*Yf9HC|g}3jlp42f-n+3FEw>7sLWt6OC~b&Yd3h#*L6HB zeT=FNaFeDvnwVKzMXBfEGFS9+EeGA1C0C7o3dHB7{PU*qSHs6U^rgZVbKh>tT*Ywi zvy5ivIOe0L?d?%@Em~GcaR46vPG?N_?ROD13Y!}i(pSO+Oia2ZO#~mGJ%4=ut#$f* zYQS$R$XUww(x}i?Q)ToU8$wCSV$b>u@%xZ@zpZkIu#xGecvKsUb*S;cVRuJy?~#_B z=8efB4lqo-^}e6JwW%4WHZ{LUz$=9XcF@!PG?_S$1C5bNjg8%oDa@Hqh6}L?f@q-> zxlWjjzVRs2^vsfpHk}6>d(Melo`L5QcExts-L%JVENd0dJF`^xatFb0!OY%x(!HI* z+tgQ*bB@_rRMb!pD&}_|dAwhJ%>aFos1wE8>X$h!gGG!gQ;8Pe-U9o99paLbpU@Js z=-vp%kE8?Uc7O4bI+a6LhxD|-k z1}@PD)-uo~Zls7*fa}Gjw_s*icaV#G*>?Y6l8)n#Jgd%i`E@ zb2nA>jmly44|RZ~utJ!pE?=y@Ze z<`w19c)_5T;Iwt$O4DPjk-t3|hUIbXkKh^>d+1sF7u{xpwTGD-7In{AXY8L`;~)oB zp0wyZHfBsCM{-sWcGb==3bi7WA3rC|uo0O`4R(R=C++Acgc{F=c~2Dexh0IV@Z_j^ zChHPf@y4yIl7l%VMH3r4A5&QU2Dk6YwS1VW{GYj4hihqJv3i7bo&?ng6VhkP%O$wP z4dYmip&W6}8u$Mm`oVTl4wxkha`MMhg(Z!RZAYNf?1$MhK`d;%(RH8L01*_jofcF` zAMoobZb;9gi2SY(17aAyYU8dG*}bnd{?C7`f3~~)Ng}vAtaksi7^zL636p#LxS-pz zq0DFTdHKr9Z`6XXvWO0D3oP2b#rSR4-F%VIT<&{l*5WzHM$DSXo9#Z{fW_>=fk`DF|wRCj6prY?bfr(PJmvX9n+f0DnJ{ImEy94B)0V zDLL@YmQuTBD#z-+xKsX`ujnIo{80x(#HRsXW0%o;&~&(VQ!8(Y#0#uE;iIXhd_AjF zeQWH_Up&6P%40!MTBW@8r-v_0CBy3b>}!2p+n4Tfd(~X53@#Lj65-$sh3co@#XW!e zJ}TpJt}`Wf?lRBs;^t-}_#8wVu$UwDgq`Cs(p5<*kjfm7G(LNT)GnQ1_f~Uyz7Hnk z$Nk=0NNf;dlcWhyNdb^dRwJROef#@S9IiAvdVwliB^gZOlc^M%>WdQ@$}-a@@|&K! zjl^3;0u^-S-#lfiMM)5g0rUtJgxv>-uT%aI2e&fN8{#S;vXogm`oq-djar`2tbtU*l?^$c% z_~jILO~28kQsZ_?5`11dTyY1dH{WyZe# zNk+Crd%z>ifwz=f$NjrI<9or)x2@{dO7ylJ;xwKEgDS(~!6%QGE2|sx#tHij=M#>L z-q~n&Ubw}VkThDlzaTNzK#DKP!R-5DJFQ0*Mk*sPLZwj~=Uqu_W%i+5&|Iz@iA8+- z*J8b=`plrSUD<)S(u`A^pO5)}k=pJa``OQHnwAuBLY$e&%ei8w>_)tU zo+v3=S3u|5V#=E@n~MbMQkm4q#$;84i~Dw{OTDeXA5pvKO+Ci!8+mS~QVg5aHEFQ; zbHaEk)W%sd7NS5H4hPzvuohF;JYQ8dw~#dc%6_*Yj;aTFv%6a^sVlG^V&J%qeel-df_YSFguXWe1*-huAK}t(Yfx63^5#3Bxv#6RKJYaTx0tq+p~`n z28OiBS_GI7Fs$Cjo|nELLtS~g)lGE_OJgN%Z7lvwr%8t40_~1#w~ny?EjL>(5G0~> zR(Dre-_WJI9IBzoUmkqLy_#D+JK;RGv+fqw#3D6eVu!t-Z20sH z^E&t{qHGy>=ErtcdRK^Th6z7EdiRVmoxc@vD`Q;R*SSp7Q+`t(!9?Z}YjMxc%du^= zg1$^85NX?`>;K?rMX6?EM7*{1^2wo0%@L()9eF7xYK^m0V!z8kJKXTpbldiG^gIcq zBB_yNo!xqQ-jqIDO0JwcX&*5IW#Ow(_q09CYikmKjK7Bm$SRs%8l2U|f0z8qKJBSG z*)SPdo-Q-}y7$Jo@<6S<{A{R~7CbvWK4t6Er;V2|u#)WOCd{-c;2Qj$G3BSoj!0Ig z{AuCb{fh<_N|amU@@1g!#&@rG8M$KiLqAP!b5%@e;l@v(IAl|dQ3{#O+K`01I%EC| zrPbeaJ1#$Lt)w^oC)P9$tks zlB{Lz7z-OHnwl0$(2zd(?`d$y$c$NocK0w%X?)C->(@jQ+o*BdEj?a6cApMxO>Qy`$8^&?W3&qS$P(H*m|?4M~OBVe*YUqVs#| zycD$sZ5Eyw=X!se1b5v0c@%H+sfTuwZ>^(kwpZEotwW`^_e0(30N$!2 zw?jR6`t6ZLVh$RgiyQDF$pV=%;P?lxu;szj!nM&IEf^9CB#s7xf@nzr(e2Jhx1(n} zAli2uVe3nDv3XjSoPtdkdnNGqwVua8#26KZNZQO&x=i|*S5cq(9Rx}*Wzt~_qao9X z8U=~QqfdS&O%Pt~F2>}>ebfPLZ&rf9=fUMeBCaw=j7rIr=@e*0l-8_yk-;JSNP zE^xv}tZJbR0(WUa;;gC+4&ZIgQvw`W{k8Zl{;>P@ZeIu|oHV#{tX-N6!+j%-fi zA}~>Pe0CxYcbvn@5v3u@qJ>z%2BFsuIv*Ri>r?u^sh zi&nyACjpQTX+P`(n zdvWphSO;V2`GsQlJ^6kZ#UU=m?k&qY`Y0}1{xR$)2dJN|?4iA9d@Th|Y%9U3#AivuKCR3C3cW$q*kaH1P zOX1r2hLWqyn0-f+cU(c4gN3yaN zUJNeRvnTW_Vq?ux6E=${2g#ET?A7uYIsWR+col7fI#myE^xnFC+@9cgv_3MI2JL6q zjI;q;mC}(Wkuh&0wgjh&iB92T_VzUBr(2y|7qcUYvu8`|i$h;W{q2AHV$s_nRfE)c z003^dTZ%109qPA}<`$yARF3G>wg{+PUs|G2P!>POv#~QMIog~Bg#&v(mb`$|2`m$B z{jAb7AEYbQHSyI3;(|$0^TA<|q_WFKG4fq!AB@tAHp1{Up8&W?D_1<{*Z+$?Jh?EhJ=xFe#;aGpPpi34bD^#|~@ zJCSI0YghS^0cS7U8D;asBMAmstj|uJ<7e1Yx;3F>nj-&I=!dO^(i?+ojLt()L59cb z?vG6kYMxWO=#ZjY>(i%?Kc$HpwN_|s?^9Fk@DA4rQgCvVsSTM*Ca0qw$)w}eYIePv zq87=Hs9UD!O<4P|O16tjL4unzfhi;8Fr~&~jc{vbrY|fHsitJB#dxU2%A>jKx*24y z1xtaU1skhtrfQIA1d8Bze1ad{Dq;wHN*xc+P`Q5OSaX2p`lFmUzqAp3iOxD*`B%Qr z`xmL%Mq&l7{uJ;M>fP`tE=CF-?uVJoHNE&qnZv zCMQhrx`pP8Y$a_h@$PI6mq&I!VIPWXq_= z5O|;nj)L=m;+-|)@zWoVd3;&+TLHR`+@9nrUy}MQdT{h^IAz>*Kfmn!ak`&p`KE1H zaQVGgH4ZYxk1oqZYWa8^lOKY8{@=6m_v4dps{XIVnYv1x13T+gm7GB8+K&;@3 z{_wuUW-xPziyN}Au&AHa&yK6ZVN(u`7Uvx)u9rl!j~q+>p+Ue|H|F!SN&DDw1N75{ zB@#1j-=SPSD`-St>&3y)ir8_Zy6g%{OMNkM%I+gal@a(~Z|i^(e1wEyM22Z`h&WAo zAF|3hv5{yy%PB;>+gK;})4*qe_TFP%4wl*SZ^U@D)(h7tWb{`Hk8O>CzfRO?t>H-2 zjAwS#uU14RZfmUGo6IX^cPq{5hVm#9Sn$)V{}fhaS?*>|g7+52zWr%;ry{^Qj`tjW zk79>gQ|5Eia4U9jv-~sJEt~!@Lnb>$1~XT^#a_2wGhr@Tf<8YjhOmy}{)mHAB^SDC zu=Cn7s4?ZB^0HjalVNslZ;PR|6%rMoT~CbT1kM{Qm}Jqx<7lrHBEt8TPpzemJ@Ez%Z= zAmpSj*Qe@OsR%MYd(j;dL8J6dR0B%wqr$h8F{g%XdKI$|iQc#OmwNB_@6WCa)#kFEA30chJUPdZzW;{V z?9lF?ro2<(;hlfCrBb1QDToJ1U^NYeJ1;PR#s3=KkzlDSqux%4xrKcS1jIyf*)FS# zdOPfQnVre9fU>p5x__iS`bnOPUW%}HtJe;%mWoQ$cI%4MmH2cF^8XS`z^_@%7evQ8 zzmnz*cXfR%Pk`G#$NE;5YnAYM0P_@i^?EIRpjLI$GXUM~!A45EwPK#(;nf~*tRnq| ziICcq@o23-_L^w)$!hVG*+i>jN58@heN@#{>hmd85nfMS$sn)yopTqnMWyeNnvT?G z2cJv0%O}o)P27HTf%;h9Kf;kr4Sn{Dypo;ylZJZpp`q6GojJu8r&R_I-5WZUMU9uv zm`ssE?d6)7qoKOyXMrcXl>OzI0oA3-^+NnK&A0u9CR&H%W+jYov?7fIRW?e>%4MrUe>-ie26*>Fz z2KoT3dzXE=9v8dIx=^+jdDwz<>S#{N`ZCY!fCmE1OfwI-PMNR$QK^p&RGwXhQ|u&z z96@YFCIBkU@B?D@0`Hfrm}mNk{4(Q<3Q?@B0et_jq%3N{sy z`31^q=_MceQ&+@6W3T*Va9Jr{1zRon>jyphZY|--4eG$tmpqv@+h6Wf$bLM2`}D&= zvQw#UN^W4&kpav``t1Wd!!#-(suzo&PI&3uYaJEsgF@+Kf3K)sMe~leYBy886A~=> zlg-cAO{EtUYKei=sZ`gG5E}7+XM!7njj0*#aRqykP)N|SqBgxqE56ddp&Sr2ui`{e z>AMTh&TN0tfs$aP(MO)uQwZqui!^MrQ z7<4|sCIR=Xqyec2@xE`z09adl8pGNjV;=uWF|ZNkSc_uqGdg_|G?nIP&&&CTFTeUl z{DfHBq{5t_sd^?{ySLSaKMgo?7N!29;FWp??9qO_Kh4&{@huDY6q+sfYR3P_OYdhJ zC6oA;QfK;Z9GqCXUrAj4c^O0sFioVux%IA?*sB+`Ghy6ge`1HYcl*DhTToV(c3xGP z1wrTJ|6CCV%FWWw_~$z;>a_z$qRvNl(SO1);3_d{f9P=K1m(XT{@1-Ccse){!dYq$ zXKWC_#nzGkt8)eP5T{YHa*|R^o|Lkhj<_Y+pQZp!r}kK)l1)HWwKOnXWdn6_UmeA* zulRfLO8Xiby;Cs|SQBOQ=!yq*#OB<8^{7Ky;gf(fOV0{^cZONQ9GT60QS6QX^x*xT zWNubs(TXHU@YY77P|bMWXa%j?Z#HKET4i1xuMcwvjaaOAn(h_=;awYfuif{I_GMie| z&SG}qUqOM>83K8@k?K5*OPNstV%)M0WY9rx#2FnAW>FK1&JOdO`rZe0B?XsGjX$j1 z?K8oset0u$nJ{RKsD0h^gG)D&>DuiMRmZ7!%Ric7Fqh{brQj+i;c^|E7lRr1oLuXm zyi}hQ`tO7TAGUJ-yI_{)L!aLR)7OBi*UDR8d!gKP1=ctQZX^J16fH1I@4i2=LfpOl z@=xsJHS_*xrb#IX)U5t`6XNN371FAo^^7hpElK#Cal6X677{_pr;cb1^xT3?+RVR= z2C5sEt74`+!4H6P_{&cXhkflyFl(VjqVv|M#+0g;$#Y~yP+#PW+VsbQSC8`cN|}ry zy_W^Fr%%DTZjndCTr-Oo-w3WUT$X>ZG0Tq@%w}H+3NeS`+Cbd|gnK&5%GfDBU@ye$ z$fVLi&$5cW6S@i@b-9xQMFdfZuT59PPRMt$iH*4i)lgcUCO{j|4Lb>okGGiWu6+$p z{wkQv5;2?OBsy!hp1H21ox-Ku8q{U7zUG6#t8+wp86k!7{ta{~e{m@W5hV2YxadAB z_afsa2kgam6VNj}%P+{EKr1RMj%T#^Kd9s3tFWmZ{PM`!2Ap&WgjISi;J7eqh*|vr zm4DRTAEs)2Q4{CVOBS7;N>@SYbcf@w7LUdarNCTgC;y9uJ*NPC_3ODi_vGAEVZ{^1 z()|OtY%Si1SaD#*53-|-#_Dux4HOLJD?>K0o>Lpe(G}Rr){FjUl`}UWuv^2874Git z=u%Ux#_{E=(CX@HkI{ZjkgIVe&-l!wIt=8@s^d=3OR49%Bs zoUr{o1Y!AI^ntbEOgC`W?`aeUZ52>S}?7pX)GD84ZvA08_5`D%>?Ux&a_XP z975yu8wmBR%EjW{-sD$Fg-?!{ZOR6k=C$5O8<37$@E>M|L8ALEH2Krrl7wAM)K4U? zO=(Q4DAa7Fae&7G9w%)|)3KzuJky5&_t`XXZpL9AL)&P)JAYHD2pwsao@Ypf3|>K$ zUbpWOzV?b$oPx;WV+OrD!$BDBka56PZW+NU^v?v6i9M;t7;|`<>>KNXVQjs3(WieD zV&ERH^+ZM-h!d4#^1gYVjsMdN?|C|=cYhcLHrePWYT|jvccH(OD}~JvGr#i=Y%vW~ z-I>^2+_Kdr)zMLD43JHe1!UzF^Vm#^VEg*k@=w!oY*LuUc@c&4)0yu#A|7MTLJm($ zP2p}@B2&PNT1CjeNl^lK=c>H!xUi|}gc}RjC-LoVA%?)+`ypsisN#>aUOJ>Ea^I_qh4;$o({QB!#~=T?cXdI* z%9m4&py6ZwS}I&G-o<^MwbWK->OrpWQ|DWW{thzIEvRX$#s}$OR~KwC{BcI@<3+l} z$M-?e2y7}Qgfzt7{43Y{`UsGqG)9%1G2dV^>!PuXJWsJ=jC>+&YL8yNWhR0DW+(e(k6Kx?0`~)2oZ2&j2x5sPpkJHm0UN zYFJ%0b3>m&!7`MqZ|Y?~QrgaX&dbqFvccTn7FqqRzC+tkDBF>H=}%-A^xL1ujGTCo zkoOISHx@U7FG6xeUK)_GN*Hq5a;+zZ+soZ3Iu+oSrZNyxjn&#k2cbn_dyR0qGpim` zGPRD2_4s|-{f!A^9jdjr+cP<&waZIp1+&<%dU&gE+1#+S)jsdCEU3ptiYTB!jUS08v$CpeRsx^Om{fJj8oUt5MWTm}xhzAQc3~waf(A#NvV|@%7>(#|mAbpfg#T>aY><0ubP|j5QJw6#HAu=mq7+^0aMm*tKxAF`gYOrt%ara8ptu8 zoHPxlTRQ#Ul%H~Cg1o7>v;1!V(E@<={CuojHgo{HB$vDuHi|p1rkB}LuW$xWV18J= zxF`ps)9p8uFM0Ay{YBxJ$kn=>wbYChuM<|AS%TF#FT^#&`ZR+ADs~d@5GZ87)Uzs; ze;Uj_JYUNm3&)R5NPv!vta?g=>Q7n3G(x>w<$-w4<0XQmYd6);qv(sYP-(+F^^wcjsq#oCsmH^j|ms61Zu@k)MRpXuQWkZg9tq zuQtEw*lS3Gmuag4x7PtbpJyl*|J_ZOXQAoeRcOWQ7Fu9omQxt)t0&}&6Q5c1s}Fn; z;P7Myx^(yR`&_CJOvcW_!RZ4r6>eXnz!**zb^H=CDt}aG_;6NN;^p+)Bon`Dq)vW+ z-w6-SBuQ5x1l5lw2*YNtrRz|7oPmSCAK5P}%>7|KgVS2bsKQRi2{R@|rK!KH7*8Aohj!W%{&kiKgMxe>`fy zFlwq?w!+JrBjXA)PbCJ%#=b7>{rbEE`2NT7? z2j9QMy*IxyML!`58x4mJI>xXCJFI)f?|T$XQd&Ps-jX3+~8(wc{JJwj-q$)r*=vkxyc}J61Zj z<{m%dPq;~6SV@YXc2`)dv%B3e-{e|S2i=~lSkfikoXXVD)FS=xE`Xc8hNu&D+^G!$! zMXTD=dYH8pez~F<<3J-A{xqJ;H_=Q0;$gPNxV+LvdE~3UU@|Jt2Ps9Jzt*}v0w^?$*(|g@F z{VL4sI+U!IigS@y?)WUWq@asnZ@8$D&TNC0$T|Evl|$niSV z;mR(@PjkLpcovUdvMEWa6p{^Itf`(XRRo1ETHLm?-`EA2XN+82gwYqj)E%0wnZVIh zCcdYZPmWSe+g5IPIn0GGFSFwO)i?|0PnGHAEt3jLkj)`Pe{zJ~J$*eVapChM*yx|K z5-AVJ4c5N&8sb!Xi)IgM_vCRJ!5*0 zq}N9v8IR~GYi)^~IUXh}5GbZrmN*pVQ?9euDyz{@*ocl(foV}(0cWT_$l6bwcT5Y| zZG&9>K{!I8(4Rhoo%1_z=ealF;KEP7PL*3YFKhI#;dDi3e%)JO%kk;>rvl02bYj@G zU0jzC!KAan4ds&Sb-cRFNi^R4k=%9(@FQM44MHZw zczOX_gTY6Yq)OaA%#4ZD%@3A6yGKW=pnqFvo$SqYQGRKGF(n1TK7Dog|J>{OV?9rr zEX(USdg6Fo&h4LL3&V;#AXZH+{gI!3$KxBDKlohwey*4PBQ!MuPM)U3g1-@RJt?1( zGtgojuFyg>aMQH1Y;YED%D3_*RlDK8B|E4$y=g4oJg^UA+RT6#lnx4rlD=L{3T3KOgXBC-1B82)CK ze=eGH@J54;(7+eDa%7ejjN=xmwY#UWU;9undTZ3y-kr&P;_+Y3;~Mq3V?u36Wb_4u z!NP8SL2-q&SmUlv0(^9YFyYnj1kh$$5=}GR+3O)YiW*1%Ui=QLeA>Hw;r?s)sZq6v!f{yw5yT z^xLiuTi9sI$vhOf{ie6*>;7Or%@q4}&ZD$9qwznne(i0$iC62&hM;~SO1u*DJ0d1N z{!@$pT$dtWD-n)fFxGFzXCd2Y_KRy2PkW8-|# z+DSM`DV<*OTo-EVV~F8qQs7N})9bzQ00J4-ayd0tbSJ@biTK^| zf5V%vax00)=hv$Zk7t?r&-|5FcZCE^DJ$2# zE>K>c5gs0^I(XW7e!LUeH9c%JEZPj6T$1rq-_%h{>h91(GZ5at6RCA}^`xmdh#WkyV;8M*FkoEiM>0xN zaps41!`qP6EFf*j$SG)G;}Tzk$)PTCKEbrzP^i3x$jJ^ZLwSSDk9Jn%cyQ6PgJk zAD7HSp~&#&_f+cQCNm#iQ)F_zVI7zJAOxFM56)vLfwB(ZxuL(CSW92qwh=vsz7~?k zwLd#SuyKZ5CvpF+i@NpO6`egNj`LpUR+6XP1<{I`kK&cBi~nU;A{$%q_HDpDq8D0P zS}$iCJzYd)-#++*Y*vnq(p6|&GC`d-GVfVWw!6G~ufa(_uiA-ec55;(9A-8^9mG^K~W19}qJq>IR`1sgR zNiHV(d#CzE{)ii`pUX&Pk_uj#jMMr!kX>dy_%bF+eKPZaoPq-5m~>5hut$C7h?Cgh`2X3J!^APc{U@h@`WA$Z^gLJ zN$Hd@^WMcfdCRNjM?sD-)`m^y=Y*s=H0P_u6TjhQExg0sqPY8&jUXV# zmN#v0l9HYAc6e8Fh0?n%2fjp8_}|vY(ZpV|ech@d#RG40a*7-7ed!vBKJ8oi8sN1V zW-I*wOIH*=UikXjzmquSL9GGL#o2t=-ib2WuQ(u{z84xF!vJfyEy zqUd}UKZnh`h*hDJp{rYThERT#q*m;Ac|AssZ)?3(Yg(`DIUCcq9>Q<#1dFOuuJ5~+ zY!dn1&?20em|zT67h4X_{@WRA#&D}Syvk5n}VEhMjaJ#PO@ zQ~9v;rFKoku>ZuHX@ko;e9kypd*1r{^Otjcn&dec3m(kbTRfz@(~oD;efmC~4UvY$ zN$m?ggeO4{^XlqIQZ&tcEWF0X-j&DjH8|WVk+l0-g+sr3J=8mUqv%+8_AS+u7$%4jRVajGu{@u&s#GOSe9Kf8erg!!$r^Lx1+mNh&kH z^HaXQ-f6%7YMFpEHx>Q@Uh~%sN!sU`To{IOX6oz()1tx9YAz1W3Rvf|g%NzsY|}UB zd;-6t$F``#=^otK^MU4o(%aFMXUhF2=83C2J<|73e`-K0U(+ZikA`3ZoA>v*L_F8Y zaX=aIXHcqQ5pDZ18(TC&S#%5n3V1yW#Ojx|IPjSG%*${$zP!zA?9bt~HA69hCPtZ2 z)k|jN&ytu&v98ZUm3VNU=e%y)iU~ERdkFjK(vRh0R6n^q=$bL>!W&F_5BrNqnQomN zm$slv!fLhvlDD{KHu6m>Df?MpPZfnDNgNFUlt}(JGs5tKEL<;WoQPZ7?h??_Fb z+}#(+?iSQI6nl#}R(j`*<6<~!lLLy2LP-aBVM0odC;8Z9hU)8Hqij=aU6OcX(B`XMeM zq^*3|hNvN0i?hRXqoRYxx%pc7^XrA9_(`L<$y#@h1WPo@pHiAv*NSEh()(|CR+Bt+%+hyETP-XIkH9s_+3Q`Qr7M!*>+R4&z^ zOBQU^iX%g686py{Y#~}4Pf>3@Hhq_4=8m|w^+V0Y?Yf{A846=BaMKcT<{hK)L`%)c zrtL=)D)ICxiEzq*`*VBzwDj46GRMPan1@)_Ahl$(GS3ihtK|i4pc7x3l`zp1^_Nb3 ze@Y|$cOeyc6^lf&3sr6O-AO^7k03tVtM8)8wmN!4e>NEp<#P?|P@tt%KYsd6?lFN| z(R(*%a=Wc9Z=SIco*W`aMlL!=$*+6+^p86@9; zLnM_>sK+@Je)d1*bVhCsb!A=ejLCiKeX}$Xu6_Ixwe*<}M_}9rJh^AXuO}-ETeFcm zdHqAVYtIjn_}@b~WvIZ8ZetzqSBMc41-T9ZB=hODBpBrl?3*)}FUYNji_a0)B@7k~ zsEqzE_TD-w$~S5k9z;Mw5J5^p5do#UK?FfbK?&(j>FyGc7LiU#>Fx&Umd+uhyK9Cy z_xOwVTkrdwwaz-? zw^C^&-7F6kZJ#Zu&9zo@)!n_*bnvz&LC(H~XR-ZiApbiR#ldIq^Bd>eg(zy;Nzz3z zEi0in7!+sBlHI*);xQjShVnL`c@Oe)j6IVoXmAk5Z6Hcs^vwI3pS2ORxL@}IAdz%u z2g>k9j;i&1DV9Kn77XooVYwaX9*!sK+#RU1w131k6n>69!>aLpzjeZY1%c{J;mZb) z-z~T`LwkFr%4N~U?R#HRT{XS@Ct6uFH*fW8H?qRlOv7DE!Fpm*>7O`WQyJOi-|d{+ zK+m1QdGIm06;%RYtj?sLjbSR^quZaP1?E?_qj*RhKuM!Taf^*1fL-v+qHv%!7{FcN z*IiQLP=G7=rRn=;)RqxYHM`?j-MB*!CR2=sTZ4xN2u|ErKJRwLfjniY^8qbm=a8+ z+VBr&=j!ZJsz03{HE2ze##*+1Rr*~{p@r07gbr+{&s=7_LqcS~aSI*P;07t7xmM6S zv^}?)=f?ZVjtL;Nvs)LlrBVMVrdq+r*DA@FeL!6`j_kMskSz+izj1zg$jzdf2NE>- zplqw&bFLOd-7|*?|M)qR5wd-$Jh+OVsotH2rGLk8O_eSOTryE7WHV zgwM-+`FajrVDHFcC}!QJG`4S7j~xd$M2)vo98q3H2>)QGUkqN>#5`pGWhHVkXOK>V z#df6;Rh>r3VD*Q=pw~0Y5L{&15@Vzo(JXiayt~R#+}llJnk8omlmZ?inwOJ85u8 zaQi+i^;jy!H9Cv5JamJ1l;CtFWoFi!$Eym@3hw?pKd05VHg-)Ct`YRQC?=h!cZmry zMrGNtGp?U+rG(y5VUh^E)Yf}`z3S1T;ST4FAy+nC?SJ`JpQ)*(?u&xntY@#x8o>1+ zy!IfoQmifKb)>bUt8FK>oGBT;kP>Clb5p=+141WZ-^l9Ca2omJWt8OwIRVwknV^z2d z?CCgWEq&5U24eU8ibjSDsl8_|{O(7thN|mDHzggSuObJ$2pIKkaD z<%hs*esd*FT4^Zm87svcgS1TSiZ8U;zkcwfQU4V+KEGMUs5JMN5BYYb2YK-ub`db) z3v_q-Yp2RZh>(5^zA>(h+>ce(+C|R#Na4QE;Rzcm*I18Zo~{Jf$55NTkFKD{v_7dh z+}p>KTqW=MN`rHPcU$ugn<8>k$w$~lLNPRP%dwLkO5F-&iK-j%?gc%1Cp{uC7}&S; zmUI>DX)D7I+lUN8i30wz&xH8V3Zxa)8~3CWpX5CV3?OIP5;hI*EuG1)j}+R6-LYCF zd)AQR=`7^Kpje;C_?m@1ZEdBe@OdzImMBV&3v4no*RW*YVs^9JFV(=5oyOLtyH^S9 z8q-g6@l#od2>F+yGbiNeuwsoOri<9053g?>V?+F{XRi-h?F@B==wgto!%BfRPx#J4 znP^cYZh#n@^;V-RW34H(EwKTPm7UsL&GMe%kDD6&aVnlRvMjvf0LI4B+xvoNkFN$d z_1*jAi&{eP8VHUq0rJr^J*x3Y%_zKi&0thK7^3c+sBH40I0KKW(Y@d5dimj+&cv%) z1s5v7aj=;Vv1OJoglZz3SdF?Oc)*4p@bT`9oMaC42IlaJw^8H@n?9iJpLLba{d|!; zr`P6l?KrMS4F7YbWbo;5i5k`EFI8KV1(!!?7c|c$r9aMauGUnaopXL*yI#J429rON z4ZdG=uNt19Zo#^?m41I=i~W6xLjdk5RUPwq#MpN!o;>JzPZ|Symx%ut3$RY#E%T6j#cL5$oZBDL5$n zzCnd2dnKfH9Nk>f89QK=`L(2Rqp1u)+>%h?Fhic9j_*8jVQ!)Q%irRZ(J_8G7f|#G za}d8-&WwXMlgOJj;jTg8$qdl^_UMPPl203=pJ#2@bua@qynw>bNkIHbgKP|KhKURA z&E+Uwy0zhw?$BS{VDM)t9cOdXYEdfxSABqP{o3)y_u4Liy<#|l-{HtjMvhxH;rXJT zq1eY{IeG{7*T3J@4@xGddkekLhK}?Xas$61-B#G54r^j?*eAYm7OhfY$tlr+`mLFv z^#^~1FrSQNHy1li5Zo*Lo1n!M+>$^6po-3q7Q;|f7ecFc+`vf=?}UhlrQ1DzyruO- zq#Y9d8j&mK4m~PFVnvYw4s%G%OK*jJF%^$0+pQjl8LbQRKVvt3IHDxN2e*{W7aH?j z4zZ<$H$c9qrTOJf^BcJp&-<-KQP4N>9x02mN7K-TL?^TK1B`8#aV*@t*!%MRZsHso zVc_4sZ~EsS?+C52RO4rUSq8FUg~V)@bFoV10#;2OX=oMGUHio|-@Nn3;i@NNeEn=# z{?~(!09qv!$F>YlYChe*=m7p zVX1hhHg_}~#3U1*3w-esm1yY#m^~48;8?MTj{V$mBK@TJfv2`{M1{^0SNUx!$#0A= z7urs2u4iJJ--SJ>{ub(%jZKem>s)}=T`3Md3S|v} zgNCDUF-!tkD^4VhjlqQc%>f60nIYk)M-|?6d+0yh`=v0*wYe)i!O|c$Q}Q$}=yf8{ zOilviJ1f_RNG;w<5L6K*EhC#-*!5MU6Dy-R@**-H9rt~1h0^lND8dY}Zt`}+z!&%& zM?Kqg-`Ssk(J~;pF{8JNkoG3VE~FEP^b$2{=^|A7AangiV!6MSs9!DKkEXO_(qW{G zJV}VG26XOrF7s_mtm|L9^Q>0Q-b@K)ntrMU_EGtFaTs6h?~F4YyMCf@Z2u*uKOwJI;?}dDxZU1A z*9oniA$h-$aNqH9uF0>==VHwCWAi4j)JXm;$dIDs)pMl^_Z>aus5chhu(#Xi^u(mJ zev9UDYjRn#ER7UUoPEk~P%>N9twt$v}rL$Rru`@yuj;2EN-OS5fjTQnTGe`qTe@plxZLAmV zw9kH4=-yLc7T!?v=R8(NmHyVQo;nmbsK&u=j8Vjx$Xv1ceClODHBn~32=`L|lmA2@koe zdyPy@VA@vFd`GR!gWru~(a{z3mqQ0}j%0`LZsTe**3Ke=Ey`~GMpKaW;4pq$+ zne+XSZ%cQYgxmntJJ0GV3Ws((rpuK3HPu;{j)CWX))4kK(@CIT^{fA&tKHbuj{MKD zy_fQTd`+7y?&oNgG9-GY+}Q7j=4u{ys0zo`T}r*VCP%JgpS{*E+O znn;cmPh+>YUvX;a_$uZgpB>wl3b{4WBqOj<;RLQ%HEQ|%rAHHJ zUvi0Xtq6j|e_YnssGS^+W#-;i33TL9{lYCq%-*2sc`Vp)19$5BxX*9y11_(X;NKjV z+9EXUel;BsH@kxmwJdz|u6^QZ1BPAdYzV->^rvlNYpUr%X$ddk|WOapwmCY6Yk>_Y|vI%+QMz_MyFubolA>g0JC+jI!^ zEtZA9uQum0`>{^*y?(?uMZ$a-PZWg22+sX{b6aLCC6|s$AFQlDuWf(;HV-!naeWl& zrR)!@XPx-l%^vxb8;VuK*k)A?xauccY$b?2PGS)$the}Xv4E!U2J68o^ZY@F4OoPr3(UZmh~l$=t|dv)W%xR zf0jkYh%R%Idbg&u_X!IS-><#Rq?G*n{-qO-l1Zku`ei*Srd=XM0OM)RHNY`J3RZh<7d*41os`Vhd68J7vWCPJ7cQ@#yfTR9k zy|+hP>8zN{VDNC?e}4PmXJ!J&jGWh$_fwuhOp=A{f${Y38qR&6azXIIcMm-TS{3sy zx#rAA$$2+YNI^heDg{?)V9CD~_&A>iI(dbHmQ1OR#A%?zmrWNEJHtOzqUTJ4l%_ z7hDZEHC5Y{q_|Jje8;08nB}vEg}5hg_*Q2x3yt?`V)I9`bYju{iabHqsV8rvie7$I z{e-eV^>u8OXGrW)V0?u#qrUZ4v;vC1@|sh9m;K+Gm``hIaF^YZqj5%<1X$#FMbX~qM!#e zOYesdxi)0ygbl+P9!TIxAxwG5ELof86|dtfR{N^Gnt%m+On~ zx)t&2yTDZGbdR(U8dWOMyI}vaxk%0F{3v3VM*8q(d-+E|b;5fPvTC#1-?uDzNZ-YELdD6yyh~7HjKGp!6if^9Mv;lz7~s}O18TBg-a%8f8y#_AF3Zr zU^`A1{5VzS>a4CV)+`aA3>00qw`E_NYtlW3 z7Dhn*F@OH;Qqam6zHKwbErzZyHK+R0z5)zh^H3nbsuyhtE&qTc?)P3}pTjwBlwRL| z#ZGhd`$yINdwF}-48bbvON-n~A=jVMhkJo{r)xpwxyh!4PpqhVGk$r-ob6G0;*gdW|<~~L^aY+4rxOZybkqmU`Vb8;y zNAGB^EqI=fuGqgiw7HB=#@jZy8r$O2=@>{SurRjsX}X?gd_%i73X!U;953M z`Fz#RRI}+8E5X$;5T6<_k+(RW4=jUzPnaEWu%3f(s|6*_r_v3zl|x6y>m9-V#!MaO zM0!5V2X1D%rUc5SGouFnFB$w4S+K7i*zrOBfSk0IYDfg%fF=2bG#ZU9!df~uw-eBl zhsD}ncksngQuF)XwoY)BQufQ0KOD5<506_A43loIBAIaPio;{k^IzIMS?5GDsH+C_ zQXxYfKSzMpxXmh$%Hpo}PU-<0h>ka;pg1S#yX7d}I@axPXn&N-zwh=pI5T+(-ro^g zQkusau`WQp+r^(Y*4O#TIzz=hRR$F<%t^8T0`&hFMX=n{j|**~Hti;WkA%ZffkPiS zuEKi{cu!S(2RGUuX2_=ydZb^Y;3^_c0n^P@ht3#Aj6Dh8)ZdLi!{Ti#xQzjcEhH|o zWv5p_Uncm>Aj>9bR(R!`ruc6;{?q-`K|=Ae?Cm_$7G9` z*P{ZyEiBQ?o|L96lDB`B$9rC$$~%ke1cR+pkV0l4s9BXlmgV0zq32urLsKY=dVY-I zNX{=2ebZ1TsXX9z^5mC7u;2KKR$?TwIa#2bHQ)Q-oDb3~MkqYHr_3W7=#1XI5+KqR zF}7_}&$u z&?%xmRpZI4!!1FBK{-*^@2&f#gXKg!o?DO58&Fp$#d4!Mcg672dnOzvd^P1MKbef} zm*NS}9cafcvZy--GzdI?s*Ul^+}~H}?opBaLL_VjCVrA6~Wn(uYEj=?2Ib5K^jc{YGRi+%p)BSvZpthL-g3QPOjD-Ki zVr|BXFb|h+$K4>>;D5~W=iYm>y|d!)ooXI*5U(z8sOLQ4m)rBTuSnV>s<4tZC0Nr? z^G4e83-=j&rMW9K)EDnwfmt;8vLj_CE=4jb?}hxkkCvv)UODKi`(rK68gxS6X!LT_--cAtNi$u$Xr${WI%dJHJ%WTfFFz;$Iw~mXEp!>vW^wVf{h!O9ubhPPD$yki~u<+Je~{oB;BHI00z@6!b&z6F}tSk2eZ7 zynti7?-^Q>+AUwW-7KFoH``$@Tzg4&dUyi1lzbLn3;Pq`qpxo?gpJ%>BT-Zq>dEU< zuV^;xxJ;iv^PqxEPQ^1s}Z;hQx!FNp*afMur3PM*`w{)e2jneQ=s{h zWYW{^fuo4Wy{e)h&HU%9u!7sfZMwGKzZcvEj4_@?22t<3t<{an^Nz#b&-WuL+k?ug z^Gi3j;y3oGNEQ}GbwrKfQ(F${s`}0XXobxH-Zvadd=_un({w8V2l`Da)jrO^9{to~ z)ACqVO5(+LA6Gyo)A2Wpo8PEb2)bPiC2&5_AX-TAEUFrv_r_Co8e-qU93E_W`{Sj# zk~0&*%gq9g_I1=E7o7)=up;~Zl>|-)@dt%bhSw`uHCH!EyeAS*4W0)<^(S!Y>ITE9 z5^$S4;j%>0lPdtxN&n+P*x6b<6a004we#xb+=!1AZ~8K;qb$zkH|Iz zN0|&#l}rB$Krl+(U)oStiaOPod1f&_e{dH^=hTZ?+{w>tF-}-JB)D;@Qs!PdNIwb| zGPa&hmL7KP^}??`d5J35BvLi`k$UD*{28;VHclz$w3XWu`?GKSq;ddEZAFl=EmDvi zYXA~L&Tcty?C^bM@#BTq7vkc&dd>nJ{mdz0y}eP^{1|S06hePeu15ZAs+A)8AL@L) z7CbE;LVss#mQeMYlt6r$21M;HEUGzqM>M8Me}9PFv@D^1R!a&{JiLlW$uX@kn~rup z)c?p!1!JyARwdoq6u0R>Wc#i51iE=*`~G47^7}C=)A&$n8fxkUPSBYoe=@Z*F`L`> zyyf!S6Rnzp>i6mhu!$L4fNDX>Qo!AdX?ZWAE(T;iWx9CFr9P}e$pPfO&RT&2>$);S z(;ti_jOR~wdQ5h$(_zVG-vANV;J-O+Xa=LS;+qZ%I6C9iq`=2l0GVjX=pd^V1l&Qn z&RdX=(?|Cl_ojR#DP2f#a|qN^r-K(3O5xqMzm93I6?3As>SOguB*_sM!3HpoCX~V&!rn{h(BEBgmhQfs0aBQI#9bPt!YZOUv*&g1?k2^s78 zw8&m_UV=p;(=oU1qfk>VhMYV6{3MkX@S4W=HO(zycrp0L9WiM8{!prCpnP-dB9-Kg z;4l;;Ok&?Dd?Z3X2Gp2F=keMr2f4_a0!}NS+!vLnOvXr?QMuky8(Pikadpc&s^|H| z3Tu3}F_huYIO&$hM}A|(udqsUF=Vv4c@4-bPKVHbhF4(7|ITa+qKo{Cbxq)x2(12? z2xjQtuW#Oea}$4v%BPS1-tPari~0HQeGK`Y!mYpW_2;c`-aikUAnD?f$;tlt#kJ_j zTuQM-a0*zQ8h@{DTaNKV-R(Ej_-CF6C|a(EhN44g-lvnrU12;6cOvghHJamrBc|M@ zB17*9dM=PKHh%&A+crpipYX6xz`f43morqqIyqER1}5V%JUAS zON2Z!6aj?N!S^;{d1$df7Hf@Dfnu`&@69c=>ubC2+T!gI-!QhW6xxUDUk`L9nR$mb&&`w%LZbQE-o*8dK*+LADNos%@vt_f3N4F_C?W5J2Wdildh+X#hve?XNQ9@aJ`e?%={zK+o|?krJ9iLL zD`Be=ASZWAc%LNJNwpq%*i{P}B220c8UnRXRVlN$g%1ho@C7G5^@mcN;8j$-Z-Y}| z7dZ!)?r-V8c@JUawEMKRy%fBuhlXFzfkaP*4=(pX!aQXJhGayHzGctco9p=Oudh3s z>vOP&gwDx9lI)x%4?q9R_xJRCH@cqVuaoKhX!ZNE)l7hVTjmfcL9)c=3g{ev}ECq2ugXoV#T>p zy4qaTk~X>2J;Dxy5+J~JzAuWmudc3^O7XZmsXz0ceS6Cr7%DROWF^9`=VHa3y@@w9 zw0M)X*kF|@5|YRx@~L?4@WIC#T~u2;yQSkx9rpNk;c&br5{>y2Is;hce`F_0dU<)Z ztXRY7B)TtB>*XJjBZ5e+U?CwP3WC5od0y{+=~}vwm-#V?7#)I9LOp)hUv%>9`Dzt| z`s)vGQ3kHCXn;7xk7hSN7w*M*hZLc6Z4G+@9-il84W&c|zPu3JYpTM9 zAlQg#={mwQVucFrWv4dH{Urm#rv=IkU^bjJH4~$uU6Ld2pt)&A5Pb~gJNrieU%Rb& zlfBr$S(-Dvzh7p~VKVt%NC!9-LJ#a;x$)3Tn)|IfoL~f zc6>LH1nf7hXG<57a@`k*v+9VIL94kSx8_SG!L#kIw{F=)BA18qd^ZY@3}6qi(5h}c z892%ALUfke{7uZv)-ELJr?y>BrZ67rhYLN$8hpGGaSUdq2AGwpp)6olTGjP4OdrRX zkY{COsg~#nr+BWtxo~d*A_-vu0qBH8M9WQ9^MSH>mRKPBqvw-~mM3Whd`$&x%oMXu z>9w_{_lrHZz#|eD*OCCG0&z=A<_T>PXSX*lA4sb#;X+*^FJE?wObJ7HZ}69|<=rt! zy>do_j1ebvFrW!0Zd-N-oaJ!_njOQlvwepVKBOE*I1zB?VjJ+dx0(XG{#o*ejUZ~6 zDx>kvLhIQrabcO!Cer88WYm#&z@LkgX5>F`Jn))DysjvO;pl!|92jC`hR)f|KBth|NWNf36*|Xj2(ohdlHCzO z3}o*;t!ir{z1UJ|_PyRm;9A!2N;{uTuOEmYjyY(ah9TqwM(ji%i&IA~t^oaOK>R7#X<-`qsbVjgo?cI~BCGBlbP^6>xBH?ojc9wo)GvY&Iz7kGJhQ zwpv!BLU5^seREPh(R+G&fLZZf^9A)Uz%8xX{qQW&A+U=sk)QSZO&I0Vz5|KPs5ztl z5hBbT$3uvCJ89(IP{6^r=>R5TvAl10W~%qBvCo>{GG9yqg|AlV%^6wKd86dW^mGsm z2KzGf>{)yW8aB~l)oj!4MyaSWU5quy=Fa* zul$boWpAv_dh&UTS>(dWQC>d-UHyk>!N+)OV^QWBk&+=HDzgT(R`i482+=7WV~LFwp;6$@IpX< zD(69$h;T^rjl;tQXZ?pahY?=sCDs9HwP{1^1onkV>c}*5%=QEBLEzLrN?)HGA8$9| z)LUq90ftK)=$gkcYX+-m!x+*6(N#wEe888wvGsC*NzFxAey*-5H=IZ(-D(WsssE(Hj1_wfyCa^R9TAVaE?X%AY zcO6$SL@w|tofp-IH!C)FPZ#5jtJ0gAC?O^0la#PMr{-3moVIv+7AUxI1E&KJWOG>WbXr z@JCetW?GO}e7=A0-*bFmvfloOc}KoQkHYr%i8mKvJXpWy_^8`wS!td=ZQpgYO_xQ^zxJ!^#*JNvmy9uYJaN~fk-xGp zdT}geLtbp$dnkIl1tYK>y#+(3#LO2t;Dc5@JzwRG1KVyVsQfuOk}LqA;6rZwuU{nt z!v1m~Wf?tBQ~%@L+1to3|MU9qoIDlgvBCf30wACC|LnA-2|q{s(j2Shj&g~Hwz0L< z##HT22`sbzxy$CO-|OocIXM`qm;J^o)_=Mj+3n(l<^>J1Z-G#hYMK3J+%Bl&BkJkt zNnZ4)HGlr1C1{_${rmV50-4nC7?0~Jsxz}zPYa$C|N8A`EYY8(3;t4|nc{wo;?k&< zR8-XYG*;tQtp;xO>|Gt!dz#gj6l7#%Y0CLsg8N(fMV%H1mxo8TcHqpQ4^U|4f*q$K zZX?fh?<_Q8D=8^Cd+lq@!KQ~^=8ppjOL7s!C1UYvdvc{b^%93r>S=FZ#1b&o59;dc z10y`z@A53hs9TE4$#J{7$~YTe3pg(gv7|LOdtFQc@XPCdaj&THOid6mms)Ce`kYa_ z3I_o{XusHK2kK1K^NtU>EgLYrTuxb>ui}kqbvgfFS7fuZX=5a59|ei~0xcJ@TyQ8f zfvbHsmRTJ=W8^ca#s7ji>QZPuJJGW3ya}PXQK7aoUr%fskCVtP^qHZkX3na4y4-jr zX2=6KI+{+mRJBB}0}BT5*3ZJ~g<$=z2;@-Yt7w0^ZaytsT6aGT(f%ohXAz*B-@1S& z+|u$wr0FUer0n1cE;{X`r@us=ujQQ`_NKJMbKGCJ3sG&nN?-C&E>uHvuCY;NA_4}Q<@08x= zj{|6S6W$?BgbE?VLTfsT7FZ-d74(1UW1I>nux!|rXt=4K1YiXna^Ng@F*DiZ)7gm* z@m9?J=?A^URo1Q`YteuEJR0d3B_t%gRZ`*#%0Ob*KQGSHk|S;@!1MsATdW|1kCZ8? zsev$9LrM6$W4(fF8ev}ylV59Epaydnm~6fPY1unEiZW92$-jAnIcCNsmn;~RnMq?{ zWJLY!8D^NQEIlLR$?W8kefs*S;~fi&UF%v6YhH3*2QyZg%Guo#HOArXQ@{?j`TAo& zj_o@=vkE)Bzg=1_17EC@K!@3)S|dKyh)e^qSF~ldNDVUiUNTh$`7^bmEPe`k)cKM zN;W+S($*by4y=g-piBIi2!7i(d)$>kh3EiTb?4p6=5RjcErjv@;7Uf`o^ZI=UO_x# z)ZP0rw{K7556=)0qS&a!RLSRCq~xD(XR4+akjo7*lE0d#$|gh8dKq|^}| zt1bG$h0o`OHz;4BS3SG3b>;2}Q7+Wzf!-*KxTt^lU^Avq8Zfe9p7IOa1MbLdkiFf990cjxoQQ2B0MA7+HQFs zctPpt=yI5C1f2F@D>JFOBtTSP!Nck?ltd^P6x68SZw!R)({XTc6zyF(Y~Q(ki_mfF z=ZwQP?^^-sDyvjsXAiBsd{kcY@XiQ_Z@d!)AH*KrTWrH-ek33ea&&~1Bu57-nLLA} zx?lAu%;IDDtDf9YON5f6v}_|h8!-!s|JgwCLA}0ETw3c!Etbp6OS+^Qp}B@yJD(|; zx!PBD4+~aKU0n{dV!L~L^(G3|Yk>@w;7OOr(M2lNdX@HqV_TPS&3fs>K7J z{IlQv!Ximh|9<`Qwfa9k z`{s3z`aieAkEF@`UHES&zrL?{r2Ow?G?>p^kfi!wudkoJH9YwDx;HPS7J~o1j^^=y z``1;8V+?P_(Popja>_z*+Fx70Kt9&j*I+nQ3tDW#u_H)SkPp-yU-;c7>uYsaYs+m^ zjUszSBJ%>7KMjuLP~#-~o}EoL%E^`(*J5F!@eFwj&dka>bR1D{&^#dmiuJ|kSN+DROS|dS zBwVfrx6;k)K()+c@Vr@SwQb%|WEz1xpdJ9i)3Q0R9*Bh~U+f%YfyJ=qgF z2QrkyUN1x>LP>0-^oSy3a=<)&eM`f|(0#ew3Mx!UNJvAUu*ReJb6mr=#~K1a12!@9 za-W5E=eQ|2rx%V4+x<#XJ62`xSY5`$o09}<$@wam6n1$?*F9j-j*55V6B-(-Zwc^M zer@GR&E8gi@y|_}rBknF+$4AHq~}0DcYWSz0c|l*q#@;Sr3+9rF4cGt5!QwKtOd?VW7^4isp~DGgFypY%JV& zwAvTpS^)G;xo6J6GacjvB7QOyH?%Fb5|32LGE2vLBCAi$XykkV6G-k|p$DUi>f0Aa zMnvTZ_C(<3Z$9{pmsy-Oi4*a|A_tlH^#q%5L)aBwZUdC;hH zbf^R<7`h%lcrYNlQM31}j~S2*!jH&td2(5rP6w=h<=PorQo3>6tN*o=jJAIOpyO!a z>@@5$NaQ$CU|;X@Xf>>Ad(p*pJ=uwbRR_YIn2dJ3&S5@*EuJx=-vqrn-;WsG)Bztu zE}-3gckqPj-R)kkx4qNTYcp#U!CKo#0!{oow^bWV*s2$K-u; zumQ`JJ`IUoww@wo|4j|iU?wKMTl?7ws;R0%_w=d%^ctD|Bk ztBT}?QW}xO)(2pHnVVtkouRcs@k7F0UWjAu0B!ap5U~9M#EgIe2LM4SlOB8iSYXsnFk!U>EL=&#jDD1yRjTfEX_qzycYc z)R2{sK;ND$ULO?jYOkx@mUG@2k859%{#`S;WNi(AK)EISiuSw%FsxEJa{dMe2E!SW zPN32*DeuNMbHiMtbI;%3-+p5_r6VL_fA%U_!0|qo{}0Tu61(*xT8kBHURE5``T;2r?E~#? zlu6*_o6z<#%U%$U-jtVH4`FH2S(vK@t>d@fupZOr%GxbQ9ASYP%tnJEYmkbBJ_-N= zXl&H3RhbE~*i4scVwmCHpm>6h$_;;Fz325qEOX<#sT))IY=K>|-8PyaY{Q)xhKkXf zFh>l;v0GSiT+cTLCYx>?xGiR6L`5G1e=|BiVShwaSaUXf*AWd5{Uz$Y@iV3Z3QMjk1>=iLEurTC!1fF#X27IOwVM=@i zOjH)HW`ibY?Dc+gY&aT4?Q$=JJ~TU!+QDIFtrBIi>3YAA0jSFUUSBP>Y45Co`Qx6DPew0Ntq^$7i!~nlXVZhAH~YbKJ-CD=%@;nbmaB;d6Iic& zdTYjsQ`Ypl)%i)JD%H%6Bs2%WjMwH=eJJmhmD%AvoBi%aw19oTgl%+*{03;vs9#q9 z84HWZ_fi#o+z6J)75@5pO1H);FFbxV?Qb!AhM*wy$>wO_@t{BiK+Lr}N7AIc2iwSB zRYN_uKg~ESwxEECA2DO(jm_NPeRO0yF*!(7bo3tOOLR1;3JV8EOvR}y-arX1C@d87 z{a&Q9{GZ*1t$w~USY9N{eu&c3btUX%Z$2cwI%F9=?)q#GW3d$j(S2c4&FVSFG=scdE%tJY#>ESfJ_ z4q$eA0qez@WdTt5+Hq%Iy>~$0YOPD;3RBzjL)*7!QEzN&dG{OaNO|^*Zcew%HLjDl zFIK3i%zg>WXYL~5R@bkYF56g55I+>UAp!0K@L^(@7>p*4hoVwa2Kj)9t%iPaop^bc zX|BI`_%;Z`LpDh_9Spm+us;tpQMfOm+3zoe8TJK#02UJ@TLd&TR25vDl&@IE>XKBL zPlf}ncKuHG&8*n5fjYFQ5?$2qZ$JB>VswsQZ>bFBWM>Bqs8>xy1L;We$zR6QUmK99 zmajtw2eLMp+?9rnUg~k}2((ils;o!J3+p0o;K=AnmVVbQu#I;9d{aF>z;BWl5##X_ z+_Yvj=FlC!t-V6W1B+wU9Ms!ua0#<42OC1$66@|PPlEx7wUB~<=$bn8iWUk2-C$Yh zczRqHm#Q*??1y`h;z{|O!Q@1HK9ZK{`N1_jJeZG_$F_QAcJ_TN^ldF^>4>`}vz1m| zfJfqLIt&|ufRbJc5f0QANNM8$V$-*F;M*2f;-QHtwzUU()$(ZPnH7&a9SOY**2qM|`GXLAnd~4ff_-z%IB( z^kCd=VE^3ix;SS!x4_`&+nZ1dff%4q4u;}vg-qf;m1bNJgzG66Iqt(1 zD^h4CZ!GhN-tr;gKAz@NSxPr|GUH?iqH`z$O7eg?eYHAvn!G zv$jfcs!I3E5ef3M%?kBvLACU3nVSdW0J(~pi&e3`rHcH+ZwfB!H|?0_CcoV92nfrcGixERPKjK zB#tqA3%N;=@>r4+<56`cdG#-WB_%YX{Khv9 z56OU&#Yr5tN;Z-2#6szQfYLd~8Sv{@W6oOKU+A?4aCINboZm!MyTOJcjzIxRg%d|O z3r4shjA>=_*D52ee1h%VZ18JaUd|#;&ST(i;+l?T-_4MS&^uVf{=n)bn%<7|ZX#FG zO6NujwX0Kr-hlP?Crk&P2hz&bH?Ro;skyUlXle)xWoMMUjFrz1aTTXCA6|GWHP>$+=zJph}i6|74_=ymaUNcfukkIOcaA*oMN zqH`nAAJ37qP<40QrhZ-~$^IcQ>dj3@D$2!Lc6o_^qFR^-8C3Ak*cjU09eS(qbK;}e zSYuRH#~l)6*zx5OE2KQd%cs_Eozkvhy4cIB$!)%J&Sl+dt9Pctv>lseFulT_CEN5q z4tm_-iHtA;c3fjY!sO)Rm%yKK!5n?UL*eu)gW1{4E z?C6l1LsAe)ow(n2jln~IYS+Ex193Z#`MM>A23WR0V4G^LWJ#?kEyp$@1uqs+RSPF- zj)0FXn9^HQ#!%Fd1cWV6SZ`tu1x2>`j0BH4{V=qr$YqhdOJuvrngupLfZ$f}>d&!S zo{=VFzXy1rnVFfN78>2&?VeALAG>%M)N!m-4fKOU1M48aP^UjGal>hluRoC|H8MlO z7TS28T*i&$f4g(5O;@t(xdHLIbs~Z0P1P~P2PN@s_yTAM&dfGm5 zak+#YRbet}UWsC#djDZ&-t^28jWv0=G1tQsJ&#r%-U)0MJXl0!)T*dyt)pFIJ2%z? zGsP2T%nQu5^KQvm{sPEy(RM~8B2 zhmnJE&;MdYPdo@>o!l;IN;D z&yw=l(g>#(gOvh|8Iirtuirs34HneGsZ_BH4g<3hi`rG;utC^6dxG`mv9|v&11`^#&*^DkV}&g+=H|GElYeuu%W7$!C@futllG zY%ET_GYL!M2|FC)#CW6Rdw(iVvle+yJ4UoYnx~9lvfs~9zSqbfiU_)ecz0aj@;J?( zKGcebV*y7C1Hku*LQUyM_aKXK|NYToZ4qIZ^)y>oA|yB0l<>j8D#oPt(0p@a{ld2S zM8Q)ouCRgv#@^mu63?Eeu%(-8lmpjBnYaUNbR3eFVO=p&Hrw0ha~|@3rKvsZg?v8{ ze_FbA{!;-;7jPwHqb3I3X2>Oa4?L`)6rqyY-KmefIPEMdEz?}-5qP%bA5 z;NDmVbA8kUATH&dF=4|?uHiAD1Ov&CZI;z5=hwMsDl7K91QW`Cwqm{jT~?P~g9R_I z$05X2bzl2dZ4XaEb9%bFA!*vpo;y>e$3OIY+AZcvB6G4u{M&Vn$9~|SYVA*#(~bOi z~ipa>}**o$4HO7M@ss^rfYb|mqQGY2WlPfuSVksMVB|pX;KD7^>Pse zEYm}x{|^K}Tc2Ol6b!!p9&R83MK9^oY3O+l$Ek0LJ|g2LAVeldz25%*H#J`IqoHWp zA>v)RV)yR1H-}JVI#z1rjyxgSGs_XzVbq$U*KSm;S4!$_GnKE9j{o`7|KRSeqoVA> zeqj_9X%#_1LM5b8x>0GQq#Nn(l2$>HmZ2HCyL$)$X@(xUq`Mg!&K{rVdC&W;^L+1l zzjgjNYkjkD4G44J``-K7SNyKuzOFQpxA&i?NTiifGIF9rTfV{dT9=DIl&FV{Jrbd$ zdE@=V(ZA z2y*=&V1C8KNP8S5j5+#=MIvs}AKcx%>O+NQ&X~3+R8}1nBq>`%v*IEr!GGA1=U?wjInlTlJ0gTX`+Mxh(Y>cDil(d1(Xem+Jmpt~CXg99sc$};c*G?v*MnqgR({)MB)H;s)tlT3TVslr zV+ovZIi3wVXxp7@dvf!+R|^smL^*DpI&@7NurR;i4IU;iFB{?y%|=`n`x$BLGn0C&GK6-8n{)y z_;BC@Oi|2Hk-L7+#p_AhfNcCOU68J7zp|cO@C!C(m!h(KP>9*y@Cnx!Q^oJ+_r|j} zb{SGx1{D}Cbm+^pS>DzPWznGj2QdTH4H3%w&%y($Mou&U$$Fbj(|mT^XjxuwzYZh}5qV?KUhAYQmw+q;CCUnlV9D-b`4W)5 zjhCQ(>tvMPG+JkLLy9XBAs`30*LNX5)f)(6I+j1zo8B(}e1CH1HzQHUPLxmg78MiU zZB=8e&$%QDY>IfT4?@0Ra$G&-Xt>w3elO8e&@$x2pmsB-oFe`k!a5i(QmMWdb>7+Y zE--Ipe5E~1Kdp@-V5^TJy_Vh=Ztb7W*@1R!_ z-+pOuo+FTH7WX*H4F(okjqwBcO3}G6X0N=iHW16HViAuc1Z{aZqkJZZiyj4LR2B@Z zrnK8_1zFiV{-pQegT6j#tj*Jr5^nv?uX0EJl?p!lVVHzg_r~EOA+CUoiN*5+;xfm2o-n! zEJE1Ow^s8H6|EF(4?;eu1x*~N!wjWpA4Jol+pfS&Z7c}heTtT%a2|+Sz5@!1lHl)H;HfzrIT_lxTdrx-+>`-nkzIuHl@;l&IMfJ1zMO;1;EW(jYE%$ z;CjK-eIXLoh!f#-J>hVV5B~_?d>Hxl{rOep2Q}RM5lS!Q^9t`O>GcqC_A9B!Xqo}IE|W^E6PW`I zs!ukV`G~DTBoua#VX@If=i@;!F~$)A+B#FGYj#Kc89Me@(_>_u`5|gID?G9=j_-T- zJ%2_JIQ?`L|Nf!huJWz*%d5gdEJ)%)^K$ubi4uK>iQUbkut_?$!kGpuLc^-Qrw>Ag zuVexhDrQx8>WOmk!W5wCT={qIeBjw%DZEqr*h+t;j?-+j={VmNkZBln2USd3ypgpZx%9U`7)Z!B+e(o(WE{-$Slr|Ej;!2-4V(M zZEw6ch4Wj9-ZhMEs(3^qswdmLe-=*Y2E(k(r+c2`_K-i)WP+CXJ{590Kz;ii$En^6@8TaxC8}g zd@waadgx~{sZ`+!gN1|>pGDu-8<~==pebRFyb^2j73`Rb<_Pb%zkY@)t=JX}O;(@P;&J?+C}j zGAdF1{Pyh_5y}q&&-aW=h^t9aqLtbcG$N;>qIz4h)7JLoxcTEd2*S^9TZd5?wlxVc z1H!uBZ`Hkt7pL-TPPwaMv59q)&HMr8sN5|K^k<}WNMSl}^e+t`>eG#XKFM{7d{fM; zv}3}~K}{$=$&VBG8Dai`rX-e)l#}q?WH3M%Cv12MB?g7^V z?;GZ*@5}rnZ(SaR%`x-FTg5z#W1F0KedFRS?{Qk5l7>7u^!xrz<_pKOi^#pkM#_}C zh{V^H&PMFWU|cC*_*r1j5$}QA%Ii2N@mJgO-+6i>8W0l@(mhH?`Bn-^ ze={HKrIfc0hL$ey{`fG95v;Eyh3365IaSd&@B>K2{$@n*WbmKYRUZt6XoEcdBD+0u z6#JxWHF1wi5ns47`mHO6>(hAy`)xra0uYduM9V{vs3yk7`nM|* zt2zVb6`S`AKU&)?YvPKDKzoo$qA`FDL-S;2F1=VMGblz;#|8?bZkVYTx^+|D>N4-3 z>yVb0x38OWHB%znO&T;XUj_pPei3A{+)ipXmp;n<1NInAm`RBGRAkDY@nxsJ>oNBZ{4lfKGJ{*F{+0;6( zHRB4^gw@q~c)3pA0x7rTPODtTqcDUHO3ZlJst*RUCv=6@R#>p@oZEwLiBP?G&L)&l zK(WK0(XR9t0Vh4u7B+oxhMbH=uChN5lkjnp?xBfxVO;&0-boPqaZ5NTo;=hK zYE6RudbYtc#;v`mX73yHyI2~|6V8a%Kq7Sc(t2OO{c8)RXm6+R>;Z7DCHr=vHf)&T zT^XO*EH%`6x|XbL+N89(Im93DcTh%yTu}I$%TSR1o z(Q+M+Qf5`%6}9h6$vc{461euQICrG;SCEWNU)Rzaz(sK#txG zjh;au-Gnr*J}xn=w{RMkz5nqEqmY8RIfg*1>nczb}5Qnauoc^+q@ww=bBzj8CBsv2v!Zme5z+7v3*Wqfpn;y- zB&SmQmNT2(pcR0FuFkr2)$85hIo1%=_KVklN5rDnIZut-prXVc4Tu#o%tEG!uE>0i zng^eN7uu#jB@imR}@MyVAR|HYGVE?v_ z0VJyU`FVXQDXH~Ji+owIozzZd&OHDtVT#|H5ujsBpe6Ld96COy+YlBiJ`W35K6HLx zwlh@LC$8c?tp@^DSsUcTCq#K~f{T7XfgnEmprxlzVt#ZQDm9FZO%iLh-k}pJ9I3N< z0~qW$#Js-kI|UU()?;Q|3ADvGpAhSX{UL# z*Jc1p&8ZoCkt7=AM{1O01>}dsX3;+9xxZb0B=jXT;Q7;I1#bvysawhTRz+HMy*{x(D5^_E7OWYPyalwu~N;b zxN*Z@6WwhJ{Zd^#CMvo+IYa=urkyIx zy#N5Hna+40Q}%$?)E#WOC@;JR?=F)*zHrAI@u+NweIB-~uz;&yp;-*e_sw;pjs*gh zsL#e3npnp)t1Jp5uR+a)j&n3&V8LDniv|Qz#niE|S8O1T2_X5K@>+F4fJIg6b@>RT zp7ehFrJm+13-S+1D|!K61{_6t_f4BW2=Gc`uh7rffC5_XgX*$`0DpzkWvEwTz5O2m z7T~aqN8SK31DQC`IPusfPa2gtd(PCH5Z$)Tf~2UBT}q@s!x7qu_(j>y!p1_@*O2yN zPBS3$O+_-e90X`r9Z;M|&sMgv!K3E#9jE2VmznzWA7Npiht?MB4tkBIC&r|I{C5!3 zFoNIX8%whrib;h8lu3$n}MxzotP$s9=a5w@?M}YB6WwJc&uX1sUB`%j~ ze*l~!x|Hi0cRi(t79+>YTu;LMUdqI1vWJ2cK|et4x6ih;gJ76Iw`%#sC-0oPp8K!j!YHN4wb0nLAU+UKQHRF`ea42;4;7dSZpuZFghRrCVfksNTUHQ>3*dC*t- ze`Hayy{xZ&Tz4l1&>Vr15;0f7x}p2sl-sSRHWC%BO-aqrq+~LB4k)lc0{arY9%Pyq z;kzUn1HMnkT4zG^#3C`25LUPNuvU{BY!?~zv(yaQzqziM3q|?QsHY1#u7C?zzGw}d zSxnb$O{BNyrFVq}#nm2)zWaxI3z1ho&s}4?$ghr-OjMv)g5^GV6af@Fo9#;0Z>#y> zLi*#6o!i;ak2KI>#uqniiY37#-};*>OO+z%>M0=L_Yk5Pqpli-5KKrlYaKp$iy0&h zZoFu?oKyII|tSz4iug5#ij;lBvdeiy6Ak#*L?J~*kVbtzY^N zsx%~`%S9Oi0`mbqFrxS$JUNvRQxK`kuPMzW20r2Z`!Eb`P;Xb zfY8sbDWis=!31tt2al+`eewY*#Y}dNFrG{z)fCgAPxSPMsT2PrLz21r(SNpIK{Ovr zWG*_K%fexH1dIM%-m24{I=oVjr?#ig@nY#84)rhr%q$iqoe~=cawULxclWFXUOO8p z3SHwUXIrZ&G@dSE?@5Suo8n5au&&hust&xUGUR1^0SYYkwP9LDyo4@@y&&s8O=r$GTHawn-nb$LL0mMyfq1~znGP}ymJjCJ&*>N^lE>ejF!4} zmb8@0mW;f}tN(~-$au?F0vL+YQ3j&|pa|!*&FX!U60p`oB zgMTKx=+gbH$O131UJz-+V6roU)#a?zYE|3#6;hSx=@oV9z_j9B_9)D@yl@IO+c&p? zQLMM-%*i+{n_atfEt@3UfJ_iXfiSE1BavI8$bMGuEmXU1!kvnacV?50Hb#>mznEU1 z7LLhuS9;c-N05U0$&0ot!JBVn#ktO_D5Ab|d^jjgczErJJJ15j%2HUeX2CY`QYhl* z=$0HUa$eMQ!8HJ_vMtps(V=XDm#P)BNxQy=rIL1hbeoe=*Mf&{QiXHG|Ms@2RMpix zfBA=Zba$_mcC!S!GyO?3pS+`Tcid^3sMy#&H?Sx#P}}3FcRMy`cjYrblwcD zZ>r^LY)nHgXQVYwsH)T3sC&+Z1{gAZs*P@ffHMUS=lN&6gI=S{oigKleseKHUV<&h zE-L#O3bG{Wjt1(f!e$S=MRD^US?V2qVRGCYifZ=SvdtV{SWh?Rk?fW=YLBar$e-vA z;mCOW7&CyI$MK2M&yl6hQ_+&Yqw)eZS!W_eFOIL9`O<>gd&J&e7k!4twIBeit@+ls zK{+(N8{GH}hqxJ>`fddTRNEQ_z-RyjadcPl0hJ!ptOyo;SC}vb=VSgE}R~nv*>liAah$D zJV<@pTc7!4ttQ8=DB7C;XY-0me89`_x5wC6>g*dsB7Or;|5myhC2Ml<4+ohtz@5pQ zvE!o@5Barn0IFo_k1%Yg^?wG8AlAJ~{}T@{5G?|V71uO7(I-G@czG0}>#%^!bw5~) znA4Ir@DbCUWJ7mM;132p5ef+vI*Z}UoEbL_(uO5Y^^7NTNjDZ9XVx-0Dx8}IV`1~r z@cObvoe92@&775C3(!*oAO%VKv4>~nLk3R21Uy%=XGKOK0}YskyK#Q)G_}z$7N88_*+wL zg$3rVn`kI!0K_iseiY1<0+jWII&NaTFz21zD$oD3=q(;ZZvr{a#x{2IYz*V2IbKhC zZ-F3A%JHrV<3DLc3YJE9JR?3rG~@C(8W;nMr@v zW@yrmE5xnZlB5@<|FLNrbkW_dLu`pOl$7Y707tyUpk4Ogys0xb5l-u|4#lCQ=HJC^ z=EL9Jia?A35)bKhb@70Qy62Xco&CMwGvB}Dj0adZmXuMJlP6CU$ppA4F`IbS^zSxw z;r2AyV}zRY&#S`un!?H%1L4WV-h$M55H>sPxW8=Zp!w;gSEBbe&(%pkMTrJ(rR8`R zC_V;aU)-XeM0iUmzpq*IRI2OdR888z<;7ZxUo3Krz0!uIY0LRw>Y(q2Ji%Q+n!x`M zM-#2<#e==fl09kM38BV$+@G{Iltk4U*<8kEq91n@ZET9S1gm?9l$$fk0P&Gn7hOlP zrQeGIJ*uANQ0IP!RdtWu=tzIOj8@0SXS0+g*S|9^ZddudUd zNB2R3*};~89;p;I{fhEc`=k3|YziuVIP}NyvE4MJX;&%$3bJGWWzsF-Pi*J5SlBq4W39GmsF;)O(OAw4e=Zm<$p z?~k|QJFW=watlMruAWmKyXyhBM28*!aUTXJ!2|P>hn;^K>bY?Qqq(?p;Jfl+cN8oaacy3&iMAxIq|F0w^s-K7`GrVKWmP$BqeMhfoz5A zOFE)u@%gnh0aVa`B`HZ+b>v^eeb|2y?(-N}oql{F;O8ji_!tC~^i|=|x-y6PtBmDz zQg>knHeOv50wI2%lLpcZmk; ze5uUsDoVu9P63>ReFM1tS?;0a{O6kvhwF%qY$rHP{Zozd7%jP9AJvWZ^3 z*ZYwd!{3E<%aouZwz@KX1g2RZF$G7ZAJWGi1&hdh zykCCmyDIx!khE!D+^Z^foV**)Pb7H0_Uf4HgNRElKD}Y9cf(Ij>tJ^#q1iV8bT}k& zCK@MW9V;urVS5mqx4=-8Sxev1P_VQ)sUn+wc36kS4HLdIDQH&DRV@{9ePu_7UbOKJ zW^MJ8J7Q%5xyBrIX|P2X`eJ9(lHS=J^LBIA#TXpLoMX3NylZc7|Iz*|$&OnA9L@O+ z!uHO(8f;LiuP8724}7uL=*z$$AYQ;`JNLBG>Kvl?2!A0XS|{`R(-jUGX9Qpujs_ZZ zM-?#j*RqA4z4|M6@)Y%kLRNS&24k2gb{0$RbQSlb)CQmBr9qXd*a|OROH#XYl1j)c z4J!23%v(-41~ftd^3|7Bw1J&B@|?QmjrEFUPz#OISD*W5IwO(tPTxm&Jd;e*_b_#n z8cIIn)lcp=;_EiK_jxpLv#ejoyYMuk>o!*6KZ&wjX~bu;vK2xW=Gb|glucwUNQYhv zJirz2VCqTl+b!$Za%4s%l1QmU&)c`N%oZ=Dk!_CmyNy2U+y3#!D93mv@%vIyL5q3d zwfbhRDc4CxH1Q4 zfgcRY{y`!{^7iiV$SX*`Y2&Zx!Uz5_C*2nj)^u5zk6Njf{AR2= zm>k<9)i+fpT>7Z9`h!i1uCK>oJGrN23EyPUVsYBKs6Q!=c`D^6A8|`!mE+dkA2TM9*jb9#A0%#K(m4pv6s_z% zb6DxAUr7xAWh3aYqLm;3p`9dr^b284#F+HK!Cz2E`+@fLuo$^c^wlr+8i|d@0LUeM z3W3y0$P3=Jo)nQ*ojn1YvT^p!qZy6UJ)(>@6Yd^-;>q5P!k9tOeU zw(HtTJ0u~`2WsH7EZ_7*gm8H=(cx^3!oq1IX>y}de3Fkp+MZrm-Hb7^32M@v_yI0^Bqb#^>P*9ClJ*KWo$2^3 zGv?=Ivq1%kIAj-lu&bMNm=#qg*!7pk#5{qYizc;nU3UIGzI%@jGZ}Xx3VO zRc$$~Tew)bHa;7td0u`R=KT#(mNlFSygz914>E-O>>cb09W4ZuZ5}888dGYekOfVK zOt2Zx!|y@ohbt~7@Ga@D*9E(z9B`ychZra7*NY+Z3Ru3C(NDP7R9Z!o=nfpJcOv;7 zUQX(DI@4uWjBmQFZZ&A|+Zn}@W3Tu$K2vlvR~@^NKtFH3uIx~?7Vu7XtM z1r3c-Ev@N5L}hX9zL-kb(JE7m1|9b=-OBE_zaoa#_AyyEM0$u5f_U-{ee?6SDZ#H0 z>lhuRtJIB8+e2@hCd&GYi>HpJV55Wrg@a&05`| zUAcj^D>a1yS4YJULb8bXb;xv`NXN7oW>{bg$`Re|GZJ6c&vLF$wlz)%h>WOo_##$c zCg<>b3X~h13f9!>p3#SU_8whINk8aHI`C^EqKyAjf2GxO>p? z`|bT~r-_*%wUPKKEbPs$w|<}DRFdZ-6nQ@ulru;i=F&AQE)s$X_i0R_f+_p^2TF%U zqM=r1Z4LA5QUt0qH-P|F+g)4@3C3QM3Lo-AWz;)cuZ3UjiR3F{i~Xo;e!(7{yyw**>Bo7+OG=c71K~TcJ8lb_qsw;2_+_wtWfbQ zWV~Y_f6unYu~GV9A7P#SgVCbPpP9L%m6@DOGp=y1Kx;=>Gk(sv)e*u>0-O354cP1^ z6?A(d0j%NOg0~#(9Nq`FHQ86s^fI?nNc~_sUVrrR&exMQwfkIt!fPUh+{sW1(~Kis z5u!ZLNq3U&7ag0$Ow@uh?kc)`YDJ@E55CP?wqyvu+*0=xsZZKY9n&_6@Yp}I=@_z;V9Q8= z5BpE6>KdG^hA5ir=K04R=NEZyoSb;s;l}i9pL3F`%ARHHu)(xN?<}c0nD6i;Ola;( z_5I{+SZB)&#Ixc_(PqGp&TSlRA)U+733X8@TpH}lhr(r;Ihc6w-M3ghg4ACKAc`HB zYHGD=<9NVmm#dO=@qUl8p1IcF-<-A8!sQ)NXbGtxt2D_p+=$XEpbB?H+Cq9sWru6V z*lTKaGz#0OV5Nghtvaw8l`?blK_-!-NyPBpw8`R#1KD)cQn*@$d|!WI$2>6~(lutvXeKR+Z8A65ff`@H5t zJ1;a45Gbg*x>0|@zer%IqA(R~H^kGd3Fa7$_H%Zz&$_*Cx4tfql9u!Ml6QDf3#Y>m z*tTn`W-JS$*mY0#$Sb$3!P`h?AXs|EMr@J5FbWpEkQDO9>2h&xt1z3e_57pza32QZ zU@t@}Gq=IE--jxJ`gZ&A~PFKWUy z1wR|RVz_p`=yF*uWc_?_-ETyB9~z?_kg zH=#O7v^*9v#9Ns*2k&;iwpTOu##gqumu-G@Q>jdn!}U}*juHl^DND*i*`XupZwj(m z8=JqWYTURB#jQJgl{a7{bX%60AzR$qi&>PFW})j9c{$FP&&49A1Zf&eYqtajbW&dr^iU3E>v&QhX@WsyT-Cz2^GxQ$IR)Afw@ z2kprpoE(n!a;H~U`4YaMgftqO{7t(76GXZ@us!#(9ohpL?6axXD;p%Ww=L8^SV_BA zmTwl6>Kbq{AW{?bFc2n|kJL*^iA*@2rc&40}dcs5s&Y$nklS$;(X~4fI*T4C}>7ia}gQq3aVf|W0hlmgif0HPB4;&OODGZ!;pc2LwwC&z0^=+X@34=QOvo{nsVwA>9$v8-U!oF zN;!kI^HSbxvJmdnj%zacOC$%v&embcP)&*HSBY*c-8gHkbg7|wM0jjo&@bs&Ei&}_ zP4+-@-#Z;uaYYxo(8zEzL~M)ye!=ob^jBp?uzhEL|XJf%Z;ygTCubZ zT-q3%P3r2<>y?URWMVH_m|o~dm;c@fI3LdofXHPsoI1w23C_GB?u>pr8jq95i({H= zHQT!SQRzbYI4r??QOn@gzd-t?B);YVFlkGMJs?UY?P(GxU}-ScsedJ@9sFO2%3WA^zqgM^yO`-m)7dhqO}>0^UE^BQ0iS-= zUpBt)$haOfNDx9tN%yXDD%Cp%Oudt$X)%dYPMV6#w_FVYZ@-m3a>y1ztP?^+bEi#q~xh7h7sR<{n8<(iHiLncjRhhkah1U+GXW>RN$FSyd@h!cvk*H znWC`xvFLgGs*93$8_wOlzs)*Ewf}T-j;F zN`_F3_x2X&A`*fl!eVlSH5VM+yx)ec5FM2n;_VL0;G!$T)HzP?Kv~>J(#yodE8MqI z^n=60$R^wr3U;s%z_EQ;O;YIECtslb(=8QE{=pxO{;BEqty`wezL*mYmoB$4RJ&)% zACl(Zr~GEI8DkOwh318tcxQkAemBF!@L{b0Cvt)-MoLBoksX#sB?)qJqcJ50hlh%C zD?+n{+I(Bnu&OSMsz+p@0jgbVjy2UfwY5XWsyQQg1?%(ry=#k!x5kU=+@S=9R7Mt~ zUBalD1@`LMHA%<@eQkBE`2ts1mjoj}T7!a$l(E`4Z6r@d%O+~xbF$!T0vOHcH z!B6s*r$=oLE0FO`cbm2%^<5I}oFCCgRwOi_S-5=EXByNO<4{cj*p-IN*`F_5;7PBgByb# z*j@1?RoEIg0$j|Q-QdRz5`(anAO2X7p=*%y>Z2L(yT!e>y+ zw>lsv3DKys&Hx2!lHXH`cN>@Gs1_w^G+NhF(IG$P*VWf^7;iqdnyi&!k@d{!5F``y zNXO~TU)R5g_bYM1Ynyg`?7s40Dj|zCNWez&2AF_IB`K*~k4ujOiu1EmdTQ#7jxE)F zB=WtQTJ_c0Zp+tgAzogwnHfPW9GnQMX}}3rDY}w@L?k8_f=b6alaH8?LCcz6}$VpR3+w;luGBbT?3fGM4r;~7s~HWT>p-a^U#@$u*iwqP_BIGVb# zRW$>dwpIn?tcvw$<0l$I%#4d z_G3guM1{r0yEm-@C%gs<6Zk4DR%9g=^z~E8T~F>y4kmy0!M5@~J2bPJtQ5E|0g~7g zTPEd!2d+gc_?Le?`Vi?5Qw9V7BfnI+!umQ}%!N+aJ&}KHD_+!vS$8i_mR* zJ3GWgg~cI!G>}RX{SIC~ps#}9J{TBMNTKMvk_RYQ zKyWbJ+Y0Qh6f2Z;bL5TEv$874g}O=&47`eCbqbjA+OPtgA)Y-;l+1!x1a*`QtvIIt zO1bUtw@b^*^n84pCcVG9CpQa>WuSC4#-X5yr%{Poxq*0uM!Dth-HFr*wE`7aH`lJA zAw{A&#rL6bE!3q%pVcvm(}YWg)5r}1xhMxI(TlC~ z;~hl+1A|JtFXZIpoHobaxa;>~z?zzxMlBb9|4spPw>4Bf9vBP;)*>=8mRb(KFF;80 zuWO<$eF0JD`_M_ZY|W}dEKJOSPoI6LB;SXH#c}NrFzJ*_?DTz9b%s48uT-$Jd!Yn# zwvur-ifiK+5Cd4<*UB)ez}D2`S!l}Clope2rBD{*+* z_K!#Rp12Zu3vl{KKPmNEoXld%RC02bti1H8K~8d`3Np{z!rJzR7unbiJu zC)6)rDJUv(m@m>wD)`}Tp%l6RIJwdgVT@5_NY&1qFY9f{NfusU8Xg@ZR zZ^qX8YxBLaaV8^Cp!4Q;g}^ALfT{MmXFi9>Nt;dt0-@*r`t>ITMY0{ZEy{9scSSeE+ToGzZ za1yT(W9{P89v*>$LoiSfBTJItMYkm~sMMp=(_+|m1rw5Z?Mo{ITJjt_pY!VLpa!XDq@D+28 zHhO2jbN!oTzMY92O|MWHnPK+Thi2x&_Rr)BbSh&7$M~X+YhMB2}^Z zDagnuZEPrR-?>Bo>eYJ}!?wYf|7!9nfifX>zhE;!B_c2!VbCw|z|=85^5(JITAy zWyVvyu#g6L7$TrLv7)SO2q+boer!_50Mo|8*UYJ|CJ^0Tsdt$j?NMiQy7-t6oT#JI zFMZUKJmceT=LJOfT4=oXwADPLJ|T8<@yL;hyH@#Fif?rUYI2S!zowBk_Oel+w? zsF(ct_}Vt+g=S+=lSb} z^Zl=g68!Z4zi&I><7BS~?%WrWd|m0>I>@X_)T1% zfC;~&tmH}j+POJSo25lh&LvjKM*B+qkX1U9b4$4@6!g>wv1droX zlS~ALSt!di4K?|bw(mw5`GEI=g;q20|ByC7DYs>eyfVczI#{R+&rMA z61KCJN8x?)9>R`{EmBk2oJZ>j#f{N|XsTB7(8#K)sjr9U(PsI<2enY-SrYj=FO% zKuvy2c~CBm-K5vjHMaf`lou_X#Aa>*AO5sq0+DtmEFD1v|7>qpr3qiVAVm9rCeR$m z3DKi<1wQ90WUEaJZARD)_avY4rb08dVl2 zwZu;Yi}H!ykHZF~Z^7&5r@Q~VZ>#l)bJKa9%r~~(x8I|L%WU3I64Gfn7%ms00?N@X z>a&_{)jFLmZ-}@cWN=B?`vF6h>Jj(7f*Ex^0PcJ=dQrqHqJ`KG0A%3)awlFq7Ja zdG~I=FD5~`=lKp6USU+@Wt3)Zt{9xe?JVb&y^|AS%GCk5xOd3-+&1HiaIx@4zTa=* zaPDCPGl*y}6H*cp@zpmlU^twj4;<`yGg>%33`*V!upwnvzaJ*m za$Gt-UJ4-DI1IvBM*Aj4w$4AC)cXd$#y=d_`zCmepUd8=vlKJi9{BI!p<;lfLIt-F|bhzr=+CB@of0yHB|lwwjvF*%+}cb3?e2?r7<35&WuwGecObNd7%!bR0aO%>|ngMqhPWznD9@a4-FLNc-f&|fu>hgP`t zl&88>1{A@J@nLl=;Q6>1<_G*)UB~UsnhUE7W_rF+lIywT*D_a)omf~n{SG z#ezUY*VA-VGt&1nz9wlEDB5*Uc3bJghfkp7uhOd5<7)`R`uVxWwGgv;ek{5Y&2UPu z@Z}omaD1=cPv^ z1fsxvSrxm^iI*5gY1|GkK`|keokGTkvg^&? z;xM0*V3BhNe}vvj>>b{jF7WCVK%JDGohwHQ)${J)uCrYkrDyN#bQ5xn^xU9lSeIW( zxpKqyNyh*Rr+&6#MtU32Zqr!Zx^M!af94-U=Q1glMSWpy|23#?DZZ5CE`TEM! ze7rOgbSYb0Tgw5zGL`F@!CF5J24ezEA=%CLsS31VVxaz1pEr|(JZAtKhG>NzUYD5N z@|+L8vH2lE+ELG#$;Az(I8}w!q{<7~Utr1q z40$CvyTlkzUj1K`g1lzwGLtQ+)#nQe(-rVI2t`dZGwC*HwTDuunwn-`Avb5)^)7s$ z>q3Fe0D<({CGk#|b#!x!3O)0-$Zh0R8xVz6J57A2jS=N%&z_OI+`SN&NPSMpVKeih z=M6D!FX=;s*aHXoJ15$UTCK#_z%DVJTsYJpZ1dSWajbzT9snh=Af$hMZ9iPB*K)yA zJ#ozI@Tu$j?Bzc~(6aMnCAp0~u}euajw)Q6YgL0-P_O|+Bj9WBtG5hIxODA_8lJB1 z$8>CbM^J8EcXB^}{{Y*OyKSit9k?{>|Jf*@_yU7VP8t42C;qKod+BDT}gmL)J(?)pcAV3 zbS(lX^hn2s2G51y(F(^ke2vn@S-0ILp-6WM@Y$e%UF03m$*=&NZ0EMLn5m0hM_}jl zd&Nj+VhNCgKj@M*H5C@PIx!&O+TL+nO=&c5`ttP*z~(;!-?d(K?k+=%7y#Jx4bSs6 z=mr?EU00_1S>q9y?p#|3XN+9lq^D&bxv@pl{mj&@8C`J+nsYgzQ46RXn=Jz!0sgiI zg#F|0Q}ARuh+O4vn^Ax{2S0wL#mvk+QEeA4c>plxU!L~icNv$hg$g}C0b8W82LEG}qP)850W&i_^}G zOqsD~dW!H)q~n!T000ZtbF$k0po<>6sOftui3JS)L0_n%xjLRsaodgF*~lFs+OqNdR|Z#^<>l;0o34L=j@{kJ z^?@`$&<&EK381*IZ_`?76BAY_hK0R9n-A;*Kv3bVmu4FH$0RvXYLr*&xRwuoC;+W9 zmBHbja6dmkP}4FQta=VOme|$VIkFHc^o)y(i%I9AqPay?3Nkf4ebmKJgrb}sjb`O} zUweV*Lp|j(SCvIXJcQI~43w&Q_0?Bt^FxP;@ zAR&&U+VvG?L&g9-U8<@%U1rV&a2tT!3qZk75uowYYn?VEAhHR(8K5Bi|6uPufST%} zc450ODxx5QNKt9hq=SMKMMR{Dl+ckTO^CE$zz{14NSEGw4>j~6(wlSwgdze0p$R0k zknr!I?>FDQ-+yQB%r|pq?##_NV@}RtpMCb(d#z_ZYwfkbHH#xfGk$(jrfqC;UPf=< zz70(SoBiz?y#pRAD~C)tVnFSHG(*Qq#n4L(?oUA`+0eA^SoXmhxajWQ-a;AH!v;() zPPR*+S6}q>2gBEDB^9(X>-fQo>W;gk0JaPja@bQ@8@&0CjyfU(-qSz=s9+Sj=RDK^MI{42szCa6pWsE~uPd5ZDyl+VF<6g=nL z+^FmCANLSjqsKt2DtOYi3Ld;Z9$>lJ+GaC}SOx=-2gVGR4In;HbCrM$Qw7av)q8C( z?9R7#f_FNgF;DMd1A$<2C%LDAwP!a{__nbOq=meB>(=VnJXnq|kKd?qD-LIK+}06ey`+Y>FfW zP)I}|TO3A<(Yf)AO?`18NepD zva(VREpwd60LA6?W?D$bV8->ODbs}q2Pc6^0uTcqA0LH7Pfn($r=<;L>mLRJrLL|H zB1% zvVsqXG9VQ?>TkeiTGVg@^S4M~<3T^z=R@HcKuYmWJ9LK~8i&T1=m7fm5?zTB7(vKN z>lYqDyKf)rmo$(}*K>z**6I)Lbw;|BhyXPt9u4$zl`tk`V|_im-BRiRb?`svWe(#l zO;o4cu+_iD8iY6k(HR0$N@lZ5FW~BSO-VJ7d{4~!`$Ok`8q~O1pUvfiK46h@?QIJQ z(qsVVzI;uu>Te1!ziEghZz?cJ;Z17yH^i*nDTE;SWDQnqMDS58wSE*W-;H`3Had-f;>t$utNSJ=$?aDm{&3Y0}fjhP0SJI9rRU!XNC9 zzEXY)NWQa(NkBY)7~;kTxzikZ6WNt4%OqYOMpF@+0^s^ z06lFW{lgFbolrL~e#45Gi$o8myKpN$RBX5Fc6)^a33)=(W?C&eS9Y#kx#AZT1Q~d) zMp!OE4gf4s`n;Pd-vIqMbR$yt_tw^MX{2c=7;{i0+^V^?HPSHgZ$FLXfCHD^9goYd zUd824H+r{Eke+f)7Mdwgvu(Zcrty?Ndh{r!_FxQcryNR4`<;I8wTVdrmwH@QTh=4$ zs}ibg2z&_0i=@o16yU1YGu+y$bcj|!%)YTZ^6yQ)2!ig@UO{UuE-vPn)+anX$9Buz z-Cg{}$P=1zIqlWe)iFRRfTZC|lgMqyi5@m?k(@ROpj|LiiEIT#@5xML4cYtWjK)%t z65CKO(Oat2u;N)V*u4QZ(#+WPCXwB?{|=S>F7T1EQ#iuX9n;4e8jxC#>w8P4tQ|Ct z?hhXF-rbK}n%qvh_kJ+jg**PP8>JK%Z)tbFjN?#{961RjUFQ?#QRYMH1K*0^7mgvC z?8vFr<dG8gS%9qUJ%4zrllyP0rz`=z z*}l5$_2n!Zn<-cho07{_zYE$eLzNlWI;WS;2Bmd6{&w5#t~;YOxCw{^AV7IaT>f&D zPr{=rHp*%)j8g<8_}rvE^w3eh3SIlZ_^YI~ds2T9)~b9S(8MaCs$5}V(iELwO|kVW zD<1Qj8NX6IR3JI|g;k+I&!`rwaJ9eK3itP4+I2Wo>N+vq5x{4_Ke_@N^}q51u)tuD zlI-~MP?n<~Xu&@Ju*gWslM-AQ={75>L+I zu>Jb=C~l&9TZd?>JFgRGK`|uScrDa;lqsAkc`SEt8%a)IL++>_yqaEvUcr@w(CJBuEx)WAE1JqB$1&nCeKUAyTeyO zzM6<%-J}O>h@B$v9aTO_&IF_x3ZB<#|F!rbg6eA)7~ikTz*$|or3>fOGuzKKL7F4EyTtwXu7t+oE4IqV&qhAaae7>X>q$jz4&U^VPw7G)1(82 z@YxzE_tiWQL}kBNoG1n;IN()NDGH=#V#=ZTiRH-n_rH)5rlh@XK+HK#JeKpx?`;g) z*8vXf%(Xhhl=Ybdlax&`!GvNuw6%uMQfbP|CxK$P8KAzHJDG1H55Gc-3uK z8VtC9Q@FT*M z$U!P9DiDuw2@IPtK@ZqR9-E2Kj)6u6C~4CHM2YG2$VjS#;0aGJN)kQ0K1u&l9A;{8 zd$kTXUg_fdogS>Wtj{J|tTGl0Fobi%ORxk2rN(>=F|X^bDMx{TjFYgAjcara zk|_nnEGe!Y;CNBdK7I;P4XfRqV=%Rus3L@Nu(3gLDo4U;fq$Rz!ONH8&CT&Q3rWAH z8~2{Is7@SwB~`gE)cOU;pakpp4Y<{fKxBi%Rprp?YR{s;@Q4^L+}Yy+ z`^b$U)Owe&R&Klm&Nm%?fRO!qBt{a7ignCEwY9Y!>j8v{3J7gmTFgus2HpsiMXR|u ze1ijSTygOu2PO0=7ipNUsqtBRBDe~3cM_B%u|gXRq3t0)Dut%gF-){b&FoyA8xQ z%ZE_LN>uZ-1-YPzIC!Ey`wm$I(Pr4EM7!j%^l;9TgeN zr|j&;W0H3dL6ubev%~Pn%TSP^YXQ_Q*R9iA-d(!=UoH)MB!zX1IUVZ<=}6Mu2!s00Hxft5XI@J+Xu9f=RlhO}DXMdVk;fl%}M{_=WI6*Pa5!E|}XvYBq zKzIOd;xuL%z;<~n?P$9Wgy;6+sP#cc`+=ms9BC&NL=t#Ux+jAFK>o(G-iPy;HUV{N zMi4Os;?zLeRt<@2V|T(TV#q0_?!*&N&RO8d?RG$mWq2ik{T%;}aN%TDPI{7bA*nT@cmh^E5#j8hQhuST*Vx0c{^Xf-fzsZH;`EQJw+wfj z&}tR5-$K5yQ_Tjt+y?((uRF`cME6M3AM}bWRRY=D`!?5u^7|e!A^26Gweuu6Qg(ek zXej`Q9Z53b>X{SAWVSqK2$J~Nsr`p+2TiSnfy9Xdri6R!pWoMa9r6EHvwXUiSz3ReB} z<8z%+le7Wl$wSl3($i~Ybv-?+>)cw)2mAVxp=7M28+n%j6gwuPO^nFqMy(W0GIA~> zs<^7kpj?2IF@`-}wfn1JaX7uaV}29oIYp|ov)kglbX-D0Vi7_km)9Pax$r9%s=Jb` zP|M1_&iSUHBlvS6Fj{U$f#}k9p>tQfue0phGJ2n5sm=Q|6=c*Oau&Bo7+FKnJ@xX^ zY_f2zr??BEbJNo`Sw-t)L;8MFr~r8!D}+h=aKi)TNa_Zh5_C@a*nXM?)bZgb?RqFB zw2JZx_I~sq7c>1Ixw!2y;b$(z5kpDl4G6~t{t4bB4_=uH@(?hD2q*`l%o(v)dxkkPB; zbB(>Y^2vUv*XE;d+X~FUdVaYPTqEQAtH(46*x0y?s|Sh^R*Ma$ce6lAn=EI~IUnMs z$S(`HT)BLZ8B4j$Vd?E@Bn1NGqVolkS45wyV*DTw3hP%z_Fg(9j=fq}s+N@8+nwSr z1yT$Fb?v;UBH;PyL6|oOp_UdxRRsIoOe;-LOSnuqVs=K|y7zh3JkIlgF?&SE0o6#^ zYjpDfQO{P`vKLrghn0K4dU~)d3h%cBqD=p-ynJXWY{f$@UP7l^O@PXQ=HvYh>)p9+ zBK^5j=vhBbgAxz%qoP1h^IX&I8j*p=+8FIlRmz8lM>28~(@B_ik=+ z?{EE`CXUcZ=4q9gS%)VhE`5wmfblf;lVKL6c zW8Dg{6ox>1btrpzgU5X2ttb5KF0pCM#MHDvuKqVfW{WtrNQ2xV`GnUrC4(y#-*;d+ zZr;4vTR-7dy?W42a?-i>fnNQUrx4@dahkg~m8(|D(hcffM6XuNz|bd;c`QWIWS5kb zw61{kv$U@+-<|};=d_KD!*GpldY_SY@ohUkji__)U3|FTVUTvhshF|6HdZ;i@4wf@}hw}%FG zi;TFs<#^Z?NN9~r`)k?6)tXH?eAA4E4G1QbK|$^8bWOY_eIhqEHcO!gvS_h#u2K>J z^b9uJtGoiT$RbwHSE!=Q)FJDKVkfrz{) z;RMQb7|7HZ037?!;2?)GrDiV(xnTyxeBy1fZeg#(zRe^sefG2yTFn+_Y>C&>6gwD#azHhX9e!TrX>({jlqHYV&@dR15I?B)|757ww-h% zc`UnQsCeY+Z~hdnG7ai_s;Mb_vtvm^=RZaq48PQ(?3RE)z(&+$#a6&&$g%ZD+XwHp zE?WFMx(yYX52b>LMHT%?4u7rpBWdhoT%ezpg+-eW7x1|myv(u^bh*hHin*+Yg4_#f zzuPE}5lq=5)AnqphdEoIKxgrpRih8~J8A#L9KCh!;xqd*L5C09PKgsd_gtS9Endpk zJ3^k8;I`?+_Zd@U?<4UPXUf@MMXGnot`>DuRTkHL@C7I2b1sGhKe?q)r*#L_mg;1T ziKwCy#qj=d^H4lr>B4(>Xitunu?Hz0a_f)8|Ng!a<9uJeNrHQ}BW0c<(}SHK%z^*F zAH+DXw9V$*4@gi9=0~SFDAc@2)S~#@hZ|U~ia@@wlbztin=kCVV2ZRo%fJL%DA~r! z15iL|<3iV(kzl3_s}A2`i3XVEZZ_ox z^5$nAySS3kne9S>^v6nBR(o?=hqFt>qrS0UtvOUrmRYDWD#xG|yf^5@N2?V6XW6&l0&kG%k= zIa-z^txmK1AFn!d{F4CDpQ2Uey+vwSNBh9TYEu}jYfc$GIN_2`s>)Ke+ebe$|G6N; zNZOzWTc~>2y+^^AwMF5QTt2wUi&4`*1OJ7d9lkV%QcVnb#|~!c*JsanyI4i5jj+iM z->9xp6nStmKbr$3fI{e{Dzqn|9QJ8;#rN--l>AftZUVdwgA|s9qhll5`CX|Rw6aNP0RLjlbC8UBKm4=Ad&-w^6w)#HXiC^DGtYkquT7>Zwj z9;g6$Jn&zE04EP_>)pF!6V6MLmiiIoohCN4Sz}9@3>a^r+i(Qy9nCSWN5*GW^h~sp zVbbkNAN9MfqF2tVA6u#79 zp?RqnyatVwZ{O}uIClzA#I?>-5sqJ8&7949tL`AH;S{nSTSjY)AY{6Me2DF34l=)g z^Z??*4qb*;oRGC>am|0_mQL~YE!{< zPIX5Fabk4whVBbJW8;Z=*`lAVIGmj^lH|q<6ey(5IbLcEOLUW?NJ;rn)dD40;k4Sx-VQv>8!7bMkm%bf zs;e_O%OEf)hg?sTQyt%$KOC+%9Qt0lO7(`>d?+#7*k0&oBc_`?xB4BXYSfTBjTU-hHF8{NTY$}`f{rh*;a*!{P zafCb646w@KIE6Nw|Aag~^!olFBpPJ)?^w@2&wa7;5B;u3nfr#mYygYo!{)lcOU(MN z+)`WN;o-+!ITo)UqEWMt;jyvo1(;mZ{hwyyV$8!dUcV6n2BkvN#OvqPP+E&JHigf+ zwBElG6@OKm*^(`?I@)X!?VU|U*P`^IyjXR&7-74;i@Y(D&{YvS>)?0AJGnbnB0Dv8 z$-l3Io)4sZo3Df)FvJaJM-D>FG!&vmduWwo|j5FcJ63{C9J0rWH_#Xgg#wbUwrGN(q(M6 z$d<32o!zSVsbz;my>$y0mlCjtgo0O<`*B0|VqW_vfS*77e?~HKXJ%wAUEe+|kFWuC zg)d@jy|lCfLTq=L2iBtkTeGt+@$m43a9CMeFD3i10XwFS!xoer9jh*X2ndp)Qrm&> z#6-dRsRH=sFT2eQ0b#&L_skEp-m2EtkRg1jF)29q_RZMwl?~!Pk7ojy3JCqW1^>s7 zjB}60UQRZzS`avOmkGkfB)2+Vsgj=FNLe4=W;Va_uH=kqDN+oAz4Tm8NY6sR-;;;W|ju3q}*1Rkrz+BdO1E-)Zg>N0;zW_#QPx4b+^7bJ)BT&q@^ zE-YOx=+x0FfrOen(wur1gip{CHO*o-}$YwL=7k8{)A;3NP9r&R~gcb78+?E56jC|ZU|9~ zT(y|$@4p$e7+^nTbwj_FU+-62%DhAUg%AohyEbsO zTI4>?F80}OJ_joW9h?&FoDX67JSovo#scRLEhpWhV6$t-fqqo~JoF}=-T(0uimHBd z8&1;gI_Xr+3-uKszSjh;waFXHKs}+cUsGs7L!TlSMJ>3X#@wpd4_-HJQ1F<1M9=3t z`D&r#!JU6DiJcRj`;gKass5=0OEw<_e42zs`#0H{U5Lf?(qeyvN4Qi(KliDrQpps4P5{%m~%i^ zlGr;nHp=AwK(3_8%U^;6wUva>)ZkuD#rqjxTOicfywi*Q5Q@CzPt3&3=Bt@fumH!` zQq@Xf&q=9boAx2Wzr9&RTb^zdE^xmes4AFz16=jfVppd4eIFmRI<&kgg4B=&dI^Sj zDa4``RpaIW$xu9eQ*~D|k_)fu-e#`7{k5pE3y0&r;r#DAHOF&=abT!AXV!1#jSN+S zMuAo`0Yjf-s9a1@N`LQ-ffB|xb){%6wXTDkyph$D)q+9&r@f5~oC$P)r=RZH*$8DN z;$-OI6k6l2zfv-l@5+HH{lrIne+cVN9j;*58v$kYn_YMcu)??FV`?v-x_w=V%gO5B zAw>{4!#G->dt3zPna?7+qct&_pjcun5j?^#g>guacbj=XG2guqRaAC>+$eK5b@3aRK9# zdjO_OW`W2F+kwL?S@xQ~b(i^54=sYd=euf^wWYJp@r8F;gT9^NUe8jeelSD1t_{!R zyay(iQt_{GVQ1?yBBtnB-%t;)UAAg6rN)>dT6v zB8>PAB5pqfR?oFYMpgwg$-I{p+q3yg6~%|$OrinPYho0#A)%^*@=DU(<+ysMSe3!l z+4p%@6e_)Z^0HfE=dF5GhzUIXoPx(&EzsUTYpH~9Lir-sp@v0FY~`Ag7y?+Nb;~EK#3*_d*bost+JzIVZfw%CZFqxF zsW6u(ng4{Ab8DKJB|BEmU8YdXygXNE-4cI>{x2XLXDOoP^P<5z?lB-s)G<>p<0Tx& z)39q5#_ptrTX)pgtSQ^ho(tC|$T9#Ms5Qgke$^*>QC$!?Q>l@+v%cV9-5rw+&+HzU z-fR2|Ri5ptp(P0kU)=OHl1&+l!m_fnmCMJSb4|QojB>Z zw7H9)vo9EeB8fKjYd*I@ETn7qGA`BOes9QJs?asf2QxgndU{D)@;@3o9r7J``ERPe zBIf86sy?HDm2n4>#4qMdXm$G{Xn3N!dbf4rqc16odMMef%C zt}_L9euzN)g9kCz?dP{xO~)i;pv3oORJJ$s7W{qiWikMZy*&*w+)co+Pl3Nc)qUKl zdhFOy8GNVD;OJ zng&qI>_0&*q19bQIFkfNULF-(czfNG#3WN@1b0-X4dO4OyRvQB63zkD%FE8yhwrwU zjKFu{W97(1gBU})F|R=oE?nyA9)3BoUM~m`*4+uBhW?iY`~5pjm)A!!#~7ltpPke4 zU)K<5kwb&0VOpLs#a+(M6E~H1U&XoAzg&Mb7x^n~Xj3iILODZE*a9$Ver%Eal}5lF z@ZD{&HOpQ)`=AB=%3L2fm4GQQnE%-HG zrQzxZq18CQ$DouXs^r}NFOZ^wMpk)5AeM_&2!mQ0?MnpS3;$%v^xmrPN$ki3u&DoY zt9^T5>U_QhoAE`)>|w)mBU2R{+;Wws7_VJ}3D+5j+gi}t(03MT0%?u(hBC!Cy72nW z_iKC3I8PJ;3!me&&E1nA9@BNcd~J{#GnM4+eM)A1XQpbq&Jz$ro#ZMbadA&hp6L|; z$+yWnvZH1B^NF4&l1P&M8K3k-P=2l>fe(-tauFChNRJ^PH#V4*Z{NUk>UO<-cJx1FGP$@>=bfxTGd*!Yr{ zRhWv{7Cyv-Fzp`ZYFVqYT?DnY{nCnWK|@O`Xg?`c9k9juF>Ve%8`pa8q!FNHHZHdt zf41~zi2KmwUiQQiyzLlhBWZO$2X08qB$*KghD9z=wM9hhY zy?g+qD`OXbJ+Sh&%Zy;X=^JHie}q0`r*&HZA23hk`WG zoB+_svsLX)j3HL%KQQZSPjY?!LxmK*e*QYV9N;c{j_FU}ZbEu)%@OO`liob%7Eb@+ zX1K^@RJXLS;AAgf=Nf=h1)L1w<-UJ_h1sE%yj^L!V{`OmGsRKysR3``1fWv1$JE7%(uS;X6 zYp?+=SY8OS>L{8y3+5`C<#IS^KwcHtIV`0!V;FQ}Qv}ta^C|dVRR)^V0It$Tf$R%a z&tc6NFyHTpC{J>Ictjzh-bcx5jGuk{C{*E)wDd583*1TG;e~<)!JG$h(?=#lScfa|RZyZ|r-6 zmV(5uWnjYeHiltV1}8>9R~OnXn1GnOG~aZq6=o9~eC;7X_}AV8t_c+;k3Au+dN@fQ zSna8fHtg(8JPxG;?Bd@wbzva(74}39J80%Yz?`Vn>6svbm0%U@hZFkQvTw8|{wvkx zx$H9GX4==xfYiQTy`Cf4MJ{gn}sg;hun-q-g6H7ZWNzb8;>?mL`kqV^Sl)Ei*Az^soU+V7+8 z^6N$X?E(nI?@tYq<;9i(FPxn{4JFt_fs6XkN$0d^7aRO&PN1ZUR&isCgO{KG2{Gh{ zZs?c+R$K4Mh(&$q&l?k4-B@iS?X2x#)Ibk%Ycr}`-15N$dAXzq$7c#Mp+5gUxG^V4 zVcGDBayvaHK?MNt97(YH#RLLta%irn0=?sQV}|W~wa7WyJdG0thloMWQm;rV)sZn5pzt- zDWYfZv!?&>b_F!tUHzC^GFS3CM#${75s~3syY4+rFN3CI+#;*(pZUOY($|xLxRaX& z*@go<9)n?;n_PZ;Ybd!%V#x^m0!Quy~pXXeFghwy!( zoZ-@?g(=5+D<1z4m< zlpoyFu5mEkzJnwmTbVRYZ=M$fQ7t4BN}jKjW`FgcC6wjk3w$ITz)HT!KLD1dwm&y& zSz~`;bJuk~{ver~6?q?2e_7lnJ)#gamNz;hZC)&1FysRCyE2Fq(gmSK#$XQz!0ePj z8VM-wj2{Sv;!qw1iANTueZBxvOBsZtL;woS7ml4=9hl!-w1_m$$xQHgY*GLLUf-3s zuwBF-*V!X!VDs3wPDOU1jid3LDwVH(c08C-Y7{EP0|zQt#$XHA(7eS(fQ8=7a55PJ zpdJx4B61P<8hn|Un1D|+XlatTs(G7VfVkv5G`Otuzr%2S_ApcyEh^gaRJs2rrKVQZ_S%vUv!oX!V=qBht+0`Ifz{UY+|mQJ1+mGYjO+XLQgd8Lzw0 zX}Ce*v$~GXPPfrH6%J*P^I<@NMn;vwz5O(YtKFgEfa;%?RK(ADv*8q0SX?UBp z37uPP(`Xd3Tv)JD-1nLd4K#X!DQNX_j}PA1^uNKHN+h)L)6>(xPm%1%_WRE;8@Y&K zuxV702Cz7LEk3AXBT#W~pxiWM;>y97im>Ywt#y?6h=Hq^IuIXmjdhWXe93DLi~^@a zhw!P*ik=#qn&Pv?rPJy0PAiGVKL&O&38>7v+Mlzmdd&Us@u24LQ-^9%#CLI%V=z(` zhf{mrCj~Uxkyc@n$;WrT`I*o9HYKh-w0!4Y-beAXsl4QT_lI{Vdd)gYRj&5ZN~D&; zzO)wYXT+VLI5-3skAs984@;UgG{kW?cZ!M)&w7 zTBPqVt4?2ZT@%)Mv|+YfZD#?}qrKTsMkDCS9|WA+rMvGPlw|_Rf1G94^Sn-A`a=@R z>l!qpL2uu12y-+J44Ly?M6GnF{>(#dM5o)Ks2B3(96ycJSuVmm1lo16(_3jQD#*CO z*_Ce}&!R!Zm<|M@5-PjytFiFYUl5ck0TGs3TB(_dUIvEcjxy0;fgTB72LW;by7E#y zu@@RPqNxORxs=N&ckwtgC#vH1%-%R?ML?W|H&*09ICMpAtO(z#tTySjRgxGE(kgYhbgRBz1fsDHb7 z^oUReoL^zHAR)WZGjF3X9i?{FbS(TgLM=}2sf%DNe>oO01d&`Mv`$9abCY+eIn2Ap zFY@wt@BVfkq3(41b_Op3Keq*D<>LziKPitbe|94p+O? z=chERXkzxktvl#e|3+egrm@a|JWE*`Dj{*;a*+Rsu8~nzN5@eFR2Q1i39EuQQ*|eLj#X&VK-w&wtovxX79nYg!C*f+-EXy&+4-L z>(_Tos96lN=DqOuf|fE&g-k>R6SAl~-SKd_lj(P2;L|4r4t8U*Fmj~YaH)f_0W_&w z=D5N2opq#X8p04a>lzzwq0bt>S$W4}lV<7kvQsdbOMo%In-8|si)u-`o5124df6~A z=3=eYJC4fh5CNYtF-z6AxNR}+xmSA@@}cEG(eZ*^qeT-!pFbfSYYDm&pcuQ<%OCPh zUZ|yqrCLXN-C%8rSMMkep&K{!9?nWg@&|D{8)gmYdb(sD3t+j(*2Jh6F*T@!k}nx-k@p;CdqN8z2k2pmijw<71@H?sPH z+1O>f*<(5ZGvt~i(2`FFkt)*5G!|B*CK`M(jH7popNh@@#*?{NYb}wReycWiiCx@V zgMWD84*M5L4nz2aiCo0g(>WaO!&(oB#t5y*`oIA%;g3I<8<<_&VqMxK>TC|!p(Wk9 z2zoDZ)jz5R=wDFtFy4-6Teex!hTV$*leY6Bew`a=ask%TP33O2x*l7-OVJ-!30*;(k~1}nY1SwM#ON_R?v_q|Kv$1yl#mI`B=$0L zQ7r#82f4%SoXH(U{f?_KVWmy^9Cwln`H{}%oqZ+VODmZY%{6r}j9Iwa6%QOY6!>=8 zcqJvvdq`)_o|RjMdj9^=@b;BWoO$s^4eu?%&m$l)y_DhoGh*d@gI;#0Bhu+PnWU{N zTDn+KRWjWhtmVb-ozt*6_r#QI+4K~>2M2GVTC zU))kP#He+sPQd32-U8Z8HUN3wrp1loHE1Rx`}sE%uI93P&rQ()KrYN zi+(5m!u6s~txQ~yxqg6mhb*MTe($ehDC7E48kCX6uaRjit~1H6a@Axkdrw9tvsDd@ z9t&9)F}qiyU7c#OCR-wYWBQGj={?&(XBFogWtD-s>zP4=QKOARuL^aDk{H{bz2n7! z_U^r}lEkN5U0x_<2v$0Pvg7Nj4#*ONw=EXjHy;Kvr)#|SEJ~=bfan=eh zP)kOVaz(0fQw+8Pxz+t^mburB_Xs6uWastT-3L86^2#1_$?mSZ{Vs$q8WVq_qrcKG z`(<0Ji`aK>gN?8{N!bc-Th0TKcC&ZK9I{6y4H?;MO&^!&%j{6*q>gt zgsOI3c(SJeyD<4nc0SduGp);O zL%N#=X7iC(?#gT*n6L%6wu`wGMCDDat!k|Vdb480S#5?Aw(ZdmuL9-xBp^mY8?L|} z#_vGAQY_7^lP{fw7JLD*w#V0R*YI)W{#@QDg(fASo*Ui1cT&Lx-(VWJB>e1y+bW1S z*nJi|iFlqn#$*mDF!>!mphIeCkj-qa!~d8PI}jDs7UCCt{$si59Q9ShwCfiB+9Fl* zm5L#276OD7-YYsu0fH;NZ&!Rg5jBWd8r!K~3QX<>(Wt%Q`$KECq2<^EcvfTI>h_lH zJeKHD%XaRF0&n~i+M@Q-yJKt?9>cy&!thTS@p337nz7+Mm^h0~pQ-6Y`V-G7(<5}# zAI8(RYT{r1dW0T4>7gA)Z+#NIRq0^xvRgHuyhcqmF<*Hgq*5S9t~+-HvG)11h*h87 zic#1r#XZe%sE+;ENw8@TZdZo~PE&=4N|)s-0#j8SO_IXv;TnXYB0gUbk$_&DGd9@y zSZP40b`#!~w%dTGiZ3qr<=jE}^$MM%59q>tYFGGT9F-n-zuthR+4uON7CP>V! zFTfi2-bz}#y2M&!`R0HFS14 zS%EwC=X@Gt$}RnPA!|c#9SBY`zhVwXv!WT)dPftZllS}E=wi(vOkf%MsnRL8D*E7>QiEz7*v_9UW-wIn7f)RT!$CuZq9APM?Elsp6gb(m&^>N*}@1dU^# zuMm9V;O4*^ua`zZvX1;}3hTL=U*yt>-?M06Q2;D-wC=_jSJy$8stmDFN3#G+n)Hf4 zNC!@zx|^$u2o*40Fn)6Brl%h`_o_3JrE;)7IgR$x~H(>pHad?_KOBD|{ZZrO0T z62c-o*KT;$MTxs6XI@@vhE7XITlYdyer=_(qSeV9qY{XHA~qs)#xLD&;%NBiTThzr z%;mfen|)Su2s8Ey>T-lR+4TC+v5cORjRQ>iACG@{XF1aPf1o&u|tOvIw z%2I3+4&-W4wshc(U8f)K(29EA_JDUJ%xu^IyBa^#FG^{>$+%|KrZHQgM)ReJG0`52462QRnBF4AVh2b7amF4_XS1+?nGs`qp=Et7?P8dU z>$9s=m9JF6TaP#>-Ihfw0Xxl>WLn0Td(BxbtWk5!BK{MAK$=SSPa?e#Z^<6W9=p+O z|JI!JTfOt!tE;%&Sc-_@b2Wl4??QT>=q6}*4in zC?mR7?Ng#mT)t{}0n6~Q=P-ADem0A<`d;3xAS;`Z59YP&K6M(pgR3^ZZP^@9xxxPa z6wlR4CE_->WZ^vxhSziWPn|y7=&X%y89V9mdYXMQFK_2S)9#!H{K&fs2dH-Li;fG% zPM#asqx|H4+p!WaWDxH)?%9P}>j6Gn?8Ml7*&;h8F}Z3&&2F5*^q@x-Hm0LhYM~Y| zws(bbyY+jyJ-!clN9AOri|5MFdyDV8 zy=S*;vZLAJvpyb(DMbiwq}HQvCFK`DvMVKmQ`ncEid5O-rZv6I9;|F*Bd5b2YZoOM z)h%22zQTmxZEf~m0czphckyM%0{Zx3BntBSLCp;N z2J7cGgoCClBUhicXT4ih0EV2~UTY@q$5!K5NQ&Q0;B_)UoE~BAeA4&nZu`R2lnC@u zSxXnsP%+W-Kx7KG96F>z_ZZprx0M}P8M1mDHTT_OR|qIWsl z-!<5Dd=nlcz?>(Ed{a5Y__JKWH35xnwhC?)Rmu%pI~Zo0?6XY`7P?y{ zZ;m^#PzSa4a@Qg163DB5N0BxCe&+Fr?icWq!1%jRVBN$}pC&XM`Hqiz`Rk?(5_9!9 z47u*Cz+Eqt^!nHD3XMt3Y8FNxDXFlYaE73lU~-oF)YuyiC@Uzw2gJbz}u%QQE0tv_p13q{tm%crySo3rmkJXkSNjU z<5t$Y{uftTt#XZksXM>J)7UD$C7c$au%b-5hWj3CW!n{Z5g2INj!u^5`MXbIw;Zz9 zHfwFi7p+7D6g<^mP8Vu*do#FaWdR>>v(kv0TLap9p?CFE?RHm0l)r5pe`5>7Vhu7b zHE{UJ1FL(67Csw3mP@tPWi9cqr03b5^AvCx8k*#WE=pfM$Mo=l5|y&C1OMk4=Mcx< zzS+%|g)R*>%rFLK-)XONG1r5wy0K8!bB)tudR207zrT!>ZxOTQ-vZ_8Xi8idjF0&ntE%xe=RMfX@U-g??@2r5aO0n<2)`=`vb6=+I@+~5? z@`tGC&xG{i^*Q6JptAC|F++t=1S1BeFn zu!#?2@VKnL5}#bc%V+W$%XNfS>Wv4!Y~NPDiXSy1F2jtSH(!UAI|(6-6KU2sN1kTa zpMQ~ySytiIei9S55mvS6_;Y`w)QcLkX8Tmh-a5#3G5@$x7UAc6=#th{wLEfX&-66j zYoL2HF+LrDq@4HRGKSf;geIcFM)xLu-}pG>t^^=BzE{}AL>%N<6Y%(;&zLKTaT3?o zyB9{&r3!0J)Nk&dZImwK>vVSxE4O$~gCXayqwDQa)WAg1(yc4#fw_!+JVY9Morw0O zlQO_EnIf&aYnFq(`i&bN)pf5=96LJUxmKHlS{=3jHnZO56nbFZ?s__<=97DEi3TOu zU^8W1Y9@!rPWIAn@V>&o)-9^2rQtDNEs zO-MtYdkc{H?H$8X*7I3tl(Cx*wN#B{U58PS?agW-71qri!97&kO!>$38IKEPtmkhs z3z+r$Ukvlfdi^}x`3yV58|V9h9oFX|N-CL5b}<0bg#m&{kppUs)oJ+6tajTz?fcHG zEBvl^&hQEZc5fCjN7JqD!M>e}%ZUN3$>0c_-5hm>&rK~cuIyEu6jn+{JA%#0crzJB zf6o7P{%L9E-KzNRl@}#CPZJ7;_s$Qk*lJI#J{($=;@_?)YX>pY0Z~RN%is9tE?e!t zuMCJX`1YndrDFvF5eaD!mZby{ky4~nBm|aj1e8WPMH*IV1Ox=6MY^SP>29QB@1XDf z-}~Wyd7lr!!|wCU%$b>U&YYR^`+?DJ*(pk6buxJcIzZ)<=IfAw!AY^XFymfS4-r)q z_do$km!>iq=BxeH+(1HIP)b||L<;U&S>q|vL+t|kF51OBSJ!#&O{eF$Y9WA{Nd{q! zv1#wJ@AYN5?H65p`*%H8aJdloToL0(eLfRuUx5@WZl6%ukXmik32aX}KY6)HB3C>h zO!R1evAkcAdc%1s;HNHdOUb8NY_(b0KUyMJ#J9CyAVis>W*@=Q8<+FwzBs1Y4+k=v zdVr$EcMqLB!}WHeZIlg!B=6pvBFj)*4Uep@^V+^f_l;Wb2HsAkM{84M+{E>$p)$-j zm1O&=iuIS8kE)}|D9HHw4-+lN={PS8QTL^Q;nePX)>nk$YGF`NZoe!mzJg&DL;;I#L z@Tmb#+g9}?`XBQJoqQ<%JR8WA$)jtjxaQsW?Hj(m?p3>)HZCYoe&GG%?RpJRcuPLW zXHTaB5TR2A1?j|l%0t5d49{gw+We0q9nrBJwCHMzgv;%T>)3F+?Q}Ha_lG;lE)Ic{ z^x_(oJMRicAzSM8uQ`fR7i_?|gXlPjDBbR6Ez}!{1i^vKvc=E!I^$_7!?(_Mlt-LC zeA9R+p%sD4lIsCBp`9KStL=Xz084oEZ5`iIuJu8l`PJ_7n=nr0HSd&0~*Isk2G()bU|); z81>@nzOFzozaMkiZE`nylcA4TKX32f(V5x+76V`rzU^uI5#A{?N z;u#S5*5S(b#A_>bLQjm4AI^hoA?O6VY##Ag1v+&K$aw|A6X9nF}ax%*e#B% z*SwFmWq88MPiR?Q7Q-?NDseT+jN|(n%)f&cqEx`f55!T!Xe1BdXpijJXygeB4Bm2W zAvaR{vPOmUxI6J1KHB=ePDtLV6R7ZY+h57v( z;akshC>JhwGo9teFmAWon*TCX$m+cG>j}!1Uc|XAtr}Kv?(fgJ$@qH_#z~2Hbaq#A zBvdbdQ}D=6)pl+F$C?cxux=OE2lV~OZ?847G_P#ovh6Qx%Vl&tBn_Nf=p27sDbrgg zD^BOs{Kk(VME2C!#2WhA82mQp2jRBPjLnw@BTYvsMH9|XoF1fn-oj{{#Leu;5;a|k zRO`+Xsbr*2-Hr#r%iry`VhU5paSErlA!AKR0EfVvuLf2PlWD{U&V=`&dj1age&ld7 z7LMIk-qF2XJ3EmZg{;xGL%oU9haP1}U6~{WIfk)HmHf2TZkD!`rVdHBALZEgi4uuc zsAMFtTnvM{%pY0gZB$=ZWvS@74yY?AsinP-20;_pfA`Y00s{yb`P5 z$D)NU%~~pBh4;q&FMs>OYWr!;+mq|*1uahTVd=}F!ga4Y-eq8gk98WVX5IJ+R=Tg< z+I(dX<|11-QzpF*j&~bSF-f6Eah=iO_P^pM(EseUkamBE&$uDIC(BHO*;Uhv?!U-+ z>t5f!I9Q}p4o-t&{rwDJ7Qd79v9(PnYFR5Ng@OZr;a?Mih#v)Ig&|X0UnKM>8yHU3T;dnDikeXbjX9Xb2j|L_Rahf~ zzw?iCNU*2i`BOCRqDwj6q~ayW;g0ndA;(F+Wzx==K{&l-^`rACK?bk4`BDiWLnrnC zW0~@T^;&f)$joh$K*+72zUD=9Qz))HYYx$1-kTO7E6@C!aH1NYz*+Kg=V;!Ll17br ziR(4oZn)J?MP8ou(D@kIpSa0*L1qP9^E2Z@5K7s0`p_=@iO;0Yb?5bdp@!nf=_+xj zCNT6L6sR~#*pxtAQMFD8V)J5tZ}I!Ko%*~(2DwWuL&Z+uHaQof-SpSeoda`)bT<;0 z0B2)s$0_{1b18?x>%`y_k^JI@6Slt4$g573yvhopE>q8oS@priu4lJ+NMO5!PL;l! zS%w$1r198eJsM8%Lf3l1Qv4)fDq@yJk7fl!c{MEE!>cDLG&D3U&qvH|V#}UAu?){1 zKjRz>I73vNH?lTjir*1^v;V9Pq<3rR(0OFT*OD+i>Og$*b~j?EBsu#yM+I(JqZDzG zeS+T0l!zew3VVb)Vd8DSafPt&NA% zLYE54dGdnldkc_LMJ-+_h(5gPr4?)tYEc={BHemrx}M#bQ3(`<ukGtGV~=M zi_tY>2?!&e9liy7``5PHS@a|o9h;TTc?Ie&we&8QZU}lSoScVNziSY}uMWt}Rw335 z=E3@qal;a2-jgdvRn#vH>?v6XUK=r+w=wJ)&>!tY!$ID~1-ey@V-xb=Yy)Si+1)?x ztn9?QXt=kU3dUDRe7j;>0UQCNd>a{6`O z2cMa1l-=&^Vs^oyKB;?Ge`RXF$E{)2%|}XJ8}ARkdI%jwVA3xREL_|>66crl3@N>lzl#-c>JZ)g z?ppF34l|crduLA2bDN898XrGMBXaBtvRq&HoOXHvZf?%*b2^X9C^btL0?>-7%FTN> ze2`X^3&1Xq1EkWYN-$q9^02Barz&^obOaN$#dKp$IgX*+E5ijd{L2|5M91fIr!+h z=p4|7Grya1iTZwArq#xBrhs1x9i~cFVKAv2w|95DtHKl`s(EMN3{ktPT5c@QS5amq z7a1WY@agw2!UyFK;pS;ZtdfR#f1Up5t$6yKyuUUOeRF@ z{=&xWqmr~YGG`4|LY=h1+F1l`Me>5BM@KNrb<@g@ZNC~bO4 zFVEU;kq@P00r?4_fC0fhlNio$+CurBB#t$TZ2N_#3ByO$mBWey5u?)N^hAN2SRV$+ zl5~630yCt7zY8akf0nbrjcF695UD_{KO>BXJ-7}G2x1n4p<6@=XF8(Zq)N>F@K;mg z%cM-ei%?^cquO$AkDjser=|PQp)?#qPxU@eTe$DGo!sY^v`|^KRobzJjp8-ph7PA6 z;&^KdELmY=nTTFL^3XnH%+bAMb-jszINZMM^UO^#K0jHkXX#=jaTe^^*?%ZGmHHp1 zS;MIrUd0{J`YKCY*KFr?cQ!ij)9ZV;Yiym))e@d_C)1L1Rvt1^%&VL5J!1;QgL6dE zWc(b=E--|o=b+pLWEJqoz22|78r0>C?P1Eu$k6g@hJhMpFAC>LnQXaaJt3zL@YW+~ zSwn=qE3n7@>bu1mzil7c*lVs(Eka~sMWsPbFkW0bG_ow zXs+F>nTx#T>a}|Mc5!F{4Gq^V@x;UWnD=AvuN{{QjPtLE@lbwRFL~~tm01t#YB6$2 z#_n{OShA;#1a**;4@8bKPiDRs(eyv3P#$Q7o>bdN4q+$-6Z5#-O75%3JbT8)4e7(C zNSTSmD-N+Ypk;A3A*;+Po!U|f)Jc$i-$u{6p(2!ZEho59C?WcK@+j(y$*u)%YNY9x zye2bI_IX~r%MsJ58nwj|`iOlkC<Air` ztB4@J76=-;eiEI;lj&lyCV* z*~3dXcH~z6#z_($0nUSZ5%M^n350a@iU0k3Pbvbcb%{=v0GD zZ)UuvG3=?bH$&`}LiF4{9vA=xwqq(O7-Ub(6$*3D{;=S9n@?;J-uVACAob!dlza9( zz3O1V9=@BUW24FC=jRn3Wl;_#?#8Z#7D|oEmWdfgO<>VJR`Pu#X=NAwKp% zebZSa&v9*NDn)f8ERWP27GB}WUsIoS|7R+jFI9Ol2d7v0s1c)NJHa0F~@|??SN_ zJ2z|p;D&QGV9VVbxC}Na)83mca8uXN2e5IC= z{|J4x7ItVzNJs~??o;r0pxr9(?6H4~Eu8;MjyheG;m~L4TAb>z1N#2JxTRfbv<`Y5rArgKbc_*Kgy?^oX*(I~Fu&~hh z;n>PPQhfax2mbr45UrS}IAmDv)ya-*IFl#|aDr(X9*jo<>=%=Z;|7ZkR`3MN54E00 zH;t)_g4todgSa|7JI~dhXriN0^BM+R!l(BB45?d{66)&>eu2e?FlY?ZvTMI!!2{1@)`yZRZ$9BBdkyPs}x(9EoGLDzdhe!LB>m_SV(cUi)Y9@HKRp9EWl{9V_maBbOnjtwfzavSt{y&p z->T=!J$nl@XmCJ~@k7<$nw>I+RWXXZ2M_+cEb(6L|LyC)xBowP1p7xMWW-cuWgnRj z&o}u2;vRRG4iOq?C4k!p@VvP!PsmC)r|im1uI_zH#D1qE+WV8IY8pcZ?PidGBgh`x zho=l~3%-EhaW0uoi-Jk?Nk4!8{p;kB@%maa zs%0w72ZbZ+_7~e#^I2!4Ew9Sv>R;|==jGwOeEHJw%I3N$?eu0qg;MagwvfX7qNB-* zbn6M^<-OFkwE9oGmsBndD&N?O2wV%)P@oCC)aU;^J`F8P!VWY~F021V{jz;@aAp zv|^`^zWDz05^r|j1(nOq4eK)3ZC!0{>WZqW#jOcJvKF~E6h<|LuNIuv_iQ>;3XLP- zX-}FcDU(n2_X|)(Nw>w4kqtnoG*`jB$QxBSj_J)b7Fp&-W!jd0N$jMyYdX8m&?D;Y z?~!B2*Z3zwx{qh#F!4!{=&W|+XMDv|%k6e>99|v{hZt%f%Qkk)J+>NPSDfkB)6*r- z4`tjkAbxVyidrL8sGYjYMw>aZVGpQwds4;X_D{&f+I{;)0JUCtV=xgV$_RCCOSwdo z`8@|zEZtYaBJ*^!LS4J=HGhSA49A1}tzB+f0@t?(#vBX`4QC-2zLqnCm#M$u34l z>Z!F5y~Nq@3+?@50`1~a$+4DGBSx|P-H9CDa8&C$?=`@-ev-_G4$jg->|x+vbZe(Z zSJ$}iE9(|~ZaY^YA3pp7fYlfJq+nbZE_!C}8fx*IZ*^wy;~q(_q%}0Z1wCy6dRl3h zAYeMSRWI8ooK6r298AST6A}{2F4{r^y=ulYH9Q=S=OTAkrZl{{Q8+1NDH;)PS-o1!(H)!m>{eP&%uKhy^-kQsA$QUlikh-3X{xI^+}nE;wa_a%`>V{CsRiWw_MCdXOFaz^3^eJ* zwa(NmL`MVjsWp`19-XgfjrdoH=`LqmI1HQM|Cln-dX7Cnn3PmhCSCEJ+E>?JK!<{) z>FjUcZiR-1x-ShXl!A$!Y)mV2qrRdkz4= zo&tzEqPWgqr=))T0ClImq0uxI2)1b#?LX6Hl@`oyv5M0dw)fHeAn4*MkKNs|uw;PI zXsT)SW0sa4b9xAqJRw?5&pezzYZob`vvP?a5@AH%1Ij*ozT479POVf{bE@Is>5a{0 zY2q)Td}xDlW6b_ugKEDPc>eo_e2`3t!*upTc1V0Y<>#Cn*^;pqV?bfc$;G9vrx&)c z@Cv}VNymO}9+&vfBQeNB%Y98s`PZ*sGYbk*WOHDku_^upShnbB`}_NRv_wyol>^xn zJNofKy=_HD=rGOG4ipE~2?#2YN*qQt@Y!c07?hHhD-P1ZcT2XrShEV`$)Mnxv zwt2Zn({hiwAfQ)L)6xnC(5cVT#Hm06w6lelr#ffCZJyc5ZIL{UER`wae{Vv|L?9rt zYNbWry_XaIHtwCN;eFhchD8=!Bw11M$CGy5^%W8yzX75{Y}_$icH z^Rg}?H1s{2B7iD#nP}Uu|Jv>y;jO2qN2nOtNbbToid{UGeGm-4Ud;m+a( z_{*163yX_pl>j8EwJU)OjFI(ky@7w0{-;oa*S5qdYA;@3s#D@J=6(m08YHP-QQIDQ z!sOHe@>B%0R%q*4h9m?4+&X%~=YJySzE9a=qfPr<6yuSS;-dX(Yjdpyk$gViHl z0e82Yyu5rP{Kx2ErOH9L(bdNB4*o(xVPQ;M`DZN#tr9OshMR^7RorEdYz8!8Vc3K| z{##qMd-K88`?bh=zs$^QUx9}!evWv4%wFC-JU~>I)lSyq0bokLwyvRN@l^dU;&@SvSn3QkEa~yt>>AF* z!I$%`HKS>IOo+{~056mD3FK>H! z|AW9@^Ejr5H?nkGurce74N6R;Mr&&DZq=?o_XZ)j7Tc=sE^I)c^d7p~(?+4X zgobAL)`8A->l7y45ZhBENOZFM?sr zbn5%1PZhuIwDz>uZVA8LtiPqLZQD09rTs9Y*Yh1Q2wE{mwsembv0*?c-}u6Ag0}#IN00$1iuO+FL5-Z|gNKewcps;T2nky* z&!UAB zv0C2Gjk~4I#0bq|nSANdTbt^G_D1HdY1Q&!bMJuM%U(Akgn#JWqo__hg zF8t*=eq5)s`o+-i-yb{^8G? z*rB1Oub2G*qv>daz~AW}+rGWPGoaICP+>`){`No-RFoAcVK5=b%Pmw(vTgxW20&xp z1MbR@>=2;s>drR`{RQ=ZqQ%`I!am#Gaj`yIZ12t*TIYN7n=L*e;SM`H&PO93v4aCi zY_z0a^MIhBE<v zA^a^!s&ZdW_*e?ac!fko7gtg&Om}A+YmYo`v}~T;&PqaASy}aBq|u#w7X0ClS^u0{ zY8Pu(R91HOVQ+43@_RaSQxafnaNcc;RNlJn*-ZYgn>-|**I|njRx5pcAg`%OEg&Ee zosbY%RK)!xb{MAkso8ifT}(5U4Ome#ZIXmJMc_KR1AOVdanmRkf#;ZU`Ix(njW~cT z5AB_w-?lq|GK`KZr=<>aX*!~!-jT7gA{hPJE3d81A$eg#0le%oi+&@A85hBQfD|6~ z;R6AqxHFzrWeyU~2z6y-@~s2w`dW;qUv3opc*VNYvvdg$wO4?TR8Ed&H^t8N)vpj* zu-iu9M;7p11iw469o!B3@6tes^Ls+2PG_5!PakIfNKpV zwgT>TpcOIynY$(Y$kr)lqup?~(x#_3t@?Xin Date: Mon, 18 Feb 2019 12:54:07 -0800 Subject: [PATCH 028/523] Set the Google Cloud minimum CPU platform to Intel Haswell * Intel Haswell or better is available in every zone around the world * Neither Kubernetes nor Typhoon have a particular minimum processor family. However, a few Google Cloud zones still default to Sandy/Ivy bridge (scheduled to shift April 2019). Price is only based on machine type so it is beneficial to opt for the next processor family * Intel Haswell is a suitable minimum since it still allows plenty of liberty in choosing any region or machine type * Likely a slight increase to preemption probability in a few zones, but any lower probability on Sandy/Ivy bridge is due to lower desirability as they're phased out * https://cloud.google.com/compute/docs/regions-zones/ --- CHANGES.md | 6 ++++++ google-cloud/container-linux/kubernetes/controllers.tf | 7 ++++--- google-cloud/container-linux/kubernetes/workers/workers.tf | 7 ++++--- google-cloud/fedora-atomic/kubernetes/controllers.tf | 7 ++++--- google-cloud/fedora-atomic/kubernetes/workers/workers.tf | 7 ++++--- 5 files changed, 22 insertions(+), 12 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index a3aa360eb..3617d74aa 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -11,6 +11,12 @@ Notable changes between versions. * Recommend updating [terraform-provider-matchbox](https://github.com/coreos/terraform-provider-matchbox) plugin from v0.2.2 to [v0.2.3](https://github.com/coreos/terraform-provider-matchbox/releases/tag/v0.2.3) ([#402](https://github.com/poseidon/typhoon/pull/402)) +#### Google Cloud + +* Set the minimum CPU platform to Intel Haswell ([#405](https://github.com/poseidon/typhoon/pull/405)) + * Haswell or better is available in every zone (no price change) + * A few zones still default to Sandy/Ivy Bridge (shifts in April 2019) + #### Addons * Improve Prometheus rules and alerts ([#404](https://github.com/poseidon/typhoon/pull/404)) diff --git a/google-cloud/container-linux/kubernetes/controllers.tf b/google-cloud/container-linux/kubernetes/controllers.tf index 1a6ff0488..26fb7fb02 100644 --- a/google-cloud/container-linux/kubernetes/controllers.tf +++ b/google-cloud/container-linux/kubernetes/controllers.tf @@ -31,9 +31,10 @@ locals { resource "google_compute_instance" "controllers" { count = "${var.controller_count}" - name = "${var.cluster_name}-controller-${count.index}" - zone = "${element(local.zones, count.index)}" - machine_type = "${var.controller_type}" + name = "${var.cluster_name}-controller-${count.index}" + zone = "${element(local.zones, count.index)}" + machine_type = "${var.controller_type}" + min_cpu_platform = "Intel Haswell" metadata { user-data = "${element(data.ct_config.controller-ignitions.*.rendered, count.index)}" diff --git a/google-cloud/container-linux/kubernetes/workers/workers.tf b/google-cloud/container-linux/kubernetes/workers/workers.tf index 511f1128d..10b0e41e4 100644 --- a/google-cloud/container-linux/kubernetes/workers/workers.tf +++ b/google-cloud/container-linux/kubernetes/workers/workers.tf @@ -23,9 +23,10 @@ resource "google_compute_region_instance_group_manager" "workers" { # Worker instance template resource "google_compute_instance_template" "worker" { - name_prefix = "${var.name}-worker-" - description = "Worker Instance template" - machine_type = "${var.machine_type}" + name_prefix = "${var.name}-worker-" + description = "Worker Instance template" + machine_type = "${var.machine_type}" + min_cpu_platform = "Intel Haswell" metadata { user-data = "${data.ct_config.worker-ignition.rendered}" diff --git a/google-cloud/fedora-atomic/kubernetes/controllers.tf b/google-cloud/fedora-atomic/kubernetes/controllers.tf index f3d40a0ac..28e84b8ec 100644 --- a/google-cloud/fedora-atomic/kubernetes/controllers.tf +++ b/google-cloud/fedora-atomic/kubernetes/controllers.tf @@ -31,9 +31,10 @@ locals { resource "google_compute_instance" "controllers" { count = "${var.controller_count}" - name = "${var.cluster_name}-controller-${count.index}" - zone = "${element(local.zones, count.index)}" - machine_type = "${var.controller_type}" + name = "${var.cluster_name}-controller-${count.index}" + zone = "${element(local.zones, count.index)}" + machine_type = "${var.controller_type}" + min_cpu_platform = "Intel Haswell" metadata { user-data = "${element(data.template_file.controller-cloudinit.*.rendered, count.index)}" diff --git a/google-cloud/fedora-atomic/kubernetes/workers/workers.tf b/google-cloud/fedora-atomic/kubernetes/workers/workers.tf index 08fde6278..180556e85 100644 --- a/google-cloud/fedora-atomic/kubernetes/workers/workers.tf +++ b/google-cloud/fedora-atomic/kubernetes/workers/workers.tf @@ -23,9 +23,10 @@ resource "google_compute_region_instance_group_manager" "workers" { # Worker instance template resource "google_compute_instance_template" "worker" { - name_prefix = "${var.name}-worker-" - description = "Worker Instance template" - machine_type = "${var.machine_type}" + name_prefix = "${var.name}-worker-" + description = "Worker Instance template" + machine_type = "${var.machine_type}" + min_cpu_platform = "Intel Haswell" metadata { user-data = "${data.template_file.worker-cloudinit.rendered}" From 4294bd029211384560cf09c36584c5e678649af9 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Tue, 19 Feb 2019 22:21:39 -0800 Subject: [PATCH 029/523] Assign Pod Priority classes to critical cluster and node components * Assign pod priorityClassNames to critical cluster and node components (higher is higher priority) to inform node out-of-resource eviction order and scheduler preemption and scheduling order * Priority Admission Controller has been enabled since Typhoon v1.11.1 --- CHANGES.md | 2 ++ aws/container-linux/kubernetes/bootkube.tf | 2 +- aws/fedora-atomic/kubernetes/bootkube.tf | 2 +- azure/container-linux/kubernetes/bootkube.tf | 2 +- bare-metal/container-linux/kubernetes/bootkube.tf | 2 +- bare-metal/fedora-atomic/kubernetes/bootkube.tf | 2 +- digital-ocean/container-linux/kubernetes/bootkube.tf | 2 +- digital-ocean/fedora-atomic/kubernetes/bootkube.tf | 2 +- google-cloud/container-linux/kubernetes/bootkube.tf | 2 +- google-cloud/fedora-atomic/kubernetes/bootkube.tf | 2 +- 10 files changed, 11 insertions(+), 9 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 3617d74aa..acde54f3f 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -6,6 +6,8 @@ Notable changes between versions. * Update etcd from v3.3.11 to [v3.3.12](https://github.com/etcd-io/etcd/releases/tag/v3.3.12) * Update Calico from v3.5.0 to v3.5.1 +* Assign priorityClassNames to critical cluster and node components + * Informs node out-of-resource eviction and scheduler preemption and ordering #### Bare-Metal diff --git a/aws/container-linux/kubernetes/bootkube.tf b/aws/container-linux/kubernetes/bootkube.tf index fbaeb6616..8ef95bb8d 100644 --- a/aws/container-linux/kubernetes/bootkube.tf +++ b/aws/container-linux/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=4d315afd41dea46fdae90bb629dfc027a6eaa2ad" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=c5f5aacce974e6961b63b0d3c136cd050083fec3" cluster_name = "${var.cluster_name}" api_servers = ["${format("%s.%s", var.cluster_name, var.dns_zone)}"] diff --git a/aws/fedora-atomic/kubernetes/bootkube.tf b/aws/fedora-atomic/kubernetes/bootkube.tf index e7ed6cda1..1377618be 100644 --- a/aws/fedora-atomic/kubernetes/bootkube.tf +++ b/aws/fedora-atomic/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=4d315afd41dea46fdae90bb629dfc027a6eaa2ad" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=c5f5aacce974e6961b63b0d3c136cd050083fec3" cluster_name = "${var.cluster_name}" api_servers = ["${format("%s.%s", var.cluster_name, var.dns_zone)}"] diff --git a/azure/container-linux/kubernetes/bootkube.tf b/azure/container-linux/kubernetes/bootkube.tf index d8b9d1e43..e7f3d1357 100644 --- a/azure/container-linux/kubernetes/bootkube.tf +++ b/azure/container-linux/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=4d315afd41dea46fdae90bb629dfc027a6eaa2ad" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=c5f5aacce974e6961b63b0d3c136cd050083fec3" cluster_name = "${var.cluster_name}" api_servers = ["${format("%s.%s", var.cluster_name, var.dns_zone)}"] diff --git a/bare-metal/container-linux/kubernetes/bootkube.tf b/bare-metal/container-linux/kubernetes/bootkube.tf index e56dc536d..afa0a1e26 100644 --- a/bare-metal/container-linux/kubernetes/bootkube.tf +++ b/bare-metal/container-linux/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=4d315afd41dea46fdae90bb629dfc027a6eaa2ad" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=c5f5aacce974e6961b63b0d3c136cd050083fec3" cluster_name = "${var.cluster_name}" api_servers = ["${var.k8s_domain_name}"] diff --git a/bare-metal/fedora-atomic/kubernetes/bootkube.tf b/bare-metal/fedora-atomic/kubernetes/bootkube.tf index bc382e386..42f164b43 100644 --- a/bare-metal/fedora-atomic/kubernetes/bootkube.tf +++ b/bare-metal/fedora-atomic/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=4d315afd41dea46fdae90bb629dfc027a6eaa2ad" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=c5f5aacce974e6961b63b0d3c136cd050083fec3" cluster_name = "${var.cluster_name}" api_servers = ["${var.k8s_domain_name}"] diff --git a/digital-ocean/container-linux/kubernetes/bootkube.tf b/digital-ocean/container-linux/kubernetes/bootkube.tf index f96d64cf6..c67a1bba2 100644 --- a/digital-ocean/container-linux/kubernetes/bootkube.tf +++ b/digital-ocean/container-linux/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=4d315afd41dea46fdae90bb629dfc027a6eaa2ad" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=c5f5aacce974e6961b63b0d3c136cd050083fec3" cluster_name = "${var.cluster_name}" api_servers = ["${format("%s.%s", var.cluster_name, var.dns_zone)}"] diff --git a/digital-ocean/fedora-atomic/kubernetes/bootkube.tf b/digital-ocean/fedora-atomic/kubernetes/bootkube.tf index 60ba58574..0903b6e55 100644 --- a/digital-ocean/fedora-atomic/kubernetes/bootkube.tf +++ b/digital-ocean/fedora-atomic/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=4d315afd41dea46fdae90bb629dfc027a6eaa2ad" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=c5f5aacce974e6961b63b0d3c136cd050083fec3" cluster_name = "${var.cluster_name}" api_servers = ["${format("%s.%s", var.cluster_name, var.dns_zone)}"] diff --git a/google-cloud/container-linux/kubernetes/bootkube.tf b/google-cloud/container-linux/kubernetes/bootkube.tf index 00e4be158..b5b8d94ec 100644 --- a/google-cloud/container-linux/kubernetes/bootkube.tf +++ b/google-cloud/container-linux/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=4d315afd41dea46fdae90bb629dfc027a6eaa2ad" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=c5f5aacce974e6961b63b0d3c136cd050083fec3" cluster_name = "${var.cluster_name}" api_servers = ["${format("%s.%s", var.cluster_name, var.dns_zone)}"] diff --git a/google-cloud/fedora-atomic/kubernetes/bootkube.tf b/google-cloud/fedora-atomic/kubernetes/bootkube.tf index f0ad310a5..c0397cc9a 100644 --- a/google-cloud/fedora-atomic/kubernetes/bootkube.tf +++ b/google-cloud/fedora-atomic/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=4d315afd41dea46fdae90bb629dfc027a6eaa2ad" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=c5f5aacce974e6961b63b0d3c136cd050083fec3" cluster_name = "${var.cluster_name}" api_servers = ["${format("%s.%s", var.cluster_name, var.dns_zone)}"] From 7f8572030dbc54aaa04bf30bfd032d5c384b7450 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Wed, 20 Feb 2019 00:54:19 -0800 Subject: [PATCH 030/523] Upgrade to support terraform-provider-google v2.0+ * Support terraform-provider-google v1.19.0, v1.19.1, v1.20.0 and v2.0+ (and allow for future 2.x.y releases) * Require terraform-provider-google v1.19.0 or newer. v1.19.0 introduced `network_interface` fields `network_ip` and `nat_ip` to deprecate `address` and `assigned_nat_ip`. Those deprecated fields are removed in terraform-provider-google v2.0 * https://github.com/terraform-providers/terraform-provider-google/releases/tag/v2.0.0 --- CHANGES.md | 4 +++- docs/atomic/google-cloud.md | 2 +- docs/cl/google-cloud.md | 2 +- google-cloud/container-linux/kubernetes/controllers.tf | 4 ++-- google-cloud/container-linux/kubernetes/require.tf | 2 +- google-cloud/fedora-atomic/kubernetes/controllers.tf | 4 ++-- google-cloud/fedora-atomic/kubernetes/require.tf | 2 +- 7 files changed, 11 insertions(+), 9 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index acde54f3f..c475641e1 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -6,7 +6,7 @@ Notable changes between versions. * Update etcd from v3.3.11 to [v3.3.12](https://github.com/etcd-io/etcd/releases/tag/v3.3.12) * Update Calico from v3.5.0 to v3.5.1 -* Assign priorityClassNames to critical cluster and node components +* Assign priorityClassNames to critical cluster and node components ([#406](https://github.com/poseidon/typhoon/pull/406)) * Informs node out-of-resource eviction and scheduler preemption and ordering #### Bare-Metal @@ -15,6 +15,8 @@ Notable changes between versions. #### Google Cloud +* Support `terraform-provider-google` v2.0+ ([#407](https://github.com/poseidon/typhoon/pull/407)) + * Require `terraform-provider-google` v1.19+ (action required) * Set the minimum CPU platform to Intel Haswell ([#405](https://github.com/poseidon/typhoon/pull/405)) * Haswell or better is available in every zone (no price change) * A few zones still default to Sandy/Ivy Bridge (shifts in April 2019) diff --git a/docs/atomic/google-cloud.md b/docs/atomic/google-cloud.md index 0e8620063..130cc9c87 100644 --- a/docs/atomic/google-cloud.md +++ b/docs/atomic/google-cloud.md @@ -45,7 +45,7 @@ Configure the Google Cloud provider to use your service account key, project-id, ```tf provider "google" { - version = "1.6" + version = "2.0.0" alias = "default" credentials = "${file("~/.config/google-cloud/terraform.json")}" diff --git a/docs/cl/google-cloud.md b/docs/cl/google-cloud.md index e8f2abcfb..3d1d411c5 100644 --- a/docs/cl/google-cloud.md +++ b/docs/cl/google-cloud.md @@ -49,7 +49,7 @@ Configure the Google Cloud provider to use your service account key, project-id, ```tf provider "google" { - version = "1.6" + version = "2.0.0" alias = "default" credentials = "${file("~/.config/google-cloud/terraform.json")}" diff --git a/google-cloud/container-linux/kubernetes/controllers.tf b/google-cloud/container-linux/kubernetes/controllers.tf index 26fb7fb02..dbb445a69 100644 --- a/google-cloud/container-linux/kubernetes/controllers.tf +++ b/google-cloud/container-linux/kubernetes/controllers.tf @@ -11,7 +11,7 @@ resource "google_dns_record_set" "etcds" { ttl = 300 # private IPv4 address for etcd - rrdatas = ["${element(google_compute_instance.controllers.*.network_interface.0.address, count.index)}"] + rrdatas = ["${element(google_compute_instance.controllers.*.network_interface.0.network_ip, count.index)}"] } # Zones in the region @@ -24,7 +24,7 @@ locals { # controllers over up to 3 zones, since all GCP regions have at least 3. zones = "${slice(data.google_compute_zones.all.names, 0, 3)}" - controllers_ipv4_public = ["${google_compute_instance.controllers.*.network_interface.0.access_config.0.assigned_nat_ip}"] + controllers_ipv4_public = ["${google_compute_instance.controllers.*.network_interface.0.access_config.0.nat_ip}"] } # Controller instances diff --git a/google-cloud/container-linux/kubernetes/require.tf b/google-cloud/container-linux/kubernetes/require.tf index 7a4ef49e4..d1f7f4d54 100644 --- a/google-cloud/container-linux/kubernetes/require.tf +++ b/google-cloud/container-linux/kubernetes/require.tf @@ -5,7 +5,7 @@ terraform { } provider "google" { - version = "~> 1.6" + version = ">= 1.19, < 3.0" } provider "local" { diff --git a/google-cloud/fedora-atomic/kubernetes/controllers.tf b/google-cloud/fedora-atomic/kubernetes/controllers.tf index 28e84b8ec..cc7d605a3 100644 --- a/google-cloud/fedora-atomic/kubernetes/controllers.tf +++ b/google-cloud/fedora-atomic/kubernetes/controllers.tf @@ -11,7 +11,7 @@ resource "google_dns_record_set" "etcds" { ttl = 300 # private IPv4 address for etcd - rrdatas = ["${element(google_compute_instance.controllers.*.network_interface.0.address, count.index)}"] + rrdatas = ["${element(google_compute_instance.controllers.*.network_interface.0.network_ip, count.index)}"] } # Zones in the region @@ -24,7 +24,7 @@ locals { # controllers over up to 3 zones, since all GCP regions have at least 3. zones = "${slice(data.google_compute_zones.all.names, 0, 3)}" - controllers_ipv4_public = ["${google_compute_instance.controllers.*.network_interface.0.access_config.0.assigned_nat_ip}"] + controllers_ipv4_public = ["${google_compute_instance.controllers.*.network_interface.0.access_config.0.nat_ip}"] } # Controller instances diff --git a/google-cloud/fedora-atomic/kubernetes/require.tf b/google-cloud/fedora-atomic/kubernetes/require.tf index 7a4ef49e4..d1f7f4d54 100644 --- a/google-cloud/fedora-atomic/kubernetes/require.tf +++ b/google-cloud/fedora-atomic/kubernetes/require.tf @@ -5,7 +5,7 @@ terraform { } provider "google" { - version = "~> 1.6" + version = ">= 1.19, < 3.0" } provider "local" { From d10c2b4cb988467dc4b5b7cf4ff26b980d2a3d1d Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Sat, 23 Feb 2019 12:49:09 -0800 Subject: [PATCH 031/523] Update Grafana from v6.0.0-beta2 to v6.0.0-beta3 * Update Grafana dashboards --- CHANGES.md | 2 +- addons/grafana/dashboards.yaml | 208 ++++++++++++++++++++++++++++++--- addons/grafana/deployment.yaml | 2 +- 3 files changed, 195 insertions(+), 17 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index c475641e1..f10ce1e2c 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -28,7 +28,7 @@ Notable changes between versions. * Add `pod` name label to metrics discovered via service endpoints * Rename `kubernetes_namespace` label to `namespace` * Improve Grafana and dashboards ([#403](https://github.com/poseidon/typhoon/pull/403), [#404](https://github.com/poseidon/typhoon/pull/404)) - * Upgrade Grafana from v5.4.3 to 6.0.0-beta2 + * Upgrade Grafana from v5.4.3 to 6.0.0-beta3 * Enable Grafana [Explore](http://docs.grafana.org/guides/whats-new-in-v6-0/#explore) UI as a Viewer (inspect/edit without saving) * Raise nginx-ingress liveness/readiness timeout to 5 seconds * Remove nginx-ingess default-backend ([#401](https://github.com/poseidon/typhoon/pull/401)) diff --git a/addons/grafana/dashboards.yaml b/addons/grafana/dashboards.yaml index 181b288b7..a6c5cc601 100644 --- a/addons/grafana/dashboards.yaml +++ b/addons/grafana/dashboards.yaml @@ -4045,7 +4045,7 @@ data: }, "yaxes": [ { - "format": "decbytes", + "format": "bytes", "label": null, "logBase": 1, "max": null, @@ -4133,7 +4133,7 @@ data: ], "type": "number", - "unit": "decbytes" + "unit": "bytes" }, { "alias": "Memory Requests", @@ -4151,7 +4151,7 @@ data: ], "type": "number", - "unit": "decbytes" + "unit": "bytes" }, { "alias": "Memory Requests %", @@ -4187,7 +4187,7 @@ data: ], "type": "number", - "unit": "decbytes" + "unit": "bytes" }, { "alias": "Memory Limits %", @@ -4833,7 +4833,7 @@ data: ], "timeFrom": null, "timeShift": null, - "title": "Memory Usage", + "title": "Memory Usage (w/o cache)", "tooltip": { "shared": true, "sort": 0, @@ -4851,7 +4851,7 @@ data: }, "yaxes": [ { - "format": "decbytes", + "format": "bytes", "label": null, "logBase": 1, "max": null, @@ -4939,7 +4939,7 @@ data: ], "type": "number", - "unit": "decbytes" + "unit": "bytes" }, { "alias": "Memory Requests", @@ -4957,7 +4957,7 @@ data: ], "type": "number", - "unit": "decbytes" + "unit": "bytes" }, { "alias": "Memory Requests %", @@ -4993,7 +4993,7 @@ data: ], "type": "number", - "unit": "decbytes" + "unit": "bytes" }, { "alias": "Memory Limits %", @@ -5013,6 +5013,60 @@ data: "type": "number", "unit": "percentunit" }, + { + "alias": "Memory Usage (RSS)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #F", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Usage (Cache)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #G", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Usage (Swap", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #H", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, { "alias": "Pod", "colorMode": null, @@ -5092,6 +5146,33 @@ data: "legendFormat": "", "refId": "E", "step": 10 + }, + { + "expr": "sum(label_replace(container_memory_rss{namespace=\"$namespace\",container_name!=\"\"}, \"pod\", \"$1\", \"pod_name\", \"(.*)\")) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "F", + "step": 10 + }, + { + "expr": "sum(label_replace(container_memory_cache{namespace=\"$namespace\",container_name!=\"\"}, \"pod\", \"$1\", \"pod_name\", \"(.*)\")) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "G", + "step": 10 + }, + { + "expr": "sum(label_replace(container_memory_swap{namespace=\"$namespace\",container_name!=\"\"}, \"pod\", \"$1\", \"pod_name\", \"(.*)\")) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "H", + "step": 10 } ], "thresholds": [ @@ -5653,10 +5734,26 @@ data: "steppedLine": false, "targets": [ { - "expr": "sum(container_memory_usage_bytes{namespace=\"$namespace\", pod_name=\"$pod\", container_name!=\"POD\", container_name!=\"\"}) by (container_name)", + "expr": "sum(container_memory_rss{namespace=\"$namespace\", pod_name=\"$pod\", container_name!=\"POD\", container_name!=\"\"}) by (container_name)", "format": "time_series", "intervalFactor": 2, - "legendFormat": "{{container_name}}", + "legendFormat": "{{container_name}} (RSS)", + "legendLink": null, + "step": 10 + }, + { + "expr": "sum(container_memory_cache{namespace=\"$namespace\", pod_name=\"$pod\", container_name!=\"POD\", container_name!=\"\"}) by (container_name)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{container_name}} (Cache)", + "legendLink": null, + "step": 10 + }, + { + "expr": "sum(container_memory_swap{namespace=\"$namespace\", pod_name=\"$pod\", container_name!=\"POD\", container_name!=\"\"}) by (container_name)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{container_name}} (Swap)", "legendLink": null, "step": 10 } @@ -5684,7 +5781,7 @@ data: }, "yaxes": [ { - "format": "short", + "format": "bytes", "label": null, "logBase": 1, "max": null, @@ -5772,7 +5869,7 @@ data: ], "type": "number", - "unit": "decbytes" + "unit": "bytes" }, { "alias": "Memory Requests", @@ -5790,7 +5887,7 @@ data: ], "type": "number", - "unit": "decbytes" + "unit": "bytes" }, { "alias": "Memory Requests %", @@ -5826,7 +5923,7 @@ data: ], "type": "number", - "unit": "decbytes" + "unit": "bytes" }, { "alias": "Memory Limits %", @@ -5846,6 +5943,60 @@ data: "type": "number", "unit": "percentunit" }, + { + "alias": "Memory Usage (RSS)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #F", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Usage (Cache)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #G", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Usage (Swap", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #H", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, { "alias": "Container", "colorMode": null, @@ -5925,6 +6076,33 @@ data: "legendFormat": "", "refId": "E", "step": 10 + }, + { + "expr": "sum(label_replace(container_memory_rss{namespace=\"$namespace\", pod_name=\"$pod\", container_name != \"\", container_name != \"POD\"}, \"container\", \"$1\", \"container_name\", \"(.*)\")) by (container)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "F", + "step": 10 + }, + { + "expr": "sum(label_replace(container_memory_cache{namespace=\"$namespace\", pod_name=\"$pod\", container_name != \"\", container_name != \"POD\"}, \"container\", \"$1\", \"container_name\", \"(.*)\")) by (container)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "G", + "step": 10 + }, + { + "expr": "sum(label_replace(container_memory_swap{namespace=\"$namespace\", pod_name=\"$pod\", container_name != \"\", container_name != \"POD\"}, \"container\", \"$1\", \"container_name\", \"(.*)\")) by (container)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "H", + "step": 10 } ], "thresholds": [ diff --git a/addons/grafana/deployment.yaml b/addons/grafana/deployment.yaml index cfe9d608c..5c5998f0b 100644 --- a/addons/grafana/deployment.yaml +++ b/addons/grafana/deployment.yaml @@ -23,7 +23,7 @@ spec: spec: containers: - name: grafana - image: grafana/grafana:6.0.0-beta2 + image: grafana/grafana:6.0.0-beta3 env: - name: GF_PATHS_CONFIG value: "/etc/grafana/custom.ini" From 42d7222f3d8cb8ee63ae37b28fc798f409a7daba Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Sat, 23 Feb 2019 13:23:11 -0800 Subject: [PATCH 032/523] Add a readiness probe to CoreDNS * https://github.com/poseidon/terraform-render-bootkube/pull/115 --- CHANGES.md | 1 + aws/container-linux/kubernetes/bootkube.tf | 2 +- aws/fedora-atomic/kubernetes/bootkube.tf | 2 +- azure/container-linux/kubernetes/bootkube.tf | 2 +- bare-metal/container-linux/kubernetes/bootkube.tf | 2 +- bare-metal/fedora-atomic/kubernetes/bootkube.tf | 2 +- digital-ocean/container-linux/kubernetes/bootkube.tf | 2 +- digital-ocean/fedora-atomic/kubernetes/bootkube.tf | 2 +- google-cloud/container-linux/kubernetes/bootkube.tf | 2 +- google-cloud/fedora-atomic/kubernetes/bootkube.tf | 2 +- 10 files changed, 10 insertions(+), 9 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index f10ce1e2c..00e729535 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -8,6 +8,7 @@ Notable changes between versions. * Update Calico from v3.5.0 to v3.5.1 * Assign priorityClassNames to critical cluster and node components ([#406](https://github.com/poseidon/typhoon/pull/406)) * Informs node out-of-resource eviction and scheduler preemption and ordering +* Add a CoreDNS readiness probe ([#410](https://github.com/poseidon/typhoon/pull/410)) #### Bare-Metal diff --git a/aws/container-linux/kubernetes/bootkube.tf b/aws/container-linux/kubernetes/bootkube.tf index 8ef95bb8d..f21f6c45b 100644 --- a/aws/container-linux/kubernetes/bootkube.tf +++ b/aws/container-linux/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=c5f5aacce974e6961b63b0d3c136cd050083fec3" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=593f0e3655294832122a133c08d0ed5c4f28f9b3" cluster_name = "${var.cluster_name}" api_servers = ["${format("%s.%s", var.cluster_name, var.dns_zone)}"] diff --git a/aws/fedora-atomic/kubernetes/bootkube.tf b/aws/fedora-atomic/kubernetes/bootkube.tf index 1377618be..161052138 100644 --- a/aws/fedora-atomic/kubernetes/bootkube.tf +++ b/aws/fedora-atomic/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=c5f5aacce974e6961b63b0d3c136cd050083fec3" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=593f0e3655294832122a133c08d0ed5c4f28f9b3" cluster_name = "${var.cluster_name}" api_servers = ["${format("%s.%s", var.cluster_name, var.dns_zone)}"] diff --git a/azure/container-linux/kubernetes/bootkube.tf b/azure/container-linux/kubernetes/bootkube.tf index e7f3d1357..6fa7b6384 100644 --- a/azure/container-linux/kubernetes/bootkube.tf +++ b/azure/container-linux/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=c5f5aacce974e6961b63b0d3c136cd050083fec3" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=593f0e3655294832122a133c08d0ed5c4f28f9b3" cluster_name = "${var.cluster_name}" api_servers = ["${format("%s.%s", var.cluster_name, var.dns_zone)}"] diff --git a/bare-metal/container-linux/kubernetes/bootkube.tf b/bare-metal/container-linux/kubernetes/bootkube.tf index afa0a1e26..d407714aa 100644 --- a/bare-metal/container-linux/kubernetes/bootkube.tf +++ b/bare-metal/container-linux/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=c5f5aacce974e6961b63b0d3c136cd050083fec3" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=593f0e3655294832122a133c08d0ed5c4f28f9b3" cluster_name = "${var.cluster_name}" api_servers = ["${var.k8s_domain_name}"] diff --git a/bare-metal/fedora-atomic/kubernetes/bootkube.tf b/bare-metal/fedora-atomic/kubernetes/bootkube.tf index 42f164b43..6b184533e 100644 --- a/bare-metal/fedora-atomic/kubernetes/bootkube.tf +++ b/bare-metal/fedora-atomic/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=c5f5aacce974e6961b63b0d3c136cd050083fec3" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=593f0e3655294832122a133c08d0ed5c4f28f9b3" cluster_name = "${var.cluster_name}" api_servers = ["${var.k8s_domain_name}"] diff --git a/digital-ocean/container-linux/kubernetes/bootkube.tf b/digital-ocean/container-linux/kubernetes/bootkube.tf index c67a1bba2..29ea717d7 100644 --- a/digital-ocean/container-linux/kubernetes/bootkube.tf +++ b/digital-ocean/container-linux/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=c5f5aacce974e6961b63b0d3c136cd050083fec3" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=593f0e3655294832122a133c08d0ed5c4f28f9b3" cluster_name = "${var.cluster_name}" api_servers = ["${format("%s.%s", var.cluster_name, var.dns_zone)}"] diff --git a/digital-ocean/fedora-atomic/kubernetes/bootkube.tf b/digital-ocean/fedora-atomic/kubernetes/bootkube.tf index 0903b6e55..3c053b97e 100644 --- a/digital-ocean/fedora-atomic/kubernetes/bootkube.tf +++ b/digital-ocean/fedora-atomic/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=c5f5aacce974e6961b63b0d3c136cd050083fec3" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=593f0e3655294832122a133c08d0ed5c4f28f9b3" cluster_name = "${var.cluster_name}" api_servers = ["${format("%s.%s", var.cluster_name, var.dns_zone)}"] diff --git a/google-cloud/container-linux/kubernetes/bootkube.tf b/google-cloud/container-linux/kubernetes/bootkube.tf index b5b8d94ec..252f3ea63 100644 --- a/google-cloud/container-linux/kubernetes/bootkube.tf +++ b/google-cloud/container-linux/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=c5f5aacce974e6961b63b0d3c136cd050083fec3" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=593f0e3655294832122a133c08d0ed5c4f28f9b3" cluster_name = "${var.cluster_name}" api_servers = ["${format("%s.%s", var.cluster_name, var.dns_zone)}"] diff --git a/google-cloud/fedora-atomic/kubernetes/bootkube.tf b/google-cloud/fedora-atomic/kubernetes/bootkube.tf index c0397cc9a..586e7a21d 100644 --- a/google-cloud/fedora-atomic/kubernetes/bootkube.tf +++ b/google-cloud/fedora-atomic/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=c5f5aacce974e6961b63b0d3c136cd050083fec3" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=593f0e3655294832122a133c08d0ed5c4f28f9b3" cluster_name = "${var.cluster_name}" api_servers = ["${format("%s.%s", var.cluster_name, var.dns_zone)}"] From 73ae5d5649c509458732cb63c50f177bef895bd9 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Mon, 25 Feb 2019 21:23:13 -0800 Subject: [PATCH 033/523] Update Calico from v3.5.1 to v3.5.2 * https://docs.projectcalico.org/v3.5/releases/ --- CHANGES.md | 2 +- aws/container-linux/kubernetes/bootkube.tf | 2 +- aws/fedora-atomic/kubernetes/bootkube.tf | 2 +- azure/container-linux/kubernetes/bootkube.tf | 2 +- bare-metal/container-linux/kubernetes/bootkube.tf | 2 +- bare-metal/fedora-atomic/kubernetes/bootkube.tf | 2 +- digital-ocean/container-linux/kubernetes/bootkube.tf | 2 +- digital-ocean/fedora-atomic/kubernetes/bootkube.tf | 2 +- google-cloud/container-linux/kubernetes/bootkube.tf | 2 +- google-cloud/fedora-atomic/kubernetes/bootkube.tf | 2 +- 10 files changed, 10 insertions(+), 10 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 00e729535..d2df3d705 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -5,7 +5,7 @@ Notable changes between versions. ## Latest * Update etcd from v3.3.11 to [v3.3.12](https://github.com/etcd-io/etcd/releases/tag/v3.3.12) -* Update Calico from v3.5.0 to v3.5.1 +* Update Calico from v3.5.0 to [v3.5.2](https://docs.projectcalico.org/v3.5/releases/) * Assign priorityClassNames to critical cluster and node components ([#406](https://github.com/poseidon/typhoon/pull/406)) * Informs node out-of-resource eviction and scheduler preemption and ordering * Add a CoreDNS readiness probe ([#410](https://github.com/poseidon/typhoon/pull/410)) diff --git a/aws/container-linux/kubernetes/bootkube.tf b/aws/container-linux/kubernetes/bootkube.tf index f21f6c45b..dbf5982b3 100644 --- a/aws/container-linux/kubernetes/bootkube.tf +++ b/aws/container-linux/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=593f0e3655294832122a133c08d0ed5c4f28f9b3" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=0a7c4fda3507b4a755c0c777d2e65d38bc152bbb" cluster_name = "${var.cluster_name}" api_servers = ["${format("%s.%s", var.cluster_name, var.dns_zone)}"] diff --git a/aws/fedora-atomic/kubernetes/bootkube.tf b/aws/fedora-atomic/kubernetes/bootkube.tf index 161052138..b35aab32c 100644 --- a/aws/fedora-atomic/kubernetes/bootkube.tf +++ b/aws/fedora-atomic/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=593f0e3655294832122a133c08d0ed5c4f28f9b3" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=0a7c4fda3507b4a755c0c777d2e65d38bc152bbb" cluster_name = "${var.cluster_name}" api_servers = ["${format("%s.%s", var.cluster_name, var.dns_zone)}"] diff --git a/azure/container-linux/kubernetes/bootkube.tf b/azure/container-linux/kubernetes/bootkube.tf index 6fa7b6384..d7a1bb8c2 100644 --- a/azure/container-linux/kubernetes/bootkube.tf +++ b/azure/container-linux/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=593f0e3655294832122a133c08d0ed5c4f28f9b3" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=0a7c4fda3507b4a755c0c777d2e65d38bc152bbb" cluster_name = "${var.cluster_name}" api_servers = ["${format("%s.%s", var.cluster_name, var.dns_zone)}"] diff --git a/bare-metal/container-linux/kubernetes/bootkube.tf b/bare-metal/container-linux/kubernetes/bootkube.tf index d407714aa..c6cf5804e 100644 --- a/bare-metal/container-linux/kubernetes/bootkube.tf +++ b/bare-metal/container-linux/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=593f0e3655294832122a133c08d0ed5c4f28f9b3" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=0a7c4fda3507b4a755c0c777d2e65d38bc152bbb" cluster_name = "${var.cluster_name}" api_servers = ["${var.k8s_domain_name}"] diff --git a/bare-metal/fedora-atomic/kubernetes/bootkube.tf b/bare-metal/fedora-atomic/kubernetes/bootkube.tf index 6b184533e..647451866 100644 --- a/bare-metal/fedora-atomic/kubernetes/bootkube.tf +++ b/bare-metal/fedora-atomic/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=593f0e3655294832122a133c08d0ed5c4f28f9b3" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=0a7c4fda3507b4a755c0c777d2e65d38bc152bbb" cluster_name = "${var.cluster_name}" api_servers = ["${var.k8s_domain_name}"] diff --git a/digital-ocean/container-linux/kubernetes/bootkube.tf b/digital-ocean/container-linux/kubernetes/bootkube.tf index 29ea717d7..f0add81ad 100644 --- a/digital-ocean/container-linux/kubernetes/bootkube.tf +++ b/digital-ocean/container-linux/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=593f0e3655294832122a133c08d0ed5c4f28f9b3" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=0a7c4fda3507b4a755c0c777d2e65d38bc152bbb" cluster_name = "${var.cluster_name}" api_servers = ["${format("%s.%s", var.cluster_name, var.dns_zone)}"] diff --git a/digital-ocean/fedora-atomic/kubernetes/bootkube.tf b/digital-ocean/fedora-atomic/kubernetes/bootkube.tf index 3c053b97e..9be511f9f 100644 --- a/digital-ocean/fedora-atomic/kubernetes/bootkube.tf +++ b/digital-ocean/fedora-atomic/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=593f0e3655294832122a133c08d0ed5c4f28f9b3" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=0a7c4fda3507b4a755c0c777d2e65d38bc152bbb" cluster_name = "${var.cluster_name}" api_servers = ["${format("%s.%s", var.cluster_name, var.dns_zone)}"] diff --git a/google-cloud/container-linux/kubernetes/bootkube.tf b/google-cloud/container-linux/kubernetes/bootkube.tf index 252f3ea63..95bc44cc8 100644 --- a/google-cloud/container-linux/kubernetes/bootkube.tf +++ b/google-cloud/container-linux/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=593f0e3655294832122a133c08d0ed5c4f28f9b3" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=0a7c4fda3507b4a755c0c777d2e65d38bc152bbb" cluster_name = "${var.cluster_name}" api_servers = ["${format("%s.%s", var.cluster_name, var.dns_zone)}"] diff --git a/google-cloud/fedora-atomic/kubernetes/bootkube.tf b/google-cloud/fedora-atomic/kubernetes/bootkube.tf index 586e7a21d..1c9f796da 100644 --- a/google-cloud/fedora-atomic/kubernetes/bootkube.tf +++ b/google-cloud/fedora-atomic/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=593f0e3655294832122a133c08d0ed5c4f28f9b3" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=0a7c4fda3507b4a755c0c777d2e65d38bc152bbb" cluster_name = "${var.cluster_name}" api_servers = ["${format("%s.%s", var.cluster_name, var.dns_zone)}"] From daee5a9d6059c247ea33b9ea3235d2d6c1caf724 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Mon, 25 Feb 2019 21:43:43 -0800 Subject: [PATCH 034/523] Update Grafana from v6.0.0-beta3 to v6.0.0 * https://github.com/grafana/grafana/releases/tag/v6.0.0 * http://docs.grafana.org/guides/whats-new-in-v6-0/ --- CHANGES.md | 2 +- addons/grafana/deployment.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index d2df3d705..d011f006b 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -29,7 +29,7 @@ Notable changes between versions. * Add `pod` name label to metrics discovered via service endpoints * Rename `kubernetes_namespace` label to `namespace` * Improve Grafana and dashboards ([#403](https://github.com/poseidon/typhoon/pull/403), [#404](https://github.com/poseidon/typhoon/pull/404)) - * Upgrade Grafana from v5.4.3 to 6.0.0-beta3 + * Upgrade Grafana from v5.4.3 to [v6.0.0](https://github.com/grafana/grafana/releases/tag/v6.0.0)! * Enable Grafana [Explore](http://docs.grafana.org/guides/whats-new-in-v6-0/#explore) UI as a Viewer (inspect/edit without saving) * Raise nginx-ingress liveness/readiness timeout to 5 seconds * Remove nginx-ingess default-backend ([#401](https://github.com/poseidon/typhoon/pull/401)) diff --git a/addons/grafana/deployment.yaml b/addons/grafana/deployment.yaml index 5c5998f0b..dfcfe9bc8 100644 --- a/addons/grafana/deployment.yaml +++ b/addons/grafana/deployment.yaml @@ -23,7 +23,7 @@ spec: spec: containers: - name: grafana - image: grafana/grafana:6.0.0-beta3 + image: grafana/grafana:6.0.0 env: - name: GF_PATHS_CONFIG value: "/etc/grafana/custom.ini" From 8ae552ebda2e348e6774ad843192fa67131b439c Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Wed, 27 Feb 2019 23:26:57 -0800 Subject: [PATCH 035/523] Update documentation for use with Ubiquiti EdgeOS * Show creation of a PXE-enabled network boot environment when using dnsmasq as the DHCP server * Recommend TFTP be served from /config/tftpboot since /config is preserved between firmware upgrades * Recommend compiling undionly.kpxe from source to enable TLS features * Add a note that equal-cost multi-path service IP routing (e.g. for ingress) requires EdgeOS v2.0. Previously, it was known that TLS handshakes couldn't be completed with packet balacing. I've verified this is no longer the case when using the v2.0 EdgeOS firmware, ECMP works as expected. --- docs/topics/hardware.md | 29 +++++++++++++++++++---------- 1 file changed, 19 insertions(+), 10 deletions(-) diff --git a/docs/topics/hardware.md b/docs/topics/hardware.md index 4715e0a84..385a1fee0 100644 --- a/docs/topics/hardware.md +++ b/docs/topics/hardware.md @@ -4,7 +4,7 @@ Typhoon ensures certain networking hardware integrates well with bare-metal Kube ## Ubiquiti -Ubiquiti EdgeRouters work well with bare-metal Kubernetes clusters. Knowledge about how to setup an EdgeRouter and use the CLI is required. +Ubiquiti EdgeRouters and EdgeOS work well with bare-metal Kubernetes clusters. Familiarity with EdgeRouter setup and CLI usage is required. ### PXE @@ -12,7 +12,7 @@ Ubiquiti EdgeRouters can provide a PXE-enabled network boot environment for clie #### ISC DHCP -Add a subnet parameter to the LAN DHCP server to include an ISC DHCP config file. +With ISC DHCP, add a subnet parameter to the LAN DHCP server to include an ISC DHCP config file. ``` configure @@ -21,7 +21,7 @@ set service dhcp-server shared-network-name NAME subnet SUBNET subnet-parameters commit-confirm ``` -Switch to root (i.e. `sudo -i`) and write the ISC DHCP config `/config/scripts/ipxe.conf`. iPXE client machines will chainload to `matchbox.example.com`, while non-iPXE clients will chainload to `undionly.kpxe` (requires TFTP to be enabled). +Switch to root (i.e. `sudo -i`) and write the ISC DHCP config `/config/scripts/ipxe.conf`. iPXE client machines will chainload to `matchbox.example.com`, while non-iPXE clients will chainload to `undionly.kpxe` (requires TFTP). ``` allow bootp; @@ -35,14 +35,23 @@ if exists user-class and option user-class = "iPXE" { } ``` +#### dnsmasq + +With dnsmasq for DHCP, add options to chainload PXE clients to iPXE `undionly.kpxe` (requires TFTP), tag iPXE clients, and chainload iPXE clients to `matchbox.example.com`. + +``` +set service dns forwarding options 'dhcp-userclass=set:ipxe,iPXE' +set service dns forwarding options 'pxe-service=tag:#ipxe,x86PC,PXE chainload to iPXE,undionly.kpxe' +set service dns forwarding options 'pxe-service=tag:ipxe,x86PC,iPXE,http://matchbox.example.com/boot.ipxe' +``` + ### TFTP -Use `dnsmasq` as a TFTP server to serve [undionly.kpxe](http://boot.ipxe.org/undionly.kpxe). +Use `dnsmasq` as a TFTP server to serve `undionly.kpxe`. Compiling from [source](https://github.com/ipxe/ipxe) with TLS support is recommended, but you may also download a [pre-compiled](http://boot.ipxe.org/undionly.kpxe) copy. ``` sudo -i -mkdir /var/lib/tftpboot -cd /var/lib/tftpboot +mkdir /config/tftpboot && cd /config/tftpboot curl http://boot.ipxe.org/undionly.kpxe -o undionly.kpxe ``` @@ -52,13 +61,10 @@ Add `dnsmasq` command line options to enable the TFTP file server. configure show service dns forwarding set service dns forwarding options enable-tftp -set service dns forwarding options tftp-root=/var/lib/tftpboot +set service dns forwarding options tftp-root=/config/tftpboot commit-confirm ``` -!!! warning - After firmware upgrades, the `/var/lib/tftpboot` directory will not exist and dnsmasq will not start properly. Repeat this process following an upgrade. - ### DHCP Assign static IPs to clients with known MAC addresses. This is called a static mapping by EdgeOS. Configure the router with the commands based on region inventory. @@ -106,6 +112,9 @@ set protocols static route 10.3.0.0/16 next-hop NODE_IP commit-confirm ``` +!!! note + Adding multiple next-hop nodes provides equal-cost multi-path (ECMP) routing. EdgeOS v2.0+ is required. The kernel in prior versions used flow-hash to balanced packets, whereas with v2.0, round-robin sessions are used. + ### Port Forwarding Expose the [Ingress Controller](/addons/ingress.md#bare-metal) by adding `port-forward` rules that DNAT a port on the router's WAN interface to an internal IP and port. By convention, a public Ingress controller is assigned a fixed service IP (e.g. 10.3.0.12). From f59830799856eddae8192f0e8bfede78e785509a Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Thu, 28 Feb 2019 22:42:04 -0800 Subject: [PATCH 036/523] Update Kubernetes from v1.13.3 to v1.13.4 * https://github.com/kubernetes/kubernetes/blob/master/CHANGELOG-1.13.md#v1134 --- CHANGES.md | 14 +++++++++----- README.md | 10 +++++----- aws/container-linux/kubernetes/README.md | 2 +- aws/container-linux/kubernetes/bootkube.tf | 2 +- .../kubernetes/cl/controller.yaml.tmpl | 2 +- .../kubernetes/workers/cl/worker.yaml.tmpl | 4 ++-- aws/fedora-atomic/kubernetes/README.md | 2 +- aws/fedora-atomic/kubernetes/bootkube.tf | 2 +- .../kubernetes/cloudinit/controller.yaml.tmpl | 2 +- .../workers/cloudinit/worker.yaml.tmpl | 2 +- azure/container-linux/kubernetes/README.md | 2 +- azure/container-linux/kubernetes/bootkube.tf | 2 +- .../kubernetes/cl/controller.yaml.tmpl | 2 +- .../kubernetes/workers/cl/worker.yaml.tmpl | 4 ++-- bare-metal/container-linux/kubernetes/README.md | 2 +- .../container-linux/kubernetes/bootkube.tf | 2 +- .../kubernetes/cl/controller.yaml.tmpl | 2 +- .../kubernetes/cl/worker.yaml.tmpl | 2 +- bare-metal/fedora-atomic/kubernetes/README.md | 2 +- bare-metal/fedora-atomic/kubernetes/bootkube.tf | 2 +- .../kubernetes/cloudinit/controller.yaml.tmpl | 2 +- .../kubernetes/cloudinit/worker.yaml.tmpl | 2 +- .../container-linux/kubernetes/README.md | 2 +- .../container-linux/kubernetes/bootkube.tf | 2 +- .../kubernetes/cl/controller.yaml.tmpl | 2 +- .../kubernetes/cl/worker.yaml.tmpl | 4 ++-- digital-ocean/fedora-atomic/kubernetes/README.md | 2 +- .../fedora-atomic/kubernetes/bootkube.tf | 2 +- .../kubernetes/cloudinit/controller.yaml.tmpl | 2 +- .../kubernetes/cloudinit/worker.yaml.tmpl | 2 +- docs/advanced/worker-pools.md | 16 ++++++++-------- docs/atomic/aws.md | 10 +++++----- docs/atomic/bare-metal.md | 10 +++++----- docs/atomic/digital-ocean.md | 10 +++++----- docs/atomic/google-cloud.md | 10 +++++----- docs/cl/aws.md | 10 +++++----- docs/cl/azure.md | 10 +++++----- docs/cl/bare-metal.md | 14 +++++++------- docs/cl/digital-ocean.md | 10 +++++----- docs/cl/google-cloud.md | 10 +++++----- docs/index.md | 10 +++++----- docs/topics/maintenance.md | 2 +- .../container-linux/kubernetes/README.md | 2 +- .../container-linux/kubernetes/bootkube.tf | 2 +- .../kubernetes/cl/controller.yaml.tmpl | 2 +- .../kubernetes/workers/cl/worker.yaml.tmpl | 4 ++-- google-cloud/fedora-atomic/kubernetes/README.md | 2 +- .../fedora-atomic/kubernetes/bootkube.tf | 2 +- .../kubernetes/cloudinit/controller.yaml.tmpl | 2 +- .../workers/cloudinit/worker.yaml.tmpl | 2 +- 50 files changed, 115 insertions(+), 111 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index d011f006b..1490d9a59 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -4,31 +4,35 @@ Notable changes between versions. ## Latest +## v1.13.4 + +* Kubernetes [v1.13.4](https://github.com/kubernetes/kubernetes/blob/master/CHANGELOG-1.13.md#v1134) * Update etcd from v3.3.11 to [v3.3.12](https://github.com/etcd-io/etcd/releases/tag/v3.3.12) * Update Calico from v3.5.0 to [v3.5.2](https://docs.projectcalico.org/v3.5/releases/) * Assign priorityClassNames to critical cluster and node components ([#406](https://github.com/poseidon/typhoon/pull/406)) - * Informs node out-of-resource eviction and scheduler preemption and ordering -* Add a CoreDNS readiness probe ([#410](https://github.com/poseidon/typhoon/pull/410)) + * Inform node out-of-resource eviction and scheduler preemption and ordering +* Add CoreDNS readiness probe ([#410](https://github.com/poseidon/typhoon/pull/410)) #### Bare-Metal * Recommend updating [terraform-provider-matchbox](https://github.com/coreos/terraform-provider-matchbox) plugin from v0.2.2 to [v0.2.3](https://github.com/coreos/terraform-provider-matchbox/releases/tag/v0.2.3) ([#402](https://github.com/poseidon/typhoon/pull/402)) +* Improve docs on using Ubiquiti EdgeOS with bare-metal clusters ([#413](https://github.com/poseidon/typhoon/pull/413)) #### Google Cloud * Support `terraform-provider-google` v2.0+ ([#407](https://github.com/poseidon/typhoon/pull/407)) - * Require `terraform-provider-google` v1.19+ (action required) + * Require `terraform-provider-google` v1.19+ (**action required**) * Set the minimum CPU platform to Intel Haswell ([#405](https://github.com/poseidon/typhoon/pull/405)) * Haswell or better is available in every zone (no price change) * A few zones still default to Sandy/Ivy Bridge (shifts in April 2019) #### Addons -* Improve Prometheus rules and alerts ([#404](https://github.com/poseidon/typhoon/pull/404)) +* Modernize Prometheus rules and alerts ([#404](https://github.com/poseidon/typhoon/pull/404)) * Drop extraneous metrics ([#397](https://github.com/poseidon/typhoon/pull/397)) * Add `pod` name label to metrics discovered via service endpoints * Rename `kubernetes_namespace` label to `namespace` -* Improve Grafana and dashboards ([#403](https://github.com/poseidon/typhoon/pull/403), [#404](https://github.com/poseidon/typhoon/pull/404)) +* Modernize Grafana and dashboards ([#403](https://github.com/poseidon/typhoon/pull/403), [#404](https://github.com/poseidon/typhoon/pull/404)) * Upgrade Grafana from v5.4.3 to [v6.0.0](https://github.com/grafana/grafana/releases/tag/v6.0.0)! * Enable Grafana [Explore](http://docs.grafana.org/guides/whats-new-in-v6-0/#explore) UI as a Viewer (inspect/edit without saving) * Raise nginx-ingress liveness/readiness timeout to 5 seconds diff --git a/README.md b/README.md index cd30ad0ae..ecec6b4aa 100644 --- a/README.md +++ b/README.md @@ -11,7 +11,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features -* Kubernetes v1.13.3 (upstream, via [kubernetes-incubator/bootkube](https://github.com/kubernetes-incubator/bootkube)) +* Kubernetes v1.13.4 (upstream, via [kubernetes-incubator/bootkube](https://github.com/kubernetes-incubator/bootkube)) * Single or multi-master, [Calico](https://www.projectcalico.org/) or [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/) * Advanced features like [worker pools](https://typhoon.psdn.io/advanced/worker-pools/), [preemptible](https://typhoon.psdn.io/cl/google-cloud/#preemption) workers, and [snippets](https://typhoon.psdn.io/advanced/customization/#container-linux) customization @@ -50,7 +50,7 @@ Define a Kubernetes cluster by using the Terraform module for your chosen platfo ```tf module "google-cloud-yavin" { - source = "git::https://github.com/poseidon/typhoon//google-cloud/container-linux/kubernetes?ref=v1.13.3" + source = "git::https://github.com/poseidon/typhoon//google-cloud/container-linux/kubernetes?ref=v1.13.4" providers = { google = "google.default" @@ -91,9 +91,9 @@ In 4-8 minutes (varies by platform), the cluster will be ready. This Google Clou $ export KUBECONFIG=/home/user/.secrets/clusters/yavin/auth/kubeconfig $ kubectl get nodes NAME ROLES STATUS AGE VERSION -yavin-controller-0.c.example-com.internal controller,master Ready 6m v1.13.3 -yavin-worker-jrbf.c.example-com.internal node Ready 5m v1.13.3 -yavin-worker-mzdm.c.example-com.internal node Ready 5m v1.13.3 +yavin-controller-0.c.example-com.internal controller,master Ready 6m v1.13.4 +yavin-worker-jrbf.c.example-com.internal node Ready 5m v1.13.4 +yavin-worker-mzdm.c.example-com.internal node Ready 5m v1.13.4 ``` List the pods. diff --git a/aws/container-linux/kubernetes/README.md b/aws/container-linux/kubernetes/README.md index 1d59ee293..909cc5959 100644 --- a/aws/container-linux/kubernetes/README.md +++ b/aws/container-linux/kubernetes/README.md @@ -11,7 +11,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features -* Kubernetes v1.13.3 (upstream, via [kubernetes-incubator/bootkube](https://github.com/kubernetes-incubator/bootkube)) +* Kubernetes v1.13.4 (upstream, via [kubernetes-incubator/bootkube](https://github.com/kubernetes-incubator/bootkube)) * Single or multi-master, [Calico](https://www.projectcalico.org/) or [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/) * Advanced features like [worker pools](https://typhoon.psdn.io/advanced/worker-pools/), [spot](https://typhoon.psdn.io/cl/aws/#spot) workers, and [snippets](https://typhoon.psdn.io/advanced/customization/#container-linux) customization diff --git a/aws/container-linux/kubernetes/bootkube.tf b/aws/container-linux/kubernetes/bootkube.tf index dbf5982b3..c06ddffe4 100644 --- a/aws/container-linux/kubernetes/bootkube.tf +++ b/aws/container-linux/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=0a7c4fda3507b4a755c0c777d2e65d38bc152bbb" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=953521dbba49eb6a39204f30a3978730eac01e11" cluster_name = "${var.cluster_name}" api_servers = ["${format("%s.%s", var.cluster_name, var.dns_zone)}"] diff --git a/aws/container-linux/kubernetes/cl/controller.yaml.tmpl b/aws/container-linux/kubernetes/cl/controller.yaml.tmpl index 63285ee56..f5d9472d3 100644 --- a/aws/container-linux/kubernetes/cl/controller.yaml.tmpl +++ b/aws/container-linux/kubernetes/cl/controller.yaml.tmpl @@ -123,7 +123,7 @@ storage: contents: inline: | KUBELET_IMAGE_URL=docker://k8s.gcr.io/hyperkube - KUBELET_IMAGE_TAG=v1.13.3 + KUBELET_IMAGE_TAG=v1.13.4 - path: /etc/sysctl.d/max-user-watches.conf filesystem: root contents: diff --git a/aws/container-linux/kubernetes/workers/cl/worker.yaml.tmpl b/aws/container-linux/kubernetes/workers/cl/worker.yaml.tmpl index 5bb864dd4..79f1bf39e 100644 --- a/aws/container-linux/kubernetes/workers/cl/worker.yaml.tmpl +++ b/aws/container-linux/kubernetes/workers/cl/worker.yaml.tmpl @@ -93,7 +93,7 @@ storage: contents: inline: | KUBELET_IMAGE_URL=docker://k8s.gcr.io/hyperkube - KUBELET_IMAGE_TAG=v1.13.3 + KUBELET_IMAGE_TAG=v1.13.4 - path: /etc/sysctl.d/max-user-watches.conf filesystem: root contents: @@ -111,7 +111,7 @@ storage: --volume config,kind=host,source=/etc/kubernetes \ --mount volume=config,target=/etc/kubernetes \ --insecure-options=image \ - docker://k8s.gcr.io/hyperkube:v1.13.3 \ + docker://k8s.gcr.io/hyperkube:v1.13.4 \ --net=host \ --dns=host \ --exec=/kubectl -- --kubeconfig=/etc/kubernetes/kubeconfig delete node $(hostname) diff --git a/aws/fedora-atomic/kubernetes/README.md b/aws/fedora-atomic/kubernetes/README.md index fb3363078..ec0102f49 100644 --- a/aws/fedora-atomic/kubernetes/README.md +++ b/aws/fedora-atomic/kubernetes/README.md @@ -11,7 +11,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features -* Kubernetes v1.13.3 (upstream, via [kubernetes-incubator/bootkube](https://github.com/kubernetes-incubator/bootkube)) +* Kubernetes v1.13.4 (upstream, via [kubernetes-incubator/bootkube](https://github.com/kubernetes-incubator/bootkube)) * Single or multi-master, [Calico](https://www.projectcalico.org/) or [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/) * Advanced features like [worker pools](https://typhoon.psdn.io/advanced/worker-pools/) and [spot](https://typhoon.psdn.io/cl/aws/#spot) workers diff --git a/aws/fedora-atomic/kubernetes/bootkube.tf b/aws/fedora-atomic/kubernetes/bootkube.tf index b35aab32c..4e2c66371 100644 --- a/aws/fedora-atomic/kubernetes/bootkube.tf +++ b/aws/fedora-atomic/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=0a7c4fda3507b4a755c0c777d2e65d38bc152bbb" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=953521dbba49eb6a39204f30a3978730eac01e11" cluster_name = "${var.cluster_name}" api_servers = ["${format("%s.%s", var.cluster_name, var.dns_zone)}"] diff --git a/aws/fedora-atomic/kubernetes/cloudinit/controller.yaml.tmpl b/aws/fedora-atomic/kubernetes/cloudinit/controller.yaml.tmpl index 483591e63..81f8b1784 100644 --- a/aws/fedora-atomic/kubernetes/cloudinit/controller.yaml.tmpl +++ b/aws/fedora-atomic/kubernetes/cloudinit/controller.yaml.tmpl @@ -79,7 +79,7 @@ runcmd: - [systemctl, daemon-reload] - [systemctl, restart, NetworkManager] - "atomic install --system --name=etcd quay.io/poseidon/etcd:v3.3.12" - - "atomic install --system --name=kubelet quay.io/poseidon/kubelet:v1.13.3" + - "atomic install --system --name=kubelet quay.io/poseidon/kubelet:v1.13.4" - "atomic install --system --name=bootkube quay.io/poseidon/bootkube:v0.14.0" - [systemctl, start, --no-block, etcd.service] - [systemctl, start, --no-block, kubelet.service] diff --git a/aws/fedora-atomic/kubernetes/workers/cloudinit/worker.yaml.tmpl b/aws/fedora-atomic/kubernetes/workers/cloudinit/worker.yaml.tmpl index c14590c19..da76beae5 100644 --- a/aws/fedora-atomic/kubernetes/workers/cloudinit/worker.yaml.tmpl +++ b/aws/fedora-atomic/kubernetes/workers/cloudinit/worker.yaml.tmpl @@ -54,7 +54,7 @@ bootcmd: runcmd: - [systemctl, daemon-reload] - [systemctl, restart, NetworkManager] - - "atomic install --system --name=kubelet quay.io/poseidon/kubelet:v1.13.3" + - "atomic install --system --name=kubelet quay.io/poseidon/kubelet:v1.13.4" - [systemctl, start, --no-block, kubelet.service] users: - default diff --git a/azure/container-linux/kubernetes/README.md b/azure/container-linux/kubernetes/README.md index 4735444da..098f5c035 100644 --- a/azure/container-linux/kubernetes/README.md +++ b/azure/container-linux/kubernetes/README.md @@ -11,7 +11,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features -* Kubernetes v1.13.3 (upstream, via [kubernetes-incubator/bootkube](https://github.com/kubernetes-incubator/bootkube)) +* Kubernetes v1.13.4 (upstream, via [kubernetes-incubator/bootkube](https://github.com/kubernetes-incubator/bootkube)) * Single or multi-master, [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled * Advanced features like [worker pools](https://typhoon.psdn.io/advanced/worker-pools/), [low-priority](https://typhoon.psdn.io/cl/azure/#low-priority) workers, and [snippets](https://typhoon.psdn.io/advanced/customization/#container-linux) customization diff --git a/azure/container-linux/kubernetes/bootkube.tf b/azure/container-linux/kubernetes/bootkube.tf index d7a1bb8c2..173fe9fa3 100644 --- a/azure/container-linux/kubernetes/bootkube.tf +++ b/azure/container-linux/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=0a7c4fda3507b4a755c0c777d2e65d38bc152bbb" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=953521dbba49eb6a39204f30a3978730eac01e11" cluster_name = "${var.cluster_name}" api_servers = ["${format("%s.%s", var.cluster_name, var.dns_zone)}"] diff --git a/azure/container-linux/kubernetes/cl/controller.yaml.tmpl b/azure/container-linux/kubernetes/cl/controller.yaml.tmpl index 63285ee56..f5d9472d3 100644 --- a/azure/container-linux/kubernetes/cl/controller.yaml.tmpl +++ b/azure/container-linux/kubernetes/cl/controller.yaml.tmpl @@ -123,7 +123,7 @@ storage: contents: inline: | KUBELET_IMAGE_URL=docker://k8s.gcr.io/hyperkube - KUBELET_IMAGE_TAG=v1.13.3 + KUBELET_IMAGE_TAG=v1.13.4 - path: /etc/sysctl.d/max-user-watches.conf filesystem: root contents: diff --git a/azure/container-linux/kubernetes/workers/cl/worker.yaml.tmpl b/azure/container-linux/kubernetes/workers/cl/worker.yaml.tmpl index decec0564..3312b7e11 100644 --- a/azure/container-linux/kubernetes/workers/cl/worker.yaml.tmpl +++ b/azure/container-linux/kubernetes/workers/cl/worker.yaml.tmpl @@ -93,7 +93,7 @@ storage: contents: inline: | KUBELET_IMAGE_URL=docker://k8s.gcr.io/hyperkube - KUBELET_IMAGE_TAG=v1.13.3 + KUBELET_IMAGE_TAG=v1.13.4 - path: /etc/sysctl.d/max-user-watches.conf filesystem: root contents: @@ -111,7 +111,7 @@ storage: --volume config,kind=host,source=/etc/kubernetes \ --mount volume=config,target=/etc/kubernetes \ --insecure-options=image \ - docker://k8s.gcr.io/hyperkube:v1.13.3 \ + docker://k8s.gcr.io/hyperkube:v1.13.4 \ --net=host \ --dns=host \ --exec=/kubectl -- --kubeconfig=/etc/kubernetes/kubeconfig delete node $(hostname | tr '[:upper:]' '[:lower:]') diff --git a/bare-metal/container-linux/kubernetes/README.md b/bare-metal/container-linux/kubernetes/README.md index 04af2f52d..2bfe8c031 100644 --- a/bare-metal/container-linux/kubernetes/README.md +++ b/bare-metal/container-linux/kubernetes/README.md @@ -11,7 +11,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features -* Kubernetes v1.13.3 (upstream, via [kubernetes-incubator/bootkube](https://github.com/kubernetes-incubator/bootkube)) +* Kubernetes v1.13.4 (upstream, via [kubernetes-incubator/bootkube](https://github.com/kubernetes-incubator/bootkube)) * Single or multi-master, [Calico](https://www.projectcalico.org/) or [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/) * Advanced features like [snippets](https://typhoon.psdn.io/advanced/customization/#container-linux) customization diff --git a/bare-metal/container-linux/kubernetes/bootkube.tf b/bare-metal/container-linux/kubernetes/bootkube.tf index c6cf5804e..9686c43c7 100644 --- a/bare-metal/container-linux/kubernetes/bootkube.tf +++ b/bare-metal/container-linux/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=0a7c4fda3507b4a755c0c777d2e65d38bc152bbb" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=953521dbba49eb6a39204f30a3978730eac01e11" cluster_name = "${var.cluster_name}" api_servers = ["${var.k8s_domain_name}"] diff --git a/bare-metal/container-linux/kubernetes/cl/controller.yaml.tmpl b/bare-metal/container-linux/kubernetes/cl/controller.yaml.tmpl index 6a1c80e95..00a27d380 100644 --- a/bare-metal/container-linux/kubernetes/cl/controller.yaml.tmpl +++ b/bare-metal/container-linux/kubernetes/cl/controller.yaml.tmpl @@ -128,7 +128,7 @@ storage: contents: inline: | KUBELET_IMAGE_URL=docker://k8s.gcr.io/hyperkube - KUBELET_IMAGE_TAG=v1.13.3 + KUBELET_IMAGE_TAG=v1.13.4 - path: /etc/hostname filesystem: root mode: 0644 diff --git a/bare-metal/container-linux/kubernetes/cl/worker.yaml.tmpl b/bare-metal/container-linux/kubernetes/cl/worker.yaml.tmpl index a3f13d03b..0e293a7a6 100644 --- a/bare-metal/container-linux/kubernetes/cl/worker.yaml.tmpl +++ b/bare-metal/container-linux/kubernetes/cl/worker.yaml.tmpl @@ -89,7 +89,7 @@ storage: contents: inline: | KUBELET_IMAGE_URL=docker://k8s.gcr.io/hyperkube - KUBELET_IMAGE_TAG=v1.13.3 + KUBELET_IMAGE_TAG=v1.13.4 - path: /etc/hostname filesystem: root mode: 0644 diff --git a/bare-metal/fedora-atomic/kubernetes/README.md b/bare-metal/fedora-atomic/kubernetes/README.md index 4d343afda..cb1fb55bc 100644 --- a/bare-metal/fedora-atomic/kubernetes/README.md +++ b/bare-metal/fedora-atomic/kubernetes/README.md @@ -11,7 +11,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features -* Kubernetes v1.13.3 (upstream, via [kubernetes-incubator/bootkube](https://github.com/kubernetes-incubator/bootkube)) +* Kubernetes v1.13.4 (upstream, via [kubernetes-incubator/bootkube](https://github.com/kubernetes-incubator/bootkube)) * Single or multi-master, [Calico](https://www.projectcalico.org/) or [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/) * Ready for Ingress, Prometheus, Grafana, and other optional [addons](https://typhoon.psdn.io/addons/overview/) diff --git a/bare-metal/fedora-atomic/kubernetes/bootkube.tf b/bare-metal/fedora-atomic/kubernetes/bootkube.tf index 647451866..c01246730 100644 --- a/bare-metal/fedora-atomic/kubernetes/bootkube.tf +++ b/bare-metal/fedora-atomic/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=0a7c4fda3507b4a755c0c777d2e65d38bc152bbb" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=953521dbba49eb6a39204f30a3978730eac01e11" cluster_name = "${var.cluster_name}" api_servers = ["${var.k8s_domain_name}"] diff --git a/bare-metal/fedora-atomic/kubernetes/cloudinit/controller.yaml.tmpl b/bare-metal/fedora-atomic/kubernetes/cloudinit/controller.yaml.tmpl index 2cba4d893..66600c7f8 100644 --- a/bare-metal/fedora-atomic/kubernetes/cloudinit/controller.yaml.tmpl +++ b/bare-metal/fedora-atomic/kubernetes/cloudinit/controller.yaml.tmpl @@ -85,7 +85,7 @@ runcmd: - [systemctl, restart, NetworkManager] - [hostnamectl, set-hostname, ${domain_name}] - "atomic install --system --name=etcd quay.io/poseidon/etcd:v3.3.12" - - "atomic install --system --name=kubelet quay.io/poseidon/kubelet:v1.13.3" + - "atomic install --system --name=kubelet quay.io/poseidon/kubelet:v1.13.4" - "atomic install --system --name=bootkube quay.io/poseidon/bootkube:v0.14.0" - [systemctl, start, --no-block, etcd.service] - [systemctl, enable, kubelet.path] diff --git a/bare-metal/fedora-atomic/kubernetes/cloudinit/worker.yaml.tmpl b/bare-metal/fedora-atomic/kubernetes/cloudinit/worker.yaml.tmpl index 52614004e..5e9d5c50e 100644 --- a/bare-metal/fedora-atomic/kubernetes/cloudinit/worker.yaml.tmpl +++ b/bare-metal/fedora-atomic/kubernetes/cloudinit/worker.yaml.tmpl @@ -60,7 +60,7 @@ runcmd: - [systemctl, daemon-reload] - [systemctl, restart, NetworkManager] - [hostnamectl, set-hostname, ${domain_name}] - - "atomic install --system --name=kubelet quay.io/poseidon/kubelet:v1.13.3" + - "atomic install --system --name=kubelet quay.io/poseidon/kubelet:v1.13.4" - [systemctl, enable, kubelet.path] - [systemctl, start, --no-block, kubelet.path] users: diff --git a/digital-ocean/container-linux/kubernetes/README.md b/digital-ocean/container-linux/kubernetes/README.md index be9f9c36b..837e27714 100644 --- a/digital-ocean/container-linux/kubernetes/README.md +++ b/digital-ocean/container-linux/kubernetes/README.md @@ -11,7 +11,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features -* Kubernetes v1.13.3 (upstream, via [kubernetes-incubator/bootkube](https://github.com/kubernetes-incubator/bootkube)) +* Kubernetes v1.13.4 (upstream, via [kubernetes-incubator/bootkube](https://github.com/kubernetes-incubator/bootkube)) * Single or multi-master, [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled * Advanced features like [snippets](https://typhoon.psdn.io/advanced/customization/#container-linux) customization diff --git a/digital-ocean/container-linux/kubernetes/bootkube.tf b/digital-ocean/container-linux/kubernetes/bootkube.tf index f0add81ad..3fdc95251 100644 --- a/digital-ocean/container-linux/kubernetes/bootkube.tf +++ b/digital-ocean/container-linux/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=0a7c4fda3507b4a755c0c777d2e65d38bc152bbb" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=953521dbba49eb6a39204f30a3978730eac01e11" cluster_name = "${var.cluster_name}" api_servers = ["${format("%s.%s", var.cluster_name, var.dns_zone)}"] diff --git a/digital-ocean/container-linux/kubernetes/cl/controller.yaml.tmpl b/digital-ocean/container-linux/kubernetes/cl/controller.yaml.tmpl index d8219f394..9586bf56f 100644 --- a/digital-ocean/container-linux/kubernetes/cl/controller.yaml.tmpl +++ b/digital-ocean/container-linux/kubernetes/cl/controller.yaml.tmpl @@ -125,7 +125,7 @@ storage: contents: inline: | KUBELET_IMAGE_URL=docker://k8s.gcr.io/hyperkube - KUBELET_IMAGE_TAG=v1.13.3 + KUBELET_IMAGE_TAG=v1.13.4 - path: /etc/sysctl.d/max-user-watches.conf filesystem: root contents: diff --git a/digital-ocean/container-linux/kubernetes/cl/worker.yaml.tmpl b/digital-ocean/container-linux/kubernetes/cl/worker.yaml.tmpl index d10fadf5c..f1a58aaf1 100644 --- a/digital-ocean/container-linux/kubernetes/cl/worker.yaml.tmpl +++ b/digital-ocean/container-linux/kubernetes/cl/worker.yaml.tmpl @@ -95,7 +95,7 @@ storage: contents: inline: | KUBELET_IMAGE_URL=docker://k8s.gcr.io/hyperkube - KUBELET_IMAGE_TAG=v1.13.3 + KUBELET_IMAGE_TAG=v1.13.4 - path: /etc/sysctl.d/max-user-watches.conf filesystem: root contents: @@ -113,7 +113,7 @@ storage: --volume config,kind=host,source=/etc/kubernetes \ --mount volume=config,target=/etc/kubernetes \ --insecure-options=image \ - docker://k8s.gcr.io/hyperkube:v1.13.3 \ + docker://k8s.gcr.io/hyperkube:v1.13.4 \ --net=host \ --dns=host \ --exec=/kubectl -- --kubeconfig=/etc/kubernetes/kubeconfig delete node $(hostname) diff --git a/digital-ocean/fedora-atomic/kubernetes/README.md b/digital-ocean/fedora-atomic/kubernetes/README.md index f5b86bf4c..c02cb1bcc 100644 --- a/digital-ocean/fedora-atomic/kubernetes/README.md +++ b/digital-ocean/fedora-atomic/kubernetes/README.md @@ -11,7 +11,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features -* Kubernetes v1.13.3 (upstream, via [kubernetes-incubator/bootkube](https://github.com/kubernetes-incubator/bootkube)) +* Kubernetes v1.13.4 (upstream, via [kubernetes-incubator/bootkube](https://github.com/kubernetes-incubator/bootkube)) * Single or multi-master, [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled * Ready for Ingress, Prometheus, Grafana, and other optional [addons](https://typhoon.psdn.io/addons/overview/) diff --git a/digital-ocean/fedora-atomic/kubernetes/bootkube.tf b/digital-ocean/fedora-atomic/kubernetes/bootkube.tf index 9be511f9f..f1776386d 100644 --- a/digital-ocean/fedora-atomic/kubernetes/bootkube.tf +++ b/digital-ocean/fedora-atomic/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=0a7c4fda3507b4a755c0c777d2e65d38bc152bbb" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=953521dbba49eb6a39204f30a3978730eac01e11" cluster_name = "${var.cluster_name}" api_servers = ["${format("%s.%s", var.cluster_name, var.dns_zone)}"] diff --git a/digital-ocean/fedora-atomic/kubernetes/cloudinit/controller.yaml.tmpl b/digital-ocean/fedora-atomic/kubernetes/cloudinit/controller.yaml.tmpl index 74798986c..9f38161c9 100644 --- a/digital-ocean/fedora-atomic/kubernetes/cloudinit/controller.yaml.tmpl +++ b/digital-ocean/fedora-atomic/kubernetes/cloudinit/controller.yaml.tmpl @@ -76,7 +76,7 @@ bootcmd: runcmd: - [systemctl, daemon-reload] - "atomic install --system --name=etcd quay.io/poseidon/etcd:v3.3.12" - - "atomic install --system --name=kubelet quay.io/poseidon/kubelet:v1.13.3" + - "atomic install --system --name=kubelet quay.io/poseidon/kubelet:v1.13.4" - "atomic install --system --name=bootkube quay.io/poseidon/bootkube:v0.14.0" - [systemctl, start, --no-block, etcd.service] - [systemctl, enable, kubelet.path] diff --git a/digital-ocean/fedora-atomic/kubernetes/cloudinit/worker.yaml.tmpl b/digital-ocean/fedora-atomic/kubernetes/cloudinit/worker.yaml.tmpl index 56f43b4d1..223e54a77 100644 --- a/digital-ocean/fedora-atomic/kubernetes/cloudinit/worker.yaml.tmpl +++ b/digital-ocean/fedora-atomic/kubernetes/cloudinit/worker.yaml.tmpl @@ -51,7 +51,7 @@ bootcmd: - [modprobe, ip_vs] runcmd: - [systemctl, daemon-reload] - - "atomic install --system --name=kubelet quay.io/poseidon/kubelet:v1.13.3" + - "atomic install --system --name=kubelet quay.io/poseidon/kubelet:v1.13.4" - [systemctl, enable, kubelet.path] - [systemctl, start, --no-block, kubelet.path] users: diff --git a/docs/advanced/worker-pools.md b/docs/advanced/worker-pools.md index 36cadd37d..c17861d43 100644 --- a/docs/advanced/worker-pools.md +++ b/docs/advanced/worker-pools.md @@ -16,7 +16,7 @@ Create a cluster following the AWS [tutorial](../cl/aws.md#cluster). Define a wo ```tf module "tempest-worker-pool" { - source = "git::https://github.com/poseidon/typhoon//aws/container-linux/kubernetes/workers?ref=v1.13.3" + source = "git::https://github.com/poseidon/typhoon//aws/container-linux/kubernetes/workers?ref=v1.13.4" providers = { aws = "aws.default" @@ -82,7 +82,7 @@ Create a cluster following the Azure [tutorial](../cl/azure.md#cluster). Define ```tf module "ramius-worker-pool" { - source = "git::https://github.com/poseidon/typhoon//azure/container-linux/kubernetes/workers?ref=v1.13.3" + source = "git::https://github.com/poseidon/typhoon//azure/container-linux/kubernetes/workers?ref=v1.13.4" providers = { azurerm = "azurerm.default" @@ -152,7 +152,7 @@ Create a cluster following the Google Cloud [tutorial](../cl/google-cloud.md#clu ```tf module "yavin-worker-pool" { - source = "git::https://github.com/poseidon/typhoon//google-cloud/container-linux/kubernetes/workers?ref=v1.13.3" + source = "git::https://github.com/poseidon/typhoon//google-cloud/container-linux/kubernetes/workers?ref=v1.13.4" providers = { google = "google.default" @@ -187,11 +187,11 @@ Verify a managed instance group of workers joins the cluster within a few minute ``` $ kubectl get nodes NAME STATUS AGE VERSION -yavin-controller-0.c.example-com.internal Ready 6m v1.13.3 -yavin-worker-jrbf.c.example-com.internal Ready 5m v1.13.3 -yavin-worker-mzdm.c.example-com.internal Ready 5m v1.13.3 -yavin-16x-worker-jrbf.c.example-com.internal Ready 3m v1.13.3 -yavin-16x-worker-mzdm.c.example-com.internal Ready 3m v1.13.3 +yavin-controller-0.c.example-com.internal Ready 6m v1.13.4 +yavin-worker-jrbf.c.example-com.internal Ready 5m v1.13.4 +yavin-worker-mzdm.c.example-com.internal Ready 5m v1.13.4 +yavin-16x-worker-jrbf.c.example-com.internal Ready 3m v1.13.4 +yavin-16x-worker-mzdm.c.example-com.internal Ready 3m v1.13.4 ``` ### Variables diff --git a/docs/atomic/aws.md b/docs/atomic/aws.md index 5a4699571..695f45a79 100644 --- a/docs/atomic/aws.md +++ b/docs/atomic/aws.md @@ -3,7 +3,7 @@ !!! danger Typhoon for Fedora Atomic is alpha. Expect rough edges and changes. -In this tutorial, we'll create a Kubernetes v1.13.3 cluster on AWS with Fedora Atomic. +In this tutorial, we'll create a Kubernetes v1.13.4 cluster on AWS with Fedora Atomic. We'll declare a Kubernetes cluster using the Typhoon Terraform module. Then apply the changes to create a VPC, gateway, subnets, security groups, controller instances, worker auto-scaling group, network load balancer, and TLS assets. Instances are provisioned on first boot with cloud-init. @@ -83,7 +83,7 @@ Define a Kubernetes cluster using the module `aws/fedora-atomic/kubernetes`. ```tf module "aws-tempest" { - source = "git::https://github.com/poseidon/typhoon//aws/fedora-atomic/kubernetes?ref=v1.13.3" + source = "git::https://github.com/poseidon/typhoon//aws/fedora-atomic/kubernetes?ref=v1.13.4" providers = { aws = "aws.default" @@ -156,9 +156,9 @@ In 5-10 minutes, the Kubernetes cluster will be ready. $ export KUBECONFIG=/home/user/.secrets/clusters/tempest/auth/kubeconfig $ kubectl get nodes NAME STATUS ROLES AGE VERSION -ip-10-0-3-155 Ready controller,master 10m v1.13.3 -ip-10-0-26-65 Ready node 10m v1.13.3 -ip-10-0-41-21 Ready node 10m v1.13.3 +ip-10-0-3-155 Ready controller,master 10m v1.13.4 +ip-10-0-26-65 Ready node 10m v1.13.4 +ip-10-0-41-21 Ready node 10m v1.13.4 ``` List the pods. diff --git a/docs/atomic/bare-metal.md b/docs/atomic/bare-metal.md index df9bbb09d..a2151208d 100644 --- a/docs/atomic/bare-metal.md +++ b/docs/atomic/bare-metal.md @@ -3,7 +3,7 @@ !!! danger Typhoon for Fedora Atomic is alpha. Expect rough edges and changes. -In this tutorial, we'll network boot and provision a Kubernetes v1.13.3 cluster on bare-metal with Fedora Atomic. +In this tutorial, we'll network boot and provision a Kubernetes v1.13.4 cluster on bare-metal with Fedora Atomic. First, we'll deploy a [Matchbox](https://github.com/coreos/matchbox) service and setup a network boot environment. Then, we'll declare a Kubernetes cluster using the Typhoon Terraform module and power on machines. On PXE boot, machines will install Fedora Atomic via kickstart, reboot into the disk install, and provision themselves as Kubernetes controllers or workers via cloud-init. @@ -228,7 +228,7 @@ Define a Kubernetes cluster using the module `bare-metal/fedora-atomic/kubernete ```tf module "bare-metal-mercury" { - source = "git::https://github.com/poseidon/typhoon//bare-metal/fedora-atomic/kubernetes?ref=v1.13.3" + source = "git::https://github.com/poseidon/typhoon//bare-metal/fedora-atomic/kubernetes?ref=v1.13.4" providers = { local = "local.default" @@ -354,9 +354,9 @@ bootkube[5]: Tearing down temporary bootstrap control plane... $ export KUBECONFIG=/home/user/.secrets/clusters/mercury/auth/kubeconfig $ kubectl get nodes NAME STATUS ROLES AGE VERSION -node1.example.com Ready controller,master 10m v1.13.3 -node2.example.com Ready node 10m v1.13.3 -node3.example.com Ready node 10m v1.13.3 +node1.example.com Ready controller,master 10m v1.13.4 +node2.example.com Ready node 10m v1.13.4 +node3.example.com Ready node 10m v1.13.4 ``` List the pods. diff --git a/docs/atomic/digital-ocean.md b/docs/atomic/digital-ocean.md index 383de3979..6a4ad686a 100644 --- a/docs/atomic/digital-ocean.md +++ b/docs/atomic/digital-ocean.md @@ -3,7 +3,7 @@ !!! danger Typhoon for Fedora Atomic is alpha. Expect rough edges and changes. -In this tutorial, we'll create a Kubernetes v1.13.3 cluster on DigitalOcean with Fedora Atomic. +In this tutorial, we'll create a Kubernetes v1.13.4 cluster on DigitalOcean with Fedora Atomic. We'll declare a Kubernetes cluster using the Typhoon Terraform module. Then apply the changes to create controller droplets, worker droplets, DNS records, tags, and TLS assets. Instances are provisioned on first boot with cloud-init. @@ -77,7 +77,7 @@ Define a Kubernetes cluster using the module `digital-ocean/fedora-atomic/kubern ```tf module "digital-ocean-nemo" { - source = "git::https://github.com/poseidon/typhoon//digital-ocean/fedora-atomic/kubernetes?ref=v1.13.3" + source = "git::https://github.com/poseidon/typhoon//digital-ocean/fedora-atomic/kubernetes?ref=v1.13.4" providers = { digitalocean = "digitalocean.default" @@ -152,9 +152,9 @@ In 3-6 minutes, the Kubernetes cluster will be ready. $ export KUBECONFIG=/home/user/.secrets/clusters/nemo/auth/kubeconfig $ kubectl get nodes NAME STATUS ROLES AGE VERSION -nemo-controller-0 Ready controller,master 10m v1.13.3 -nemo-worker-0 Ready node 10m v1.13.3 -nemo-worker-1 Ready node 10m v1.13.3 +nemo-controller-0 Ready controller,master 10m v1.13.4 +nemo-worker-0 Ready node 10m v1.13.4 +nemo-worker-1 Ready node 10m v1.13.4 ``` List the pods. diff --git a/docs/atomic/google-cloud.md b/docs/atomic/google-cloud.md index 130cc9c87..d96267685 100644 --- a/docs/atomic/google-cloud.md +++ b/docs/atomic/google-cloud.md @@ -3,7 +3,7 @@ !!! danger Typhoon for Fedora Atomic is alpha. Fedora does not publish official images for Google Cloud so you must prepare them yourself. Expect rough edges and changes. -In this tutorial, we'll create a Kubernetes v1.13.3 cluster on Google Compute Engine with Fedora Atomic. +In this tutorial, we'll create a Kubernetes v1.13.4 cluster on Google Compute Engine with Fedora Atomic. We'll declare a Kubernetes cluster using the Typhoon Terraform module. Then apply the changes to create a network, firewall rules, health checks, controller instances, worker managed instance group, load balancers, and TLS assets. Instances are provisioned on first boot with cloud-init. @@ -121,7 +121,7 @@ Define a Kubernetes cluster using the module `google-cloud/fedora-atomic/kuberne ```tf module "google-cloud-yavin" { - source = "git::https://github.com/poseidon/typhoon//google-cloud/fedora-atomic/kubernetes?ref=v1.13.3" + source = "git::https://github.com/poseidon/typhoon//google-cloud/fedora-atomic/kubernetes?ref=v1.13.4" providers = { google = "google.default" @@ -197,9 +197,9 @@ In 5-10 minutes, the Kubernetes cluster will be ready. $ export KUBECONFIG=/home/user/.secrets/clusters/yavin/auth/kubeconfig $ kubectl get nodes NAME ROLES STATUS AGE VERSION -yavin-controller-0.c.example-com.internal controller,master Ready 6m v1.13.3 -yavin-worker-jrbf.c.example-com.internal node Ready 5m v1.13.3 -yavin-worker-mzdm.c.example-com.internal node Ready 5m v1.13.3 +yavin-controller-0.c.example-com.internal controller,master Ready 6m v1.13.4 +yavin-worker-jrbf.c.example-com.internal node Ready 5m v1.13.4 +yavin-worker-mzdm.c.example-com.internal node Ready 5m v1.13.4 ``` List the pods. diff --git a/docs/cl/aws.md b/docs/cl/aws.md index b1f8ab389..0339460bd 100644 --- a/docs/cl/aws.md +++ b/docs/cl/aws.md @@ -1,6 +1,6 @@ # AWS -In this tutorial, we'll create a Kubernetes v1.13.3 cluster on AWS with Container Linux. +In this tutorial, we'll create a Kubernetes v1.13.4 cluster on AWS with Container Linux. We'll declare a Kubernetes cluster using the Typhoon Terraform module. Then apply the changes to create a VPC, gateway, subnets, security groups, controller instances, worker auto-scaling group, network load balancer, and TLS assets. @@ -92,7 +92,7 @@ Define a Kubernetes cluster using the module `aws/container-linux/kubernetes`. ```tf module "aws-tempest" { - source = "git::https://github.com/poseidon/typhoon//aws/container-linux/kubernetes?ref=v1.13.3" + source = "git::https://github.com/poseidon/typhoon//aws/container-linux/kubernetes?ref=v1.13.4" providers = { aws = "aws.default" @@ -165,9 +165,9 @@ In 4-8 minutes, the Kubernetes cluster will be ready. $ export KUBECONFIG=/home/user/.secrets/clusters/tempest/auth/kubeconfig $ kubectl get nodes NAME STATUS ROLES AGE VERSION -ip-10-0-3-155 Ready controller,master 10m v1.13.3 -ip-10-0-26-65 Ready node 10m v1.13.3 -ip-10-0-41-21 Ready node 10m v1.13.3 +ip-10-0-3-155 Ready controller,master 10m v1.13.4 +ip-10-0-26-65 Ready node 10m v1.13.4 +ip-10-0-41-21 Ready node 10m v1.13.4 ``` List the pods. diff --git a/docs/cl/azure.md b/docs/cl/azure.md index b992412fd..299b1d656 100644 --- a/docs/cl/azure.md +++ b/docs/cl/azure.md @@ -3,7 +3,7 @@ !!! danger Typhoon for Azure is alpha. For production, use AWS, Google Cloud, or bare-metal. As Azure matures, check [errata](https://github.com/poseidon/typhoon/wiki/Errata) for known shortcomings. -In this tutorial, we'll create a Kubernetes v1.13.3 cluster on Azure with Container Linux. +In this tutorial, we'll create a Kubernetes v1.13.4 cluster on Azure with Container Linux. We'll declare a Kubernetes cluster using the Typhoon Terraform module. Then apply the changes to create a resource group, virtual network, subnets, security groups, controller availability set, worker scale set, load balancer, and TLS assets. @@ -87,7 +87,7 @@ Define a Kubernetes cluster using the module `azure/container-linux/kubernetes`. ```tf module "azure-ramius" { - source = "git::https://github.com/poseidon/typhoon//azure/container-linux/kubernetes?ref=v1.13.3" + source = "git::https://github.com/poseidon/typhoon//azure/container-linux/kubernetes?ref=v1.13.4" providers = { azurerm = "azurerm.default" @@ -161,9 +161,9 @@ In 4-8 minutes, the Kubernetes cluster will be ready. $ export KUBECONFIG=/home/user/.secrets/clusters/ramius/auth/kubeconfig $ kubectl get nodes NAME STATUS ROLES AGE VERSION -ramius-controller-0 Ready controller,master 24m v1.13.3 -ramius-worker-000001 Ready node 25m v1.13.3 -ramius-worker-000002 Ready node 24m v1.13.3 +ramius-controller-0 Ready controller,master 24m v1.13.4 +ramius-worker-000001 Ready node 25m v1.13.4 +ramius-worker-000002 Ready node 24m v1.13.4 ``` List the pods. diff --git a/docs/cl/bare-metal.md b/docs/cl/bare-metal.md index af1b518ec..970547fd8 100644 --- a/docs/cl/bare-metal.md +++ b/docs/cl/bare-metal.md @@ -1,6 +1,6 @@ # Bare-Metal -In this tutorial, we'll network boot and provision a Kubernetes v1.13.3 cluster on bare-metal with Container Linux. +In this tutorial, we'll network boot and provision a Kubernetes v1.13.4 cluster on bare-metal with Container Linux. First, we'll deploy a [Matchbox](https://github.com/coreos/matchbox) service and setup a network boot environment. Then, we'll declare a Kubernetes cluster using the Typhoon Terraform module and power on machines. On PXE boot, machines will install Container Linux to disk, reboot into the disk install, and provision themselves as Kubernetes controllers or workers via Ignition. @@ -179,7 +179,7 @@ Define a Kubernetes cluster using the module `bare-metal/container-linux/kuberne ```tf module "bare-metal-mercury" { - source = "git::https://github.com/poseidon/typhoon//bare-metal/container-linux/kubernetes?ref=v1.13.3" + source = "git::https://github.com/poseidon/typhoon//bare-metal/container-linux/kubernetes?ref=v1.13.4" providers = { local = "local.default" @@ -288,9 +288,9 @@ Apply complete! Resources: 55 added, 0 changed, 0 destroyed. To watch the install to disk (until machines reboot from disk), SSH to port 2222. ``` -# before v1.13.3 +# before v1.13.4 $ ssh debug@node1.example.com -# after v1.13.3 +# after v1.13.4 $ ssh -p 2222 core@node1.example.com ``` @@ -315,9 +315,9 @@ bootkube[5]: Tearing down temporary bootstrap control plane... $ export KUBECONFIG=/home/user/.secrets/clusters/mercury/auth/kubeconfig $ kubectl get nodes NAME STATUS ROLES AGE VERSION -node1.example.com Ready controller,master 10m v1.13.3 -node2.example.com Ready node 10m v1.13.3 -node3.example.com Ready node 10m v1.13.3 +node1.example.com Ready controller,master 10m v1.13.4 +node2.example.com Ready node 10m v1.13.4 +node3.example.com Ready node 10m v1.13.4 ``` List the pods. diff --git a/docs/cl/digital-ocean.md b/docs/cl/digital-ocean.md index 25d12a3b7..fc4937dae 100644 --- a/docs/cl/digital-ocean.md +++ b/docs/cl/digital-ocean.md @@ -1,6 +1,6 @@ # Digital Ocean -In this tutorial, we'll create a Kubernetes v1.13.3 cluster on DigitalOcean with Container Linux. +In this tutorial, we'll create a Kubernetes v1.13.4 cluster on DigitalOcean with Container Linux. We'll declare a Kubernetes cluster using the Typhoon Terraform module. Then apply the changes to create controller droplets, worker droplets, DNS records, tags, and TLS assets. @@ -86,7 +86,7 @@ Define a Kubernetes cluster using the module `digital-ocean/container-linux/kube ```tf module "digital-ocean-nemo" { - source = "git::https://github.com/poseidon/typhoon//digital-ocean/container-linux/kubernetes?ref=v1.13.3" + source = "git::https://github.com/poseidon/typhoon//digital-ocean/container-linux/kubernetes?ref=v1.13.4" providers = { digitalocean = "digitalocean.default" @@ -160,9 +160,9 @@ In 3-6 minutes, the Kubernetes cluster will be ready. $ export KUBECONFIG=/home/user/.secrets/clusters/nemo/auth/kubeconfig $ kubectl get nodes NAME STATUS ROLES AGE VERSION -nemo-controller-0 Ready controller,master 10m v1.13.3 -nemo-worker-0 Ready node 10m v1.13.3 -nemo-worker-1 Ready node 10m v1.13.3 +nemo-controller-0 Ready controller,master 10m v1.13.4 +nemo-worker-0 Ready node 10m v1.13.4 +nemo-worker-1 Ready node 10m v1.13.4 ``` List the pods. diff --git a/docs/cl/google-cloud.md b/docs/cl/google-cloud.md index 3d1d411c5..d7be23301 100644 --- a/docs/cl/google-cloud.md +++ b/docs/cl/google-cloud.md @@ -1,6 +1,6 @@ # Google Cloud -In this tutorial, we'll create a Kubernetes v1.13.3 cluster on Google Compute Engine with Container Linux. +In this tutorial, we'll create a Kubernetes v1.13.4 cluster on Google Compute Engine with Container Linux. We'll declare a Kubernetes cluster using the Typhoon Terraform module. Then apply the changes to create a network, firewall rules, health checks, controller instances, worker managed instance group, load balancers, and TLS assets. @@ -93,7 +93,7 @@ Define a Kubernetes cluster using the module `google-cloud/container-linux/kuber ```tf module "google-cloud-yavin" { - source = "git::https://github.com/poseidon/typhoon//google-cloud/container-linux/kubernetes?ref=v1.13.3" + source = "git::https://github.com/poseidon/typhoon//google-cloud/container-linux/kubernetes?ref=v1.13.4" providers = { google = "google.default" @@ -168,9 +168,9 @@ In 4-8 minutes, the Kubernetes cluster will be ready. $ export KUBECONFIG=/home/user/.secrets/clusters/yavin/auth/kubeconfig $ kubectl get nodes NAME ROLES STATUS AGE VERSION -yavin-controller-0.c.example-com.internal controller,master Ready 6m v1.13.3 -yavin-worker-jrbf.c.example-com.internal node Ready 5m v1.13.3 -yavin-worker-mzdm.c.example-com.internal node Ready 5m v1.13.3 +yavin-controller-0.c.example-com.internal controller,master Ready 6m v1.13.4 +yavin-worker-jrbf.c.example-com.internal node Ready 5m v1.13.4 +yavin-worker-mzdm.c.example-com.internal node Ready 5m v1.13.4 ``` List the pods. diff --git a/docs/index.md b/docs/index.md index 586a36133..ed7b2118c 100644 --- a/docs/index.md +++ b/docs/index.md @@ -11,7 +11,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features -* Kubernetes v1.13.3 (upstream, via [kubernetes-incubator/bootkube](https://github.com/kubernetes-incubator/bootkube)) +* Kubernetes v1.13.4 (upstream, via [kubernetes-incubator/bootkube](https://github.com/kubernetes-incubator/bootkube)) * Single or multi-master, [Calico](https://www.projectcalico.org/) or [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/) * Advanced features like [worker pools](advanced/worker-pools/), [preemptible](cl/google-cloud/#preemption) workers, and [snippets](advanced/customization/#container-linux) customization @@ -49,7 +49,7 @@ Define a Kubernetes cluster by using the Terraform module for your chosen platfo ```tf module "google-cloud-yavin" { - source = "git::https://github.com/poseidon/typhoon//google-cloud/container-linux/kubernetes?ref=v1.13.3" + source = "git::https://github.com/poseidon/typhoon//google-cloud/container-linux/kubernetes?ref=v1.13.4" providers = { google = "google.default" @@ -90,9 +90,9 @@ In 4-8 minutes (varies by platform), the cluster will be ready. This Google Clou $ export KUBECONFIG=/home/user/.secrets/clusters/yavin/auth/kubeconfig $ kubectl get nodes NAME ROLES STATUS AGE VERSION -yavin-controller-0.c.example-com.internal controller,master Ready 6m v1.13.3 -yavin-worker-jrbf.c.example-com.internal node Ready 5m v1.13.3 -yavin-worker-mzdm.c.example-com.internal node Ready 5m v1.13.3 +yavin-controller-0.c.example-com.internal controller,master Ready 6m v1.13.4 +yavin-worker-jrbf.c.example-com.internal node Ready 5m v1.13.4 +yavin-worker-mzdm.c.example-com.internal node Ready 5m v1.13.4 ``` List the pods. diff --git a/docs/topics/maintenance.md b/docs/topics/maintenance.md index 0675ea479..77d3b5a03 100644 --- a/docs/topics/maintenance.md +++ b/docs/topics/maintenance.md @@ -18,7 +18,7 @@ module "google-cloud-yavin" { } module "bare-metal-mercury" { - source = "git::https://github.com/poseidon/typhoon//bare-metal/container-linux/kubernetes?ref=v1.13.3" + source = "git::https://github.com/poseidon/typhoon//bare-metal/container-linux/kubernetes?ref=v1.13.4" ... } ``` diff --git a/google-cloud/container-linux/kubernetes/README.md b/google-cloud/container-linux/kubernetes/README.md index 2f4cdef51..dd56442be 100644 --- a/google-cloud/container-linux/kubernetes/README.md +++ b/google-cloud/container-linux/kubernetes/README.md @@ -11,7 +11,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features -* Kubernetes v1.13.3 (upstream, via [kubernetes-incubator/bootkube](https://github.com/kubernetes-incubator/bootkube)) +* Kubernetes v1.13.4 (upstream, via [kubernetes-incubator/bootkube](https://github.com/kubernetes-incubator/bootkube)) * Single or multi-master, [Calico](https://www.projectcalico.org/) or [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/) * Advanced features like [worker pools](https://typhoon.psdn.io/advanced/worker-pools/), [preemptible](https://typhoon.psdn.io/cl/google-cloud/#preemption) workers, and [snippets](https://typhoon.psdn.io/advanced/customization/#container-linux) customization diff --git a/google-cloud/container-linux/kubernetes/bootkube.tf b/google-cloud/container-linux/kubernetes/bootkube.tf index 95bc44cc8..634f30197 100644 --- a/google-cloud/container-linux/kubernetes/bootkube.tf +++ b/google-cloud/container-linux/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=0a7c4fda3507b4a755c0c777d2e65d38bc152bbb" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=953521dbba49eb6a39204f30a3978730eac01e11" cluster_name = "${var.cluster_name}" api_servers = ["${format("%s.%s", var.cluster_name, var.dns_zone)}"] diff --git a/google-cloud/container-linux/kubernetes/cl/controller.yaml.tmpl b/google-cloud/container-linux/kubernetes/cl/controller.yaml.tmpl index 1fc03ce0c..1f40f856f 100644 --- a/google-cloud/container-linux/kubernetes/cl/controller.yaml.tmpl +++ b/google-cloud/container-linux/kubernetes/cl/controller.yaml.tmpl @@ -124,7 +124,7 @@ storage: contents: inline: | KUBELET_IMAGE_URL=docker://k8s.gcr.io/hyperkube - KUBELET_IMAGE_TAG=v1.13.3 + KUBELET_IMAGE_TAG=v1.13.4 - path: /etc/sysctl.d/max-user-watches.conf filesystem: root contents: diff --git a/google-cloud/container-linux/kubernetes/workers/cl/worker.yaml.tmpl b/google-cloud/container-linux/kubernetes/workers/cl/worker.yaml.tmpl index 6e114d0eb..04f442208 100644 --- a/google-cloud/container-linux/kubernetes/workers/cl/worker.yaml.tmpl +++ b/google-cloud/container-linux/kubernetes/workers/cl/worker.yaml.tmpl @@ -94,7 +94,7 @@ storage: contents: inline: | KUBELET_IMAGE_URL=docker://k8s.gcr.io/hyperkube - KUBELET_IMAGE_TAG=v1.13.3 + KUBELET_IMAGE_TAG=v1.13.4 - path: /etc/sysctl.d/max-user-watches.conf filesystem: root contents: @@ -112,7 +112,7 @@ storage: --volume config,kind=host,source=/etc/kubernetes \ --mount volume=config,target=/etc/kubernetes \ --insecure-options=image \ - docker://k8s.gcr.io/hyperkube:v1.13.3 \ + docker://k8s.gcr.io/hyperkube:v1.13.4 \ --net=host \ --dns=host \ --exec=/kubectl -- --kubeconfig=/etc/kubernetes/kubeconfig delete node $(hostname) diff --git a/google-cloud/fedora-atomic/kubernetes/README.md b/google-cloud/fedora-atomic/kubernetes/README.md index ae7e6758f..2e7ec11e0 100644 --- a/google-cloud/fedora-atomic/kubernetes/README.md +++ b/google-cloud/fedora-atomic/kubernetes/README.md @@ -11,7 +11,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features -* Kubernetes v1.13.3 (upstream, via [kubernetes-incubator/bootkube](https://github.com/kubernetes-incubator/bootkube)) +* Kubernetes v1.13.4 (upstream, via [kubernetes-incubator/bootkube](https://github.com/kubernetes-incubator/bootkube)) * Single or multi-master, [Calico](https://www.projectcalico.org/) or [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/) * Advanced features like [worker pools](https://typhoon.psdn.io/advanced/worker-pools/) and [preemptible](https://typhoon.psdn.io/cl/google-cloud/#preemption) workers diff --git a/google-cloud/fedora-atomic/kubernetes/bootkube.tf b/google-cloud/fedora-atomic/kubernetes/bootkube.tf index 1c9f796da..454f6f5d7 100644 --- a/google-cloud/fedora-atomic/kubernetes/bootkube.tf +++ b/google-cloud/fedora-atomic/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=0a7c4fda3507b4a755c0c777d2e65d38bc152bbb" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=953521dbba49eb6a39204f30a3978730eac01e11" cluster_name = "${var.cluster_name}" api_servers = ["${format("%s.%s", var.cluster_name, var.dns_zone)}"] diff --git a/google-cloud/fedora-atomic/kubernetes/cloudinit/controller.yaml.tmpl b/google-cloud/fedora-atomic/kubernetes/cloudinit/controller.yaml.tmpl index 483591e63..81f8b1784 100644 --- a/google-cloud/fedora-atomic/kubernetes/cloudinit/controller.yaml.tmpl +++ b/google-cloud/fedora-atomic/kubernetes/cloudinit/controller.yaml.tmpl @@ -79,7 +79,7 @@ runcmd: - [systemctl, daemon-reload] - [systemctl, restart, NetworkManager] - "atomic install --system --name=etcd quay.io/poseidon/etcd:v3.3.12" - - "atomic install --system --name=kubelet quay.io/poseidon/kubelet:v1.13.3" + - "atomic install --system --name=kubelet quay.io/poseidon/kubelet:v1.13.4" - "atomic install --system --name=bootkube quay.io/poseidon/bootkube:v0.14.0" - [systemctl, start, --no-block, etcd.service] - [systemctl, start, --no-block, kubelet.service] diff --git a/google-cloud/fedora-atomic/kubernetes/workers/cloudinit/worker.yaml.tmpl b/google-cloud/fedora-atomic/kubernetes/workers/cloudinit/worker.yaml.tmpl index c14590c19..da76beae5 100644 --- a/google-cloud/fedora-atomic/kubernetes/workers/cloudinit/worker.yaml.tmpl +++ b/google-cloud/fedora-atomic/kubernetes/workers/cloudinit/worker.yaml.tmpl @@ -54,7 +54,7 @@ bootcmd: runcmd: - [systemctl, daemon-reload] - [systemctl, restart, NetworkManager] - - "atomic install --system --name=kubelet quay.io/poseidon/kubelet:v1.13.3" + - "atomic install --system --name=kubelet quay.io/poseidon/kubelet:v1.13.4" - [systemctl, start, --no-block, kubelet.service] users: - default From 4ff7fe2c29d1c563dea9fed2ad9763a907d53a54 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Thu, 28 Feb 2019 23:22:07 -0800 Subject: [PATCH 037/523] Update Grafana dashboards from upstreams --- addons/grafana/dashboards.yaml | 132 ++++++++++++++++++++------------- 1 file changed, 81 insertions(+), 51 deletions(-) diff --git a/addons/grafana/dashboards.yaml b/addons/grafana/dashboards.yaml index a6c5cc601..909124660 100644 --- a/addons/grafana/dashboards.yaml +++ b/addons/grafana/dashboards.yaml @@ -1301,7 +1301,7 @@ data: "timeShift": null, "title": "CPU Utilisation", "tooltip": { - "shared": true, + "shared": false, "sort": 0, "value_type": "individual" }, @@ -1387,7 +1387,7 @@ data: "timeShift": null, "title": "CPU Saturation (Load1)", "tooltip": { - "shared": true, + "shared": false, "sort": 0, "value_type": "individual" }, @@ -1485,7 +1485,7 @@ data: "timeShift": null, "title": "Memory Utilisation", "tooltip": { - "shared": true, + "shared": false, "sort": 0, "value_type": "individual" }, @@ -1571,7 +1571,7 @@ data: "timeShift": null, "title": "Memory Saturation (Swap I/O)", "tooltip": { - "shared": true, + "shared": false, "sort": 0, "value_type": "individual" }, @@ -1669,7 +1669,7 @@ data: "timeShift": null, "title": "Disk IO Utilisation", "tooltip": { - "shared": true, + "shared": false, "sort": 0, "value_type": "individual" }, @@ -1755,7 +1755,7 @@ data: "timeShift": null, "title": "Disk IO Saturation", "tooltip": { - "shared": true, + "shared": false, "sort": 0, "value_type": "individual" }, @@ -1853,7 +1853,7 @@ data: "timeShift": null, "title": "Net Utilisation (Transmitted)", "tooltip": { - "shared": true, + "shared": false, "sort": 0, "value_type": "individual" }, @@ -1939,7 +1939,7 @@ data: "timeShift": null, "title": "Net Saturation (Dropped)", "tooltip": { - "shared": true, + "shared": false, "sort": 0, "value_type": "individual" }, @@ -2037,7 +2037,7 @@ data: "timeShift": null, "title": "Disk Capacity", "tooltip": { - "shared": true, + "shared": false, "sort": 0, "value_type": "individual" }, @@ -2211,7 +2211,7 @@ data: "timeShift": null, "title": "CPU Utilisation", "tooltip": { - "shared": true, + "shared": false, "sort": 0, "value_type": "individual" }, @@ -2297,7 +2297,7 @@ data: "timeShift": null, "title": "CPU Saturation (Load1)", "tooltip": { - "shared": true, + "shared": false, "sort": 0, "value_type": "individual" }, @@ -2395,7 +2395,7 @@ data: "timeShift": null, "title": "Memory Utilisation", "tooltip": { - "shared": true, + "shared": false, "sort": 0, "value_type": "individual" }, @@ -2481,7 +2481,7 @@ data: "timeShift": null, "title": "Memory Saturation (Swap I/O)", "tooltip": { - "shared": true, + "shared": false, "sort": 0, "value_type": "individual" }, @@ -2579,7 +2579,7 @@ data: "timeShift": null, "title": "Disk IO Utilisation", "tooltip": { - "shared": true, + "shared": false, "sort": 0, "value_type": "individual" }, @@ -2665,7 +2665,7 @@ data: "timeShift": null, "title": "Disk IO Saturation", "tooltip": { - "shared": true, + "shared": false, "sort": 0, "value_type": "individual" }, @@ -2763,7 +2763,7 @@ data: "timeShift": null, "title": "Net Utilisation (Transmitted)", "tooltip": { - "shared": true, + "shared": false, "sort": 0, "value_type": "individual" }, @@ -2849,7 +2849,7 @@ data: "timeShift": null, "title": "Net Saturation (Dropped)", "tooltip": { - "shared": true, + "shared": false, "sort": 0, "value_type": "individual" }, @@ -2947,7 +2947,7 @@ data: "timeShift": null, "title": "Disk Utilisation", "tooltip": { - "shared": true, + "shared": false, "sort": 0, "value_type": "individual" }, @@ -3146,7 +3146,7 @@ data: "timeShift": null, "title": "CPU Utilisation", "tooltip": { - "shared": true, + "shared": false, "sort": 0, "value_type": "individual" }, @@ -3230,7 +3230,7 @@ data: "timeShift": null, "title": "CPU Requests Commitment", "tooltip": { - "shared": true, + "shared": false, "sort": 0, "value_type": "individual" }, @@ -3314,7 +3314,7 @@ data: "timeShift": null, "title": "CPU Limits Commitment", "tooltip": { - "shared": true, + "shared": false, "sort": 0, "value_type": "individual" }, @@ -3398,7 +3398,7 @@ data: "timeShift": null, "title": "Memory Utilisation", "tooltip": { - "shared": true, + "shared": false, "sort": 0, "value_type": "individual" }, @@ -3482,7 +3482,7 @@ data: "timeShift": null, "title": "Memory Requests Commitment", "tooltip": { - "shared": true, + "shared": false, "sort": 0, "value_type": "individual" }, @@ -3566,7 +3566,7 @@ data: "timeShift": null, "title": "Memory Limits Commitment", "tooltip": { - "shared": true, + "shared": false, "sort": 0, "value_type": "individual" }, @@ -3664,7 +3664,7 @@ data: "timeShift": null, "title": "CPU Usage", "tooltip": { - "shared": true, + "shared": false, "sort": 0, "value_type": "individual" }, @@ -3930,7 +3930,7 @@ data: "timeShift": null, "title": "CPU Quota", "tooltip": { - "shared": true, + "shared": false, "sort": 0, "value_type": "individual" }, @@ -4029,7 +4029,7 @@ data: "timeShift": null, "title": "Memory Usage (w/o cache)", "tooltip": { - "shared": true, + "shared": false, "sort": 0, "value_type": "individual" }, @@ -4295,7 +4295,7 @@ data: "timeShift": null, "title": "Requests by Namespace", "tooltip": { - "shared": true, + "shared": false, "sort": 0, "value_type": "individual" }, @@ -4470,7 +4470,7 @@ data: "timeShift": null, "title": "CPU Usage", "tooltip": { - "shared": true, + "shared": false, "sort": 0, "value_type": "individual" }, @@ -4736,7 +4736,7 @@ data: "timeShift": null, "title": "CPU Quota", "tooltip": { - "shared": true, + "shared": false, "sort": 0, "value_type": "individual" }, @@ -4835,7 +4835,7 @@ data: "timeShift": null, "title": "Memory Usage (w/o cache)", "tooltip": { - "shared": true, + "shared": false, "sort": 0, "value_type": "individual" }, @@ -5182,7 +5182,7 @@ data: "timeShift": null, "title": "Memory Quota", "tooltip": { - "shared": true, + "shared": false, "sort": 0, "value_type": "individual" }, @@ -5384,7 +5384,7 @@ data: "timeShift": null, "title": "CPU Usage", "tooltip": { - "shared": true, + "shared": false, "sort": 0, "value_type": "individual" }, @@ -5650,7 +5650,7 @@ data: "timeShift": null, "title": "CPU Quota", "tooltip": { - "shared": true, + "shared": false, "sort": 0, "value_type": "individual" }, @@ -5765,7 +5765,7 @@ data: "timeShift": null, "title": "Memory Usage", "tooltip": { - "shared": true, + "shared": false, "sort": 0, "value_type": "individual" }, @@ -6112,7 +6112,7 @@ data: "timeShift": null, "title": "Memory Quota", "tooltip": { - "shared": true, + "shared": false, "sort": 0, "value_type": "individual" }, @@ -6367,7 +6367,7 @@ data: "timeShift": null, "title": "System load", "tooltip": { - "shared": true, + "shared": false, "sort": 0, "value_type": "individual" }, @@ -6458,7 +6458,7 @@ data: "timeShift": null, "title": "Usage Per Core", "tooltip": { - "shared": true, + "shared": false, "sort": 0, "value_type": "individual" }, @@ -6562,7 +6562,7 @@ data: "timeShift": null, "title": "CPU Utilizaion", "tooltip": { - "shared": true, + "shared": false, "sort": 0, "value_type": "individual" }, @@ -6665,6 +6665,9 @@ data: ], "thresholds": "80, 90", "title": "CPU Usage", + "tooltip": { + "shared": false + }, "type": "singlestat", "valueFontSize": "80%", "valueMaps": [ @@ -6768,7 +6771,7 @@ data: "timeShift": null, "title": "Memory Usage", "tooltip": { - "shared": true, + "shared": false, "sort": 0, "value_type": "individual" }, @@ -6871,6 +6874,9 @@ data: ], "thresholds": "80, 90", "title": "Memory Usage", + "tooltip": { + "shared": false + }, "type": "singlestat", "valueFontSize": "80%", "valueMaps": [ @@ -6974,7 +6980,7 @@ data: "timeShift": null, "title": "Disk I/O", "tooltip": { - "shared": true, + "shared": false, "sort": 0, "value_type": "individual" }, @@ -7065,7 +7071,7 @@ data: "timeShift": null, "title": "Disk Space Usage", "tooltip": { - "shared": true, + "shared": false, "sort": 0, "value_type": "individual" }, @@ -7169,7 +7175,7 @@ data: "timeShift": null, "title": "Network Received", "tooltip": { - "shared": true, + "shared": false, "sort": 0, "value_type": "individual" }, @@ -7260,7 +7266,7 @@ data: "timeShift": null, "title": "Network Transmitted", "tooltip": { - "shared": true, + "shared": false, "sort": 0, "value_type": "individual" }, @@ -7371,7 +7377,7 @@ data: "timeShift": null, "title": "Inodes Usage", "tooltip": { - "shared": true, + "shared": false, "sort": 0, "value_type": "individual" }, @@ -7474,6 +7480,9 @@ data: ], "thresholds": "80, 90", "title": "Inodes Usage", + "tooltip": { + "shared": false + }, "type": "singlestat", "valueFontSize": "80%", "valueMaps": [ @@ -7665,7 +7674,7 @@ data: "timeShift": null, "title": "Volume Space Usage", "tooltip": { - "shared": true, + "shared": false, "sort": 0, "value_type": "individual" }, @@ -7769,7 +7778,7 @@ data: "timeShift": null, "title": "Volume inodes Usage", "tooltip": { - "shared": true, + "shared": false, "sort": 0, "value_type": "individual" }, @@ -8021,7 +8030,7 @@ data: "timeShift": null, "title": "Memory Usage", "tooltip": { - "shared": true, + "shared": false, "sort": 0, "value_type": "individual" }, @@ -8124,7 +8133,7 @@ data: "timeShift": null, "title": "CPU Usage", "tooltip": { - "shared": true, + "shared": false, "sort": 0, "value_type": "individual" }, @@ -8227,7 +8236,7 @@ data: "timeShift": null, "title": "Network I/O", "tooltip": { - "shared": true, + "shared": false, "sort": 0, "value_type": "individual" }, @@ -8503,6 +8512,9 @@ data: ], "thresholds": "", "title": "CPU", + "tooltip": { + "shared": false + }, "type": "singlestat", "valueFontSize": "80%", "valueMaps": [ @@ -8583,6 +8595,9 @@ data: ], "thresholds": "", "title": "Memory", + "tooltip": { + "shared": false + }, "type": "singlestat", "valueFontSize": "80%", "valueMaps": [ @@ -8663,6 +8678,9 @@ data: ], "thresholds": "", "title": "Network", + "tooltip": { + "shared": false + }, "type": "singlestat", "valueFontSize": "80%", "valueMaps": [ @@ -8758,6 +8776,9 @@ data: ], "thresholds": "", "title": "Desired Replicas", + "tooltip": { + "shared": false + }, "type": "singlestat", "valueFontSize": "80%", "valueMaps": [ @@ -8839,6 +8860,9 @@ data: ], "thresholds": "", "title": "Replicas of current version", + "tooltip": { + "shared": false + }, "type": "singlestat", "valueFontSize": "80%", "valueMaps": [ @@ -8920,6 +8944,9 @@ data: ], "thresholds": "", "title": "Observed Generation", + "tooltip": { + "shared": false + }, "type": "singlestat", "valueFontSize": "80%", "valueMaps": [ @@ -9001,6 +9028,9 @@ data: ], "thresholds": "", "title": "Metadata Generation", + "tooltip": { + "shared": false + }, "type": "singlestat", "valueFontSize": "80%", "valueMaps": [ @@ -9110,7 +9140,7 @@ data: "timeShift": null, "title": "Replicas", "tooltip": { - "shared": true, + "shared": false, "sort": 0, "value_type": "individual" }, From d42f42df4e05fd3085141a142c50395ba2c4a2ae Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Fri, 1 Mar 2019 01:15:08 -0800 Subject: [PATCH 038/523] Re-measure cluster provision times and document --- docs/topics/performance.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/topics/performance.md b/docs/topics/performance.md index 5b0d16f32..20f32f42b 100644 --- a/docs/topics/performance.md +++ b/docs/topics/performance.md @@ -6,11 +6,11 @@ Provisioning times vary based on the operating system and platform. Sampling the | Platform | Apply | Destroy | |---------------|-------|---------| -| AWS | 6 min | 4 min | -| Azure | 7 min | 7 min | +| AWS | 5 min | 3 min | +| Azure | 10 min | 7 min | | Bare-Metal | 10-15 min | NA | | Digital Ocean | 3 min 30 sec | 20 sec | -| Google Cloud | 7 min | 6 min | +| Google Cloud | 8 min | 5 min | Notes: From a08adc92b53f4ef8847c520f4a8b27c6e2379e6c Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Fri, 1 Mar 2019 01:18:54 -0800 Subject: [PATCH 039/523] Update nginx-ingress from v0.22.0 to v0.23.0 * https://github.com/kubernetes/ingress-nginx/releases/tag/nginx-0.23.0 --- CHANGES.md | 6 +++--- addons/nginx-ingress/aws/deployment.yaml | 2 +- addons/nginx-ingress/azure/deployment.yaml | 2 +- addons/nginx-ingress/bare-metal/deployment.yaml | 2 +- addons/nginx-ingress/digital-ocean/daemonset.yaml | 2 +- addons/nginx-ingress/google-cloud/deployment.yaml | 2 +- 6 files changed, 8 insertions(+), 8 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 1490d9a59..013f31e89 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -35,9 +35,9 @@ Notable changes between versions. * Modernize Grafana and dashboards ([#403](https://github.com/poseidon/typhoon/pull/403), [#404](https://github.com/poseidon/typhoon/pull/404)) * Upgrade Grafana from v5.4.3 to [v6.0.0](https://github.com/grafana/grafana/releases/tag/v6.0.0)! * Enable Grafana [Explore](http://docs.grafana.org/guides/whats-new-in-v6-0/#explore) UI as a Viewer (inspect/edit without saving) -* Raise nginx-ingress liveness/readiness timeout to 5 seconds -* Remove nginx-ingess default-backend ([#401](https://github.com/poseidon/typhoon/pull/401)) - * nginx-ingress now responds with its own 404 page by default +* Update nginx-ingress from v0.22.0 to v0.23.0 + * Raise nginx-ingress liveness/readiness timeout to 5 seconds + * Remove nginx-ingess default-backend ([#401](https://github.com/poseidon/typhoon/pull/401)) ## v1.13.3 diff --git a/addons/nginx-ingress/aws/deployment.yaml b/addons/nginx-ingress/aws/deployment.yaml index 1f537e253..c42ba9195 100644 --- a/addons/nginx-ingress/aws/deployment.yaml +++ b/addons/nginx-ingress/aws/deployment.yaml @@ -24,7 +24,7 @@ spec: node-role.kubernetes.io/node: "" containers: - name: nginx-ingress-controller - image: quay.io/kubernetes-ingress-controller/nginx-ingress-controller:0.22.0 + image: quay.io/kubernetes-ingress-controller/nginx-ingress-controller:0.23.0 args: - /nginx-ingress-controller - --ingress-class=public diff --git a/addons/nginx-ingress/azure/deployment.yaml b/addons/nginx-ingress/azure/deployment.yaml index 1f537e253..c42ba9195 100644 --- a/addons/nginx-ingress/azure/deployment.yaml +++ b/addons/nginx-ingress/azure/deployment.yaml @@ -24,7 +24,7 @@ spec: node-role.kubernetes.io/node: "" containers: - name: nginx-ingress-controller - image: quay.io/kubernetes-ingress-controller/nginx-ingress-controller:0.22.0 + image: quay.io/kubernetes-ingress-controller/nginx-ingress-controller:0.23.0 args: - /nginx-ingress-controller - --ingress-class=public diff --git a/addons/nginx-ingress/bare-metal/deployment.yaml b/addons/nginx-ingress/bare-metal/deployment.yaml index 8c61aa0f5..ad5040fde 100644 --- a/addons/nginx-ingress/bare-metal/deployment.yaml +++ b/addons/nginx-ingress/bare-metal/deployment.yaml @@ -22,7 +22,7 @@ spec: spec: containers: - name: nginx-ingress-controller - image: quay.io/kubernetes-ingress-controller/nginx-ingress-controller:0.22.0 + image: quay.io/kubernetes-ingress-controller/nginx-ingress-controller:0.23.0 args: - /nginx-ingress-controller - --ingress-class=public diff --git a/addons/nginx-ingress/digital-ocean/daemonset.yaml b/addons/nginx-ingress/digital-ocean/daemonset.yaml index 29885d6ac..a8ab80854 100644 --- a/addons/nginx-ingress/digital-ocean/daemonset.yaml +++ b/addons/nginx-ingress/digital-ocean/daemonset.yaml @@ -24,7 +24,7 @@ spec: node-role.kubernetes.io/node: "" containers: - name: nginx-ingress-controller - image: quay.io/kubernetes-ingress-controller/nginx-ingress-controller:0.22.0 + image: quay.io/kubernetes-ingress-controller/nginx-ingress-controller:0.23.0 args: - /nginx-ingress-controller - --ingress-class=public diff --git a/addons/nginx-ingress/google-cloud/deployment.yaml b/addons/nginx-ingress/google-cloud/deployment.yaml index 1f537e253..c42ba9195 100644 --- a/addons/nginx-ingress/google-cloud/deployment.yaml +++ b/addons/nginx-ingress/google-cloud/deployment.yaml @@ -24,7 +24,7 @@ spec: node-role.kubernetes.io/node: "" containers: - name: nginx-ingress-controller - image: quay.io/kubernetes-ingress-controller/nginx-ingress-controller:0.22.0 + image: quay.io/kubernetes-ingress-controller/nginx-ingress-controller:0.23.0 args: - /nginx-ingress-controller - --ingress-class=public From fc277eaab671ba8fb9fd5f4304d66f9f85a5e1b7 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Sat, 2 Mar 2019 10:54:35 -0800 Subject: [PATCH 040/523] Document the GCP DNS admin requirement for cluster provisioning * Configure the google terraform provider to use GCP service account credentials with compute and dns admin privileges --- docs/atomic/google-cloud.md | 2 +- docs/cl/google-cloud.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/atomic/google-cloud.md b/docs/atomic/google-cloud.md index d96267685..3abcd06c1 100644 --- a/docs/atomic/google-cloud.md +++ b/docs/atomic/google-cloud.md @@ -35,7 +35,7 @@ cd infra/clusters Login to your Google Console [API Manager](https://console.cloud.google.com/apis/dashboard) and select a project, or [signup](https://cloud.google.com/free/) if you don't have an account. -Select "Credentials" and create a service account key. Choose the "Compute Engine Admin" role and save the JSON private key to a file that can be referenced in configs. +Select "Credentials" and create a service account key. Choose the "Compute Engine Admin" and "DNS Administrator" roles and save the JSON private key to a file that can be referenced in configs. ```sh mv ~/Downloads/project-id-43048204.json ~/.config/google-cloud/terraform.json diff --git a/docs/cl/google-cloud.md b/docs/cl/google-cloud.md index d7be23301..a8687da25 100644 --- a/docs/cl/google-cloud.md +++ b/docs/cl/google-cloud.md @@ -39,7 +39,7 @@ cd infra/clusters Login to your Google Console [API Manager](https://console.cloud.google.com/apis/dashboard) and select a project, or [signup](https://cloud.google.com/free/) if you don't have an account. -Select "Credentials" and create a service account key. Choose the "Compute Engine Admin" role and save the JSON private key to a file that can be referenced in configs. +Select "Credentials" and create a service account key. Choose the "Compute Engine Admin" and "DNS Administrator" roles and save the JSON private key to a file that can be referenced in configs. ```sh mv ~/Downloads/project-id-43048204.json ~/.config/google-cloud/terraform.json From de251bd94f9fd60807b6128be28f499aeeb8d6e4 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Sat, 2 Mar 2019 11:00:21 -0800 Subject: [PATCH 041/523] Update tutorials to prefer newer provider plugins over min version * Minimum versions of Terraform provider plugins are enforced in each module already. Its better to provide examples with newer versions. Some folks don't update them * Previously, tutorials showed the minimum viable version of each terraform provider that might be used --- docs/atomic/aws.md | 2 +- docs/atomic/digital-ocean.md | 2 +- docs/atomic/google-cloud.md | 2 +- docs/cl/aws.md | 2 +- docs/cl/azure.md | 2 +- docs/cl/digital-ocean.md | 2 +- docs/cl/google-cloud.md | 2 +- 7 files changed, 7 insertions(+), 7 deletions(-) diff --git a/docs/atomic/aws.md b/docs/atomic/aws.md index 695f45a79..ec481ce32 100644 --- a/docs/atomic/aws.md +++ b/docs/atomic/aws.md @@ -44,7 +44,7 @@ Configure the AWS provider to use your access key credentials in a `providers.tf ```tf provider "aws" { - version = "~> 1.13.0" + version = "~> 1.60.0" alias = "default" region = "eu-central-1" diff --git a/docs/atomic/digital-ocean.md b/docs/atomic/digital-ocean.md index 6a4ad686a..465b49fe0 100644 --- a/docs/atomic/digital-ocean.md +++ b/docs/atomic/digital-ocean.md @@ -45,7 +45,7 @@ Configure the DigitalOcean provider to use your token in a `providers.tf` file. ```tf provider "digitalocean" { - version = "1.0.0" + version = "~> 1.1.0" token = "${chomp(file("~/.config/digital-ocean/token"))}" alias = "default" } diff --git a/docs/atomic/google-cloud.md b/docs/atomic/google-cloud.md index 3abcd06c1..2f362b4f8 100644 --- a/docs/atomic/google-cloud.md +++ b/docs/atomic/google-cloud.md @@ -45,7 +45,7 @@ Configure the Google Cloud provider to use your service account key, project-id, ```tf provider "google" { - version = "2.0.0" + version = "~> 2.1.0" alias = "default" credentials = "${file("~/.config/google-cloud/terraform.json")}" diff --git a/docs/cl/aws.md b/docs/cl/aws.md index 0339460bd..91338bedc 100644 --- a/docs/cl/aws.md +++ b/docs/cl/aws.md @@ -49,7 +49,7 @@ Configure the AWS provider to use your access key credentials in a `providers.tf ```tf provider "aws" { - version = "~> 1.13.0" + version = "~> 1.60.0" alias = "default" region = "eu-central-1" diff --git a/docs/cl/azure.md b/docs/cl/azure.md index 299b1d656..e29aca220 100644 --- a/docs/cl/azure.md +++ b/docs/cl/azure.md @@ -50,7 +50,7 @@ Configure the Azure provider in a `providers.tf` file. ```tf provider "azurerm" { - version = "1.16.0" + version = "~> 1.22.1" alias = "default" } diff --git a/docs/cl/digital-ocean.md b/docs/cl/digital-ocean.md index fc4937dae..8dfcdc165 100644 --- a/docs/cl/digital-ocean.md +++ b/docs/cl/digital-ocean.md @@ -50,7 +50,7 @@ Configure the DigitalOcean provider to use your token in a `providers.tf` file. ```tf provider "digitalocean" { - version = "1.0.0" + version = "~> 1.1.0" token = "${chomp(file("~/.config/digital-ocean/token"))}" alias = "default" } diff --git a/docs/cl/google-cloud.md b/docs/cl/google-cloud.md index a8687da25..3f27114b5 100644 --- a/docs/cl/google-cloud.md +++ b/docs/cl/google-cloud.md @@ -49,7 +49,7 @@ Configure the Google Cloud provider to use your service account key, project-id, ```tf provider "google" { - version = "2.0.0" + version = "~> 2.1.0" alias = "default" credentials = "${file("~/.config/google-cloud/terraform.json")}" From 5066a25d895e408d26476342f13689dd4748c6cf Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Sat, 2 Mar 2019 11:26:12 -0800 Subject: [PATCH 042/523] Add links and clarifications in CHANGES for release --- CHANGES.md | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index 013f31e89..b5c459941 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -32,13 +32,17 @@ Notable changes between versions. * Drop extraneous metrics ([#397](https://github.com/poseidon/typhoon/pull/397)) * Add `pod` name label to metrics discovered via service endpoints * Rename `kubernetes_namespace` label to `namespace` -* Modernize Grafana and dashboards ([#403](https://github.com/poseidon/typhoon/pull/403), [#404](https://github.com/poseidon/typhoon/pull/404)) +* Modernize Grafana and dashboards, see [docs](https://typhoon.psdn.io/addons/grafana/) ([#403](https://github.com/poseidon/typhoon/pull/403), [#404](https://github.com/poseidon/typhoon/pull/404)) * Upgrade Grafana from v5.4.3 to [v6.0.0](https://github.com/grafana/grafana/releases/tag/v6.0.0)! * Enable Grafana [Explore](http://docs.grafana.org/guides/whats-new-in-v6-0/#explore) UI as a Viewer (inspect/edit without saving) * Update nginx-ingress from v0.22.0 to v0.23.0 * Raise nginx-ingress liveness/readiness timeout to 5 seconds * Remove nginx-ingess default-backend ([#401](https://github.com/poseidon/typhoon/pull/401)) +#### Fedora Atomic + +* Build Kubelet [system container](https://github.com/poseidon/system-containers) with buildah. The image is an OCI format and slightly larger. + ## v1.13.3 * Kubernetes [v1.13.3](https://github.com/kubernetes/kubernetes/blob/master/CHANGELOG-1.13.md#v1133) From deec512c1470aa92a4cfd338de0486f1a1468ad0 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Mon, 4 Mar 2019 23:00:13 -0800 Subject: [PATCH 043/523] Resolve in-addr.arpa and ip6.arpa zones with CoreDNS kubernetes plugin * Resolve in-addr.arpa and ip6.arpa DNS PTR requests for Kubernetes service IPs and pod IPs * Previously, CoreDNS was configured to resolve in-addr.arpa PTR records for service IPs (but not pod IPs) --- CHANGES.md | 3 +++ aws/container-linux/kubernetes/bootkube.tf | 2 +- aws/fedora-atomic/kubernetes/bootkube.tf | 2 +- azure/container-linux/kubernetes/bootkube.tf | 2 +- bare-metal/container-linux/kubernetes/bootkube.tf | 2 +- bare-metal/fedora-atomic/kubernetes/bootkube.tf | 2 +- digital-ocean/container-linux/kubernetes/bootkube.tf | 2 +- digital-ocean/fedora-atomic/kubernetes/bootkube.tf | 2 +- google-cloud/container-linux/kubernetes/bootkube.tf | 2 +- google-cloud/fedora-atomic/kubernetes/bootkube.tf | 2 +- 10 files changed, 12 insertions(+), 9 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index b5c459941..c06b15573 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -4,6 +4,9 @@ Notable changes between versions. ## Latest +* Resolve in-addr.arpa reverse DNS lookups (PTR) for pod IPv4 addresses ([#415](https://github.com/poseidon/typhoon/pull/415)) + * Reverse DNS lookups for service IPv4 addresses unchanged + ## v1.13.4 * Kubernetes [v1.13.4](https://github.com/kubernetes/kubernetes/blob/master/CHANGELOG-1.13.md#v1134) diff --git a/aws/container-linux/kubernetes/bootkube.tf b/aws/container-linux/kubernetes/bootkube.tf index c06ddffe4..4ffb48852 100644 --- a/aws/container-linux/kubernetes/bootkube.tf +++ b/aws/container-linux/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=953521dbba49eb6a39204f30a3978730eac01e11" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=1528266595b91d40f824a3281563f13dd0a5b6df" cluster_name = "${var.cluster_name}" api_servers = ["${format("%s.%s", var.cluster_name, var.dns_zone)}"] diff --git a/aws/fedora-atomic/kubernetes/bootkube.tf b/aws/fedora-atomic/kubernetes/bootkube.tf index 4e2c66371..a2116f5ed 100644 --- a/aws/fedora-atomic/kubernetes/bootkube.tf +++ b/aws/fedora-atomic/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=953521dbba49eb6a39204f30a3978730eac01e11" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=1528266595b91d40f824a3281563f13dd0a5b6df" cluster_name = "${var.cluster_name}" api_servers = ["${format("%s.%s", var.cluster_name, var.dns_zone)}"] diff --git a/azure/container-linux/kubernetes/bootkube.tf b/azure/container-linux/kubernetes/bootkube.tf index 173fe9fa3..6d2513731 100644 --- a/azure/container-linux/kubernetes/bootkube.tf +++ b/azure/container-linux/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=953521dbba49eb6a39204f30a3978730eac01e11" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=1528266595b91d40f824a3281563f13dd0a5b6df" cluster_name = "${var.cluster_name}" api_servers = ["${format("%s.%s", var.cluster_name, var.dns_zone)}"] diff --git a/bare-metal/container-linux/kubernetes/bootkube.tf b/bare-metal/container-linux/kubernetes/bootkube.tf index 9686c43c7..c2937f085 100644 --- a/bare-metal/container-linux/kubernetes/bootkube.tf +++ b/bare-metal/container-linux/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=953521dbba49eb6a39204f30a3978730eac01e11" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=1528266595b91d40f824a3281563f13dd0a5b6df" cluster_name = "${var.cluster_name}" api_servers = ["${var.k8s_domain_name}"] diff --git a/bare-metal/fedora-atomic/kubernetes/bootkube.tf b/bare-metal/fedora-atomic/kubernetes/bootkube.tf index c01246730..0eb1a1c5c 100644 --- a/bare-metal/fedora-atomic/kubernetes/bootkube.tf +++ b/bare-metal/fedora-atomic/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=953521dbba49eb6a39204f30a3978730eac01e11" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=1528266595b91d40f824a3281563f13dd0a5b6df" cluster_name = "${var.cluster_name}" api_servers = ["${var.k8s_domain_name}"] diff --git a/digital-ocean/container-linux/kubernetes/bootkube.tf b/digital-ocean/container-linux/kubernetes/bootkube.tf index 3fdc95251..8082608e5 100644 --- a/digital-ocean/container-linux/kubernetes/bootkube.tf +++ b/digital-ocean/container-linux/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=953521dbba49eb6a39204f30a3978730eac01e11" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=1528266595b91d40f824a3281563f13dd0a5b6df" cluster_name = "${var.cluster_name}" api_servers = ["${format("%s.%s", var.cluster_name, var.dns_zone)}"] diff --git a/digital-ocean/fedora-atomic/kubernetes/bootkube.tf b/digital-ocean/fedora-atomic/kubernetes/bootkube.tf index f1776386d..ae48813e0 100644 --- a/digital-ocean/fedora-atomic/kubernetes/bootkube.tf +++ b/digital-ocean/fedora-atomic/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=953521dbba49eb6a39204f30a3978730eac01e11" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=1528266595b91d40f824a3281563f13dd0a5b6df" cluster_name = "${var.cluster_name}" api_servers = ["${format("%s.%s", var.cluster_name, var.dns_zone)}"] diff --git a/google-cloud/container-linux/kubernetes/bootkube.tf b/google-cloud/container-linux/kubernetes/bootkube.tf index 634f30197..fa7962e6e 100644 --- a/google-cloud/container-linux/kubernetes/bootkube.tf +++ b/google-cloud/container-linux/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=953521dbba49eb6a39204f30a3978730eac01e11" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=1528266595b91d40f824a3281563f13dd0a5b6df" cluster_name = "${var.cluster_name}" api_servers = ["${format("%s.%s", var.cluster_name, var.dns_zone)}"] diff --git a/google-cloud/fedora-atomic/kubernetes/bootkube.tf b/google-cloud/fedora-atomic/kubernetes/bootkube.tf index 454f6f5d7..a4ea41afd 100644 --- a/google-cloud/fedora-atomic/kubernetes/bootkube.tf +++ b/google-cloud/fedora-atomic/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=953521dbba49eb6a39204f30a3978730eac01e11" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=1528266595b91d40f824a3281563f13dd0a5b6df" cluster_name = "${var.cluster_name}" api_servers = ["${format("%s.%s", var.cluster_name, var.dns_zone)}"] From 4d9a692424fc993735ca576d945169dde3f34999 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Mon, 4 Mar 2019 23:08:12 -0800 Subject: [PATCH 044/523] Update Prometheus from v2.7.1 to v2.7.2 * https://github.com/prometheus/prometheus/releases/tag/v2.7.2 --- CHANGES.md | 4 ++++ addons/prometheus/deployment.yaml | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index c06b15573..2da2ab121 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -7,6 +7,10 @@ Notable changes between versions. * Resolve in-addr.arpa reverse DNS lookups (PTR) for pod IPv4 addresses ([#415](https://github.com/poseidon/typhoon/pull/415)) * Reverse DNS lookups for service IPv4 addresses unchanged +#### Addons + +* Update Prometheus from v2.7.1 to v2.7.2 + ## v1.13.4 * Kubernetes [v1.13.4](https://github.com/kubernetes/kubernetes/blob/master/CHANGELOG-1.13.md#v1134) diff --git a/addons/prometheus/deployment.yaml b/addons/prometheus/deployment.yaml index 8e9f26a79..abfb0eec0 100644 --- a/addons/prometheus/deployment.yaml +++ b/addons/prometheus/deployment.yaml @@ -20,7 +20,7 @@ spec: serviceAccountName: prometheus containers: - name: prometheus - image: quay.io/prometheus/prometheus:v2.7.1 + image: quay.io/prometheus/prometheus:v2.7.2 args: - --web.listen-address=0.0.0.0:9090 - --config.file=/etc/prometheus/prometheus.yaml From 3afd114f8cf6684d948912c21bc3be953cd21bbc Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Mon, 4 Mar 2019 23:11:02 -0800 Subject: [PATCH 045/523] Update mkdocs-material from v4.0.1 to v4.0.2 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 37b4849cb..899424e7c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ mkdocs==1.0.4 -mkdocs-material==4.0.1 +mkdocs-material==4.0.2 pygments==2.2.0 pymdown-extensions==5.0.0 From fe96da27d7dc7f94b90f773948176594e0c165f6 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Sat, 9 Mar 2019 12:03:10 -0800 Subject: [PATCH 046/523] Add support for terraform-provider-aws v2.0+ * Allow terraform-provider-aws >= v1.13, but < 3.0. No change to the minimum version, but allow using v2.x.y releases * Verify compatability with terraform-provider-aws v2.1.0 --- CHANGES.md | 4 ++++ aws/container-linux/kubernetes/require.tf | 2 +- aws/fedora-atomic/kubernetes/require.tf | 2 +- docs/atomic/aws.md | 4 ++-- docs/atomic/bare-metal.md | 2 +- docs/atomic/digital-ocean.md | 2 +- docs/atomic/google-cloud.md | 2 +- docs/cl/aws.md | 4 ++-- docs/cl/azure.md | 2 +- docs/cl/bare-metal.md | 2 +- docs/cl/digital-ocean.md | 2 +- docs/cl/google-cloud.md | 2 +- 12 files changed, 17 insertions(+), 13 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 2da2ab121..4b61c5ca3 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -7,6 +7,10 @@ Notable changes between versions. * Resolve in-addr.arpa reverse DNS lookups (PTR) for pod IPv4 addresses ([#415](https://github.com/poseidon/typhoon/pull/415)) * Reverse DNS lookups for service IPv4 addresses unchanged +#### AWS + +* Support `terraform-provider-aws` v2.0+ ([#419](https://github.com/poseidon/typhoon/pull/419)) + #### Addons * Update Prometheus from v2.7.1 to v2.7.2 diff --git a/aws/container-linux/kubernetes/require.tf b/aws/container-linux/kubernetes/require.tf index d47547530..68f475d62 100644 --- a/aws/container-linux/kubernetes/require.tf +++ b/aws/container-linux/kubernetes/require.tf @@ -5,7 +5,7 @@ terraform { } provider "aws" { - version = "~> 1.13" + version = ">= 1.13, < 3.0" } provider "local" { diff --git a/aws/fedora-atomic/kubernetes/require.tf b/aws/fedora-atomic/kubernetes/require.tf index d47547530..68f475d62 100644 --- a/aws/fedora-atomic/kubernetes/require.tf +++ b/aws/fedora-atomic/kubernetes/require.tf @@ -5,7 +5,7 @@ terraform { } provider "aws" { - version = "~> 1.13" + version = ">= 1.13, < 3.0" } provider "local" { diff --git a/docs/atomic/aws.md b/docs/atomic/aws.md index ec481ce32..d66798c3c 100644 --- a/docs/atomic/aws.md +++ b/docs/atomic/aws.md @@ -21,7 +21,7 @@ Install [Terraform](https://www.terraform.io/downloads.html) v0.11.x on your sys ```sh $ terraform version -Terraform v0.11.7 +Terraform v0.11.12 ``` Read [concepts](/architecture/concepts/) to learn about Terraform, modules, and organizing resources. Change to your infrastructure repository (e.g. `infra`). @@ -44,7 +44,7 @@ Configure the AWS provider to use your access key credentials in a `providers.tf ```tf provider "aws" { - version = "~> 1.60.0" + version = "~> 2.1.0" alias = "default" region = "eu-central-1" diff --git a/docs/atomic/bare-metal.md b/docs/atomic/bare-metal.md index a2151208d..870b1b6ee 100644 --- a/docs/atomic/bare-metal.md +++ b/docs/atomic/bare-metal.md @@ -171,7 +171,7 @@ Install [Terraform](https://www.terraform.io/downloads.html) v0.11.x on your sys ```sh $ terraform version -Terraform v0.11.7 +Terraform v0.11.12 ``` Add the [terraform-provider-matchbox](https://github.com/coreos/terraform-provider-matchbox) plugin binary for your system to `~/.terraform.d/plugins/`, noting the final name. diff --git a/docs/atomic/digital-ocean.md b/docs/atomic/digital-ocean.md index 465b49fe0..f17089acb 100644 --- a/docs/atomic/digital-ocean.md +++ b/docs/atomic/digital-ocean.md @@ -21,7 +21,7 @@ Install [Terraform](https://www.terraform.io/downloads.html) v0.11.x on your sys ```sh $ terraform version -Terraform v0.11.7 +Terraform v0.11.12 ``` Read [concepts](/architecture/concepts/) to learn about Terraform, modules, and organizing resources. Change to your infrastructure repository (e.g. `infra`). diff --git a/docs/atomic/google-cloud.md b/docs/atomic/google-cloud.md index 2f362b4f8..b94e9749f 100644 --- a/docs/atomic/google-cloud.md +++ b/docs/atomic/google-cloud.md @@ -22,7 +22,7 @@ Install [Terraform](https://www.terraform.io/downloads.html) v0.11.x on your sys ```sh $ terraform version -Terraform v0.11.7 +Terraform v0.11.12 ``` Read [concepts](/architecture/concepts/) to learn about Terraform, modules, and organizing resources. Change to your infrastructure repository (e.g. `infra`). diff --git a/docs/cl/aws.md b/docs/cl/aws.md index 91338bedc..b53513dc0 100644 --- a/docs/cl/aws.md +++ b/docs/cl/aws.md @@ -18,7 +18,7 @@ Install [Terraform](https://www.terraform.io/downloads.html) v0.11.x on your sys ```sh $ terraform version -Terraform v0.11.11 +Terraform v0.11.12 ``` Add the [terraform-provider-ct](https://github.com/coreos/terraform-provider-ct) plugin binary for your system to `~/.terraform.d/plugins/`, noting the final name. @@ -49,7 +49,7 @@ Configure the AWS provider to use your access key credentials in a `providers.tf ```tf provider "aws" { - version = "~> 1.60.0" + version = "~> 2.1.0" alias = "default" region = "eu-central-1" diff --git a/docs/cl/azure.md b/docs/cl/azure.md index e29aca220..d888628c2 100644 --- a/docs/cl/azure.md +++ b/docs/cl/azure.md @@ -21,7 +21,7 @@ Install [Terraform](https://www.terraform.io/downloads.html) v0.11.x on your sys ```sh $ terraform version -Terraform v0.11.11 +Terraform v0.11.12 ``` Add the [terraform-provider-ct](https://github.com/coreos/terraform-provider-ct) plugin binary for your system to `~/.terraform.d/plugins/`, noting the final name. diff --git a/docs/cl/bare-metal.md b/docs/cl/bare-metal.md index 970547fd8..73b14542e 100644 --- a/docs/cl/bare-metal.md +++ b/docs/cl/bare-metal.md @@ -110,7 +110,7 @@ Install [Terraform](https://www.terraform.io/downloads.html) v0.11.x on your sys ```sh $ terraform version -Terraform v0.11.7 +Terraform v0.11.12 ``` Add the [terraform-provider-matchbox](https://github.com/coreos/terraform-provider-matchbox) plugin binary for your system to `~/.terraform.d/plugins/`, noting the final name. diff --git a/docs/cl/digital-ocean.md b/docs/cl/digital-ocean.md index 8dfcdc165..ade88a96a 100644 --- a/docs/cl/digital-ocean.md +++ b/docs/cl/digital-ocean.md @@ -18,7 +18,7 @@ Install [Terraform](https://www.terraform.io/downloads.html) v0.11.x on your sys ```sh $ terraform version -Terraform v0.11.11 +Terraform v0.11.12 ``` Add the [terraform-provider-ct](https://github.com/coreos/terraform-provider-ct) plugin binary for your system to `~/.terraform.d/plugins/`, noting the final name. diff --git a/docs/cl/google-cloud.md b/docs/cl/google-cloud.md index 3f27114b5..3b60371e5 100644 --- a/docs/cl/google-cloud.md +++ b/docs/cl/google-cloud.md @@ -18,7 +18,7 @@ Install [Terraform](https://www.terraform.io/downloads.html) v0.11.x on your sys ```sh $ terraform version -Terraform v0.11.7 +Terraform v0.11.12 ``` Add the [terraform-provider-ct](https://github.com/coreos/terraform-provider-ct) plugin binary for your system to `~/.terraform.d/plugins/`, noting the final name. From 4201eb1efa76924242233f80d4947a10bec649ec Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Sat, 9 Mar 2019 12:43:43 -0800 Subject: [PATCH 047/523] Update Grafana from v6.0.0 to v6.0.1 * https://github.com/grafana/grafana/releases/tag/v6.0.1 --- CHANGES.md | 1 + addons/grafana/deployment.yaml | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index 4b61c5ca3..4b23c31d0 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -14,6 +14,7 @@ Notable changes between versions. #### Addons * Update Prometheus from v2.7.1 to v2.7.2 +* Update Grafana from v6.0.0 to v6.0.1 ## v1.13.4 diff --git a/addons/grafana/deployment.yaml b/addons/grafana/deployment.yaml index dfcfe9bc8..226233fbb 100644 --- a/addons/grafana/deployment.yaml +++ b/addons/grafana/deployment.yaml @@ -23,7 +23,7 @@ spec: spec: containers: - name: grafana - image: grafana/grafana:6.0.0 + image: grafana/grafana:6.0.1 env: - name: GF_PATHS_CONFIG value: "/etc/grafana/custom.ini" From 9493ed3b1dc075d75032975ac0c95f4578a0e405 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Sat, 9 Mar 2019 19:34:15 -0800 Subject: [PATCH 048/523] Change default iPXE kernel/initrd download from HTTP to HTTPS * Require an iPXE-enabled network boot environment with support for TLS downloads. PXE clients must chainload to iPXE firmware compiled with `DOWNLOAD_PROTO_HTTPS` enabled ([crypto](https://ipxe.org/crypto)) * iPXE's pre-compiled firmware binaries do _not_ enable HTTPS. Admins should build iPXE from source with support enabled * Affects the Container Linux and Flatcar Linux install profiles that pull from public downloads. No effect when cached_install=true or using Fedora Atomic, as those download from Matchbox * Add `download_protocol` variable. Recognizing boot firmware TLS support is difficult in some environments, set the protocol to "http" for the old behavior (discouraged) --- CHANGES.md | 7 +++++++ bare-metal/container-linux/kubernetes/profiles.tf | 8 ++++---- bare-metal/container-linux/kubernetes/variables.tf | 6 ++++++ docs/cl/bare-metal.md | 13 +++++++++---- docs/topics/hardware.md | 2 +- 5 files changed, 27 insertions(+), 9 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 4b23c31d0..fe38ec9b8 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -11,6 +11,13 @@ Notable changes between versions. * Support `terraform-provider-aws` v2.0+ ([#419](https://github.com/poseidon/typhoon/pull/419)) +#### Bare-Metal + +* Change the default iPXE kernel and initrd download protocol from HTTP to HTTPS ([#420](https://github.com/poseidon/typhoon/pull/420)) + * Require an iPXE-enabled network boot environment with support for TLS downloads. PXE clients must chainload to iPXE firmware compiled with `DOWNLOAD_PROTO_HTTPS` [enabled](https://ipxe.org/crypto). (**action required**) + * Affects Container Linux and Flatcar Linux install profiles that pull from public images (default). No affect when `cached_install=true` or Fedora Atomic, since those download from Matchbox + * Add `download_protocol` variable. Recognizing boot firmware TLS support is difficult in some environments, set the protocol to "http" for the old behavior (discouraged) + #### Addons * Update Prometheus from v2.7.1 to v2.7.2 diff --git a/bare-metal/container-linux/kubernetes/profiles.tf b/bare-metal/container-linux/kubernetes/profiles.tf index 4b576efeb..11e718c70 100644 --- a/bare-metal/container-linux/kubernetes/profiles.tf +++ b/bare-metal/container-linux/kubernetes/profiles.tf @@ -11,10 +11,10 @@ resource "matchbox_profile" "container-linux-install" { count = "${length(var.controller_names) + length(var.worker_names)}" name = "${format("%s-container-linux-install-%s", var.cluster_name, element(concat(var.controller_names, var.worker_names), count.index))}" - kernel = "http://${local.channel}.release.core-os.net/amd64-usr/${var.os_version}/coreos_production_pxe.vmlinuz" + kernel = "${var.download_protocol}://${local.channel}.release.core-os.net/amd64-usr/${var.os_version}/coreos_production_pxe.vmlinuz" initrd = [ - "http://${local.channel}.release.core-os.net/amd64-usr/${var.os_version}/coreos_production_pxe_image.cpio.gz", + "${var.download_protocol}://${local.channel}.release.core-os.net/amd64-usr/${var.os_version}/coreos_production_pxe_image.cpio.gz", ] args = [ @@ -96,10 +96,10 @@ resource "matchbox_profile" "flatcar-install" { count = "${length(var.controller_names) + length(var.worker_names)}" name = "${format("%s-flatcar-install-%s", var.cluster_name, element(concat(var.controller_names, var.worker_names), count.index))}" - kernel = "http://${local.channel}.release.flatcar-linux.net/amd64-usr/${var.os_version}/flatcar_production_pxe.vmlinuz" + kernel = "${var.download_protocol}://${local.channel}.release.flatcar-linux.net/amd64-usr/${var.os_version}/flatcar_production_pxe.vmlinuz" initrd = [ - "http://${local.channel}.release.flatcar-linux.net/amd64-usr/${var.os_version}/flatcar_production_pxe_image.cpio.gz", + "${var.download_protocol}://${local.channel}.release.flatcar-linux.net/amd64-usr/${var.os_version}/flatcar_production_pxe_image.cpio.gz", ] args = [ diff --git a/bare-metal/container-linux/kubernetes/variables.tf b/bare-metal/container-linux/kubernetes/variables.tf index da09eeeef..3e0a40434 100644 --- a/bare-metal/container-linux/kubernetes/variables.tf +++ b/bare-metal/container-linux/kubernetes/variables.tf @@ -118,6 +118,12 @@ variable "cluster_domain_suffix" { default = "cluster.local" } +variable "download_protocol" { + type = "string" + default = "https" + description = "Protocol iPXE should use to download the kernel and initrd. Defaults to https, which requires iPXE compiled with crypto support. Unused if cached_install is true." +} + variable "cached_install" { type = "string" default = "false" diff --git a/docs/cl/bare-metal.md b/docs/cl/bare-metal.md index 73b14542e..a6d11b3a7 100644 --- a/docs/cl/bare-metal.md +++ b/docs/cl/bare-metal.md @@ -9,7 +9,7 @@ Controllers are provisioned to run an `etcd-member` peer and a `kubelet` service ## Requirements * Machines with 2GB RAM, 30GB disk, PXE-enabled NIC, IPMI -* PXE-enabled [network boot](https://coreos.com/matchbox/docs/latest/network-setup.html) environment +* PXE-enabled [network boot](https://coreos.com/matchbox/docs/latest/network-setup.html) environment (with HTTPS support) * Matchbox v0.6+ deployment with API enabled * Matchbox credentials `client.crt`, `client.key`, `ca.crt` * Terraform v0.11.x, [terraform-provider-matchbox](https://github.com/coreos/terraform-provider-matchbox), and [terraform-provider-ct](https://github.com/coreos/terraform-provider-ct) installed locally @@ -82,7 +82,7 @@ $ openssl s_client -connect matchbox.example.com:8081 \ ## PXE Environment -Create a iPXE-enabled network boot environment. Configure PXE clients to chainload [iPXE](http://ipxe.org/cmd) and instruct iPXE clients to chainload from your Matchbox service's `/boot.ipxe` endpoint. +Create an iPXE-enabled network boot environment. Configure PXE clients to chainload [iPXE](http://ipxe.org/cmd) firmware compiled to support [HTTPS downloads](https://ipxe.org/crypto). Instruct iPXE clients to chainload from your Matchbox service's `/boot.ipxe` endpoint. For networks already supporting iPXE clients, you can add a `default.ipxe` config. @@ -93,8 +93,6 @@ chain http://matchbox.foo:8080/boot.ipxe For networks with Ubiquiti Routers, you can [configure the router](/topics/hardware/#ubiquiti) itself to chainload machines to iPXE and Matchbox. -For a small lab, you may wish to checkout the [quay.io/coreos/dnsmasq](https://quay.io/repository/coreos/dnsmasq) container image and [copy-paste examples](https://github.com/coreos/matchbox/blob/master/Documentation/network-setup.md#coreosdnsmasq). - Read about the [many ways](https://coreos.com/matchbox/docs/latest/network-setup.html) to setup a compliant iPXE-enabled network. There is quite a bit of flexibility: * Continue using existing DHCP, TFTP, or DNS services @@ -104,6 +102,9 @@ Read about the [many ways](https://coreos.com/matchbox/docs/latest/network-setup !!! note "" TFTP chainloading to modern boot firmware, like iPXE, avoids issues with old NICs and allows faster transfer protocols like HTTP to be used. +!!! warning + Compile iPXE from [source](https://github.com/ipxe/ipxe) with support for [HTTPS downloads](https://ipxe.org/crypto). iPXE's pre-built firmware binaries do not enable this. If you cannot enable HTTPS downloads, set `download_protocol = "http"` (discouraged). + ## Terraform Setup Install [Terraform](https://www.terraform.io/downloads.html) v0.11.x on your system. @@ -215,6 +216,9 @@ module "bare-metal-mercury" { "node2.example.com", "node3.example.com", ] + + # set to http only if you cannot chainload to iPXE firmware with https support + # download_protocol = "http" } ``` @@ -375,6 +379,7 @@ Check the [variables.tf](https://github.com/poseidon/typhoon/blob/master/bare-me | Name | Description | Default | Example | |:-----|:------------|:--------|:--------| +| download_protocol | Protocol iPXE uses to download the kernel and initrd. iPXE must be compiled with [crypto](https://ipxe.org/crypto) support for https. Unused if cached_install is true | "https" | "http" | | cached_install | PXE boot and install from the Matchbox `/assets` cache. Admin MUST have downloaded Container Linux or Flatcar images into the cache | false | true | | install_disk | Disk device where Container Linux should be installed | "/dev/sda" | "/dev/sdb" | | networking | Choice of networking provider | "calico" | "calico" or "flannel" | diff --git a/docs/topics/hardware.md b/docs/topics/hardware.md index 385a1fee0..5751b18bf 100644 --- a/docs/topics/hardware.md +++ b/docs/topics/hardware.md @@ -47,7 +47,7 @@ set service dns forwarding options 'pxe-service=tag:ipxe,x86PC,iPXE,http://match ### TFTP -Use `dnsmasq` as a TFTP server to serve `undionly.kpxe`. Compiling from [source](https://github.com/ipxe/ipxe) with TLS support is recommended, but you may also download a [pre-compiled](http://boot.ipxe.org/undionly.kpxe) copy. +Use `dnsmasq` as a TFTP server to serve `undionly.kpxe`. Compiling from [source](https://github.com/ipxe/ipxe) with TLS support is strongly recommended. If you use a [pre-compiled](http://boot.ipxe.org/undionly.kpxe) copy, you must set `download_protocol = "http"` in your cluster definition (discouraged). ``` sudo -i From 2019177b6b823e8b557c5faee3ebf7eb34921937 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Tue, 12 Mar 2019 01:19:54 -0700 Subject: [PATCH 049/523] Fix implicit map assignments to be explicit * Terraform v0.12 will require map assignments be explicit, part of v0.12 readiness --- aws/container-linux/kubernetes/controllers.tf | 2 +- aws/fedora-atomic/kubernetes/controllers.tf | 2 +- azure/container-linux/kubernetes/controllers.tf | 2 +- bare-metal/container-linux/kubernetes/groups.tf | 6 +++--- bare-metal/container-linux/kubernetes/profiles.tf | 8 ++++---- bare-metal/fedora-atomic/kubernetes/groups.tf | 8 ++++---- bare-metal/fedora-atomic/kubernetes/profiles.tf | 6 +++--- digital-ocean/container-linux/kubernetes/controllers.tf | 2 +- digital-ocean/fedora-atomic/kubernetes/controllers.tf | 2 +- google-cloud/container-linux/kubernetes/controllers.tf | 4 ++-- .../container-linux/kubernetes/workers/workers.tf | 2 +- google-cloud/fedora-atomic/kubernetes/controllers.tf | 4 ++-- google-cloud/fedora-atomic/kubernetes/workers/workers.tf | 2 +- 13 files changed, 25 insertions(+), 25 deletions(-) diff --git a/aws/container-linux/kubernetes/controllers.tf b/aws/container-linux/kubernetes/controllers.tf index d43e495ba..7cac72c08 100644 --- a/aws/container-linux/kubernetes/controllers.tf +++ b/aws/container-linux/kubernetes/controllers.tf @@ -79,7 +79,7 @@ data "template_file" "etcds" { count = "${var.controller_count}" template = "etcd$${index}=https://$${cluster_name}-etcd$${index}.$${dns_zone}:2380" - vars { + vars = { index = "${count.index}" cluster_name = "${var.cluster_name}" dns_zone = "${var.dns_zone}" diff --git a/aws/fedora-atomic/kubernetes/controllers.tf b/aws/fedora-atomic/kubernetes/controllers.tf index 939c4d49d..26ba3ca36 100644 --- a/aws/fedora-atomic/kubernetes/controllers.tf +++ b/aws/fedora-atomic/kubernetes/controllers.tf @@ -71,7 +71,7 @@ data "template_file" "etcds" { count = "${var.controller_count}" template = "etcd$${index}=https://$${cluster_name}-etcd$${index}.$${dns_zone}:2380" - vars { + vars = { index = "${count.index}" cluster_name = "${var.cluster_name}" dns_zone = "${var.dns_zone}" diff --git a/azure/container-linux/kubernetes/controllers.tf b/azure/container-linux/kubernetes/controllers.tf index ffa05a1ec..63a8fb17f 100644 --- a/azure/container-linux/kubernetes/controllers.tf +++ b/azure/container-linux/kubernetes/controllers.tf @@ -160,7 +160,7 @@ data "template_file" "etcds" { count = "${var.controller_count}" template = "etcd$${index}=https://$${cluster_name}-etcd$${index}.$${dns_zone}:2380" - vars { + vars = { index = "${count.index}" cluster_name = "${var.cluster_name}" dns_zone = "${var.dns_zone}" diff --git a/bare-metal/container-linux/kubernetes/groups.tf b/bare-metal/container-linux/kubernetes/groups.tf index 6a467af37..728ddc4b6 100644 --- a/bare-metal/container-linux/kubernetes/groups.tf +++ b/bare-metal/container-linux/kubernetes/groups.tf @@ -5,7 +5,7 @@ resource "matchbox_group" "install" { profile = "${local.flavor == "flatcar" ? var.cached_install == "true" ? element(matchbox_profile.cached-flatcar-linux-install.*.name, count.index) : element(matchbox_profile.flatcar-install.*.name, count.index) : var.cached_install == "true" ? element(matchbox_profile.cached-container-linux-install.*.name, count.index) : element(matchbox_profile.container-linux-install.*.name, count.index)}" - selector { + selector = { mac = "${element(concat(var.controller_macs, var.worker_macs), count.index)}" } } @@ -15,7 +15,7 @@ resource "matchbox_group" "controller" { name = "${format("%s-%s", var.cluster_name, element(var.controller_names, count.index))}" profile = "${element(matchbox_profile.controllers.*.name, count.index)}" - selector { + selector = { mac = "${element(var.controller_macs, count.index)}" os = "installed" } @@ -26,7 +26,7 @@ resource "matchbox_group" "worker" { name = "${format("%s-%s", var.cluster_name, element(var.worker_names, count.index))}" profile = "${element(matchbox_profile.workers.*.name, count.index)}" - selector { + selector = { mac = "${element(var.worker_macs, count.index)}" os = "installed" } diff --git a/bare-metal/container-linux/kubernetes/profiles.tf b/bare-metal/container-linux/kubernetes/profiles.tf index 11e718c70..7d0d481cc 100644 --- a/bare-metal/container-linux/kubernetes/profiles.tf +++ b/bare-metal/container-linux/kubernetes/profiles.tf @@ -34,7 +34,7 @@ data "template_file" "container-linux-install-configs" { template = "${file("${path.module}/cl/install.yaml.tmpl")}" - vars { + vars = { os_flavor = "${local.flavor}" os_channel = "${local.channel}" os_version = "${var.os_version}" @@ -77,7 +77,7 @@ data "template_file" "cached-container-linux-install-configs" { template = "${file("${path.module}/cl/install.yaml.tmpl")}" - vars { + vars = { os_flavor = "${local.flavor}" os_channel = "${local.channel}" os_version = "${var.os_version}" @@ -159,7 +159,7 @@ data "template_file" "controller-configs" { template = "${file("${path.module}/cl/controller.yaml.tmpl")}" - vars { + vars = { domain_name = "${element(var.controller_domains, count.index)}" etcd_name = "${element(var.controller_names, count.index)}" etcd_initial_cluster = "${join(",", formatlist("%s=https://%s:2380", var.controller_names, var.controller_domains))}" @@ -190,7 +190,7 @@ data "template_file" "worker-configs" { template = "${file("${path.module}/cl/worker.yaml.tmpl")}" - vars { + vars = { domain_name = "${element(var.worker_domains, count.index)}" cluster_dns_service_ip = "${module.bootkube.cluster_dns_service_ip}" cluster_domain_suffix = "${var.cluster_domain_suffix}" diff --git a/bare-metal/fedora-atomic/kubernetes/groups.tf b/bare-metal/fedora-atomic/kubernetes/groups.tf index a18147bd8..200fd3b96 100644 --- a/bare-metal/fedora-atomic/kubernetes/groups.tf +++ b/bare-metal/fedora-atomic/kubernetes/groups.tf @@ -5,11 +5,11 @@ resource "matchbox_group" "install" { name = "${format("fedora-install-%s", element(concat(var.controller_names, var.worker_names), count.index))}" profile = "${element(matchbox_profile.cached-fedora-install.*.name, count.index)}" - selector { + selector = { mac = "${element(concat(var.controller_macs, var.worker_macs), count.index)}" } - metadata { + metadata = { ssh_authorized_key = "${var.ssh_authorized_key}" } } @@ -19,7 +19,7 @@ resource "matchbox_group" "controller" { name = "${format("%s-%s", var.cluster_name, element(var.controller_names, count.index))}" profile = "${element(matchbox_profile.controllers.*.name, count.index)}" - selector { + selector = { mac = "${element(var.controller_macs, count.index)}" os = "installed" } @@ -30,7 +30,7 @@ resource "matchbox_group" "worker" { name = "${format("%s-%s", var.cluster_name, element(var.worker_names, count.index))}" profile = "${element(matchbox_profile.workers.*.name, count.index)}" - selector { + selector = { mac = "${element(var.worker_macs, count.index)}" os = "installed" } diff --git a/bare-metal/fedora-atomic/kubernetes/profiles.tf b/bare-metal/fedora-atomic/kubernetes/profiles.tf index 30aec83c3..1f8b1c5ea 100644 --- a/bare-metal/fedora-atomic/kubernetes/profiles.tf +++ b/bare-metal/fedora-atomic/kubernetes/profiles.tf @@ -33,7 +33,7 @@ data "template_file" "install-kickstarts" { template = "${file("${path.module}/kickstart/fedora-atomic.ks.tmpl")}" - vars { + vars = { matchbox_http_endpoint = "${var.matchbox_http_endpoint}" atomic_assets_endpoint = "${local.atomic_assets_endpoint}" mac = "${element(concat(var.controller_macs, var.worker_macs), count.index)}" @@ -54,7 +54,7 @@ data "template_file" "controller-configs" { template = "${file("${path.module}/cloudinit/controller.yaml.tmpl")}" - vars { + vars = { domain_name = "${element(var.controller_domains, count.index)}" etcd_name = "${element(var.controller_names, count.index)}" etcd_initial_cluster = "${join(",", formatlist("%s=https://%s:2380", var.controller_names, var.controller_domains))}" @@ -78,7 +78,7 @@ data "template_file" "worker-configs" { template = "${file("${path.module}/cloudinit/worker.yaml.tmpl")}" - vars { + vars = { domain_name = "${element(var.worker_domains, count.index)}" cluster_dns_service_ip = "${module.bootkube.cluster_dns_service_ip}" cluster_domain_suffix = "${var.cluster_domain_suffix}" diff --git a/digital-ocean/container-linux/kubernetes/controllers.tf b/digital-ocean/container-linux/kubernetes/controllers.tf index fadaf9208..1f8f15c79 100644 --- a/digital-ocean/container-linux/kubernetes/controllers.tf +++ b/digital-ocean/container-linux/kubernetes/controllers.tf @@ -93,7 +93,7 @@ data "template_file" "etcds" { count = "${var.controller_count}" template = "etcd$${index}=https://$${cluster_name}-etcd$${index}.$${dns_zone}:2380" - vars { + vars = { index = "${count.index}" cluster_name = "${var.cluster_name}" dns_zone = "${var.dns_zone}" diff --git a/digital-ocean/fedora-atomic/kubernetes/controllers.tf b/digital-ocean/fedora-atomic/kubernetes/controllers.tf index 19a950414..deaa87b17 100644 --- a/digital-ocean/fedora-atomic/kubernetes/controllers.tf +++ b/digital-ocean/fedora-atomic/kubernetes/controllers.tf @@ -87,7 +87,7 @@ data "template_file" "etcds" { count = "${var.controller_count}" template = "etcd$${index}=https://$${cluster_name}-etcd$${index}.$${dns_zone}:2380" - vars { + vars = { index = "${count.index}" cluster_name = "${var.cluster_name}" dns_zone = "${var.dns_zone}" diff --git a/google-cloud/container-linux/kubernetes/controllers.tf b/google-cloud/container-linux/kubernetes/controllers.tf index dbb445a69..3578f04f4 100644 --- a/google-cloud/container-linux/kubernetes/controllers.tf +++ b/google-cloud/container-linux/kubernetes/controllers.tf @@ -36,7 +36,7 @@ resource "google_compute_instance" "controllers" { machine_type = "${var.controller_type}" min_cpu_platform = "Intel Haswell" - metadata { + metadata = { user-data = "${element(data.ct_config.controller-ignitions.*.rendered, count.index)}" } @@ -99,7 +99,7 @@ data "template_file" "etcds" { count = "${var.controller_count}" template = "etcd$${index}=https://$${cluster_name}-etcd$${index}.$${dns_zone}:2380" - vars { + vars = { index = "${count.index}" cluster_name = "${var.cluster_name}" dns_zone = "${var.dns_zone}" diff --git a/google-cloud/container-linux/kubernetes/workers/workers.tf b/google-cloud/container-linux/kubernetes/workers/workers.tf index 10b0e41e4..422ff0398 100644 --- a/google-cloud/container-linux/kubernetes/workers/workers.tf +++ b/google-cloud/container-linux/kubernetes/workers/workers.tf @@ -28,7 +28,7 @@ resource "google_compute_instance_template" "worker" { machine_type = "${var.machine_type}" min_cpu_platform = "Intel Haswell" - metadata { + metadata = { user-data = "${data.ct_config.worker-ignition.rendered}" } diff --git a/google-cloud/fedora-atomic/kubernetes/controllers.tf b/google-cloud/fedora-atomic/kubernetes/controllers.tf index cc7d605a3..c166975e4 100644 --- a/google-cloud/fedora-atomic/kubernetes/controllers.tf +++ b/google-cloud/fedora-atomic/kubernetes/controllers.tf @@ -36,7 +36,7 @@ resource "google_compute_instance" "controllers" { machine_type = "${var.controller_type}" min_cpu_platform = "Intel Haswell" - metadata { + metadata = { user-data = "${element(data.template_file.controller-cloudinit.*.rendered, count.index)}" } @@ -91,7 +91,7 @@ data "template_file" "etcds" { count = "${var.controller_count}" template = "etcd$${index}=https://$${cluster_name}-etcd$${index}.$${dns_zone}:2380" - vars { + vars = { index = "${count.index}" cluster_name = "${var.cluster_name}" dns_zone = "${var.dns_zone}" diff --git a/google-cloud/fedora-atomic/kubernetes/workers/workers.tf b/google-cloud/fedora-atomic/kubernetes/workers/workers.tf index 180556e85..95ea6e5a1 100644 --- a/google-cloud/fedora-atomic/kubernetes/workers/workers.tf +++ b/google-cloud/fedora-atomic/kubernetes/workers/workers.tf @@ -28,7 +28,7 @@ resource "google_compute_instance_template" "worker" { machine_type = "${var.machine_type}" min_cpu_platform = "Intel Haswell" - metadata { + metadata = { user-data = "${data.template_file.worker-cloudinit.rendered}" } From e0bee2e4173a71633c75509dcf2a1b8d6a2a6398 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Wed, 13 Mar 2019 22:11:38 -0700 Subject: [PATCH 050/523] Update Prometheus from v2.7.2 to v2.8.0 * https://github.com/prometheus/prometheus/releases/tag/v2.8.0 --- CHANGES.md | 2 +- addons/prometheus/deployment.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index fe38ec9b8..a27bfe099 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -20,7 +20,7 @@ Notable changes between versions. #### Addons -* Update Prometheus from v2.7.1 to v2.7.2 +* Update Prometheus from v2.7.1 to [v2.8.0](https://github.com/prometheus/prometheus/releases/tag/v2.8.0) * Update Grafana from v6.0.0 to v6.0.1 ## v1.13.4 diff --git a/addons/prometheus/deployment.yaml b/addons/prometheus/deployment.yaml index abfb0eec0..5d62fd4a6 100644 --- a/addons/prometheus/deployment.yaml +++ b/addons/prometheus/deployment.yaml @@ -20,7 +20,7 @@ spec: serviceAccountName: prometheus containers: - name: prometheus - image: quay.io/prometheus/prometheus:v2.7.2 + image: quay.io/prometheus/prometheus:v2.8.0 args: - --web.listen-address=0.0.0.0:9090 - --config.file=/etc/prometheus/prometheus.yaml From 3d6a6d4adb7d7dda018bb6fbac57beb2fa3ac7cd Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Thu, 14 Mar 2019 00:55:55 -0700 Subject: [PATCH 051/523] Re-add Kubelet metadata service dependency on DigitalOcean * Restore the original special-casing of DigitalOcean Kubelets * Fix node metadata InternalIP being set to the IP of the default gateway on DigitalOcean nodes (regressed in v1.12.3) * Reverts the "pretty" node names on DigitalOcean (worker-2 vs IP) * Closes #424 (full details) --- CHANGES.md | 6 ++++++ .../kubernetes/cl/controller.yaml.tmpl | 4 ++++ .../kubernetes/cl/worker.yaml.tmpl | 4 ++++ .../kubernetes/cloudinit/controller.yaml.tmpl | 16 ++++++++++++++++ .../kubernetes/cloudinit/worker.yaml.tmpl | 16 ++++++++++++++++ docs/atomic/digital-ocean.md | 8 ++++---- docs/cl/digital-ocean.md | 8 ++++---- 7 files changed, 54 insertions(+), 8 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index a27bfe099..9b01e277d 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -18,6 +18,12 @@ Notable changes between versions. * Affects Container Linux and Flatcar Linux install profiles that pull from public images (default). No affect when `cached_install=true` or Fedora Atomic, since those download from Matchbox * Add `download_protocol` variable. Recognizing boot firmware TLS support is difficult in some environments, set the protocol to "http" for the old behavior (discouraged) +#### DigitalOcean + +* Fix kubelet hostname-override to set node metadata InternalIP correctly ([#424](https://github.com/poseidon/typhoon/issues/424)) + * Uniquely, DigitalOcean does not resolve hostnames to instance private IPs. Kubelet auto-detect mechanisms require the internal IP be set directly. + * Regressed in v1.12.3 ([#337](https://github.com/poseidon/typhoon/pull/337)) which aimed to provide friendly hostname-based node names on DigitalOcean + #### Addons * Update Prometheus from v2.7.1 to [v2.8.0](https://github.com/prometheus/prometheus/releases/tag/v2.8.0) diff --git a/digital-ocean/container-linux/kubernetes/cl/controller.yaml.tmpl b/digital-ocean/container-linux/kubernetes/cl/controller.yaml.tmpl index 9586bf56f..01fca5df3 100644 --- a/digital-ocean/container-linux/kubernetes/cl/controller.yaml.tmpl +++ b/digital-ocean/container-linux/kubernetes/cl/controller.yaml.tmpl @@ -56,9 +56,12 @@ systemd: contents: | [Unit] Description=Kubelet via Hyperkube + Requires=coreos-metadata.service + After=coreos-metadata.service Wants=rpc-statd.service [Service] EnvironmentFile=/etc/kubernetes/kubelet.env + EnvironmentFile=/run/metadata/coreos Environment="RKT_RUN_ARGS=--uuid-file-save=/var/cache/kubelet-pod.uuid \ --volume=resolv,kind=host,source=/etc/resolv.conf \ --mount volume=resolv,target=/etc/resolv.conf \ @@ -90,6 +93,7 @@ systemd: --cluster_domain=${cluster_domain_suffix} \ --cni-conf-dir=/etc/kubernetes/cni/net.d \ --exit-on-lock-contention \ + --hostname-override=$${COREOS_DIGITALOCEAN_IPV4_PRIVATE_0} \ --kubeconfig=/etc/kubernetes/kubeconfig \ --lock-file=/var/run/lock/kubelet.lock \ --network-plugin=cni \ diff --git a/digital-ocean/container-linux/kubernetes/cl/worker.yaml.tmpl b/digital-ocean/container-linux/kubernetes/cl/worker.yaml.tmpl index f1a58aaf1..256c864fc 100644 --- a/digital-ocean/container-linux/kubernetes/cl/worker.yaml.tmpl +++ b/digital-ocean/container-linux/kubernetes/cl/worker.yaml.tmpl @@ -31,9 +31,12 @@ systemd: contents: | [Unit] Description=Kubelet via Hyperkube + Requires=coreos-metadata.service + After=coreos-metadata.service Wants=rpc-statd.service [Service] EnvironmentFile=/etc/kubernetes/kubelet.env + EnvironmentFile=/run/metadata/coreos Environment="RKT_RUN_ARGS=--uuid-file-save=/var/cache/kubelet-pod.uuid \ --volume=resolv,kind=host,source=/etc/resolv.conf \ --mount volume=resolv,target=/etc/resolv.conf \ @@ -63,6 +66,7 @@ systemd: --cluster_domain=${cluster_domain_suffix} \ --cni-conf-dir=/etc/kubernetes/cni/net.d \ --exit-on-lock-contention \ + --hostname-override=$${COREOS_DIGITALOCEAN_IPV4_PRIVATE_0} \ --kubeconfig=/etc/kubernetes/kubeconfig \ --lock-file=/var/run/lock/kubelet.lock \ --network-plugin=cni \ diff --git a/digital-ocean/fedora-atomic/kubernetes/cloudinit/controller.yaml.tmpl b/digital-ocean/fedora-atomic/kubernetes/cloudinit/controller.yaml.tmpl index 9f38161c9..79747dcda 100644 --- a/digital-ocean/fedora-atomic/kubernetes/cloudinit/controller.yaml.tmpl +++ b/digital-ocean/fedora-atomic/kubernetes/cloudinit/controller.yaml.tmpl @@ -19,9 +19,24 @@ write_files: ETCD_PEER_CERT_FILE=/etc/ssl/certs/etcd/peer.crt ETCD_PEER_KEY_FILE=/etc/ssl/certs/etcd/peer.key ETCD_PEER_CLIENT_CERT_AUTH=true + - path: /etc/systemd/system/cloud-metadata.service + content: | + [Unit] + Description=Cloud metadata agent + [Service] + Type=oneshot + Environment=OUTPUT=/run/metadata/cloud + ExecStart=/usr/bin/mkdir -p /run/metadata + ExecStart=/usr/bin/bash -c 'echo "HOSTNAME_OVERRIDE=$(curl\ + --url http://169.254.169.254/metadata/v1/interfaces/private/0/ipv4/address\ + --retry 10)" > $${OUTPUT}' + [Install] + WantedBy=multi-user.target - path: /etc/systemd/system/kubelet.service.d/10-typhoon.conf content: | [Unit] + Requires=cloud-metadata.service + After=cloud-metadata.service Wants=rpc-statd.service [Service] ExecStartPre=/bin/mkdir -p /opt/cni/bin @@ -79,6 +94,7 @@ runcmd: - "atomic install --system --name=kubelet quay.io/poseidon/kubelet:v1.13.4" - "atomic install --system --name=bootkube quay.io/poseidon/bootkube:v0.14.0" - [systemctl, start, --no-block, etcd.service] + - [systemctl, enable, cloud-metadata.service] - [systemctl, enable, kubelet.path] - [systemctl, start, --no-block, kubelet.path] users: diff --git a/digital-ocean/fedora-atomic/kubernetes/cloudinit/worker.yaml.tmpl b/digital-ocean/fedora-atomic/kubernetes/cloudinit/worker.yaml.tmpl index 223e54a77..b87b6acc9 100644 --- a/digital-ocean/fedora-atomic/kubernetes/cloudinit/worker.yaml.tmpl +++ b/digital-ocean/fedora-atomic/kubernetes/cloudinit/worker.yaml.tmpl @@ -1,8 +1,23 @@ #cloud-config write_files: + - path: /etc/systemd/system/cloud-metadata.service + content: | + [Unit] + Description=Cloud metadata agent + [Service] + Type=oneshot + Environment=OUTPUT=/run/metadata/cloud + ExecStart=/usr/bin/mkdir -p /run/metadata + ExecStart=/usr/bin/bash -c 'echo "HOSTNAME_OVERRIDE=$(curl\ + --url http://169.254.169.254/metadata/v1/interfaces/private/0/ipv4/address\ + --retry 10)" > $${OUTPUT}' + [Install] + WantedBy=multi-user.target - path: /etc/systemd/system/kubelet.service.d/10-typhoon.conf content: | [Unit] + Requires=cloud-metadata.service + After=cloud-metadata.service Wants=rpc-statd.service [Service] ExecStartPre=/bin/mkdir -p /opt/cni/bin @@ -51,6 +66,7 @@ bootcmd: - [modprobe, ip_vs] runcmd: - [systemctl, daemon-reload] + - [systemctl, enable, cloud-metadata.service] - "atomic install --system --name=kubelet quay.io/poseidon/kubelet:v1.13.4" - [systemctl, enable, kubelet.path] - [systemctl, start, --no-block, kubelet.path] diff --git a/docs/atomic/digital-ocean.md b/docs/atomic/digital-ocean.md index f17089acb..f302d1cf9 100644 --- a/docs/atomic/digital-ocean.md +++ b/docs/atomic/digital-ocean.md @@ -152,9 +152,9 @@ In 3-6 minutes, the Kubernetes cluster will be ready. $ export KUBECONFIG=/home/user/.secrets/clusters/nemo/auth/kubeconfig $ kubectl get nodes NAME STATUS ROLES AGE VERSION -nemo-controller-0 Ready controller,master 10m v1.13.4 -nemo-worker-0 Ready node 10m v1.13.4 -nemo-worker-1 Ready node 10m v1.13.4 +10.132.110.130 Ready controller,master 10m v1.13.4 +10.132.115.81 Ready node 10m v1.13.4 +10.132.124.107 Ready node 10m v1.13.4 ``` List the pods. @@ -175,7 +175,7 @@ kube-system kube-proxy-k35rc 1/1 Running 0 kube-system kube-scheduler-3895335239-2bc4c 1/1 Running 0 11m kube-system kube-scheduler-3895335239-b7q47 1/1 Running 1 11m kube-system pod-checkpointer-pr1lq 1/1 Running 0 11m -kube-system pod-checkpointer-pr1lq-nemo-controller-0 1/1 Running 0 10m +kube-system pod-checkpointer-pr1lq-10.132.115.81 1/1 Running 0 10m ``` ## Going Further diff --git a/docs/cl/digital-ocean.md b/docs/cl/digital-ocean.md index ade88a96a..b04d9914c 100644 --- a/docs/cl/digital-ocean.md +++ b/docs/cl/digital-ocean.md @@ -160,9 +160,9 @@ In 3-6 minutes, the Kubernetes cluster will be ready. $ export KUBECONFIG=/home/user/.secrets/clusters/nemo/auth/kubeconfig $ kubectl get nodes NAME STATUS ROLES AGE VERSION -nemo-controller-0 Ready controller,master 10m v1.13.4 -nemo-worker-0 Ready node 10m v1.13.4 -nemo-worker-1 Ready node 10m v1.13.4 +10.132.110.130 Ready controller,master 10m v1.13.4 +10.132.115.81 Ready node 10m v1.13.4 +10.132.124.107 Ready node 10m v1.13.4 ``` List the pods. @@ -183,7 +183,7 @@ kube-system kube-proxy-k35rc 1/1 Running 0 kube-system kube-scheduler-3895335239-2bc4c 1/1 Running 0 11m kube-system kube-scheduler-3895335239-b7q47 1/1 Running 1 11m kube-system pod-checkpointer-pr1lq 1/1 Running 0 11m -kube-system pod-checkpointer-pr1lq-nemo-controller-0 1/1 Running 0 10m +kube-system pod-checkpointer-pr1lq-10.132.115.81 1/1 Running 0 10m ``` ## Going Further From bf97a45b9d8da5750a7c88db51822d7d1d6ccb93 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Wed, 13 Mar 2019 23:46:04 -0700 Subject: [PATCH 052/523] Remove heapster manifests from addons * Heapster addon powers `kubectl top` * In early Kubernetes, people legitimately used and expected `kubectl top` to work, so the optional addon was provided * Today the standards are different. Many better monitoring tools exist, that are also less coupled to Kubernetes "kubectl top" reliance on a non-core extensions means its not in-scope for minimal Kubernetes clusters. No more exceptionalism * Finally, Heapster isn't that useful anymore. Its manifests have no need for Typhoon-specific modification * Look to prior releases if you still wish to apply heapster --- CHANGES.md | 4 ++ addons/heapster/cluster-role-binding.yaml | 12 ----- addons/heapster/cluster-role.yaml | 30 ----------- addons/heapster/deployment.yaml | 62 ----------------------- addons/heapster/role-binding.yaml | 13 ----- addons/heapster/role.yaml | 19 ------- addons/heapster/service-account.yaml | 5 -- addons/heapster/service.yaml | 12 ----- docs/addons/heapster.md | 19 ------- docs/addons/overview.md | 1 - mkdocs.yml | 1 - 11 files changed, 4 insertions(+), 174 deletions(-) delete mode 100644 addons/heapster/cluster-role-binding.yaml delete mode 100644 addons/heapster/cluster-role.yaml delete mode 100644 addons/heapster/deployment.yaml delete mode 100644 addons/heapster/role-binding.yaml delete mode 100644 addons/heapster/role.yaml delete mode 100644 addons/heapster/service-account.yaml delete mode 100644 addons/heapster/service.yaml delete mode 100644 docs/addons/heapster.md diff --git a/CHANGES.md b/CHANGES.md index 9b01e277d..14b7592e7 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -28,6 +28,10 @@ Notable changes between versions. * Update Prometheus from v2.7.1 to [v2.8.0](https://github.com/prometheus/prometheus/releases/tag/v2.8.0) * Update Grafana from v6.0.0 to v6.0.1 +* Remove heapster manifests from addons ([#427](https://github.com/poseidon/typhoon/pull/427)) + * Heapster addon powers `kubectl top` (in early Kubernetes, running the addon was expected). Today, there are better monitoring options. + * `kubectl top` reliance on a non-core extension means its not in-scope for minimal Kubernetes + * Look to prior releases if you still wish to apply heapster ## v1.13.4 diff --git a/addons/heapster/cluster-role-binding.yaml b/addons/heapster/cluster-role-binding.yaml deleted file mode 100644 index 6a36f0809..000000000 --- a/addons/heapster/cluster-role-binding.yaml +++ /dev/null @@ -1,12 +0,0 @@ -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: heapster -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: heapster -subjects: -- kind: ServiceAccount - name: heapster - namespace: kube-system diff --git a/addons/heapster/cluster-role.yaml b/addons/heapster/cluster-role.yaml deleted file mode 100644 index 7f4ac13f7..000000000 --- a/addons/heapster/cluster-role.yaml +++ /dev/null @@ -1,30 +0,0 @@ -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: heapster -rules: -- apiGroups: - - "" - resources: - - events - - namespaces - - nodes - - pods - verbs: - - get - - list - - watch -- apiGroups: - - extensions - resources: - - deployments - verbs: - - get - - list - - watch -- apiGroups: - - "" - resources: - - nodes/stats - verbs: - - get diff --git a/addons/heapster/deployment.yaml b/addons/heapster/deployment.yaml deleted file mode 100644 index 3e1424d30..000000000 --- a/addons/heapster/deployment.yaml +++ /dev/null @@ -1,62 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: heapster - namespace: kube-system -spec: - replicas: 1 - selector: - matchLabels: - name: heapster - phase: prod - template: - metadata: - labels: - name: heapster - phase: prod - annotations: - seccomp.security.alpha.kubernetes.io/pod: 'docker/default' - spec: - serviceAccountName: heapster - containers: - - name: heapster - image: k8s.gcr.io/heapster-amd64:v1.5.4 - command: - - /heapster - - --source=kubernetes.summary_api:''?useServiceAccount=true&kubeletHttps=true&kubeletPort=10250&insecure=true - livenessProbe: - httpGet: - path: /healthz - port: 8082 - scheme: HTTP - initialDelaySeconds: 180 - timeoutSeconds: 5 - - name: heapster-nanny - image: k8s.gcr.io/addon-resizer:1.7 - command: - - /pod_nanny - - --cpu=80m - - --extra-cpu=0.5m - - --memory=140Mi - - --extra-memory=4Mi - - --threshold=5 - - --deployment=heapster - - --container=heapster - - --poll-period=300000 - - --estimator=exponential - env: - - name: MY_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: MY_POD_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - resources: - limits: - cpu: 50m - memory: 90Mi - requests: - cpu: 50m - memory: 90Mi diff --git a/addons/heapster/role-binding.yaml b/addons/heapster/role-binding.yaml deleted file mode 100644 index c83567292..000000000 --- a/addons/heapster/role-binding.yaml +++ /dev/null @@ -1,13 +0,0 @@ -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - name: heapster - namespace: kube-system -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: Role - name: system:pod-nanny -subjects: -- kind: ServiceAccount - name: heapster - namespace: kube-system diff --git a/addons/heapster/role.yaml b/addons/heapster/role.yaml deleted file mode 100644 index 5c1296723..000000000 --- a/addons/heapster/role.yaml +++ /dev/null @@ -1,19 +0,0 @@ -apiVersion: rbac.authorization.k8s.io/v1 -kind: Role -metadata: - name: system:pod-nanny - namespace: kube-system -rules: -- apiGroups: - - "" - resources: - - pods - verbs: - - get -- apiGroups: - - "extensions" - resources: - - deployments - verbs: - - get - - update diff --git a/addons/heapster/service-account.yaml b/addons/heapster/service-account.yaml deleted file mode 100644 index c91d5b6bd..000000000 --- a/addons/heapster/service-account.yaml +++ /dev/null @@ -1,5 +0,0 @@ -apiVersion: v1 -kind: ServiceAccount -metadata: - name: heapster - namespace: kube-system diff --git a/addons/heapster/service.yaml b/addons/heapster/service.yaml deleted file mode 100644 index cff31adb2..000000000 --- a/addons/heapster/service.yaml +++ /dev/null @@ -1,12 +0,0 @@ -apiVersion: v1 -kind: Service -metadata: - name: heapster - namespace: kube-system -spec: - type: ClusterIP - selector: - name: heapster - ports: - - port: 80 - targetPort: 8082 diff --git a/docs/addons/heapster.md b/docs/addons/heapster.md deleted file mode 100644 index e10135446..000000000 --- a/docs/addons/heapster.md +++ /dev/null @@ -1,19 +0,0 @@ -# Heapster - -[Heapster](https://kubernetes.io/docs/user-guide/monitoring/) collects data from apiservers and kubelets and exposes it through a REST API. This API powers the `kubectl top` command. - -## Create - -```sh -kubectl apply -f addons/heapster -R -``` - -## Usage - -Allow heapster to run for a few minutes, then check CPU and memory usage. - -```sh -kubectl top node -kubectl top pod -``` - diff --git a/docs/addons/overview.md b/docs/addons/overview.md index 9c57b1499..c56fbd54d 100644 --- a/docs/addons/overview.md +++ b/docs/addons/overview.md @@ -4,7 +4,6 @@ Every Typhoon cluster is verified to work well with several post-install addons. * [CLUO](cluo.md) (Container Linux only) * Nginx [Ingress Controller](ingress.md) -* [Heapster](heapster.md) * [Prometheus](prometheus.md) * [Grafana](grafana.md) diff --git a/mkdocs.yml b/mkdocs.yml index ca655d5b2..dc6be25fb 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -77,7 +77,6 @@ nav: - 'Addons': - 'Overview': 'addons/overview.md' - 'CLUO': 'addons/cluo.md' - - 'Heapster': 'addons/heapster.md' - 'Nginx Ingress': 'addons/ingress.md' - 'Prometheus': 'addons/prometheus.md' - 'Grafana': 'addons/grafana.md' From aa630003a46508083d95144ebd72c56d7096a3ef Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Wed, 13 Mar 2019 23:05:14 -0700 Subject: [PATCH 053/523] Refresh Prometheus rules and Grafana dashboards * Refresh rules and dashboards from upstreams * Organize dashboards and stay below the ConfigMap size limit --- CHANGES.md | 2 + addons/grafana/dashboards-etcd.yaml | 1230 ++++++++++++ .../{dashboards.yaml => dashboards-k8s.yaml} | 1710 ++++------------- addons/grafana/deployment.yaml | 14 +- addons/prometheus/rules.yaml | 38 +- 5 files changed, 1637 insertions(+), 1357 deletions(-) create mode 100644 addons/grafana/dashboards-etcd.yaml rename addons/grafana/{dashboards.yaml => dashboards-k8s.yaml} (82%) diff --git a/CHANGES.md b/CHANGES.md index 14b7592e7..e145e7bb7 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -27,7 +27,9 @@ Notable changes between versions. #### Addons * Update Prometheus from v2.7.1 to [v2.8.0](https://github.com/prometheus/prometheus/releases/tag/v2.8.0) + * Refresh rules based on upstreams ([#426](https://github.com/poseidon/typhoon/pull/426)) * Update Grafana from v6.0.0 to v6.0.1 + * Refresh dashboards and organize to stay below ConfigMap size limit ([#426](https://github.com/poseidon/typhoon/pull/426)) * Remove heapster manifests from addons ([#427](https://github.com/poseidon/typhoon/pull/427)) * Heapster addon powers `kubectl top` (in early Kubernetes, running the addon was expected). Today, there are better monitoring options. * `kubectl top` reliance on a non-core extension means its not in-scope for minimal Kubernetes diff --git a/addons/grafana/dashboards-etcd.yaml b/addons/grafana/dashboards-etcd.yaml new file mode 100644 index 000000000..21f9880ff --- /dev/null +++ b/addons/grafana/dashboards-etcd.yaml @@ -0,0 +1,1230 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: grafana-dashboards-etcd + namespace: monitoring +data: + etcd.json: |- + { + "annotations": { + "list": [ + + ] + }, + "description": "etcd sample Grafana dashboard with Prometheus", + "editable": true, + "gnetId": null, + "hideControls": false, + "id": 6, + "links": [ + + ], + "refresh": false, + "rows": [ + { + "collapse": false, + "editable": true, + "height": "250px", + "panels": [ + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": "$datasource", + "editable": true, + "error": false, + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "id": 28, + "interval": null, + "isNew": true, + "links": [ + + ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "span": 3, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "targets": [ + { + "expr": "sum(etcd_server_has_leader{job=\"$cluster\"})", + "intervalFactor": 2, + "legendFormat": "", + "metric": "etcd_server_has_leader", + "refId": "A", + "step": 20 + } + ], + "thresholds": "", + "title": "Up", + "type": "singlestat", + "valueFontSize": "200%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "aliasColors": { + + }, + "bars": false, + "datasource": "$datasource", + "editable": true, + "error": false, + "fill": 0, + "id": 23, + "isNew": true, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "span": 5, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(grpc_server_started_total{job=\"$cluster\",grpc_type=\"unary\"}[5m]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "RPC Rate", + "metric": "grpc_server_started_total", + "refId": "A", + "step": 2 + }, + { + "expr": "sum(rate(grpc_server_handled_total{job=\"$cluster\",grpc_type=\"unary\",grpc_code!=\"OK\"}[5m]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "RPC Failed Rate", + "metric": "grpc_server_handled_total", + "refId": "B", + "step": 2 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "RPC Rate", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "datasource": "$datasource", + "editable": true, + "error": false, + "fill": 0, + "id": 41, + "isNew": true, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "span": 4, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(grpc_server_started_total{job=\"$cluster\",grpc_service=\"etcdserverpb.Watch\",grpc_type=\"bidi_stream\"}) - sum(grpc_server_handled_total{job=\"$cluster\",grpc_service=\"etcdserverpb.Watch\",grpc_type=\"bidi_stream\"})", + "intervalFactor": 2, + "legendFormat": "Watch Streams", + "metric": "grpc_server_handled_total", + "refId": "A", + "step": 4 + }, + { + "expr": "sum(grpc_server_started_total{job=\"$cluster\",grpc_service=\"etcdserverpb.Lease\",grpc_type=\"bidi_stream\"}) - sum(grpc_server_handled_total{job=\"$cluster\",grpc_service=\"etcdserverpb.Lease\",grpc_type=\"bidi_stream\"})", + "intervalFactor": 2, + "legendFormat": "Lease Streams", + "metric": "grpc_server_handled_total", + "refId": "B", + "step": 4 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Active Streams", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": "", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "showTitle": false, + "title": "Row" + }, + { + "collapse": false, + "editable": true, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "datasource": "$datasource", + "decimals": null, + "editable": true, + "error": false, + "fill": 0, + "grid": { + + }, + "id": 1, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "etcd_debugging_mvcc_db_total_size_in_bytes{job=\"$cluster\"}", + "hide": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{instance}} DB Size", + "metric": "", + "refId": "A", + "step": 4 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "DB Size", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "bytes", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "datasource": "$datasource", + "editable": true, + "error": false, + "fill": 0, + "grid": { + + }, + "id": 3, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 1, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "span": 4, + "stack": false, + "steppedLine": true, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(etcd_disk_wal_fsync_duration_seconds_bucket{job=\"$cluster\"}[5m])) by (instance, le))", + "hide": false, + "intervalFactor": 2, + "legendFormat": "{{instance}} WAL fsync", + "metric": "etcd_disk_wal_fsync_duration_seconds_bucket", + "refId": "A", + "step": 4 + }, + { + "expr": "histogram_quantile(0.99, sum(rate(etcd_disk_backend_commit_duration_seconds_bucket{job=\"$cluster\"}[5m])) by (instance, le))", + "intervalFactor": 2, + "legendFormat": "{{instance}} DB fsync", + "metric": "etcd_disk_backend_commit_duration_seconds_bucket", + "refId": "B", + "step": 4 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Disk Sync Duration", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "s", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "datasource": "$datasource", + "editable": true, + "error": false, + "fill": 0, + "id": 29, + "isNew": true, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "process_resident_memory_bytes{job=\"$cluster\"}", + "intervalFactor": 2, + "legendFormat": "{{instance}} Resident Memory", + "metric": "process_resident_memory_bytes", + "refId": "A", + "step": 4 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Memory", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "title": "New row" + }, + { + "collapse": false, + "editable": true, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "datasource": "$datasource", + "editable": true, + "error": false, + "fill": 5, + "id": 22, + "isNew": true, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "span": 3, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "rate(etcd_network_client_grpc_received_bytes_total{job=\"$cluster\"}[5m])", + "intervalFactor": 2, + "legendFormat": "{{instance}} Client Traffic In", + "metric": "etcd_network_client_grpc_received_bytes_total", + "refId": "A", + "step": 4 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Client Traffic In", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "datasource": "$datasource", + "editable": true, + "error": false, + "fill": 5, + "id": 21, + "isNew": true, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "span": 3, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "rate(etcd_network_client_grpc_sent_bytes_total{job=\"$cluster\"}[5m])", + "intervalFactor": 2, + "legendFormat": "{{instance}} Client Traffic Out", + "metric": "etcd_network_client_grpc_sent_bytes_total", + "refId": "A", + "step": 4 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Client Traffic Out", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "datasource": "$datasource", + "editable": true, + "error": false, + "fill": 0, + "id": 20, + "isNew": true, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "span": 3, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(etcd_network_peer_received_bytes_total{job=\"$cluster\"}[5m])) by (instance)", + "intervalFactor": 2, + "legendFormat": "{{instance}} Peer Traffic In", + "metric": "etcd_network_peer_received_bytes_total", + "refId": "A", + "step": 4 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Peer Traffic In", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "datasource": "$datasource", + "decimals": null, + "editable": true, + "error": false, + "fill": 0, + "grid": { + + }, + "id": 16, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "span": 3, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(etcd_network_peer_sent_bytes_total{job=\"$cluster\"}[5m])) by (instance)", + "hide": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{instance}} Peer Traffic Out", + "metric": "etcd_network_peer_sent_bytes_total", + "refId": "A", + "step": 4 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Peer Traffic Out", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "title": "New row" + }, + { + "collapse": false, + "editable": true, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "datasource": "$datasource", + "editable": true, + "error": false, + "fill": 0, + "id": 40, + "isNew": true, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(etcd_server_proposals_failed_total{job=\"$cluster\"}[5m]))", + "intervalFactor": 2, + "legendFormat": "Proposal Failure Rate", + "metric": "etcd_server_proposals_failed_total", + "refId": "A", + "step": 2 + }, + { + "expr": "sum(etcd_server_proposals_pending{job=\"$cluster\"})", + "intervalFactor": 2, + "legendFormat": "Proposal Pending Total", + "metric": "etcd_server_proposals_pending", + "refId": "B", + "step": 2 + }, + { + "expr": "sum(rate(etcd_server_proposals_committed_total{job=\"$cluster\"}[5m]))", + "intervalFactor": 2, + "legendFormat": "Proposal Commit Rate", + "metric": "etcd_server_proposals_committed_total", + "refId": "C", + "step": 2 + }, + { + "expr": "sum(rate(etcd_server_proposals_applied_total{job=\"$cluster\"}[5m]))", + "intervalFactor": 2, + "legendFormat": "Proposal Apply Rate", + "refId": "D", + "step": 2 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Raft Proposals", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": "", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "datasource": "$datasource", + "decimals": 0, + "editable": true, + "error": false, + "fill": 0, + "id": 19, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "changes(etcd_server_leader_changes_seen_total{job=\"$cluster\"}[1d])", + "intervalFactor": 2, + "legendFormat": "{{instance}} Total Leader Elections Per Day", + "metric": "etcd_server_leader_changes_seen_total", + "refId": "A", + "step": 2 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Total Leader Elections Per Day", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "title": "New row" + } + ], + "schemaVersion": 13, + "sharedCrosshair": false, + "style": "dark", + "tags": [ + + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": null, + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + "text": "prod", + "value": "prod" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": "cluster", + "multi": false, + "name": "cluster", + "options": [ + + ], + "query": "label_values(etcd_server_has_leader, job)", + "refresh": 1, + "regex": "", + "sort": 2, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-15m", + "to": "now" + }, + "timepicker": { + "now": true, + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "browser", + "title": "etcd", + "version": 215 + } diff --git a/addons/grafana/dashboards.yaml b/addons/grafana/dashboards-k8s.yaml similarity index 82% rename from addons/grafana/dashboards.yaml rename to addons/grafana/dashboards-k8s.yaml index 909124660..7c0e429cf 100644 --- a/addons/grafana/dashboards.yaml +++ b/addons/grafana/dashboards-k8s.yaml @@ -1,1233 +1,9 @@ apiVersion: v1 kind: ConfigMap metadata: - name: grafana-dashboards + name: grafana-dashboards-k8s namespace: monitoring data: - etcd.json: |- - { - "annotations": { - "list": [ - - ] - }, - "description": "etcd sample Grafana dashboard with Prometheus", - "editable": true, - "gnetId": null, - "hideControls": false, - "id": 6, - "links": [ - - ], - "refresh": false, - "rows": [ - { - "collapse": false, - "editable": true, - "height": "250px", - "panels": [ - { - "cacheTimeout": null, - "colorBackground": false, - "colorValue": false, - "colors": [ - "rgba(245, 54, 54, 0.9)", - "rgba(237, 129, 40, 0.89)", - "rgba(50, 172, 45, 0.97)" - ], - "datasource": "$datasource", - "editable": true, - "error": false, - "format": "none", - "gauge": { - "maxValue": 100, - "minValue": 0, - "show": false, - "thresholdLabels": false, - "thresholdMarkers": true - }, - "id": 28, - "interval": null, - "isNew": true, - "links": [ - - ], - "mappingType": 1, - "mappingTypes": [ - { - "name": "value to text", - "value": 1 - }, - { - "name": "range to text", - "value": 2 - } - ], - "maxDataPoints": 100, - "nullPointMode": "connected", - "nullText": null, - "postfix": "", - "postfixFontSize": "50%", - "prefix": "", - "prefixFontSize": "50%", - "rangeMaps": [ - { - "from": "null", - "text": "N/A", - "to": "null" - } - ], - "span": 3, - "sparkline": { - "fillColor": "rgba(31, 118, 189, 0.18)", - "full": false, - "lineColor": "rgb(31, 120, 193)", - "show": false - }, - "targets": [ - { - "expr": "sum(etcd_server_has_leader{job=\"$cluster\"})", - "intervalFactor": 2, - "legendFormat": "", - "metric": "etcd_server_has_leader", - "refId": "A", - "step": 20 - } - ], - "thresholds": "", - "title": "Up", - "type": "singlestat", - "valueFontSize": "200%", - "valueMaps": [ - { - "op": "=", - "text": "N/A", - "value": "null" - } - ], - "valueName": "avg" - }, - { - "aliasColors": { - - }, - "bars": false, - "datasource": "$datasource", - "editable": true, - "error": false, - "fill": 0, - "id": 23, - "isNew": true, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": false, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 2, - "links": [ - - ], - "nullPointMode": "connected", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - - ], - "span": 5, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum(rate(grpc_server_started_total{job=\"$cluster\",grpc_type=\"unary\"}[5m]))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "RPC Rate", - "metric": "grpc_server_started_total", - "refId": "A", - "step": 2 - }, - { - "expr": "sum(rate(grpc_server_handled_total{job=\"$cluster\",grpc_type=\"unary\",grpc_code!=\"OK\"}[5m]))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "RPC Failed Rate", - "metric": "grpc_server_handled_total", - "refId": "B", - "step": 2 - } - ], - "thresholds": [ - - ], - "timeFrom": null, - "timeShift": null, - "title": "RPC Rate", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "name": null, - "show": true, - "values": [ - - ] - }, - "yaxes": [ - { - "format": "ops", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] - }, - { - "aliasColors": { - - }, - "bars": false, - "datasource": "$datasource", - "editable": true, - "error": false, - "fill": 0, - "id": 41, - "isNew": true, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": false, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 2, - "links": [ - - ], - "nullPointMode": "connected", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - - ], - "span": 4, - "stack": true, - "steppedLine": false, - "targets": [ - { - "expr": "sum(grpc_server_started_total{job=\"$cluster\",grpc_service=\"etcdserverpb.Watch\",grpc_type=\"bidi_stream\"}) - sum(grpc_server_handled_total{job=\"$cluster\",grpc_service=\"etcdserverpb.Watch\",grpc_type=\"bidi_stream\"})", - "intervalFactor": 2, - "legendFormat": "Watch Streams", - "metric": "grpc_server_handled_total", - "refId": "A", - "step": 4 - }, - { - "expr": "sum(grpc_server_started_total{job=\"$cluster\",grpc_service=\"etcdserverpb.Lease\",grpc_type=\"bidi_stream\"}) - sum(grpc_server_handled_total{job=\"$cluster\",grpc_service=\"etcdserverpb.Lease\",grpc_type=\"bidi_stream\"})", - "intervalFactor": 2, - "legendFormat": "Lease Streams", - "metric": "grpc_server_handled_total", - "refId": "B", - "step": 4 - } - ], - "thresholds": [ - - ], - "timeFrom": null, - "timeShift": null, - "title": "Active Streams", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "name": null, - "show": true, - "values": [ - - ] - }, - "yaxes": [ - { - "format": "short", - "label": "", - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] - } - ], - "showTitle": false, - "title": "Row" - }, - { - "collapse": false, - "editable": true, - "height": "250px", - "panels": [ - { - "aliasColors": { - - }, - "bars": false, - "datasource": "$datasource", - "decimals": null, - "editable": true, - "error": false, - "fill": 0, - "grid": { - - }, - "id": 1, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": false, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 2, - "links": [ - - ], - "nullPointMode": "connected", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - - ], - "span": 4, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "etcd_debugging_mvcc_db_total_size_in_bytes{job=\"$cluster\"}", - "hide": false, - "interval": "", - "intervalFactor": 2, - "legendFormat": "{{instance}} DB Size", - "metric": "", - "refId": "A", - "step": 4 - } - ], - "thresholds": [ - - ], - "timeFrom": null, - "timeShift": null, - "title": "DB Size", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "name": null, - "show": true, - "values": [ - - ] - }, - "yaxes": [ - { - "format": "bytes", - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] - }, - { - "aliasColors": { - - }, - "bars": false, - "datasource": "$datasource", - "editable": true, - "error": false, - "fill": 0, - "grid": { - - }, - "id": 3, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": false, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 2, - "links": [ - - ], - "nullPointMode": "connected", - "percentage": false, - "pointradius": 1, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - - ], - "span": 4, - "stack": false, - "steppedLine": true, - "targets": [ - { - "expr": "histogram_quantile(0.99, sum(rate(etcd_disk_wal_fsync_duration_seconds_bucket{job=\"$cluster\"}[5m])) by (instance, le))", - "hide": false, - "intervalFactor": 2, - "legendFormat": "{{instance}} WAL fsync", - "metric": "etcd_disk_wal_fsync_duration_seconds_bucket", - "refId": "A", - "step": 4 - }, - { - "expr": "histogram_quantile(0.99, sum(rate(etcd_disk_backend_commit_duration_seconds_bucket{job=\"$cluster\"}[5m])) by (instance, le))", - "intervalFactor": 2, - "legendFormat": "{{instance}} DB fsync", - "metric": "etcd_disk_backend_commit_duration_seconds_bucket", - "refId": "B", - "step": 4 - } - ], - "thresholds": [ - - ], - "timeFrom": null, - "timeShift": null, - "title": "Disk Sync Duration", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "name": null, - "show": true, - "values": [ - - ] - }, - "yaxes": [ - { - "format": "s", - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] - }, - { - "aliasColors": { - - }, - "bars": false, - "datasource": "$datasource", - "editable": true, - "error": false, - "fill": 0, - "id": 29, - "isNew": true, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": false, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 2, - "links": [ - - ], - "nullPointMode": "connected", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - - ], - "span": 4, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "process_resident_memory_bytes{job=\"$cluster\"}", - "intervalFactor": 2, - "legendFormat": "{{instance}} Resident Memory", - "metric": "process_resident_memory_bytes", - "refId": "A", - "step": 4 - } - ], - "thresholds": [ - - ], - "timeFrom": null, - "timeShift": null, - "title": "Memory", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "name": null, - "show": true, - "values": [ - - ] - }, - "yaxes": [ - { - "format": "bytes", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] - } - ], - "title": "New row" - }, - { - "collapse": false, - "editable": true, - "height": "250px", - "panels": [ - { - "aliasColors": { - - }, - "bars": false, - "datasource": "$datasource", - "editable": true, - "error": false, - "fill": 5, - "id": 22, - "isNew": true, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": false, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 2, - "links": [ - - ], - "nullPointMode": "connected", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - - ], - "span": 3, - "stack": true, - "steppedLine": false, - "targets": [ - { - "expr": "rate(etcd_network_client_grpc_received_bytes_total{job=\"$cluster\"}[5m])", - "intervalFactor": 2, - "legendFormat": "{{instance}} Client Traffic In", - "metric": "etcd_network_client_grpc_received_bytes_total", - "refId": "A", - "step": 4 - } - ], - "thresholds": [ - - ], - "timeFrom": null, - "timeShift": null, - "title": "Client Traffic In", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "name": null, - "show": true, - "values": [ - - ] - }, - "yaxes": [ - { - "format": "Bps", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] - }, - { - "aliasColors": { - - }, - "bars": false, - "datasource": "$datasource", - "editable": true, - "error": false, - "fill": 5, - "id": 21, - "isNew": true, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": false, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 2, - "links": [ - - ], - "nullPointMode": "connected", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - - ], - "span": 3, - "stack": true, - "steppedLine": false, - "targets": [ - { - "expr": "rate(etcd_network_client_grpc_sent_bytes_total{job=\"$cluster\"}[5m])", - "intervalFactor": 2, - "legendFormat": "{{instance}} Client Traffic Out", - "metric": "etcd_network_client_grpc_sent_bytes_total", - "refId": "A", - "step": 4 - } - ], - "thresholds": [ - - ], - "timeFrom": null, - "timeShift": null, - "title": "Client Traffic Out", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "name": null, - "show": true, - "values": [ - - ] - }, - "yaxes": [ - { - "format": "Bps", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] - }, - { - "aliasColors": { - - }, - "bars": false, - "datasource": "$datasource", - "editable": true, - "error": false, - "fill": 0, - "id": 20, - "isNew": true, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": false, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 2, - "links": [ - - ], - "nullPointMode": "connected", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - - ], - "span": 3, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum(rate(etcd_network_peer_received_bytes_total{job=\"$cluster\"}[5m])) by (instance)", - "intervalFactor": 2, - "legendFormat": "{{instance}} Peer Traffic In", - "metric": "etcd_network_peer_received_bytes_total", - "refId": "A", - "step": 4 - } - ], - "thresholds": [ - - ], - "timeFrom": null, - "timeShift": null, - "title": "Peer Traffic In", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "name": null, - "show": true, - "values": [ - - ] - }, - "yaxes": [ - { - "format": "Bps", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] - }, - { - "aliasColors": { - - }, - "bars": false, - "datasource": "$datasource", - "decimals": null, - "editable": true, - "error": false, - "fill": 0, - "grid": { - - }, - "id": 16, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": false, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 2, - "links": [ - - ], - "nullPointMode": "connected", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - - ], - "span": 3, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum(rate(etcd_network_peer_sent_bytes_total{job=\"$cluster\"}[5m])) by (instance)", - "hide": false, - "interval": "", - "intervalFactor": 2, - "legendFormat": "{{instance}} Peer Traffic Out", - "metric": "etcd_network_peer_sent_bytes_total", - "refId": "A", - "step": 4 - } - ], - "thresholds": [ - - ], - "timeFrom": null, - "timeShift": null, - "title": "Peer Traffic Out", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "name": null, - "show": true, - "values": [ - - ] - }, - "yaxes": [ - { - "format": "Bps", - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] - } - ], - "title": "New row" - }, - { - "collapse": false, - "editable": true, - "height": "250px", - "panels": [ - { - "aliasColors": { - - }, - "bars": false, - "datasource": "$datasource", - "editable": true, - "error": false, - "fill": 0, - "id": 40, - "isNew": true, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": false, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 2, - "links": [ - - ], - "nullPointMode": "connected", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - - ], - "span": 6, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum(rate(etcd_server_proposals_failed_total{job=\"$cluster\"}[5m]))", - "intervalFactor": 2, - "legendFormat": "Proposal Failure Rate", - "metric": "etcd_server_proposals_failed_total", - "refId": "A", - "step": 2 - }, - { - "expr": "sum(etcd_server_proposals_pending{job=\"$cluster\"})", - "intervalFactor": 2, - "legendFormat": "Proposal Pending Total", - "metric": "etcd_server_proposals_pending", - "refId": "B", - "step": 2 - }, - { - "expr": "sum(rate(etcd_server_proposals_committed_total{job=\"$cluster\"}[5m]))", - "intervalFactor": 2, - "legendFormat": "Proposal Commit Rate", - "metric": "etcd_server_proposals_committed_total", - "refId": "C", - "step": 2 - }, - { - "expr": "sum(rate(etcd_server_proposals_applied_total{job=\"$cluster\"}[5m]))", - "intervalFactor": 2, - "legendFormat": "Proposal Apply Rate", - "refId": "D", - "step": 2 - } - ], - "thresholds": [ - - ], - "timeFrom": null, - "timeShift": null, - "title": "Raft Proposals", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "name": null, - "show": true, - "values": [ - - ] - }, - "yaxes": [ - { - "format": "short", - "label": "", - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] - }, - { - "aliasColors": { - - }, - "bars": false, - "datasource": "$datasource", - "decimals": 0, - "editable": true, - "error": false, - "fill": 0, - "id": 19, - "isNew": true, - "legend": { - "alignAsTable": false, - "avg": false, - "current": false, - "max": false, - "min": false, - "rightSide": false, - "show": false, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 2, - "links": [ - - ], - "nullPointMode": "connected", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - - ], - "span": 6, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "changes(etcd_server_leader_changes_seen_total{job=\"$cluster\"}[1d])", - "intervalFactor": 2, - "legendFormat": "{{instance}} Total Leader Elections Per Day", - "metric": "etcd_server_leader_changes_seen_total", - "refId": "A", - "step": 2 - } - ], - "thresholds": [ - - ], - "timeFrom": null, - "timeShift": null, - "title": "Total Leader Elections Per Day", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "name": null, - "show": true, - "values": [ - - ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] - } - ], - "title": "New row" - } - ], - "schemaVersion": 13, - "sharedCrosshair": false, - "style": "dark", - "tags": [ - - ], - "templating": { - "list": [ - { - "current": { - "text": "Prometheus", - "value": "Prometheus" - }, - "hide": 0, - "label": null, - "name": "datasource", - "options": [ - - ], - "query": "prometheus", - "refresh": 1, - "regex": "", - "type": "datasource" - }, - { - "allValue": null, - "current": { - "text": "prod", - "value": "prod" - }, - "datasource": "$datasource", - "hide": 0, - "includeAll": false, - "label": "cluster", - "multi": false, - "name": "cluster", - "options": [ - - ], - "query": "label_values(etcd_server_has_leader, job)", - "refresh": 1, - "regex": "", - "sort": 2, - "tagValuesQuery": "", - "tags": [ - - ], - "tagsQuery": "", - "type": "query", - "useTags": false - } - ] - }, - "time": { - "from": "now-15m", - "to": "now" - }, - "timepicker": { - "now": true, - "refresh_intervals": [ - "5s", - "10s", - "30s", - "1m", - "5m", - "15m", - "30m", - "1h", - "2h", - "1d" - ], - "time_options": [ - "5m", - "15m", - "1h", - "6h", - "12h", - "24h", - "2d", - "7d", - "30d" - ] - }, - "timezone": "browser", - "title": "etcd", - "version": 215 - } k8s-cluster-rsrc-use.json: |- { "annotations": { @@ -1286,7 +62,7 @@ data: "steppedLine": false, "targets": [ { - "expr": "node:cluster_cpu_utilisation:ratio", + "expr": "node:cluster_cpu_utilisation:ratio{cluster=\"$cluster\"}", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{node}}", @@ -1372,7 +148,7 @@ data: "steppedLine": false, "targets": [ { - "expr": "node:node_cpu_saturation_load1: / scalar(sum(min(kube_pod_info) by (node)))", + "expr": "node:node_cpu_saturation_load1:{cluster=\"$cluster\"} / scalar(sum(min(kube_pod_info{cluster=\"$cluster\"}) by (node)))", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{node}}", @@ -1470,7 +246,7 @@ data: "steppedLine": false, "targets": [ { - "expr": "node:cluster_memory_utilisation:ratio", + "expr": "node:cluster_memory_utilisation:ratio{cluster=\"$cluster\"}", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{node}}", @@ -1556,7 +332,7 @@ data: "steppedLine": false, "targets": [ { - "expr": "node:node_memory_swap_io_bytes:sum_rate", + "expr": "node:node_memory_swap_io_bytes:sum_rate{cluster=\"$cluster\"}", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{node}}", @@ -1654,7 +430,7 @@ data: "steppedLine": false, "targets": [ { - "expr": "node:node_disk_utilisation:avg_irate / scalar(:kube_pod_info_node_count:)", + "expr": "node:node_disk_utilisation:avg_irate{cluster=\"$cluster\"} / scalar(:kube_pod_info_node_count:{cluster=\"$cluster\"})", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{node}}", @@ -1740,7 +516,7 @@ data: "steppedLine": false, "targets": [ { - "expr": "node:node_disk_saturation:avg_irate / scalar(:kube_pod_info_node_count:)", + "expr": "node:node_disk_saturation:avg_irate{cluster=\"$cluster\"} / scalar(:kube_pod_info_node_count:{cluster=\"$cluster\"})", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{node}}", @@ -1838,7 +614,7 @@ data: "steppedLine": false, "targets": [ { - "expr": "node:node_net_utilisation:sum_irate", + "expr": "node:node_net_utilisation:sum_irate{cluster=\"$cluster\"}", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{node}}", @@ -1924,7 +700,7 @@ data: "steppedLine": false, "targets": [ { - "expr": "node:node_net_saturation:sum_irate", + "expr": "node:node_net_saturation:sum_irate{cluster=\"$cluster\"}", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{node}}", @@ -2022,7 +798,7 @@ data: "steppedLine": false, "targets": [ { - "expr": "sum(max(node_filesystem_size_bytes{fstype=~\"ext[234]|btrfs|xfs|zfs\"} - node_filesystem_avail_bytes{fstype=~\"ext[234]|btrfs|xfs|zfs\"}) by (device,pod,namespace)) by (pod,namespace)\n/ scalar(sum(max(node_filesystem_size_bytes{fstype=~\"ext[234]|btrfs|xfs|zfs\"}) by (device,pod,namespace)))\n* on (namespace, pod) group_left (node) node_namespace_pod:kube_pod_info:\n", + "expr": "sum(max(node_filesystem_size_bytes{fstype=~\"ext[234]|btrfs|xfs|zfs\", cluster=\"$cluster\"} - node_filesystem_avail_bytes{fstype=~\"ext[234]|btrfs|xfs|zfs\", cluster=\"$cluster\"}) by (device,pod,namespace)) by (pod,namespace)\n/ scalar(sum(max(node_filesystem_size_bytes{fstype=~\"ext[234]|btrfs|xfs|zfs\", cluster=\"$cluster\"}) by (device,pod,namespace)))\n* on (namespace, pod) group_left (node) node_namespace_pod:kube_pod_info:{cluster=\"$cluster\"}\n", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{node}}", @@ -2101,6 +877,33 @@ data: "refresh": 1, "regex": "", "type": "datasource" + }, + { + "allValue": null, + "current": { + "text": "prod", + "value": "prod" + }, + "datasource": "$datasource", + "hide": 2, + "includeAll": false, + "label": "cluster", + "multi": false, + "name": "cluster", + "options": [ + + ], + "query": "label_values(kube_node_info, cluster)", + "refresh": 1, + "regex": "", + "sort": 2, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false } ] }, @@ -2196,7 +999,7 @@ data: "steppedLine": false, "targets": [ { - "expr": "node:node_cpu_utilisation:avg1m{node=\"$node\"}", + "expr": "node:node_cpu_utilisation:avg1m{cluster=\"$cluster\", node=\"$node\"}", "format": "time_series", "intervalFactor": 2, "legendFormat": "Utilisation", @@ -2282,7 +1085,7 @@ data: "steppedLine": false, "targets": [ { - "expr": "node:node_cpu_saturation_load1:{node=\"$node\"}", + "expr": "node:node_cpu_saturation_load1:{cluster=\"$cluster\", node=\"$node\"}", "format": "time_series", "intervalFactor": 2, "legendFormat": "Saturation", @@ -2380,7 +1183,7 @@ data: "steppedLine": false, "targets": [ { - "expr": "node:node_memory_utilisation:{node=\"$node\"}", + "expr": "node:node_memory_utilisation:{cluster=\"$cluster\", node=\"$node\"}", "format": "time_series", "intervalFactor": 2, "legendFormat": "Memory", @@ -2466,7 +1269,7 @@ data: "steppedLine": false, "targets": [ { - "expr": "node:node_memory_swap_io_bytes:sum_rate{node=\"$node\"}", + "expr": "node:node_memory_swap_io_bytes:sum_rate{cluster=\"$cluster\", node=\"$node\"}", "format": "time_series", "intervalFactor": 2, "legendFormat": "Swap IO", @@ -2564,7 +1367,7 @@ data: "steppedLine": false, "targets": [ { - "expr": "node:node_disk_utilisation:avg_irate{node=\"$node\"}", + "expr": "node:node_disk_utilisation:avg_irate{cluster=\"$cluster\", node=\"$node\"}", "format": "time_series", "intervalFactor": 2, "legendFormat": "Utilisation", @@ -2650,7 +1453,7 @@ data: "steppedLine": false, "targets": [ { - "expr": "node:node_disk_saturation:avg_irate{node=\"$node\"}", + "expr": "node:node_disk_saturation:avg_irate{cluster=\"$cluster\", node=\"$node\"}", "format": "time_series", "intervalFactor": 2, "legendFormat": "Saturation", @@ -2748,7 +1551,7 @@ data: "steppedLine": false, "targets": [ { - "expr": "node:node_net_utilisation:sum_irate{node=\"$node\"}", + "expr": "node:node_net_utilisation:sum_irate{cluster=\"$cluster\", node=\"$node\"}", "format": "time_series", "intervalFactor": 2, "legendFormat": "Utilisation", @@ -2834,7 +1637,7 @@ data: "steppedLine": false, "targets": [ { - "expr": "node:node_net_saturation:sum_irate{node=\"$node\"}", + "expr": "node:node_net_saturation:sum_irate{cluster=\"$cluster\", node=\"$node\"}", "format": "time_series", "intervalFactor": 2, "legendFormat": "Saturation", @@ -2932,7 +1735,7 @@ data: "steppedLine": false, "targets": [ { - "expr": "node:node_filesystem_usage:\n* on (namespace, pod) group_left (node) node_namespace_pod:kube_pod_info:{node=\"$node\"}\n", + "expr": "node:node_filesystem_usage:{cluster=\"$cluster\"}\n* on (namespace, pod) group_left (node) node_namespace_pod:kube_pod_info:{cluster=\"$cluster\", node=\"$node\"}\n", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{device}}", @@ -3012,6 +1815,33 @@ data: "regex": "", "type": "datasource" }, + { + "allValue": null, + "current": { + "text": "prod", + "value": "prod" + }, + "datasource": "$datasource", + "hide": 2, + "includeAll": false, + "label": "cluster", + "multi": false, + "name": "cluster", + "options": [ + + ], + "query": "label_values(kube_node_info, cluster)", + "refresh": 1, + "regex": "", + "sort": 2, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, { "allValue": null, "current": { @@ -3027,7 +1857,7 @@ data: "options": [ ], - "query": "label_values(kube_node_info, node)", + "query": "label_values(kube_node_info{cluster=\"$cluster\"}, node)", "refresh": 1, "regex": "", "sort": 2, @@ -3134,7 +1964,7 @@ data: "steppedLine": false, "targets": [ { - "expr": "1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[1m]))", + "expr": "1 - avg(rate(node_cpu_seconds_total{mode=\"idle\", cluster=\"$cluster\"}[1m]))", "format": "time_series", "instant": true, "intervalFactor": 2, @@ -3218,7 +2048,7 @@ data: "steppedLine": false, "targets": [ { - "expr": "sum(kube_pod_container_resource_requests_cpu_cores) / sum(node:node_num_cpu:sum)", + "expr": "sum(kube_pod_container_resource_requests_cpu_cores{cluster=\"$cluster\"}) / sum(node:node_num_cpu:sum{cluster=\"$cluster\"})", "format": "time_series", "instant": true, "intervalFactor": 2, @@ -3302,7 +2132,7 @@ data: "steppedLine": false, "targets": [ { - "expr": "sum(kube_pod_container_resource_limits_cpu_cores) / sum(node:node_num_cpu:sum)", + "expr": "sum(kube_pod_container_resource_limits_cpu_cores{cluster=\"$cluster\"}) / sum(node:node_num_cpu:sum{cluster=\"$cluster\"})", "format": "time_series", "instant": true, "intervalFactor": 2, @@ -3386,7 +2216,7 @@ data: "steppedLine": false, "targets": [ { - "expr": "1 - sum(:node_memory_MemFreeCachedBuffers_bytes:sum) / sum(:node_memory_MemTotal_bytes:sum)", + "expr": "1 - sum(:node_memory_MemFreeCachedBuffers_bytes:sum{cluster=\"$cluster\"}) / sum(:node_memory_MemTotal_bytes:sum{cluster=\"$cluster\"})", "format": "time_series", "instant": true, "intervalFactor": 2, @@ -3470,7 +2300,7 @@ data: "steppedLine": false, "targets": [ { - "expr": "sum(kube_pod_container_resource_requests_memory_bytes) / sum(:node_memory_MemTotal_bytes:sum)", + "expr": "sum(kube_pod_container_resource_requests_memory_bytes{cluster=\"$cluster\"}) / sum(:node_memory_MemTotal_bytes:sum{cluster=\"$cluster\"})", "format": "time_series", "instant": true, "intervalFactor": 2, @@ -3554,7 +2384,7 @@ data: "steppedLine": false, "targets": [ { - "expr": "sum(kube_pod_container_resource_limits_memory_bytes) / sum(:node_memory_MemTotal_bytes:sum)", + "expr": "sum(kube_pod_container_resource_limits_memory_bytes{cluster=\"$cluster\"}) / sum(:node_memory_MemTotal_bytes:sum{cluster=\"$cluster\"})", "format": "time_series", "instant": true, "intervalFactor": 2, @@ -3649,7 +2479,7 @@ data: "steppedLine": false, "targets": [ { - "expr": "sum(namespace_pod_name_container_name:container_cpu_usage_seconds_total:sum_rate) by (namespace)", + "expr": "sum(namespace_pod_name_container_name:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\"}) by (namespace)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{namespace}}", @@ -3852,7 +2682,7 @@ data: "decimals": 2, "link": true, "linkTooltip": "Drill down", - "linkUrl": "/d/85a562078cdf77779eaa1add43ccec1e/k8s-resources-namespace?var-datasource=$datasource&var-namespace=$__cell", + "linkUrl": "/d/85a562078cdf77779eaa1add43ccec1e/k8s-resources-namespace?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$__cell", "pattern": "namespace", "thresholds": [ @@ -3878,7 +2708,7 @@ data: ], "targets": [ { - "expr": "sum(namespace_pod_name_container_name:container_cpu_usage_seconds_total:sum_rate) by (namespace)", + "expr": "sum(namespace_pod_name_container_name:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\"}) by (namespace)", "format": "table", "instant": true, "intervalFactor": 2, @@ -3887,7 +2717,7 @@ data: "step": 10 }, { - "expr": "sum(kube_pod_container_resource_requests_cpu_cores) by (namespace)", + "expr": "sum(kube_pod_container_resource_requests_cpu_cores{cluster=\"$cluster\"}) by (namespace)", "format": "table", "instant": true, "intervalFactor": 2, @@ -3896,7 +2726,7 @@ data: "step": 10 }, { - "expr": "sum(namespace_pod_name_container_name:container_cpu_usage_seconds_total:sum_rate) by (namespace) / sum(kube_pod_container_resource_requests_cpu_cores) by (namespace)", + "expr": "sum(namespace_pod_name_container_name:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\"}) by (namespace) / sum(kube_pod_container_resource_requests_cpu_cores{cluster=\"$cluster\"}) by (namespace)", "format": "table", "instant": true, "intervalFactor": 2, @@ -3905,7 +2735,7 @@ data: "step": 10 }, { - "expr": "sum(kube_pod_container_resource_limits_cpu_cores) by (namespace)", + "expr": "sum(kube_pod_container_resource_limits_cpu_cores{cluster=\"$cluster\"}) by (namespace)", "format": "table", "instant": true, "intervalFactor": 2, @@ -3914,7 +2744,7 @@ data: "step": 10 }, { - "expr": "sum(namespace_pod_name_container_name:container_cpu_usage_seconds_total:sum_rate) by (namespace) / sum(kube_pod_container_resource_limits_cpu_cores) by (namespace)", + "expr": "sum(namespace_pod_name_container_name:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\"}) by (namespace) / sum(kube_pod_container_resource_limits_cpu_cores{cluster=\"$cluster\"}) by (namespace)", "format": "table", "instant": true, "intervalFactor": 2, @@ -4014,7 +2844,7 @@ data: "steppedLine": false, "targets": [ { - "expr": "sum(container_memory_rss{container_name!=\"\"}) by (namespace)", + "expr": "sum(container_memory_rss{cluster=\"$cluster\", container_name!=\"\"}) by (namespace)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{namespace}}", @@ -4217,7 +3047,7 @@ data: "decimals": 2, "link": true, "linkTooltip": "Drill down", - "linkUrl": "/d/85a562078cdf77779eaa1add43ccec1e/k8s-resources-namespace?var-datasource=$datasource&var-namespace=$__cell", + "linkUrl": "/d/85a562078cdf77779eaa1add43ccec1e/k8s-resources-namespace?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$__cell", "pattern": "namespace", "thresholds": [ @@ -4243,7 +3073,7 @@ data: ], "targets": [ { - "expr": "sum(container_memory_rss{container_name!=\"\"}) by (namespace)", + "expr": "sum(container_memory_rss{cluster=\"$cluster\", container_name!=\"\"}) by (namespace)", "format": "table", "instant": true, "intervalFactor": 2, @@ -4252,7 +3082,7 @@ data: "step": 10 }, { - "expr": "sum(kube_pod_container_resource_requests_memory_bytes) by (namespace)", + "expr": "sum(kube_pod_container_resource_requests_memory_bytes{cluster=\"$cluster\"}) by (namespace)", "format": "table", "instant": true, "intervalFactor": 2, @@ -4261,7 +3091,7 @@ data: "step": 10 }, { - "expr": "sum(container_memory_rss{container_name!=\"\"}) by (namespace) / sum(kube_pod_container_resource_requests_memory_bytes) by (namespace)", + "expr": "sum(container_memory_rss{cluster=\"$cluster\", container_name!=\"\"}) by (namespace) / sum(kube_pod_container_resource_requests_memory_bytes{cluster=\"$cluster\"}) by (namespace)", "format": "table", "instant": true, "intervalFactor": 2, @@ -4270,7 +3100,7 @@ data: "step": 10 }, { - "expr": "sum(kube_pod_container_resource_limits_memory_bytes) by (namespace)", + "expr": "sum(kube_pod_container_resource_limits_memory_bytes{cluster=\"$cluster\"}) by (namespace)", "format": "table", "instant": true, "intervalFactor": 2, @@ -4279,7 +3109,7 @@ data: "step": 10 }, { - "expr": "sum(container_memory_rss{container_name!=\"\"}) by (namespace) / sum(kube_pod_container_resource_limits_memory_bytes) by (namespace)", + "expr": "sum(container_memory_rss{cluster=\"$cluster\", container_name!=\"\"}) by (namespace) / sum(kube_pod_container_resource_limits_memory_bytes{cluster=\"$cluster\"}) by (namespace)", "format": "table", "instant": true, "intervalFactor": 2, @@ -4360,6 +3190,33 @@ data: "refresh": 1, "regex": "", "type": "datasource" + }, + { + "allValue": null, + "current": { + "text": "prod", + "value": "prod" + }, + "datasource": "$datasource", + "hide": 2, + "includeAll": false, + "label": "cluster", + "multi": false, + "name": "cluster", + "options": [ + + ], + "query": "label_values(node_cpu_seconds_total, cluster)", + "refresh": 1, + "regex": "", + "sort": 2, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false } ] }, @@ -4455,7 +3312,7 @@ data: "steppedLine": false, "targets": [ { - "expr": "sum(namespace_pod_name_container_name:container_cpu_usage_seconds_total:sum_rate{namespace=\"$namespace\"}) by (pod_name)", + "expr": "sum(namespace_pod_name_container_name:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod_name)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{pod_name}}", @@ -4658,7 +3515,7 @@ data: "decimals": 2, "link": true, "linkTooltip": "Drill down", - "linkUrl": "/d/6581e46e4e5c7ba40a07646395ef7b23/k8s-resources-pod?var-datasource=$datasource&var-namespace=$namespace&var-pod=$__cell", + "linkUrl": "/d/6581e46e4e5c7ba40a07646395ef7b23/k8s-resources-pod?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$namespace&var-pod=$__cell", "pattern": "pod", "thresholds": [ @@ -4684,7 +3541,7 @@ data: ], "targets": [ { - "expr": "sum(label_replace(namespace_pod_name_container_name:container_cpu_usage_seconds_total:sum_rate{namespace=\"$namespace\"}, \"pod\", \"$1\", \"pod_name\", \"(.*)\")) by (pod)", + "expr": "sum(label_replace(namespace_pod_name_container_name:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\"}, \"pod\", \"$1\", \"pod_name\", \"(.*)\")) by (pod)", "format": "table", "instant": true, "intervalFactor": 2, @@ -4693,7 +3550,7 @@ data: "step": 10 }, { - "expr": "sum(kube_pod_container_resource_requests_cpu_cores{namespace=\"$namespace\"}) by (pod)", + "expr": "sum(kube_pod_container_resource_requests_cpu_cores{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)", "format": "table", "instant": true, "intervalFactor": 2, @@ -4702,7 +3559,7 @@ data: "step": 10 }, { - "expr": "sum(label_replace(namespace_pod_name_container_name:container_cpu_usage_seconds_total:sum_rate{namespace=\"$namespace\"}, \"pod\", \"$1\", \"pod_name\", \"(.*)\")) by (pod) / sum(kube_pod_container_resource_requests_cpu_cores{namespace=\"$namespace\"}) by (pod)", + "expr": "sum(label_replace(namespace_pod_name_container_name:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\"}, \"pod\", \"$1\", \"pod_name\", \"(.*)\")) by (pod) / sum(kube_pod_container_resource_requests_cpu_cores{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)", "format": "table", "instant": true, "intervalFactor": 2, @@ -4711,7 +3568,7 @@ data: "step": 10 }, { - "expr": "sum(kube_pod_container_resource_limits_cpu_cores{namespace=\"$namespace\"}) by (pod)", + "expr": "sum(kube_pod_container_resource_limits_cpu_cores{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)", "format": "table", "instant": true, "intervalFactor": 2, @@ -4720,7 +3577,7 @@ data: "step": 10 }, { - "expr": "sum(label_replace(namespace_pod_name_container_name:container_cpu_usage_seconds_total:sum_rate{namespace=\"$namespace\"}, \"pod\", \"$1\", \"pod_name\", \"(.*)\")) by (pod) / sum(kube_pod_container_resource_limits_cpu_cores{namespace=\"$namespace\"}) by (pod)", + "expr": "sum(label_replace(namespace_pod_name_container_name:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\"}, \"pod\", \"$1\", \"pod_name\", \"(.*)\")) by (pod) / sum(kube_pod_container_resource_limits_cpu_cores{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)", "format": "table", "instant": true, "intervalFactor": 2, @@ -4820,7 +3677,7 @@ data: "steppedLine": false, "targets": [ { - "expr": "sum(container_memory_usage_bytes{namespace=\"$namespace\", container_name!=\"\"}) by (pod_name)", + "expr": "sum(container_memory_usage_bytes{cluster=\"$cluster\", namespace=\"$namespace\", container_name!=\"\"}) by (pod_name)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{pod_name}}", @@ -5077,7 +3934,7 @@ data: "decimals": 2, "link": true, "linkTooltip": "Drill down", - "linkUrl": "/d/6581e46e4e5c7ba40a07646395ef7b23/k8s-resources-pod?var-datasource=$datasource&var-namespace=$namespace&var-pod=$__cell", + "linkUrl": "/d/6581e46e4e5c7ba40a07646395ef7b23/k8s-resources-pod?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$namespace&var-pod=$__cell", "pattern": "pod", "thresholds": [ @@ -5103,7 +3960,7 @@ data: ], "targets": [ { - "expr": "sum(label_replace(container_memory_usage_bytes{namespace=\"$namespace\",container_name!=\"\"}, \"pod\", \"$1\", \"pod_name\", \"(.*)\")) by (pod)", + "expr": "sum(label_replace(container_memory_usage_bytes{cluster=\"$cluster\", namespace=\"$namespace\",container_name!=\"\"}, \"pod\", \"$1\", \"pod_name\", \"(.*)\")) by (pod)", "format": "table", "instant": true, "intervalFactor": 2, @@ -5112,7 +3969,7 @@ data: "step": 10 }, { - "expr": "sum(kube_pod_container_resource_requests_memory_bytes{namespace=\"$namespace\"}) by (pod)", + "expr": "sum(kube_pod_container_resource_requests_memory_bytes{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)", "format": "table", "instant": true, "intervalFactor": 2, @@ -5121,7 +3978,7 @@ data: "step": 10 }, { - "expr": "sum(label_replace(container_memory_usage_bytes{namespace=\"$namespace\",container_name!=\"\"}, \"pod\", \"$1\", \"pod_name\", \"(.*)\")) by (pod) / sum(kube_pod_container_resource_requests_memory_bytes{namespace=\"$namespace\"}) by (pod)", + "expr": "sum(label_replace(container_memory_usage_bytes{cluster=\"$cluster\", namespace=\"$namespace\",container_name!=\"\"}, \"pod\", \"$1\", \"pod_name\", \"(.*)\")) by (pod) / sum(kube_pod_container_resource_requests_memory_bytes{namespace=\"$namespace\"}) by (pod)", "format": "table", "instant": true, "intervalFactor": 2, @@ -5130,7 +3987,7 @@ data: "step": 10 }, { - "expr": "sum(kube_pod_container_resource_limits_memory_bytes{namespace=\"$namespace\"}) by (pod)", + "expr": "sum(kube_pod_container_resource_limits_memory_bytes{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)", "format": "table", "instant": true, "intervalFactor": 2, @@ -5139,7 +3996,7 @@ data: "step": 10 }, { - "expr": "sum(label_replace(container_memory_usage_bytes{namespace=\"$namespace\",container_name!=\"\"}, \"pod\", \"$1\", \"pod_name\", \"(.*)\")) by (pod) / sum(kube_pod_container_resource_limits_memory_bytes{namespace=\"$namespace\"}) by (pod)", + "expr": "sum(label_replace(container_memory_usage_bytes{cluster=\"$cluster\", namespace=\"$namespace\",container_name!=\"\"}, \"pod\", \"$1\", \"pod_name\", \"(.*)\")) by (pod) / sum(kube_pod_container_resource_limits_memory_bytes{namespace=\"$namespace\"}) by (pod)", "format": "table", "instant": true, "intervalFactor": 2, @@ -5148,7 +4005,7 @@ data: "step": 10 }, { - "expr": "sum(label_replace(container_memory_rss{namespace=\"$namespace\",container_name!=\"\"}, \"pod\", \"$1\", \"pod_name\", \"(.*)\")) by (pod)", + "expr": "sum(label_replace(container_memory_rss{cluster=\"$cluster\", namespace=\"$namespace\",container_name!=\"\"}, \"pod\", \"$1\", \"pod_name\", \"(.*)\")) by (pod)", "format": "table", "instant": true, "intervalFactor": 2, @@ -5157,7 +4014,7 @@ data: "step": 10 }, { - "expr": "sum(label_replace(container_memory_cache{namespace=\"$namespace\",container_name!=\"\"}, \"pod\", \"$1\", \"pod_name\", \"(.*)\")) by (pod)", + "expr": "sum(label_replace(container_memory_cache{cluster=\"$cluster\", namespace=\"$namespace\",container_name!=\"\"}, \"pod\", \"$1\", \"pod_name\", \"(.*)\")) by (pod)", "format": "table", "instant": true, "intervalFactor": 2, @@ -5166,7 +4023,7 @@ data: "step": 10 }, { - "expr": "sum(label_replace(container_memory_swap{namespace=\"$namespace\",container_name!=\"\"}, \"pod\", \"$1\", \"pod_name\", \"(.*)\")) by (pod)", + "expr": "sum(label_replace(container_memory_swap{cluster=\"$cluster\", namespace=\"$namespace\",container_name!=\"\"}, \"pod\", \"$1\", \"pod_name\", \"(.*)\")) by (pod)", "format": "table", "instant": true, "intervalFactor": 2, @@ -5248,6 +4105,33 @@ data: "regex": "", "type": "datasource" }, + { + "allValue": null, + "current": { + "text": "prod", + "value": "prod" + }, + "datasource": "$datasource", + "hide": 2, + "includeAll": false, + "label": "cluster", + "multi": false, + "name": "cluster", + "options": [ + + ], + "query": "label_values(kube_pod_info, cluster)", + "refresh": 1, + "regex": "", + "sort": 2, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, { "allValue": null, "current": { @@ -5263,7 +4147,7 @@ data: "options": [ ], - "query": "label_values(kube_pod_info, namespace)", + "query": "label_values(kube_pod_info{cluster=\"$cluster\"}, namespace)", "refresh": 1, "regex": "", "sort": 2, @@ -5369,7 +4253,7 @@ data: "steppedLine": false, "targets": [ { - "expr": "sum(namespace_pod_name_container_name:container_cpu_usage_seconds_total:sum_rate{namespace=\"$namespace\", pod_name=\"$pod\", container_name!=\"POD\"}) by (container_name)", + "expr": "sum(namespace_pod_name_container_name:container_cpu_usage_seconds_total:sum_rate{namespace=\"$namespace\", pod_name=\"$pod\", container_name!=\"POD\", cluster=\"$cluster\"}) by (container_name)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{container_name}}", @@ -5598,7 +4482,7 @@ data: ], "targets": [ { - "expr": "sum(label_replace(namespace_pod_name_container_name:container_cpu_usage_seconds_total:sum_rate{namespace=\"$namespace\", pod_name=\"$pod\", container_name!=\"POD\"}, \"container\", \"$1\", \"container_name\", \"(.*)\")) by (container)", + "expr": "sum(label_replace(namespace_pod_name_container_name:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\", pod_name=\"$pod\", container_name!=\"POD\"}, \"container\", \"$1\", \"container_name\", \"(.*)\")) by (container)", "format": "table", "instant": true, "intervalFactor": 2, @@ -5607,7 +4491,7 @@ data: "step": 10 }, { - "expr": "sum(kube_pod_container_resource_requests_cpu_cores{namespace=\"$namespace\", pod=\"$pod\"}) by (container)", + "expr": "sum(kube_pod_container_resource_requests_cpu_cores{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container)", "format": "table", "instant": true, "intervalFactor": 2, @@ -5616,7 +4500,7 @@ data: "step": 10 }, { - "expr": "sum(label_replace(namespace_pod_name_container_name:container_cpu_usage_seconds_total:sum_rate{namespace=\"$namespace\", pod_name=\"$pod\"}, \"container\", \"$1\", \"container_name\", \"(.*)\")) by (container) / sum(kube_pod_container_resource_requests_cpu_cores{namespace=\"$namespace\", pod=\"$pod\"}) by (container)", + "expr": "sum(label_replace(namespace_pod_name_container_name:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\", pod_name=\"$pod\"}, \"container\", \"$1\", \"container_name\", \"(.*)\")) by (container) / sum(kube_pod_container_resource_requests_cpu_cores{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container)", "format": "table", "instant": true, "intervalFactor": 2, @@ -5625,7 +4509,7 @@ data: "step": 10 }, { - "expr": "sum(kube_pod_container_resource_limits_cpu_cores{namespace=\"$namespace\", pod=\"$pod\"}) by (container)", + "expr": "sum(kube_pod_container_resource_limits_cpu_cores{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container)", "format": "table", "instant": true, "intervalFactor": 2, @@ -5634,7 +4518,7 @@ data: "step": 10 }, { - "expr": "sum(label_replace(namespace_pod_name_container_name:container_cpu_usage_seconds_total:sum_rate{namespace=\"$namespace\", pod_name=\"$pod\"}, \"container\", \"$1\", \"container_name\", \"(.*)\")) by (container) / sum(kube_pod_container_resource_limits_cpu_cores{namespace=\"$namespace\", pod=\"$pod\"}) by (container)", + "expr": "sum(label_replace(namespace_pod_name_container_name:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\", pod_name=\"$pod\"}, \"container\", \"$1\", \"container_name\", \"(.*)\")) by (container) / sum(kube_pod_container_resource_limits_cpu_cores{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container)", "format": "table", "instant": true, "intervalFactor": 2, @@ -5734,7 +4618,7 @@ data: "steppedLine": false, "targets": [ { - "expr": "sum(container_memory_rss{namespace=\"$namespace\", pod_name=\"$pod\", container_name!=\"POD\", container_name!=\"\"}) by (container_name)", + "expr": "sum(container_memory_rss{cluster=\"$cluster\", namespace=\"$namespace\", pod_name=\"$pod\", container_name!=\"POD\", container_name!=\"\"}) by (container_name)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{container_name}} (RSS)", @@ -5742,7 +4626,7 @@ data: "step": 10 }, { - "expr": "sum(container_memory_cache{namespace=\"$namespace\", pod_name=\"$pod\", container_name!=\"POD\", container_name!=\"\"}) by (container_name)", + "expr": "sum(container_memory_cache{cluster=\"$cluster\", namespace=\"$namespace\", pod_name=\"$pod\", container_name!=\"POD\", container_name!=\"\"}) by (container_name)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{container_name}} (Cache)", @@ -5750,7 +4634,7 @@ data: "step": 10 }, { - "expr": "sum(container_memory_swap{namespace=\"$namespace\", pod_name=\"$pod\", container_name!=\"POD\", container_name!=\"\"}) by (container_name)", + "expr": "sum(container_memory_swap{cluster=\"$cluster\", namespace=\"$namespace\", pod_name=\"$pod\", container_name!=\"POD\", container_name!=\"\"}) by (container_name)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{container_name}} (Swap)", @@ -6033,7 +4917,7 @@ data: ], "targets": [ { - "expr": "sum(label_replace(container_memory_usage_bytes{namespace=\"$namespace\", pod_name=\"$pod\", container_name!=\"POD\", container_name!=\"\"}, \"container\", \"$1\", \"container_name\", \"(.*)\")) by (container)", + "expr": "sum(label_replace(container_memory_usage_bytes{cluster=\"$cluster\", namespace=\"$namespace\", pod_name=\"$pod\", container_name!=\"POD\", container_name!=\"\"}, \"container\", \"$1\", \"container_name\", \"(.*)\")) by (container)", "format": "table", "instant": true, "intervalFactor": 2, @@ -6042,7 +4926,7 @@ data: "step": 10 }, { - "expr": "sum(kube_pod_container_resource_requests_memory_bytes{namespace=\"$namespace\", pod=\"$pod\"}) by (container)", + "expr": "sum(kube_pod_container_resource_requests_memory_bytes{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container)", "format": "table", "instant": true, "intervalFactor": 2, @@ -6051,7 +4935,7 @@ data: "step": 10 }, { - "expr": "sum(label_replace(container_memory_usage_bytes{namespace=\"$namespace\", pod_name=\"$pod\"}, \"container\", \"$1\", \"container_name\", \"(.*)\")) by (container) / sum(kube_pod_container_resource_requests_memory_bytes{namespace=\"$namespace\", pod=\"$pod\"}) by (container)", + "expr": "sum(label_replace(container_memory_usage_bytes{cluster=\"$cluster\", namespace=\"$namespace\", pod_name=\"$pod\"}, \"container\", \"$1\", \"container_name\", \"(.*)\")) by (container) / sum(kube_pod_container_resource_requests_memory_bytes{namespace=\"$namespace\", pod=\"$pod\"}) by (container)", "format": "table", "instant": true, "intervalFactor": 2, @@ -6060,7 +4944,7 @@ data: "step": 10 }, { - "expr": "sum(kube_pod_container_resource_limits_memory_bytes{namespace=\"$namespace\", pod=\"$pod\", container!=\"\"}) by (container)", + "expr": "sum(kube_pod_container_resource_limits_memory_bytes{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\", container!=\"\"}) by (container)", "format": "table", "instant": true, "intervalFactor": 2, @@ -6069,7 +4953,7 @@ data: "step": 10 }, { - "expr": "sum(label_replace(container_memory_usage_bytes{namespace=\"$namespace\", pod_name=\"$pod\", container_name!=\"\"}, \"container\", \"$1\", \"container_name\", \"(.*)\")) by (container) / sum(kube_pod_container_resource_limits_memory_bytes{namespace=\"$namespace\", pod=\"$pod\"}) by (container)", + "expr": "sum(label_replace(container_memory_usage_bytes{cluster=\"$cluster\", namespace=\"$namespace\", pod_name=\"$pod\", container_name!=\"\"}, \"container\", \"$1\", \"container_name\", \"(.*)\")) by (container) / sum(kube_pod_container_resource_limits_memory_bytes{namespace=\"$namespace\", pod=\"$pod\"}) by (container)", "format": "table", "instant": true, "intervalFactor": 2, @@ -6078,7 +4962,7 @@ data: "step": 10 }, { - "expr": "sum(label_replace(container_memory_rss{namespace=\"$namespace\", pod_name=\"$pod\", container_name != \"\", container_name != \"POD\"}, \"container\", \"$1\", \"container_name\", \"(.*)\")) by (container)", + "expr": "sum(label_replace(container_memory_rss{cluster=\"$cluster\", namespace=\"$namespace\", pod_name=\"$pod\", container_name != \"\", container_name != \"POD\"}, \"container\", \"$1\", \"container_name\", \"(.*)\")) by (container)", "format": "table", "instant": true, "intervalFactor": 2, @@ -6087,7 +4971,7 @@ data: "step": 10 }, { - "expr": "sum(label_replace(container_memory_cache{namespace=\"$namespace\", pod_name=\"$pod\", container_name != \"\", container_name != \"POD\"}, \"container\", \"$1\", \"container_name\", \"(.*)\")) by (container)", + "expr": "sum(label_replace(container_memory_cache{cluster=\"$cluster\", namespace=\"$namespace\", pod_name=\"$pod\", container_name != \"\", container_name != \"POD\"}, \"container\", \"$1\", \"container_name\", \"(.*)\")) by (container)", "format": "table", "instant": true, "intervalFactor": 2, @@ -6096,7 +4980,7 @@ data: "step": 10 }, { - "expr": "sum(label_replace(container_memory_swap{namespace=\"$namespace\", pod_name=\"$pod\", container_name != \"\", container_name != \"POD\"}, \"container\", \"$1\", \"container_name\", \"(.*)\")) by (container)", + "expr": "sum(label_replace(container_memory_swap{cluster=\"$cluster\", namespace=\"$namespace\", pod_name=\"$pod\", container_name != \"\", container_name != \"POD\"}, \"container\", \"$1\", \"container_name\", \"(.*)\")) by (container)", "format": "table", "instant": true, "intervalFactor": 2, @@ -6178,6 +5062,33 @@ data: "regex": "", "type": "datasource" }, + { + "allValue": null, + "current": { + "text": "prod", + "value": "prod" + }, + "datasource": "$datasource", + "hide": 2, + "includeAll": false, + "label": "cluster", + "multi": false, + "name": "cluster", + "options": [ + + ], + "query": "label_values(kube_pod_info, cluster)", + "refresh": 1, + "regex": "", + "sort": 2, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, { "allValue": null, "current": { @@ -6193,7 +5104,7 @@ data: "options": [ ], - "query": "label_values(kube_pod_info, namespace)", + "query": "label_values(kube_pod_info{cluster=\"$cluster\"}, namespace)", "refresh": 1, "regex": "", "sort": 2, @@ -6220,7 +5131,7 @@ data: "options": [ ], - "query": "label_values(kube_pod_info{namespace=\"$namespace\"}, pod)", + "query": "label_values(kube_pod_info{cluster=\"$cluster\", namespace=\"$namespace\"}, pod)", "refresh": 1, "regex": "", "sort": 2, @@ -6339,21 +5250,21 @@ data: "steppedLine": false, "targets": [ { - "expr": "max(node_load1{job=\"node-exporter\", instance=\"$instance\"})", + "expr": "max(node_load1{cluster=\"$cluster\", job=\"node-exporter\", instance=\"$instance\"})", "format": "time_series", "intervalFactor": 2, "legendFormat": "load 1m", "refId": "A" }, { - "expr": "max(node_load5{job=\"node-exporter\", instance=\"$instance\"})", + "expr": "max(node_load5{cluster=\"$cluster\", job=\"node-exporter\", instance=\"$instance\"})", "format": "time_series", "intervalFactor": 2, "legendFormat": "load 5m", "refId": "B" }, { - "expr": "max(node_load15{job=\"node-exporter\", instance=\"$instance\"})", + "expr": "max(node_load15{cluster=\"$cluster\", job=\"node-exporter\", instance=\"$instance\"})", "format": "time_series", "intervalFactor": 2, "legendFormat": "load 15m", @@ -6444,7 +5355,7 @@ data: "steppedLine": false, "targets": [ { - "expr": "sum by (cpu) (irate(node_cpu_seconds_total{job=\"node-exporter\", mode!=\"idle\", instance=\"$instance\"}[5m]))", + "expr": "sum by (cpu) (irate(node_cpu_seconds_total{cluster=\"$cluster\", job=\"node-exporter\", mode!=\"idle\", instance=\"$instance\"}[5m]))", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{cpu}}", @@ -6548,7 +5459,7 @@ data: "steppedLine": false, "targets": [ { - "expr": "max (sum by (cpu) (irate(node_cpu_seconds_total{job=\"node-exporter\", mode!=\"idle\", instance=\"$instance\"}[2m])) ) * 100\n", + "expr": "max (sum by (cpu) (irate(node_cpu_seconds_total{cluster=\"$cluster\", job=\"node-exporter\", mode!=\"idle\", instance=\"$instance\"}[2m])) ) * 100\n", "format": "time_series", "intervalFactor": 10, "legendFormat": "{{ cpu }}", @@ -6656,7 +5567,7 @@ data: "tableColumn": "", "targets": [ { - "expr": "avg(sum by (cpu) (irate(node_cpu_seconds_total{job=\"node-exporter\", mode!=\"idle\", instance=\"$instance\"}[2m]))) * 100\n", + "expr": "avg(sum by (cpu) (irate(node_cpu_seconds_total{cluster=\"$cluster\", job=\"node-exporter\", mode!=\"idle\", instance=\"$instance\"}[2m]))) * 100\n", "format": "time_series", "intervalFactor": 2, "legendFormat": "", @@ -6736,28 +5647,28 @@ data: "steppedLine": false, "targets": [ { - "expr": "max(\n node_memory_MemTotal_bytes{job=\"node-exporter\", instance=\"$instance\"}\n - node_memory_MemFree_bytes{job=\"node-exporter\", instance=\"$instance\"}\n - node_memory_Buffers_bytes{job=\"node-exporter\", instance=\"$instance\"}\n - node_memory_Cached_bytes{job=\"node-exporter\", instance=\"$instance\"}\n)\n", + "expr": "max(\n node_memory_MemTotal_bytes{cluster=\"$cluster\", job=\"node-exporter\", instance=\"$instance\"}\n - node_memory_MemFree_bytes{cluster=\"$cluster\", job=\"node-exporter\", instance=\"$instance\"}\n - node_memory_Buffers_bytes{cluster=\"$cluster\", job=\"node-exporter\", instance=\"$instance\"}\n - node_memory_Cached_bytes{cluster=\"$cluster\", job=\"node-exporter\", instance=\"$instance\"}\n)\n", "format": "time_series", "intervalFactor": 2, "legendFormat": "memory used", "refId": "A" }, { - "expr": "max(node_memory_Buffers_bytes{job=\"node-exporter\", instance=\"$instance\"})", + "expr": "max(node_memory_Buffers_bytes{cluster=\"$cluster\", job=\"node-exporter\", instance=\"$instance\"})", "format": "time_series", "intervalFactor": 2, "legendFormat": "memory buffers", "refId": "B" }, { - "expr": "max(node_memory_Cached_bytes{job=\"node-exporter\", instance=\"$instance\"})", + "expr": "max(node_memory_Cached_bytes{cluster=\"$cluster\", job=\"node-exporter\", instance=\"$instance\"})", "format": "time_series", "intervalFactor": 2, "legendFormat": "memory cached", "refId": "C" }, { - "expr": "max(node_memory_MemFree_bytes{job=\"node-exporter\", instance=\"$instance\"})", + "expr": "max(node_memory_MemFree_bytes{cluster=\"$cluster\", job=\"node-exporter\", instance=\"$instance\"})", "format": "time_series", "intervalFactor": 2, "legendFormat": "memory free", @@ -6865,7 +5776,7 @@ data: "tableColumn": "", "targets": [ { - "expr": "max(\n (\n (\n node_memory_MemTotal_bytes{job=\"node-exporter\", instance=\"$instance\"}\n - node_memory_MemFree_bytes{job=\"node-exporter\", instance=\"$instance\"}\n - node_memory_Buffers_bytes{job=\"node-exporter\", instance=\"$instance\"}\n - node_memory_Cached_bytes{job=\"node-exporter\", instance=\"$instance\"}\n )\n / node_memory_MemTotal_bytes{job=\"node-exporter\", instance=\"$instance\"}\n ) * 100)\n", + "expr": "max(\n (\n (\n node_memory_MemTotal_bytes{cluster=\"$cluster\", job=\"node-exporter\", instance=\"$instance\"}\n - node_memory_MemFree_bytes{cluster=\"$cluster\", job=\"node-exporter\", instance=\"$instance\"}\n - node_memory_Buffers_bytes{cluster=\"$cluster\", job=\"node-exporter\", instance=\"$instance\"}\n - node_memory_Cached_bytes{cluster=\"$cluster\", job=\"node-exporter\", instance=\"$instance\"}\n )\n / node_memory_MemTotal_bytes{cluster=\"$cluster\", job=\"node-exporter\", instance=\"$instance\"}\n ) * 100)\n", "format": "time_series", "intervalFactor": 2, "legendFormat": "", @@ -6952,21 +5863,21 @@ data: "steppedLine": false, "targets": [ { - "expr": "max(rate(node_disk_read_bytes_total{job=\"node-exporter\", instance=\"$instance\"}[2m]))", + "expr": "max(rate(node_disk_read_bytes_total{cluster=\"$cluster\", job=\"node-exporter\", instance=\"$instance\"}[2m]))", "format": "time_series", "intervalFactor": 2, "legendFormat": "read", "refId": "A" }, { - "expr": "max(rate(node_disk_written_bytes_total{job=\"node-exporter\", instance=\"$instance\"}[2m]))", + "expr": "max(rate(node_disk_written_bytes_total{cluster=\"$cluster\", job=\"node-exporter\", instance=\"$instance\"}[2m]))", "format": "time_series", "intervalFactor": 2, "legendFormat": "written", "refId": "B" }, { - "expr": "max(rate(node_disk_io_time_seconds_total{job=\"node-exporter\", instance=\"$instance\"}[2m]))", + "expr": "max(rate(node_disk_io_time_seconds_total{cluster=\"$cluster\", job=\"node-exporter\", instance=\"$instance\"}[2m]))", "format": "time_series", "intervalFactor": 2, "legendFormat": "io time", @@ -7057,7 +5968,7 @@ data: "steppedLine": false, "targets": [ { - "expr": "node:node_filesystem_usage:\n", + "expr": "node:node_filesystem_usage:{cluster=\"$cluster\"}", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{device}}", @@ -7161,7 +6072,7 @@ data: "steppedLine": false, "targets": [ { - "expr": "max(rate(node_network_receive_bytes_total{job=\"node-exporter\", instance=\"$instance\", device!~\"lo\"}[5m]))", + "expr": "max(rate(node_network_receive_bytes_total{cluster=\"$cluster\", job=\"node-exporter\", instance=\"$instance\", device!~\"lo\"}[5m]))", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{device}}", @@ -7252,7 +6163,7 @@ data: "steppedLine": false, "targets": [ { - "expr": "max(rate(node_network_transmit_bytes_total{job=\"node-exporter\", instance=\"$instance\", device!~\"lo\"}[5m]))", + "expr": "max(rate(node_network_transmit_bytes_total{cluster=\"$cluster\", job=\"node-exporter\", instance=\"$instance\", device!~\"lo\"}[5m]))", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{device}}", @@ -7356,14 +6267,14 @@ data: "steppedLine": false, "targets": [ { - "expr": "max(\n node_filesystem_files{job=\"node-exporter\", instance=\"$instance\"}\n - node_filesystem_files_free{job=\"node-exporter\", instance=\"$instance\"}\n)\n", + "expr": "max(\n node_filesystem_files{cluster=\"$cluster\", job=\"node-exporter\", instance=\"$instance\"}\n - node_filesystem_files_free{cluster=\"$cluster\", job=\"node-exporter\", instance=\"$instance\"}\n)\n", "format": "time_series", "intervalFactor": 2, "legendFormat": "inodes used", "refId": "A" }, { - "expr": "max(node_filesystem_files_free{job=\"node-exporter\", instance=\"$instance\"})", + "expr": "max(node_filesystem_files_free{cluster=\"$cluster\", job=\"node-exporter\", instance=\"$instance\"})", "format": "time_series", "intervalFactor": 2, "legendFormat": "inodes free", @@ -7471,7 +6382,7 @@ data: "tableColumn": "", "targets": [ { - "expr": "max(\n (\n (\n node_filesystem_files{job=\"node-exporter\", instance=\"$instance\"}\n - node_filesystem_files_free{job=\"node-exporter\", instance=\"$instance\"}\n )\n / node_filesystem_files{job=\"node-exporter\", instance=\"$instance\"}\n ) * 100)\n", + "expr": "max(\n (\n (\n node_filesystem_files{cluster=\"$cluster\", job=\"node-exporter\", instance=\"$instance\"}\n - node_filesystem_files_free{cluster=\"$cluster\", job=\"node-exporter\", instance=\"$instance\"}\n )\n / node_filesystem_files{cluster=\"$cluster\", job=\"node-exporter\", instance=\"$instance\"}\n ) * 100)\n", "format": "time_series", "intervalFactor": 2, "legendFormat": "", @@ -7531,6 +6442,32 @@ data: "allValue": null, "current": { + }, + "datasource": "$datasource", + "hide": 2, + "includeAll": false, + "label": "cluster", + "multi": false, + "name": "cluster", + "options": [ + + ], + "query": "label_values(kube_pod_info, cluster)", + "refresh": 2, + "regex": "", + "sort": 0, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + }, "datasource": "$datasource", "hide": 0, @@ -7541,7 +6478,7 @@ data: "options": [ ], - "query": "label_values(node_boot_time_seconds{job=\"node-exporter\"}, instance)", + "query": "label_values(node_boot_time_seconds{cluster=\"$cluster\", job=\"node-exporter\"}, instance)", "refresh": 2, "regex": "", "sort": 0, @@ -7660,7 +6597,7 @@ data: "steppedLine": false, "targets": [ { - "expr": "(kubelet_volume_stats_capacity_bytes{job=\"kubelet\", persistentvolumeclaim=\"$volume\"} - kubelet_volume_stats_available_bytes{job=\"kubelet\", persistentvolumeclaim=\"$volume\"}) / kubelet_volume_stats_capacity_bytes{job=\"kubelet\", persistentvolumeclaim=\"$volume\"} * 100\n", + "expr": "(kubelet_volume_stats_capacity_bytes{cluster=\"$cluster\", job=\"kubelet\", persistentvolumeclaim=\"$volume\"} - kubelet_volume_stats_available_bytes{cluster=\"$cluster\", job=\"kubelet\", persistentvolumeclaim=\"$volume\"}) / kubelet_volume_stats_capacity_bytes{cluster=\"$cluster\", job=\"kubelet\", persistentvolumeclaim=\"$volume\"} * 100\n", "format": "time_series", "intervalFactor": 1, "legendFormat": "{{ Usage }}", @@ -7764,7 +6701,7 @@ data: "steppedLine": false, "targets": [ { - "expr": "kubelet_volume_stats_inodes_used{job=\"kubelet\", persistentvolumeclaim=\"$volume\"} / kubelet_volume_stats_inodes{job=\"kubelet\", persistentvolumeclaim=\"$volume\"} * 100\n", + "expr": "kubelet_volume_stats_inodes_used{cluster=\"$cluster\", job=\"kubelet\", persistentvolumeclaim=\"$volume\"} / kubelet_volume_stats_inodes{cluster=\"$cluster\", job=\"kubelet\", persistentvolumeclaim=\"$volume\"} * 100\n", "format": "time_series", "intervalFactor": 1, "legendFormat": "{{ Usage }}", @@ -7848,6 +6785,32 @@ data: "allValue": null, "current": { + }, + "datasource": "$datasource", + "hide": 2, + "includeAll": false, + "label": "cluster", + "multi": false, + "name": "cluster", + "options": [ + + ], + "query": "label_values(kubelet_volume_stats_capacity_bytes, cluster)", + "refresh": 2, + "regex": "", + "sort": 0, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + }, "datasource": "$datasource", "hide": 0, @@ -7858,7 +6821,7 @@ data: "options": [ ], - "query": "label_values(kubelet_volume_stats_capacity_bytes{job=\"kubelet\"}, exported_namespace)", + "query": "label_values(kubelet_volume_stats_capacity_bytes{cluster=\"$cluster\", job=\"kubelet\"}, exported_namespace)", "refresh": 2, "regex": "", "sort": 0, @@ -7884,7 +6847,7 @@ data: "options": [ ], - "query": "label_values(kubelet_volume_stats_capacity_bytes{job=\"kubelet\", exported_namespace=\"$namespace\"}, persistentvolumeclaim)", + "query": "label_values(kubelet_volume_stats_capacity_bytes{cluster=\"$cluster\", job=\"kubelet\", exported_namespace=\"$namespace\"}, persistentvolumeclaim)", "refresh": 2, "regex": "", "sort": 0, @@ -7998,25 +6961,26 @@ data: ], "spaceLength": 10, + "span": 12, "stack": false, "steppedLine": false, "targets": [ { - "expr": "sum by(container_name) (container_memory_usage_bytes{job=\"kubernetes-cadvisor\", namespace=\"$namespace\", pod_name=\"$pod\", container_name=~\"$container\", container_name!=\"POD\"})", + "expr": "sum by(container_name) (container_memory_usage_bytes{job=\"kubernetes-cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\", pod_name=\"$pod\", container_name=~\"$container\", container_name!=\"POD\"})", "format": "time_series", "intervalFactor": 2, "legendFormat": "Current: {{ container_name }}", "refId": "A" }, { - "expr": "sum by(container) (kube_pod_container_resource_requests_memory_bytes{job=\"kube-state-metrics\", namespace=\"$namespace\", pod=\"$pod\", container=~\"$container\"})", + "expr": "sum by(container) (kube_pod_container_resource_requests_memory_bytes{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\", container=~\"$container\"})", "format": "time_series", "intervalFactor": 2, "legendFormat": "Requested: {{ container }}", "refId": "B" }, { - "expr": "sum by(container) (kube_pod_container_resource_limits_memory_bytes{job=\"kube-state-metrics\", namespace=\"$namespace\", pod=\"$pod\", container=~\"$container\"})", + "expr": "sum by(container) (kube_pod_container_resource_limits_memory_bytes{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\", container=~\"$container\"})", "format": "time_series", "intervalFactor": 2, "legendFormat": "Limit: {{ container }}", @@ -8115,11 +7079,12 @@ data: ], "spaceLength": 10, + "span": 12, "stack": false, "steppedLine": false, "targets": [ { - "expr": "sum by (container_name) (rate(container_cpu_usage_seconds_total{job=\"kubernetes-cadvisor\", namespace=\"$namespace\", image!=\"\",container_name!=\"POD\",pod_name=\"$pod\"}[1m]))", + "expr": "sum by (container_name) (rate(container_cpu_usage_seconds_total{job=\"kubernetes-cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\", image!=\"\",container_name!=\"POD\",pod_name=\"$pod\"}[1m]))", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{ container_name }}", @@ -8218,11 +7183,12 @@ data: ], "spaceLength": 10, + "span": 12, "stack": false, "steppedLine": false, "targets": [ { - "expr": "sort_desc(sum by (pod_name) (rate(container_network_receive_bytes_total{job=\"kubernetes-cadvisor\", namespace=\"$namespace\", pod_name=\"$pod\"}[1m])))", + "expr": "sort_desc(sum by (pod_name) (rate(container_network_receive_bytes_total{job=\"kubernetes-cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\", pod_name=\"$pod\"}[1m])))", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{ pod_name }}", @@ -8306,6 +7272,32 @@ data: "allValue": null, "current": { + }, + "datasource": "$datasource", + "hide": 2, + "includeAll": false, + "label": "cluster", + "multi": false, + "name": "cluster", + "options": [ + + ], + "query": "label_values(kube_pod_info, cluster)", + "refresh": 2, + "regex": "", + "sort": 0, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + }, "datasource": "$datasource", "hide": 0, @@ -8316,7 +7308,7 @@ data: "options": [ ], - "query": "label_values(kube_pod_info, namespace)", + "query": "label_values(kube_pod_info{cluster=\"$cluster\"}, namespace)", "refresh": 2, "regex": "", "sort": 0, @@ -8342,7 +7334,7 @@ data: "options": [ ], - "query": "label_values(kube_pod_info{namespace=~\"$namespace\"}, pod)", + "query": "label_values(kube_pod_info{cluster=\"$cluster\", namespace=~\"$namespace\"}, pod)", "refresh": 2, "regex": "", "sort": 0, @@ -8368,7 +7360,7 @@ data: "options": [ ], - "query": "label_values(kube_pod_container_info{namespace=\"$namespace\", pod=\"$pod\"}, container)", + "query": "label_values(kube_pod_container_info{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}, container)", "refresh": 2, "regex": "", "sort": 0, @@ -8503,7 +7495,7 @@ data: "tableColumn": "", "targets": [ { - "expr": "sum(rate(container_cpu_usage_seconds_total{job=\"kubernetes-cadvisor\", namespace=\"$namespace\", pod_name=~\"$statefulset.*\"}[3m]))", + "expr": "sum(rate(container_cpu_usage_seconds_total{job=\"kubernetes-cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\", pod_name=~\"$statefulset.*\"}[3m]))", "format": "time_series", "intervalFactor": 2, "legendFormat": "", @@ -8586,7 +7578,7 @@ data: "tableColumn": "", "targets": [ { - "expr": "sum(container_memory_usage_bytes{job=\"kubernetes-cadvisor\", namespace=\"$namespace\", pod_name=~\"$statefulset.*\"}) / 1024^3", + "expr": "sum(container_memory_usage_bytes{job=\"kubernetes-cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\", pod_name=~\"$statefulset.*\"}) / 1024^3", "format": "time_series", "intervalFactor": 2, "legendFormat": "", @@ -8669,7 +7661,7 @@ data: "tableColumn": "", "targets": [ { - "expr": "sum(rate(container_network_transmit_bytes_total{job=\"kubernetes-cadvisor\", namespace=\"$namespace\", pod_name=~\"$statefulset.*\"}[3m])) + sum(rate(container_network_receive_bytes_total{namespace=\"$namespace\",pod_name=~\"$statefulset.*\"}[3m]))", + "expr": "sum(rate(container_network_transmit_bytes_total{job=\"kubernetes-cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\", pod_name=~\"$statefulset.*\"}[3m])) + sum(rate(container_network_receive_bytes_total{cluster=\"$cluster\", namespace=\"$namespace\",pod_name=~\"$statefulset.*\"}[3m]))", "format": "time_series", "intervalFactor": 2, "legendFormat": "", @@ -8767,7 +7759,7 @@ data: "tableColumn": "", "targets": [ { - "expr": "max(kube_statefulset_replicas{job=\"kube-state-metrics\", namespace=\"$namespace\", statefulset=\"$statefulset\"}) without (instance, pod)", + "expr": "max(kube_statefulset_replicas{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", statefulset=\"$statefulset\"}) without (instance, pod)", "format": "time_series", "intervalFactor": 2, "legendFormat": "", @@ -8851,7 +7843,7 @@ data: "tableColumn": "", "targets": [ { - "expr": "min(kube_statefulset_status_replicas_current{job=\"kube-state-metrics\", namespace=\"$namespace\", statefulset=\"$statefulset\"}) without (instance, pod)", + "expr": "min(kube_statefulset_status_replicas_current{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", statefulset=\"$statefulset\"}) without (instance, pod)", "format": "time_series", "intervalFactor": 2, "legendFormat": "", @@ -8935,7 +7927,7 @@ data: "tableColumn": "", "targets": [ { - "expr": "max(kube_statefulset_status_observed_generation{job=\"kube-state-metrics\", namespace=\"$namespace\", statefulset=\"$statefulset\"}) without (instance, pod)", + "expr": "max(kube_statefulset_status_observed_generation{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", statefulset=\"$statefulset\"}) without (instance, pod)", "format": "time_series", "intervalFactor": 2, "legendFormat": "", @@ -9019,7 +8011,7 @@ data: "tableColumn": "", "targets": [ { - "expr": "max(kube_statefulset_metadata_generation{job=\"kube-state-metrics\", statefulset=\"$statefulset\", namespace=\"$namespace\"}) without (instance, pod)", + "expr": "max(kube_statefulset_metadata_generation{job=\"kube-state-metrics\", statefulset=\"$statefulset\", cluster=\"$cluster\", namespace=\"$namespace\"}) without (instance, pod)", "format": "time_series", "intervalFactor": 2, "legendFormat": "", @@ -9098,35 +8090,35 @@ data: "steppedLine": false, "targets": [ { - "expr": "max(kube_statefulset_replicas{job=\"kube-state-metrics\", statefulset=\"$statefulset\",namespace=\"$namespace\"}) without (instance, pod)", + "expr": "max(kube_statefulset_replicas{job=\"kube-state-metrics\", statefulset=\"$statefulset\", cluster=\"$cluster\", namespace=\"$namespace\"}) without (instance, pod)", "format": "time_series", "intervalFactor": 2, "legendFormat": "replicas specified", "refId": "A" }, { - "expr": "max(kube_statefulset_status_replicas{job=\"kube-state-metrics\", statefulset=\"$statefulset\",namespace=\"$namespace\"}) without (instance, pod)", + "expr": "max(kube_statefulset_status_replicas{job=\"kube-state-metrics\", statefulset=\"$statefulset\", cluster=\"$cluster\", namespace=\"$namespace\"}) without (instance, pod)", "format": "time_series", "intervalFactor": 2, "legendFormat": "replicas created", "refId": "B" }, { - "expr": "min(kube_statefulset_status_replicas_ready{job=\"kube-state-metrics\", statefulset=\"$statefulset\",namespace=\"$namespace\"}) without (instance, pod)", + "expr": "min(kube_statefulset_status_replicas_ready{job=\"kube-state-metrics\", statefulset=\"$statefulset\", cluster=\"$cluster\", namespace=\"$namespace\"}) without (instance, pod)", "format": "time_series", "intervalFactor": 2, "legendFormat": "ready", "refId": "C" }, { - "expr": "min(kube_statefulset_status_replicas_current{job=\"kube-state-metrics\", statefulset=\"$statefulset\",namespace=\"$namespace\"}) without (instance, pod)", + "expr": "min(kube_statefulset_status_replicas_current{job=\"kube-state-metrics\", statefulset=\"$statefulset\", cluster=\"$cluster\", namespace=\"$namespace\"}) without (instance, pod)", "format": "time_series", "intervalFactor": 2, "legendFormat": "replicas of current version", "refId": "D" }, { - "expr": "min(kube_statefulset_status_replicas_updated{job=\"kube-state-metrics\", statefulset=\"$statefulset\",namespace=\"$namespace\"}) without (instance, pod)", + "expr": "min(kube_statefulset_status_replicas_updated{job=\"kube-state-metrics\", statefulset=\"$statefulset\", cluster=\"$cluster\", namespace=\"$namespace\"}) without (instance, pod)", "format": "time_series", "intervalFactor": 2, "legendFormat": "updated", @@ -9210,6 +8202,32 @@ data: "allValue": null, "current": { + }, + "datasource": "$datasource", + "hide": 2, + "includeAll": false, + "label": "cluster", + "multi": false, + "name": "cluster", + "options": [ + + ], + "query": "label_values(kube_statefulset_metadata_generation, cluster)", + "refresh": 2, + "regex": "", + "sort": 0, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + }, "datasource": "$datasource", "hide": 0, diff --git a/addons/grafana/deployment.yaml b/addons/grafana/deployment.yaml index 226233fbb..09b057854 100644 --- a/addons/grafana/deployment.yaml +++ b/addons/grafana/deployment.yaml @@ -44,8 +44,10 @@ spec: mountPath: /etc/grafana/provisioning/datasources - name: providers mountPath: /etc/grafana/provisioning/dashboards - - name: dashboards - mountPath: /etc/grafana/dashboards + - name: dashboards-etcd + mountPath: /etc/grafana/dashboards/etcd + - name: dashboards-k8s + mountPath: /etc/grafana/dashboards/k8s volumes: - name: config configMap: @@ -56,6 +58,10 @@ spec: - name: providers configMap: name: grafana-providers - - name: dashboards + - name: dashboards-etcd configMap: - name: grafana-dashboards + name: grafana-dashboards-etcd + - name: dashboards-k8s + configMap: + name: grafana-dashboards-k8s + diff --git a/addons/prometheus/rules.yaml b/addons/prometheus/rules.yaml index f88d94c06..38d5d6084 100644 --- a/addons/prometheus/rules.yaml +++ b/addons/prometheus/rules.yaml @@ -366,19 +366,19 @@ data: "record": "node:node_memory_swap_io_bytes:sum_rate" }, { - "expr": "avg(irate(node_disk_io_time_seconds_total{job=\"node-exporter\",device=~\"nvme.+|rbd.+|sd.+|vd.+|xvd.+\"}[1m]))\n", + "expr": "avg(irate(node_disk_io_time_seconds_total{job=\"node-exporter\",device=~\"nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+\"}[1m]))\n", "record": ":node_disk_utilisation:avg_irate" }, { - "expr": "avg by (node) (\n irate(node_disk_io_time_seconds_total{job=\"node-exporter\",device=~\"nvme.+|rbd.+|sd.+|vd.+|xvd.+\"}[1m])\n* on (namespace, pod) group_left(node)\n node_namespace_pod:kube_pod_info:\n)\n", + "expr": "avg by (node) (\n irate(node_disk_io_time_seconds_total{job=\"node-exporter\",device=~\"nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+\"}[1m])\n* on (namespace, pod) group_left(node)\n node_namespace_pod:kube_pod_info:\n)\n", "record": "node:node_disk_utilisation:avg_irate" }, { - "expr": "avg(irate(node_disk_io_time_weighted_seconds_total{job=\"node-exporter\",device=~\"nvme.+|rbd.+|sd.+|vd.+|xvd.+\"}[1m]) / 1e3)\n", + "expr": "avg(irate(node_disk_io_time_weighted_seconds_total{job=\"node-exporter\",device=~\"nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+\"}[1m]) / 1e3)\n", "record": ":node_disk_saturation:avg_irate" }, { - "expr": "avg by (node) (\n irate(node_disk_io_time_weighted_seconds_total{job=\"node-exporter\",device=~\"nvme.+|rbd.+|sd.+|vd.+|xvd.+\"}[1m]) / 1e3\n* on (namespace, pod) group_left(node)\n node_namespace_pod:kube_pod_info:\n)\n", + "expr": "avg by (node) (\n irate(node_disk_io_time_weighted_seconds_total{job=\"node-exporter\",device=~\"nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+\"}[1m]) / 1e3\n* on (namespace, pod) group_left(node)\n node_namespace_pod:kube_pod_info:\n)\n", "record": "node:node_disk_saturation:avg_irate" }, { @@ -840,7 +840,7 @@ data: "message": "API server is returning errors for {{ $value }}% of requests.", "runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubeapierrorshigh" }, - "expr": "sum(rate(apiserver_request_count{job=\"apiserver\",code=~\"^(?:5..)$\"}[5m])) without(instance, pod)\n /\nsum(rate(apiserver_request_count{job=\"apiserver\"}[5m])) without(instance, pod) * 100 > 10\n", + "expr": "sum(rate(apiserver_request_count{job=\"apiserver\",code=~\"^(?:5..)$\"}[5m]))\n /\nsum(rate(apiserver_request_count{job=\"apiserver\"}[5m])) * 100 > 3\n", "for": "10m", "labels": { "severity": "critical" @@ -852,7 +852,31 @@ data: "message": "API server is returning errors for {{ $value }}% of requests.", "runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubeapierrorshigh" }, - "expr": "sum(rate(apiserver_request_count{job=\"apiserver\",code=~\"^(?:5..)$\"}[5m])) without(instance, pod)\n /\nsum(rate(apiserver_request_count{job=\"apiserver\"}[5m])) without(instance, pod) * 100 > 5\n", + "expr": "sum(rate(apiserver_request_count{job=\"apiserver\",code=~\"^(?:5..)$\"}[5m]))\n /\nsum(rate(apiserver_request_count{job=\"apiserver\"}[5m])) * 100 > 1\n", + "for": "10m", + "labels": { + "severity": "warning" + } + }, + { + "alert": "KubeAPIErrorsHigh", + "annotations": { + "message": "API server is returning errors for {{ $value }}% of requests for {{ $labels.verb }} {{ $labels.resource }} {{ $labels.subresource }}.", + "runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubeapierrorshigh" + }, + "expr": "sum(rate(apiserver_request_count{job=\"apiserver\",code=~\"^(?:5..)$\"}[5m])) by (resource,subresource,verb)\n /\nsum(rate(apiserver_request_count{job=\"apiserver\"}[5m])) by (resource,subresource,verb) * 100 > 10\n", + "for": "10m", + "labels": { + "severity": "critical" + } + }, + { + "alert": "KubeAPIErrorsHigh", + "annotations": { + "message": "API server is returning errors for {{ $value }}% of requests for {{ $labels.verb }} {{ $labels.resource }} {{ $labels.subresource }}.", + "runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubeapierrorshigh" + }, + "expr": "sum(rate(apiserver_request_count{job=\"apiserver\",code=~\"^(?:5..)$\"}[5m])) by (resource,subresource,verb)\n /\nsum(rate(apiserver_request_count{job=\"apiserver\"}[5m])) by (resource,subresource,verb) * 100 > 5\n", "for": "10m", "labels": { "severity": "warning" @@ -1040,7 +1064,7 @@ data: "description": "{{$labels.job}} at {{$labels.instance}} has a corrupted write-ahead log (WAL).", "summary": "Prometheus write-ahead log is corrupted" }, - "expr": "tsdb_wal_corruptions_total{job=\"prometheus\"} > 0\n", + "expr": "prometheus_tsdb_wal_corruptions_total{job=\"prometheus\"} > 0\n", "for": "4h", "labels": { "severity": "warning" From 1feefbe9c66bc8e7dcffe1eb085ad6c121678ac1 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Mon, 18 Mar 2019 21:07:07 -0700 Subject: [PATCH 054/523] Update Calico from v3.5.2 to v3.6.0 * Add calico-ipam CRDs and RBAC permissions * Switch IPAM from host-local to calico-ipam * `calico-ipam` subnets `ippools` (defaults to pod CIDR) into `ipamblocks` (defaults to /26, but set to /24 in Typhoon) * `host-local` subnets the pod CIDR based on the node PodCIDR field (set via kube-controller-manager as /24's) * Create a custom default IPv4 IPPool to ensure the block size is kept at /24 to allow 110 pods per node (Kubernetes default) * Retaining host-local was slightly preferred, but Calico v3.6 is migrating all usage to calico-ipam. The codepath that skipped calico-ipam for KDD was removed * https://docs.projectcalico.org/v3.6/release-notes/ --- CHANGES.md | 2 ++ aws/container-linux/kubernetes/bootkube.tf | 2 +- aws/fedora-atomic/kubernetes/bootkube.tf | 2 +- azure/container-linux/kubernetes/bootkube.tf | 2 +- bare-metal/container-linux/kubernetes/bootkube.tf | 2 +- bare-metal/fedora-atomic/kubernetes/bootkube.tf | 2 +- digital-ocean/container-linux/kubernetes/bootkube.tf | 2 +- digital-ocean/fedora-atomic/kubernetes/bootkube.tf | 2 +- google-cloud/container-linux/kubernetes/bootkube.tf | 2 +- google-cloud/fedora-atomic/kubernetes/bootkube.tf | 2 +- 10 files changed, 11 insertions(+), 9 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index e145e7bb7..d7676bd5f 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -6,6 +6,8 @@ Notable changes between versions. * Resolve in-addr.arpa reverse DNS lookups (PTR) for pod IPv4 addresses ([#415](https://github.com/poseidon/typhoon/pull/415)) * Reverse DNS lookups for service IPv4 addresses unchanged +* Upgrade Calico from v3.5.2 to [v3.6.0](https://docs.projectcalico.org/v3.6/release-notes/) ([#430](https://github.com/poseidon/typhoon/pull/430)) + * Change pod IPAM from `host-local` to `calico-ipam`. `pod_cidr` is still divided into `/24` subnets per node, but managed as `ippools` and `ipamblocks` #### AWS diff --git a/aws/container-linux/kubernetes/bootkube.tf b/aws/container-linux/kubernetes/bootkube.tf index 4ffb48852..aaa35f055 100644 --- a/aws/container-linux/kubernetes/bootkube.tf +++ b/aws/container-linux/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=1528266595b91d40f824a3281563f13dd0a5b6df" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=23f81a5e8c4a975750beabc6f603a60a04621225" cluster_name = "${var.cluster_name}" api_servers = ["${format("%s.%s", var.cluster_name, var.dns_zone)}"] diff --git a/aws/fedora-atomic/kubernetes/bootkube.tf b/aws/fedora-atomic/kubernetes/bootkube.tf index a2116f5ed..4c45f2e1c 100644 --- a/aws/fedora-atomic/kubernetes/bootkube.tf +++ b/aws/fedora-atomic/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=1528266595b91d40f824a3281563f13dd0a5b6df" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=23f81a5e8c4a975750beabc6f603a60a04621225" cluster_name = "${var.cluster_name}" api_servers = ["${format("%s.%s", var.cluster_name, var.dns_zone)}"] diff --git a/azure/container-linux/kubernetes/bootkube.tf b/azure/container-linux/kubernetes/bootkube.tf index 6d2513731..07d12ca8d 100644 --- a/azure/container-linux/kubernetes/bootkube.tf +++ b/azure/container-linux/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=1528266595b91d40f824a3281563f13dd0a5b6df" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=23f81a5e8c4a975750beabc6f603a60a04621225" cluster_name = "${var.cluster_name}" api_servers = ["${format("%s.%s", var.cluster_name, var.dns_zone)}"] diff --git a/bare-metal/container-linux/kubernetes/bootkube.tf b/bare-metal/container-linux/kubernetes/bootkube.tf index c2937f085..aceff83ab 100644 --- a/bare-metal/container-linux/kubernetes/bootkube.tf +++ b/bare-metal/container-linux/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=1528266595b91d40f824a3281563f13dd0a5b6df" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=23f81a5e8c4a975750beabc6f603a60a04621225" cluster_name = "${var.cluster_name}" api_servers = ["${var.k8s_domain_name}"] diff --git a/bare-metal/fedora-atomic/kubernetes/bootkube.tf b/bare-metal/fedora-atomic/kubernetes/bootkube.tf index 0eb1a1c5c..0d854f794 100644 --- a/bare-metal/fedora-atomic/kubernetes/bootkube.tf +++ b/bare-metal/fedora-atomic/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=1528266595b91d40f824a3281563f13dd0a5b6df" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=23f81a5e8c4a975750beabc6f603a60a04621225" cluster_name = "${var.cluster_name}" api_servers = ["${var.k8s_domain_name}"] diff --git a/digital-ocean/container-linux/kubernetes/bootkube.tf b/digital-ocean/container-linux/kubernetes/bootkube.tf index 8082608e5..5f679cfc9 100644 --- a/digital-ocean/container-linux/kubernetes/bootkube.tf +++ b/digital-ocean/container-linux/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=1528266595b91d40f824a3281563f13dd0a5b6df" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=23f81a5e8c4a975750beabc6f603a60a04621225" cluster_name = "${var.cluster_name}" api_servers = ["${format("%s.%s", var.cluster_name, var.dns_zone)}"] diff --git a/digital-ocean/fedora-atomic/kubernetes/bootkube.tf b/digital-ocean/fedora-atomic/kubernetes/bootkube.tf index ae48813e0..f33c4a993 100644 --- a/digital-ocean/fedora-atomic/kubernetes/bootkube.tf +++ b/digital-ocean/fedora-atomic/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=1528266595b91d40f824a3281563f13dd0a5b6df" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=23f81a5e8c4a975750beabc6f603a60a04621225" cluster_name = "${var.cluster_name}" api_servers = ["${format("%s.%s", var.cluster_name, var.dns_zone)}"] diff --git a/google-cloud/container-linux/kubernetes/bootkube.tf b/google-cloud/container-linux/kubernetes/bootkube.tf index fa7962e6e..f4d34467d 100644 --- a/google-cloud/container-linux/kubernetes/bootkube.tf +++ b/google-cloud/container-linux/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=1528266595b91d40f824a3281563f13dd0a5b6df" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=23f81a5e8c4a975750beabc6f603a60a04621225" cluster_name = "${var.cluster_name}" api_servers = ["${format("%s.%s", var.cluster_name, var.dns_zone)}"] diff --git a/google-cloud/fedora-atomic/kubernetes/bootkube.tf b/google-cloud/fedora-atomic/kubernetes/bootkube.tf index a4ea41afd..d80a295d4 100644 --- a/google-cloud/fedora-atomic/kubernetes/bootkube.tf +++ b/google-cloud/fedora-atomic/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=1528266595b91d40f824a3281563f13dd0a5b6df" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=23f81a5e8c4a975750beabc6f603a60a04621225" cluster_name = "${var.cluster_name}" api_servers = ["${format("%s.%s", var.cluster_name, var.dns_zone)}"] From 6dd2731046cb484e3634138c5b003d5f59006ef0 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Wed, 20 Mar 2019 00:15:08 -0700 Subject: [PATCH 055/523] Set cpu/memory resources requests/limits for some addons * Set resource requests and limits for Grafana and CLUO * Set resource requests for Prometheus, but allow usage to grow since needs vary widely * Leave nginx without resource requests/limits for now, its typically well behaved --- addons/cluo/update-agent.yaml | 61 +++++++++++++++++-------------- addons/cluo/update-operator.yaml | 28 +++++++++----- addons/grafana/deployment.yaml | 4 +- addons/prometheus/deployment.yaml | 4 ++ 4 files changed, 58 insertions(+), 39 deletions(-) diff --git a/addons/cluo/update-agent.yaml b/addons/cluo/update-agent.yaml index c40f3fe5c..880615f3e 100644 --- a/addons/cluo/update-agent.yaml +++ b/addons/cluo/update-agent.yaml @@ -18,34 +18,41 @@ spec: annotations: seccomp.security.alpha.kubernetes.io/pod: 'docker/default' spec: - containers: - - name: update-agent - image: quay.io/coreos/container-linux-update-operator:v0.7.0 - command: - - "/bin/update-agent" - volumeMounts: - - mountPath: /var/run/dbus - name: var-run-dbus - - mountPath: /etc/coreos - name: etc-coreos - - mountPath: /usr/share/coreos - name: usr-share-coreos - - mountPath: /etc/os-release - name: etc-os-release - env: - # read by update-agent as the node name to manage reboots for - - name: UPDATE_AGENT_NODE - valueFrom: - fieldRef: - fieldPath: spec.nodeName - - name: POD_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace tolerations: - - key: node-role.kubernetes.io/master - operator: Exists - effect: NoSchedule + - key: node-role.kubernetes.io/master + operator: Exists + effect: NoSchedule + containers: + - name: update-agent + image: quay.io/coreos/container-linux-update-operator:v0.7.0 + command: + - "/bin/update-agent" + env: + # read by update-agent as the node name to manage reboots for + - name: UPDATE_AGENT_NODE + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + resources: + requests: + cpu: 10m + memory: 20Mi + limits: + cpu: 20m + memory: 40Mi + volumeMounts: + - mountPath: /var/run/dbus + name: var-run-dbus + - mountPath: /etc/coreos + name: etc-coreos + - mountPath: /usr/share/coreos + name: usr-share-coreos + - mountPath: /etc/os-release + name: etc-os-release volumes: - name: var-run-dbus hostPath: diff --git a/addons/cluo/update-operator.yaml b/addons/cluo/update-operator.yaml index 62b04a605..bb922ffab 100644 --- a/addons/cluo/update-operator.yaml +++ b/addons/cluo/update-operator.yaml @@ -15,17 +15,25 @@ spec: annotations: seccomp.security.alpha.kubernetes.io/pod: 'docker/default' spec: - containers: - - name: update-operator - image: quay.io/coreos/container-linux-update-operator:v0.7.0 - command: - - "/bin/update-operator" - env: - - name: POD_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace tolerations: - key: node-role.kubernetes.io/master operator: Exists effect: NoSchedule + containers: + - name: update-operator + image: quay.io/coreos/container-linux-update-operator:v0.7.0 + command: + - "/bin/update-operator" + env: + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + resources: + requests: + cpu: 10m + memory: 20Mi + limits: + cpu: 20m + memory: 40Mi + diff --git a/addons/grafana/deployment.yaml b/addons/grafana/deployment.yaml index 09b057854..bb04e6019 100644 --- a/addons/grafana/deployment.yaml +++ b/addons/grafana/deployment.yaml @@ -32,11 +32,11 @@ spec: containerPort: 8080 resources: requests: - memory: 100Mi cpu: 100m + memory: 100Mi limits: - memory: 200Mi cpu: 200m + memory: 200Mi volumeMounts: - name: config mountPath: /etc/grafana diff --git a/addons/prometheus/deployment.yaml b/addons/prometheus/deployment.yaml index 5d62fd4a6..5088f2356 100644 --- a/addons/prometheus/deployment.yaml +++ b/addons/prometheus/deployment.yaml @@ -28,6 +28,10 @@ spec: ports: - name: web containerPort: 9090 + resources: + requests: + cpu: 100m + memory: 200Mi volumeMounts: - name: config mountPath: /etc/prometheus From 619a0370dca0dccc140ea2d6bf2cc52e684369d2 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Thu, 21 Mar 2019 23:41:25 -0700 Subject: [PATCH 056/523] Update Grafana from v6.0.1 to v6.0.2 * https://github.com/grafana/grafana/releases/tag/v6.0.2 --- CHANGES.md | 2 +- addons/grafana/deployment.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index d7676bd5f..35cf369c4 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -30,7 +30,7 @@ Notable changes between versions. * Update Prometheus from v2.7.1 to [v2.8.0](https://github.com/prometheus/prometheus/releases/tag/v2.8.0) * Refresh rules based on upstreams ([#426](https://github.com/poseidon/typhoon/pull/426)) -* Update Grafana from v6.0.0 to v6.0.1 +* Update Grafana from v6.0.0 to v6.0.2 * Refresh dashboards and organize to stay below ConfigMap size limit ([#426](https://github.com/poseidon/typhoon/pull/426)) * Remove heapster manifests from addons ([#427](https://github.com/poseidon/typhoon/pull/427)) * Heapster addon powers `kubectl top` (in early Kubernetes, running the addon was expected). Today, there are better monitoring options. diff --git a/addons/grafana/deployment.yaml b/addons/grafana/deployment.yaml index bb04e6019..31dc662b9 100644 --- a/addons/grafana/deployment.yaml +++ b/addons/grafana/deployment.yaml @@ -23,7 +23,7 @@ spec: spec: containers: - name: grafana - image: grafana/grafana:6.0.1 + image: grafana/grafana:6.0.2 env: - name: GF_PATHS_CONFIG value: "/etc/grafana/custom.ini" From 36e31fc9fa28f482604fc449380c1c822c46dd15 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Sat, 23 Mar 2019 17:54:16 -0700 Subject: [PATCH 057/523] Add liveness and readiness probes to Grafana * https://github.com/grafana/grafana/issues/3302 --- CHANGES.md | 1 + addons/grafana/deployment.yaml | 10 ++++++++++ addons/prometheus/deployment.yaml | 8 ++++---- 3 files changed, 15 insertions(+), 4 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 35cf369c4..38fdecee8 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -31,6 +31,7 @@ Notable changes between versions. * Update Prometheus from v2.7.1 to [v2.8.0](https://github.com/prometheus/prometheus/releases/tag/v2.8.0) * Refresh rules based on upstreams ([#426](https://github.com/poseidon/typhoon/pull/426)) * Update Grafana from v6.0.0 to v6.0.2 + * Add liveness and readiness probes * Refresh dashboards and organize to stay below ConfigMap size limit ([#426](https://github.com/poseidon/typhoon/pull/426)) * Remove heapster manifests from addons ([#427](https://github.com/poseidon/typhoon/pull/427)) * Heapster addon powers `kubectl top` (in early Kubernetes, running the addon was expected). Today, there are better monitoring options. diff --git a/addons/grafana/deployment.yaml b/addons/grafana/deployment.yaml index 31dc662b9..a72972b1a 100644 --- a/addons/grafana/deployment.yaml +++ b/addons/grafana/deployment.yaml @@ -30,6 +30,16 @@ spec: ports: - name: http containerPort: 8080 + livenessProbe: + httpGet: + path: /metrics + port: 8080 + initialDelaySeconds: 10 + readinessProbe: + httpGet: + path: /api/health + port: 8080 + initialDelaySeconds: 10 resources: requests: cpu: 100m diff --git a/addons/prometheus/deployment.yaml b/addons/prometheus/deployment.yaml index 5088f2356..ccb727bf3 100644 --- a/addons/prometheus/deployment.yaml +++ b/addons/prometheus/deployment.yaml @@ -39,15 +39,15 @@ spec: mountPath: /etc/prometheus/rules - name: data mountPath: /var/lib/prometheus - readinessProbe: + livenessProbe: httpGet: - path: /-/ready + path: /-/healthy port: 9090 initialDelaySeconds: 10 timeoutSeconds: 10 - livenessProbe: + readinessProbe: httpGet: - path: /-/healthy + path: /-/ready port: 9090 initialDelaySeconds: 10 timeoutSeconds: 10 From 41a9d86bc37a8d757935b78b6b821ee7c958561e Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Sat, 23 Mar 2019 17:59:47 -0700 Subject: [PATCH 058/523] Add NetworkPolicy to limit traffic into Prometheus * Allow traffic from Grafana to Prometheus in monitoring * Allow traffic from Prometheus to Prometheus in monitoring * NetworkPolicy denies non-whitelisted traffic. Define policy to allow other access --- CHANGES.md | 1 + addons/prometheus/network-policy.yaml | 28 +++++++++++++++++++++++++++ 2 files changed, 29 insertions(+) create mode 100644 addons/prometheus/network-policy.yaml diff --git a/CHANGES.md b/CHANGES.md index 38fdecee8..ccc5a758f 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -30,6 +30,7 @@ Notable changes between versions. * Update Prometheus from v2.7.1 to [v2.8.0](https://github.com/prometheus/prometheus/releases/tag/v2.8.0) * Refresh rules based on upstreams ([#426](https://github.com/poseidon/typhoon/pull/426)) + * Define NetworkPolicy to allow only traffic from the Grafana addon * Update Grafana from v6.0.0 to v6.0.2 * Add liveness and readiness probes * Refresh dashboards and organize to stay below ConfigMap size limit ([#426](https://github.com/poseidon/typhoon/pull/426)) diff --git a/addons/prometheus/network-policy.yaml b/addons/prometheus/network-policy.yaml new file mode 100644 index 000000000..e64cdb546 --- /dev/null +++ b/addons/prometheus/network-policy.yaml @@ -0,0 +1,28 @@ +# Allow Grafana access and in-cluster Prometheus scraping +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: prometheus + namespace: monitoring +spec: + podSelector: + matchLabels: + name: prometheus + ingress: + - ports: + - protocol: TCP + port: 9090 + from: + - namespaceSelector: + matchLabels: + name: monitoring + podSelector: + matchLabels: + name: grafana + - namespaceSelector: + matchLabels: + name: monitoring + podSelector: + matchLabels: + name: prometheus + From 4fea526ebf9a1051652914015a8aac77dc73b246 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Mon, 25 Mar 2019 21:43:47 -0700 Subject: [PATCH 059/523] Update Kubernetes from v1.13.4 to v1.13.5 * https://github.com/kubernetes/kubernetes/blob/master/CHANGELOG-1.13.md#v1135 --- CHANGES.md | 4 ++++ README.md | 10 +++++----- aws/container-linux/kubernetes/README.md | 2 +- aws/container-linux/kubernetes/bootkube.tf | 2 +- .../kubernetes/cl/controller.yaml.tmpl | 2 +- .../kubernetes/workers/cl/worker.yaml.tmpl | 4 ++-- aws/fedora-atomic/kubernetes/README.md | 2 +- aws/fedora-atomic/kubernetes/bootkube.tf | 2 +- .../kubernetes/cloudinit/controller.yaml.tmpl | 2 +- .../workers/cloudinit/worker.yaml.tmpl | 2 +- azure/container-linux/kubernetes/README.md | 2 +- azure/container-linux/kubernetes/bootkube.tf | 2 +- .../kubernetes/cl/controller.yaml.tmpl | 2 +- .../kubernetes/workers/cl/worker.yaml.tmpl | 4 ++-- bare-metal/container-linux/kubernetes/README.md | 2 +- .../container-linux/kubernetes/bootkube.tf | 2 +- .../kubernetes/cl/controller.yaml.tmpl | 2 +- .../kubernetes/cl/worker.yaml.tmpl | 2 +- bare-metal/fedora-atomic/kubernetes/README.md | 2 +- bare-metal/fedora-atomic/kubernetes/bootkube.tf | 2 +- .../kubernetes/cloudinit/controller.yaml.tmpl | 2 +- .../kubernetes/cloudinit/worker.yaml.tmpl | 2 +- .../container-linux/kubernetes/README.md | 2 +- .../container-linux/kubernetes/bootkube.tf | 2 +- .../kubernetes/cl/controller.yaml.tmpl | 2 +- .../kubernetes/cl/worker.yaml.tmpl | 4 ++-- digital-ocean/fedora-atomic/kubernetes/README.md | 2 +- .../fedora-atomic/kubernetes/bootkube.tf | 2 +- .../kubernetes/cloudinit/controller.yaml.tmpl | 2 +- .../kubernetes/cloudinit/worker.yaml.tmpl | 2 +- docs/advanced/worker-pools.md | 16 ++++++++-------- docs/atomic/aws.md | 10 +++++----- docs/atomic/bare-metal.md | 10 +++++----- docs/atomic/digital-ocean.md | 10 +++++----- docs/atomic/google-cloud.md | 10 +++++----- docs/cl/aws.md | 10 +++++----- docs/cl/azure.md | 10 +++++----- docs/cl/bare-metal.md | 14 +++++++------- docs/cl/digital-ocean.md | 10 +++++----- docs/cl/google-cloud.md | 10 +++++----- docs/index.md | 10 +++++----- docs/topics/maintenance.md | 2 +- .../container-linux/kubernetes/README.md | 2 +- .../container-linux/kubernetes/bootkube.tf | 2 +- .../kubernetes/cl/controller.yaml.tmpl | 2 +- .../kubernetes/workers/cl/worker.yaml.tmpl | 4 ++-- google-cloud/fedora-atomic/kubernetes/README.md | 2 +- .../fedora-atomic/kubernetes/bootkube.tf | 2 +- .../kubernetes/cloudinit/controller.yaml.tmpl | 2 +- .../workers/cloudinit/worker.yaml.tmpl | 2 +- 50 files changed, 110 insertions(+), 106 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index ccc5a758f..ac50d7cc7 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -4,6 +4,10 @@ Notable changes between versions. ## Latest + +## v1.13.5 + +* Kubernetes [v1.13.5](https://github.com/kubernetes/kubernetes/blob/master/CHANGELOG-1.13.md#v1135) * Resolve in-addr.arpa reverse DNS lookups (PTR) for pod IPv4 addresses ([#415](https://github.com/poseidon/typhoon/pull/415)) * Reverse DNS lookups for service IPv4 addresses unchanged * Upgrade Calico from v3.5.2 to [v3.6.0](https://docs.projectcalico.org/v3.6/release-notes/) ([#430](https://github.com/poseidon/typhoon/pull/430)) diff --git a/README.md b/README.md index ecec6b4aa..b55ca38f7 100644 --- a/README.md +++ b/README.md @@ -11,7 +11,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features -* Kubernetes v1.13.4 (upstream, via [kubernetes-incubator/bootkube](https://github.com/kubernetes-incubator/bootkube)) +* Kubernetes v1.13.5 (upstream, via [kubernetes-incubator/bootkube](https://github.com/kubernetes-incubator/bootkube)) * Single or multi-master, [Calico](https://www.projectcalico.org/) or [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/) * Advanced features like [worker pools](https://typhoon.psdn.io/advanced/worker-pools/), [preemptible](https://typhoon.psdn.io/cl/google-cloud/#preemption) workers, and [snippets](https://typhoon.psdn.io/advanced/customization/#container-linux) customization @@ -50,7 +50,7 @@ Define a Kubernetes cluster by using the Terraform module for your chosen platfo ```tf module "google-cloud-yavin" { - source = "git::https://github.com/poseidon/typhoon//google-cloud/container-linux/kubernetes?ref=v1.13.4" + source = "git::https://github.com/poseidon/typhoon//google-cloud/container-linux/kubernetes?ref=v1.13.5" providers = { google = "google.default" @@ -91,9 +91,9 @@ In 4-8 minutes (varies by platform), the cluster will be ready. This Google Clou $ export KUBECONFIG=/home/user/.secrets/clusters/yavin/auth/kubeconfig $ kubectl get nodes NAME ROLES STATUS AGE VERSION -yavin-controller-0.c.example-com.internal controller,master Ready 6m v1.13.4 -yavin-worker-jrbf.c.example-com.internal node Ready 5m v1.13.4 -yavin-worker-mzdm.c.example-com.internal node Ready 5m v1.13.4 +yavin-controller-0.c.example-com.internal controller,master Ready 6m v1.13.5 +yavin-worker-jrbf.c.example-com.internal node Ready 5m v1.13.5 +yavin-worker-mzdm.c.example-com.internal node Ready 5m v1.13.5 ``` List the pods. diff --git a/aws/container-linux/kubernetes/README.md b/aws/container-linux/kubernetes/README.md index 909cc5959..5eeef2945 100644 --- a/aws/container-linux/kubernetes/README.md +++ b/aws/container-linux/kubernetes/README.md @@ -11,7 +11,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features -* Kubernetes v1.13.4 (upstream, via [kubernetes-incubator/bootkube](https://github.com/kubernetes-incubator/bootkube)) +* Kubernetes v1.13.5 (upstream, via [kubernetes-incubator/bootkube](https://github.com/kubernetes-incubator/bootkube)) * Single or multi-master, [Calico](https://www.projectcalico.org/) or [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/) * Advanced features like [worker pools](https://typhoon.psdn.io/advanced/worker-pools/), [spot](https://typhoon.psdn.io/cl/aws/#spot) workers, and [snippets](https://typhoon.psdn.io/advanced/customization/#container-linux) customization diff --git a/aws/container-linux/kubernetes/bootkube.tf b/aws/container-linux/kubernetes/bootkube.tf index aaa35f055..11f3f2f86 100644 --- a/aws/container-linux/kubernetes/bootkube.tf +++ b/aws/container-linux/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=23f81a5e8c4a975750beabc6f603a60a04621225" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=da0321287ba8cd48e4492894483fcdc1da057657" cluster_name = "${var.cluster_name}" api_servers = ["${format("%s.%s", var.cluster_name, var.dns_zone)}"] diff --git a/aws/container-linux/kubernetes/cl/controller.yaml.tmpl b/aws/container-linux/kubernetes/cl/controller.yaml.tmpl index f5d9472d3..7252aa80e 100644 --- a/aws/container-linux/kubernetes/cl/controller.yaml.tmpl +++ b/aws/container-linux/kubernetes/cl/controller.yaml.tmpl @@ -123,7 +123,7 @@ storage: contents: inline: | KUBELET_IMAGE_URL=docker://k8s.gcr.io/hyperkube - KUBELET_IMAGE_TAG=v1.13.4 + KUBELET_IMAGE_TAG=v1.13.5 - path: /etc/sysctl.d/max-user-watches.conf filesystem: root contents: diff --git a/aws/container-linux/kubernetes/workers/cl/worker.yaml.tmpl b/aws/container-linux/kubernetes/workers/cl/worker.yaml.tmpl index 79f1bf39e..9cc13a6e9 100644 --- a/aws/container-linux/kubernetes/workers/cl/worker.yaml.tmpl +++ b/aws/container-linux/kubernetes/workers/cl/worker.yaml.tmpl @@ -93,7 +93,7 @@ storage: contents: inline: | KUBELET_IMAGE_URL=docker://k8s.gcr.io/hyperkube - KUBELET_IMAGE_TAG=v1.13.4 + KUBELET_IMAGE_TAG=v1.13.5 - path: /etc/sysctl.d/max-user-watches.conf filesystem: root contents: @@ -111,7 +111,7 @@ storage: --volume config,kind=host,source=/etc/kubernetes \ --mount volume=config,target=/etc/kubernetes \ --insecure-options=image \ - docker://k8s.gcr.io/hyperkube:v1.13.4 \ + docker://k8s.gcr.io/hyperkube:v1.13.5 \ --net=host \ --dns=host \ --exec=/kubectl -- --kubeconfig=/etc/kubernetes/kubeconfig delete node $(hostname) diff --git a/aws/fedora-atomic/kubernetes/README.md b/aws/fedora-atomic/kubernetes/README.md index ec0102f49..4069a93c8 100644 --- a/aws/fedora-atomic/kubernetes/README.md +++ b/aws/fedora-atomic/kubernetes/README.md @@ -11,7 +11,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features -* Kubernetes v1.13.4 (upstream, via [kubernetes-incubator/bootkube](https://github.com/kubernetes-incubator/bootkube)) +* Kubernetes v1.13.5 (upstream, via [kubernetes-incubator/bootkube](https://github.com/kubernetes-incubator/bootkube)) * Single or multi-master, [Calico](https://www.projectcalico.org/) or [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/) * Advanced features like [worker pools](https://typhoon.psdn.io/advanced/worker-pools/) and [spot](https://typhoon.psdn.io/cl/aws/#spot) workers diff --git a/aws/fedora-atomic/kubernetes/bootkube.tf b/aws/fedora-atomic/kubernetes/bootkube.tf index 4c45f2e1c..371140ce1 100644 --- a/aws/fedora-atomic/kubernetes/bootkube.tf +++ b/aws/fedora-atomic/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=23f81a5e8c4a975750beabc6f603a60a04621225" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=da0321287ba8cd48e4492894483fcdc1da057657" cluster_name = "${var.cluster_name}" api_servers = ["${format("%s.%s", var.cluster_name, var.dns_zone)}"] diff --git a/aws/fedora-atomic/kubernetes/cloudinit/controller.yaml.tmpl b/aws/fedora-atomic/kubernetes/cloudinit/controller.yaml.tmpl index 81f8b1784..8afdfc050 100644 --- a/aws/fedora-atomic/kubernetes/cloudinit/controller.yaml.tmpl +++ b/aws/fedora-atomic/kubernetes/cloudinit/controller.yaml.tmpl @@ -79,7 +79,7 @@ runcmd: - [systemctl, daemon-reload] - [systemctl, restart, NetworkManager] - "atomic install --system --name=etcd quay.io/poseidon/etcd:v3.3.12" - - "atomic install --system --name=kubelet quay.io/poseidon/kubelet:v1.13.4" + - "atomic install --system --name=kubelet quay.io/poseidon/kubelet:v1.13.5" - "atomic install --system --name=bootkube quay.io/poseidon/bootkube:v0.14.0" - [systemctl, start, --no-block, etcd.service] - [systemctl, start, --no-block, kubelet.service] diff --git a/aws/fedora-atomic/kubernetes/workers/cloudinit/worker.yaml.tmpl b/aws/fedora-atomic/kubernetes/workers/cloudinit/worker.yaml.tmpl index da76beae5..a05cd877c 100644 --- a/aws/fedora-atomic/kubernetes/workers/cloudinit/worker.yaml.tmpl +++ b/aws/fedora-atomic/kubernetes/workers/cloudinit/worker.yaml.tmpl @@ -54,7 +54,7 @@ bootcmd: runcmd: - [systemctl, daemon-reload] - [systemctl, restart, NetworkManager] - - "atomic install --system --name=kubelet quay.io/poseidon/kubelet:v1.13.4" + - "atomic install --system --name=kubelet quay.io/poseidon/kubelet:v1.13.5" - [systemctl, start, --no-block, kubelet.service] users: - default diff --git a/azure/container-linux/kubernetes/README.md b/azure/container-linux/kubernetes/README.md index 098f5c035..a8aeacbbd 100644 --- a/azure/container-linux/kubernetes/README.md +++ b/azure/container-linux/kubernetes/README.md @@ -11,7 +11,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features -* Kubernetes v1.13.4 (upstream, via [kubernetes-incubator/bootkube](https://github.com/kubernetes-incubator/bootkube)) +* Kubernetes v1.13.5 (upstream, via [kubernetes-incubator/bootkube](https://github.com/kubernetes-incubator/bootkube)) * Single or multi-master, [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled * Advanced features like [worker pools](https://typhoon.psdn.io/advanced/worker-pools/), [low-priority](https://typhoon.psdn.io/cl/azure/#low-priority) workers, and [snippets](https://typhoon.psdn.io/advanced/customization/#container-linux) customization diff --git a/azure/container-linux/kubernetes/bootkube.tf b/azure/container-linux/kubernetes/bootkube.tf index 07d12ca8d..58290863a 100644 --- a/azure/container-linux/kubernetes/bootkube.tf +++ b/azure/container-linux/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=23f81a5e8c4a975750beabc6f603a60a04621225" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=da0321287ba8cd48e4492894483fcdc1da057657" cluster_name = "${var.cluster_name}" api_servers = ["${format("%s.%s", var.cluster_name, var.dns_zone)}"] diff --git a/azure/container-linux/kubernetes/cl/controller.yaml.tmpl b/azure/container-linux/kubernetes/cl/controller.yaml.tmpl index f5d9472d3..7252aa80e 100644 --- a/azure/container-linux/kubernetes/cl/controller.yaml.tmpl +++ b/azure/container-linux/kubernetes/cl/controller.yaml.tmpl @@ -123,7 +123,7 @@ storage: contents: inline: | KUBELET_IMAGE_URL=docker://k8s.gcr.io/hyperkube - KUBELET_IMAGE_TAG=v1.13.4 + KUBELET_IMAGE_TAG=v1.13.5 - path: /etc/sysctl.d/max-user-watches.conf filesystem: root contents: diff --git a/azure/container-linux/kubernetes/workers/cl/worker.yaml.tmpl b/azure/container-linux/kubernetes/workers/cl/worker.yaml.tmpl index 3312b7e11..c5211f278 100644 --- a/azure/container-linux/kubernetes/workers/cl/worker.yaml.tmpl +++ b/azure/container-linux/kubernetes/workers/cl/worker.yaml.tmpl @@ -93,7 +93,7 @@ storage: contents: inline: | KUBELET_IMAGE_URL=docker://k8s.gcr.io/hyperkube - KUBELET_IMAGE_TAG=v1.13.4 + KUBELET_IMAGE_TAG=v1.13.5 - path: /etc/sysctl.d/max-user-watches.conf filesystem: root contents: @@ -111,7 +111,7 @@ storage: --volume config,kind=host,source=/etc/kubernetes \ --mount volume=config,target=/etc/kubernetes \ --insecure-options=image \ - docker://k8s.gcr.io/hyperkube:v1.13.4 \ + docker://k8s.gcr.io/hyperkube:v1.13.5 \ --net=host \ --dns=host \ --exec=/kubectl -- --kubeconfig=/etc/kubernetes/kubeconfig delete node $(hostname | tr '[:upper:]' '[:lower:]') diff --git a/bare-metal/container-linux/kubernetes/README.md b/bare-metal/container-linux/kubernetes/README.md index 2bfe8c031..7ff0ca7de 100644 --- a/bare-metal/container-linux/kubernetes/README.md +++ b/bare-metal/container-linux/kubernetes/README.md @@ -11,7 +11,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features -* Kubernetes v1.13.4 (upstream, via [kubernetes-incubator/bootkube](https://github.com/kubernetes-incubator/bootkube)) +* Kubernetes v1.13.5 (upstream, via [kubernetes-incubator/bootkube](https://github.com/kubernetes-incubator/bootkube)) * Single or multi-master, [Calico](https://www.projectcalico.org/) or [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/) * Advanced features like [snippets](https://typhoon.psdn.io/advanced/customization/#container-linux) customization diff --git a/bare-metal/container-linux/kubernetes/bootkube.tf b/bare-metal/container-linux/kubernetes/bootkube.tf index aceff83ab..f43bf8b30 100644 --- a/bare-metal/container-linux/kubernetes/bootkube.tf +++ b/bare-metal/container-linux/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=23f81a5e8c4a975750beabc6f603a60a04621225" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=da0321287ba8cd48e4492894483fcdc1da057657" cluster_name = "${var.cluster_name}" api_servers = ["${var.k8s_domain_name}"] diff --git a/bare-metal/container-linux/kubernetes/cl/controller.yaml.tmpl b/bare-metal/container-linux/kubernetes/cl/controller.yaml.tmpl index 00a27d380..7863f04a4 100644 --- a/bare-metal/container-linux/kubernetes/cl/controller.yaml.tmpl +++ b/bare-metal/container-linux/kubernetes/cl/controller.yaml.tmpl @@ -128,7 +128,7 @@ storage: contents: inline: | KUBELET_IMAGE_URL=docker://k8s.gcr.io/hyperkube - KUBELET_IMAGE_TAG=v1.13.4 + KUBELET_IMAGE_TAG=v1.13.5 - path: /etc/hostname filesystem: root mode: 0644 diff --git a/bare-metal/container-linux/kubernetes/cl/worker.yaml.tmpl b/bare-metal/container-linux/kubernetes/cl/worker.yaml.tmpl index 0e293a7a6..5e5e5bcec 100644 --- a/bare-metal/container-linux/kubernetes/cl/worker.yaml.tmpl +++ b/bare-metal/container-linux/kubernetes/cl/worker.yaml.tmpl @@ -89,7 +89,7 @@ storage: contents: inline: | KUBELET_IMAGE_URL=docker://k8s.gcr.io/hyperkube - KUBELET_IMAGE_TAG=v1.13.4 + KUBELET_IMAGE_TAG=v1.13.5 - path: /etc/hostname filesystem: root mode: 0644 diff --git a/bare-metal/fedora-atomic/kubernetes/README.md b/bare-metal/fedora-atomic/kubernetes/README.md index cb1fb55bc..65aac5d10 100644 --- a/bare-metal/fedora-atomic/kubernetes/README.md +++ b/bare-metal/fedora-atomic/kubernetes/README.md @@ -11,7 +11,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features -* Kubernetes v1.13.4 (upstream, via [kubernetes-incubator/bootkube](https://github.com/kubernetes-incubator/bootkube)) +* Kubernetes v1.13.5 (upstream, via [kubernetes-incubator/bootkube](https://github.com/kubernetes-incubator/bootkube)) * Single or multi-master, [Calico](https://www.projectcalico.org/) or [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/) * Ready for Ingress, Prometheus, Grafana, and other optional [addons](https://typhoon.psdn.io/addons/overview/) diff --git a/bare-metal/fedora-atomic/kubernetes/bootkube.tf b/bare-metal/fedora-atomic/kubernetes/bootkube.tf index 0d854f794..c07e687d6 100644 --- a/bare-metal/fedora-atomic/kubernetes/bootkube.tf +++ b/bare-metal/fedora-atomic/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=23f81a5e8c4a975750beabc6f603a60a04621225" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=da0321287ba8cd48e4492894483fcdc1da057657" cluster_name = "${var.cluster_name}" api_servers = ["${var.k8s_domain_name}"] diff --git a/bare-metal/fedora-atomic/kubernetes/cloudinit/controller.yaml.tmpl b/bare-metal/fedora-atomic/kubernetes/cloudinit/controller.yaml.tmpl index 66600c7f8..1e23ec0d9 100644 --- a/bare-metal/fedora-atomic/kubernetes/cloudinit/controller.yaml.tmpl +++ b/bare-metal/fedora-atomic/kubernetes/cloudinit/controller.yaml.tmpl @@ -85,7 +85,7 @@ runcmd: - [systemctl, restart, NetworkManager] - [hostnamectl, set-hostname, ${domain_name}] - "atomic install --system --name=etcd quay.io/poseidon/etcd:v3.3.12" - - "atomic install --system --name=kubelet quay.io/poseidon/kubelet:v1.13.4" + - "atomic install --system --name=kubelet quay.io/poseidon/kubelet:v1.13.5" - "atomic install --system --name=bootkube quay.io/poseidon/bootkube:v0.14.0" - [systemctl, start, --no-block, etcd.service] - [systemctl, enable, kubelet.path] diff --git a/bare-metal/fedora-atomic/kubernetes/cloudinit/worker.yaml.tmpl b/bare-metal/fedora-atomic/kubernetes/cloudinit/worker.yaml.tmpl index 5e9d5c50e..bae7973f0 100644 --- a/bare-metal/fedora-atomic/kubernetes/cloudinit/worker.yaml.tmpl +++ b/bare-metal/fedora-atomic/kubernetes/cloudinit/worker.yaml.tmpl @@ -60,7 +60,7 @@ runcmd: - [systemctl, daemon-reload] - [systemctl, restart, NetworkManager] - [hostnamectl, set-hostname, ${domain_name}] - - "atomic install --system --name=kubelet quay.io/poseidon/kubelet:v1.13.4" + - "atomic install --system --name=kubelet quay.io/poseidon/kubelet:v1.13.5" - [systemctl, enable, kubelet.path] - [systemctl, start, --no-block, kubelet.path] users: diff --git a/digital-ocean/container-linux/kubernetes/README.md b/digital-ocean/container-linux/kubernetes/README.md index 837e27714..2f8b0a4fd 100644 --- a/digital-ocean/container-linux/kubernetes/README.md +++ b/digital-ocean/container-linux/kubernetes/README.md @@ -11,7 +11,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features -* Kubernetes v1.13.4 (upstream, via [kubernetes-incubator/bootkube](https://github.com/kubernetes-incubator/bootkube)) +* Kubernetes v1.13.5 (upstream, via [kubernetes-incubator/bootkube](https://github.com/kubernetes-incubator/bootkube)) * Single or multi-master, [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled * Advanced features like [snippets](https://typhoon.psdn.io/advanced/customization/#container-linux) customization diff --git a/digital-ocean/container-linux/kubernetes/bootkube.tf b/digital-ocean/container-linux/kubernetes/bootkube.tf index 5f679cfc9..b33120245 100644 --- a/digital-ocean/container-linux/kubernetes/bootkube.tf +++ b/digital-ocean/container-linux/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=23f81a5e8c4a975750beabc6f603a60a04621225" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=da0321287ba8cd48e4492894483fcdc1da057657" cluster_name = "${var.cluster_name}" api_servers = ["${format("%s.%s", var.cluster_name, var.dns_zone)}"] diff --git a/digital-ocean/container-linux/kubernetes/cl/controller.yaml.tmpl b/digital-ocean/container-linux/kubernetes/cl/controller.yaml.tmpl index 01fca5df3..95a34b9aa 100644 --- a/digital-ocean/container-linux/kubernetes/cl/controller.yaml.tmpl +++ b/digital-ocean/container-linux/kubernetes/cl/controller.yaml.tmpl @@ -129,7 +129,7 @@ storage: contents: inline: | KUBELET_IMAGE_URL=docker://k8s.gcr.io/hyperkube - KUBELET_IMAGE_TAG=v1.13.4 + KUBELET_IMAGE_TAG=v1.13.5 - path: /etc/sysctl.d/max-user-watches.conf filesystem: root contents: diff --git a/digital-ocean/container-linux/kubernetes/cl/worker.yaml.tmpl b/digital-ocean/container-linux/kubernetes/cl/worker.yaml.tmpl index 256c864fc..2c72c238a 100644 --- a/digital-ocean/container-linux/kubernetes/cl/worker.yaml.tmpl +++ b/digital-ocean/container-linux/kubernetes/cl/worker.yaml.tmpl @@ -99,7 +99,7 @@ storage: contents: inline: | KUBELET_IMAGE_URL=docker://k8s.gcr.io/hyperkube - KUBELET_IMAGE_TAG=v1.13.4 + KUBELET_IMAGE_TAG=v1.13.5 - path: /etc/sysctl.d/max-user-watches.conf filesystem: root contents: @@ -117,7 +117,7 @@ storage: --volume config,kind=host,source=/etc/kubernetes \ --mount volume=config,target=/etc/kubernetes \ --insecure-options=image \ - docker://k8s.gcr.io/hyperkube:v1.13.4 \ + docker://k8s.gcr.io/hyperkube:v1.13.5 \ --net=host \ --dns=host \ --exec=/kubectl -- --kubeconfig=/etc/kubernetes/kubeconfig delete node $(hostname) diff --git a/digital-ocean/fedora-atomic/kubernetes/README.md b/digital-ocean/fedora-atomic/kubernetes/README.md index c02cb1bcc..48a6c57d3 100644 --- a/digital-ocean/fedora-atomic/kubernetes/README.md +++ b/digital-ocean/fedora-atomic/kubernetes/README.md @@ -11,7 +11,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features -* Kubernetes v1.13.4 (upstream, via [kubernetes-incubator/bootkube](https://github.com/kubernetes-incubator/bootkube)) +* Kubernetes v1.13.5 (upstream, via [kubernetes-incubator/bootkube](https://github.com/kubernetes-incubator/bootkube)) * Single or multi-master, [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled * Ready for Ingress, Prometheus, Grafana, and other optional [addons](https://typhoon.psdn.io/addons/overview/) diff --git a/digital-ocean/fedora-atomic/kubernetes/bootkube.tf b/digital-ocean/fedora-atomic/kubernetes/bootkube.tf index f33c4a993..1a3b4b827 100644 --- a/digital-ocean/fedora-atomic/kubernetes/bootkube.tf +++ b/digital-ocean/fedora-atomic/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=23f81a5e8c4a975750beabc6f603a60a04621225" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=da0321287ba8cd48e4492894483fcdc1da057657" cluster_name = "${var.cluster_name}" api_servers = ["${format("%s.%s", var.cluster_name, var.dns_zone)}"] diff --git a/digital-ocean/fedora-atomic/kubernetes/cloudinit/controller.yaml.tmpl b/digital-ocean/fedora-atomic/kubernetes/cloudinit/controller.yaml.tmpl index 79747dcda..23770028a 100644 --- a/digital-ocean/fedora-atomic/kubernetes/cloudinit/controller.yaml.tmpl +++ b/digital-ocean/fedora-atomic/kubernetes/cloudinit/controller.yaml.tmpl @@ -91,7 +91,7 @@ bootcmd: runcmd: - [systemctl, daemon-reload] - "atomic install --system --name=etcd quay.io/poseidon/etcd:v3.3.12" - - "atomic install --system --name=kubelet quay.io/poseidon/kubelet:v1.13.4" + - "atomic install --system --name=kubelet quay.io/poseidon/kubelet:v1.13.5" - "atomic install --system --name=bootkube quay.io/poseidon/bootkube:v0.14.0" - [systemctl, start, --no-block, etcd.service] - [systemctl, enable, cloud-metadata.service] diff --git a/digital-ocean/fedora-atomic/kubernetes/cloudinit/worker.yaml.tmpl b/digital-ocean/fedora-atomic/kubernetes/cloudinit/worker.yaml.tmpl index b87b6acc9..014af57f1 100644 --- a/digital-ocean/fedora-atomic/kubernetes/cloudinit/worker.yaml.tmpl +++ b/digital-ocean/fedora-atomic/kubernetes/cloudinit/worker.yaml.tmpl @@ -67,7 +67,7 @@ bootcmd: runcmd: - [systemctl, daemon-reload] - [systemctl, enable, cloud-metadata.service] - - "atomic install --system --name=kubelet quay.io/poseidon/kubelet:v1.13.4" + - "atomic install --system --name=kubelet quay.io/poseidon/kubelet:v1.13.5" - [systemctl, enable, kubelet.path] - [systemctl, start, --no-block, kubelet.path] users: diff --git a/docs/advanced/worker-pools.md b/docs/advanced/worker-pools.md index c17861d43..4569e0755 100644 --- a/docs/advanced/worker-pools.md +++ b/docs/advanced/worker-pools.md @@ -16,7 +16,7 @@ Create a cluster following the AWS [tutorial](../cl/aws.md#cluster). Define a wo ```tf module "tempest-worker-pool" { - source = "git::https://github.com/poseidon/typhoon//aws/container-linux/kubernetes/workers?ref=v1.13.4" + source = "git::https://github.com/poseidon/typhoon//aws/container-linux/kubernetes/workers?ref=v1.13.5" providers = { aws = "aws.default" @@ -82,7 +82,7 @@ Create a cluster following the Azure [tutorial](../cl/azure.md#cluster). Define ```tf module "ramius-worker-pool" { - source = "git::https://github.com/poseidon/typhoon//azure/container-linux/kubernetes/workers?ref=v1.13.4" + source = "git::https://github.com/poseidon/typhoon//azure/container-linux/kubernetes/workers?ref=v1.13.5" providers = { azurerm = "azurerm.default" @@ -152,7 +152,7 @@ Create a cluster following the Google Cloud [tutorial](../cl/google-cloud.md#clu ```tf module "yavin-worker-pool" { - source = "git::https://github.com/poseidon/typhoon//google-cloud/container-linux/kubernetes/workers?ref=v1.13.4" + source = "git::https://github.com/poseidon/typhoon//google-cloud/container-linux/kubernetes/workers?ref=v1.13.5" providers = { google = "google.default" @@ -187,11 +187,11 @@ Verify a managed instance group of workers joins the cluster within a few minute ``` $ kubectl get nodes NAME STATUS AGE VERSION -yavin-controller-0.c.example-com.internal Ready 6m v1.13.4 -yavin-worker-jrbf.c.example-com.internal Ready 5m v1.13.4 -yavin-worker-mzdm.c.example-com.internal Ready 5m v1.13.4 -yavin-16x-worker-jrbf.c.example-com.internal Ready 3m v1.13.4 -yavin-16x-worker-mzdm.c.example-com.internal Ready 3m v1.13.4 +yavin-controller-0.c.example-com.internal Ready 6m v1.13.5 +yavin-worker-jrbf.c.example-com.internal Ready 5m v1.13.5 +yavin-worker-mzdm.c.example-com.internal Ready 5m v1.13.5 +yavin-16x-worker-jrbf.c.example-com.internal Ready 3m v1.13.5 +yavin-16x-worker-mzdm.c.example-com.internal Ready 3m v1.13.5 ``` ### Variables diff --git a/docs/atomic/aws.md b/docs/atomic/aws.md index d66798c3c..97f1d8df0 100644 --- a/docs/atomic/aws.md +++ b/docs/atomic/aws.md @@ -3,7 +3,7 @@ !!! danger Typhoon for Fedora Atomic is alpha. Expect rough edges and changes. -In this tutorial, we'll create a Kubernetes v1.13.4 cluster on AWS with Fedora Atomic. +In this tutorial, we'll create a Kubernetes v1.13.5 cluster on AWS with Fedora Atomic. We'll declare a Kubernetes cluster using the Typhoon Terraform module. Then apply the changes to create a VPC, gateway, subnets, security groups, controller instances, worker auto-scaling group, network load balancer, and TLS assets. Instances are provisioned on first boot with cloud-init. @@ -83,7 +83,7 @@ Define a Kubernetes cluster using the module `aws/fedora-atomic/kubernetes`. ```tf module "aws-tempest" { - source = "git::https://github.com/poseidon/typhoon//aws/fedora-atomic/kubernetes?ref=v1.13.4" + source = "git::https://github.com/poseidon/typhoon//aws/fedora-atomic/kubernetes?ref=v1.13.5" providers = { aws = "aws.default" @@ -156,9 +156,9 @@ In 5-10 minutes, the Kubernetes cluster will be ready. $ export KUBECONFIG=/home/user/.secrets/clusters/tempest/auth/kubeconfig $ kubectl get nodes NAME STATUS ROLES AGE VERSION -ip-10-0-3-155 Ready controller,master 10m v1.13.4 -ip-10-0-26-65 Ready node 10m v1.13.4 -ip-10-0-41-21 Ready node 10m v1.13.4 +ip-10-0-3-155 Ready controller,master 10m v1.13.5 +ip-10-0-26-65 Ready node 10m v1.13.5 +ip-10-0-41-21 Ready node 10m v1.13.5 ``` List the pods. diff --git a/docs/atomic/bare-metal.md b/docs/atomic/bare-metal.md index 870b1b6ee..fd4fcd0e0 100644 --- a/docs/atomic/bare-metal.md +++ b/docs/atomic/bare-metal.md @@ -3,7 +3,7 @@ !!! danger Typhoon for Fedora Atomic is alpha. Expect rough edges and changes. -In this tutorial, we'll network boot and provision a Kubernetes v1.13.4 cluster on bare-metal with Fedora Atomic. +In this tutorial, we'll network boot and provision a Kubernetes v1.13.5 cluster on bare-metal with Fedora Atomic. First, we'll deploy a [Matchbox](https://github.com/coreos/matchbox) service and setup a network boot environment. Then, we'll declare a Kubernetes cluster using the Typhoon Terraform module and power on machines. On PXE boot, machines will install Fedora Atomic via kickstart, reboot into the disk install, and provision themselves as Kubernetes controllers or workers via cloud-init. @@ -228,7 +228,7 @@ Define a Kubernetes cluster using the module `bare-metal/fedora-atomic/kubernete ```tf module "bare-metal-mercury" { - source = "git::https://github.com/poseidon/typhoon//bare-metal/fedora-atomic/kubernetes?ref=v1.13.4" + source = "git::https://github.com/poseidon/typhoon//bare-metal/fedora-atomic/kubernetes?ref=v1.13.5" providers = { local = "local.default" @@ -354,9 +354,9 @@ bootkube[5]: Tearing down temporary bootstrap control plane... $ export KUBECONFIG=/home/user/.secrets/clusters/mercury/auth/kubeconfig $ kubectl get nodes NAME STATUS ROLES AGE VERSION -node1.example.com Ready controller,master 10m v1.13.4 -node2.example.com Ready node 10m v1.13.4 -node3.example.com Ready node 10m v1.13.4 +node1.example.com Ready controller,master 10m v1.13.5 +node2.example.com Ready node 10m v1.13.5 +node3.example.com Ready node 10m v1.13.5 ``` List the pods. diff --git a/docs/atomic/digital-ocean.md b/docs/atomic/digital-ocean.md index f302d1cf9..c504688e7 100644 --- a/docs/atomic/digital-ocean.md +++ b/docs/atomic/digital-ocean.md @@ -3,7 +3,7 @@ !!! danger Typhoon for Fedora Atomic is alpha. Expect rough edges and changes. -In this tutorial, we'll create a Kubernetes v1.13.4 cluster on DigitalOcean with Fedora Atomic. +In this tutorial, we'll create a Kubernetes v1.13.5 cluster on DigitalOcean with Fedora Atomic. We'll declare a Kubernetes cluster using the Typhoon Terraform module. Then apply the changes to create controller droplets, worker droplets, DNS records, tags, and TLS assets. Instances are provisioned on first boot with cloud-init. @@ -77,7 +77,7 @@ Define a Kubernetes cluster using the module `digital-ocean/fedora-atomic/kubern ```tf module "digital-ocean-nemo" { - source = "git::https://github.com/poseidon/typhoon//digital-ocean/fedora-atomic/kubernetes?ref=v1.13.4" + source = "git::https://github.com/poseidon/typhoon//digital-ocean/fedora-atomic/kubernetes?ref=v1.13.5" providers = { digitalocean = "digitalocean.default" @@ -152,9 +152,9 @@ In 3-6 minutes, the Kubernetes cluster will be ready. $ export KUBECONFIG=/home/user/.secrets/clusters/nemo/auth/kubeconfig $ kubectl get nodes NAME STATUS ROLES AGE VERSION -10.132.110.130 Ready controller,master 10m v1.13.4 -10.132.115.81 Ready node 10m v1.13.4 -10.132.124.107 Ready node 10m v1.13.4 +10.132.110.130 Ready controller,master 10m v1.13.5 +10.132.115.81 Ready node 10m v1.13.5 +10.132.124.107 Ready node 10m v1.13.5 ``` List the pods. diff --git a/docs/atomic/google-cloud.md b/docs/atomic/google-cloud.md index b94e9749f..8ef22066a 100644 --- a/docs/atomic/google-cloud.md +++ b/docs/atomic/google-cloud.md @@ -3,7 +3,7 @@ !!! danger Typhoon for Fedora Atomic is alpha. Fedora does not publish official images for Google Cloud so you must prepare them yourself. Expect rough edges and changes. -In this tutorial, we'll create a Kubernetes v1.13.4 cluster on Google Compute Engine with Fedora Atomic. +In this tutorial, we'll create a Kubernetes v1.13.5 cluster on Google Compute Engine with Fedora Atomic. We'll declare a Kubernetes cluster using the Typhoon Terraform module. Then apply the changes to create a network, firewall rules, health checks, controller instances, worker managed instance group, load balancers, and TLS assets. Instances are provisioned on first boot with cloud-init. @@ -121,7 +121,7 @@ Define a Kubernetes cluster using the module `google-cloud/fedora-atomic/kuberne ```tf module "google-cloud-yavin" { - source = "git::https://github.com/poseidon/typhoon//google-cloud/fedora-atomic/kubernetes?ref=v1.13.4" + source = "git::https://github.com/poseidon/typhoon//google-cloud/fedora-atomic/kubernetes?ref=v1.13.5" providers = { google = "google.default" @@ -197,9 +197,9 @@ In 5-10 minutes, the Kubernetes cluster will be ready. $ export KUBECONFIG=/home/user/.secrets/clusters/yavin/auth/kubeconfig $ kubectl get nodes NAME ROLES STATUS AGE VERSION -yavin-controller-0.c.example-com.internal controller,master Ready 6m v1.13.4 -yavin-worker-jrbf.c.example-com.internal node Ready 5m v1.13.4 -yavin-worker-mzdm.c.example-com.internal node Ready 5m v1.13.4 +yavin-controller-0.c.example-com.internal controller,master Ready 6m v1.13.5 +yavin-worker-jrbf.c.example-com.internal node Ready 5m v1.13.5 +yavin-worker-mzdm.c.example-com.internal node Ready 5m v1.13.5 ``` List the pods. diff --git a/docs/cl/aws.md b/docs/cl/aws.md index b53513dc0..c43c1cecb 100644 --- a/docs/cl/aws.md +++ b/docs/cl/aws.md @@ -1,6 +1,6 @@ # AWS -In this tutorial, we'll create a Kubernetes v1.13.4 cluster on AWS with Container Linux. +In this tutorial, we'll create a Kubernetes v1.13.5 cluster on AWS with Container Linux. We'll declare a Kubernetes cluster using the Typhoon Terraform module. Then apply the changes to create a VPC, gateway, subnets, security groups, controller instances, worker auto-scaling group, network load balancer, and TLS assets. @@ -92,7 +92,7 @@ Define a Kubernetes cluster using the module `aws/container-linux/kubernetes`. ```tf module "aws-tempest" { - source = "git::https://github.com/poseidon/typhoon//aws/container-linux/kubernetes?ref=v1.13.4" + source = "git::https://github.com/poseidon/typhoon//aws/container-linux/kubernetes?ref=v1.13.5" providers = { aws = "aws.default" @@ -165,9 +165,9 @@ In 4-8 minutes, the Kubernetes cluster will be ready. $ export KUBECONFIG=/home/user/.secrets/clusters/tempest/auth/kubeconfig $ kubectl get nodes NAME STATUS ROLES AGE VERSION -ip-10-0-3-155 Ready controller,master 10m v1.13.4 -ip-10-0-26-65 Ready node 10m v1.13.4 -ip-10-0-41-21 Ready node 10m v1.13.4 +ip-10-0-3-155 Ready controller,master 10m v1.13.5 +ip-10-0-26-65 Ready node 10m v1.13.5 +ip-10-0-41-21 Ready node 10m v1.13.5 ``` List the pods. diff --git a/docs/cl/azure.md b/docs/cl/azure.md index d888628c2..2804c3687 100644 --- a/docs/cl/azure.md +++ b/docs/cl/azure.md @@ -3,7 +3,7 @@ !!! danger Typhoon for Azure is alpha. For production, use AWS, Google Cloud, or bare-metal. As Azure matures, check [errata](https://github.com/poseidon/typhoon/wiki/Errata) for known shortcomings. -In this tutorial, we'll create a Kubernetes v1.13.4 cluster on Azure with Container Linux. +In this tutorial, we'll create a Kubernetes v1.13.5 cluster on Azure with Container Linux. We'll declare a Kubernetes cluster using the Typhoon Terraform module. Then apply the changes to create a resource group, virtual network, subnets, security groups, controller availability set, worker scale set, load balancer, and TLS assets. @@ -87,7 +87,7 @@ Define a Kubernetes cluster using the module `azure/container-linux/kubernetes`. ```tf module "azure-ramius" { - source = "git::https://github.com/poseidon/typhoon//azure/container-linux/kubernetes?ref=v1.13.4" + source = "git::https://github.com/poseidon/typhoon//azure/container-linux/kubernetes?ref=v1.13.5" providers = { azurerm = "azurerm.default" @@ -161,9 +161,9 @@ In 4-8 minutes, the Kubernetes cluster will be ready. $ export KUBECONFIG=/home/user/.secrets/clusters/ramius/auth/kubeconfig $ kubectl get nodes NAME STATUS ROLES AGE VERSION -ramius-controller-0 Ready controller,master 24m v1.13.4 -ramius-worker-000001 Ready node 25m v1.13.4 -ramius-worker-000002 Ready node 24m v1.13.4 +ramius-controller-0 Ready controller,master 24m v1.13.5 +ramius-worker-000001 Ready node 25m v1.13.5 +ramius-worker-000002 Ready node 24m v1.13.5 ``` List the pods. diff --git a/docs/cl/bare-metal.md b/docs/cl/bare-metal.md index a6d11b3a7..9e7fed2b7 100644 --- a/docs/cl/bare-metal.md +++ b/docs/cl/bare-metal.md @@ -1,6 +1,6 @@ # Bare-Metal -In this tutorial, we'll network boot and provision a Kubernetes v1.13.4 cluster on bare-metal with Container Linux. +In this tutorial, we'll network boot and provision a Kubernetes v1.13.5 cluster on bare-metal with Container Linux. First, we'll deploy a [Matchbox](https://github.com/coreos/matchbox) service and setup a network boot environment. Then, we'll declare a Kubernetes cluster using the Typhoon Terraform module and power on machines. On PXE boot, machines will install Container Linux to disk, reboot into the disk install, and provision themselves as Kubernetes controllers or workers via Ignition. @@ -180,7 +180,7 @@ Define a Kubernetes cluster using the module `bare-metal/container-linux/kuberne ```tf module "bare-metal-mercury" { - source = "git::https://github.com/poseidon/typhoon//bare-metal/container-linux/kubernetes?ref=v1.13.4" + source = "git::https://github.com/poseidon/typhoon//bare-metal/container-linux/kubernetes?ref=v1.13.5" providers = { local = "local.default" @@ -292,9 +292,9 @@ Apply complete! Resources: 55 added, 0 changed, 0 destroyed. To watch the install to disk (until machines reboot from disk), SSH to port 2222. ``` -# before v1.13.4 +# before v1.13.5 $ ssh debug@node1.example.com -# after v1.13.4 +# after v1.13.5 $ ssh -p 2222 core@node1.example.com ``` @@ -319,9 +319,9 @@ bootkube[5]: Tearing down temporary bootstrap control plane... $ export KUBECONFIG=/home/user/.secrets/clusters/mercury/auth/kubeconfig $ kubectl get nodes NAME STATUS ROLES AGE VERSION -node1.example.com Ready controller,master 10m v1.13.4 -node2.example.com Ready node 10m v1.13.4 -node3.example.com Ready node 10m v1.13.4 +node1.example.com Ready controller,master 10m v1.13.5 +node2.example.com Ready node 10m v1.13.5 +node3.example.com Ready node 10m v1.13.5 ``` List the pods. diff --git a/docs/cl/digital-ocean.md b/docs/cl/digital-ocean.md index b04d9914c..2ecd59934 100644 --- a/docs/cl/digital-ocean.md +++ b/docs/cl/digital-ocean.md @@ -1,6 +1,6 @@ # Digital Ocean -In this tutorial, we'll create a Kubernetes v1.13.4 cluster on DigitalOcean with Container Linux. +In this tutorial, we'll create a Kubernetes v1.13.5 cluster on DigitalOcean with Container Linux. We'll declare a Kubernetes cluster using the Typhoon Terraform module. Then apply the changes to create controller droplets, worker droplets, DNS records, tags, and TLS assets. @@ -86,7 +86,7 @@ Define a Kubernetes cluster using the module `digital-ocean/container-linux/kube ```tf module "digital-ocean-nemo" { - source = "git::https://github.com/poseidon/typhoon//digital-ocean/container-linux/kubernetes?ref=v1.13.4" + source = "git::https://github.com/poseidon/typhoon//digital-ocean/container-linux/kubernetes?ref=v1.13.5" providers = { digitalocean = "digitalocean.default" @@ -160,9 +160,9 @@ In 3-6 minutes, the Kubernetes cluster will be ready. $ export KUBECONFIG=/home/user/.secrets/clusters/nemo/auth/kubeconfig $ kubectl get nodes NAME STATUS ROLES AGE VERSION -10.132.110.130 Ready controller,master 10m v1.13.4 -10.132.115.81 Ready node 10m v1.13.4 -10.132.124.107 Ready node 10m v1.13.4 +10.132.110.130 Ready controller,master 10m v1.13.5 +10.132.115.81 Ready node 10m v1.13.5 +10.132.124.107 Ready node 10m v1.13.5 ``` List the pods. diff --git a/docs/cl/google-cloud.md b/docs/cl/google-cloud.md index 3b60371e5..bc8fe9a26 100644 --- a/docs/cl/google-cloud.md +++ b/docs/cl/google-cloud.md @@ -1,6 +1,6 @@ # Google Cloud -In this tutorial, we'll create a Kubernetes v1.13.4 cluster on Google Compute Engine with Container Linux. +In this tutorial, we'll create a Kubernetes v1.13.5 cluster on Google Compute Engine with Container Linux. We'll declare a Kubernetes cluster using the Typhoon Terraform module. Then apply the changes to create a network, firewall rules, health checks, controller instances, worker managed instance group, load balancers, and TLS assets. @@ -93,7 +93,7 @@ Define a Kubernetes cluster using the module `google-cloud/container-linux/kuber ```tf module "google-cloud-yavin" { - source = "git::https://github.com/poseidon/typhoon//google-cloud/container-linux/kubernetes?ref=v1.13.4" + source = "git::https://github.com/poseidon/typhoon//google-cloud/container-linux/kubernetes?ref=v1.13.5" providers = { google = "google.default" @@ -168,9 +168,9 @@ In 4-8 minutes, the Kubernetes cluster will be ready. $ export KUBECONFIG=/home/user/.secrets/clusters/yavin/auth/kubeconfig $ kubectl get nodes NAME ROLES STATUS AGE VERSION -yavin-controller-0.c.example-com.internal controller,master Ready 6m v1.13.4 -yavin-worker-jrbf.c.example-com.internal node Ready 5m v1.13.4 -yavin-worker-mzdm.c.example-com.internal node Ready 5m v1.13.4 +yavin-controller-0.c.example-com.internal controller,master Ready 6m v1.13.5 +yavin-worker-jrbf.c.example-com.internal node Ready 5m v1.13.5 +yavin-worker-mzdm.c.example-com.internal node Ready 5m v1.13.5 ``` List the pods. diff --git a/docs/index.md b/docs/index.md index ed7b2118c..3abc081bc 100644 --- a/docs/index.md +++ b/docs/index.md @@ -11,7 +11,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features -* Kubernetes v1.13.4 (upstream, via [kubernetes-incubator/bootkube](https://github.com/kubernetes-incubator/bootkube)) +* Kubernetes v1.13.5 (upstream, via [kubernetes-incubator/bootkube](https://github.com/kubernetes-incubator/bootkube)) * Single or multi-master, [Calico](https://www.projectcalico.org/) or [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/) * Advanced features like [worker pools](advanced/worker-pools/), [preemptible](cl/google-cloud/#preemption) workers, and [snippets](advanced/customization/#container-linux) customization @@ -49,7 +49,7 @@ Define a Kubernetes cluster by using the Terraform module for your chosen platfo ```tf module "google-cloud-yavin" { - source = "git::https://github.com/poseidon/typhoon//google-cloud/container-linux/kubernetes?ref=v1.13.4" + source = "git::https://github.com/poseidon/typhoon//google-cloud/container-linux/kubernetes?ref=v1.13.5" providers = { google = "google.default" @@ -90,9 +90,9 @@ In 4-8 minutes (varies by platform), the cluster will be ready. This Google Clou $ export KUBECONFIG=/home/user/.secrets/clusters/yavin/auth/kubeconfig $ kubectl get nodes NAME ROLES STATUS AGE VERSION -yavin-controller-0.c.example-com.internal controller,master Ready 6m v1.13.4 -yavin-worker-jrbf.c.example-com.internal node Ready 5m v1.13.4 -yavin-worker-mzdm.c.example-com.internal node Ready 5m v1.13.4 +yavin-controller-0.c.example-com.internal controller,master Ready 6m v1.13.5 +yavin-worker-jrbf.c.example-com.internal node Ready 5m v1.13.5 +yavin-worker-mzdm.c.example-com.internal node Ready 5m v1.13.5 ``` List the pods. diff --git a/docs/topics/maintenance.md b/docs/topics/maintenance.md index 77d3b5a03..9baca2db4 100644 --- a/docs/topics/maintenance.md +++ b/docs/topics/maintenance.md @@ -18,7 +18,7 @@ module "google-cloud-yavin" { } module "bare-metal-mercury" { - source = "git::https://github.com/poseidon/typhoon//bare-metal/container-linux/kubernetes?ref=v1.13.4" + source = "git::https://github.com/poseidon/typhoon//bare-metal/container-linux/kubernetes?ref=v1.13.5" ... } ``` diff --git a/google-cloud/container-linux/kubernetes/README.md b/google-cloud/container-linux/kubernetes/README.md index dd56442be..bbe9188dd 100644 --- a/google-cloud/container-linux/kubernetes/README.md +++ b/google-cloud/container-linux/kubernetes/README.md @@ -11,7 +11,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features -* Kubernetes v1.13.4 (upstream, via [kubernetes-incubator/bootkube](https://github.com/kubernetes-incubator/bootkube)) +* Kubernetes v1.13.5 (upstream, via [kubernetes-incubator/bootkube](https://github.com/kubernetes-incubator/bootkube)) * Single or multi-master, [Calico](https://www.projectcalico.org/) or [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/) * Advanced features like [worker pools](https://typhoon.psdn.io/advanced/worker-pools/), [preemptible](https://typhoon.psdn.io/cl/google-cloud/#preemption) workers, and [snippets](https://typhoon.psdn.io/advanced/customization/#container-linux) customization diff --git a/google-cloud/container-linux/kubernetes/bootkube.tf b/google-cloud/container-linux/kubernetes/bootkube.tf index f4d34467d..ac4f16b4b 100644 --- a/google-cloud/container-linux/kubernetes/bootkube.tf +++ b/google-cloud/container-linux/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=23f81a5e8c4a975750beabc6f603a60a04621225" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=da0321287ba8cd48e4492894483fcdc1da057657" cluster_name = "${var.cluster_name}" api_servers = ["${format("%s.%s", var.cluster_name, var.dns_zone)}"] diff --git a/google-cloud/container-linux/kubernetes/cl/controller.yaml.tmpl b/google-cloud/container-linux/kubernetes/cl/controller.yaml.tmpl index 1f40f856f..e25ed6b25 100644 --- a/google-cloud/container-linux/kubernetes/cl/controller.yaml.tmpl +++ b/google-cloud/container-linux/kubernetes/cl/controller.yaml.tmpl @@ -124,7 +124,7 @@ storage: contents: inline: | KUBELET_IMAGE_URL=docker://k8s.gcr.io/hyperkube - KUBELET_IMAGE_TAG=v1.13.4 + KUBELET_IMAGE_TAG=v1.13.5 - path: /etc/sysctl.d/max-user-watches.conf filesystem: root contents: diff --git a/google-cloud/container-linux/kubernetes/workers/cl/worker.yaml.tmpl b/google-cloud/container-linux/kubernetes/workers/cl/worker.yaml.tmpl index 04f442208..7e00d0ad3 100644 --- a/google-cloud/container-linux/kubernetes/workers/cl/worker.yaml.tmpl +++ b/google-cloud/container-linux/kubernetes/workers/cl/worker.yaml.tmpl @@ -94,7 +94,7 @@ storage: contents: inline: | KUBELET_IMAGE_URL=docker://k8s.gcr.io/hyperkube - KUBELET_IMAGE_TAG=v1.13.4 + KUBELET_IMAGE_TAG=v1.13.5 - path: /etc/sysctl.d/max-user-watches.conf filesystem: root contents: @@ -112,7 +112,7 @@ storage: --volume config,kind=host,source=/etc/kubernetes \ --mount volume=config,target=/etc/kubernetes \ --insecure-options=image \ - docker://k8s.gcr.io/hyperkube:v1.13.4 \ + docker://k8s.gcr.io/hyperkube:v1.13.5 \ --net=host \ --dns=host \ --exec=/kubectl -- --kubeconfig=/etc/kubernetes/kubeconfig delete node $(hostname) diff --git a/google-cloud/fedora-atomic/kubernetes/README.md b/google-cloud/fedora-atomic/kubernetes/README.md index 2e7ec11e0..3f621dd7b 100644 --- a/google-cloud/fedora-atomic/kubernetes/README.md +++ b/google-cloud/fedora-atomic/kubernetes/README.md @@ -11,7 +11,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features -* Kubernetes v1.13.4 (upstream, via [kubernetes-incubator/bootkube](https://github.com/kubernetes-incubator/bootkube)) +* Kubernetes v1.13.5 (upstream, via [kubernetes-incubator/bootkube](https://github.com/kubernetes-incubator/bootkube)) * Single or multi-master, [Calico](https://www.projectcalico.org/) or [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/) * Advanced features like [worker pools](https://typhoon.psdn.io/advanced/worker-pools/) and [preemptible](https://typhoon.psdn.io/cl/google-cloud/#preemption) workers diff --git a/google-cloud/fedora-atomic/kubernetes/bootkube.tf b/google-cloud/fedora-atomic/kubernetes/bootkube.tf index d80a295d4..35c9b2a63 100644 --- a/google-cloud/fedora-atomic/kubernetes/bootkube.tf +++ b/google-cloud/fedora-atomic/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=23f81a5e8c4a975750beabc6f603a60a04621225" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=da0321287ba8cd48e4492894483fcdc1da057657" cluster_name = "${var.cluster_name}" api_servers = ["${format("%s.%s", var.cluster_name, var.dns_zone)}"] diff --git a/google-cloud/fedora-atomic/kubernetes/cloudinit/controller.yaml.tmpl b/google-cloud/fedora-atomic/kubernetes/cloudinit/controller.yaml.tmpl index 81f8b1784..8afdfc050 100644 --- a/google-cloud/fedora-atomic/kubernetes/cloudinit/controller.yaml.tmpl +++ b/google-cloud/fedora-atomic/kubernetes/cloudinit/controller.yaml.tmpl @@ -79,7 +79,7 @@ runcmd: - [systemctl, daemon-reload] - [systemctl, restart, NetworkManager] - "atomic install --system --name=etcd quay.io/poseidon/etcd:v3.3.12" - - "atomic install --system --name=kubelet quay.io/poseidon/kubelet:v1.13.4" + - "atomic install --system --name=kubelet quay.io/poseidon/kubelet:v1.13.5" - "atomic install --system --name=bootkube quay.io/poseidon/bootkube:v0.14.0" - [systemctl, start, --no-block, etcd.service] - [systemctl, start, --no-block, kubelet.service] diff --git a/google-cloud/fedora-atomic/kubernetes/workers/cloudinit/worker.yaml.tmpl b/google-cloud/fedora-atomic/kubernetes/workers/cloudinit/worker.yaml.tmpl index da76beae5..a05cd877c 100644 --- a/google-cloud/fedora-atomic/kubernetes/workers/cloudinit/worker.yaml.tmpl +++ b/google-cloud/fedora-atomic/kubernetes/workers/cloudinit/worker.yaml.tmpl @@ -54,7 +54,7 @@ bootcmd: runcmd: - [systemctl, daemon-reload] - [systemctl, restart, NetworkManager] - - "atomic install --system --name=kubelet quay.io/poseidon/kubelet:v1.13.4" + - "atomic install --system --name=kubelet quay.io/poseidon/kubelet:v1.13.5" - [systemctl, start, --no-block, kubelet.service] users: - default From 32fe72fb2d80758a389c6d4ebe84525020dc5ba5 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Tue, 26 Mar 2019 00:46:11 -0700 Subject: [PATCH 060/523] Update mkdocs and plugin versions used in tutorials * Recommend provider plugin versions that are currently used by the author * Recommend updating terraform-provider-ct plugin from v0.3.0 to v0.3.1 * https://github.com/coreos/terraform-provider-ct/releases --- CHANGES.md | 4 ++-- docs/atomic/aws.md | 2 +- docs/atomic/google-cloud.md | 2 +- docs/cl/aws.md | 10 +++++----- docs/cl/azure.md | 10 +++++----- docs/cl/bare-metal.md | 8 ++++---- docs/cl/digital-ocean.md | 8 ++++---- docs/cl/google-cloud.md | 10 +++++----- docs/topics/maintenance.md | 7 ++++--- requirements.txt | 2 +- 10 files changed, 32 insertions(+), 31 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index ac50d7cc7..fa2979449 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -4,7 +4,6 @@ Notable changes between versions. ## Latest - ## v1.13.5 * Kubernetes [v1.13.5](https://github.com/kubernetes/kubernetes/blob/master/CHANGELOG-1.13.md#v1135) @@ -12,6 +11,7 @@ Notable changes between versions. * Reverse DNS lookups for service IPv4 addresses unchanged * Upgrade Calico from v3.5.2 to [v3.6.0](https://docs.projectcalico.org/v3.6/release-notes/) ([#430](https://github.com/poseidon/typhoon/pull/430)) * Change pod IPAM from `host-local` to `calico-ipam`. `pod_cidr` is still divided into `/24` subnets per node, but managed as `ippools` and `ipamblocks` +* Suggest updating [terraform-provider-ct](https://github.com/coreos/terraform-provider-ct) from v0.3.0 to [v0.3.1](https://github.com/coreos/terraform-provider-ct/releases/tag/v0.3.1) ([#434](https://github.com/poseidon/typhoon/pull/434)) #### AWS @@ -21,7 +21,7 @@ Notable changes between versions. * Change the default iPXE kernel and initrd download protocol from HTTP to HTTPS ([#420](https://github.com/poseidon/typhoon/pull/420)) * Require an iPXE-enabled network boot environment with support for TLS downloads. PXE clients must chainload to iPXE firmware compiled with `DOWNLOAD_PROTO_HTTPS` [enabled](https://ipxe.org/crypto). (**action required**) - * Affects Container Linux and Flatcar Linux install profiles that pull from public images (default). No affect when `cached_install=true` or Fedora Atomic, since those download from Matchbox + * Only affects Container Linux and Flatcar Linux install profiles that pull public images (default) * Add `download_protocol` variable. Recognizing boot firmware TLS support is difficult in some environments, set the protocol to "http" for the old behavior (discouraged) #### DigitalOcean diff --git a/docs/atomic/aws.md b/docs/atomic/aws.md index 97f1d8df0..433b30db3 100644 --- a/docs/atomic/aws.md +++ b/docs/atomic/aws.md @@ -44,7 +44,7 @@ Configure the AWS provider to use your access key credentials in a `providers.tf ```tf provider "aws" { - version = "~> 2.1.0" + version = "~> 2.3.0" alias = "default" region = "eu-central-1" diff --git a/docs/atomic/google-cloud.md b/docs/atomic/google-cloud.md index 8ef22066a..25e22e24c 100644 --- a/docs/atomic/google-cloud.md +++ b/docs/atomic/google-cloud.md @@ -45,7 +45,7 @@ Configure the Google Cloud provider to use your service account key, project-id, ```tf provider "google" { - version = "~> 2.1.0" + version = "~> 2.2.0" alias = "default" credentials = "${file("~/.config/google-cloud/terraform.json")}" diff --git a/docs/cl/aws.md b/docs/cl/aws.md index c43c1cecb..7acd7fd85 100644 --- a/docs/cl/aws.md +++ b/docs/cl/aws.md @@ -24,9 +24,9 @@ Terraform v0.11.12 Add the [terraform-provider-ct](https://github.com/coreos/terraform-provider-ct) plugin binary for your system to `~/.terraform.d/plugins/`, noting the final name. ```sh -wget https://github.com/coreos/terraform-provider-ct/releases/download/v0.3.0/terraform-provider-ct-v0.3.0-linux-amd64.tar.gz -tar xzf terraform-provider-ct-v0.3.0-linux-amd64.tar.gz -mv terraform-provider-ct-v0.3.0-linux-amd64/terraform-provider-ct ~/.terraform.d/plugins/terraform-provider-ct_v0.3.0 +wget https://github.com/coreos/terraform-provider-ct/releases/download/v0.3.1/terraform-provider-ct-v0.3.1-linux-amd64.tar.gz +tar xzf terraform-provider-ct-v0.3.1-linux-amd64.tar.gz +mv terraform-provider-ct-v0.3.1-linux-amd64/terraform-provider-ct ~/.terraform.d/plugins/terraform-provider-ct_v0.3.1 ``` Read [concepts](/architecture/concepts/) to learn about Terraform, modules, and organizing resources. Change to your infrastructure repository (e.g. `infra`). @@ -49,7 +49,7 @@ Configure the AWS provider to use your access key credentials in a `providers.tf ```tf provider "aws" { - version = "~> 2.1.0" + version = "~> 2.3.0" alias = "default" region = "eu-central-1" @@ -57,7 +57,7 @@ provider "aws" { } provider "ct" { - version = "0.3.0" + version = "0.3.1" } provider "local" { diff --git a/docs/cl/azure.md b/docs/cl/azure.md index 2804c3687..5f128d996 100644 --- a/docs/cl/azure.md +++ b/docs/cl/azure.md @@ -27,9 +27,9 @@ Terraform v0.11.12 Add the [terraform-provider-ct](https://github.com/coreos/terraform-provider-ct) plugin binary for your system to `~/.terraform.d/plugins/`, noting the final name. ```sh -wget https://github.com/coreos/terraform-provider-ct/releases/download/v0.3.0/terraform-provider-ct-v0.3.0-linux-amd64.tar.gz -tar xzf terraform-provider-ct-v0.3.0-linux-amd64.tar.gz -mv terraform-provider-ct-v0.3.0-linux-amd64/terraform-provider-ct ~/.terraform.d/plugins/terraform-provider-ct_v0.3.0 +wget https://github.com/coreos/terraform-provider-ct/releases/download/v0.3.1/terraform-provider-ct-v0.3.1-linux-amd64.tar.gz +tar xzf terraform-provider-ct-v0.3.1-linux-amd64.tar.gz +mv terraform-provider-ct-v0.3.1-linux-amd64/terraform-provider-ct ~/.terraform.d/plugins/terraform-provider-ct_v0.3.1 ``` Read [concepts](/architecture/concepts/) to learn about Terraform, modules, and organizing resources. Change to your infrastructure repository (e.g. `infra`). @@ -50,12 +50,12 @@ Configure the Azure provider in a `providers.tf` file. ```tf provider "azurerm" { - version = "~> 1.22.1" + version = "~> 1.23.0" alias = "default" } provider "ct" { - version = "0.3.0" + version = "0.3.1" } provider "local" { diff --git a/docs/cl/bare-metal.md b/docs/cl/bare-metal.md index 9e7fed2b7..ff49622ab 100644 --- a/docs/cl/bare-metal.md +++ b/docs/cl/bare-metal.md @@ -125,9 +125,9 @@ mv terraform-provider-matchbox-v0.2.3-linux-amd64/terraform-provider-matchbox ~/ Add the [terraform-provider-ct](https://github.com/coreos/terraform-provider-ct) plugin binary for your system to `~/.terraform.d/plugins/`, noting the final name. ```sh -wget https://github.com/coreos/terraform-provider-ct/releases/download/v0.3.0/terraform-provider-ct-v0.3.0-linux-amd64.tar.gz -tar xzf terraform-provider-ct-v0.3.0-linux-amd64.tar.gz -mv terraform-provider-ct-v0.3.0-linux-amd64/terraform-provider-ct ~/.terraform.d/plugins/terraform-provider-ct_v0.3.0 +wget https://github.com/coreos/terraform-provider-ct/releases/download/v0.3.1/terraform-provider-ct-v0.3.1-linux-amd64.tar.gz +tar xzf terraform-provider-ct-v0.3.1-linux-amd64.tar.gz +mv terraform-provider-ct-v0.3.1-linux-amd64/terraform-provider-ct ~/.terraform.d/plugins/terraform-provider-ct_v0.3.1 ``` Read [concepts](/architecture/concepts/) to learn about Terraform, modules, and organizing resources. Change to your infrastructure repository (e.g. `infra`). @@ -150,7 +150,7 @@ provider "matchbox" { } provider "ct" { - version = "0.3.0" + version = "0.3.1" } provider "local" { diff --git a/docs/cl/digital-ocean.md b/docs/cl/digital-ocean.md index 2ecd59934..7235a42e6 100644 --- a/docs/cl/digital-ocean.md +++ b/docs/cl/digital-ocean.md @@ -24,9 +24,9 @@ Terraform v0.11.12 Add the [terraform-provider-ct](https://github.com/coreos/terraform-provider-ct) plugin binary for your system to `~/.terraform.d/plugins/`, noting the final name. ```sh -wget https://github.com/coreos/terraform-provider-ct/releases/download/v0.3.0/terraform-provider-ct-v0.3.0-linux-amd64.tar.gz -tar xzf terraform-provider-ct-v0.3.0-linux-amd64.tar.gz -mv terraform-provider-ct-v0.3.0-linux-amd64/terraform-provider-ct ~/.terraform.d/plugins/terraform-provider-ct_v0.3.0 +wget https://github.com/coreos/terraform-provider-ct/releases/download/v0.3.1/terraform-provider-ct-v0.3.1-linux-amd64.tar.gz +tar xzf terraform-provider-ct-v0.3.1-linux-amd64.tar.gz +mv terraform-provider-ct-v0.3.1-linux-amd64/terraform-provider-ct ~/.terraform.d/plugins/terraform-provider-ct_v0.3.1 ``` Read [concepts](/architecture/concepts/) to learn about Terraform, modules, and organizing resources. Change to your infrastructure repository (e.g. `infra`). @@ -56,7 +56,7 @@ provider "digitalocean" { } provider "ct" { - version = "0.3.0" + version = "0.3.1" } provider "local" { diff --git a/docs/cl/google-cloud.md b/docs/cl/google-cloud.md index bc8fe9a26..393d5840b 100644 --- a/docs/cl/google-cloud.md +++ b/docs/cl/google-cloud.md @@ -24,9 +24,9 @@ Terraform v0.11.12 Add the [terraform-provider-ct](https://github.com/coreos/terraform-provider-ct) plugin binary for your system to `~/.terraform.d/plugins/`, noting the final name. ```sh -wget https://github.com/coreos/terraform-provider-ct/releases/download/v0.3.0/terraform-provider-ct-v0.3.0-linux-amd64.tar.gz -tar xzf terraform-provider-ct-v0.3.0-linux-amd64.tar.gz -mv terraform-provider-ct-v0.3.0-linux-amd64/terraform-provider-ct ~/.terraform.d/plugins/terraform-provider-ct_v0.3.0 +wget https://github.com/coreos/terraform-provider-ct/releases/download/v0.3.1/terraform-provider-ct-v0.3.1-linux-amd64.tar.gz +tar xzf terraform-provider-ct-v0.3.1-linux-amd64.tar.gz +mv terraform-provider-ct-v0.3.1-linux-amd64/terraform-provider-ct ~/.terraform.d/plugins/terraform-provider-ct_v0.3.1 ``` Read [concepts](/architecture/concepts/) to learn about Terraform, modules, and organizing resources. Change to your infrastructure repository (e.g. `infra`). @@ -49,7 +49,7 @@ Configure the Google Cloud provider to use your service account key, project-id, ```tf provider "google" { - version = "~> 2.1.0" + version = "~> 2.2.0" alias = "default" credentials = "${file("~/.config/google-cloud/terraform.json")}" @@ -58,7 +58,7 @@ provider "google" { } provider "ct" { - version = "0.3.0" + version = "0.3.1" } provider "local" { diff --git a/docs/topics/maintenance.md b/docs/topics/maintenance.md index 9baca2db4..c452bb8bb 100644 --- a/docs/topics/maintenance.md +++ b/docs/topics/maintenance.md @@ -202,9 +202,9 @@ First, [migrate](#terraform-plugins-directory) to the Terraform 3rd-party plugin Add the [terraform-provider-ct](https://github.com/coreos/terraform-provider-ct) plugin binary for your system to `~/.terraform.d/plugins/`, noting the final name. ```sh -wget https://github.com/coreos/terraform-provider-ct/releases/download/v0.3.0/terraform-provider-ct-v0.3.0-linux-amd64.tar.gz -tar xzf terraform-provider-ct-v0.3.0-linux-amd64.tar.gz -mv terraform-provider-ct-v0.3.0-linux-amd64/terraform-provider-ct ~/.terraform.d/plugins/terraform-provider-ct_v0.3.0 +wget https://github.com/coreos/terraform-provider-ct/releases/download/v0.3.1/terraform-provider-ct-v0.3.1-linux-amd64.tar.gz +tar xzf terraform-provider-ct-v0.3.1-linux-amd64.tar.gz +mv terraform-provider-ct-v0.3.1-linux-amd64/terraform-provider-ct ~/.terraform.d/plugins/terraform-provider-ct_v0.3.1 ``` Binary names are versioned. This enables the ability to upgrade different plugins and have clusters pin different versions. @@ -215,6 +215,7 @@ $ tree ~/.terraform.d/ └── plugins ├── terraform-provider-ct_v0.2.1 ├── terraform-provider-ct_v0.3.0 + ├── terraform-provider-ct_v0.3.1 └── terraform-provider-matchbox_v0.2.3 ``` diff --git a/requirements.txt b/requirements.txt index 899424e7c..4e29be567 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ mkdocs==1.0.4 -mkdocs-material==4.0.2 +mkdocs-material==4.1.0 pygments==2.2.0 pymdown-extensions==5.0.0 From 5a1bc423a1c6e218906c64c0bb22928285a92657 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Tue, 26 Mar 2019 23:47:14 -0700 Subject: [PATCH 061/523] Announce Fedora Atomic modules won't be updated beyond v1.13.x * Thank you Project Atomic team and users * See the deprecation announcement https://typhoon.psdn.io/announce/#march-27-2019 --- CHANGES.md | 2 ++ README.md | 8 ++++---- docs/announce.md | 14 ++++++++++++++ docs/atomic/aws.md | 2 +- docs/atomic/bare-metal.md | 2 +- docs/atomic/digital-ocean.md | 2 +- docs/atomic/google-cloud.md | 2 +- docs/index.md | 8 ++++---- 8 files changed, 28 insertions(+), 12 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index fa2979449..4e28646ee 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -12,6 +12,8 @@ Notable changes between versions. * Upgrade Calico from v3.5.2 to [v3.6.0](https://docs.projectcalico.org/v3.6/release-notes/) ([#430](https://github.com/poseidon/typhoon/pull/430)) * Change pod IPAM from `host-local` to `calico-ipam`. `pod_cidr` is still divided into `/24` subnets per node, but managed as `ippools` and `ipamblocks` * Suggest updating [terraform-provider-ct](https://github.com/coreos/terraform-provider-ct) from v0.3.0 to [v0.3.1](https://github.com/coreos/terraform-provider-ct/releases/tag/v0.3.1) ([#434](https://github.com/poseidon/typhoon/pull/434)) +* Announce: Fedora Atomic modules will be not be updated beyond Kubernetes v1.13.x ([#437](https://github.com/poseidon/typhoon/pull/437)) + * Thank you Project Atomic team and users, please see the deprecation [notice](https://typhoon.psdn.io/announce/#march-27-2019) #### AWS diff --git a/README.md b/README.md index b55ca38f7..55d4b6b62 100644 --- a/README.md +++ b/README.md @@ -33,10 +33,10 @@ Fedora Atomic support is alpha and will evolve as Fedora Atomic is replaced by F | Platform | Operating System | Terraform Module | Status | |---------------|------------------|------------------|--------| -| AWS | Fedora Atomic | [aws/fedora-atomic/kubernetes](aws/fedora-atomic/kubernetes) | alpha | -| Bare-Metal | Fedora Atomic | [bare-metal/fedora-atomic/kubernetes](bare-metal/fedora-atomic/kubernetes) | alpha | -| Digital Ocean | Fedora Atomic | [digital-ocean/fedora-atomic/kubernetes](digital-ocean/fedora-atomic/kubernetes) | alpha | -| Google Cloud | Fedora Atomic | [google-cloud/fedora-atomic/kubernetes](google-cloud/fedora-atomic/kubernetes) | alpha | +| AWS | Fedora Atomic | [aws/fedora-atomic/kubernetes](aws/fedora-atomic/kubernetes) | deprecated | +| Bare-Metal | Fedora Atomic | [bare-metal/fedora-atomic/kubernetes](bare-metal/fedora-atomic/kubernetes) | deprecated | +| Digital Ocean | Fedora Atomic | [digital-ocean/fedora-atomic/kubernetes](digital-ocean/fedora-atomic/kubernetes) | deprecated | +| Google Cloud | Fedora Atomic | [google-cloud/fedora-atomic/kubernetes](google-cloud/fedora-atomic/kubernetes) | deprecated | ## Documentation diff --git a/docs/announce.md b/docs/announce.md index ab1467cb4..789ebf382 100644 --- a/docs/announce.md +++ b/docs/announce.md @@ -1,5 +1,19 @@ # Announce +## March 27, 2019 + +Last April, Typhoon [introduced](#april-26-2018) alpha support for creating Kubernetes clusters with Fedora Atomic on AWS, Google Cloud, DigitalOcean, and bare-metal. Fedora Atomic shared many of Container Linux's aims for a container-optimized operating system, introduced novel ideas, and provided technical diversification for an uncertain future. However, Project Atomic efforts were merged into Fedora CoreOS and future Fedora Atomic releases are [not expected](http://www.projectatomic.io/blog/2018/06/welcome-to-fedora-coreos/). *Typhoon modules for Fedora Atomic will not be updated much beyond Kubernetes v1.13*. They may later be removed. + +Typhoon for Fedora Atomic fell short of goals to provide a consistent, practical experience across operating systems and platforms. The modules have remained alpha, despite improvements. Features like coordinated OS updates and boot-time declarative customization were not realized. Inelegance of Cloud-Init/kickstart loomed large. With that brief but obligatory summary, I'd like to change gears and celebrate the many positives. + +Fedora Atomic showcased [rpm-ostree](https://github.com/projectatomic/rpm-ostree) as a different approach to Container Linux's AB update scheme. It provided a viable route toward [CRI-O](https://github.com/kubernetes-sigs/cri-o) to replace Docker as the container engine. And Fedora Atomic devised [system containers](http://www.projectatomic.io/blog/2016/09/intro-to-system-containers/) as a way to package and run raw OCI images through runc for host-level containers[^2]. Many of these ideas will live on in Fedora CoreOS, which is exciting! + +For Typhoon, Fedora Atomic brought fresh ideas and broader perspectives about different container-optimized base operating systems and related tools. Its sad to let go of so much work, but I think its time. Many of the concepts and technologies that were explored will surface again and Typhoon is better positioned as a result. + +Thank you Project Atomic team members for your work! - dghubble + +[^2]: Container Linux's own primordial rkt-fly shim dates back to the pre-OCI era. In some ways, rkt drove the OCI standards that made newer ideas, like system containers, appealing. + ## May 23, 2018 Starting in v1.10.3, Typhoon AWS and bare-metal `container-linux` modules allow picking between the Red Hat [Container Linux](https://coreos.com/os/docs/latest/) (formerly CoreOS Container Linux) and Kinvolk [Flatcar Linux](https://www.flatcar-linux.org/) operating system. Flatcar Linux serves as a drop-in compatible "friendly fork" of Container Linux. Flatcar Linux publishes the same channels and versions as Container Linux and gets provisioned, managed, and operated in an identical way (e.g. login as user "core"). diff --git a/docs/atomic/aws.md b/docs/atomic/aws.md index 433b30db3..e0f575de9 100644 --- a/docs/atomic/aws.md +++ b/docs/atomic/aws.md @@ -1,7 +1,7 @@ # AWS !!! danger - Typhoon for Fedora Atomic is alpha. Expect rough edges and changes. + Typhoon for Fedora Atomic will not be updated much beyond Kubernetes v1.13. In this tutorial, we'll create a Kubernetes v1.13.5 cluster on AWS with Fedora Atomic. diff --git a/docs/atomic/bare-metal.md b/docs/atomic/bare-metal.md index fd4fcd0e0..f04acd246 100644 --- a/docs/atomic/bare-metal.md +++ b/docs/atomic/bare-metal.md @@ -1,7 +1,7 @@ # Bare-Metal !!! danger - Typhoon for Fedora Atomic is alpha. Expect rough edges and changes. + Typhoon for Fedora Atomic will not be updated much beyond Kubernetes v1.13. In this tutorial, we'll network boot and provision a Kubernetes v1.13.5 cluster on bare-metal with Fedora Atomic. diff --git a/docs/atomic/digital-ocean.md b/docs/atomic/digital-ocean.md index c504688e7..c33ea5146 100644 --- a/docs/atomic/digital-ocean.md +++ b/docs/atomic/digital-ocean.md @@ -1,7 +1,7 @@ # Digital Ocean !!! danger - Typhoon for Fedora Atomic is alpha. Expect rough edges and changes. + Typhoon for Fedora Atomic will not be updated much beyond Kubernetes v1.13. In this tutorial, we'll create a Kubernetes v1.13.5 cluster on DigitalOcean with Fedora Atomic. diff --git a/docs/atomic/google-cloud.md b/docs/atomic/google-cloud.md index 25e22e24c..111eb4995 100644 --- a/docs/atomic/google-cloud.md +++ b/docs/atomic/google-cloud.md @@ -1,7 +1,7 @@ # Google Cloud !!! danger - Typhoon for Fedora Atomic is alpha. Fedora does not publish official images for Google Cloud so you must prepare them yourself. Expect rough edges and changes. + Typhoon for Fedora Atomic will not be updated much beyond Kubernetes v1.13. Fedora does not publish official images for Google Cloud so you must prepare them yourself. Expect rough edges and changes. In this tutorial, we'll create a Kubernetes v1.13.5 cluster on Google Compute Engine with Fedora Atomic. diff --git a/docs/index.md b/docs/index.md index 3abc081bc..77c788c45 100644 --- a/docs/index.md +++ b/docs/index.md @@ -33,10 +33,10 @@ Fedora Atomic support is alpha and will evolve as Fedora Atomic is replaced by F | Platform | Operating System | Terraform Module | Status | |---------------|------------------|------------------|--------| -| AWS | Fedora Atomic | [aws/fedora-atomic/kubernetes](atomic/aws.md) | alpha | -| Bare-Metal | Fedora Atomic | [bare-metal/fedora-atomic/kubernetes](atomic/bare-metal.md) | alpha | -| Digital Ocean | Fedora Atomic | [digital-ocean/fedora-atomic/kubernetes](atomic/digital-ocean.md) | alpha | -| Google Cloud | Fedora Atomic | [google-cloud/fedora-atomic/kubernetes](atomic/google-cloud.md) | alpha | +| AWS | Fedora Atomic | [aws/fedora-atomic/kubernetes](atomic/aws.md) | deprecated | +| Bare-Metal | Fedora Atomic | [bare-metal/fedora-atomic/kubernetes](atomic/bare-metal.md) | deprecated | +| Digital Ocean | Fedora Atomic | [digital-ocean/fedora-atomic/kubernetes](atomic/digital-ocean.md) | deprecated | +| Google Cloud | Fedora Atomic | [google-cloud/fedora-atomic/kubernetes](atomic/google-cloud.md) | deprecated | ## Documentation From 46196af5009fb2abef89a979248cb4bd55c6b941 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Wed, 27 Mar 2019 19:45:10 -0700 Subject: [PATCH 062/523] Remove Haswell minimum CPU platform requirement * Google Cloud API implements `min_cpu_platform` to mean "use exactly this CPU" * Fix error creating clusters in newer regions lacking Haswell platform (e.g. europe-west2) (#438) * Reverts #405, added in v1.13.4 * Original goal of ignoring old Ivy/Sandy bridge CPUs in older regions will be achieved shortly anyway. Google Cloud is deprecating those CPUs in April 2019 * https://cloud.google.com/compute/docs/instances/specify-min-cpu-platform#how_selecting_a_minimum_cpu_platform_works --- CHANGES.md | 6 ++++++ bare-metal/container-linux/kubernetes/variables.tf | 4 ++-- google-cloud/container-linux/kubernetes/controllers.tf | 7 +++---- google-cloud/container-linux/kubernetes/workers/workers.tf | 7 +++---- google-cloud/fedora-atomic/kubernetes/controllers.tf | 7 +++---- google-cloud/fedora-atomic/kubernetes/workers/workers.tf | 7 +++---- 6 files changed, 20 insertions(+), 18 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 4e28646ee..d3e8a9994 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -4,6 +4,12 @@ Notable changes between versions. ## Latest +#### Google Cloud + +* Remove Haswell minimum CPU platform requirement ([#439](https://github.com/poseidon/typhoon/pull/439)) + * Google Cloud API implements `min_cpu_platform` to mean "use exactly this CPU". Revert [#405](https://github.com/poseidon/typhoon/pull/405) added in v1.13.4. + * Fix error creating clusters in new regions without Haswell (e.g. europe-west2) ([#438](https://github.com/poseidon/typhoon/issues/438)) + ## v1.13.5 * Kubernetes [v1.13.5](https://github.com/kubernetes/kubernetes/blob/master/CHANGELOG-1.13.md#v1135) diff --git a/bare-metal/container-linux/kubernetes/variables.tf b/bare-metal/container-linux/kubernetes/variables.tf index 3e0a40434..bfba58dba 100644 --- a/bare-metal/container-linux/kubernetes/variables.tf +++ b/bare-metal/container-linux/kubernetes/variables.tf @@ -119,8 +119,8 @@ variable "cluster_domain_suffix" { } variable "download_protocol" { - type = "string" - default = "https" + type = "string" + default = "https" description = "Protocol iPXE should use to download the kernel and initrd. Defaults to https, which requires iPXE compiled with crypto support. Unused if cached_install is true." } diff --git a/google-cloud/container-linux/kubernetes/controllers.tf b/google-cloud/container-linux/kubernetes/controllers.tf index 3578f04f4..f61face6d 100644 --- a/google-cloud/container-linux/kubernetes/controllers.tf +++ b/google-cloud/container-linux/kubernetes/controllers.tf @@ -31,10 +31,9 @@ locals { resource "google_compute_instance" "controllers" { count = "${var.controller_count}" - name = "${var.cluster_name}-controller-${count.index}" - zone = "${element(local.zones, count.index)}" - machine_type = "${var.controller_type}" - min_cpu_platform = "Intel Haswell" + name = "${var.cluster_name}-controller-${count.index}" + zone = "${element(local.zones, count.index)}" + machine_type = "${var.controller_type}" metadata = { user-data = "${element(data.ct_config.controller-ignitions.*.rendered, count.index)}" diff --git a/google-cloud/container-linux/kubernetes/workers/workers.tf b/google-cloud/container-linux/kubernetes/workers/workers.tf index 422ff0398..f0114ef3d 100644 --- a/google-cloud/container-linux/kubernetes/workers/workers.tf +++ b/google-cloud/container-linux/kubernetes/workers/workers.tf @@ -23,10 +23,9 @@ resource "google_compute_region_instance_group_manager" "workers" { # Worker instance template resource "google_compute_instance_template" "worker" { - name_prefix = "${var.name}-worker-" - description = "Worker Instance template" - machine_type = "${var.machine_type}" - min_cpu_platform = "Intel Haswell" + name_prefix = "${var.name}-worker-" + description = "Worker Instance template" + machine_type = "${var.machine_type}" metadata = { user-data = "${data.ct_config.worker-ignition.rendered}" diff --git a/google-cloud/fedora-atomic/kubernetes/controllers.tf b/google-cloud/fedora-atomic/kubernetes/controllers.tf index c166975e4..5ccba436b 100644 --- a/google-cloud/fedora-atomic/kubernetes/controllers.tf +++ b/google-cloud/fedora-atomic/kubernetes/controllers.tf @@ -31,10 +31,9 @@ locals { resource "google_compute_instance" "controllers" { count = "${var.controller_count}" - name = "${var.cluster_name}-controller-${count.index}" - zone = "${element(local.zones, count.index)}" - machine_type = "${var.controller_type}" - min_cpu_platform = "Intel Haswell" + name = "${var.cluster_name}-controller-${count.index}" + zone = "${element(local.zones, count.index)}" + machine_type = "${var.controller_type}" metadata = { user-data = "${element(data.template_file.controller-cloudinit.*.rendered, count.index)}" diff --git a/google-cloud/fedora-atomic/kubernetes/workers/workers.tf b/google-cloud/fedora-atomic/kubernetes/workers/workers.tf index 95ea6e5a1..fb76f77e7 100644 --- a/google-cloud/fedora-atomic/kubernetes/workers/workers.tf +++ b/google-cloud/fedora-atomic/kubernetes/workers/workers.tf @@ -23,10 +23,9 @@ resource "google_compute_region_instance_group_manager" "workers" { # Worker instance template resource "google_compute_instance_template" "worker" { - name_prefix = "${var.name}-worker-" - description = "Worker Instance template" - machine_type = "${var.machine_type}" - min_cpu_platform = "Intel Haswell" + name_prefix = "${var.name}-worker-" + description = "Worker Instance template" + machine_type = "${var.machine_type}" metadata = { user-data = "${data.template_file.worker-cloudinit.rendered}" From 3e9dc28a0083faa4139a1cba91853589c6d6ea90 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Sun, 31 Mar 2019 17:03:22 -0700 Subject: [PATCH 063/523] Update Prometheus from v2.8.0 to v2.8.1 * https://github.com/prometheus/prometheus/releases/tag/v2.8.1 --- CHANGES.md | 4 ++++ addons/prometheus/deployment.yaml | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index d3e8a9994..d8857764d 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -10,6 +10,10 @@ Notable changes between versions. * Google Cloud API implements `min_cpu_platform` to mean "use exactly this CPU". Revert [#405](https://github.com/poseidon/typhoon/pull/405) added in v1.13.4. * Fix error creating clusters in new regions without Haswell (e.g. europe-west2) ([#438](https://github.com/poseidon/typhoon/issues/438)) +#### Addons + +* Update Prometheus from v2.8.0 to v2.8.1 + ## v1.13.5 * Kubernetes [v1.13.5](https://github.com/kubernetes/kubernetes/blob/master/CHANGELOG-1.13.md#v1135) diff --git a/addons/prometheus/deployment.yaml b/addons/prometheus/deployment.yaml index ccb727bf3..254475982 100644 --- a/addons/prometheus/deployment.yaml +++ b/addons/prometheus/deployment.yaml @@ -20,7 +20,7 @@ spec: serviceAccountName: prometheus containers: - name: prometheus - image: quay.io/prometheus/prometheus:v2.8.0 + image: quay.io/prometheus/prometheus:v2.8.1 args: - --web.listen-address=0.0.0.0:9090 - --config.file=/etc/prometheus/prometheus.yaml From b3ec5f73e38dd792d14f2f27144bdfdd08f34f84 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Sun, 31 Mar 2019 17:43:43 -0700 Subject: [PATCH 064/523] Update Calico from v3.6.0 to v3.6.1 * https://docs.projectcalico.org/v3.6/release-notes/ --- CHANGES.md | 2 ++ aws/container-linux/kubernetes/bootkube.tf | 2 +- aws/fedora-atomic/kubernetes/bootkube.tf | 2 +- azure/container-linux/kubernetes/bootkube.tf | 2 +- bare-metal/container-linux/kubernetes/bootkube.tf | 2 +- bare-metal/fedora-atomic/kubernetes/bootkube.tf | 2 +- digital-ocean/container-linux/kubernetes/bootkube.tf | 2 +- digital-ocean/fedora-atomic/kubernetes/bootkube.tf | 2 +- google-cloud/container-linux/kubernetes/bootkube.tf | 2 +- google-cloud/fedora-atomic/kubernetes/bootkube.tf | 2 +- 10 files changed, 11 insertions(+), 9 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index d8857764d..95b392c95 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -4,6 +4,8 @@ Notable changes between versions. ## Latest +* Update Calico from v3.6.0 to v3.6.1 + #### Google Cloud * Remove Haswell minimum CPU platform requirement ([#439](https://github.com/poseidon/typhoon/pull/439)) diff --git a/aws/container-linux/kubernetes/bootkube.tf b/aws/container-linux/kubernetes/bootkube.tf index 11f3f2f86..0103059b3 100644 --- a/aws/container-linux/kubernetes/bootkube.tf +++ b/aws/container-linux/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=da0321287ba8cd48e4492894483fcdc1da057657" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=bcb015e1052192b166b19a1e3b5da0f25051eaf4" cluster_name = "${var.cluster_name}" api_servers = ["${format("%s.%s", var.cluster_name, var.dns_zone)}"] diff --git a/aws/fedora-atomic/kubernetes/bootkube.tf b/aws/fedora-atomic/kubernetes/bootkube.tf index 371140ce1..fc45dc315 100644 --- a/aws/fedora-atomic/kubernetes/bootkube.tf +++ b/aws/fedora-atomic/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=da0321287ba8cd48e4492894483fcdc1da057657" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=bcb015e1052192b166b19a1e3b5da0f25051eaf4" cluster_name = "${var.cluster_name}" api_servers = ["${format("%s.%s", var.cluster_name, var.dns_zone)}"] diff --git a/azure/container-linux/kubernetes/bootkube.tf b/azure/container-linux/kubernetes/bootkube.tf index 58290863a..4055d2b2b 100644 --- a/azure/container-linux/kubernetes/bootkube.tf +++ b/azure/container-linux/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=da0321287ba8cd48e4492894483fcdc1da057657" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=bcb015e1052192b166b19a1e3b5da0f25051eaf4" cluster_name = "${var.cluster_name}" api_servers = ["${format("%s.%s", var.cluster_name, var.dns_zone)}"] diff --git a/bare-metal/container-linux/kubernetes/bootkube.tf b/bare-metal/container-linux/kubernetes/bootkube.tf index f43bf8b30..e94c60fac 100644 --- a/bare-metal/container-linux/kubernetes/bootkube.tf +++ b/bare-metal/container-linux/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=da0321287ba8cd48e4492894483fcdc1da057657" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=bcb015e1052192b166b19a1e3b5da0f25051eaf4" cluster_name = "${var.cluster_name}" api_servers = ["${var.k8s_domain_name}"] diff --git a/bare-metal/fedora-atomic/kubernetes/bootkube.tf b/bare-metal/fedora-atomic/kubernetes/bootkube.tf index c07e687d6..117c2c2c7 100644 --- a/bare-metal/fedora-atomic/kubernetes/bootkube.tf +++ b/bare-metal/fedora-atomic/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=da0321287ba8cd48e4492894483fcdc1da057657" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=bcb015e1052192b166b19a1e3b5da0f25051eaf4" cluster_name = "${var.cluster_name}" api_servers = ["${var.k8s_domain_name}"] diff --git a/digital-ocean/container-linux/kubernetes/bootkube.tf b/digital-ocean/container-linux/kubernetes/bootkube.tf index b33120245..1ae13f383 100644 --- a/digital-ocean/container-linux/kubernetes/bootkube.tf +++ b/digital-ocean/container-linux/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=da0321287ba8cd48e4492894483fcdc1da057657" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=bcb015e1052192b166b19a1e3b5da0f25051eaf4" cluster_name = "${var.cluster_name}" api_servers = ["${format("%s.%s", var.cluster_name, var.dns_zone)}"] diff --git a/digital-ocean/fedora-atomic/kubernetes/bootkube.tf b/digital-ocean/fedora-atomic/kubernetes/bootkube.tf index 1a3b4b827..8f18569a8 100644 --- a/digital-ocean/fedora-atomic/kubernetes/bootkube.tf +++ b/digital-ocean/fedora-atomic/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=da0321287ba8cd48e4492894483fcdc1da057657" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=bcb015e1052192b166b19a1e3b5da0f25051eaf4" cluster_name = "${var.cluster_name}" api_servers = ["${format("%s.%s", var.cluster_name, var.dns_zone)}"] diff --git a/google-cloud/container-linux/kubernetes/bootkube.tf b/google-cloud/container-linux/kubernetes/bootkube.tf index ac4f16b4b..b179f4b13 100644 --- a/google-cloud/container-linux/kubernetes/bootkube.tf +++ b/google-cloud/container-linux/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=da0321287ba8cd48e4492894483fcdc1da057657" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=bcb015e1052192b166b19a1e3b5da0f25051eaf4" cluster_name = "${var.cluster_name}" api_servers = ["${format("%s.%s", var.cluster_name, var.dns_zone)}"] diff --git a/google-cloud/fedora-atomic/kubernetes/bootkube.tf b/google-cloud/fedora-atomic/kubernetes/bootkube.tf index 35c9b2a63..21a91dba5 100644 --- a/google-cloud/fedora-atomic/kubernetes/bootkube.tf +++ b/google-cloud/fedora-atomic/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=da0321287ba8cd48e4492894483fcdc1da057657" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=bcb015e1052192b166b19a1e3b5da0f25051eaf4" cluster_name = "${var.cluster_name}" api_servers = ["${format("%s.%s", var.cluster_name, var.dns_zone)}"] From ae3a8a5770f3111157c6b54acb2886d79ebee0a1 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Sun, 31 Mar 2019 18:23:18 -0700 Subject: [PATCH 065/523] Update mkdocs-material from v4.1.0 to v4.1.1 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 4e29be567..1d43bfcb4 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ mkdocs==1.0.4 -mkdocs-material==4.1.0 +mkdocs-material==4.1.1 pygments==2.2.0 pymdown-extensions==5.0.0 From aaa8e0261a01a62fa4a343788b3df20b9e93ee38 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Sat, 30 Mar 2019 15:12:55 -0700 Subject: [PATCH 066/523] Add Google Cloud worker instances to a target pool * Background: A managed instance group of workers is used in backend services for global load balancing (HTTP/HTTPS Ingress) and output for custom global load balancing use cases * Add worker instances to a target pool load balancing TCP/UDP applications (NodePort or proxied). Output as `worker_target_pool` * Health check for workers with a healthy Ingress controller. Forward rules (regional) to target pools don't support different external and internal ports so choosing nodes with Ingress allows proxying as a workaround * A target pool is a logical grouping only. It doesn't add costs to clusters or worker pools --- CHANGES.md | 3 +++ .../container-linux/kubernetes/network.tf | 22 ++++++++++++++----- .../container-linux/kubernetes/outputs.tf | 7 +++++- .../kubernetes/workers/outputs.tf | 11 +++++++++- .../kubernetes/workers/target_pool.tf | 21 ++++++++++++++++++ .../kubernetes/workers/workers.tf | 5 +++-- 6 files changed, 60 insertions(+), 9 deletions(-) create mode 100644 google-cloud/container-linux/kubernetes/workers/target_pool.tf diff --git a/CHANGES.md b/CHANGES.md index 95b392c95..6a774efe3 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -8,6 +8,9 @@ Notable changes between versions. #### Google Cloud +* Add ability to load balance TCP/UDP applications ([#442](https://github.com/poseidon/typhoon/pull/442)) + * Add worker instances to a target pool, output as `worker_target_pool` + * Health check for workers with Ingress controllers. Forward rules don't support differing internal/external ports, but some Ingress controllers support TCP/UDP proxy as a workaround * Remove Haswell minimum CPU platform requirement ([#439](https://github.com/poseidon/typhoon/pull/439)) * Google Cloud API implements `min_cpu_platform` to mean "use exactly this CPU". Revert [#405](https://github.com/poseidon/typhoon/pull/405) added in v1.13.4. * Fix error creating clusters in new regions without Haswell (e.g. europe-west2) ([#438](https://github.com/poseidon/typhoon/issues/438)) diff --git a/google-cloud/container-linux/kubernetes/network.tf b/google-cloud/container-linux/kubernetes/network.tf index a507ce8ea..04496186e 100644 --- a/google-cloud/container-linux/kubernetes/network.tf +++ b/google-cloud/container-linux/kubernetes/network.tf @@ -138,8 +138,8 @@ resource "google_compute_firewall" "allow-ingress" { target_tags = ["${var.cluster_name}-worker"] } -resource "google_compute_firewall" "google-health-checks" { - name = "${var.cluster_name}-google-health-checks" +resource "google_compute_firewall" "google-ingress-health-checks" { + name = "${var.cluster_name}-ingress-health" network = "${google_compute_network.network.name}" allow { @@ -147,7 +147,19 @@ resource "google_compute_firewall" "google-health-checks" { ports = [10254] } - # https://cloud.google.com/compute/docs/load-balancing/tcp-ssl/tcp-proxy#health-checking - source_ranges = ["130.211.0.0/22", "35.191.0.0/16"] - target_tags = ["${var.cluster_name}-worker"] + # https://cloud.google.com/load-balancing/docs/health-check-concepts#method + source_ranges = [ + # Global LB health checks + "35.191.0.0/16", + + "130.211.0.0/22", + + # Region LB health checks + "35.191.0.0/16", + + "209.85.152.0/22", + "209.85.204.0/22", + ] + + target_tags = ["${var.cluster_name}-worker"] } diff --git a/google-cloud/container-linux/kubernetes/outputs.tf b/google-cloud/container-linux/kubernetes/outputs.tf index f1a09ece4..aefdd9b81 100644 --- a/google-cloud/container-linux/kubernetes/outputs.tf +++ b/google-cloud/container-linux/kubernetes/outputs.tf @@ -33,6 +33,11 @@ output "network_self_link" { # Outputs for custom load balancing output "worker_instance_group" { - description = "Full URL of the worker managed instance group" + description = "Worker managed instance group full URL" value = "${module.workers.instance_group}" } + +output "worker_target_pool" { + description = "Worker target pool self link" + value = "${module.workers.target_pool}" +} diff --git a/google-cloud/container-linux/kubernetes/workers/outputs.tf b/google-cloud/container-linux/kubernetes/workers/outputs.tf index 4521d7923..0817d675d 100644 --- a/google-cloud/container-linux/kubernetes/workers/outputs.tf +++ b/google-cloud/container-linux/kubernetes/workers/outputs.tf @@ -1,4 +1,13 @@ +# Outputs for global load balancing + output "instance_group" { - description = "Full URL of the worker managed instance group" + description = "Worker managed instance group full URL" value = "${google_compute_region_instance_group_manager.workers.instance_group}" } + +# Outputs for regional load balancing + +output "target_pool" { + description = "Worker target pool self link" + value = "${google_compute_target_pool.workers.self_link}" +} diff --git a/google-cloud/container-linux/kubernetes/workers/target_pool.tf b/google-cloud/container-linux/kubernetes/workers/target_pool.tf new file mode 100644 index 000000000..949591d57 --- /dev/null +++ b/google-cloud/container-linux/kubernetes/workers/target_pool.tf @@ -0,0 +1,21 @@ +# Target pool for TCP/UDP load balancing +resource "google_compute_target_pool" "workers" { + name = "${var.name}-worker-pool" + session_affinity = "NONE" + + health_checks = [ + "${google_compute_http_health_check.workers.name}", + ] +} + +# HTTP Health Check (for TCP/UDP load balancing) +# Forward rules (regional) to target pools don't support different external +# and internal ports. Health check for nodes with Ingress controllers that +# may support proxying or otherwise satisfy the check. +resource "google_compute_http_health_check" "workers" { + name = "${var.name}-target-pool-health" + description = "Health check for the worker target pool" + + port = 10254 + request_path = "/healthz" +} diff --git a/google-cloud/container-linux/kubernetes/workers/workers.tf b/google-cloud/container-linux/kubernetes/workers/workers.tf index f0114ef3d..1dcf0cbc8 100644 --- a/google-cloud/container-linux/kubernetes/workers/workers.tf +++ b/google-cloud/container-linux/kubernetes/workers/workers.tf @@ -1,4 +1,4 @@ -# Regional managed instance group of workers +# Managed instance group of workers resource "google_compute_region_instance_group_manager" "workers" { name = "${var.name}-worker-group" description = "Compute instance group of ${var.name} workers" @@ -8,7 +8,8 @@ resource "google_compute_region_instance_group_manager" "workers" { instance_template = "${google_compute_instance_template.worker.self_link}" region = "${var.region}" - target_size = "${var.count}" + target_size = "${var.count}" + target_pools = ["${google_compute_target_pool.workers.self_link}"] named_port { name = "http" From 60265f9b5890169a7f450249e202220ab064f68f Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Sun, 31 Mar 2019 23:22:47 -0700 Subject: [PATCH 067/523] Add ability to load balance TCP applications on AWS * Add ability to load balance TCP applications (e.g. NodePort) * Output the network load balancer ARN as `nlb_id` * Accept a `worker_target_groups` (ARN) list to which worker instances should be added * AWS NLBs and target groups don't support UDP --- CHANGES.md | 6 ++++++ aws/container-linux/kubernetes/outputs.tf | 5 +++++ aws/container-linux/kubernetes/variables.tf | 6 ++++++ aws/container-linux/kubernetes/workers.tf | 1 + aws/container-linux/kubernetes/workers/variables.tf | 6 ++++++ aws/container-linux/kubernetes/workers/workers.tf | 1 + docs/cl/aws.md | 1 + 7 files changed, 26 insertions(+) diff --git a/CHANGES.md b/CHANGES.md index 6a774efe3..5148fd852 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -6,6 +6,12 @@ Notable changes between versions. * Update Calico from v3.6.0 to v3.6.1 +#### AWS + +* Add ability to load balance TCP applications ([#443](https://github.com/poseidon/typhoon/pull/443)) + * Output the network load balancer ARN as `nlb_id` + * Accept a `worker_target_groups` (ARN) list to which worker instances should be added + #### Google Cloud * Add ability to load balance TCP/UDP applications ([#442](https://github.com/poseidon/typhoon/pull/442)) diff --git a/aws/container-linux/kubernetes/outputs.tf b/aws/container-linux/kubernetes/outputs.tf index d042a3d4b..ac66d0fd1 100644 --- a/aws/container-linux/kubernetes/outputs.tf +++ b/aws/container-linux/kubernetes/outputs.tf @@ -37,6 +37,11 @@ output "kubeconfig" { # Outputs for custom load balancing +output "nlb_id" { + description = "ARN of the Network Load Balancer" + value = "${aws_lb.nlb.id}" +} + output "worker_target_group_http" { description = "ARN of a target group of workers for HTTP traffic" value = "${module.workers.target_group_http}" diff --git a/aws/container-linux/kubernetes/variables.tf b/aws/container-linux/kubernetes/variables.tf index c1912c99b..10e5dce3f 100644 --- a/aws/container-linux/kubernetes/variables.tf +++ b/aws/container-linux/kubernetes/variables.tf @@ -71,6 +71,12 @@ variable "worker_price" { description = "Spot price in USD for autoscaling group spot instances. Leave as default empty string for autoscaling group to use on-demand instances. Note, switching in-place from spot to on-demand is not possible: https://github.com/terraform-providers/terraform-provider-aws/issues/4320" } +variable "worker_target_groups" { + type = "list" + description = "Additional target group ARNs to which worker instances should be added" + default = [] +} + variable "controller_clc_snippets" { type = "list" description = "Controller Container Linux Config snippets" diff --git a/aws/container-linux/kubernetes/workers.tf b/aws/container-linux/kubernetes/workers.tf index 0d39e0134..987d8d135 100644 --- a/aws/container-linux/kubernetes/workers.tf +++ b/aws/container-linux/kubernetes/workers.tf @@ -11,6 +11,7 @@ module "workers" { os_image = "${var.os_image}" disk_size = "${var.disk_size}" spot_price = "${var.worker_price}" + target_groups = ["${var.worker_target_groups}"] # configuration kubeconfig = "${module.bootkube.kubeconfig-kubelet}" diff --git a/aws/container-linux/kubernetes/workers/variables.tf b/aws/container-linux/kubernetes/workers/variables.tf index c412a4b33..253b3dc94 100644 --- a/aws/container-linux/kubernetes/workers/variables.tf +++ b/aws/container-linux/kubernetes/workers/variables.tf @@ -64,6 +64,12 @@ variable "spot_price" { description = "Spot price in USD for autoscaling group spot instances. Leave as default empty string for autoscaling group to use on-demand instances. Note, switching in-place from spot to on-demand is not possible: https://github.com/terraform-providers/terraform-provider-aws/issues/4320" } +variable "target_groups" { + type = "list" + description = "Additional target group ARNs to which instances should be added" + default = [] +} + variable "clc_snippets" { type = "list" description = "Container Linux Config snippets" diff --git a/aws/container-linux/kubernetes/workers/workers.tf b/aws/container-linux/kubernetes/workers/workers.tf index 2e1dbbfe3..ef21c8258 100644 --- a/aws/container-linux/kubernetes/workers/workers.tf +++ b/aws/container-linux/kubernetes/workers/workers.tf @@ -19,6 +19,7 @@ resource "aws_autoscaling_group" "workers" { target_group_arns = [ "${aws_lb_target_group.workers-http.id}", "${aws_lb_target_group.workers-https.id}", + "${var.target_groups}", ] lifecycle { diff --git a/docs/cl/aws.md b/docs/cl/aws.md index 7acd7fd85..59b802542 100644 --- a/docs/cl/aws.md +++ b/docs/cl/aws.md @@ -242,6 +242,7 @@ Reference the DNS zone id with `"${aws_route53_zone.zone-for-clusters.zone_id}"` | disk_size | Size of the EBS volume in GB | "40" | "100" | | disk_type | Type of the EBS volume | "gp2" | standard, gp2, io1 | | disk_iops | IOPS of the EBS volume | "0" (i.e. auto) | "400" | +| worker_target_groups | Target group ARNs to which worker instances should be added | [] | ["${aws_lb_target_group.app.id}"] | | worker_price | Spot price in USD for workers. Leave as default empty string for regular on-demand instances | "" | "0.10" | | controller_clc_snippets | Controller Container Linux Config snippets | [] | [example](/advanced/customization/) | | worker_clc_snippets | Worker Container Linux Config snippets | [] | [example](/advanced/customization/) | From 2a07c975388d5073900348cf08cf528070736eb2 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Tue, 2 Apr 2019 23:02:04 -0700 Subject: [PATCH 068/523] Harden internal firewall rules on DigitalOcean * Define firewall rules on DigitialOcean to match rules used on AWS, GCP, and Azure * Output `controller_tag` and `worker_tag` to simplify custom firewall rule creation --- CHANGES.md | 5 ++ .../container-linux/kubernetes/network.tf | 77 ++++++++++++++----- .../container-linux/kubernetes/outputs.tf | 13 ++++ 3 files changed, 77 insertions(+), 18 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 5148fd852..b3592f945 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -12,6 +12,11 @@ Notable changes between versions. * Output the network load balancer ARN as `nlb_id` * Accept a `worker_target_groups` (ARN) list to which worker instances should be added +#### DigitalOcean + +* Harden internal (node-to-node) firewall rules to align with other platforms +* Output `controller_tag` and `worker_tag` to simplify custom firewall rule creation + #### Google Cloud * Add ability to load balance TCP/UDP applications ([#442](https://github.com/poseidon/typhoon/pull/442)) diff --git a/digital-ocean/container-linux/kubernetes/network.tf b/digital-ocean/container-linux/kubernetes/network.tf index 312d7966e..27a4e0c28 100644 --- a/digital-ocean/container-linux/kubernetes/network.tf +++ b/digital-ocean/container-linux/kubernetes/network.tf @@ -3,36 +3,26 @@ resource "digitalocean_firewall" "rules" { tags = ["${var.cluster_name}-controller", "${var.cluster_name}-worker"] - # allow ssh, apiserver, http/https ingress, and peer-to-peer traffic + # allow ssh, internal flannel, internal node-exporter, internal kubelet inbound_rule = [ { protocol = "tcp" port_range = "22" source_addresses = ["0.0.0.0/0", "::/0"] }, - { - protocol = "tcp" - port_range = "80" - source_addresses = ["0.0.0.0/0", "::/0"] - }, - { - protocol = "tcp" - port_range = "443" - source_addresses = ["0.0.0.0/0", "::/0"] - }, - { - protocol = "tcp" - port_range = "6443" - source_addresses = ["0.0.0.0/0", "::/0"] - }, { protocol = "udp" - port_range = "1-65535" + port_range = "8472" source_tags = ["${digitalocean_tag.controllers.name}", "${digitalocean_tag.workers.name}"] }, { protocol = "tcp" - port_range = "1-65535" + port_range = "9100" + source_tags = ["${digitalocean_tag.workers.name}"] + }, + { + protocol = "tcp" + port_range = "10250" source_tags = ["${digitalocean_tag.controllers.name}", "${digitalocean_tag.workers.name}"] }, ] @@ -56,3 +46,54 @@ resource "digitalocean_firewall" "rules" { }, ] } + +resource "digitalocean_firewall" "controllers" { + name = "${var.cluster_name}-controllers" + + tags = ["${var.cluster_name}-controller"] + + # etcd, kube-apiserver, kubelet + inbound_rule = [ + { + protocol = "tcp" + port_range = "2379-2380" + source_tags = ["${digitalocean_tag.controllers.name}"] + }, + { + protocol = "tcp" + port_range = "2381" + source_tags = ["${digitalocean_tag.workers.name}"] + }, + { + protocol = "tcp" + port_range = "6443" + source_addresses = ["0.0.0.0/0", "::/0"] + }, + ] +} + +resource "digitalocean_firewall" "workers" { + name = "${var.cluster_name}-workers" + + tags = ["${var.cluster_name}-worker"] + + # allow HTTP/HTTPS ingress + inbound_rule = [ + { + protocol = "tcp" + port_range = "80" + source_addresses = ["0.0.0.0/0", "::/0"] + }, + { + protocol = "tcp" + port_range = "443" + source_addresses = ["0.0.0.0/0", "::/0"] + }, + { + protocol = "tcp" + port_range = "10254" + source_addresses = ["0.0.0.0/0"] + }, + ] +} + diff --git a/digital-ocean/container-linux/kubernetes/outputs.tf b/digital-ocean/container-linux/kubernetes/outputs.tf index 86a4c76ac..7ca6b81cf 100644 --- a/digital-ocean/container-linux/kubernetes/outputs.tf +++ b/digital-ocean/container-linux/kubernetes/outputs.tf @@ -26,3 +26,16 @@ output "workers_ipv4" { output "workers_ipv6" { value = ["${digitalocean_droplet.workers.*.ipv6_address}"] } + +# Outputs for custom firewalls + +output "controller_tag" { + description = "Tag applied to controller droplets" + value = "${digitalocean_tag.controllers.name}" +} + +output "worker_tag" { + description = "Tag applied to worker droplets" + value = "${digitalocean_tag.workers.name}" +} + From 3e7a38cb136d4d3f0c948d9df2a1713517722969 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Wed, 3 Apr 2019 20:44:24 -0700 Subject: [PATCH 069/523] Update Grafana from v6.0.2 to v6.1.0 * https://github.com/grafana/grafana/releases/tag/v6.1.0 --- CHANGES.md | 1 + addons/grafana/deployment.yaml | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index b3592f945..bc4206baf 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -29,6 +29,7 @@ Notable changes between versions. #### Addons * Update Prometheus from v2.8.0 to v2.8.1 +* Update Grafana from v6.0.2 to [v6.1.0](http://docs.grafana.org/guides/whats-new-in-v6-1/) ## v1.13.5 diff --git a/addons/grafana/deployment.yaml b/addons/grafana/deployment.yaml index a72972b1a..0a3f450df 100644 --- a/addons/grafana/deployment.yaml +++ b/addons/grafana/deployment.yaml @@ -23,7 +23,7 @@ spec: spec: containers: - name: grafana - image: grafana/grafana:6.0.2 + image: grafana/grafana:6.1.0 env: - name: GF_PATHS_CONFIG value: "/etc/grafana/custom.ini" From 29a303524514c2f6df9701768651da6e5eb359cc Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Sat, 6 Apr 2019 18:31:43 -0700 Subject: [PATCH 070/523] Update Grafana from v6.1.0 to v6.1.1 --- CHANGES.md | 2 +- addons/grafana/deployment.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index bc4206baf..0e4ef9762 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -29,7 +29,7 @@ Notable changes between versions. #### Addons * Update Prometheus from v2.8.0 to v2.8.1 -* Update Grafana from v6.0.2 to [v6.1.0](http://docs.grafana.org/guides/whats-new-in-v6-1/) +* Update Grafana from v6.0.2 to [v6.1.1](http://docs.grafana.org/guides/whats-new-in-v6-1/) ## v1.13.5 diff --git a/addons/grafana/deployment.yaml b/addons/grafana/deployment.yaml index 0a3f450df..5dbbdffa9 100644 --- a/addons/grafana/deployment.yaml +++ b/addons/grafana/deployment.yaml @@ -23,7 +23,7 @@ spec: spec: containers: - name: grafana - image: grafana/grafana:6.1.0 + image: grafana/grafana:6.1.1 env: - name: GF_PATHS_CONFIG value: "/etc/grafana/custom.ini" From ce78d5988e355bf21882b337b9cc180c9478ef20 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Sat, 6 Apr 2019 23:27:11 -0700 Subject: [PATCH 071/523] Refresh Prometheus rules and Grafana dashboards * Refresh rules and dashboards from upstreams * Add new Kubernetes "workload" dashboards * View pods in a workload (deployment/daemonset/statefulset) * View workloads in a namespace --- CHANGES.md | 2 + addons/grafana/dashboards-k8s-resources.yaml | 5270 ++++++++++++++++++ addons/grafana/dashboards-k8s.yaml | 3302 +---------- addons/grafana/deployment.yaml | 5 + addons/prometheus/rules.yaml | 41 +- 5 files changed, 5329 insertions(+), 3291 deletions(-) create mode 100644 addons/grafana/dashboards-k8s-resources.yaml diff --git a/CHANGES.md b/CHANGES.md index 0e4ef9762..ef835fad9 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -30,6 +30,8 @@ Notable changes between versions. * Update Prometheus from v2.8.0 to v2.8.1 * Update Grafana from v6.0.2 to [v6.1.1](http://docs.grafana.org/guides/whats-new-in-v6-1/) + * Add dashboard for pods in a workload (deployment/daemonset/statefulset) ([#446](https://github.com/poseidon/typhoon/pull/446)) + * Add dashboard for workloads by namespace ## v1.13.5 diff --git a/addons/grafana/dashboards-k8s-resources.yaml b/addons/grafana/dashboards-k8s-resources.yaml new file mode 100644 index 000000000..eedcc7e46 --- /dev/null +++ b/addons/grafana/dashboards-k8s-resources.yaml @@ -0,0 +1,5270 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: grafana-dashboards-k8s-resources + namespace: monitoring +data: + k8s-resources-cluster.json: |- + { + "annotations": { + "list": [ + + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "links": [ + + ], + "refresh": "10s", + "rows": [ + { + "collapse": false, + "height": "100px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "format": "percentunit", + "id": 1, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "1 - avg(rate(node_cpu_seconds_total{mode=\"idle\", cluster=\"$cluster\"}[1m]))", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "refId": "A" + } + ], + "thresholds": "70,80", + "timeFrom": null, + "timeShift": null, + "title": "CPU Utilisation", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "singlestat", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "format": "percentunit", + "id": 2, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(kube_pod_container_resource_requests_cpu_cores{cluster=\"$cluster\"}) / sum(node:node_num_cpu:sum{cluster=\"$cluster\"})", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "refId": "A" + } + ], + "thresholds": "70,80", + "timeFrom": null, + "timeShift": null, + "title": "CPU Requests Commitment", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "singlestat", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "format": "percentunit", + "id": 3, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(kube_pod_container_resource_limits_cpu_cores{cluster=\"$cluster\"}) / sum(node:node_num_cpu:sum{cluster=\"$cluster\"})", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "refId": "A" + } + ], + "thresholds": "70,80", + "timeFrom": null, + "timeShift": null, + "title": "CPU Limits Commitment", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "singlestat", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "format": "percentunit", + "id": 4, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "1 - sum(:node_memory_MemFreeCachedBuffers_bytes:sum{cluster=\"$cluster\"}) / sum(:node_memory_MemTotal_bytes:sum{cluster=\"$cluster\"})", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "refId": "A" + } + ], + "thresholds": "70,80", + "timeFrom": null, + "timeShift": null, + "title": "Memory Utilisation", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "singlestat", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "format": "percentunit", + "id": 5, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(kube_pod_container_resource_requests_memory_bytes{cluster=\"$cluster\"}) / sum(:node_memory_MemTotal_bytes:sum{cluster=\"$cluster\"})", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "refId": "A" + } + ], + "thresholds": "70,80", + "timeFrom": null, + "timeShift": null, + "title": "Memory Requests Commitment", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "singlestat", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "format": "percentunit", + "id": 6, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(kube_pod_container_resource_limits_memory_bytes{cluster=\"$cluster\"}) / sum(:node_memory_MemTotal_bytes:sum{cluster=\"$cluster\"})", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "refId": "A" + } + ], + "thresholds": "70,80", + "timeFrom": null, + "timeShift": null, + "title": "Memory Limits Commitment", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "singlestat", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Headlines", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 7, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(namespace_pod_name_container_name:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\"}) by (namespace)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{namespace}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU Usage", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "CPU", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 8, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "Pods", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 0, + "link": true, + "linkTooltip": "Drill down to pods", + "linkUrl": "/d/85a562078cdf77779eaa1add43ccec1e/k8s-resources-namespace?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$__cell_1", + "pattern": "Value #A", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "Workloads", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 0, + "link": true, + "linkTooltip": "Drill down to workloads", + "linkUrl": "/d/a87fb0d919ec0ea5f6543124e16c42a5/k8s-resources-workloads-namespace?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$__cell_1", + "pattern": "Value #B", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "CPU Usage", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #C", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "CPU Requests", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #D", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "CPU Requests %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #E", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "CPU Limits", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #F", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "CPU Limits %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #G", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "Namespace", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": true, + "linkTooltip": "Drill down to pods", + "linkUrl": "/d/85a562078cdf77779eaa1add43ccec1e/k8s-resources-namespace?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$__cell", + "pattern": "namespace", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "/.*/", + "thresholds": [ + + ], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "expr": "count(mixin_pod_workload{cluster=\"$cluster\"}) by (namespace)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10 + }, + { + "expr": "count(avg(mixin_pod_workload{cluster=\"$cluster\"}) by (workload, namespace)) by (namespace)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + }, + { + "expr": "sum(namespace_pod_name_container_name:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\"}) by (namespace)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "C", + "step": 10 + }, + { + "expr": "sum(kube_pod_container_resource_requests_cpu_cores{cluster=\"$cluster\"}) by (namespace)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "D", + "step": 10 + }, + { + "expr": "sum(namespace_pod_name_container_name:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\"}) by (namespace) / sum(kube_pod_container_resource_requests_cpu_cores{cluster=\"$cluster\"}) by (namespace)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "E", + "step": 10 + }, + { + "expr": "sum(kube_pod_container_resource_limits_cpu_cores{cluster=\"$cluster\"}) by (namespace)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "F", + "step": 10 + }, + { + "expr": "sum(namespace_pod_name_container_name:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\"}) by (namespace) / sum(kube_pod_container_resource_limits_cpu_cores{cluster=\"$cluster\"}) by (namespace)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "G", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU Quota", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "transform": "table", + "type": "table", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "CPU Quota", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 9, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(container_memory_rss{cluster=\"$cluster\", container_name!=\"\"}) by (namespace)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{namespace}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Memory Usage (w/o cache)", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Memory", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 10, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "Pods", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 0, + "link": true, + "linkTooltip": "Drill down to pods", + "linkUrl": "/d/85a562078cdf77779eaa1add43ccec1e/k8s-resources-namespace?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$__cell_1", + "pattern": "Value #A", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "Workloads", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 0, + "link": true, + "linkTooltip": "Drill down to workloads", + "linkUrl": "/d/a87fb0d919ec0ea5f6543124e16c42a5/k8s-resources-workloads-namespace?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$__cell_1", + "pattern": "Value #B", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "CPU Usage", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #C", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "Memory Usage", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #D", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Requests", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #E", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Requests %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #F", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "Memory Limits", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #G", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Limits %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #H", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "Namespace", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": true, + "linkTooltip": "Drill down to pods", + "linkUrl": "/d/85a562078cdf77779eaa1add43ccec1e/k8s-resources-namespace?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$__cell", + "pattern": "namespace", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "/.*/", + "thresholds": [ + + ], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "expr": "count(mixin_pod_workload{cluster=\"$cluster\"}) by (namespace)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10 + }, + { + "expr": "count(avg(mixin_pod_workload{cluster=\"$cluster\"}) by (workload, namespace)) by (namespace)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + }, + { + "expr": "sum(container_memory_rss{cluster=\"$cluster\", container_name!=\"\"}) by (namespace)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "C", + "step": 10 + }, + { + "expr": "sum(kube_pod_container_resource_requests_memory_bytes{cluster=\"$cluster\"}) by (namespace)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "D", + "step": 10 + }, + { + "expr": "sum(container_memory_rss{cluster=\"$cluster\", container_name!=\"\"}) by (namespace) / sum(kube_pod_container_resource_requests_memory_bytes{cluster=\"$cluster\"}) by (namespace)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "E", + "step": 10 + }, + { + "expr": "sum(kube_pod_container_resource_limits_memory_bytes{cluster=\"$cluster\"}) by (namespace)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "F", + "step": 10 + }, + { + "expr": "sum(container_memory_rss{cluster=\"$cluster\", container_name!=\"\"}) by (namespace) / sum(kube_pod_container_resource_limits_memory_bytes{cluster=\"$cluster\"}) by (namespace)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "G", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Requests by Namespace", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "transform": "table", + "type": "table", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Memory Requests", + "titleSize": "h6" + } + ], + "schemaVersion": 14, + "style": "dark", + "tags": [ + "kubernetes-mixin" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": null, + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + "text": "prod", + "value": "prod" + }, + "datasource": "$datasource", + "hide": 2, + "includeAll": false, + "label": "cluster", + "multi": false, + "name": "cluster", + "options": [ + + ], + "query": "label_values(node_cpu_seconds_total, cluster)", + "refresh": 1, + "regex": "", + "sort": 2, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "", + "title": "Kubernetes / Compute Resources / Cluster", + "uid": "efa86fd1d0c121a26444b636a3f509a8", + "version": 0 + } + k8s-resources-namespace.json: |- + { + "annotations": { + "list": [ + + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "links": [ + + ], + "refresh": "10s", + "rows": [ + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 1, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(namespace_pod_name_container_name:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod_name)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{pod_name}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU Usage", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "CPU Usage", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 2, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "CPU Usage", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #A", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "CPU Requests", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #B", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "CPU Requests %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #C", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "CPU Limits", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #D", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "CPU Limits %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #E", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "Pod", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": true, + "linkTooltip": "Drill down", + "linkUrl": "/d/6581e46e4e5c7ba40a07646395ef7b23/k8s-resources-pod?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$namespace&var-pod=$__cell", + "pattern": "pod", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "/.*/", + "thresholds": [ + + ], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "expr": "sum(label_replace(namespace_pod_name_container_name:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\"}, \"pod\", \"$1\", \"pod_name\", \"(.*)\")) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10 + }, + { + "expr": "sum(kube_pod_container_resource_requests_cpu_cores{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + }, + { + "expr": "sum(label_replace(namespace_pod_name_container_name:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\"}, \"pod\", \"$1\", \"pod_name\", \"(.*)\")) by (pod) / sum(kube_pod_container_resource_requests_cpu_cores{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "C", + "step": 10 + }, + { + "expr": "sum(kube_pod_container_resource_limits_cpu_cores{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "D", + "step": 10 + }, + { + "expr": "sum(label_replace(namespace_pod_name_container_name:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\"}, \"pod\", \"$1\", \"pod_name\", \"(.*)\")) by (pod) / sum(kube_pod_container_resource_limits_cpu_cores{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "E", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU Quota", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "transform": "table", + "type": "table", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "CPU Quota", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 3, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(container_memory_usage_bytes{cluster=\"$cluster\", namespace=\"$namespace\", container_name!=\"\"}) by (pod_name)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{pod_name}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Memory Usage (w/o cache)", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Memory Usage", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 4, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "Memory Usage", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #A", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Requests", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #B", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Requests %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #C", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "Memory Limits", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #D", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Limits %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #E", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "Memory Usage (RSS)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #F", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Usage (Cache)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #G", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Usage (Swap", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #H", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Pod", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": true, + "linkTooltip": "Drill down", + "linkUrl": "/d/6581e46e4e5c7ba40a07646395ef7b23/k8s-resources-pod?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$namespace&var-pod=$__cell", + "pattern": "pod", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "/.*/", + "thresholds": [ + + ], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "expr": "sum(label_replace(container_memory_usage_bytes{cluster=\"$cluster\", namespace=\"$namespace\",container_name!=\"\"}, \"pod\", \"$1\", \"pod_name\", \"(.*)\")) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10 + }, + { + "expr": "sum(kube_pod_container_resource_requests_memory_bytes{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + }, + { + "expr": "sum(label_replace(container_memory_usage_bytes{cluster=\"$cluster\", namespace=\"$namespace\",container_name!=\"\"}, \"pod\", \"$1\", \"pod_name\", \"(.*)\")) by (pod) / sum(kube_pod_container_resource_requests_memory_bytes{namespace=\"$namespace\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "C", + "step": 10 + }, + { + "expr": "sum(kube_pod_container_resource_limits_memory_bytes{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "D", + "step": 10 + }, + { + "expr": "sum(label_replace(container_memory_usage_bytes{cluster=\"$cluster\", namespace=\"$namespace\",container_name!=\"\"}, \"pod\", \"$1\", \"pod_name\", \"(.*)\")) by (pod) / sum(kube_pod_container_resource_limits_memory_bytes{namespace=\"$namespace\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "E", + "step": 10 + }, + { + "expr": "sum(label_replace(container_memory_rss{cluster=\"$cluster\", namespace=\"$namespace\",container_name!=\"\"}, \"pod\", \"$1\", \"pod_name\", \"(.*)\")) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "F", + "step": 10 + }, + { + "expr": "sum(label_replace(container_memory_cache{cluster=\"$cluster\", namespace=\"$namespace\",container_name!=\"\"}, \"pod\", \"$1\", \"pod_name\", \"(.*)\")) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "G", + "step": 10 + }, + { + "expr": "sum(label_replace(container_memory_swap{cluster=\"$cluster\", namespace=\"$namespace\",container_name!=\"\"}, \"pod\", \"$1\", \"pod_name\", \"(.*)\")) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "H", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Memory Quota", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "transform": "table", + "type": "table", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Memory Quota", + "titleSize": "h6" + } + ], + "schemaVersion": 14, + "style": "dark", + "tags": [ + "kubernetes-mixin" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": null, + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + "text": "prod", + "value": "prod" + }, + "datasource": "$datasource", + "hide": 2, + "includeAll": false, + "label": "cluster", + "multi": false, + "name": "cluster", + "options": [ + + ], + "query": "label_values(kube_pod_info, cluster)", + "refresh": 1, + "regex": "", + "sort": 2, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + "text": "prod", + "value": "prod" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": "namespace", + "multi": false, + "name": "namespace", + "options": [ + + ], + "query": "label_values(kube_pod_info{cluster=\"$cluster\"}, namespace)", + "refresh": 1, + "regex": "", + "sort": 2, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "", + "title": "Kubernetes / Compute Resources / Namespace (Pods)", + "uid": "85a562078cdf77779eaa1add43ccec1e", + "version": 0 + } + k8s-resources-pod.json: |- + { + "annotations": { + "list": [ + + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "links": [ + + ], + "refresh": "10s", + "rows": [ + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 1, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(namespace_pod_name_container_name:container_cpu_usage_seconds_total:sum_rate{namespace=\"$namespace\", pod_name=\"$pod\", container_name!=\"POD\", cluster=\"$cluster\"}) by (container_name)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{container_name}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU Usage", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "CPU Usage", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 2, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "CPU Usage", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #A", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "CPU Requests", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #B", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "CPU Requests %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #C", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "CPU Limits", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #D", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "CPU Limits %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #E", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "Container", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "container", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "/.*/", + "thresholds": [ + + ], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "expr": "sum(label_replace(namespace_pod_name_container_name:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\", pod_name=\"$pod\", container_name!=\"POD\"}, \"container\", \"$1\", \"container_name\", \"(.*)\")) by (container)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10 + }, + { + "expr": "sum(kube_pod_container_resource_requests_cpu_cores{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + }, + { + "expr": "sum(label_replace(namespace_pod_name_container_name:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\", pod_name=\"$pod\"}, \"container\", \"$1\", \"container_name\", \"(.*)\")) by (container) / sum(kube_pod_container_resource_requests_cpu_cores{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "C", + "step": 10 + }, + { + "expr": "sum(kube_pod_container_resource_limits_cpu_cores{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "D", + "step": 10 + }, + { + "expr": "sum(label_replace(namespace_pod_name_container_name:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\", pod_name=\"$pod\"}, \"container\", \"$1\", \"container_name\", \"(.*)\")) by (container) / sum(kube_pod_container_resource_limits_cpu_cores{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "E", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU Quota", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "transform": "table", + "type": "table", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "CPU Quota", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 3, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(container_memory_rss{cluster=\"$cluster\", namespace=\"$namespace\", pod_name=\"$pod\", container_name!=\"POD\", container_name!=\"\"}) by (container_name)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{container_name}} (RSS)", + "legendLink": null, + "step": 10 + }, + { + "expr": "sum(container_memory_cache{cluster=\"$cluster\", namespace=\"$namespace\", pod_name=\"$pod\", container_name!=\"POD\", container_name!=\"\"}) by (container_name)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{container_name}} (Cache)", + "legendLink": null, + "step": 10 + }, + { + "expr": "sum(container_memory_swap{cluster=\"$cluster\", namespace=\"$namespace\", pod_name=\"$pod\", container_name!=\"POD\", container_name!=\"\"}) by (container_name)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{container_name}} (Swap)", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Memory Usage", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Memory Usage", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 4, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "Memory Usage", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #A", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Requests", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #B", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Requests %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #C", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "Memory Limits", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #D", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Limits %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #E", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "Memory Usage (RSS)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #F", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Usage (Cache)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #G", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Usage (Swap", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #H", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Container", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "container", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "/.*/", + "thresholds": [ + + ], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "expr": "sum(label_replace(container_memory_usage_bytes{cluster=\"$cluster\", namespace=\"$namespace\", pod_name=\"$pod\", container_name!=\"POD\", container_name!=\"\"}, \"container\", \"$1\", \"container_name\", \"(.*)\")) by (container)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10 + }, + { + "expr": "sum(kube_pod_container_resource_requests_memory_bytes{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + }, + { + "expr": "sum(label_replace(container_memory_usage_bytes{cluster=\"$cluster\", namespace=\"$namespace\", pod_name=\"$pod\"}, \"container\", \"$1\", \"container_name\", \"(.*)\")) by (container) / sum(kube_pod_container_resource_requests_memory_bytes{namespace=\"$namespace\", pod=\"$pod\"}) by (container)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "C", + "step": 10 + }, + { + "expr": "sum(kube_pod_container_resource_limits_memory_bytes{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\", container!=\"\"}) by (container)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "D", + "step": 10 + }, + { + "expr": "sum(label_replace(container_memory_usage_bytes{cluster=\"$cluster\", namespace=\"$namespace\", pod_name=\"$pod\", container_name!=\"\"}, \"container\", \"$1\", \"container_name\", \"(.*)\")) by (container) / sum(kube_pod_container_resource_limits_memory_bytes{namespace=\"$namespace\", pod=\"$pod\"}) by (container)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "E", + "step": 10 + }, + { + "expr": "sum(label_replace(container_memory_rss{cluster=\"$cluster\", namespace=\"$namespace\", pod_name=\"$pod\", container_name != \"\", container_name != \"POD\"}, \"container\", \"$1\", \"container_name\", \"(.*)\")) by (container)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "F", + "step": 10 + }, + { + "expr": "sum(label_replace(container_memory_cache{cluster=\"$cluster\", namespace=\"$namespace\", pod_name=\"$pod\", container_name != \"\", container_name != \"POD\"}, \"container\", \"$1\", \"container_name\", \"(.*)\")) by (container)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "G", + "step": 10 + }, + { + "expr": "sum(label_replace(container_memory_swap{cluster=\"$cluster\", namespace=\"$namespace\", pod_name=\"$pod\", container_name != \"\", container_name != \"POD\"}, \"container\", \"$1\", \"container_name\", \"(.*)\")) by (container)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "H", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Memory Quota", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "transform": "table", + "type": "table", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Memory Quota", + "titleSize": "h6" + } + ], + "schemaVersion": 14, + "style": "dark", + "tags": [ + "kubernetes-mixin" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": null, + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + "text": "prod", + "value": "prod" + }, + "datasource": "$datasource", + "hide": 2, + "includeAll": false, + "label": "cluster", + "multi": false, + "name": "cluster", + "options": [ + + ], + "query": "label_values(kube_pod_info, cluster)", + "refresh": 1, + "regex": "", + "sort": 2, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + "text": "prod", + "value": "prod" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": "namespace", + "multi": false, + "name": "namespace", + "options": [ + + ], + "query": "label_values(kube_pod_info{cluster=\"$cluster\"}, namespace)", + "refresh": 1, + "regex": "", + "sort": 2, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + "text": "prod", + "value": "prod" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": "pod", + "multi": false, + "name": "pod", + "options": [ + + ], + "query": "label_values(kube_pod_info{cluster=\"$cluster\", namespace=\"$namespace\"}, pod)", + "refresh": 1, + "regex": "", + "sort": 2, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "", + "title": "Kubernetes / Compute Resources / Pod", + "uid": "6581e46e4e5c7ba40a07646395ef7b23", + "version": 0 + } + k8s-resources-workload.json: |- + { + "annotations": { + "list": [ + + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "links": [ + + ], + "refresh": "10s", + "rows": [ + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 1, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(\n label_replace(\n namespace_pod_name_container_name:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\"},\n \"pod\", \"$1\", \"pod_name\", \"(.*)\"\n ) * on(namespace,pod) group_left(workload, workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{pod}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU Usage", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "CPU Usage", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 2, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "CPU Usage", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #A", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "CPU Requests", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #B", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "CPU Requests %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #C", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "CPU Limits", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #D", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "CPU Limits %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #E", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "Pod", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": true, + "linkTooltip": "Drill down", + "linkUrl": "/d/6581e46e4e5c7ba40a07646395ef7b23/k8s-resources-pod?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$namespace&var-pod=$__cell", + "pattern": "pod", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "/.*/", + "thresholds": [ + + ], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "expr": "sum(\n label_replace(\n namespace_pod_name_container_name:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\"},\n \"pod\", \"$1\", \"pod_name\", \"(.*)\"\n ) * on(namespace,pod) group_left(workload, workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10 + }, + { + "expr": "sum(\n kube_pod_container_resource_requests_cpu_cores{cluster=\"$cluster\", namespace=\"$namespace\"}\n * on(namespace,pod) group_left(workload, workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + }, + { + "expr": "sum(\n label_replace(\n namespace_pod_name_container_name:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\"},\n \"pod\", \"$1\", \"pod_name\", \"(.*)\"\n ) * on(namespace,pod) group_left(workload, workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n/sum(\n kube_pod_container_resource_requests_cpu_cores{cluster=\"$cluster\", namespace=\"$namespace\"}\n * on(namespace,pod) group_left(workload, workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "C", + "step": 10 + }, + { + "expr": "sum(\n kube_pod_container_resource_limits_cpu_cores{cluster=\"$cluster\", namespace=\"$namespace\"}\n * on(namespace,pod) group_left(workload, workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "D", + "step": 10 + }, + { + "expr": "sum(\n label_replace(\n namespace_pod_name_container_name:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\"},\n \"pod\", \"$1\", \"pod_name\", \"(.*)\"\n ) * on(namespace,pod) group_left(workload, workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n/sum(\n kube_pod_container_resource_limits_cpu_cores{cluster=\"$cluster\", namespace=\"$namespace\"}\n * on(namespace,pod) group_left(workload, workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "E", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU Quota", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "transform": "table", + "type": "table", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "CPU Quota", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 3, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(\n label_replace(\n container_memory_usage_bytes{cluster=\"$cluster\", namespace=\"$namespace\", container_name!=\"\"},\n \"pod\", \"$1\", \"pod_name\", \"(.*)\"\n ) * on(namespace,pod) group_left(workload, workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n ) by (pod)\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{pod}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Memory Usage", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Memory Usage", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 4, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "Memory Usage", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #A", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Requests", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #B", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Requests %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #C", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "Memory Limits", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #D", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Limits %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #E", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "Pod", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": true, + "linkTooltip": "Drill down", + "linkUrl": "/d/6581e46e4e5c7ba40a07646395ef7b23/k8s-resources-pod?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$namespace&var-pod=$__cell", + "pattern": "pod", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "/.*/", + "thresholds": [ + + ], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "expr": "sum(\n label_replace(\n container_memory_usage_bytes{cluster=\"$cluster\", namespace=\"$namespace\", container_name!=\"\"},\n \"pod\", \"$1\", \"pod_name\", \"(.*)\"\n ) * on(namespace,pod) group_left(workload, workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n ) by (pod)\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10 + }, + { + "expr": "sum(\n kube_pod_container_resource_requests_memory_bytes{cluster=\"$cluster\", namespace=\"$namespace\"}\n * on(namespace,pod) group_left(workload, workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + }, + { + "expr": "sum(\n label_replace(\n container_memory_usage_bytes{cluster=\"$cluster\", namespace=\"$namespace\", container_name!=\"\"},\n \"pod\", \"$1\", \"pod_name\", \"(.*)\"\n ) * on(namespace,pod) group_left(workload, workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n ) by (pod)\n/sum(\n kube_pod_container_resource_requests_memory_bytes{cluster=\"$cluster\", namespace=\"$namespace\"}\n * on(namespace,pod) group_left(workload, workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "C", + "step": 10 + }, + { + "expr": "sum(\n kube_pod_container_resource_limits_memory_bytes{cluster=\"$cluster\", namespace=\"$namespace\"}\n * on(namespace,pod) group_left(workload, workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "D", + "step": 10 + }, + { + "expr": "sum(\n label_replace(\n container_memory_usage_bytes{cluster=\"$cluster\", namespace=\"$namespace\", container_name!=\"\"},\n \"pod\", \"$1\", \"pod_name\", \"(.*)\"\n ) * on(namespace,pod) group_left(workload, workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n ) by (pod)\n/sum(\n kube_pod_container_resource_limits_memory_bytes{cluster=\"$cluster\", namespace=\"$namespace\"}\n * on(namespace,pod) group_left(workload, workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "E", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Memory Quota", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "transform": "table", + "type": "table", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Memory Quota", + "titleSize": "h6" + } + ], + "schemaVersion": 14, + "style": "dark", + "tags": [ + "kubernetes-mixin" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": null, + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + "text": "prod", + "value": "prod" + }, + "datasource": "$datasource", + "hide": 2, + "includeAll": false, + "label": "cluster", + "multi": false, + "name": "cluster", + "options": [ + + ], + "query": "label_values(kube_pod_info, cluster)", + "refresh": 1, + "regex": "", + "sort": 2, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + "text": "prod", + "value": "prod" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": "namespace", + "multi": false, + "name": "namespace", + "options": [ + + ], + "query": "label_values(kube_pod_info{cluster=\"$cluster\"}, namespace)", + "refresh": 1, + "regex": "", + "sort": 2, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + "text": "prod", + "value": "prod" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": "workload", + "multi": false, + "name": "workload", + "options": [ + + ], + "query": "label_values(mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\"}, workload)", + "refresh": 1, + "regex": "", + "sort": 2, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + "text": "prod", + "value": "prod" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": "type", + "multi": false, + "name": "type", + "options": [ + + ], + "query": "label_values(mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\"}, workload_type)", + "refresh": 1, + "regex": "", + "sort": 2, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "", + "title": "Kubernetes / Compute Resources / Workload", + "uid": "a164a7f0339f99e89cea5cb47e9be617", + "version": 0 + } + k8s-resources-workloads-namespace.json: |- + { + "annotations": { + "list": [ + + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "links": [ + + ], + "refresh": "10s", + "rows": [ + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 1, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(\n label_replace(\n namespace_pod_name_container_name:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\"},\n \"pod\", \"$1\", \"pod_name\", \"(.*)\"\n ) * on(namespace,pod) group_left(workload, workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\"}\n) by (workload, workload_type)\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{workload}} - {{workload_type}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU Usage", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "CPU Usage", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 2, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "Running Pods", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 0, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #A", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "CPU Usage", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #B", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "CPU Requests", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #C", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "CPU Requests %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #D", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "CPU Limits", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #E", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "CPU Limits %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #F", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "Workload", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": true, + "linkTooltip": "Drill down", + "linkUrl": "/d/a164a7f0339f99e89cea5cb47e9be617/k8s-resources-workload?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$namespace&var-workload=$__cell&var-type=$__cell_2", + "pattern": "workload", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "Workload Type", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "workload_type", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "/.*/", + "thresholds": [ + + ], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "expr": "count(mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\"}) by (workload, workload_type)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10 + }, + { + "expr": "sum(\n label_replace(\n namespace_pod_name_container_name:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\"},\n \"pod\", \"$1\", \"pod_name\", \"(.*)\"\n ) * on(namespace,pod) group_left(workload, workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\"}\n) by (workload, workload_type)\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + }, + { + "expr": "sum(\n kube_pod_container_resource_requests_cpu_cores{cluster=\"$cluster\", namespace=\"$namespace\"}\n * on(namespace,pod) group_left(workload, workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\"}\n) by (workload, workload_type)\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "C", + "step": 10 + }, + { + "expr": "sum(\n label_replace(\n namespace_pod_name_container_name:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\"},\n \"pod\", \"$1\", \"pod_name\", \"(.*)\"\n ) * on(namespace,pod) group_left(workload, workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\"}\n) by (workload, workload_type)\n/sum(\n kube_pod_container_resource_requests_cpu_cores{cluster=\"$cluster\", namespace=\"$namespace\"}\n * on(namespace,pod) group_left(workload, workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\"}\n) by (workload, workload_type)\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "D", + "step": 10 + }, + { + "expr": "sum(\n kube_pod_container_resource_limits_cpu_cores{cluster=\"$cluster\", namespace=\"$namespace\"}\n * on(namespace,pod) group_left(workload, workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\"}\n) by (workload, workload_type)\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "E", + "step": 10 + }, + { + "expr": "sum(\n label_replace(\n namespace_pod_name_container_name:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\"},\n \"pod\", \"$1\", \"pod_name\", \"(.*)\"\n ) * on(namespace,pod) group_left(workload, workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\"}\n) by (workload, workload_type)\n/sum(\n kube_pod_container_resource_limits_cpu_cores{cluster=\"$cluster\", namespace=\"$namespace\"}\n * on(namespace,pod) group_left(workload, workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\"}\n) by (workload, workload_type)\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "F", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU Quota", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "transform": "table", + "type": "table", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "CPU Quota", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 3, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(\n label_replace(\n container_memory_usage_bytes{cluster=\"$cluster\", namespace=\"$namespace\", container_name!=\"\"},\n \"pod\", \"$1\", \"pod_name\", \"(.*)\"\n ) * on(namespace,pod) group_left(workload, workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\"}\n ) by (workload, workload_type)\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{workload}} - {{workload_type}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Memory Usage", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Memory Usage", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 4, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "Running Pods", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 0, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #A", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "Memory Usage", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #B", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Requests", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #C", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Requests %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #D", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "Memory Limits", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #E", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Limits %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #F", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "Workload", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": true, + "linkTooltip": "Drill down", + "linkUrl": "/d/a164a7f0339f99e89cea5cb47e9be617/k8s-resources-workload?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$namespace&var-workload=$__cell&var-type=$__cell_2", + "pattern": "workload", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "Workload Type", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "workload_type", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "/.*/", + "thresholds": [ + + ], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "expr": "count(mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\"}) by (workload, workload_type)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10 + }, + { + "expr": "sum(\n label_replace(\n container_memory_usage_bytes{cluster=\"$cluster\", namespace=\"$namespace\", container_name!=\"\"},\n \"pod\", \"$1\", \"pod_name\", \"(.*)\"\n ) * on(namespace,pod) group_left(workload, workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\"}\n ) by (workload, workload_type)\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + }, + { + "expr": "sum(\n kube_pod_container_resource_requests_memory_bytes{cluster=\"$cluster\", namespace=\"$namespace\"}\n * on(namespace,pod) group_left(workload, workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\"}\n) by (workload, workload_type)\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "C", + "step": 10 + }, + { + "expr": "sum(\n label_replace(\n container_memory_usage_bytes{cluster=\"$cluster\", namespace=\"$namespace\", container_name!=\"\"},\n \"pod\", \"$1\", \"pod_name\", \"(.*)\"\n ) * on(namespace,pod) group_left(workload, workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\"}\n ) by (workload, workload_type)\n/sum(\n kube_pod_container_resource_requests_memory_bytes{cluster=\"$cluster\", namespace=\"$namespace\"}\n * on(namespace,pod) group_left(workload, workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\"}\n) by (workload, workload_type)\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "D", + "step": 10 + }, + { + "expr": "sum(\n kube_pod_container_resource_limits_memory_bytes{cluster=\"$cluster\", namespace=\"$namespace\"}\n * on(namespace,pod) group_left(workload, workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\"}\n) by (workload, workload_type)\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "E", + "step": 10 + }, + { + "expr": "sum(\n label_replace(\n container_memory_usage_bytes{cluster=\"$cluster\", namespace=\"$namespace\", container_name!=\"\"},\n \"pod\", \"$1\", \"pod_name\", \"(.*)\"\n ) * on(namespace,pod) group_left(workload, workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\"}\n ) by (workload, workload_type)\n/sum(\n kube_pod_container_resource_limits_memory_bytes{cluster=\"$cluster\", namespace=\"$namespace\"}\n * on(namespace,pod) group_left(workload, workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\"}\n) by (workload, workload_type)\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "F", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Memory Quota", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "transform": "table", + "type": "table", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Memory Quota", + "titleSize": "h6" + } + ], + "schemaVersion": 14, + "style": "dark", + "tags": [ + "kubernetes-mixin" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": null, + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + "text": "prod", + "value": "prod" + }, + "datasource": "$datasource", + "hide": 2, + "includeAll": false, + "label": "cluster", + "multi": false, + "name": "cluster", + "options": [ + + ], + "query": "label_values(kube_pod_info, cluster)", + "refresh": 1, + "regex": "", + "sort": 2, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + "text": "prod", + "value": "prod" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": "namespace", + "multi": false, + "name": "namespace", + "options": [ + + ], + "query": "label_values(kube_pod_info{cluster=\"$cluster\"}, namespace)", + "refresh": 1, + "regex": "", + "sort": 2, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "", + "title": "Kubernetes / Compute Resources / Namespace (Workloads)", + "uid": "a87fb0d919ec0ea5f6543124e16c42a5", + "version": 0 + } diff --git a/addons/grafana/dashboards-k8s.yaml b/addons/grafana/dashboards-k8s.yaml index 7c0e429cf..c8322aeb2 100644 --- a/addons/grafana/dashboards-k8s.yaml +++ b/addons/grafana/dashboards-k8s.yaml @@ -1905,3280 +1905,6 @@ data: "uid": "4ac4f123aae0ff6dbaf4f4f66120033b", "version": 0 } - k8s-resources-cluster.json: |- - { - "annotations": { - "list": [ - - ] - }, - "editable": true, - "gnetId": null, - "graphTooltip": 0, - "hideControls": false, - "links": [ - - ], - "refresh": "10s", - "rows": [ - { - "collapse": false, - "height": "100px", - "panels": [ - { - "aliasColors": { - - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 1, - "format": "percentunit", - "id": 1, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [ - - ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - - ], - "spaceLength": 10, - "span": 2, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "1 - avg(rate(node_cpu_seconds_total{mode=\"idle\", cluster=\"$cluster\"}[1m]))", - "format": "time_series", - "instant": true, - "intervalFactor": 2, - "refId": "A" - } - ], - "thresholds": "70,80", - "timeFrom": null, - "timeShift": null, - "title": "CPU Utilisation", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "singlestat", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ - - ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] - }, - { - "aliasColors": { - - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 1, - "format": "percentunit", - "id": 2, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [ - - ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - - ], - "spaceLength": 10, - "span": 2, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum(kube_pod_container_resource_requests_cpu_cores{cluster=\"$cluster\"}) / sum(node:node_num_cpu:sum{cluster=\"$cluster\"})", - "format": "time_series", - "instant": true, - "intervalFactor": 2, - "refId": "A" - } - ], - "thresholds": "70,80", - "timeFrom": null, - "timeShift": null, - "title": "CPU Requests Commitment", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "singlestat", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ - - ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] - }, - { - "aliasColors": { - - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 1, - "format": "percentunit", - "id": 3, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [ - - ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - - ], - "spaceLength": 10, - "span": 2, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum(kube_pod_container_resource_limits_cpu_cores{cluster=\"$cluster\"}) / sum(node:node_num_cpu:sum{cluster=\"$cluster\"})", - "format": "time_series", - "instant": true, - "intervalFactor": 2, - "refId": "A" - } - ], - "thresholds": "70,80", - "timeFrom": null, - "timeShift": null, - "title": "CPU Limits Commitment", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "singlestat", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ - - ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] - }, - { - "aliasColors": { - - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 1, - "format": "percentunit", - "id": 4, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [ - - ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - - ], - "spaceLength": 10, - "span": 2, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "1 - sum(:node_memory_MemFreeCachedBuffers_bytes:sum{cluster=\"$cluster\"}) / sum(:node_memory_MemTotal_bytes:sum{cluster=\"$cluster\"})", - "format": "time_series", - "instant": true, - "intervalFactor": 2, - "refId": "A" - } - ], - "thresholds": "70,80", - "timeFrom": null, - "timeShift": null, - "title": "Memory Utilisation", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "singlestat", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ - - ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] - }, - { - "aliasColors": { - - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 1, - "format": "percentunit", - "id": 5, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [ - - ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - - ], - "spaceLength": 10, - "span": 2, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum(kube_pod_container_resource_requests_memory_bytes{cluster=\"$cluster\"}) / sum(:node_memory_MemTotal_bytes:sum{cluster=\"$cluster\"})", - "format": "time_series", - "instant": true, - "intervalFactor": 2, - "refId": "A" - } - ], - "thresholds": "70,80", - "timeFrom": null, - "timeShift": null, - "title": "Memory Requests Commitment", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "singlestat", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ - - ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] - }, - { - "aliasColors": { - - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 1, - "format": "percentunit", - "id": 6, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [ - - ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - - ], - "spaceLength": 10, - "span": 2, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum(kube_pod_container_resource_limits_memory_bytes{cluster=\"$cluster\"}) / sum(:node_memory_MemTotal_bytes:sum{cluster=\"$cluster\"})", - "format": "time_series", - "instant": true, - "intervalFactor": 2, - "refId": "A" - } - ], - "thresholds": "70,80", - "timeFrom": null, - "timeShift": null, - "title": "Memory Limits Commitment", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "singlestat", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ - - ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] - } - ], - "repeat": null, - "repeatIteration": null, - "repeatRowId": null, - "showTitle": false, - "title": "Headlines", - "titleSize": "h6" - }, - { - "collapse": false, - "height": "250px", - "panels": [ - { - "aliasColors": { - - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 10, - "id": 7, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 0, - "links": [ - - ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - - ], - "spaceLength": 10, - "span": 12, - "stack": true, - "steppedLine": false, - "targets": [ - { - "expr": "sum(namespace_pod_name_container_name:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\"}) by (namespace)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{namespace}}", - "legendLink": null, - "step": 10 - } - ], - "thresholds": [ - - ], - "timeFrom": null, - "timeShift": null, - "title": "CPU Usage", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ - - ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] - } - ], - "repeat": null, - "repeatIteration": null, - "repeatRowId": null, - "showTitle": true, - "title": "CPU", - "titleSize": "h6" - }, - { - "collapse": false, - "height": "250px", - "panels": [ - { - "aliasColors": { - - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 1, - "id": 8, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [ - - ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - - ], - "spaceLength": 10, - "span": 12, - "stack": false, - "steppedLine": false, - "styles": [ - { - "alias": "Time", - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "pattern": "Time", - "type": "hidden" - }, - { - "alias": "CPU Usage", - "colorMode": null, - "colors": [ - - ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "link": false, - "linkTooltip": "Drill down", - "linkUrl": "", - "pattern": "Value #A", - "thresholds": [ - - ], - "type": "number", - "unit": "short" - }, - { - "alias": "CPU Requests", - "colorMode": null, - "colors": [ - - ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "link": false, - "linkTooltip": "Drill down", - "linkUrl": "", - "pattern": "Value #B", - "thresholds": [ - - ], - "type": "number", - "unit": "short" - }, - { - "alias": "CPU Requests %", - "colorMode": null, - "colors": [ - - ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "link": false, - "linkTooltip": "Drill down", - "linkUrl": "", - "pattern": "Value #C", - "thresholds": [ - - ], - "type": "number", - "unit": "percentunit" - }, - { - "alias": "CPU Limits", - "colorMode": null, - "colors": [ - - ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "link": false, - "linkTooltip": "Drill down", - "linkUrl": "", - "pattern": "Value #D", - "thresholds": [ - - ], - "type": "number", - "unit": "short" - }, - { - "alias": "CPU Limits %", - "colorMode": null, - "colors": [ - - ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "link": false, - "linkTooltip": "Drill down", - "linkUrl": "", - "pattern": "Value #E", - "thresholds": [ - - ], - "type": "number", - "unit": "percentunit" - }, - { - "alias": "Namespace", - "colorMode": null, - "colors": [ - - ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "link": true, - "linkTooltip": "Drill down", - "linkUrl": "/d/85a562078cdf77779eaa1add43ccec1e/k8s-resources-namespace?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$__cell", - "pattern": "namespace", - "thresholds": [ - - ], - "type": "number", - "unit": "short" - }, - { - "alias": "", - "colorMode": null, - "colors": [ - - ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "pattern": "/.*/", - "thresholds": [ - - ], - "type": "string", - "unit": "short" - } - ], - "targets": [ - { - "expr": "sum(namespace_pod_name_container_name:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\"}) by (namespace)", - "format": "table", - "instant": true, - "intervalFactor": 2, - "legendFormat": "", - "refId": "A", - "step": 10 - }, - { - "expr": "sum(kube_pod_container_resource_requests_cpu_cores{cluster=\"$cluster\"}) by (namespace)", - "format": "table", - "instant": true, - "intervalFactor": 2, - "legendFormat": "", - "refId": "B", - "step": 10 - }, - { - "expr": "sum(namespace_pod_name_container_name:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\"}) by (namespace) / sum(kube_pod_container_resource_requests_cpu_cores{cluster=\"$cluster\"}) by (namespace)", - "format": "table", - "instant": true, - "intervalFactor": 2, - "legendFormat": "", - "refId": "C", - "step": 10 - }, - { - "expr": "sum(kube_pod_container_resource_limits_cpu_cores{cluster=\"$cluster\"}) by (namespace)", - "format": "table", - "instant": true, - "intervalFactor": 2, - "legendFormat": "", - "refId": "D", - "step": 10 - }, - { - "expr": "sum(namespace_pod_name_container_name:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\"}) by (namespace) / sum(kube_pod_container_resource_limits_cpu_cores{cluster=\"$cluster\"}) by (namespace)", - "format": "table", - "instant": true, - "intervalFactor": 2, - "legendFormat": "", - "refId": "E", - "step": 10 - } - ], - "thresholds": [ - - ], - "timeFrom": null, - "timeShift": null, - "title": "CPU Quota", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "transform": "table", - "type": "table", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ - - ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] - } - ], - "repeat": null, - "repeatIteration": null, - "repeatRowId": null, - "showTitle": true, - "title": "CPU Quota", - "titleSize": "h6" - }, - { - "collapse": false, - "height": "250px", - "panels": [ - { - "aliasColors": { - - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 10, - "id": 9, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 0, - "links": [ - - ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - - ], - "spaceLength": 10, - "span": 12, - "stack": true, - "steppedLine": false, - "targets": [ - { - "expr": "sum(container_memory_rss{cluster=\"$cluster\", container_name!=\"\"}) by (namespace)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{namespace}}", - "legendLink": null, - "step": 10 - } - ], - "thresholds": [ - - ], - "timeFrom": null, - "timeShift": null, - "title": "Memory Usage (w/o cache)", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ - - ] - }, - "yaxes": [ - { - "format": "bytes", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] - } - ], - "repeat": null, - "repeatIteration": null, - "repeatRowId": null, - "showTitle": true, - "title": "Memory", - "titleSize": "h6" - }, - { - "collapse": false, - "height": "250px", - "panels": [ - { - "aliasColors": { - - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 1, - "id": 10, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [ - - ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - - ], - "spaceLength": 10, - "span": 12, - "stack": false, - "steppedLine": false, - "styles": [ - { - "alias": "Time", - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "pattern": "Time", - "type": "hidden" - }, - { - "alias": "Memory Usage", - "colorMode": null, - "colors": [ - - ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "link": false, - "linkTooltip": "Drill down", - "linkUrl": "", - "pattern": "Value #A", - "thresholds": [ - - ], - "type": "number", - "unit": "bytes" - }, - { - "alias": "Memory Requests", - "colorMode": null, - "colors": [ - - ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "link": false, - "linkTooltip": "Drill down", - "linkUrl": "", - "pattern": "Value #B", - "thresholds": [ - - ], - "type": "number", - "unit": "bytes" - }, - { - "alias": "Memory Requests %", - "colorMode": null, - "colors": [ - - ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "link": false, - "linkTooltip": "Drill down", - "linkUrl": "", - "pattern": "Value #C", - "thresholds": [ - - ], - "type": "number", - "unit": "percentunit" - }, - { - "alias": "Memory Limits", - "colorMode": null, - "colors": [ - - ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "link": false, - "linkTooltip": "Drill down", - "linkUrl": "", - "pattern": "Value #D", - "thresholds": [ - - ], - "type": "number", - "unit": "bytes" - }, - { - "alias": "Memory Limits %", - "colorMode": null, - "colors": [ - - ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "link": false, - "linkTooltip": "Drill down", - "linkUrl": "", - "pattern": "Value #E", - "thresholds": [ - - ], - "type": "number", - "unit": "percentunit" - }, - { - "alias": "Namespace", - "colorMode": null, - "colors": [ - - ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "link": true, - "linkTooltip": "Drill down", - "linkUrl": "/d/85a562078cdf77779eaa1add43ccec1e/k8s-resources-namespace?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$__cell", - "pattern": "namespace", - "thresholds": [ - - ], - "type": "number", - "unit": "short" - }, - { - "alias": "", - "colorMode": null, - "colors": [ - - ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "pattern": "/.*/", - "thresholds": [ - - ], - "type": "string", - "unit": "short" - } - ], - "targets": [ - { - "expr": "sum(container_memory_rss{cluster=\"$cluster\", container_name!=\"\"}) by (namespace)", - "format": "table", - "instant": true, - "intervalFactor": 2, - "legendFormat": "", - "refId": "A", - "step": 10 - }, - { - "expr": "sum(kube_pod_container_resource_requests_memory_bytes{cluster=\"$cluster\"}) by (namespace)", - "format": "table", - "instant": true, - "intervalFactor": 2, - "legendFormat": "", - "refId": "B", - "step": 10 - }, - { - "expr": "sum(container_memory_rss{cluster=\"$cluster\", container_name!=\"\"}) by (namespace) / sum(kube_pod_container_resource_requests_memory_bytes{cluster=\"$cluster\"}) by (namespace)", - "format": "table", - "instant": true, - "intervalFactor": 2, - "legendFormat": "", - "refId": "C", - "step": 10 - }, - { - "expr": "sum(kube_pod_container_resource_limits_memory_bytes{cluster=\"$cluster\"}) by (namespace)", - "format": "table", - "instant": true, - "intervalFactor": 2, - "legendFormat": "", - "refId": "D", - "step": 10 - }, - { - "expr": "sum(container_memory_rss{cluster=\"$cluster\", container_name!=\"\"}) by (namespace) / sum(kube_pod_container_resource_limits_memory_bytes{cluster=\"$cluster\"}) by (namespace)", - "format": "table", - "instant": true, - "intervalFactor": 2, - "legendFormat": "", - "refId": "E", - "step": 10 - } - ], - "thresholds": [ - - ], - "timeFrom": null, - "timeShift": null, - "title": "Requests by Namespace", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "transform": "table", - "type": "table", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ - - ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] - } - ], - "repeat": null, - "repeatIteration": null, - "repeatRowId": null, - "showTitle": true, - "title": "Memory Requests", - "titleSize": "h6" - } - ], - "schemaVersion": 14, - "style": "dark", - "tags": [ - "kubernetes-mixin" - ], - "templating": { - "list": [ - { - "current": { - "text": "Prometheus", - "value": "Prometheus" - }, - "hide": 0, - "label": null, - "name": "datasource", - "options": [ - - ], - "query": "prometheus", - "refresh": 1, - "regex": "", - "type": "datasource" - }, - { - "allValue": null, - "current": { - "text": "prod", - "value": "prod" - }, - "datasource": "$datasource", - "hide": 2, - "includeAll": false, - "label": "cluster", - "multi": false, - "name": "cluster", - "options": [ - - ], - "query": "label_values(node_cpu_seconds_total, cluster)", - "refresh": 1, - "regex": "", - "sort": 2, - "tagValuesQuery": "", - "tags": [ - - ], - "tagsQuery": "", - "type": "query", - "useTags": false - } - ] - }, - "time": { - "from": "now-1h", - "to": "now" - }, - "timepicker": { - "refresh_intervals": [ - "5s", - "10s", - "30s", - "1m", - "5m", - "15m", - "30m", - "1h", - "2h", - "1d" - ], - "time_options": [ - "5m", - "15m", - "1h", - "6h", - "12h", - "24h", - "2d", - "7d", - "30d" - ] - }, - "timezone": "", - "title": "Kubernetes / Compute Resources / Cluster", - "uid": "efa86fd1d0c121a26444b636a3f509a8", - "version": 0 - } - k8s-resources-namespace.json: |- - { - "annotations": { - "list": [ - - ] - }, - "editable": true, - "gnetId": null, - "graphTooltip": 0, - "hideControls": false, - "links": [ - - ], - "refresh": "10s", - "rows": [ - { - "collapse": false, - "height": "250px", - "panels": [ - { - "aliasColors": { - - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 10, - "id": 1, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 0, - "links": [ - - ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - - ], - "spaceLength": 10, - "span": 12, - "stack": true, - "steppedLine": false, - "targets": [ - { - "expr": "sum(namespace_pod_name_container_name:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod_name)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{pod_name}}", - "legendLink": null, - "step": 10 - } - ], - "thresholds": [ - - ], - "timeFrom": null, - "timeShift": null, - "title": "CPU Usage", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ - - ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] - } - ], - "repeat": null, - "repeatIteration": null, - "repeatRowId": null, - "showTitle": true, - "title": "CPU Usage", - "titleSize": "h6" - }, - { - "collapse": false, - "height": "250px", - "panels": [ - { - "aliasColors": { - - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 1, - "id": 2, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [ - - ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - - ], - "spaceLength": 10, - "span": 12, - "stack": false, - "steppedLine": false, - "styles": [ - { - "alias": "Time", - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "pattern": "Time", - "type": "hidden" - }, - { - "alias": "CPU Usage", - "colorMode": null, - "colors": [ - - ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "link": false, - "linkTooltip": "Drill down", - "linkUrl": "", - "pattern": "Value #A", - "thresholds": [ - - ], - "type": "number", - "unit": "short" - }, - { - "alias": "CPU Requests", - "colorMode": null, - "colors": [ - - ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "link": false, - "linkTooltip": "Drill down", - "linkUrl": "", - "pattern": "Value #B", - "thresholds": [ - - ], - "type": "number", - "unit": "short" - }, - { - "alias": "CPU Requests %", - "colorMode": null, - "colors": [ - - ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "link": false, - "linkTooltip": "Drill down", - "linkUrl": "", - "pattern": "Value #C", - "thresholds": [ - - ], - "type": "number", - "unit": "percentunit" - }, - { - "alias": "CPU Limits", - "colorMode": null, - "colors": [ - - ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "link": false, - "linkTooltip": "Drill down", - "linkUrl": "", - "pattern": "Value #D", - "thresholds": [ - - ], - "type": "number", - "unit": "short" - }, - { - "alias": "CPU Limits %", - "colorMode": null, - "colors": [ - - ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "link": false, - "linkTooltip": "Drill down", - "linkUrl": "", - "pattern": "Value #E", - "thresholds": [ - - ], - "type": "number", - "unit": "percentunit" - }, - { - "alias": "Pod", - "colorMode": null, - "colors": [ - - ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "link": true, - "linkTooltip": "Drill down", - "linkUrl": "/d/6581e46e4e5c7ba40a07646395ef7b23/k8s-resources-pod?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$namespace&var-pod=$__cell", - "pattern": "pod", - "thresholds": [ - - ], - "type": "number", - "unit": "short" - }, - { - "alias": "", - "colorMode": null, - "colors": [ - - ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "pattern": "/.*/", - "thresholds": [ - - ], - "type": "string", - "unit": "short" - } - ], - "targets": [ - { - "expr": "sum(label_replace(namespace_pod_name_container_name:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\"}, \"pod\", \"$1\", \"pod_name\", \"(.*)\")) by (pod)", - "format": "table", - "instant": true, - "intervalFactor": 2, - "legendFormat": "", - "refId": "A", - "step": 10 - }, - { - "expr": "sum(kube_pod_container_resource_requests_cpu_cores{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)", - "format": "table", - "instant": true, - "intervalFactor": 2, - "legendFormat": "", - "refId": "B", - "step": 10 - }, - { - "expr": "sum(label_replace(namespace_pod_name_container_name:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\"}, \"pod\", \"$1\", \"pod_name\", \"(.*)\")) by (pod) / sum(kube_pod_container_resource_requests_cpu_cores{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)", - "format": "table", - "instant": true, - "intervalFactor": 2, - "legendFormat": "", - "refId": "C", - "step": 10 - }, - { - "expr": "sum(kube_pod_container_resource_limits_cpu_cores{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)", - "format": "table", - "instant": true, - "intervalFactor": 2, - "legendFormat": "", - "refId": "D", - "step": 10 - }, - { - "expr": "sum(label_replace(namespace_pod_name_container_name:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\"}, \"pod\", \"$1\", \"pod_name\", \"(.*)\")) by (pod) / sum(kube_pod_container_resource_limits_cpu_cores{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)", - "format": "table", - "instant": true, - "intervalFactor": 2, - "legendFormat": "", - "refId": "E", - "step": 10 - } - ], - "thresholds": [ - - ], - "timeFrom": null, - "timeShift": null, - "title": "CPU Quota", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "transform": "table", - "type": "table", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ - - ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] - } - ], - "repeat": null, - "repeatIteration": null, - "repeatRowId": null, - "showTitle": true, - "title": "CPU Quota", - "titleSize": "h6" - }, - { - "collapse": false, - "height": "250px", - "panels": [ - { - "aliasColors": { - - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 10, - "id": 3, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 0, - "links": [ - - ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - - ], - "spaceLength": 10, - "span": 12, - "stack": true, - "steppedLine": false, - "targets": [ - { - "expr": "sum(container_memory_usage_bytes{cluster=\"$cluster\", namespace=\"$namespace\", container_name!=\"\"}) by (pod_name)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{pod_name}}", - "legendLink": null, - "step": 10 - } - ], - "thresholds": [ - - ], - "timeFrom": null, - "timeShift": null, - "title": "Memory Usage (w/o cache)", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ - - ] - }, - "yaxes": [ - { - "format": "bytes", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] - } - ], - "repeat": null, - "repeatIteration": null, - "repeatRowId": null, - "showTitle": true, - "title": "Memory Usage", - "titleSize": "h6" - }, - { - "collapse": false, - "height": "250px", - "panels": [ - { - "aliasColors": { - - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 1, - "id": 4, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [ - - ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - - ], - "spaceLength": 10, - "span": 12, - "stack": false, - "steppedLine": false, - "styles": [ - { - "alias": "Time", - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "pattern": "Time", - "type": "hidden" - }, - { - "alias": "Memory Usage", - "colorMode": null, - "colors": [ - - ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "link": false, - "linkTooltip": "Drill down", - "linkUrl": "", - "pattern": "Value #A", - "thresholds": [ - - ], - "type": "number", - "unit": "bytes" - }, - { - "alias": "Memory Requests", - "colorMode": null, - "colors": [ - - ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "link": false, - "linkTooltip": "Drill down", - "linkUrl": "", - "pattern": "Value #B", - "thresholds": [ - - ], - "type": "number", - "unit": "bytes" - }, - { - "alias": "Memory Requests %", - "colorMode": null, - "colors": [ - - ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "link": false, - "linkTooltip": "Drill down", - "linkUrl": "", - "pattern": "Value #C", - "thresholds": [ - - ], - "type": "number", - "unit": "percentunit" - }, - { - "alias": "Memory Limits", - "colorMode": null, - "colors": [ - - ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "link": false, - "linkTooltip": "Drill down", - "linkUrl": "", - "pattern": "Value #D", - "thresholds": [ - - ], - "type": "number", - "unit": "bytes" - }, - { - "alias": "Memory Limits %", - "colorMode": null, - "colors": [ - - ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "link": false, - "linkTooltip": "Drill down", - "linkUrl": "", - "pattern": "Value #E", - "thresholds": [ - - ], - "type": "number", - "unit": "percentunit" - }, - { - "alias": "Memory Usage (RSS)", - "colorMode": null, - "colors": [ - - ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "link": false, - "linkTooltip": "Drill down", - "linkUrl": "", - "pattern": "Value #F", - "thresholds": [ - - ], - "type": "number", - "unit": "bytes" - }, - { - "alias": "Memory Usage (Cache)", - "colorMode": null, - "colors": [ - - ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "link": false, - "linkTooltip": "Drill down", - "linkUrl": "", - "pattern": "Value #G", - "thresholds": [ - - ], - "type": "number", - "unit": "bytes" - }, - { - "alias": "Memory Usage (Swap", - "colorMode": null, - "colors": [ - - ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "link": false, - "linkTooltip": "Drill down", - "linkUrl": "", - "pattern": "Value #H", - "thresholds": [ - - ], - "type": "number", - "unit": "bytes" - }, - { - "alias": "Pod", - "colorMode": null, - "colors": [ - - ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "link": true, - "linkTooltip": "Drill down", - "linkUrl": "/d/6581e46e4e5c7ba40a07646395ef7b23/k8s-resources-pod?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$namespace&var-pod=$__cell", - "pattern": "pod", - "thresholds": [ - - ], - "type": "number", - "unit": "short" - }, - { - "alias": "", - "colorMode": null, - "colors": [ - - ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "pattern": "/.*/", - "thresholds": [ - - ], - "type": "string", - "unit": "short" - } - ], - "targets": [ - { - "expr": "sum(label_replace(container_memory_usage_bytes{cluster=\"$cluster\", namespace=\"$namespace\",container_name!=\"\"}, \"pod\", \"$1\", \"pod_name\", \"(.*)\")) by (pod)", - "format": "table", - "instant": true, - "intervalFactor": 2, - "legendFormat": "", - "refId": "A", - "step": 10 - }, - { - "expr": "sum(kube_pod_container_resource_requests_memory_bytes{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)", - "format": "table", - "instant": true, - "intervalFactor": 2, - "legendFormat": "", - "refId": "B", - "step": 10 - }, - { - "expr": "sum(label_replace(container_memory_usage_bytes{cluster=\"$cluster\", namespace=\"$namespace\",container_name!=\"\"}, \"pod\", \"$1\", \"pod_name\", \"(.*)\")) by (pod) / sum(kube_pod_container_resource_requests_memory_bytes{namespace=\"$namespace\"}) by (pod)", - "format": "table", - "instant": true, - "intervalFactor": 2, - "legendFormat": "", - "refId": "C", - "step": 10 - }, - { - "expr": "sum(kube_pod_container_resource_limits_memory_bytes{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)", - "format": "table", - "instant": true, - "intervalFactor": 2, - "legendFormat": "", - "refId": "D", - "step": 10 - }, - { - "expr": "sum(label_replace(container_memory_usage_bytes{cluster=\"$cluster\", namespace=\"$namespace\",container_name!=\"\"}, \"pod\", \"$1\", \"pod_name\", \"(.*)\")) by (pod) / sum(kube_pod_container_resource_limits_memory_bytes{namespace=\"$namespace\"}) by (pod)", - "format": "table", - "instant": true, - "intervalFactor": 2, - "legendFormat": "", - "refId": "E", - "step": 10 - }, - { - "expr": "sum(label_replace(container_memory_rss{cluster=\"$cluster\", namespace=\"$namespace\",container_name!=\"\"}, \"pod\", \"$1\", \"pod_name\", \"(.*)\")) by (pod)", - "format": "table", - "instant": true, - "intervalFactor": 2, - "legendFormat": "", - "refId": "F", - "step": 10 - }, - { - "expr": "sum(label_replace(container_memory_cache{cluster=\"$cluster\", namespace=\"$namespace\",container_name!=\"\"}, \"pod\", \"$1\", \"pod_name\", \"(.*)\")) by (pod)", - "format": "table", - "instant": true, - "intervalFactor": 2, - "legendFormat": "", - "refId": "G", - "step": 10 - }, - { - "expr": "sum(label_replace(container_memory_swap{cluster=\"$cluster\", namespace=\"$namespace\",container_name!=\"\"}, \"pod\", \"$1\", \"pod_name\", \"(.*)\")) by (pod)", - "format": "table", - "instant": true, - "intervalFactor": 2, - "legendFormat": "", - "refId": "H", - "step": 10 - } - ], - "thresholds": [ - - ], - "timeFrom": null, - "timeShift": null, - "title": "Memory Quota", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "transform": "table", - "type": "table", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ - - ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] - } - ], - "repeat": null, - "repeatIteration": null, - "repeatRowId": null, - "showTitle": true, - "title": "Memory Quota", - "titleSize": "h6" - } - ], - "schemaVersion": 14, - "style": "dark", - "tags": [ - "kubernetes-mixin" - ], - "templating": { - "list": [ - { - "current": { - "text": "Prometheus", - "value": "Prometheus" - }, - "hide": 0, - "label": null, - "name": "datasource", - "options": [ - - ], - "query": "prometheus", - "refresh": 1, - "regex": "", - "type": "datasource" - }, - { - "allValue": null, - "current": { - "text": "prod", - "value": "prod" - }, - "datasource": "$datasource", - "hide": 2, - "includeAll": false, - "label": "cluster", - "multi": false, - "name": "cluster", - "options": [ - - ], - "query": "label_values(kube_pod_info, cluster)", - "refresh": 1, - "regex": "", - "sort": 2, - "tagValuesQuery": "", - "tags": [ - - ], - "tagsQuery": "", - "type": "query", - "useTags": false - }, - { - "allValue": null, - "current": { - "text": "prod", - "value": "prod" - }, - "datasource": "$datasource", - "hide": 0, - "includeAll": false, - "label": "namespace", - "multi": false, - "name": "namespace", - "options": [ - - ], - "query": "label_values(kube_pod_info{cluster=\"$cluster\"}, namespace)", - "refresh": 1, - "regex": "", - "sort": 2, - "tagValuesQuery": "", - "tags": [ - - ], - "tagsQuery": "", - "type": "query", - "useTags": false - } - ] - }, - "time": { - "from": "now-1h", - "to": "now" - }, - "timepicker": { - "refresh_intervals": [ - "5s", - "10s", - "30s", - "1m", - "5m", - "15m", - "30m", - "1h", - "2h", - "1d" - ], - "time_options": [ - "5m", - "15m", - "1h", - "6h", - "12h", - "24h", - "2d", - "7d", - "30d" - ] - }, - "timezone": "", - "title": "Kubernetes / Compute Resources / Namespace", - "uid": "85a562078cdf77779eaa1add43ccec1e", - "version": 0 - } - k8s-resources-pod.json: |- - { - "annotations": { - "list": [ - - ] - }, - "editable": true, - "gnetId": null, - "graphTooltip": 0, - "hideControls": false, - "links": [ - - ], - "refresh": "10s", - "rows": [ - { - "collapse": false, - "height": "250px", - "panels": [ - { - "aliasColors": { - - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 10, - "id": 1, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 0, - "links": [ - - ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - - ], - "spaceLength": 10, - "span": 12, - "stack": true, - "steppedLine": false, - "targets": [ - { - "expr": "sum(namespace_pod_name_container_name:container_cpu_usage_seconds_total:sum_rate{namespace=\"$namespace\", pod_name=\"$pod\", container_name!=\"POD\", cluster=\"$cluster\"}) by (container_name)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{container_name}}", - "legendLink": null, - "step": 10 - } - ], - "thresholds": [ - - ], - "timeFrom": null, - "timeShift": null, - "title": "CPU Usage", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ - - ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] - } - ], - "repeat": null, - "repeatIteration": null, - "repeatRowId": null, - "showTitle": true, - "title": "CPU Usage", - "titleSize": "h6" - }, - { - "collapse": false, - "height": "250px", - "panels": [ - { - "aliasColors": { - - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 1, - "id": 2, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [ - - ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - - ], - "spaceLength": 10, - "span": 12, - "stack": false, - "steppedLine": false, - "styles": [ - { - "alias": "Time", - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "pattern": "Time", - "type": "hidden" - }, - { - "alias": "CPU Usage", - "colorMode": null, - "colors": [ - - ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "link": false, - "linkTooltip": "Drill down", - "linkUrl": "", - "pattern": "Value #A", - "thresholds": [ - - ], - "type": "number", - "unit": "short" - }, - { - "alias": "CPU Requests", - "colorMode": null, - "colors": [ - - ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "link": false, - "linkTooltip": "Drill down", - "linkUrl": "", - "pattern": "Value #B", - "thresholds": [ - - ], - "type": "number", - "unit": "short" - }, - { - "alias": "CPU Requests %", - "colorMode": null, - "colors": [ - - ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "link": false, - "linkTooltip": "Drill down", - "linkUrl": "", - "pattern": "Value #C", - "thresholds": [ - - ], - "type": "number", - "unit": "percentunit" - }, - { - "alias": "CPU Limits", - "colorMode": null, - "colors": [ - - ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "link": false, - "linkTooltip": "Drill down", - "linkUrl": "", - "pattern": "Value #D", - "thresholds": [ - - ], - "type": "number", - "unit": "short" - }, - { - "alias": "CPU Limits %", - "colorMode": null, - "colors": [ - - ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "link": false, - "linkTooltip": "Drill down", - "linkUrl": "", - "pattern": "Value #E", - "thresholds": [ - - ], - "type": "number", - "unit": "percentunit" - }, - { - "alias": "Container", - "colorMode": null, - "colors": [ - - ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "link": false, - "linkTooltip": "Drill down", - "linkUrl": "", - "pattern": "container", - "thresholds": [ - - ], - "type": "number", - "unit": "short" - }, - { - "alias": "", - "colorMode": null, - "colors": [ - - ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "pattern": "/.*/", - "thresholds": [ - - ], - "type": "string", - "unit": "short" - } - ], - "targets": [ - { - "expr": "sum(label_replace(namespace_pod_name_container_name:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\", pod_name=\"$pod\", container_name!=\"POD\"}, \"container\", \"$1\", \"container_name\", \"(.*)\")) by (container)", - "format": "table", - "instant": true, - "intervalFactor": 2, - "legendFormat": "", - "refId": "A", - "step": 10 - }, - { - "expr": "sum(kube_pod_container_resource_requests_cpu_cores{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container)", - "format": "table", - "instant": true, - "intervalFactor": 2, - "legendFormat": "", - "refId": "B", - "step": 10 - }, - { - "expr": "sum(label_replace(namespace_pod_name_container_name:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\", pod_name=\"$pod\"}, \"container\", \"$1\", \"container_name\", \"(.*)\")) by (container) / sum(kube_pod_container_resource_requests_cpu_cores{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container)", - "format": "table", - "instant": true, - "intervalFactor": 2, - "legendFormat": "", - "refId": "C", - "step": 10 - }, - { - "expr": "sum(kube_pod_container_resource_limits_cpu_cores{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container)", - "format": "table", - "instant": true, - "intervalFactor": 2, - "legendFormat": "", - "refId": "D", - "step": 10 - }, - { - "expr": "sum(label_replace(namespace_pod_name_container_name:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\", pod_name=\"$pod\"}, \"container\", \"$1\", \"container_name\", \"(.*)\")) by (container) / sum(kube_pod_container_resource_limits_cpu_cores{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container)", - "format": "table", - "instant": true, - "intervalFactor": 2, - "legendFormat": "", - "refId": "E", - "step": 10 - } - ], - "thresholds": [ - - ], - "timeFrom": null, - "timeShift": null, - "title": "CPU Quota", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "transform": "table", - "type": "table", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ - - ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] - } - ], - "repeat": null, - "repeatIteration": null, - "repeatRowId": null, - "showTitle": true, - "title": "CPU Quota", - "titleSize": "h6" - }, - { - "collapse": false, - "height": "250px", - "panels": [ - { - "aliasColors": { - - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 10, - "id": 3, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 0, - "links": [ - - ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - - ], - "spaceLength": 10, - "span": 12, - "stack": true, - "steppedLine": false, - "targets": [ - { - "expr": "sum(container_memory_rss{cluster=\"$cluster\", namespace=\"$namespace\", pod_name=\"$pod\", container_name!=\"POD\", container_name!=\"\"}) by (container_name)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{container_name}} (RSS)", - "legendLink": null, - "step": 10 - }, - { - "expr": "sum(container_memory_cache{cluster=\"$cluster\", namespace=\"$namespace\", pod_name=\"$pod\", container_name!=\"POD\", container_name!=\"\"}) by (container_name)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{container_name}} (Cache)", - "legendLink": null, - "step": 10 - }, - { - "expr": "sum(container_memory_swap{cluster=\"$cluster\", namespace=\"$namespace\", pod_name=\"$pod\", container_name!=\"POD\", container_name!=\"\"}) by (container_name)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{container_name}} (Swap)", - "legendLink": null, - "step": 10 - } - ], - "thresholds": [ - - ], - "timeFrom": null, - "timeShift": null, - "title": "Memory Usage", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ - - ] - }, - "yaxes": [ - { - "format": "bytes", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] - } - ], - "repeat": null, - "repeatIteration": null, - "repeatRowId": null, - "showTitle": true, - "title": "Memory Usage", - "titleSize": "h6" - }, - { - "collapse": false, - "height": "250px", - "panels": [ - { - "aliasColors": { - - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 1, - "id": 4, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [ - - ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - - ], - "spaceLength": 10, - "span": 12, - "stack": false, - "steppedLine": false, - "styles": [ - { - "alias": "Time", - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "pattern": "Time", - "type": "hidden" - }, - { - "alias": "Memory Usage", - "colorMode": null, - "colors": [ - - ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "link": false, - "linkTooltip": "Drill down", - "linkUrl": "", - "pattern": "Value #A", - "thresholds": [ - - ], - "type": "number", - "unit": "bytes" - }, - { - "alias": "Memory Requests", - "colorMode": null, - "colors": [ - - ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "link": false, - "linkTooltip": "Drill down", - "linkUrl": "", - "pattern": "Value #B", - "thresholds": [ - - ], - "type": "number", - "unit": "bytes" - }, - { - "alias": "Memory Requests %", - "colorMode": null, - "colors": [ - - ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "link": false, - "linkTooltip": "Drill down", - "linkUrl": "", - "pattern": "Value #C", - "thresholds": [ - - ], - "type": "number", - "unit": "percentunit" - }, - { - "alias": "Memory Limits", - "colorMode": null, - "colors": [ - - ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "link": false, - "linkTooltip": "Drill down", - "linkUrl": "", - "pattern": "Value #D", - "thresholds": [ - - ], - "type": "number", - "unit": "bytes" - }, - { - "alias": "Memory Limits %", - "colorMode": null, - "colors": [ - - ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "link": false, - "linkTooltip": "Drill down", - "linkUrl": "", - "pattern": "Value #E", - "thresholds": [ - - ], - "type": "number", - "unit": "percentunit" - }, - { - "alias": "Memory Usage (RSS)", - "colorMode": null, - "colors": [ - - ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "link": false, - "linkTooltip": "Drill down", - "linkUrl": "", - "pattern": "Value #F", - "thresholds": [ - - ], - "type": "number", - "unit": "bytes" - }, - { - "alias": "Memory Usage (Cache)", - "colorMode": null, - "colors": [ - - ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "link": false, - "linkTooltip": "Drill down", - "linkUrl": "", - "pattern": "Value #G", - "thresholds": [ - - ], - "type": "number", - "unit": "bytes" - }, - { - "alias": "Memory Usage (Swap", - "colorMode": null, - "colors": [ - - ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "link": false, - "linkTooltip": "Drill down", - "linkUrl": "", - "pattern": "Value #H", - "thresholds": [ - - ], - "type": "number", - "unit": "bytes" - }, - { - "alias": "Container", - "colorMode": null, - "colors": [ - - ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "link": false, - "linkTooltip": "Drill down", - "linkUrl": "", - "pattern": "container", - "thresholds": [ - - ], - "type": "number", - "unit": "short" - }, - { - "alias": "", - "colorMode": null, - "colors": [ - - ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "pattern": "/.*/", - "thresholds": [ - - ], - "type": "string", - "unit": "short" - } - ], - "targets": [ - { - "expr": "sum(label_replace(container_memory_usage_bytes{cluster=\"$cluster\", namespace=\"$namespace\", pod_name=\"$pod\", container_name!=\"POD\", container_name!=\"\"}, \"container\", \"$1\", \"container_name\", \"(.*)\")) by (container)", - "format": "table", - "instant": true, - "intervalFactor": 2, - "legendFormat": "", - "refId": "A", - "step": 10 - }, - { - "expr": "sum(kube_pod_container_resource_requests_memory_bytes{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container)", - "format": "table", - "instant": true, - "intervalFactor": 2, - "legendFormat": "", - "refId": "B", - "step": 10 - }, - { - "expr": "sum(label_replace(container_memory_usage_bytes{cluster=\"$cluster\", namespace=\"$namespace\", pod_name=\"$pod\"}, \"container\", \"$1\", \"container_name\", \"(.*)\")) by (container) / sum(kube_pod_container_resource_requests_memory_bytes{namespace=\"$namespace\", pod=\"$pod\"}) by (container)", - "format": "table", - "instant": true, - "intervalFactor": 2, - "legendFormat": "", - "refId": "C", - "step": 10 - }, - { - "expr": "sum(kube_pod_container_resource_limits_memory_bytes{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\", container!=\"\"}) by (container)", - "format": "table", - "instant": true, - "intervalFactor": 2, - "legendFormat": "", - "refId": "D", - "step": 10 - }, - { - "expr": "sum(label_replace(container_memory_usage_bytes{cluster=\"$cluster\", namespace=\"$namespace\", pod_name=\"$pod\", container_name!=\"\"}, \"container\", \"$1\", \"container_name\", \"(.*)\")) by (container) / sum(kube_pod_container_resource_limits_memory_bytes{namespace=\"$namespace\", pod=\"$pod\"}) by (container)", - "format": "table", - "instant": true, - "intervalFactor": 2, - "legendFormat": "", - "refId": "E", - "step": 10 - }, - { - "expr": "sum(label_replace(container_memory_rss{cluster=\"$cluster\", namespace=\"$namespace\", pod_name=\"$pod\", container_name != \"\", container_name != \"POD\"}, \"container\", \"$1\", \"container_name\", \"(.*)\")) by (container)", - "format": "table", - "instant": true, - "intervalFactor": 2, - "legendFormat": "", - "refId": "F", - "step": 10 - }, - { - "expr": "sum(label_replace(container_memory_cache{cluster=\"$cluster\", namespace=\"$namespace\", pod_name=\"$pod\", container_name != \"\", container_name != \"POD\"}, \"container\", \"$1\", \"container_name\", \"(.*)\")) by (container)", - "format": "table", - "instant": true, - "intervalFactor": 2, - "legendFormat": "", - "refId": "G", - "step": 10 - }, - { - "expr": "sum(label_replace(container_memory_swap{cluster=\"$cluster\", namespace=\"$namespace\", pod_name=\"$pod\", container_name != \"\", container_name != \"POD\"}, \"container\", \"$1\", \"container_name\", \"(.*)\")) by (container)", - "format": "table", - "instant": true, - "intervalFactor": 2, - "legendFormat": "", - "refId": "H", - "step": 10 - } - ], - "thresholds": [ - - ], - "timeFrom": null, - "timeShift": null, - "title": "Memory Quota", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "transform": "table", - "type": "table", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ - - ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] - } - ], - "repeat": null, - "repeatIteration": null, - "repeatRowId": null, - "showTitle": true, - "title": "Memory Quota", - "titleSize": "h6" - } - ], - "schemaVersion": 14, - "style": "dark", - "tags": [ - "kubernetes-mixin" - ], - "templating": { - "list": [ - { - "current": { - "text": "Prometheus", - "value": "Prometheus" - }, - "hide": 0, - "label": null, - "name": "datasource", - "options": [ - - ], - "query": "prometheus", - "refresh": 1, - "regex": "", - "type": "datasource" - }, - { - "allValue": null, - "current": { - "text": "prod", - "value": "prod" - }, - "datasource": "$datasource", - "hide": 2, - "includeAll": false, - "label": "cluster", - "multi": false, - "name": "cluster", - "options": [ - - ], - "query": "label_values(kube_pod_info, cluster)", - "refresh": 1, - "regex": "", - "sort": 2, - "tagValuesQuery": "", - "tags": [ - - ], - "tagsQuery": "", - "type": "query", - "useTags": false - }, - { - "allValue": null, - "current": { - "text": "prod", - "value": "prod" - }, - "datasource": "$datasource", - "hide": 0, - "includeAll": false, - "label": "namespace", - "multi": false, - "name": "namespace", - "options": [ - - ], - "query": "label_values(kube_pod_info{cluster=\"$cluster\"}, namespace)", - "refresh": 1, - "regex": "", - "sort": 2, - "tagValuesQuery": "", - "tags": [ - - ], - "tagsQuery": "", - "type": "query", - "useTags": false - }, - { - "allValue": null, - "current": { - "text": "prod", - "value": "prod" - }, - "datasource": "$datasource", - "hide": 0, - "includeAll": false, - "label": "pod", - "multi": false, - "name": "pod", - "options": [ - - ], - "query": "label_values(kube_pod_info{cluster=\"$cluster\", namespace=\"$namespace\"}, pod)", - "refresh": 1, - "regex": "", - "sort": 2, - "tagValuesQuery": "", - "tags": [ - - ], - "tagsQuery": "", - "type": "query", - "useTags": false - } - ] - }, - "time": { - "from": "now-1h", - "to": "now" - }, - "timepicker": { - "refresh_intervals": [ - "5s", - "10s", - "30s", - "1m", - "5m", - "15m", - "30m", - "1h", - "2h", - "1d" - ], - "time_options": [ - "5m", - "15m", - "1h", - "6h", - "12h", - "24h", - "2d", - "7d", - "30d" - ] - }, - "timezone": "", - "title": "Kubernetes / Compute Resources / Pod", - "uid": "6581e46e4e5c7ba40a07646395ef7b23", - "version": 0 - } nodes.json: |- { "__inputs": [ @@ -5471,7 +2197,7 @@ data: ], "timeFrom": null, "timeShift": null, - "title": "CPU Utilizaion", + "title": "CPU Utilization", "tooltip": { "shared": false, "sort": 0, @@ -6821,7 +3547,7 @@ data: "options": [ ], - "query": "label_values(kubelet_volume_stats_capacity_bytes{cluster=\"$cluster\", job=\"kubelet\"}, exported_namespace)", + "query": "label_values(kubelet_volume_stats_capacity_bytes{cluster=\"$cluster\", job=\"kubelet\"}, namespace)", "refresh": 2, "regex": "", "sort": 0, @@ -6847,7 +3573,7 @@ data: "options": [ ], - "query": "label_values(kubelet_volume_stats_capacity_bytes{cluster=\"$cluster\", job=\"kubelet\", exported_namespace=\"$namespace\"}, persistentvolumeclaim)", + "query": "label_values(kubelet_volume_stats_capacity_bytes{cluster=\"$cluster\", job=\"kubelet\", namespace=\"$namespace\"}, persistentvolumeclaim)", "refresh": 2, "regex": "", "sort": 0, @@ -6973,14 +3699,14 @@ data: "refId": "A" }, { - "expr": "sum by(container) (kube_pod_container_resource_requests_memory_bytes{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\", container=~\"$container\"})", + "expr": "sum by(container) (kube_pod_container_resource_requests{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"memory\", pod=\"$pod\", container=~\"$container\"})", "format": "time_series", "intervalFactor": 2, "legendFormat": "Requested: {{ container }}", "refId": "B" }, { - "expr": "sum by(container) (kube_pod_container_resource_limits_memory_bytes{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\", container=~\"$container\"})", + "expr": "sum by(container) (kube_pod_container_resource_limits{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"memory\", pod=\"$pod\", container=~\"$container\"})", "format": "time_series", "intervalFactor": 2, "legendFormat": "Limit: {{ container }}", @@ -7084,11 +3810,25 @@ data: "steppedLine": false, "targets": [ { - "expr": "sum by (container_name) (rate(container_cpu_usage_seconds_total{job=\"kubernetes-cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\", image!=\"\",container_name!=\"POD\",pod_name=\"$pod\"}[1m]))", + "expr": "sum by (container_name) (rate(container_cpu_usage_seconds_total{job=\"kubernetes-cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\", image!=\"\", pod_name=\"$pod\", container_name=~\"$container\", container_name!=\"POD\"}[1m]))", "format": "time_series", "intervalFactor": 2, - "legendFormat": "{{ container_name }}", + "legendFormat": "Current: {{ container_name }}", "refId": "A" + }, + { + "expr": "sum by(container) (kube_pod_container_resource_requests{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"cpu\", pod=\"$pod\", container=~\"$container\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Requested: {{ container }}", + "refId": "B" + }, + { + "expr": "sum by(container) (kube_pod_container_resource_limits{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"cpu\", pod=\"$pod\", container=~\"$container\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Limit: {{ container }}", + "refId": "C" } ], "thresholds": [ diff --git a/addons/grafana/deployment.yaml b/addons/grafana/deployment.yaml index 5dbbdffa9..b6625e8fb 100644 --- a/addons/grafana/deployment.yaml +++ b/addons/grafana/deployment.yaml @@ -58,6 +58,8 @@ spec: mountPath: /etc/grafana/dashboards/etcd - name: dashboards-k8s mountPath: /etc/grafana/dashboards/k8s + - name: dashboards-k8s-resources + mountPath: /etc/grafana/dashboards/k8s-resources volumes: - name: config configMap: @@ -74,4 +76,7 @@ spec: - name: dashboards-k8s configMap: name: grafana-dashboards-k8s + - name: dashboards-k8s-resources + configMap: + name: grafana-dashboards-k8s-resources diff --git a/addons/prometheus/rules.yaml b/addons/prometheus/rules.yaml index 38d5d6084..1b2718167 100644 --- a/addons/prometheus/rules.yaml +++ b/addons/prometheus/rules.yaml @@ -183,12 +183,33 @@ data: "record": "namespace_name:container_memory_usage_bytes:sum" }, { - "expr": "sum by (namespace, label_name) (\n sum(kube_pod_container_resource_requests_memory_bytes{job=\"kube-state-metrics\"}) by (namespace, pod)\n* on (namespace, pod) group_left(label_name)\n label_replace(kube_pod_labels{job=\"kube-state-metrics\"}, \"pod_name\", \"$1\", \"pod\", \"(.*)\")\n)\n", + "expr": "sum by (namespace, label_name) (\n sum(kube_pod_container_resource_requests_memory_bytes{job=\"kube-state-metrics\"} * on (endpoint, instance, job, namespace, pod, service) group_left(phase) (kube_pod_status_phase{phase=~\"^(Pending|Running)$\"} == 1)) by (namespace, pod)\n* on (namespace, pod) group_left(label_name)\n label_replace(kube_pod_labels{job=\"kube-state-metrics\"}, \"pod_name\", \"$1\", \"pod\", \"(.*)\")\n)\n", "record": "namespace_name:kube_pod_container_resource_requests_memory_bytes:sum" }, { - "expr": "sum by (namespace, label_name) (\n sum(kube_pod_container_resource_requests_cpu_cores{job=\"kube-state-metrics\"} and on(pod) kube_pod_status_scheduled{condition=\"true\"}) by (namespace, pod)\n* on (namespace, pod) group_left(label_name)\n label_replace(kube_pod_labels{job=\"kube-state-metrics\"}, \"pod_name\", \"$1\", \"pod\", \"(.*)\")\n)\n", + "expr": "sum by (namespace, label_name) (\n sum(kube_pod_container_resource_requests_cpu_cores{job=\"kube-state-metrics\"} * on (endpoint, instance, job, namespace, pod, service) group_left(phase) (kube_pod_status_phase{phase=~\"^(Pending|Running)$\"} == 1)) by (namespace, pod)\n* on (namespace, pod) group_left(label_name)\n label_replace(kube_pod_labels{job=\"kube-state-metrics\"}, \"pod_name\", \"$1\", \"pod\", \"(.*)\")\n)\n", "record": "namespace_name:kube_pod_container_resource_requests_cpu_cores:sum" + }, + { + "expr": "sum(\n label_replace(\n label_replace(\n kube_pod_owner{job=\"kube-state-metrics\", owner_kind=\"ReplicaSet\"},\n \"replicaset\", \"$1\", \"owner_name\", \"(.*)\"\n ) * on(replicaset, namespace) group_left(owner_name) kube_replicaset_owner{job=\"kube-state-metrics\"},\n \"workload\", \"$1\", \"owner_name\", \"(.*)\"\n )\n) by (namespace, workload, pod)\n", + "labels": { + "workload_type": "deployment" + }, + "record": "mixin_pod_workload" + }, + { + "expr": "sum(\n label_replace(\n kube_pod_owner{job=\"kube-state-metrics\", owner_kind=\"DaemonSet\"},\n \"workload\", \"$1\", \"owner_name\", \"(.*)\"\n )\n) by (namespace, workload, pod)\n", + "labels": { + "workload_type": "daemonset" + }, + "record": "mixin_pod_workload" + }, + { + "expr": "sum(\n label_replace(\n kube_pod_owner{job=\"kube-state-metrics\", owner_kind=\"StatefulSet\"},\n \"workload\", \"$1\", \"owner_name\", \"(.*)\"\n )\n) by (namespace, workload, pod)\n", + "labels": { + "workload_type": "statefulset" + }, + "record": "mixin_pod_workload" } ] }, @@ -374,11 +395,11 @@ data: "record": "node:node_disk_utilisation:avg_irate" }, { - "expr": "avg(irate(node_disk_io_time_weighted_seconds_total{job=\"node-exporter\",device=~\"nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+\"}[1m]) / 1e3)\n", + "expr": "avg(irate(node_disk_io_time_weighted_seconds_total{job=\"node-exporter\",device=~\"nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+\"}[1m]))\n", "record": ":node_disk_saturation:avg_irate" }, { - "expr": "avg by (node) (\n irate(node_disk_io_time_weighted_seconds_total{job=\"node-exporter\",device=~\"nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+\"}[1m]) / 1e3\n* on (namespace, pod) group_left(node)\n node_namespace_pod:kube_pod_info:\n)\n", + "expr": "avg by (node) (\n irate(node_disk_io_time_weighted_seconds_total{job=\"node-exporter\",device=~\"nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+\"}[1m])\n* on (namespace, pod) group_left(node)\n node_namespace_pod:kube_pod_info:\n)\n", "record": "node:node_disk_saturation:avg_irate" }, { @@ -662,7 +683,7 @@ data: "message": "Cluster has overcommitted CPU resource requests for Namespaces.", "runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubecpuovercommit" }, - "expr": "sum(kube_resourcequota{job=\"kube-state-metrics\", type=\"hard\", resource=\"requests.cpu\"})\n /\nsum(node:node_num_cpu:sum)\n > 1.5\n", + "expr": "sum(kube_resourcequota{job=\"kube-state-metrics\", type=\"hard\", resource=\"cpu\"})\n /\nsum(node:node_num_cpu:sum)\n > 1.5\n", "for": "5m", "labels": { "severity": "warning" @@ -674,7 +695,7 @@ data: "message": "Cluster has overcommitted memory resource requests for Namespaces.", "runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubememovercommit" }, - "expr": "sum(kube_resourcequota{job=\"kube-state-metrics\", type=\"hard\", resource=\"requests.memory\"})\n /\nsum(node_memory_MemTotal_bytes{job=\"node-exporter\"})\n > 1.5\n", + "expr": "sum(kube_resourcequota{job=\"kube-state-metrics\", type=\"hard\", resource=\"memory\"})\n /\nsum(node_memory_MemTotal_bytes{job=\"node-exporter\"})\n > 1.5\n", "for": "5m", "labels": { "severity": "warning" @@ -885,10 +906,10 @@ data: { "alert": "KubeClientCertificateExpiration", "annotations": { - "message": "A client certificate used to authenticate to the apiserver is expiring in less than 7 days.", + "message": "A client certificate used to authenticate to the apiserver is expiring in less than 7.0 days.", "runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubeclientcertificateexpiration" }, - "expr": "histogram_quantile(0.01, sum by (job, le) (rate(apiserver_client_certificate_expiration_seconds_bucket{job=\"apiserver\"}[5m]))) < 604800\n", + "expr": "apiserver_client_certificate_expiration_seconds_count{job=\"apiserver\"} > 0 and histogram_quantile(0.01, sum by (job, le) (rate(apiserver_client_certificate_expiration_seconds_bucket{job=\"apiserver\"}[5m]))) < 604800\n", "labels": { "severity": "warning" } @@ -896,10 +917,10 @@ data: { "alert": "KubeClientCertificateExpiration", "annotations": { - "message": "A client certificate used to authenticate to the apiserver is expiring in less than 24 hours.", + "message": "A client certificate used to authenticate to the apiserver is expiring in less than 24.0 hours.", "runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubeclientcertificateexpiration" }, - "expr": "histogram_quantile(0.01, sum by (job, le) (rate(apiserver_client_certificate_expiration_seconds_bucket{job=\"apiserver\"}[5m]))) < 86400\n", + "expr": "apiserver_client_certificate_expiration_seconds_count{job=\"apiserver\"} > 0 and histogram_quantile(0.01, sum by (job, le) (rate(apiserver_client_certificate_expiration_seconds_bucket{job=\"apiserver\"}[5m]))) < 86400\n", "labels": { "severity": "critical" } From 5271e410ebbe4c89a3237e8e6d2eddb4ec614ecf Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Sun, 7 Apr 2019 00:09:00 -0700 Subject: [PATCH 072/523] Update Kubernetes from v1.13.5 to v1.14.0 * https://github.com/kubernetes/kubernetes/blob/master/CHANGELOG-1.14.md#v1140 --- CHANGES.md | 1 + README.md | 10 +++++----- aws/container-linux/kubernetes/README.md | 2 +- aws/container-linux/kubernetes/bootkube.tf | 2 +- .../kubernetes/cl/controller.yaml.tmpl | 2 +- .../kubernetes/workers/cl/worker.yaml.tmpl | 4 ++-- aws/fedora-atomic/kubernetes/README.md | 2 +- aws/fedora-atomic/kubernetes/bootkube.tf | 2 +- .../kubernetes/cloudinit/controller.yaml.tmpl | 2 +- .../workers/cloudinit/worker.yaml.tmpl | 2 +- azure/container-linux/kubernetes/README.md | 2 +- azure/container-linux/kubernetes/bootkube.tf | 2 +- .../kubernetes/cl/controller.yaml.tmpl | 2 +- .../kubernetes/workers/cl/worker.yaml.tmpl | 4 ++-- bare-metal/container-linux/kubernetes/README.md | 2 +- .../container-linux/kubernetes/bootkube.tf | 2 +- .../kubernetes/cl/controller.yaml.tmpl | 2 +- .../kubernetes/cl/worker.yaml.tmpl | 2 +- bare-metal/fedora-atomic/kubernetes/README.md | 2 +- bare-metal/fedora-atomic/kubernetes/bootkube.tf | 2 +- .../kubernetes/cloudinit/controller.yaml.tmpl | 2 +- .../kubernetes/cloudinit/worker.yaml.tmpl | 2 +- .../container-linux/kubernetes/README.md | 2 +- .../container-linux/kubernetes/bootkube.tf | 2 +- .../kubernetes/cl/controller.yaml.tmpl | 2 +- .../kubernetes/cl/worker.yaml.tmpl | 4 ++-- digital-ocean/fedora-atomic/kubernetes/README.md | 2 +- .../fedora-atomic/kubernetes/bootkube.tf | 2 +- .../kubernetes/cloudinit/controller.yaml.tmpl | 2 +- .../kubernetes/cloudinit/worker.yaml.tmpl | 2 +- docs/advanced/worker-pools.md | 16 ++++++++-------- docs/atomic/aws.md | 10 +++++----- docs/atomic/bare-metal.md | 10 +++++----- docs/atomic/digital-ocean.md | 10 +++++----- docs/atomic/google-cloud.md | 10 +++++----- docs/cl/aws.md | 10 +++++----- docs/cl/azure.md | 10 +++++----- docs/cl/bare-metal.md | 14 +++++++------- docs/cl/digital-ocean.md | 10 +++++----- docs/cl/google-cloud.md | 10 +++++----- docs/index.md | 10 +++++----- docs/topics/maintenance.md | 2 +- .../container-linux/kubernetes/README.md | 2 +- .../container-linux/kubernetes/bootkube.tf | 2 +- .../kubernetes/cl/controller.yaml.tmpl | 2 +- .../kubernetes/workers/cl/worker.yaml.tmpl | 4 ++-- google-cloud/fedora-atomic/kubernetes/README.md | 2 +- .../fedora-atomic/kubernetes/bootkube.tf | 2 +- .../kubernetes/cloudinit/controller.yaml.tmpl | 2 +- .../workers/cloudinit/worker.yaml.tmpl | 2 +- 50 files changed, 107 insertions(+), 106 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index ef835fad9..a2b85ff3e 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -4,6 +4,7 @@ Notable changes between versions. ## Latest +* Kubernetes [v1.14.0](https://github.com/kubernetes/kubernetes/blob/master/CHANGELOG-1.14.md#v1140) * Update Calico from v3.6.0 to v3.6.1 #### AWS diff --git a/README.md b/README.md index 55d4b6b62..4861b6cc0 100644 --- a/README.md +++ b/README.md @@ -11,7 +11,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features -* Kubernetes v1.13.5 (upstream, via [kubernetes-incubator/bootkube](https://github.com/kubernetes-incubator/bootkube)) +* Kubernetes v1.14.0 (upstream, via [kubernetes-incubator/bootkube](https://github.com/kubernetes-incubator/bootkube)) * Single or multi-master, [Calico](https://www.projectcalico.org/) or [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/) * Advanced features like [worker pools](https://typhoon.psdn.io/advanced/worker-pools/), [preemptible](https://typhoon.psdn.io/cl/google-cloud/#preemption) workers, and [snippets](https://typhoon.psdn.io/advanced/customization/#container-linux) customization @@ -50,7 +50,7 @@ Define a Kubernetes cluster by using the Terraform module for your chosen platfo ```tf module "google-cloud-yavin" { - source = "git::https://github.com/poseidon/typhoon//google-cloud/container-linux/kubernetes?ref=v1.13.5" + source = "git::https://github.com/poseidon/typhoon//google-cloud/container-linux/kubernetes?ref=v1.14.0" providers = { google = "google.default" @@ -91,9 +91,9 @@ In 4-8 minutes (varies by platform), the cluster will be ready. This Google Clou $ export KUBECONFIG=/home/user/.secrets/clusters/yavin/auth/kubeconfig $ kubectl get nodes NAME ROLES STATUS AGE VERSION -yavin-controller-0.c.example-com.internal controller,master Ready 6m v1.13.5 -yavin-worker-jrbf.c.example-com.internal node Ready 5m v1.13.5 -yavin-worker-mzdm.c.example-com.internal node Ready 5m v1.13.5 +yavin-controller-0.c.example-com.internal controller,master Ready 6m v1.14.0 +yavin-worker-jrbf.c.example-com.internal node Ready 5m v1.14.0 +yavin-worker-mzdm.c.example-com.internal node Ready 5m v1.14.0 ``` List the pods. diff --git a/aws/container-linux/kubernetes/README.md b/aws/container-linux/kubernetes/README.md index 5eeef2945..18239c431 100644 --- a/aws/container-linux/kubernetes/README.md +++ b/aws/container-linux/kubernetes/README.md @@ -11,7 +11,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features -* Kubernetes v1.13.5 (upstream, via [kubernetes-incubator/bootkube](https://github.com/kubernetes-incubator/bootkube)) +* Kubernetes v1.14.0 (upstream, via [kubernetes-incubator/bootkube](https://github.com/kubernetes-incubator/bootkube)) * Single or multi-master, [Calico](https://www.projectcalico.org/) or [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/) * Advanced features like [worker pools](https://typhoon.psdn.io/advanced/worker-pools/), [spot](https://typhoon.psdn.io/cl/aws/#spot) workers, and [snippets](https://typhoon.psdn.io/advanced/customization/#container-linux) customization diff --git a/aws/container-linux/kubernetes/bootkube.tf b/aws/container-linux/kubernetes/bootkube.tf index 0103059b3..c40108978 100644 --- a/aws/container-linux/kubernetes/bootkube.tf +++ b/aws/container-linux/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=bcb015e1052192b166b19a1e3b5da0f25051eaf4" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=ce5db83663b1de2096afc1787c2b622bc08987b3" cluster_name = "${var.cluster_name}" api_servers = ["${format("%s.%s", var.cluster_name, var.dns_zone)}"] diff --git a/aws/container-linux/kubernetes/cl/controller.yaml.tmpl b/aws/container-linux/kubernetes/cl/controller.yaml.tmpl index 7252aa80e..d8248e7e3 100644 --- a/aws/container-linux/kubernetes/cl/controller.yaml.tmpl +++ b/aws/container-linux/kubernetes/cl/controller.yaml.tmpl @@ -123,7 +123,7 @@ storage: contents: inline: | KUBELET_IMAGE_URL=docker://k8s.gcr.io/hyperkube - KUBELET_IMAGE_TAG=v1.13.5 + KUBELET_IMAGE_TAG=v1.14.0 - path: /etc/sysctl.d/max-user-watches.conf filesystem: root contents: diff --git a/aws/container-linux/kubernetes/workers/cl/worker.yaml.tmpl b/aws/container-linux/kubernetes/workers/cl/worker.yaml.tmpl index 9cc13a6e9..a42da3852 100644 --- a/aws/container-linux/kubernetes/workers/cl/worker.yaml.tmpl +++ b/aws/container-linux/kubernetes/workers/cl/worker.yaml.tmpl @@ -93,7 +93,7 @@ storage: contents: inline: | KUBELET_IMAGE_URL=docker://k8s.gcr.io/hyperkube - KUBELET_IMAGE_TAG=v1.13.5 + KUBELET_IMAGE_TAG=v1.14.0 - path: /etc/sysctl.d/max-user-watches.conf filesystem: root contents: @@ -111,7 +111,7 @@ storage: --volume config,kind=host,source=/etc/kubernetes \ --mount volume=config,target=/etc/kubernetes \ --insecure-options=image \ - docker://k8s.gcr.io/hyperkube:v1.13.5 \ + docker://k8s.gcr.io/hyperkube:v1.14.0 \ --net=host \ --dns=host \ --exec=/kubectl -- --kubeconfig=/etc/kubernetes/kubeconfig delete node $(hostname) diff --git a/aws/fedora-atomic/kubernetes/README.md b/aws/fedora-atomic/kubernetes/README.md index 4069a93c8..66a22c3d9 100644 --- a/aws/fedora-atomic/kubernetes/README.md +++ b/aws/fedora-atomic/kubernetes/README.md @@ -11,7 +11,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features -* Kubernetes v1.13.5 (upstream, via [kubernetes-incubator/bootkube](https://github.com/kubernetes-incubator/bootkube)) +* Kubernetes v1.14.0 (upstream, via [kubernetes-incubator/bootkube](https://github.com/kubernetes-incubator/bootkube)) * Single or multi-master, [Calico](https://www.projectcalico.org/) or [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/) * Advanced features like [worker pools](https://typhoon.psdn.io/advanced/worker-pools/) and [spot](https://typhoon.psdn.io/cl/aws/#spot) workers diff --git a/aws/fedora-atomic/kubernetes/bootkube.tf b/aws/fedora-atomic/kubernetes/bootkube.tf index fc45dc315..cfa874170 100644 --- a/aws/fedora-atomic/kubernetes/bootkube.tf +++ b/aws/fedora-atomic/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=bcb015e1052192b166b19a1e3b5da0f25051eaf4" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=ce5db83663b1de2096afc1787c2b622bc08987b3" cluster_name = "${var.cluster_name}" api_servers = ["${format("%s.%s", var.cluster_name, var.dns_zone)}"] diff --git a/aws/fedora-atomic/kubernetes/cloudinit/controller.yaml.tmpl b/aws/fedora-atomic/kubernetes/cloudinit/controller.yaml.tmpl index 8afdfc050..5949d55d2 100644 --- a/aws/fedora-atomic/kubernetes/cloudinit/controller.yaml.tmpl +++ b/aws/fedora-atomic/kubernetes/cloudinit/controller.yaml.tmpl @@ -79,7 +79,7 @@ runcmd: - [systemctl, daemon-reload] - [systemctl, restart, NetworkManager] - "atomic install --system --name=etcd quay.io/poseidon/etcd:v3.3.12" - - "atomic install --system --name=kubelet quay.io/poseidon/kubelet:v1.13.5" + - "atomic install --system --name=kubelet quay.io/poseidon/kubelet:v1.14.0" - "atomic install --system --name=bootkube quay.io/poseidon/bootkube:v0.14.0" - [systemctl, start, --no-block, etcd.service] - [systemctl, start, --no-block, kubelet.service] diff --git a/aws/fedora-atomic/kubernetes/workers/cloudinit/worker.yaml.tmpl b/aws/fedora-atomic/kubernetes/workers/cloudinit/worker.yaml.tmpl index a05cd877c..373a7163a 100644 --- a/aws/fedora-atomic/kubernetes/workers/cloudinit/worker.yaml.tmpl +++ b/aws/fedora-atomic/kubernetes/workers/cloudinit/worker.yaml.tmpl @@ -54,7 +54,7 @@ bootcmd: runcmd: - [systemctl, daemon-reload] - [systemctl, restart, NetworkManager] - - "atomic install --system --name=kubelet quay.io/poseidon/kubelet:v1.13.5" + - "atomic install --system --name=kubelet quay.io/poseidon/kubelet:v1.14.0" - [systemctl, start, --no-block, kubelet.service] users: - default diff --git a/azure/container-linux/kubernetes/README.md b/azure/container-linux/kubernetes/README.md index a8aeacbbd..96ea18d76 100644 --- a/azure/container-linux/kubernetes/README.md +++ b/azure/container-linux/kubernetes/README.md @@ -11,7 +11,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features -* Kubernetes v1.13.5 (upstream, via [kubernetes-incubator/bootkube](https://github.com/kubernetes-incubator/bootkube)) +* Kubernetes v1.14.0 (upstream, via [kubernetes-incubator/bootkube](https://github.com/kubernetes-incubator/bootkube)) * Single or multi-master, [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled * Advanced features like [worker pools](https://typhoon.psdn.io/advanced/worker-pools/), [low-priority](https://typhoon.psdn.io/cl/azure/#low-priority) workers, and [snippets](https://typhoon.psdn.io/advanced/customization/#container-linux) customization diff --git a/azure/container-linux/kubernetes/bootkube.tf b/azure/container-linux/kubernetes/bootkube.tf index 4055d2b2b..7b0bd0627 100644 --- a/azure/container-linux/kubernetes/bootkube.tf +++ b/azure/container-linux/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=bcb015e1052192b166b19a1e3b5da0f25051eaf4" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=ce5db83663b1de2096afc1787c2b622bc08987b3" cluster_name = "${var.cluster_name}" api_servers = ["${format("%s.%s", var.cluster_name, var.dns_zone)}"] diff --git a/azure/container-linux/kubernetes/cl/controller.yaml.tmpl b/azure/container-linux/kubernetes/cl/controller.yaml.tmpl index 7252aa80e..d8248e7e3 100644 --- a/azure/container-linux/kubernetes/cl/controller.yaml.tmpl +++ b/azure/container-linux/kubernetes/cl/controller.yaml.tmpl @@ -123,7 +123,7 @@ storage: contents: inline: | KUBELET_IMAGE_URL=docker://k8s.gcr.io/hyperkube - KUBELET_IMAGE_TAG=v1.13.5 + KUBELET_IMAGE_TAG=v1.14.0 - path: /etc/sysctl.d/max-user-watches.conf filesystem: root contents: diff --git a/azure/container-linux/kubernetes/workers/cl/worker.yaml.tmpl b/azure/container-linux/kubernetes/workers/cl/worker.yaml.tmpl index c5211f278..6459a9661 100644 --- a/azure/container-linux/kubernetes/workers/cl/worker.yaml.tmpl +++ b/azure/container-linux/kubernetes/workers/cl/worker.yaml.tmpl @@ -93,7 +93,7 @@ storage: contents: inline: | KUBELET_IMAGE_URL=docker://k8s.gcr.io/hyperkube - KUBELET_IMAGE_TAG=v1.13.5 + KUBELET_IMAGE_TAG=v1.14.0 - path: /etc/sysctl.d/max-user-watches.conf filesystem: root contents: @@ -111,7 +111,7 @@ storage: --volume config,kind=host,source=/etc/kubernetes \ --mount volume=config,target=/etc/kubernetes \ --insecure-options=image \ - docker://k8s.gcr.io/hyperkube:v1.13.5 \ + docker://k8s.gcr.io/hyperkube:v1.14.0 \ --net=host \ --dns=host \ --exec=/kubectl -- --kubeconfig=/etc/kubernetes/kubeconfig delete node $(hostname | tr '[:upper:]' '[:lower:]') diff --git a/bare-metal/container-linux/kubernetes/README.md b/bare-metal/container-linux/kubernetes/README.md index 7ff0ca7de..8a368327b 100644 --- a/bare-metal/container-linux/kubernetes/README.md +++ b/bare-metal/container-linux/kubernetes/README.md @@ -11,7 +11,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features -* Kubernetes v1.13.5 (upstream, via [kubernetes-incubator/bootkube](https://github.com/kubernetes-incubator/bootkube)) +* Kubernetes v1.14.0 (upstream, via [kubernetes-incubator/bootkube](https://github.com/kubernetes-incubator/bootkube)) * Single or multi-master, [Calico](https://www.projectcalico.org/) or [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/) * Advanced features like [snippets](https://typhoon.psdn.io/advanced/customization/#container-linux) customization diff --git a/bare-metal/container-linux/kubernetes/bootkube.tf b/bare-metal/container-linux/kubernetes/bootkube.tf index e94c60fac..3e834398f 100644 --- a/bare-metal/container-linux/kubernetes/bootkube.tf +++ b/bare-metal/container-linux/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=bcb015e1052192b166b19a1e3b5da0f25051eaf4" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=ce5db83663b1de2096afc1787c2b622bc08987b3" cluster_name = "${var.cluster_name}" api_servers = ["${var.k8s_domain_name}"] diff --git a/bare-metal/container-linux/kubernetes/cl/controller.yaml.tmpl b/bare-metal/container-linux/kubernetes/cl/controller.yaml.tmpl index 7863f04a4..143fc178f 100644 --- a/bare-metal/container-linux/kubernetes/cl/controller.yaml.tmpl +++ b/bare-metal/container-linux/kubernetes/cl/controller.yaml.tmpl @@ -128,7 +128,7 @@ storage: contents: inline: | KUBELET_IMAGE_URL=docker://k8s.gcr.io/hyperkube - KUBELET_IMAGE_TAG=v1.13.5 + KUBELET_IMAGE_TAG=v1.14.0 - path: /etc/hostname filesystem: root mode: 0644 diff --git a/bare-metal/container-linux/kubernetes/cl/worker.yaml.tmpl b/bare-metal/container-linux/kubernetes/cl/worker.yaml.tmpl index 5e5e5bcec..da3b29703 100644 --- a/bare-metal/container-linux/kubernetes/cl/worker.yaml.tmpl +++ b/bare-metal/container-linux/kubernetes/cl/worker.yaml.tmpl @@ -89,7 +89,7 @@ storage: contents: inline: | KUBELET_IMAGE_URL=docker://k8s.gcr.io/hyperkube - KUBELET_IMAGE_TAG=v1.13.5 + KUBELET_IMAGE_TAG=v1.14.0 - path: /etc/hostname filesystem: root mode: 0644 diff --git a/bare-metal/fedora-atomic/kubernetes/README.md b/bare-metal/fedora-atomic/kubernetes/README.md index 65aac5d10..dab414ff7 100644 --- a/bare-metal/fedora-atomic/kubernetes/README.md +++ b/bare-metal/fedora-atomic/kubernetes/README.md @@ -11,7 +11,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features -* Kubernetes v1.13.5 (upstream, via [kubernetes-incubator/bootkube](https://github.com/kubernetes-incubator/bootkube)) +* Kubernetes v1.14.0 (upstream, via [kubernetes-incubator/bootkube](https://github.com/kubernetes-incubator/bootkube)) * Single or multi-master, [Calico](https://www.projectcalico.org/) or [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/) * Ready for Ingress, Prometheus, Grafana, and other optional [addons](https://typhoon.psdn.io/addons/overview/) diff --git a/bare-metal/fedora-atomic/kubernetes/bootkube.tf b/bare-metal/fedora-atomic/kubernetes/bootkube.tf index 117c2c2c7..a6fbb57cf 100644 --- a/bare-metal/fedora-atomic/kubernetes/bootkube.tf +++ b/bare-metal/fedora-atomic/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=bcb015e1052192b166b19a1e3b5da0f25051eaf4" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=ce5db83663b1de2096afc1787c2b622bc08987b3" cluster_name = "${var.cluster_name}" api_servers = ["${var.k8s_domain_name}"] diff --git a/bare-metal/fedora-atomic/kubernetes/cloudinit/controller.yaml.tmpl b/bare-metal/fedora-atomic/kubernetes/cloudinit/controller.yaml.tmpl index 1e23ec0d9..bed3dce5d 100644 --- a/bare-metal/fedora-atomic/kubernetes/cloudinit/controller.yaml.tmpl +++ b/bare-metal/fedora-atomic/kubernetes/cloudinit/controller.yaml.tmpl @@ -85,7 +85,7 @@ runcmd: - [systemctl, restart, NetworkManager] - [hostnamectl, set-hostname, ${domain_name}] - "atomic install --system --name=etcd quay.io/poseidon/etcd:v3.3.12" - - "atomic install --system --name=kubelet quay.io/poseidon/kubelet:v1.13.5" + - "atomic install --system --name=kubelet quay.io/poseidon/kubelet:v1.14.0" - "atomic install --system --name=bootkube quay.io/poseidon/bootkube:v0.14.0" - [systemctl, start, --no-block, etcd.service] - [systemctl, enable, kubelet.path] diff --git a/bare-metal/fedora-atomic/kubernetes/cloudinit/worker.yaml.tmpl b/bare-metal/fedora-atomic/kubernetes/cloudinit/worker.yaml.tmpl index bae7973f0..a01c2cd3a 100644 --- a/bare-metal/fedora-atomic/kubernetes/cloudinit/worker.yaml.tmpl +++ b/bare-metal/fedora-atomic/kubernetes/cloudinit/worker.yaml.tmpl @@ -60,7 +60,7 @@ runcmd: - [systemctl, daemon-reload] - [systemctl, restart, NetworkManager] - [hostnamectl, set-hostname, ${domain_name}] - - "atomic install --system --name=kubelet quay.io/poseidon/kubelet:v1.13.5" + - "atomic install --system --name=kubelet quay.io/poseidon/kubelet:v1.14.0" - [systemctl, enable, kubelet.path] - [systemctl, start, --no-block, kubelet.path] users: diff --git a/digital-ocean/container-linux/kubernetes/README.md b/digital-ocean/container-linux/kubernetes/README.md index 2f8b0a4fd..0e81f7822 100644 --- a/digital-ocean/container-linux/kubernetes/README.md +++ b/digital-ocean/container-linux/kubernetes/README.md @@ -11,7 +11,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features -* Kubernetes v1.13.5 (upstream, via [kubernetes-incubator/bootkube](https://github.com/kubernetes-incubator/bootkube)) +* Kubernetes v1.14.0 (upstream, via [kubernetes-incubator/bootkube](https://github.com/kubernetes-incubator/bootkube)) * Single or multi-master, [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled * Advanced features like [snippets](https://typhoon.psdn.io/advanced/customization/#container-linux) customization diff --git a/digital-ocean/container-linux/kubernetes/bootkube.tf b/digital-ocean/container-linux/kubernetes/bootkube.tf index 1ae13f383..cf67d6724 100644 --- a/digital-ocean/container-linux/kubernetes/bootkube.tf +++ b/digital-ocean/container-linux/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=bcb015e1052192b166b19a1e3b5da0f25051eaf4" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=ce5db83663b1de2096afc1787c2b622bc08987b3" cluster_name = "${var.cluster_name}" api_servers = ["${format("%s.%s", var.cluster_name, var.dns_zone)}"] diff --git a/digital-ocean/container-linux/kubernetes/cl/controller.yaml.tmpl b/digital-ocean/container-linux/kubernetes/cl/controller.yaml.tmpl index 95a34b9aa..1fb193f11 100644 --- a/digital-ocean/container-linux/kubernetes/cl/controller.yaml.tmpl +++ b/digital-ocean/container-linux/kubernetes/cl/controller.yaml.tmpl @@ -129,7 +129,7 @@ storage: contents: inline: | KUBELET_IMAGE_URL=docker://k8s.gcr.io/hyperkube - KUBELET_IMAGE_TAG=v1.13.5 + KUBELET_IMAGE_TAG=v1.14.0 - path: /etc/sysctl.d/max-user-watches.conf filesystem: root contents: diff --git a/digital-ocean/container-linux/kubernetes/cl/worker.yaml.tmpl b/digital-ocean/container-linux/kubernetes/cl/worker.yaml.tmpl index 2c72c238a..51b836752 100644 --- a/digital-ocean/container-linux/kubernetes/cl/worker.yaml.tmpl +++ b/digital-ocean/container-linux/kubernetes/cl/worker.yaml.tmpl @@ -99,7 +99,7 @@ storage: contents: inline: | KUBELET_IMAGE_URL=docker://k8s.gcr.io/hyperkube - KUBELET_IMAGE_TAG=v1.13.5 + KUBELET_IMAGE_TAG=v1.14.0 - path: /etc/sysctl.d/max-user-watches.conf filesystem: root contents: @@ -117,7 +117,7 @@ storage: --volume config,kind=host,source=/etc/kubernetes \ --mount volume=config,target=/etc/kubernetes \ --insecure-options=image \ - docker://k8s.gcr.io/hyperkube:v1.13.5 \ + docker://k8s.gcr.io/hyperkube:v1.14.0 \ --net=host \ --dns=host \ --exec=/kubectl -- --kubeconfig=/etc/kubernetes/kubeconfig delete node $(hostname) diff --git a/digital-ocean/fedora-atomic/kubernetes/README.md b/digital-ocean/fedora-atomic/kubernetes/README.md index 48a6c57d3..eaa5934bd 100644 --- a/digital-ocean/fedora-atomic/kubernetes/README.md +++ b/digital-ocean/fedora-atomic/kubernetes/README.md @@ -11,7 +11,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features -* Kubernetes v1.13.5 (upstream, via [kubernetes-incubator/bootkube](https://github.com/kubernetes-incubator/bootkube)) +* Kubernetes v1.14.0 (upstream, via [kubernetes-incubator/bootkube](https://github.com/kubernetes-incubator/bootkube)) * Single or multi-master, [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled * Ready for Ingress, Prometheus, Grafana, and other optional [addons](https://typhoon.psdn.io/addons/overview/) diff --git a/digital-ocean/fedora-atomic/kubernetes/bootkube.tf b/digital-ocean/fedora-atomic/kubernetes/bootkube.tf index 8f18569a8..41dd57bbc 100644 --- a/digital-ocean/fedora-atomic/kubernetes/bootkube.tf +++ b/digital-ocean/fedora-atomic/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=bcb015e1052192b166b19a1e3b5da0f25051eaf4" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=ce5db83663b1de2096afc1787c2b622bc08987b3" cluster_name = "${var.cluster_name}" api_servers = ["${format("%s.%s", var.cluster_name, var.dns_zone)}"] diff --git a/digital-ocean/fedora-atomic/kubernetes/cloudinit/controller.yaml.tmpl b/digital-ocean/fedora-atomic/kubernetes/cloudinit/controller.yaml.tmpl index 23770028a..71cf564a6 100644 --- a/digital-ocean/fedora-atomic/kubernetes/cloudinit/controller.yaml.tmpl +++ b/digital-ocean/fedora-atomic/kubernetes/cloudinit/controller.yaml.tmpl @@ -91,7 +91,7 @@ bootcmd: runcmd: - [systemctl, daemon-reload] - "atomic install --system --name=etcd quay.io/poseidon/etcd:v3.3.12" - - "atomic install --system --name=kubelet quay.io/poseidon/kubelet:v1.13.5" + - "atomic install --system --name=kubelet quay.io/poseidon/kubelet:v1.14.0" - "atomic install --system --name=bootkube quay.io/poseidon/bootkube:v0.14.0" - [systemctl, start, --no-block, etcd.service] - [systemctl, enable, cloud-metadata.service] diff --git a/digital-ocean/fedora-atomic/kubernetes/cloudinit/worker.yaml.tmpl b/digital-ocean/fedora-atomic/kubernetes/cloudinit/worker.yaml.tmpl index 014af57f1..8db4e0f47 100644 --- a/digital-ocean/fedora-atomic/kubernetes/cloudinit/worker.yaml.tmpl +++ b/digital-ocean/fedora-atomic/kubernetes/cloudinit/worker.yaml.tmpl @@ -67,7 +67,7 @@ bootcmd: runcmd: - [systemctl, daemon-reload] - [systemctl, enable, cloud-metadata.service] - - "atomic install --system --name=kubelet quay.io/poseidon/kubelet:v1.13.5" + - "atomic install --system --name=kubelet quay.io/poseidon/kubelet:v1.14.0" - [systemctl, enable, kubelet.path] - [systemctl, start, --no-block, kubelet.path] users: diff --git a/docs/advanced/worker-pools.md b/docs/advanced/worker-pools.md index 4569e0755..955cd8517 100644 --- a/docs/advanced/worker-pools.md +++ b/docs/advanced/worker-pools.md @@ -16,7 +16,7 @@ Create a cluster following the AWS [tutorial](../cl/aws.md#cluster). Define a wo ```tf module "tempest-worker-pool" { - source = "git::https://github.com/poseidon/typhoon//aws/container-linux/kubernetes/workers?ref=v1.13.5" + source = "git::https://github.com/poseidon/typhoon//aws/container-linux/kubernetes/workers?ref=v1.14.0" providers = { aws = "aws.default" @@ -82,7 +82,7 @@ Create a cluster following the Azure [tutorial](../cl/azure.md#cluster). Define ```tf module "ramius-worker-pool" { - source = "git::https://github.com/poseidon/typhoon//azure/container-linux/kubernetes/workers?ref=v1.13.5" + source = "git::https://github.com/poseidon/typhoon//azure/container-linux/kubernetes/workers?ref=v1.14.0" providers = { azurerm = "azurerm.default" @@ -152,7 +152,7 @@ Create a cluster following the Google Cloud [tutorial](../cl/google-cloud.md#clu ```tf module "yavin-worker-pool" { - source = "git::https://github.com/poseidon/typhoon//google-cloud/container-linux/kubernetes/workers?ref=v1.13.5" + source = "git::https://github.com/poseidon/typhoon//google-cloud/container-linux/kubernetes/workers?ref=v1.14.0" providers = { google = "google.default" @@ -187,11 +187,11 @@ Verify a managed instance group of workers joins the cluster within a few minute ``` $ kubectl get nodes NAME STATUS AGE VERSION -yavin-controller-0.c.example-com.internal Ready 6m v1.13.5 -yavin-worker-jrbf.c.example-com.internal Ready 5m v1.13.5 -yavin-worker-mzdm.c.example-com.internal Ready 5m v1.13.5 -yavin-16x-worker-jrbf.c.example-com.internal Ready 3m v1.13.5 -yavin-16x-worker-mzdm.c.example-com.internal Ready 3m v1.13.5 +yavin-controller-0.c.example-com.internal Ready 6m v1.14.0 +yavin-worker-jrbf.c.example-com.internal Ready 5m v1.14.0 +yavin-worker-mzdm.c.example-com.internal Ready 5m v1.14.0 +yavin-16x-worker-jrbf.c.example-com.internal Ready 3m v1.14.0 +yavin-16x-worker-mzdm.c.example-com.internal Ready 3m v1.14.0 ``` ### Variables diff --git a/docs/atomic/aws.md b/docs/atomic/aws.md index e0f575de9..3200f55c0 100644 --- a/docs/atomic/aws.md +++ b/docs/atomic/aws.md @@ -3,7 +3,7 @@ !!! danger Typhoon for Fedora Atomic will not be updated much beyond Kubernetes v1.13. -In this tutorial, we'll create a Kubernetes v1.13.5 cluster on AWS with Fedora Atomic. +In this tutorial, we'll create a Kubernetes v1.14.0 cluster on AWS with Fedora Atomic. We'll declare a Kubernetes cluster using the Typhoon Terraform module. Then apply the changes to create a VPC, gateway, subnets, security groups, controller instances, worker auto-scaling group, network load balancer, and TLS assets. Instances are provisioned on first boot with cloud-init. @@ -83,7 +83,7 @@ Define a Kubernetes cluster using the module `aws/fedora-atomic/kubernetes`. ```tf module "aws-tempest" { - source = "git::https://github.com/poseidon/typhoon//aws/fedora-atomic/kubernetes?ref=v1.13.5" + source = "git::https://github.com/poseidon/typhoon//aws/fedora-atomic/kubernetes?ref=v1.14.0" providers = { aws = "aws.default" @@ -156,9 +156,9 @@ In 5-10 minutes, the Kubernetes cluster will be ready. $ export KUBECONFIG=/home/user/.secrets/clusters/tempest/auth/kubeconfig $ kubectl get nodes NAME STATUS ROLES AGE VERSION -ip-10-0-3-155 Ready controller,master 10m v1.13.5 -ip-10-0-26-65 Ready node 10m v1.13.5 -ip-10-0-41-21 Ready node 10m v1.13.5 +ip-10-0-3-155 Ready controller,master 10m v1.14.0 +ip-10-0-26-65 Ready node 10m v1.14.0 +ip-10-0-41-21 Ready node 10m v1.14.0 ``` List the pods. diff --git a/docs/atomic/bare-metal.md b/docs/atomic/bare-metal.md index f04acd246..7dde69893 100644 --- a/docs/atomic/bare-metal.md +++ b/docs/atomic/bare-metal.md @@ -3,7 +3,7 @@ !!! danger Typhoon for Fedora Atomic will not be updated much beyond Kubernetes v1.13. -In this tutorial, we'll network boot and provision a Kubernetes v1.13.5 cluster on bare-metal with Fedora Atomic. +In this tutorial, we'll network boot and provision a Kubernetes v1.14.0 cluster on bare-metal with Fedora Atomic. First, we'll deploy a [Matchbox](https://github.com/coreos/matchbox) service and setup a network boot environment. Then, we'll declare a Kubernetes cluster using the Typhoon Terraform module and power on machines. On PXE boot, machines will install Fedora Atomic via kickstart, reboot into the disk install, and provision themselves as Kubernetes controllers or workers via cloud-init. @@ -228,7 +228,7 @@ Define a Kubernetes cluster using the module `bare-metal/fedora-atomic/kubernete ```tf module "bare-metal-mercury" { - source = "git::https://github.com/poseidon/typhoon//bare-metal/fedora-atomic/kubernetes?ref=v1.13.5" + source = "git::https://github.com/poseidon/typhoon//bare-metal/fedora-atomic/kubernetes?ref=v1.14.0" providers = { local = "local.default" @@ -354,9 +354,9 @@ bootkube[5]: Tearing down temporary bootstrap control plane... $ export KUBECONFIG=/home/user/.secrets/clusters/mercury/auth/kubeconfig $ kubectl get nodes NAME STATUS ROLES AGE VERSION -node1.example.com Ready controller,master 10m v1.13.5 -node2.example.com Ready node 10m v1.13.5 -node3.example.com Ready node 10m v1.13.5 +node1.example.com Ready controller,master 10m v1.14.0 +node2.example.com Ready node 10m v1.14.0 +node3.example.com Ready node 10m v1.14.0 ``` List the pods. diff --git a/docs/atomic/digital-ocean.md b/docs/atomic/digital-ocean.md index c33ea5146..d8d97ac7a 100644 --- a/docs/atomic/digital-ocean.md +++ b/docs/atomic/digital-ocean.md @@ -3,7 +3,7 @@ !!! danger Typhoon for Fedora Atomic will not be updated much beyond Kubernetes v1.13. -In this tutorial, we'll create a Kubernetes v1.13.5 cluster on DigitalOcean with Fedora Atomic. +In this tutorial, we'll create a Kubernetes v1.14.0 cluster on DigitalOcean with Fedora Atomic. We'll declare a Kubernetes cluster using the Typhoon Terraform module. Then apply the changes to create controller droplets, worker droplets, DNS records, tags, and TLS assets. Instances are provisioned on first boot with cloud-init. @@ -77,7 +77,7 @@ Define a Kubernetes cluster using the module `digital-ocean/fedora-atomic/kubern ```tf module "digital-ocean-nemo" { - source = "git::https://github.com/poseidon/typhoon//digital-ocean/fedora-atomic/kubernetes?ref=v1.13.5" + source = "git::https://github.com/poseidon/typhoon//digital-ocean/fedora-atomic/kubernetes?ref=v1.14.0" providers = { digitalocean = "digitalocean.default" @@ -152,9 +152,9 @@ In 3-6 minutes, the Kubernetes cluster will be ready. $ export KUBECONFIG=/home/user/.secrets/clusters/nemo/auth/kubeconfig $ kubectl get nodes NAME STATUS ROLES AGE VERSION -10.132.110.130 Ready controller,master 10m v1.13.5 -10.132.115.81 Ready node 10m v1.13.5 -10.132.124.107 Ready node 10m v1.13.5 +10.132.110.130 Ready controller,master 10m v1.14.0 +10.132.115.81 Ready node 10m v1.14.0 +10.132.124.107 Ready node 10m v1.14.0 ``` List the pods. diff --git a/docs/atomic/google-cloud.md b/docs/atomic/google-cloud.md index 111eb4995..fc87174b6 100644 --- a/docs/atomic/google-cloud.md +++ b/docs/atomic/google-cloud.md @@ -3,7 +3,7 @@ !!! danger Typhoon for Fedora Atomic will not be updated much beyond Kubernetes v1.13. Fedora does not publish official images for Google Cloud so you must prepare them yourself. Expect rough edges and changes. -In this tutorial, we'll create a Kubernetes v1.13.5 cluster on Google Compute Engine with Fedora Atomic. +In this tutorial, we'll create a Kubernetes v1.14.0 cluster on Google Compute Engine with Fedora Atomic. We'll declare a Kubernetes cluster using the Typhoon Terraform module. Then apply the changes to create a network, firewall rules, health checks, controller instances, worker managed instance group, load balancers, and TLS assets. Instances are provisioned on first boot with cloud-init. @@ -121,7 +121,7 @@ Define a Kubernetes cluster using the module `google-cloud/fedora-atomic/kuberne ```tf module "google-cloud-yavin" { - source = "git::https://github.com/poseidon/typhoon//google-cloud/fedora-atomic/kubernetes?ref=v1.13.5" + source = "git::https://github.com/poseidon/typhoon//google-cloud/fedora-atomic/kubernetes?ref=v1.14.0" providers = { google = "google.default" @@ -197,9 +197,9 @@ In 5-10 minutes, the Kubernetes cluster will be ready. $ export KUBECONFIG=/home/user/.secrets/clusters/yavin/auth/kubeconfig $ kubectl get nodes NAME ROLES STATUS AGE VERSION -yavin-controller-0.c.example-com.internal controller,master Ready 6m v1.13.5 -yavin-worker-jrbf.c.example-com.internal node Ready 5m v1.13.5 -yavin-worker-mzdm.c.example-com.internal node Ready 5m v1.13.5 +yavin-controller-0.c.example-com.internal controller,master Ready 6m v1.14.0 +yavin-worker-jrbf.c.example-com.internal node Ready 5m v1.14.0 +yavin-worker-mzdm.c.example-com.internal node Ready 5m v1.14.0 ``` List the pods. diff --git a/docs/cl/aws.md b/docs/cl/aws.md index 59b802542..a7607f878 100644 --- a/docs/cl/aws.md +++ b/docs/cl/aws.md @@ -1,6 +1,6 @@ # AWS -In this tutorial, we'll create a Kubernetes v1.13.5 cluster on AWS with Container Linux. +In this tutorial, we'll create a Kubernetes v1.14.0 cluster on AWS with Container Linux. We'll declare a Kubernetes cluster using the Typhoon Terraform module. Then apply the changes to create a VPC, gateway, subnets, security groups, controller instances, worker auto-scaling group, network load balancer, and TLS assets. @@ -92,7 +92,7 @@ Define a Kubernetes cluster using the module `aws/container-linux/kubernetes`. ```tf module "aws-tempest" { - source = "git::https://github.com/poseidon/typhoon//aws/container-linux/kubernetes?ref=v1.13.5" + source = "git::https://github.com/poseidon/typhoon//aws/container-linux/kubernetes?ref=v1.14.0" providers = { aws = "aws.default" @@ -165,9 +165,9 @@ In 4-8 minutes, the Kubernetes cluster will be ready. $ export KUBECONFIG=/home/user/.secrets/clusters/tempest/auth/kubeconfig $ kubectl get nodes NAME STATUS ROLES AGE VERSION -ip-10-0-3-155 Ready controller,master 10m v1.13.5 -ip-10-0-26-65 Ready node 10m v1.13.5 -ip-10-0-41-21 Ready node 10m v1.13.5 +ip-10-0-3-155 Ready controller,master 10m v1.14.0 +ip-10-0-26-65 Ready node 10m v1.14.0 +ip-10-0-41-21 Ready node 10m v1.14.0 ``` List the pods. diff --git a/docs/cl/azure.md b/docs/cl/azure.md index 5f128d996..590a5ef5a 100644 --- a/docs/cl/azure.md +++ b/docs/cl/azure.md @@ -3,7 +3,7 @@ !!! danger Typhoon for Azure is alpha. For production, use AWS, Google Cloud, or bare-metal. As Azure matures, check [errata](https://github.com/poseidon/typhoon/wiki/Errata) for known shortcomings. -In this tutorial, we'll create a Kubernetes v1.13.5 cluster on Azure with Container Linux. +In this tutorial, we'll create a Kubernetes v1.14.0 cluster on Azure with Container Linux. We'll declare a Kubernetes cluster using the Typhoon Terraform module. Then apply the changes to create a resource group, virtual network, subnets, security groups, controller availability set, worker scale set, load balancer, and TLS assets. @@ -87,7 +87,7 @@ Define a Kubernetes cluster using the module `azure/container-linux/kubernetes`. ```tf module "azure-ramius" { - source = "git::https://github.com/poseidon/typhoon//azure/container-linux/kubernetes?ref=v1.13.5" + source = "git::https://github.com/poseidon/typhoon//azure/container-linux/kubernetes?ref=v1.14.0" providers = { azurerm = "azurerm.default" @@ -161,9 +161,9 @@ In 4-8 minutes, the Kubernetes cluster will be ready. $ export KUBECONFIG=/home/user/.secrets/clusters/ramius/auth/kubeconfig $ kubectl get nodes NAME STATUS ROLES AGE VERSION -ramius-controller-0 Ready controller,master 24m v1.13.5 -ramius-worker-000001 Ready node 25m v1.13.5 -ramius-worker-000002 Ready node 24m v1.13.5 +ramius-controller-0 Ready controller,master 24m v1.14.0 +ramius-worker-000001 Ready node 25m v1.14.0 +ramius-worker-000002 Ready node 24m v1.14.0 ``` List the pods. diff --git a/docs/cl/bare-metal.md b/docs/cl/bare-metal.md index ff49622ab..418a8a239 100644 --- a/docs/cl/bare-metal.md +++ b/docs/cl/bare-metal.md @@ -1,6 +1,6 @@ # Bare-Metal -In this tutorial, we'll network boot and provision a Kubernetes v1.13.5 cluster on bare-metal with Container Linux. +In this tutorial, we'll network boot and provision a Kubernetes v1.14.0 cluster on bare-metal with Container Linux. First, we'll deploy a [Matchbox](https://github.com/coreos/matchbox) service and setup a network boot environment. Then, we'll declare a Kubernetes cluster using the Typhoon Terraform module and power on machines. On PXE boot, machines will install Container Linux to disk, reboot into the disk install, and provision themselves as Kubernetes controllers or workers via Ignition. @@ -180,7 +180,7 @@ Define a Kubernetes cluster using the module `bare-metal/container-linux/kuberne ```tf module "bare-metal-mercury" { - source = "git::https://github.com/poseidon/typhoon//bare-metal/container-linux/kubernetes?ref=v1.13.5" + source = "git::https://github.com/poseidon/typhoon//bare-metal/container-linux/kubernetes?ref=v1.14.0" providers = { local = "local.default" @@ -292,9 +292,9 @@ Apply complete! Resources: 55 added, 0 changed, 0 destroyed. To watch the install to disk (until machines reboot from disk), SSH to port 2222. ``` -# before v1.13.5 +# before v1.14.0 $ ssh debug@node1.example.com -# after v1.13.5 +# after v1.14.0 $ ssh -p 2222 core@node1.example.com ``` @@ -319,9 +319,9 @@ bootkube[5]: Tearing down temporary bootstrap control plane... $ export KUBECONFIG=/home/user/.secrets/clusters/mercury/auth/kubeconfig $ kubectl get nodes NAME STATUS ROLES AGE VERSION -node1.example.com Ready controller,master 10m v1.13.5 -node2.example.com Ready node 10m v1.13.5 -node3.example.com Ready node 10m v1.13.5 +node1.example.com Ready controller,master 10m v1.14.0 +node2.example.com Ready node 10m v1.14.0 +node3.example.com Ready node 10m v1.14.0 ``` List the pods. diff --git a/docs/cl/digital-ocean.md b/docs/cl/digital-ocean.md index 7235a42e6..7a86097ab 100644 --- a/docs/cl/digital-ocean.md +++ b/docs/cl/digital-ocean.md @@ -1,6 +1,6 @@ # Digital Ocean -In this tutorial, we'll create a Kubernetes v1.13.5 cluster on DigitalOcean with Container Linux. +In this tutorial, we'll create a Kubernetes v1.14.0 cluster on DigitalOcean with Container Linux. We'll declare a Kubernetes cluster using the Typhoon Terraform module. Then apply the changes to create controller droplets, worker droplets, DNS records, tags, and TLS assets. @@ -86,7 +86,7 @@ Define a Kubernetes cluster using the module `digital-ocean/container-linux/kube ```tf module "digital-ocean-nemo" { - source = "git::https://github.com/poseidon/typhoon//digital-ocean/container-linux/kubernetes?ref=v1.13.5" + source = "git::https://github.com/poseidon/typhoon//digital-ocean/container-linux/kubernetes?ref=v1.14.0" providers = { digitalocean = "digitalocean.default" @@ -160,9 +160,9 @@ In 3-6 minutes, the Kubernetes cluster will be ready. $ export KUBECONFIG=/home/user/.secrets/clusters/nemo/auth/kubeconfig $ kubectl get nodes NAME STATUS ROLES AGE VERSION -10.132.110.130 Ready controller,master 10m v1.13.5 -10.132.115.81 Ready node 10m v1.13.5 -10.132.124.107 Ready node 10m v1.13.5 +10.132.110.130 Ready controller,master 10m v1.14.0 +10.132.115.81 Ready node 10m v1.14.0 +10.132.124.107 Ready node 10m v1.14.0 ``` List the pods. diff --git a/docs/cl/google-cloud.md b/docs/cl/google-cloud.md index 393d5840b..d8710c421 100644 --- a/docs/cl/google-cloud.md +++ b/docs/cl/google-cloud.md @@ -1,6 +1,6 @@ # Google Cloud -In this tutorial, we'll create a Kubernetes v1.13.5 cluster on Google Compute Engine with Container Linux. +In this tutorial, we'll create a Kubernetes v1.14.0 cluster on Google Compute Engine with Container Linux. We'll declare a Kubernetes cluster using the Typhoon Terraform module. Then apply the changes to create a network, firewall rules, health checks, controller instances, worker managed instance group, load balancers, and TLS assets. @@ -93,7 +93,7 @@ Define a Kubernetes cluster using the module `google-cloud/container-linux/kuber ```tf module "google-cloud-yavin" { - source = "git::https://github.com/poseidon/typhoon//google-cloud/container-linux/kubernetes?ref=v1.13.5" + source = "git::https://github.com/poseidon/typhoon//google-cloud/container-linux/kubernetes?ref=v1.14.0" providers = { google = "google.default" @@ -168,9 +168,9 @@ In 4-8 minutes, the Kubernetes cluster will be ready. $ export KUBECONFIG=/home/user/.secrets/clusters/yavin/auth/kubeconfig $ kubectl get nodes NAME ROLES STATUS AGE VERSION -yavin-controller-0.c.example-com.internal controller,master Ready 6m v1.13.5 -yavin-worker-jrbf.c.example-com.internal node Ready 5m v1.13.5 -yavin-worker-mzdm.c.example-com.internal node Ready 5m v1.13.5 +yavin-controller-0.c.example-com.internal controller,master Ready 6m v1.14.0 +yavin-worker-jrbf.c.example-com.internal node Ready 5m v1.14.0 +yavin-worker-mzdm.c.example-com.internal node Ready 5m v1.14.0 ``` List the pods. diff --git a/docs/index.md b/docs/index.md index 77c788c45..d62565c83 100644 --- a/docs/index.md +++ b/docs/index.md @@ -11,7 +11,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features -* Kubernetes v1.13.5 (upstream, via [kubernetes-incubator/bootkube](https://github.com/kubernetes-incubator/bootkube)) +* Kubernetes v1.14.0 (upstream, via [kubernetes-incubator/bootkube](https://github.com/kubernetes-incubator/bootkube)) * Single or multi-master, [Calico](https://www.projectcalico.org/) or [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/) * Advanced features like [worker pools](advanced/worker-pools/), [preemptible](cl/google-cloud/#preemption) workers, and [snippets](advanced/customization/#container-linux) customization @@ -49,7 +49,7 @@ Define a Kubernetes cluster by using the Terraform module for your chosen platfo ```tf module "google-cloud-yavin" { - source = "git::https://github.com/poseidon/typhoon//google-cloud/container-linux/kubernetes?ref=v1.13.5" + source = "git::https://github.com/poseidon/typhoon//google-cloud/container-linux/kubernetes?ref=v1.14.0" providers = { google = "google.default" @@ -90,9 +90,9 @@ In 4-8 minutes (varies by platform), the cluster will be ready. This Google Clou $ export KUBECONFIG=/home/user/.secrets/clusters/yavin/auth/kubeconfig $ kubectl get nodes NAME ROLES STATUS AGE VERSION -yavin-controller-0.c.example-com.internal controller,master Ready 6m v1.13.5 -yavin-worker-jrbf.c.example-com.internal node Ready 5m v1.13.5 -yavin-worker-mzdm.c.example-com.internal node Ready 5m v1.13.5 +yavin-controller-0.c.example-com.internal controller,master Ready 6m v1.14.0 +yavin-worker-jrbf.c.example-com.internal node Ready 5m v1.14.0 +yavin-worker-mzdm.c.example-com.internal node Ready 5m v1.14.0 ``` List the pods. diff --git a/docs/topics/maintenance.md b/docs/topics/maintenance.md index c452bb8bb..347f48146 100644 --- a/docs/topics/maintenance.md +++ b/docs/topics/maintenance.md @@ -18,7 +18,7 @@ module "google-cloud-yavin" { } module "bare-metal-mercury" { - source = "git::https://github.com/poseidon/typhoon//bare-metal/container-linux/kubernetes?ref=v1.13.5" + source = "git::https://github.com/poseidon/typhoon//bare-metal/container-linux/kubernetes?ref=v1.14.0" ... } ``` diff --git a/google-cloud/container-linux/kubernetes/README.md b/google-cloud/container-linux/kubernetes/README.md index bbe9188dd..74cc0a9c3 100644 --- a/google-cloud/container-linux/kubernetes/README.md +++ b/google-cloud/container-linux/kubernetes/README.md @@ -11,7 +11,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features -* Kubernetes v1.13.5 (upstream, via [kubernetes-incubator/bootkube](https://github.com/kubernetes-incubator/bootkube)) +* Kubernetes v1.14.0 (upstream, via [kubernetes-incubator/bootkube](https://github.com/kubernetes-incubator/bootkube)) * Single or multi-master, [Calico](https://www.projectcalico.org/) or [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/) * Advanced features like [worker pools](https://typhoon.psdn.io/advanced/worker-pools/), [preemptible](https://typhoon.psdn.io/cl/google-cloud/#preemption) workers, and [snippets](https://typhoon.psdn.io/advanced/customization/#container-linux) customization diff --git a/google-cloud/container-linux/kubernetes/bootkube.tf b/google-cloud/container-linux/kubernetes/bootkube.tf index b179f4b13..eeca64d6b 100644 --- a/google-cloud/container-linux/kubernetes/bootkube.tf +++ b/google-cloud/container-linux/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=bcb015e1052192b166b19a1e3b5da0f25051eaf4" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=ce5db83663b1de2096afc1787c2b622bc08987b3" cluster_name = "${var.cluster_name}" api_servers = ["${format("%s.%s", var.cluster_name, var.dns_zone)}"] diff --git a/google-cloud/container-linux/kubernetes/cl/controller.yaml.tmpl b/google-cloud/container-linux/kubernetes/cl/controller.yaml.tmpl index e25ed6b25..9cb60d496 100644 --- a/google-cloud/container-linux/kubernetes/cl/controller.yaml.tmpl +++ b/google-cloud/container-linux/kubernetes/cl/controller.yaml.tmpl @@ -124,7 +124,7 @@ storage: contents: inline: | KUBELET_IMAGE_URL=docker://k8s.gcr.io/hyperkube - KUBELET_IMAGE_TAG=v1.13.5 + KUBELET_IMAGE_TAG=v1.14.0 - path: /etc/sysctl.d/max-user-watches.conf filesystem: root contents: diff --git a/google-cloud/container-linux/kubernetes/workers/cl/worker.yaml.tmpl b/google-cloud/container-linux/kubernetes/workers/cl/worker.yaml.tmpl index 7e00d0ad3..e1bf59bfc 100644 --- a/google-cloud/container-linux/kubernetes/workers/cl/worker.yaml.tmpl +++ b/google-cloud/container-linux/kubernetes/workers/cl/worker.yaml.tmpl @@ -94,7 +94,7 @@ storage: contents: inline: | KUBELET_IMAGE_URL=docker://k8s.gcr.io/hyperkube - KUBELET_IMAGE_TAG=v1.13.5 + KUBELET_IMAGE_TAG=v1.14.0 - path: /etc/sysctl.d/max-user-watches.conf filesystem: root contents: @@ -112,7 +112,7 @@ storage: --volume config,kind=host,source=/etc/kubernetes \ --mount volume=config,target=/etc/kubernetes \ --insecure-options=image \ - docker://k8s.gcr.io/hyperkube:v1.13.5 \ + docker://k8s.gcr.io/hyperkube:v1.14.0 \ --net=host \ --dns=host \ --exec=/kubectl -- --kubeconfig=/etc/kubernetes/kubeconfig delete node $(hostname) diff --git a/google-cloud/fedora-atomic/kubernetes/README.md b/google-cloud/fedora-atomic/kubernetes/README.md index 3f621dd7b..ffb34c71d 100644 --- a/google-cloud/fedora-atomic/kubernetes/README.md +++ b/google-cloud/fedora-atomic/kubernetes/README.md @@ -11,7 +11,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features -* Kubernetes v1.13.5 (upstream, via [kubernetes-incubator/bootkube](https://github.com/kubernetes-incubator/bootkube)) +* Kubernetes v1.14.0 (upstream, via [kubernetes-incubator/bootkube](https://github.com/kubernetes-incubator/bootkube)) * Single or multi-master, [Calico](https://www.projectcalico.org/) or [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/) * Advanced features like [worker pools](https://typhoon.psdn.io/advanced/worker-pools/) and [preemptible](https://typhoon.psdn.io/cl/google-cloud/#preemption) workers diff --git a/google-cloud/fedora-atomic/kubernetes/bootkube.tf b/google-cloud/fedora-atomic/kubernetes/bootkube.tf index 21a91dba5..851eeedcb 100644 --- a/google-cloud/fedora-atomic/kubernetes/bootkube.tf +++ b/google-cloud/fedora-atomic/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=bcb015e1052192b166b19a1e3b5da0f25051eaf4" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=ce5db83663b1de2096afc1787c2b622bc08987b3" cluster_name = "${var.cluster_name}" api_servers = ["${format("%s.%s", var.cluster_name, var.dns_zone)}"] diff --git a/google-cloud/fedora-atomic/kubernetes/cloudinit/controller.yaml.tmpl b/google-cloud/fedora-atomic/kubernetes/cloudinit/controller.yaml.tmpl index 8afdfc050..5949d55d2 100644 --- a/google-cloud/fedora-atomic/kubernetes/cloudinit/controller.yaml.tmpl +++ b/google-cloud/fedora-atomic/kubernetes/cloudinit/controller.yaml.tmpl @@ -79,7 +79,7 @@ runcmd: - [systemctl, daemon-reload] - [systemctl, restart, NetworkManager] - "atomic install --system --name=etcd quay.io/poseidon/etcd:v3.3.12" - - "atomic install --system --name=kubelet quay.io/poseidon/kubelet:v1.13.5" + - "atomic install --system --name=kubelet quay.io/poseidon/kubelet:v1.14.0" - "atomic install --system --name=bootkube quay.io/poseidon/bootkube:v0.14.0" - [systemctl, start, --no-block, etcd.service] - [systemctl, start, --no-block, kubelet.service] diff --git a/google-cloud/fedora-atomic/kubernetes/workers/cloudinit/worker.yaml.tmpl b/google-cloud/fedora-atomic/kubernetes/workers/cloudinit/worker.yaml.tmpl index a05cd877c..373a7163a 100644 --- a/google-cloud/fedora-atomic/kubernetes/workers/cloudinit/worker.yaml.tmpl +++ b/google-cloud/fedora-atomic/kubernetes/workers/cloudinit/worker.yaml.tmpl @@ -54,7 +54,7 @@ bootcmd: runcmd: - [systemctl, daemon-reload] - [systemctl, restart, NetworkManager] - - "atomic install --system --name=kubelet quay.io/poseidon/kubelet:v1.13.5" + - "atomic install --system --name=kubelet quay.io/poseidon/kubelet:v1.14.0" - [systemctl, start, --no-block, kubelet.service] users: - default From be29f5203988ae042448885d3978eb88c099cb04 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Sun, 7 Apr 2019 02:29:07 -0700 Subject: [PATCH 073/523] Add enable_aggregation option (defaults to false) * Add an `enable_aggregation` variable to enable the kube-apiserver aggregation layer for adding extension apiservers to clusters * Aggregation is **disabled** by default. Typhoon recommends you not enable aggregation. Consider whether less invasive ways to achieve your goals are possible and whether those goals are well-founded * Enabling aggregation and extension apiservers increases the attack surface of a cluster and makes extensions a part of the control plane. Admins must scrutinize and trust any extension apiserver used. * Passing a v1.14 CNCF conformance test requires aggregation be enabled. Having an option for aggregation keeps compliance, but retains the stricter security posture on default clusters --- CHANGES.md | 3 +++ aws/container-linux/kubernetes/bootkube.tf | 3 ++- aws/container-linux/kubernetes/variables.tf | 6 ++++++ aws/fedora-atomic/kubernetes/bootkube.tf | 2 +- azure/container-linux/kubernetes/bootkube.tf | 3 ++- azure/container-linux/kubernetes/variables.tf | 6 ++++++ bare-metal/container-linux/kubernetes/bootkube.tf | 3 ++- bare-metal/container-linux/kubernetes/variables.tf | 6 ++++++ bare-metal/fedora-atomic/kubernetes/bootkube.tf | 2 +- .../container-linux/kubernetes/bootkube.tf | 3 ++- digital-ocean/container-linux/kubernetes/network.tf | 13 ++++++------- digital-ocean/container-linux/kubernetes/outputs.tf | 5 ++--- .../container-linux/kubernetes/variables.tf | 6 ++++++ digital-ocean/fedora-atomic/kubernetes/bootkube.tf | 2 +- google-cloud/container-linux/kubernetes/bootkube.tf | 3 ++- .../container-linux/kubernetes/variables.tf | 6 ++++++ google-cloud/fedora-atomic/kubernetes/bootkube.tf | 2 +- 17 files changed, 55 insertions(+), 19 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index a2b85ff3e..ddc9d7855 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -6,6 +6,9 @@ Notable changes between versions. * Kubernetes [v1.14.0](https://github.com/kubernetes/kubernetes/blob/master/CHANGELOG-1.14.md#v1140) * Update Calico from v3.6.0 to v3.6.1 +* Add `enable_aggregation` option for CNCF conformance + * Aggregation is disabled by default to retain our security stance + * Aggregation increases the security surface area. Extensions become part of the control plane and must be scrutinized carefully and trusted. Favor leaving aggregation disabled. #### AWS diff --git a/aws/container-linux/kubernetes/bootkube.tf b/aws/container-linux/kubernetes/bootkube.tf index c40108978..be6212077 100644 --- a/aws/container-linux/kubernetes/bootkube.tf +++ b/aws/container-linux/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=ce5db83663b1de2096afc1787c2b622bc08987b3" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=feb6e4cb3e479b20dfc269f65e76ceb62d8d2ec4" cluster_name = "${var.cluster_name}" api_servers = ["${format("%s.%s", var.cluster_name, var.dns_zone)}"] @@ -12,4 +12,5 @@ module "bootkube" { service_cidr = "${var.service_cidr}" cluster_domain_suffix = "${var.cluster_domain_suffix}" enable_reporting = "${var.enable_reporting}" + enable_aggregation = "${var.enable_aggregation}" } diff --git a/aws/container-linux/kubernetes/variables.tf b/aws/container-linux/kubernetes/variables.tf index 10e5dce3f..33be7faac 100644 --- a/aws/container-linux/kubernetes/variables.tf +++ b/aws/container-linux/kubernetes/variables.tf @@ -146,3 +146,9 @@ variable "enable_reporting" { description = "Enable usage or analytics reporting to upstreams (Calico)" default = "false" } + +variable "enable_aggregation" { + description = "Enable the Kubernetes Aggregation Layer (defaults to false)" + type = "string" + default = "false" +} diff --git a/aws/fedora-atomic/kubernetes/bootkube.tf b/aws/fedora-atomic/kubernetes/bootkube.tf index cfa874170..054e4b09e 100644 --- a/aws/fedora-atomic/kubernetes/bootkube.tf +++ b/aws/fedora-atomic/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=ce5db83663b1de2096afc1787c2b622bc08987b3" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=feb6e4cb3e479b20dfc269f65e76ceb62d8d2ec4" cluster_name = "${var.cluster_name}" api_servers = ["${format("%s.%s", var.cluster_name, var.dns_zone)}"] diff --git a/azure/container-linux/kubernetes/bootkube.tf b/azure/container-linux/kubernetes/bootkube.tf index 7b0bd0627..a1b1732ec 100644 --- a/azure/container-linux/kubernetes/bootkube.tf +++ b/azure/container-linux/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=ce5db83663b1de2096afc1787c2b622bc08987b3" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=feb6e4cb3e479b20dfc269f65e76ceb62d8d2ec4" cluster_name = "${var.cluster_name}" api_servers = ["${format("%s.%s", var.cluster_name, var.dns_zone)}"] @@ -11,4 +11,5 @@ module "bootkube" { service_cidr = "${var.service_cidr}" cluster_domain_suffix = "${var.cluster_domain_suffix}" enable_reporting = "${var.enable_reporting}" + enable_aggregation = "${var.enable_aggregation}" } diff --git a/azure/container-linux/kubernetes/variables.tf b/azure/container-linux/kubernetes/variables.tf index d55e4bc8d..b5e378dc6 100644 --- a/azure/container-linux/kubernetes/variables.tf +++ b/azure/container-linux/kubernetes/variables.tf @@ -121,3 +121,9 @@ variable "enable_reporting" { description = "Enable usage or analytics reporting to upstreams (Calico)" default = "false" } + +variable "enable_aggregation" { + description = "Enable the Kubernetes Aggregation Layer (defaults to false)" + type = "string" + default = "false" +} diff --git a/bare-metal/container-linux/kubernetes/bootkube.tf b/bare-metal/container-linux/kubernetes/bootkube.tf index 3e834398f..75f923453 100644 --- a/bare-metal/container-linux/kubernetes/bootkube.tf +++ b/bare-metal/container-linux/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=ce5db83663b1de2096afc1787c2b622bc08987b3" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=feb6e4cb3e479b20dfc269f65e76ceb62d8d2ec4" cluster_name = "${var.cluster_name}" api_servers = ["${var.k8s_domain_name}"] @@ -13,4 +13,5 @@ module "bootkube" { service_cidr = "${var.service_cidr}" cluster_domain_suffix = "${var.cluster_domain_suffix}" enable_reporting = "${var.enable_reporting}" + enable_aggregation = "${var.enable_aggregation}" } diff --git a/bare-metal/container-linux/kubernetes/variables.tf b/bare-metal/container-linux/kubernetes/variables.tf index bfba58dba..788ecd4aa 100644 --- a/bare-metal/container-linux/kubernetes/variables.tf +++ b/bare-metal/container-linux/kubernetes/variables.tf @@ -153,3 +153,9 @@ variable "enable_reporting" { description = "Enable usage or analytics reporting to upstreams (Calico)" default = "false" } + +variable "enable_aggregation" { + description = "Enable the Kubernetes Aggregation Layer (defaults to false)" + type = "string" + default = "false" +} diff --git a/bare-metal/fedora-atomic/kubernetes/bootkube.tf b/bare-metal/fedora-atomic/kubernetes/bootkube.tf index a6fbb57cf..5cb852fb5 100644 --- a/bare-metal/fedora-atomic/kubernetes/bootkube.tf +++ b/bare-metal/fedora-atomic/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=ce5db83663b1de2096afc1787c2b622bc08987b3" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=feb6e4cb3e479b20dfc269f65e76ceb62d8d2ec4" cluster_name = "${var.cluster_name}" api_servers = ["${var.k8s_domain_name}"] diff --git a/digital-ocean/container-linux/kubernetes/bootkube.tf b/digital-ocean/container-linux/kubernetes/bootkube.tf index cf67d6724..878af4f16 100644 --- a/digital-ocean/container-linux/kubernetes/bootkube.tf +++ b/digital-ocean/container-linux/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=ce5db83663b1de2096afc1787c2b622bc08987b3" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=feb6e4cb3e479b20dfc269f65e76ceb62d8d2ec4" cluster_name = "${var.cluster_name}" api_servers = ["${format("%s.%s", var.cluster_name, var.dns_zone)}"] @@ -12,4 +12,5 @@ module "bootkube" { service_cidr = "${var.service_cidr}" cluster_domain_suffix = "${var.cluster_domain_suffix}" enable_reporting = "${var.enable_reporting}" + enable_aggregation = "${var.enable_aggregation}" } diff --git a/digital-ocean/container-linux/kubernetes/network.tf b/digital-ocean/container-linux/kubernetes/network.tf index 27a4e0c28..52cf7fb12 100644 --- a/digital-ocean/container-linux/kubernetes/network.tf +++ b/digital-ocean/container-linux/kubernetes/network.tf @@ -55,13 +55,13 @@ resource "digitalocean_firewall" "controllers" { # etcd, kube-apiserver, kubelet inbound_rule = [ { - protocol = "tcp" - port_range = "2379-2380" + protocol = "tcp" + port_range = "2379-2380" source_tags = ["${digitalocean_tag.controllers.name}"] }, { - protocol = "tcp" - port_range = "2381" + protocol = "tcp" + port_range = "2381" source_tags = ["${digitalocean_tag.workers.name}"] }, { @@ -90,10 +90,9 @@ resource "digitalocean_firewall" "workers" { source_addresses = ["0.0.0.0/0", "::/0"] }, { - protocol = "tcp" - port_range = "10254" + protocol = "tcp" + port_range = "10254" source_addresses = ["0.0.0.0/0"] }, ] } - diff --git a/digital-ocean/container-linux/kubernetes/outputs.tf b/digital-ocean/container-linux/kubernetes/outputs.tf index 7ca6b81cf..15172d7f5 100644 --- a/digital-ocean/container-linux/kubernetes/outputs.tf +++ b/digital-ocean/container-linux/kubernetes/outputs.tf @@ -31,11 +31,10 @@ output "workers_ipv6" { output "controller_tag" { description = "Tag applied to controller droplets" - value = "${digitalocean_tag.controllers.name}" + value = "${digitalocean_tag.controllers.name}" } output "worker_tag" { description = "Tag applied to worker droplets" - value = "${digitalocean_tag.workers.name}" + value = "${digitalocean_tag.workers.name}" } - diff --git a/digital-ocean/container-linux/kubernetes/variables.tf b/digital-ocean/container-linux/kubernetes/variables.tf index 535797e64..9606fed08 100644 --- a/digital-ocean/container-linux/kubernetes/variables.tf +++ b/digital-ocean/container-linux/kubernetes/variables.tf @@ -98,3 +98,9 @@ variable "enable_reporting" { description = "Enable usage or analytics reporting to upstreams (Calico)" default = "false" } + +variable "enable_aggregation" { + description = "Enable the Kubernetes Aggregation Layer (defaults to false)" + type = "string" + default = "false" +} diff --git a/digital-ocean/fedora-atomic/kubernetes/bootkube.tf b/digital-ocean/fedora-atomic/kubernetes/bootkube.tf index 41dd57bbc..11e59d4d1 100644 --- a/digital-ocean/fedora-atomic/kubernetes/bootkube.tf +++ b/digital-ocean/fedora-atomic/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=ce5db83663b1de2096afc1787c2b622bc08987b3" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=feb6e4cb3e479b20dfc269f65e76ceb62d8d2ec4" cluster_name = "${var.cluster_name}" api_servers = ["${format("%s.%s", var.cluster_name, var.dns_zone)}"] diff --git a/google-cloud/container-linux/kubernetes/bootkube.tf b/google-cloud/container-linux/kubernetes/bootkube.tf index eeca64d6b..446362c6c 100644 --- a/google-cloud/container-linux/kubernetes/bootkube.tf +++ b/google-cloud/container-linux/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=ce5db83663b1de2096afc1787c2b622bc08987b3" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=feb6e4cb3e479b20dfc269f65e76ceb62d8d2ec4" cluster_name = "${var.cluster_name}" api_servers = ["${format("%s.%s", var.cluster_name, var.dns_zone)}"] @@ -12,6 +12,7 @@ module "bootkube" { service_cidr = "${var.service_cidr}" cluster_domain_suffix = "${var.cluster_domain_suffix}" enable_reporting = "${var.enable_reporting}" + enable_aggregation = "${var.enable_aggregation}" // temporary apiserver_port = 443 diff --git a/google-cloud/container-linux/kubernetes/variables.tf b/google-cloud/container-linux/kubernetes/variables.tf index 61d1cdf46..971dc3140 100644 --- a/google-cloud/container-linux/kubernetes/variables.tf +++ b/google-cloud/container-linux/kubernetes/variables.tf @@ -121,3 +121,9 @@ variable "enable_reporting" { description = "Enable usage or analytics reporting to upstreams (Calico)" default = "false" } + +variable "enable_aggregation" { + description = "Enable the Kubernetes Aggregation Layer (defaults to false)" + type = "string" + default = "false" +} diff --git a/google-cloud/fedora-atomic/kubernetes/bootkube.tf b/google-cloud/fedora-atomic/kubernetes/bootkube.tf index 851eeedcb..a2ca4d8a9 100644 --- a/google-cloud/fedora-atomic/kubernetes/bootkube.tf +++ b/google-cloud/fedora-atomic/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=ce5db83663b1de2096afc1787c2b622bc08987b3" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=feb6e4cb3e479b20dfc269f65e76ceb62d8d2ec4" cluster_name = "${var.cluster_name}" api_servers = ["${format("%s.%s", var.cluster_name, var.dns_zone)}"] From c1fe41d34a84d48dabca0fe45bd6affeccd31144 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Sun, 7 Apr 2019 18:04:02 -0700 Subject: [PATCH 074/523] Add ability to load balance TCP/UDP applications on Azure * Add ability to load balance TCP/UDP applications (e.g. NodePort) * Output the load balancer ID as `loadbalancer_id` * Output `worker_security_group_name` and `worker_address_prefix` for extending firewall rules --- CHANGES.md | 12 ++++++--- azure/container-linux/kubernetes/outputs.tf | 27 ++++++++++++++++++--- 2 files changed, 32 insertions(+), 7 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index ddc9d7855..fa433274b 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -6,7 +6,7 @@ Notable changes between versions. * Kubernetes [v1.14.0](https://github.com/kubernetes/kubernetes/blob/master/CHANGELOG-1.14.md#v1140) * Update Calico from v3.6.0 to v3.6.1 -* Add `enable_aggregation` option for CNCF conformance +* Add `enable_aggregation` option for CNCF conformance ([#436](https://github.com/poseidon/typhoon/pull/436)) * Aggregation is disabled by default to retain our security stance * Aggregation increases the security surface area. Extensions become part of the control plane and must be scrutinized carefully and trusted. Favor leaving aggregation disabled. @@ -16,10 +16,16 @@ Notable changes between versions. * Output the network load balancer ARN as `nlb_id` * Accept a `worker_target_groups` (ARN) list to which worker instances should be added +#### Azure + +* Add ability to load balance TCP/UDP applications ([#447](https://github.com/poseidon/typhoon/pull/447)) + * Output the load balancer ID as `loadbalancer_id` +* Output `worker_security_group_name` and `worker_address_prefix` for extending firewall rules ([#447](https://github.com/poseidon/typhoon/pull/447)) + #### DigitalOcean -* Harden internal (node-to-node) firewall rules to align with other platforms -* Output `controller_tag` and `worker_tag` to simplify custom firewall rule creation +* Harden internal (node-to-node) firewall rules to align with other platforms ([#444](https://github.com/poseidon/typhoon/pull/444)) +* Output `controller_tag` and `worker_tag` to simplify extending firewall rules ([#444](https://github.com/poseidon/typhoon/pull/444)) #### Google Cloud diff --git a/azure/container-linux/kubernetes/outputs.tf b/azure/container-linux/kubernetes/outputs.tf index 75fdfc463..bfd2a1439 100644 --- a/azure/container-linux/kubernetes/outputs.tf +++ b/azure/container-linux/kubernetes/outputs.tf @@ -27,10 +27,29 @@ output "security_group_id" { value = "${azurerm_network_security_group.worker.id}" } -output "backend_address_pool_id" { - value = "${azurerm_lb_backend_address_pool.worker.id}" -} - output "kubeconfig" { value = "${module.bootkube.kubeconfig-kubelet}" } + +# Outputs for custom firewalling + +output "worker_security_group_name" { + value = "${azurerm_network_security_group.worker.name}" +} + +output "worker_address_prefix" { + description = "Worker network subnet CIDR address (for source/destination)" + value = "${azurerm_subnet.worker.address_prefix}" +} + +# Outputs for custom load balancing + +output "loadbalancer_id" { + description = "ID of the cluster load balancer" + value = "${azurerm_lb.cluster.id}" +} + +output "backend_address_pool_id" { + description = "ID of the worker backend address pool" + value = "${azurerm_lb_backend_address_pool.worker.id}" +} From 1aa4d2cdc1f1da3bf8391d343d301b60bc79bf19 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Mon, 8 Apr 2019 18:48:49 -0700 Subject: [PATCH 075/523] Update CHANGES for v1.14.0 release --- CHANGES.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index fa433274b..fdc0a8170 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -4,6 +4,8 @@ Notable changes between versions. ## Latest +## v1.14.0 + * Kubernetes [v1.14.0](https://github.com/kubernetes/kubernetes/blob/master/CHANGELOG-1.14.md#v1140) * Update Calico from v3.6.0 to v3.6.1 * Add `enable_aggregation` option for CNCF conformance ([#436](https://github.com/poseidon/typhoon/pull/436)) @@ -25,7 +27,8 @@ Notable changes between versions. #### DigitalOcean * Harden internal (node-to-node) firewall rules to align with other platforms ([#444](https://github.com/poseidon/typhoon/pull/444)) -* Output `controller_tag` and `worker_tag` to simplify extending firewall rules ([#444](https://github.com/poseidon/typhoon/pull/444)) +* Add ability to load balance TCP applications ([#444](https://github.com/poseidon/typhoon/pull/444)) + * Output `controller_tag` and `worker_tag` for extending firewall rules ([#444](https://github.com/poseidon/typhoon/pull/444)) #### Google Cloud From 452253081b8cfaa7c866be924126729889a9b034 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Tue, 9 Apr 2019 21:47:23 -0700 Subject: [PATCH 076/523] Update Kubernetes from v1.14.0 to v1.14.1 * https://github.com/kubernetes/kubernetes/blob/master/CHANGELOG-1.14.md#changelog-since-v1140 --- CHANGES.md | 2 ++ README.md | 10 +++++----- aws/container-linux/kubernetes/README.md | 2 +- aws/container-linux/kubernetes/bootkube.tf | 2 +- .../kubernetes/cl/controller.yaml.tmpl | 2 +- .../kubernetes/workers/cl/worker.yaml.tmpl | 4 ++-- aws/fedora-atomic/kubernetes/README.md | 2 +- aws/fedora-atomic/kubernetes/bootkube.tf | 2 +- .../kubernetes/cloudinit/controller.yaml.tmpl | 2 +- .../workers/cloudinit/worker.yaml.tmpl | 2 +- azure/container-linux/kubernetes/README.md | 2 +- azure/container-linux/kubernetes/bootkube.tf | 2 +- .../kubernetes/cl/controller.yaml.tmpl | 2 +- .../kubernetes/workers/cl/worker.yaml.tmpl | 4 ++-- bare-metal/container-linux/kubernetes/README.md | 2 +- .../container-linux/kubernetes/bootkube.tf | 2 +- .../kubernetes/cl/controller.yaml.tmpl | 2 +- .../kubernetes/cl/worker.yaml.tmpl | 2 +- bare-metal/fedora-atomic/kubernetes/README.md | 2 +- bare-metal/fedora-atomic/kubernetes/bootkube.tf | 2 +- .../kubernetes/cloudinit/controller.yaml.tmpl | 2 +- .../kubernetes/cloudinit/worker.yaml.tmpl | 2 +- .../container-linux/kubernetes/README.md | 2 +- .../container-linux/kubernetes/bootkube.tf | 2 +- .../kubernetes/cl/controller.yaml.tmpl | 2 +- .../kubernetes/cl/worker.yaml.tmpl | 4 ++-- digital-ocean/fedora-atomic/kubernetes/README.md | 2 +- .../fedora-atomic/kubernetes/bootkube.tf | 2 +- .../kubernetes/cloudinit/controller.yaml.tmpl | 2 +- .../kubernetes/cloudinit/worker.yaml.tmpl | 2 +- docs/advanced/worker-pools.md | 16 ++++++++-------- docs/atomic/aws.md | 10 +++++----- docs/atomic/bare-metal.md | 10 +++++----- docs/atomic/digital-ocean.md | 10 +++++----- docs/atomic/google-cloud.md | 10 +++++----- docs/cl/aws.md | 10 +++++----- docs/cl/azure.md | 10 +++++----- docs/cl/bare-metal.md | 14 +++++++------- docs/cl/digital-ocean.md | 10 +++++----- docs/cl/google-cloud.md | 10 +++++----- docs/index.md | 10 +++++----- docs/topics/maintenance.md | 2 +- .../container-linux/kubernetes/README.md | 2 +- .../container-linux/kubernetes/bootkube.tf | 2 +- .../kubernetes/cl/controller.yaml.tmpl | 2 +- .../kubernetes/workers/cl/worker.yaml.tmpl | 4 ++-- google-cloud/fedora-atomic/kubernetes/README.md | 2 +- .../fedora-atomic/kubernetes/bootkube.tf | 2 +- .../kubernetes/cloudinit/controller.yaml.tmpl | 2 +- .../workers/cloudinit/worker.yaml.tmpl | 2 +- 50 files changed, 108 insertions(+), 106 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index fdc0a8170..4581fb7b1 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -4,6 +4,8 @@ Notable changes between versions. ## Latest +* Kubernetes [v1.14.1](https://github.com/kubernetes/kubernetes/blob/master/CHANGELOG-1.14.md#v1141) + ## v1.14.0 * Kubernetes [v1.14.0](https://github.com/kubernetes/kubernetes/blob/master/CHANGELOG-1.14.md#v1140) diff --git a/README.md b/README.md index 4861b6cc0..3bfbb4038 100644 --- a/README.md +++ b/README.md @@ -11,7 +11,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features -* Kubernetes v1.14.0 (upstream, via [kubernetes-incubator/bootkube](https://github.com/kubernetes-incubator/bootkube)) +* Kubernetes v1.14.1 (upstream, via [kubernetes-incubator/bootkube](https://github.com/kubernetes-incubator/bootkube)) * Single or multi-master, [Calico](https://www.projectcalico.org/) or [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/) * Advanced features like [worker pools](https://typhoon.psdn.io/advanced/worker-pools/), [preemptible](https://typhoon.psdn.io/cl/google-cloud/#preemption) workers, and [snippets](https://typhoon.psdn.io/advanced/customization/#container-linux) customization @@ -50,7 +50,7 @@ Define a Kubernetes cluster by using the Terraform module for your chosen platfo ```tf module "google-cloud-yavin" { - source = "git::https://github.com/poseidon/typhoon//google-cloud/container-linux/kubernetes?ref=v1.14.0" + source = "git::https://github.com/poseidon/typhoon//google-cloud/container-linux/kubernetes?ref=v1.14.1" providers = { google = "google.default" @@ -91,9 +91,9 @@ In 4-8 minutes (varies by platform), the cluster will be ready. This Google Clou $ export KUBECONFIG=/home/user/.secrets/clusters/yavin/auth/kubeconfig $ kubectl get nodes NAME ROLES STATUS AGE VERSION -yavin-controller-0.c.example-com.internal controller,master Ready 6m v1.14.0 -yavin-worker-jrbf.c.example-com.internal node Ready 5m v1.14.0 -yavin-worker-mzdm.c.example-com.internal node Ready 5m v1.14.0 +yavin-controller-0.c.example-com.internal controller,master Ready 6m v1.14.1 +yavin-worker-jrbf.c.example-com.internal node Ready 5m v1.14.1 +yavin-worker-mzdm.c.example-com.internal node Ready 5m v1.14.1 ``` List the pods. diff --git a/aws/container-linux/kubernetes/README.md b/aws/container-linux/kubernetes/README.md index 18239c431..7a66e9b57 100644 --- a/aws/container-linux/kubernetes/README.md +++ b/aws/container-linux/kubernetes/README.md @@ -11,7 +11,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features -* Kubernetes v1.14.0 (upstream, via [kubernetes-incubator/bootkube](https://github.com/kubernetes-incubator/bootkube)) +* Kubernetes v1.14.1 (upstream, via [kubernetes-incubator/bootkube](https://github.com/kubernetes-incubator/bootkube)) * Single or multi-master, [Calico](https://www.projectcalico.org/) or [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/) * Advanced features like [worker pools](https://typhoon.psdn.io/advanced/worker-pools/), [spot](https://typhoon.psdn.io/cl/aws/#spot) workers, and [snippets](https://typhoon.psdn.io/advanced/customization/#container-linux) customization diff --git a/aws/container-linux/kubernetes/bootkube.tf b/aws/container-linux/kubernetes/bootkube.tf index be6212077..605f57a5d 100644 --- a/aws/container-linux/kubernetes/bootkube.tf +++ b/aws/container-linux/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=feb6e4cb3e479b20dfc269f65e76ceb62d8d2ec4" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=a80eed2b6ac489243a6454dc2f46b17eefa7d84d" cluster_name = "${var.cluster_name}" api_servers = ["${format("%s.%s", var.cluster_name, var.dns_zone)}"] diff --git a/aws/container-linux/kubernetes/cl/controller.yaml.tmpl b/aws/container-linux/kubernetes/cl/controller.yaml.tmpl index d8248e7e3..d6472d4ce 100644 --- a/aws/container-linux/kubernetes/cl/controller.yaml.tmpl +++ b/aws/container-linux/kubernetes/cl/controller.yaml.tmpl @@ -123,7 +123,7 @@ storage: contents: inline: | KUBELET_IMAGE_URL=docker://k8s.gcr.io/hyperkube - KUBELET_IMAGE_TAG=v1.14.0 + KUBELET_IMAGE_TAG=v1.14.1 - path: /etc/sysctl.d/max-user-watches.conf filesystem: root contents: diff --git a/aws/container-linux/kubernetes/workers/cl/worker.yaml.tmpl b/aws/container-linux/kubernetes/workers/cl/worker.yaml.tmpl index a42da3852..84b816d9c 100644 --- a/aws/container-linux/kubernetes/workers/cl/worker.yaml.tmpl +++ b/aws/container-linux/kubernetes/workers/cl/worker.yaml.tmpl @@ -93,7 +93,7 @@ storage: contents: inline: | KUBELET_IMAGE_URL=docker://k8s.gcr.io/hyperkube - KUBELET_IMAGE_TAG=v1.14.0 + KUBELET_IMAGE_TAG=v1.14.1 - path: /etc/sysctl.d/max-user-watches.conf filesystem: root contents: @@ -111,7 +111,7 @@ storage: --volume config,kind=host,source=/etc/kubernetes \ --mount volume=config,target=/etc/kubernetes \ --insecure-options=image \ - docker://k8s.gcr.io/hyperkube:v1.14.0 \ + docker://k8s.gcr.io/hyperkube:v1.14.1 \ --net=host \ --dns=host \ --exec=/kubectl -- --kubeconfig=/etc/kubernetes/kubeconfig delete node $(hostname) diff --git a/aws/fedora-atomic/kubernetes/README.md b/aws/fedora-atomic/kubernetes/README.md index 66a22c3d9..cdb64baa2 100644 --- a/aws/fedora-atomic/kubernetes/README.md +++ b/aws/fedora-atomic/kubernetes/README.md @@ -11,7 +11,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features -* Kubernetes v1.14.0 (upstream, via [kubernetes-incubator/bootkube](https://github.com/kubernetes-incubator/bootkube)) +* Kubernetes v1.14.1 (upstream, via [kubernetes-incubator/bootkube](https://github.com/kubernetes-incubator/bootkube)) * Single or multi-master, [Calico](https://www.projectcalico.org/) or [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/) * Advanced features like [worker pools](https://typhoon.psdn.io/advanced/worker-pools/) and [spot](https://typhoon.psdn.io/cl/aws/#spot) workers diff --git a/aws/fedora-atomic/kubernetes/bootkube.tf b/aws/fedora-atomic/kubernetes/bootkube.tf index 054e4b09e..6d00cbeda 100644 --- a/aws/fedora-atomic/kubernetes/bootkube.tf +++ b/aws/fedora-atomic/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=feb6e4cb3e479b20dfc269f65e76ceb62d8d2ec4" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=a80eed2b6ac489243a6454dc2f46b17eefa7d84d" cluster_name = "${var.cluster_name}" api_servers = ["${format("%s.%s", var.cluster_name, var.dns_zone)}"] diff --git a/aws/fedora-atomic/kubernetes/cloudinit/controller.yaml.tmpl b/aws/fedora-atomic/kubernetes/cloudinit/controller.yaml.tmpl index 5949d55d2..64c1058e7 100644 --- a/aws/fedora-atomic/kubernetes/cloudinit/controller.yaml.tmpl +++ b/aws/fedora-atomic/kubernetes/cloudinit/controller.yaml.tmpl @@ -79,7 +79,7 @@ runcmd: - [systemctl, daemon-reload] - [systemctl, restart, NetworkManager] - "atomic install --system --name=etcd quay.io/poseidon/etcd:v3.3.12" - - "atomic install --system --name=kubelet quay.io/poseidon/kubelet:v1.14.0" + - "atomic install --system --name=kubelet quay.io/poseidon/kubelet:v1.14.1" - "atomic install --system --name=bootkube quay.io/poseidon/bootkube:v0.14.0" - [systemctl, start, --no-block, etcd.service] - [systemctl, start, --no-block, kubelet.service] diff --git a/aws/fedora-atomic/kubernetes/workers/cloudinit/worker.yaml.tmpl b/aws/fedora-atomic/kubernetes/workers/cloudinit/worker.yaml.tmpl index 373a7163a..680c013a6 100644 --- a/aws/fedora-atomic/kubernetes/workers/cloudinit/worker.yaml.tmpl +++ b/aws/fedora-atomic/kubernetes/workers/cloudinit/worker.yaml.tmpl @@ -54,7 +54,7 @@ bootcmd: runcmd: - [systemctl, daemon-reload] - [systemctl, restart, NetworkManager] - - "atomic install --system --name=kubelet quay.io/poseidon/kubelet:v1.14.0" + - "atomic install --system --name=kubelet quay.io/poseidon/kubelet:v1.14.1" - [systemctl, start, --no-block, kubelet.service] users: - default diff --git a/azure/container-linux/kubernetes/README.md b/azure/container-linux/kubernetes/README.md index 96ea18d76..819a70ae9 100644 --- a/azure/container-linux/kubernetes/README.md +++ b/azure/container-linux/kubernetes/README.md @@ -11,7 +11,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features -* Kubernetes v1.14.0 (upstream, via [kubernetes-incubator/bootkube](https://github.com/kubernetes-incubator/bootkube)) +* Kubernetes v1.14.1 (upstream, via [kubernetes-incubator/bootkube](https://github.com/kubernetes-incubator/bootkube)) * Single or multi-master, [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled * Advanced features like [worker pools](https://typhoon.psdn.io/advanced/worker-pools/), [low-priority](https://typhoon.psdn.io/cl/azure/#low-priority) workers, and [snippets](https://typhoon.psdn.io/advanced/customization/#container-linux) customization diff --git a/azure/container-linux/kubernetes/bootkube.tf b/azure/container-linux/kubernetes/bootkube.tf index a1b1732ec..d9de6b92e 100644 --- a/azure/container-linux/kubernetes/bootkube.tf +++ b/azure/container-linux/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=feb6e4cb3e479b20dfc269f65e76ceb62d8d2ec4" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=a80eed2b6ac489243a6454dc2f46b17eefa7d84d" cluster_name = "${var.cluster_name}" api_servers = ["${format("%s.%s", var.cluster_name, var.dns_zone)}"] diff --git a/azure/container-linux/kubernetes/cl/controller.yaml.tmpl b/azure/container-linux/kubernetes/cl/controller.yaml.tmpl index d8248e7e3..d6472d4ce 100644 --- a/azure/container-linux/kubernetes/cl/controller.yaml.tmpl +++ b/azure/container-linux/kubernetes/cl/controller.yaml.tmpl @@ -123,7 +123,7 @@ storage: contents: inline: | KUBELET_IMAGE_URL=docker://k8s.gcr.io/hyperkube - KUBELET_IMAGE_TAG=v1.14.0 + KUBELET_IMAGE_TAG=v1.14.1 - path: /etc/sysctl.d/max-user-watches.conf filesystem: root contents: diff --git a/azure/container-linux/kubernetes/workers/cl/worker.yaml.tmpl b/azure/container-linux/kubernetes/workers/cl/worker.yaml.tmpl index 6459a9661..aade6ef8b 100644 --- a/azure/container-linux/kubernetes/workers/cl/worker.yaml.tmpl +++ b/azure/container-linux/kubernetes/workers/cl/worker.yaml.tmpl @@ -93,7 +93,7 @@ storage: contents: inline: | KUBELET_IMAGE_URL=docker://k8s.gcr.io/hyperkube - KUBELET_IMAGE_TAG=v1.14.0 + KUBELET_IMAGE_TAG=v1.14.1 - path: /etc/sysctl.d/max-user-watches.conf filesystem: root contents: @@ -111,7 +111,7 @@ storage: --volume config,kind=host,source=/etc/kubernetes \ --mount volume=config,target=/etc/kubernetes \ --insecure-options=image \ - docker://k8s.gcr.io/hyperkube:v1.14.0 \ + docker://k8s.gcr.io/hyperkube:v1.14.1 \ --net=host \ --dns=host \ --exec=/kubectl -- --kubeconfig=/etc/kubernetes/kubeconfig delete node $(hostname | tr '[:upper:]' '[:lower:]') diff --git a/bare-metal/container-linux/kubernetes/README.md b/bare-metal/container-linux/kubernetes/README.md index 8a368327b..835ae8bb9 100644 --- a/bare-metal/container-linux/kubernetes/README.md +++ b/bare-metal/container-linux/kubernetes/README.md @@ -11,7 +11,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features -* Kubernetes v1.14.0 (upstream, via [kubernetes-incubator/bootkube](https://github.com/kubernetes-incubator/bootkube)) +* Kubernetes v1.14.1 (upstream, via [kubernetes-incubator/bootkube](https://github.com/kubernetes-incubator/bootkube)) * Single or multi-master, [Calico](https://www.projectcalico.org/) or [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/) * Advanced features like [snippets](https://typhoon.psdn.io/advanced/customization/#container-linux) customization diff --git a/bare-metal/container-linux/kubernetes/bootkube.tf b/bare-metal/container-linux/kubernetes/bootkube.tf index 75f923453..062ddcf8f 100644 --- a/bare-metal/container-linux/kubernetes/bootkube.tf +++ b/bare-metal/container-linux/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=feb6e4cb3e479b20dfc269f65e76ceb62d8d2ec4" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=a80eed2b6ac489243a6454dc2f46b17eefa7d84d" cluster_name = "${var.cluster_name}" api_servers = ["${var.k8s_domain_name}"] diff --git a/bare-metal/container-linux/kubernetes/cl/controller.yaml.tmpl b/bare-metal/container-linux/kubernetes/cl/controller.yaml.tmpl index 143fc178f..b8e29c37e 100644 --- a/bare-metal/container-linux/kubernetes/cl/controller.yaml.tmpl +++ b/bare-metal/container-linux/kubernetes/cl/controller.yaml.tmpl @@ -128,7 +128,7 @@ storage: contents: inline: | KUBELET_IMAGE_URL=docker://k8s.gcr.io/hyperkube - KUBELET_IMAGE_TAG=v1.14.0 + KUBELET_IMAGE_TAG=v1.14.1 - path: /etc/hostname filesystem: root mode: 0644 diff --git a/bare-metal/container-linux/kubernetes/cl/worker.yaml.tmpl b/bare-metal/container-linux/kubernetes/cl/worker.yaml.tmpl index da3b29703..f7a1b5fb0 100644 --- a/bare-metal/container-linux/kubernetes/cl/worker.yaml.tmpl +++ b/bare-metal/container-linux/kubernetes/cl/worker.yaml.tmpl @@ -89,7 +89,7 @@ storage: contents: inline: | KUBELET_IMAGE_URL=docker://k8s.gcr.io/hyperkube - KUBELET_IMAGE_TAG=v1.14.0 + KUBELET_IMAGE_TAG=v1.14.1 - path: /etc/hostname filesystem: root mode: 0644 diff --git a/bare-metal/fedora-atomic/kubernetes/README.md b/bare-metal/fedora-atomic/kubernetes/README.md index dab414ff7..6ddd86098 100644 --- a/bare-metal/fedora-atomic/kubernetes/README.md +++ b/bare-metal/fedora-atomic/kubernetes/README.md @@ -11,7 +11,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features -* Kubernetes v1.14.0 (upstream, via [kubernetes-incubator/bootkube](https://github.com/kubernetes-incubator/bootkube)) +* Kubernetes v1.14.1 (upstream, via [kubernetes-incubator/bootkube](https://github.com/kubernetes-incubator/bootkube)) * Single or multi-master, [Calico](https://www.projectcalico.org/) or [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/) * Ready for Ingress, Prometheus, Grafana, and other optional [addons](https://typhoon.psdn.io/addons/overview/) diff --git a/bare-metal/fedora-atomic/kubernetes/bootkube.tf b/bare-metal/fedora-atomic/kubernetes/bootkube.tf index 5cb852fb5..934b39643 100644 --- a/bare-metal/fedora-atomic/kubernetes/bootkube.tf +++ b/bare-metal/fedora-atomic/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=feb6e4cb3e479b20dfc269f65e76ceb62d8d2ec4" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=a80eed2b6ac489243a6454dc2f46b17eefa7d84d" cluster_name = "${var.cluster_name}" api_servers = ["${var.k8s_domain_name}"] diff --git a/bare-metal/fedora-atomic/kubernetes/cloudinit/controller.yaml.tmpl b/bare-metal/fedora-atomic/kubernetes/cloudinit/controller.yaml.tmpl index bed3dce5d..a43d58fd2 100644 --- a/bare-metal/fedora-atomic/kubernetes/cloudinit/controller.yaml.tmpl +++ b/bare-metal/fedora-atomic/kubernetes/cloudinit/controller.yaml.tmpl @@ -85,7 +85,7 @@ runcmd: - [systemctl, restart, NetworkManager] - [hostnamectl, set-hostname, ${domain_name}] - "atomic install --system --name=etcd quay.io/poseidon/etcd:v3.3.12" - - "atomic install --system --name=kubelet quay.io/poseidon/kubelet:v1.14.0" + - "atomic install --system --name=kubelet quay.io/poseidon/kubelet:v1.14.1" - "atomic install --system --name=bootkube quay.io/poseidon/bootkube:v0.14.0" - [systemctl, start, --no-block, etcd.service] - [systemctl, enable, kubelet.path] diff --git a/bare-metal/fedora-atomic/kubernetes/cloudinit/worker.yaml.tmpl b/bare-metal/fedora-atomic/kubernetes/cloudinit/worker.yaml.tmpl index a01c2cd3a..2d4c5cf01 100644 --- a/bare-metal/fedora-atomic/kubernetes/cloudinit/worker.yaml.tmpl +++ b/bare-metal/fedora-atomic/kubernetes/cloudinit/worker.yaml.tmpl @@ -60,7 +60,7 @@ runcmd: - [systemctl, daemon-reload] - [systemctl, restart, NetworkManager] - [hostnamectl, set-hostname, ${domain_name}] - - "atomic install --system --name=kubelet quay.io/poseidon/kubelet:v1.14.0" + - "atomic install --system --name=kubelet quay.io/poseidon/kubelet:v1.14.1" - [systemctl, enable, kubelet.path] - [systemctl, start, --no-block, kubelet.path] users: diff --git a/digital-ocean/container-linux/kubernetes/README.md b/digital-ocean/container-linux/kubernetes/README.md index 0e81f7822..13ec7d334 100644 --- a/digital-ocean/container-linux/kubernetes/README.md +++ b/digital-ocean/container-linux/kubernetes/README.md @@ -11,7 +11,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features -* Kubernetes v1.14.0 (upstream, via [kubernetes-incubator/bootkube](https://github.com/kubernetes-incubator/bootkube)) +* Kubernetes v1.14.1 (upstream, via [kubernetes-incubator/bootkube](https://github.com/kubernetes-incubator/bootkube)) * Single or multi-master, [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled * Advanced features like [snippets](https://typhoon.psdn.io/advanced/customization/#container-linux) customization diff --git a/digital-ocean/container-linux/kubernetes/bootkube.tf b/digital-ocean/container-linux/kubernetes/bootkube.tf index 878af4f16..16dd9805c 100644 --- a/digital-ocean/container-linux/kubernetes/bootkube.tf +++ b/digital-ocean/container-linux/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=feb6e4cb3e479b20dfc269f65e76ceb62d8d2ec4" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=a80eed2b6ac489243a6454dc2f46b17eefa7d84d" cluster_name = "${var.cluster_name}" api_servers = ["${format("%s.%s", var.cluster_name, var.dns_zone)}"] diff --git a/digital-ocean/container-linux/kubernetes/cl/controller.yaml.tmpl b/digital-ocean/container-linux/kubernetes/cl/controller.yaml.tmpl index 1fb193f11..84ac9c41f 100644 --- a/digital-ocean/container-linux/kubernetes/cl/controller.yaml.tmpl +++ b/digital-ocean/container-linux/kubernetes/cl/controller.yaml.tmpl @@ -129,7 +129,7 @@ storage: contents: inline: | KUBELET_IMAGE_URL=docker://k8s.gcr.io/hyperkube - KUBELET_IMAGE_TAG=v1.14.0 + KUBELET_IMAGE_TAG=v1.14.1 - path: /etc/sysctl.d/max-user-watches.conf filesystem: root contents: diff --git a/digital-ocean/container-linux/kubernetes/cl/worker.yaml.tmpl b/digital-ocean/container-linux/kubernetes/cl/worker.yaml.tmpl index 51b836752..c724a6e9e 100644 --- a/digital-ocean/container-linux/kubernetes/cl/worker.yaml.tmpl +++ b/digital-ocean/container-linux/kubernetes/cl/worker.yaml.tmpl @@ -99,7 +99,7 @@ storage: contents: inline: | KUBELET_IMAGE_URL=docker://k8s.gcr.io/hyperkube - KUBELET_IMAGE_TAG=v1.14.0 + KUBELET_IMAGE_TAG=v1.14.1 - path: /etc/sysctl.d/max-user-watches.conf filesystem: root contents: @@ -117,7 +117,7 @@ storage: --volume config,kind=host,source=/etc/kubernetes \ --mount volume=config,target=/etc/kubernetes \ --insecure-options=image \ - docker://k8s.gcr.io/hyperkube:v1.14.0 \ + docker://k8s.gcr.io/hyperkube:v1.14.1 \ --net=host \ --dns=host \ --exec=/kubectl -- --kubeconfig=/etc/kubernetes/kubeconfig delete node $(hostname) diff --git a/digital-ocean/fedora-atomic/kubernetes/README.md b/digital-ocean/fedora-atomic/kubernetes/README.md index eaa5934bd..2ccbfa8e7 100644 --- a/digital-ocean/fedora-atomic/kubernetes/README.md +++ b/digital-ocean/fedora-atomic/kubernetes/README.md @@ -11,7 +11,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features -* Kubernetes v1.14.0 (upstream, via [kubernetes-incubator/bootkube](https://github.com/kubernetes-incubator/bootkube)) +* Kubernetes v1.14.1 (upstream, via [kubernetes-incubator/bootkube](https://github.com/kubernetes-incubator/bootkube)) * Single or multi-master, [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled * Ready for Ingress, Prometheus, Grafana, and other optional [addons](https://typhoon.psdn.io/addons/overview/) diff --git a/digital-ocean/fedora-atomic/kubernetes/bootkube.tf b/digital-ocean/fedora-atomic/kubernetes/bootkube.tf index 11e59d4d1..6d37b646f 100644 --- a/digital-ocean/fedora-atomic/kubernetes/bootkube.tf +++ b/digital-ocean/fedora-atomic/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=feb6e4cb3e479b20dfc269f65e76ceb62d8d2ec4" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=a80eed2b6ac489243a6454dc2f46b17eefa7d84d" cluster_name = "${var.cluster_name}" api_servers = ["${format("%s.%s", var.cluster_name, var.dns_zone)}"] diff --git a/digital-ocean/fedora-atomic/kubernetes/cloudinit/controller.yaml.tmpl b/digital-ocean/fedora-atomic/kubernetes/cloudinit/controller.yaml.tmpl index 71cf564a6..e03312815 100644 --- a/digital-ocean/fedora-atomic/kubernetes/cloudinit/controller.yaml.tmpl +++ b/digital-ocean/fedora-atomic/kubernetes/cloudinit/controller.yaml.tmpl @@ -91,7 +91,7 @@ bootcmd: runcmd: - [systemctl, daemon-reload] - "atomic install --system --name=etcd quay.io/poseidon/etcd:v3.3.12" - - "atomic install --system --name=kubelet quay.io/poseidon/kubelet:v1.14.0" + - "atomic install --system --name=kubelet quay.io/poseidon/kubelet:v1.14.1" - "atomic install --system --name=bootkube quay.io/poseidon/bootkube:v0.14.0" - [systemctl, start, --no-block, etcd.service] - [systemctl, enable, cloud-metadata.service] diff --git a/digital-ocean/fedora-atomic/kubernetes/cloudinit/worker.yaml.tmpl b/digital-ocean/fedora-atomic/kubernetes/cloudinit/worker.yaml.tmpl index 8db4e0f47..0bc273bab 100644 --- a/digital-ocean/fedora-atomic/kubernetes/cloudinit/worker.yaml.tmpl +++ b/digital-ocean/fedora-atomic/kubernetes/cloudinit/worker.yaml.tmpl @@ -67,7 +67,7 @@ bootcmd: runcmd: - [systemctl, daemon-reload] - [systemctl, enable, cloud-metadata.service] - - "atomic install --system --name=kubelet quay.io/poseidon/kubelet:v1.14.0" + - "atomic install --system --name=kubelet quay.io/poseidon/kubelet:v1.14.1" - [systemctl, enable, kubelet.path] - [systemctl, start, --no-block, kubelet.path] users: diff --git a/docs/advanced/worker-pools.md b/docs/advanced/worker-pools.md index 955cd8517..26b58f0a9 100644 --- a/docs/advanced/worker-pools.md +++ b/docs/advanced/worker-pools.md @@ -16,7 +16,7 @@ Create a cluster following the AWS [tutorial](../cl/aws.md#cluster). Define a wo ```tf module "tempest-worker-pool" { - source = "git::https://github.com/poseidon/typhoon//aws/container-linux/kubernetes/workers?ref=v1.14.0" + source = "git::https://github.com/poseidon/typhoon//aws/container-linux/kubernetes/workers?ref=v1.14.1" providers = { aws = "aws.default" @@ -82,7 +82,7 @@ Create a cluster following the Azure [tutorial](../cl/azure.md#cluster). Define ```tf module "ramius-worker-pool" { - source = "git::https://github.com/poseidon/typhoon//azure/container-linux/kubernetes/workers?ref=v1.14.0" + source = "git::https://github.com/poseidon/typhoon//azure/container-linux/kubernetes/workers?ref=v1.14.1" providers = { azurerm = "azurerm.default" @@ -152,7 +152,7 @@ Create a cluster following the Google Cloud [tutorial](../cl/google-cloud.md#clu ```tf module "yavin-worker-pool" { - source = "git::https://github.com/poseidon/typhoon//google-cloud/container-linux/kubernetes/workers?ref=v1.14.0" + source = "git::https://github.com/poseidon/typhoon//google-cloud/container-linux/kubernetes/workers?ref=v1.14.1" providers = { google = "google.default" @@ -187,11 +187,11 @@ Verify a managed instance group of workers joins the cluster within a few minute ``` $ kubectl get nodes NAME STATUS AGE VERSION -yavin-controller-0.c.example-com.internal Ready 6m v1.14.0 -yavin-worker-jrbf.c.example-com.internal Ready 5m v1.14.0 -yavin-worker-mzdm.c.example-com.internal Ready 5m v1.14.0 -yavin-16x-worker-jrbf.c.example-com.internal Ready 3m v1.14.0 -yavin-16x-worker-mzdm.c.example-com.internal Ready 3m v1.14.0 +yavin-controller-0.c.example-com.internal Ready 6m v1.14.1 +yavin-worker-jrbf.c.example-com.internal Ready 5m v1.14.1 +yavin-worker-mzdm.c.example-com.internal Ready 5m v1.14.1 +yavin-16x-worker-jrbf.c.example-com.internal Ready 3m v1.14.1 +yavin-16x-worker-mzdm.c.example-com.internal Ready 3m v1.14.1 ``` ### Variables diff --git a/docs/atomic/aws.md b/docs/atomic/aws.md index 3200f55c0..7e1668bba 100644 --- a/docs/atomic/aws.md +++ b/docs/atomic/aws.md @@ -3,7 +3,7 @@ !!! danger Typhoon for Fedora Atomic will not be updated much beyond Kubernetes v1.13. -In this tutorial, we'll create a Kubernetes v1.14.0 cluster on AWS with Fedora Atomic. +In this tutorial, we'll create a Kubernetes v1.14.1 cluster on AWS with Fedora Atomic. We'll declare a Kubernetes cluster using the Typhoon Terraform module. Then apply the changes to create a VPC, gateway, subnets, security groups, controller instances, worker auto-scaling group, network load balancer, and TLS assets. Instances are provisioned on first boot with cloud-init. @@ -83,7 +83,7 @@ Define a Kubernetes cluster using the module `aws/fedora-atomic/kubernetes`. ```tf module "aws-tempest" { - source = "git::https://github.com/poseidon/typhoon//aws/fedora-atomic/kubernetes?ref=v1.14.0" + source = "git::https://github.com/poseidon/typhoon//aws/fedora-atomic/kubernetes?ref=v1.14.1" providers = { aws = "aws.default" @@ -156,9 +156,9 @@ In 5-10 minutes, the Kubernetes cluster will be ready. $ export KUBECONFIG=/home/user/.secrets/clusters/tempest/auth/kubeconfig $ kubectl get nodes NAME STATUS ROLES AGE VERSION -ip-10-0-3-155 Ready controller,master 10m v1.14.0 -ip-10-0-26-65 Ready node 10m v1.14.0 -ip-10-0-41-21 Ready node 10m v1.14.0 +ip-10-0-3-155 Ready controller,master 10m v1.14.1 +ip-10-0-26-65 Ready node 10m v1.14.1 +ip-10-0-41-21 Ready node 10m v1.14.1 ``` List the pods. diff --git a/docs/atomic/bare-metal.md b/docs/atomic/bare-metal.md index 7dde69893..c83a6e3da 100644 --- a/docs/atomic/bare-metal.md +++ b/docs/atomic/bare-metal.md @@ -3,7 +3,7 @@ !!! danger Typhoon for Fedora Atomic will not be updated much beyond Kubernetes v1.13. -In this tutorial, we'll network boot and provision a Kubernetes v1.14.0 cluster on bare-metal with Fedora Atomic. +In this tutorial, we'll network boot and provision a Kubernetes v1.14.1 cluster on bare-metal with Fedora Atomic. First, we'll deploy a [Matchbox](https://github.com/coreos/matchbox) service and setup a network boot environment. Then, we'll declare a Kubernetes cluster using the Typhoon Terraform module and power on machines. On PXE boot, machines will install Fedora Atomic via kickstart, reboot into the disk install, and provision themselves as Kubernetes controllers or workers via cloud-init. @@ -228,7 +228,7 @@ Define a Kubernetes cluster using the module `bare-metal/fedora-atomic/kubernete ```tf module "bare-metal-mercury" { - source = "git::https://github.com/poseidon/typhoon//bare-metal/fedora-atomic/kubernetes?ref=v1.14.0" + source = "git::https://github.com/poseidon/typhoon//bare-metal/fedora-atomic/kubernetes?ref=v1.14.1" providers = { local = "local.default" @@ -354,9 +354,9 @@ bootkube[5]: Tearing down temporary bootstrap control plane... $ export KUBECONFIG=/home/user/.secrets/clusters/mercury/auth/kubeconfig $ kubectl get nodes NAME STATUS ROLES AGE VERSION -node1.example.com Ready controller,master 10m v1.14.0 -node2.example.com Ready node 10m v1.14.0 -node3.example.com Ready node 10m v1.14.0 +node1.example.com Ready controller,master 10m v1.14.1 +node2.example.com Ready node 10m v1.14.1 +node3.example.com Ready node 10m v1.14.1 ``` List the pods. diff --git a/docs/atomic/digital-ocean.md b/docs/atomic/digital-ocean.md index d8d97ac7a..30fc8723d 100644 --- a/docs/atomic/digital-ocean.md +++ b/docs/atomic/digital-ocean.md @@ -3,7 +3,7 @@ !!! danger Typhoon for Fedora Atomic will not be updated much beyond Kubernetes v1.13. -In this tutorial, we'll create a Kubernetes v1.14.0 cluster on DigitalOcean with Fedora Atomic. +In this tutorial, we'll create a Kubernetes v1.14.1 cluster on DigitalOcean with Fedora Atomic. We'll declare a Kubernetes cluster using the Typhoon Terraform module. Then apply the changes to create controller droplets, worker droplets, DNS records, tags, and TLS assets. Instances are provisioned on first boot with cloud-init. @@ -77,7 +77,7 @@ Define a Kubernetes cluster using the module `digital-ocean/fedora-atomic/kubern ```tf module "digital-ocean-nemo" { - source = "git::https://github.com/poseidon/typhoon//digital-ocean/fedora-atomic/kubernetes?ref=v1.14.0" + source = "git::https://github.com/poseidon/typhoon//digital-ocean/fedora-atomic/kubernetes?ref=v1.14.1" providers = { digitalocean = "digitalocean.default" @@ -152,9 +152,9 @@ In 3-6 minutes, the Kubernetes cluster will be ready. $ export KUBECONFIG=/home/user/.secrets/clusters/nemo/auth/kubeconfig $ kubectl get nodes NAME STATUS ROLES AGE VERSION -10.132.110.130 Ready controller,master 10m v1.14.0 -10.132.115.81 Ready node 10m v1.14.0 -10.132.124.107 Ready node 10m v1.14.0 +10.132.110.130 Ready controller,master 10m v1.14.1 +10.132.115.81 Ready node 10m v1.14.1 +10.132.124.107 Ready node 10m v1.14.1 ``` List the pods. diff --git a/docs/atomic/google-cloud.md b/docs/atomic/google-cloud.md index fc87174b6..13a6eb0d0 100644 --- a/docs/atomic/google-cloud.md +++ b/docs/atomic/google-cloud.md @@ -3,7 +3,7 @@ !!! danger Typhoon for Fedora Atomic will not be updated much beyond Kubernetes v1.13. Fedora does not publish official images for Google Cloud so you must prepare them yourself. Expect rough edges and changes. -In this tutorial, we'll create a Kubernetes v1.14.0 cluster on Google Compute Engine with Fedora Atomic. +In this tutorial, we'll create a Kubernetes v1.14.1 cluster on Google Compute Engine with Fedora Atomic. We'll declare a Kubernetes cluster using the Typhoon Terraform module. Then apply the changes to create a network, firewall rules, health checks, controller instances, worker managed instance group, load balancers, and TLS assets. Instances are provisioned on first boot with cloud-init. @@ -121,7 +121,7 @@ Define a Kubernetes cluster using the module `google-cloud/fedora-atomic/kuberne ```tf module "google-cloud-yavin" { - source = "git::https://github.com/poseidon/typhoon//google-cloud/fedora-atomic/kubernetes?ref=v1.14.0" + source = "git::https://github.com/poseidon/typhoon//google-cloud/fedora-atomic/kubernetes?ref=v1.14.1" providers = { google = "google.default" @@ -197,9 +197,9 @@ In 5-10 minutes, the Kubernetes cluster will be ready. $ export KUBECONFIG=/home/user/.secrets/clusters/yavin/auth/kubeconfig $ kubectl get nodes NAME ROLES STATUS AGE VERSION -yavin-controller-0.c.example-com.internal controller,master Ready 6m v1.14.0 -yavin-worker-jrbf.c.example-com.internal node Ready 5m v1.14.0 -yavin-worker-mzdm.c.example-com.internal node Ready 5m v1.14.0 +yavin-controller-0.c.example-com.internal controller,master Ready 6m v1.14.1 +yavin-worker-jrbf.c.example-com.internal node Ready 5m v1.14.1 +yavin-worker-mzdm.c.example-com.internal node Ready 5m v1.14.1 ``` List the pods. diff --git a/docs/cl/aws.md b/docs/cl/aws.md index a7607f878..c2ed02f92 100644 --- a/docs/cl/aws.md +++ b/docs/cl/aws.md @@ -1,6 +1,6 @@ # AWS -In this tutorial, we'll create a Kubernetes v1.14.0 cluster on AWS with Container Linux. +In this tutorial, we'll create a Kubernetes v1.14.1 cluster on AWS with Container Linux. We'll declare a Kubernetes cluster using the Typhoon Terraform module. Then apply the changes to create a VPC, gateway, subnets, security groups, controller instances, worker auto-scaling group, network load balancer, and TLS assets. @@ -92,7 +92,7 @@ Define a Kubernetes cluster using the module `aws/container-linux/kubernetes`. ```tf module "aws-tempest" { - source = "git::https://github.com/poseidon/typhoon//aws/container-linux/kubernetes?ref=v1.14.0" + source = "git::https://github.com/poseidon/typhoon//aws/container-linux/kubernetes?ref=v1.14.1" providers = { aws = "aws.default" @@ -165,9 +165,9 @@ In 4-8 minutes, the Kubernetes cluster will be ready. $ export KUBECONFIG=/home/user/.secrets/clusters/tempest/auth/kubeconfig $ kubectl get nodes NAME STATUS ROLES AGE VERSION -ip-10-0-3-155 Ready controller,master 10m v1.14.0 -ip-10-0-26-65 Ready node 10m v1.14.0 -ip-10-0-41-21 Ready node 10m v1.14.0 +ip-10-0-3-155 Ready controller,master 10m v1.14.1 +ip-10-0-26-65 Ready node 10m v1.14.1 +ip-10-0-41-21 Ready node 10m v1.14.1 ``` List the pods. diff --git a/docs/cl/azure.md b/docs/cl/azure.md index 590a5ef5a..144ee3982 100644 --- a/docs/cl/azure.md +++ b/docs/cl/azure.md @@ -3,7 +3,7 @@ !!! danger Typhoon for Azure is alpha. For production, use AWS, Google Cloud, or bare-metal. As Azure matures, check [errata](https://github.com/poseidon/typhoon/wiki/Errata) for known shortcomings. -In this tutorial, we'll create a Kubernetes v1.14.0 cluster on Azure with Container Linux. +In this tutorial, we'll create a Kubernetes v1.14.1 cluster on Azure with Container Linux. We'll declare a Kubernetes cluster using the Typhoon Terraform module. Then apply the changes to create a resource group, virtual network, subnets, security groups, controller availability set, worker scale set, load balancer, and TLS assets. @@ -87,7 +87,7 @@ Define a Kubernetes cluster using the module `azure/container-linux/kubernetes`. ```tf module "azure-ramius" { - source = "git::https://github.com/poseidon/typhoon//azure/container-linux/kubernetes?ref=v1.14.0" + source = "git::https://github.com/poseidon/typhoon//azure/container-linux/kubernetes?ref=v1.14.1" providers = { azurerm = "azurerm.default" @@ -161,9 +161,9 @@ In 4-8 minutes, the Kubernetes cluster will be ready. $ export KUBECONFIG=/home/user/.secrets/clusters/ramius/auth/kubeconfig $ kubectl get nodes NAME STATUS ROLES AGE VERSION -ramius-controller-0 Ready controller,master 24m v1.14.0 -ramius-worker-000001 Ready node 25m v1.14.0 -ramius-worker-000002 Ready node 24m v1.14.0 +ramius-controller-0 Ready controller,master 24m v1.14.1 +ramius-worker-000001 Ready node 25m v1.14.1 +ramius-worker-000002 Ready node 24m v1.14.1 ``` List the pods. diff --git a/docs/cl/bare-metal.md b/docs/cl/bare-metal.md index 418a8a239..fcd869626 100644 --- a/docs/cl/bare-metal.md +++ b/docs/cl/bare-metal.md @@ -1,6 +1,6 @@ # Bare-Metal -In this tutorial, we'll network boot and provision a Kubernetes v1.14.0 cluster on bare-metal with Container Linux. +In this tutorial, we'll network boot and provision a Kubernetes v1.14.1 cluster on bare-metal with Container Linux. First, we'll deploy a [Matchbox](https://github.com/coreos/matchbox) service and setup a network boot environment. Then, we'll declare a Kubernetes cluster using the Typhoon Terraform module and power on machines. On PXE boot, machines will install Container Linux to disk, reboot into the disk install, and provision themselves as Kubernetes controllers or workers via Ignition. @@ -180,7 +180,7 @@ Define a Kubernetes cluster using the module `bare-metal/container-linux/kuberne ```tf module "bare-metal-mercury" { - source = "git::https://github.com/poseidon/typhoon//bare-metal/container-linux/kubernetes?ref=v1.14.0" + source = "git::https://github.com/poseidon/typhoon//bare-metal/container-linux/kubernetes?ref=v1.14.1" providers = { local = "local.default" @@ -292,9 +292,9 @@ Apply complete! Resources: 55 added, 0 changed, 0 destroyed. To watch the install to disk (until machines reboot from disk), SSH to port 2222. ``` -# before v1.14.0 +# before v1.14.1 $ ssh debug@node1.example.com -# after v1.14.0 +# after v1.14.1 $ ssh -p 2222 core@node1.example.com ``` @@ -319,9 +319,9 @@ bootkube[5]: Tearing down temporary bootstrap control plane... $ export KUBECONFIG=/home/user/.secrets/clusters/mercury/auth/kubeconfig $ kubectl get nodes NAME STATUS ROLES AGE VERSION -node1.example.com Ready controller,master 10m v1.14.0 -node2.example.com Ready node 10m v1.14.0 -node3.example.com Ready node 10m v1.14.0 +node1.example.com Ready controller,master 10m v1.14.1 +node2.example.com Ready node 10m v1.14.1 +node3.example.com Ready node 10m v1.14.1 ``` List the pods. diff --git a/docs/cl/digital-ocean.md b/docs/cl/digital-ocean.md index 7a86097ab..6801c3185 100644 --- a/docs/cl/digital-ocean.md +++ b/docs/cl/digital-ocean.md @@ -1,6 +1,6 @@ # Digital Ocean -In this tutorial, we'll create a Kubernetes v1.14.0 cluster on DigitalOcean with Container Linux. +In this tutorial, we'll create a Kubernetes v1.14.1 cluster on DigitalOcean with Container Linux. We'll declare a Kubernetes cluster using the Typhoon Terraform module. Then apply the changes to create controller droplets, worker droplets, DNS records, tags, and TLS assets. @@ -86,7 +86,7 @@ Define a Kubernetes cluster using the module `digital-ocean/container-linux/kube ```tf module "digital-ocean-nemo" { - source = "git::https://github.com/poseidon/typhoon//digital-ocean/container-linux/kubernetes?ref=v1.14.0" + source = "git::https://github.com/poseidon/typhoon//digital-ocean/container-linux/kubernetes?ref=v1.14.1" providers = { digitalocean = "digitalocean.default" @@ -160,9 +160,9 @@ In 3-6 minutes, the Kubernetes cluster will be ready. $ export KUBECONFIG=/home/user/.secrets/clusters/nemo/auth/kubeconfig $ kubectl get nodes NAME STATUS ROLES AGE VERSION -10.132.110.130 Ready controller,master 10m v1.14.0 -10.132.115.81 Ready node 10m v1.14.0 -10.132.124.107 Ready node 10m v1.14.0 +10.132.110.130 Ready controller,master 10m v1.14.1 +10.132.115.81 Ready node 10m v1.14.1 +10.132.124.107 Ready node 10m v1.14.1 ``` List the pods. diff --git a/docs/cl/google-cloud.md b/docs/cl/google-cloud.md index d8710c421..4afc2235d 100644 --- a/docs/cl/google-cloud.md +++ b/docs/cl/google-cloud.md @@ -1,6 +1,6 @@ # Google Cloud -In this tutorial, we'll create a Kubernetes v1.14.0 cluster on Google Compute Engine with Container Linux. +In this tutorial, we'll create a Kubernetes v1.14.1 cluster on Google Compute Engine with Container Linux. We'll declare a Kubernetes cluster using the Typhoon Terraform module. Then apply the changes to create a network, firewall rules, health checks, controller instances, worker managed instance group, load balancers, and TLS assets. @@ -93,7 +93,7 @@ Define a Kubernetes cluster using the module `google-cloud/container-linux/kuber ```tf module "google-cloud-yavin" { - source = "git::https://github.com/poseidon/typhoon//google-cloud/container-linux/kubernetes?ref=v1.14.0" + source = "git::https://github.com/poseidon/typhoon//google-cloud/container-linux/kubernetes?ref=v1.14.1" providers = { google = "google.default" @@ -168,9 +168,9 @@ In 4-8 minutes, the Kubernetes cluster will be ready. $ export KUBECONFIG=/home/user/.secrets/clusters/yavin/auth/kubeconfig $ kubectl get nodes NAME ROLES STATUS AGE VERSION -yavin-controller-0.c.example-com.internal controller,master Ready 6m v1.14.0 -yavin-worker-jrbf.c.example-com.internal node Ready 5m v1.14.0 -yavin-worker-mzdm.c.example-com.internal node Ready 5m v1.14.0 +yavin-controller-0.c.example-com.internal controller,master Ready 6m v1.14.1 +yavin-worker-jrbf.c.example-com.internal node Ready 5m v1.14.1 +yavin-worker-mzdm.c.example-com.internal node Ready 5m v1.14.1 ``` List the pods. diff --git a/docs/index.md b/docs/index.md index d62565c83..1408af720 100644 --- a/docs/index.md +++ b/docs/index.md @@ -11,7 +11,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features -* Kubernetes v1.14.0 (upstream, via [kubernetes-incubator/bootkube](https://github.com/kubernetes-incubator/bootkube)) +* Kubernetes v1.14.1 (upstream, via [kubernetes-incubator/bootkube](https://github.com/kubernetes-incubator/bootkube)) * Single or multi-master, [Calico](https://www.projectcalico.org/) or [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/) * Advanced features like [worker pools](advanced/worker-pools/), [preemptible](cl/google-cloud/#preemption) workers, and [snippets](advanced/customization/#container-linux) customization @@ -49,7 +49,7 @@ Define a Kubernetes cluster by using the Terraform module for your chosen platfo ```tf module "google-cloud-yavin" { - source = "git::https://github.com/poseidon/typhoon//google-cloud/container-linux/kubernetes?ref=v1.14.0" + source = "git::https://github.com/poseidon/typhoon//google-cloud/container-linux/kubernetes?ref=v1.14.1" providers = { google = "google.default" @@ -90,9 +90,9 @@ In 4-8 minutes (varies by platform), the cluster will be ready. This Google Clou $ export KUBECONFIG=/home/user/.secrets/clusters/yavin/auth/kubeconfig $ kubectl get nodes NAME ROLES STATUS AGE VERSION -yavin-controller-0.c.example-com.internal controller,master Ready 6m v1.14.0 -yavin-worker-jrbf.c.example-com.internal node Ready 5m v1.14.0 -yavin-worker-mzdm.c.example-com.internal node Ready 5m v1.14.0 +yavin-controller-0.c.example-com.internal controller,master Ready 6m v1.14.1 +yavin-worker-jrbf.c.example-com.internal node Ready 5m v1.14.1 +yavin-worker-mzdm.c.example-com.internal node Ready 5m v1.14.1 ``` List the pods. diff --git a/docs/topics/maintenance.md b/docs/topics/maintenance.md index 347f48146..6cd0141e6 100644 --- a/docs/topics/maintenance.md +++ b/docs/topics/maintenance.md @@ -18,7 +18,7 @@ module "google-cloud-yavin" { } module "bare-metal-mercury" { - source = "git::https://github.com/poseidon/typhoon//bare-metal/container-linux/kubernetes?ref=v1.14.0" + source = "git::https://github.com/poseidon/typhoon//bare-metal/container-linux/kubernetes?ref=v1.14.1" ... } ``` diff --git a/google-cloud/container-linux/kubernetes/README.md b/google-cloud/container-linux/kubernetes/README.md index 74cc0a9c3..40787c28c 100644 --- a/google-cloud/container-linux/kubernetes/README.md +++ b/google-cloud/container-linux/kubernetes/README.md @@ -11,7 +11,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features -* Kubernetes v1.14.0 (upstream, via [kubernetes-incubator/bootkube](https://github.com/kubernetes-incubator/bootkube)) +* Kubernetes v1.14.1 (upstream, via [kubernetes-incubator/bootkube](https://github.com/kubernetes-incubator/bootkube)) * Single or multi-master, [Calico](https://www.projectcalico.org/) or [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/) * Advanced features like [worker pools](https://typhoon.psdn.io/advanced/worker-pools/), [preemptible](https://typhoon.psdn.io/cl/google-cloud/#preemption) workers, and [snippets](https://typhoon.psdn.io/advanced/customization/#container-linux) customization diff --git a/google-cloud/container-linux/kubernetes/bootkube.tf b/google-cloud/container-linux/kubernetes/bootkube.tf index 446362c6c..9d8279a85 100644 --- a/google-cloud/container-linux/kubernetes/bootkube.tf +++ b/google-cloud/container-linux/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=feb6e4cb3e479b20dfc269f65e76ceb62d8d2ec4" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=a80eed2b6ac489243a6454dc2f46b17eefa7d84d" cluster_name = "${var.cluster_name}" api_servers = ["${format("%s.%s", var.cluster_name, var.dns_zone)}"] diff --git a/google-cloud/container-linux/kubernetes/cl/controller.yaml.tmpl b/google-cloud/container-linux/kubernetes/cl/controller.yaml.tmpl index 9cb60d496..b7a09a55f 100644 --- a/google-cloud/container-linux/kubernetes/cl/controller.yaml.tmpl +++ b/google-cloud/container-linux/kubernetes/cl/controller.yaml.tmpl @@ -124,7 +124,7 @@ storage: contents: inline: | KUBELET_IMAGE_URL=docker://k8s.gcr.io/hyperkube - KUBELET_IMAGE_TAG=v1.14.0 + KUBELET_IMAGE_TAG=v1.14.1 - path: /etc/sysctl.d/max-user-watches.conf filesystem: root contents: diff --git a/google-cloud/container-linux/kubernetes/workers/cl/worker.yaml.tmpl b/google-cloud/container-linux/kubernetes/workers/cl/worker.yaml.tmpl index e1bf59bfc..ced7def3e 100644 --- a/google-cloud/container-linux/kubernetes/workers/cl/worker.yaml.tmpl +++ b/google-cloud/container-linux/kubernetes/workers/cl/worker.yaml.tmpl @@ -94,7 +94,7 @@ storage: contents: inline: | KUBELET_IMAGE_URL=docker://k8s.gcr.io/hyperkube - KUBELET_IMAGE_TAG=v1.14.0 + KUBELET_IMAGE_TAG=v1.14.1 - path: /etc/sysctl.d/max-user-watches.conf filesystem: root contents: @@ -112,7 +112,7 @@ storage: --volume config,kind=host,source=/etc/kubernetes \ --mount volume=config,target=/etc/kubernetes \ --insecure-options=image \ - docker://k8s.gcr.io/hyperkube:v1.14.0 \ + docker://k8s.gcr.io/hyperkube:v1.14.1 \ --net=host \ --dns=host \ --exec=/kubectl -- --kubeconfig=/etc/kubernetes/kubeconfig delete node $(hostname) diff --git a/google-cloud/fedora-atomic/kubernetes/README.md b/google-cloud/fedora-atomic/kubernetes/README.md index ffb34c71d..25712fee1 100644 --- a/google-cloud/fedora-atomic/kubernetes/README.md +++ b/google-cloud/fedora-atomic/kubernetes/README.md @@ -11,7 +11,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features -* Kubernetes v1.14.0 (upstream, via [kubernetes-incubator/bootkube](https://github.com/kubernetes-incubator/bootkube)) +* Kubernetes v1.14.1 (upstream, via [kubernetes-incubator/bootkube](https://github.com/kubernetes-incubator/bootkube)) * Single or multi-master, [Calico](https://www.projectcalico.org/) or [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/) * Advanced features like [worker pools](https://typhoon.psdn.io/advanced/worker-pools/) and [preemptible](https://typhoon.psdn.io/cl/google-cloud/#preemption) workers diff --git a/google-cloud/fedora-atomic/kubernetes/bootkube.tf b/google-cloud/fedora-atomic/kubernetes/bootkube.tf index a2ca4d8a9..78469044d 100644 --- a/google-cloud/fedora-atomic/kubernetes/bootkube.tf +++ b/google-cloud/fedora-atomic/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=feb6e4cb3e479b20dfc269f65e76ceb62d8d2ec4" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=a80eed2b6ac489243a6454dc2f46b17eefa7d84d" cluster_name = "${var.cluster_name}" api_servers = ["${format("%s.%s", var.cluster_name, var.dns_zone)}"] diff --git a/google-cloud/fedora-atomic/kubernetes/cloudinit/controller.yaml.tmpl b/google-cloud/fedora-atomic/kubernetes/cloudinit/controller.yaml.tmpl index 5949d55d2..64c1058e7 100644 --- a/google-cloud/fedora-atomic/kubernetes/cloudinit/controller.yaml.tmpl +++ b/google-cloud/fedora-atomic/kubernetes/cloudinit/controller.yaml.tmpl @@ -79,7 +79,7 @@ runcmd: - [systemctl, daemon-reload] - [systemctl, restart, NetworkManager] - "atomic install --system --name=etcd quay.io/poseidon/etcd:v3.3.12" - - "atomic install --system --name=kubelet quay.io/poseidon/kubelet:v1.14.0" + - "atomic install --system --name=kubelet quay.io/poseidon/kubelet:v1.14.1" - "atomic install --system --name=bootkube quay.io/poseidon/bootkube:v0.14.0" - [systemctl, start, --no-block, etcd.service] - [systemctl, start, --no-block, kubelet.service] diff --git a/google-cloud/fedora-atomic/kubernetes/workers/cloudinit/worker.yaml.tmpl b/google-cloud/fedora-atomic/kubernetes/workers/cloudinit/worker.yaml.tmpl index 373a7163a..680c013a6 100644 --- a/google-cloud/fedora-atomic/kubernetes/workers/cloudinit/worker.yaml.tmpl +++ b/google-cloud/fedora-atomic/kubernetes/workers/cloudinit/worker.yaml.tmpl @@ -54,7 +54,7 @@ bootcmd: runcmd: - [systemctl, daemon-reload] - [systemctl, restart, NetworkManager] - - "atomic install --system --name=kubelet quay.io/poseidon/kubelet:v1.14.0" + - "atomic install --system --name=kubelet quay.io/poseidon/kubelet:v1.14.1" - [systemctl, start, --no-block, kubelet.service] users: - default From 44c293888b9f1afadf375bb63bbec7405773aa69 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Tue, 9 Apr 2019 22:06:27 -0700 Subject: [PATCH 077/523] Update Grafana from v6.1.1 to v6.1.3 * https://github.com/grafana/grafana/releases/tag/v6.1.3 --- CHANGES.md | 5 +++++ addons/grafana/deployment.yaml | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index 4581fb7b1..a032a1552 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -6,6 +6,11 @@ Notable changes between versions. * Kubernetes [v1.14.1](https://github.com/kubernetes/kubernetes/blob/master/CHANGELOG-1.14.md#v1141) +#### Addons + +* Update Grafana from v6.1.1 to v6.1.3 + + ## v1.14.0 * Kubernetes [v1.14.0](https://github.com/kubernetes/kubernetes/blob/master/CHANGELOG-1.14.md#v1140) diff --git a/addons/grafana/deployment.yaml b/addons/grafana/deployment.yaml index b6625e8fb..b292454be 100644 --- a/addons/grafana/deployment.yaml +++ b/addons/grafana/deployment.yaml @@ -23,7 +23,7 @@ spec: spec: containers: - name: grafana - image: grafana/grafana:6.1.1 + image: grafana/grafana:6.1.3 env: - name: GF_PATHS_CONFIG value: "/etc/grafana/custom.ini" From 6e5d66cf666a2700ad3e540698a6e243651b96d9 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Tue, 9 Apr 2019 22:12:08 -0700 Subject: [PATCH 078/523] Update kube-state-metrics from v1.5.0 to v1.6.0-rc.0 * Adds a metrics collector for Ingress resources and other improvements * https://github.com/kubernetes/kube-state-metrics/pull/640 * https://github.com/kubernetes/kube-state-metrics/releases/tag/v1.6.0-rc.0 --- CHANGES.md | 2 +- .../prometheus/exporters/kube-state-metrics/cluster-role.yaml | 1 + addons/prometheus/exporters/kube-state-metrics/deployment.yaml | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index a032a1552..e9891e6e8 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -8,9 +8,9 @@ Notable changes between versions. #### Addons +* Update kube-state-metrics from v1.5.0 to v1.6.0-rc.0 ([#449](https://github.com/poseidon/typhoon/pull/449)) * Update Grafana from v6.1.1 to v6.1.3 - ## v1.14.0 * Kubernetes [v1.14.0](https://github.com/kubernetes/kubernetes/blob/master/CHANGELOG-1.14.md#v1140) diff --git a/addons/prometheus/exporters/kube-state-metrics/cluster-role.yaml b/addons/prometheus/exporters/kube-state-metrics/cluster-role.yaml index 1b3f2cde8..d83ee3226 100644 --- a/addons/prometheus/exporters/kube-state-metrics/cluster-role.yaml +++ b/addons/prometheus/exporters/kube-state-metrics/cluster-role.yaml @@ -27,6 +27,7 @@ rules: - daemonsets - deployments - replicasets + - ingresses verbs: - list - watch diff --git a/addons/prometheus/exporters/kube-state-metrics/deployment.yaml b/addons/prometheus/exporters/kube-state-metrics/deployment.yaml index 598b0f574..9f3073bf1 100644 --- a/addons/prometheus/exporters/kube-state-metrics/deployment.yaml +++ b/addons/prometheus/exporters/kube-state-metrics/deployment.yaml @@ -24,7 +24,7 @@ spec: serviceAccountName: kube-state-metrics containers: - name: kube-state-metrics - image: quay.io/coreos/kube-state-metrics:v1.5.0 + image: quay.io/coreos/kube-state-metrics:v1.6.0-rc.0 ports: - name: metrics containerPort: 8080 From 8da17fb7a271cd083deac58abb2b99bb7e4d0dc8 Mon Sep 17 00:00:00 2001 From: JordanP Date: Thu, 11 Apr 2019 17:50:33 +0200 Subject: [PATCH 079/523] Fix "google_compute_target_pool.workers: Cannot determine region" If no region is set at the Google provider level, Terraform fails to create the google_compute_target_pool.workers resource and complains with "Cannot determine region: set in this resource, or set provider-level 'region' or 'zone'." This commit fixes the issue by explicitly setting the region for the google_compute_target_pool.workers resource. --- google-cloud/container-linux/kubernetes/workers/target_pool.tf | 1 + 1 file changed, 1 insertion(+) diff --git a/google-cloud/container-linux/kubernetes/workers/target_pool.tf b/google-cloud/container-linux/kubernetes/workers/target_pool.tf index 949591d57..9197d9781 100644 --- a/google-cloud/container-linux/kubernetes/workers/target_pool.tf +++ b/google-cloud/container-linux/kubernetes/workers/target_pool.tf @@ -1,6 +1,7 @@ # Target pool for TCP/UDP load balancing resource "google_compute_target_pool" "workers" { name = "${var.name}-worker-pool" + region = "${var.region}" session_affinity = "NONE" health_checks = [ From 1b157a2fa48a26338086e224cfaa60dbdc801868 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Sat, 13 Apr 2019 12:37:53 -0700 Subject: [PATCH 080/523] Revert "Update kube-state-metrics from v1.5.0 to v1.6.0-rc.0" * This reverts commit 6e5d66cf666a2700ad3e540698a6e243651b96d9 * kube-state-metrics v1.6.0-rc.0 fires KubeDeploymentReplicasMismatch alerts where its own Deployment doesn't have replicas available, (kube_deployment_status_replicas_available) even though all replicas are available according to kubectl inspection * This problem was present even with the CSR ClusterRole fix (https://github.com/kubernetes/kube-state-metrics/pull/717) --- CHANGES.md | 2 +- .../prometheus/exporters/kube-state-metrics/cluster-role.yaml | 1 - addons/prometheus/exporters/kube-state-metrics/deployment.yaml | 2 +- 3 files changed, 2 insertions(+), 3 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index e9891e6e8..a032a1552 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -8,9 +8,9 @@ Notable changes between versions. #### Addons -* Update kube-state-metrics from v1.5.0 to v1.6.0-rc.0 ([#449](https://github.com/poseidon/typhoon/pull/449)) * Update Grafana from v6.1.1 to v6.1.3 + ## v1.14.0 * Kubernetes [v1.14.0](https://github.com/kubernetes/kubernetes/blob/master/CHANGELOG-1.14.md#v1140) diff --git a/addons/prometheus/exporters/kube-state-metrics/cluster-role.yaml b/addons/prometheus/exporters/kube-state-metrics/cluster-role.yaml index d83ee3226..1b3f2cde8 100644 --- a/addons/prometheus/exporters/kube-state-metrics/cluster-role.yaml +++ b/addons/prometheus/exporters/kube-state-metrics/cluster-role.yaml @@ -27,7 +27,6 @@ rules: - daemonsets - deployments - replicasets - - ingresses verbs: - list - watch diff --git a/addons/prometheus/exporters/kube-state-metrics/deployment.yaml b/addons/prometheus/exporters/kube-state-metrics/deployment.yaml index 9f3073bf1..598b0f574 100644 --- a/addons/prometheus/exporters/kube-state-metrics/deployment.yaml +++ b/addons/prometheus/exporters/kube-state-metrics/deployment.yaml @@ -24,7 +24,7 @@ spec: serviceAccountName: kube-state-metrics containers: - name: kube-state-metrics - image: quay.io/coreos/kube-state-metrics:v1.6.0-rc.0 + image: quay.io/coreos/kube-state-metrics:v1.5.0 ports: - name: metrics containerPort: 8080 From a141c5fe9e020879b25b1e8dee5d3c4b0ad6608e Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Mon, 15 Apr 2019 21:07:12 -0700 Subject: [PATCH 081/523] Update nginx-ingress from v0.23.0 to v0.24.1 * https://github.com/kubernetes/ingress-nginx/releases/tag/nginx-0.24.1 * https://github.com/kubernetes/ingress-nginx/releases/tag/nginx-0.24.0 --- CHANGES.md | 2 +- addons/nginx-ingress/aws/deployment.yaml | 2 +- addons/nginx-ingress/azure/deployment.yaml | 2 +- addons/nginx-ingress/bare-metal/deployment.yaml | 2 +- addons/nginx-ingress/digital-ocean/daemonset.yaml | 2 +- addons/nginx-ingress/google-cloud/deployment.yaml | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index a032a1552..010c910bb 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -9,7 +9,7 @@ Notable changes between versions. #### Addons * Update Grafana from v6.1.1 to v6.1.3 - +* Update nginx-ingress from v0.23.0 to v0.24.1 ## v1.14.0 diff --git a/addons/nginx-ingress/aws/deployment.yaml b/addons/nginx-ingress/aws/deployment.yaml index c42ba9195..e283c76d5 100644 --- a/addons/nginx-ingress/aws/deployment.yaml +++ b/addons/nginx-ingress/aws/deployment.yaml @@ -24,7 +24,7 @@ spec: node-role.kubernetes.io/node: "" containers: - name: nginx-ingress-controller - image: quay.io/kubernetes-ingress-controller/nginx-ingress-controller:0.23.0 + image: quay.io/kubernetes-ingress-controller/nginx-ingress-controller:0.24.1 args: - /nginx-ingress-controller - --ingress-class=public diff --git a/addons/nginx-ingress/azure/deployment.yaml b/addons/nginx-ingress/azure/deployment.yaml index c42ba9195..e283c76d5 100644 --- a/addons/nginx-ingress/azure/deployment.yaml +++ b/addons/nginx-ingress/azure/deployment.yaml @@ -24,7 +24,7 @@ spec: node-role.kubernetes.io/node: "" containers: - name: nginx-ingress-controller - image: quay.io/kubernetes-ingress-controller/nginx-ingress-controller:0.23.0 + image: quay.io/kubernetes-ingress-controller/nginx-ingress-controller:0.24.1 args: - /nginx-ingress-controller - --ingress-class=public diff --git a/addons/nginx-ingress/bare-metal/deployment.yaml b/addons/nginx-ingress/bare-metal/deployment.yaml index ad5040fde..2048180cc 100644 --- a/addons/nginx-ingress/bare-metal/deployment.yaml +++ b/addons/nginx-ingress/bare-metal/deployment.yaml @@ -22,7 +22,7 @@ spec: spec: containers: - name: nginx-ingress-controller - image: quay.io/kubernetes-ingress-controller/nginx-ingress-controller:0.23.0 + image: quay.io/kubernetes-ingress-controller/nginx-ingress-controller:0.24.1 args: - /nginx-ingress-controller - --ingress-class=public diff --git a/addons/nginx-ingress/digital-ocean/daemonset.yaml b/addons/nginx-ingress/digital-ocean/daemonset.yaml index a8ab80854..11990ad82 100644 --- a/addons/nginx-ingress/digital-ocean/daemonset.yaml +++ b/addons/nginx-ingress/digital-ocean/daemonset.yaml @@ -24,7 +24,7 @@ spec: node-role.kubernetes.io/node: "" containers: - name: nginx-ingress-controller - image: quay.io/kubernetes-ingress-controller/nginx-ingress-controller:0.23.0 + image: quay.io/kubernetes-ingress-controller/nginx-ingress-controller:0.24.1 args: - /nginx-ingress-controller - --ingress-class=public diff --git a/addons/nginx-ingress/google-cloud/deployment.yaml b/addons/nginx-ingress/google-cloud/deployment.yaml index c42ba9195..e283c76d5 100644 --- a/addons/nginx-ingress/google-cloud/deployment.yaml +++ b/addons/nginx-ingress/google-cloud/deployment.yaml @@ -24,7 +24,7 @@ spec: node-role.kubernetes.io/node: "" containers: - name: nginx-ingress-controller - image: quay.io/kubernetes-ingress-controller/nginx-ingress-controller:0.23.0 + image: quay.io/kubernetes-ingress-controller/nginx-ingress-controller:0.24.1 args: - /nginx-ingress-controller - --ingress-class=public From e73cccd7ebc669bc5de4af026a750786126095dd Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Tue, 16 Apr 2019 00:05:13 -0700 Subject: [PATCH 082/523] Update provider versions in tutorial docs * Update terraform provider plugin version in docs to reflect the recommended current versions that are currently used --- CHANGES.md | 2 ++ docs/cl/aws.md | 2 +- docs/cl/azure.md | 2 +- docs/cl/google-cloud.md | 2 +- 4 files changed, 5 insertions(+), 3 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 010c910bb..591d8c221 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -4,6 +4,8 @@ Notable changes between versions. ## Latest +## v1.14.1 + * Kubernetes [v1.14.1](https://github.com/kubernetes/kubernetes/blob/master/CHANGELOG-1.14.md#v1141) #### Addons diff --git a/docs/cl/aws.md b/docs/cl/aws.md index c2ed02f92..b0ffc41cb 100644 --- a/docs/cl/aws.md +++ b/docs/cl/aws.md @@ -49,7 +49,7 @@ Configure the AWS provider to use your access key credentials in a `providers.tf ```tf provider "aws" { - version = "~> 2.3.0" + version = "~> 2.6.0" alias = "default" region = "eu-central-1" diff --git a/docs/cl/azure.md b/docs/cl/azure.md index 144ee3982..147e90202 100644 --- a/docs/cl/azure.md +++ b/docs/cl/azure.md @@ -50,7 +50,7 @@ Configure the Azure provider in a `providers.tf` file. ```tf provider "azurerm" { - version = "~> 1.23.0" + version = "~> 1.24.0" alias = "default" } diff --git a/docs/cl/google-cloud.md b/docs/cl/google-cloud.md index 4afc2235d..ab9bc2496 100644 --- a/docs/cl/google-cloud.md +++ b/docs/cl/google-cloud.md @@ -49,7 +49,7 @@ Configure the Google Cloud provider to use your service account key, project-id, ```tf provider "google" { - version = "~> 2.2.0" + version = "~> 2.3.0" alias = "default" credentials = "${file("~/.config/google-cloud/terraform.json")}" From f3174c2b7a7fbc6352767fef7515dced3ad12021 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Thu, 18 Apr 2019 23:26:32 -0700 Subject: [PATCH 083/523] Update Prometheus from v2.8.1 to v2.9.1 * https://github.com/prometheus/prometheus/releases/tag/v2.9.1 * https://github.com/prometheus/prometheus/releases/tag/v2.9.0 --- CHANGES.md | 4 ++++ addons/prometheus/deployment.yaml | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index 591d8c221..9e32d4f56 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -4,6 +4,10 @@ Notable changes between versions. ## Latest +#### Addons + +* Update Prometheus from v2.8.1 to v2.9.1 + ## v1.14.1 * Kubernetes [v1.14.1](https://github.com/kubernetes/kubernetes/blob/master/CHANGELOG-1.14.md#v1141) diff --git a/addons/prometheus/deployment.yaml b/addons/prometheus/deployment.yaml index 254475982..fb514139a 100644 --- a/addons/prometheus/deployment.yaml +++ b/addons/prometheus/deployment.yaml @@ -20,7 +20,7 @@ spec: serviceAccountName: prometheus containers: - name: prometheus - image: quay.io/prometheus/prometheus:v2.8.1 + image: quay.io/prometheus/prometheus:v2.9.1 args: - --web.listen-address=0.0.0.0:9090 - --config.file=/etc/prometheus/prometheus.yaml From 418597aa599a32a035b18f4f80efb8f7babc648e Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Thu, 18 Apr 2019 23:30:43 -0700 Subject: [PATCH 084/523] Update Grafana from v6.1.3 to v6.1.4 * https://github.com/grafana/grafana/releases/tag/v6.1.4 --- CHANGES.md | 1 + addons/grafana/deployment.yaml | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index 9e32d4f56..aa3e5dfea 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -7,6 +7,7 @@ Notable changes between versions. #### Addons * Update Prometheus from v2.8.1 to v2.9.1 +* Update Grafana from v6.1.3 to v6.1.4 ## v1.14.1 diff --git a/addons/grafana/deployment.yaml b/addons/grafana/deployment.yaml index b292454be..809290fd4 100644 --- a/addons/grafana/deployment.yaml +++ b/addons/grafana/deployment.yaml @@ -23,7 +23,7 @@ spec: spec: containers: - name: grafana - image: grafana/grafana:6.1.3 + image: grafana/grafana:6.1.4 env: - name: GF_PATHS_CONFIG value: "/etc/grafana/custom.ini" From 68377a61f804bd3056bbf4632d2f376a3f599d47 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Thu, 18 Apr 2019 23:40:36 -0700 Subject: [PATCH 085/523] Update mkdocs-material from v4.1.1 to v4.1.2 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 1d43bfcb4..61422abb4 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ mkdocs==1.0.4 -mkdocs-material==4.1.1 +mkdocs-material==4.1.2 pygments==2.2.0 pymdown-extensions==5.0.0 From 034a1a9d402a1e76c299538dd23cad969ded2030 Mon Sep 17 00:00:00 2001 From: JordanP Date: Fri, 19 Apr 2019 12:14:26 +0200 Subject: [PATCH 086/523] Remove mention of nginx-ingress default-backend from docs * Default backend was removed in 170ef74eea50fa1b8c0aa8012f23b40c522137e4 --- docs/addons/ingress.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/addons/ingress.md b/docs/addons/ingress.md index cc6d0620a..7e2bf84be 100644 --- a/docs/addons/ingress.md +++ b/docs/addons/ingress.md @@ -6,7 +6,7 @@ Nginx Ingress controller pods accept and demultiplex HTTP, HTTPS, TCP, or UDP tr On AWS, a network load balancer (NLB) distributes TCP traffic across two target groups (port 80 and 443) of worker nodes running an Ingress controller deployment. Security groups rules allow traffic to ports 80 and 443. Health checks ensure only workers with a healthy Ingress controller receive traffic. -Create the Ingress controller deployment, service, RBAC roles, RBAC bindings, default backend, and namespace. +Create the Ingress controller deployment, service, RBAC roles, RBAC bindings, and namespace. ``` kubectl apply -R -f addons/nginx-ingress/aws @@ -39,7 +39,7 @@ resource "google_dns_record_set" "some-application" { On Azure, a load balancer distributes traffic across a backend address pool of worker nodes running an Ingress controller deployment. Security group rules allow traffic to ports 80 and 443. Health probes ensure only workers with a healthy Ingress controller receive traffic. -Create the Ingress controller deployment, service, RBAC roles, RBAC bindings, default backend, and namespace. +Create the Ingress controller deployment, service, RBAC roles, RBAC bindings, and namespace. ``` kubectl apply -R -f addons/nginx-ingress/azure @@ -74,7 +74,7 @@ On bare-metal, routing traffic to Ingress controller pods can be done in number ### Equal-Cost Multi-Path -Create the Ingress controller deployment, service, RBAC roles, RBAC bindings, and default backend. The service should use a fixed ClusterIP (e.g. 10.3.0.12) in the Kubernetes service IPv4 CIDR range. +Create the Ingress controller deployment, service, RBAC roles, and RBAC bindings. The service should use a fixed ClusterIP (e.g. 10.3.0.12) in the Kubernetes service IPv4 CIDR range. ``` kubectl apply -R -f addons/nginx-ingress/bare-metal @@ -103,7 +103,7 @@ resource "google_dns_record_set" "some-application" { On Digital Ocean, DNS A and AAAA records (e.g. FQDN `nemo-workers.example.com`) resolve to each worker[^1] running an Ingress controller DaemonSet on host ports 80 and 443. Firewall rules allow IPv4 and IPv6 traffic to ports 80 and 443. -Create the Ingress controller daemonset, service, RBAC roles, RBAC bindings, default backend, and namespace. +Create the Ingress controller daemonset, service, RBAC roles, RBAC bindings, and namespace. ``` kubectl apply -R -f addons/nginx-ingress/digital-ocean @@ -133,7 +133,7 @@ resource "google_dns_record_set" "some-application" { On Google Cloud, a TCP Proxy load balancer distributes IPv4 and IPv6 TCP traffic across a backend service of worker nodes running an Ingress controller deployment. Firewall rules allow traffic to ports 80 and 443. Health check rules ensure only workers with a healthy Ingress controller receive traffic. -Create the Ingress controller deployment, service, RBAC roles, RBAC bindings, default backend, and namespace. +Create the Ingress controller deployment, service, RBAC roles, RBAC bindings, and namespace. ``` kubectl apply -R -f addons/nginx-ingress/google-cloud From 2c11bad43953f2b6bce38a0c919fec3887a9810d Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Sat, 27 Apr 2019 20:39:55 -0700 Subject: [PATCH 087/523] Update Prometheus from v2.9.1 to v2.9.2 * https://github.com/prometheus/prometheus/releases/tag/v2.9.2 --- CHANGES.md | 2 +- addons/prometheus/deployment.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index aa3e5dfea..348d70d4d 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -6,7 +6,7 @@ Notable changes between versions. #### Addons -* Update Prometheus from v2.8.1 to v2.9.1 +* Update Prometheus from v2.8.1 to v2.9.2 * Update Grafana from v6.1.3 to v6.1.4 ## v1.14.1 diff --git a/addons/prometheus/deployment.yaml b/addons/prometheus/deployment.yaml index fb514139a..691c8df94 100644 --- a/addons/prometheus/deployment.yaml +++ b/addons/prometheus/deployment.yaml @@ -20,7 +20,7 @@ spec: serviceAccountName: prometheus containers: - name: prometheus - image: quay.io/prometheus/prometheus:v2.9.1 + image: quay.io/prometheus/prometheus:v2.9.2 args: - --web.listen-address=0.0.0.0:9090 - --config.file=/etc/prometheus/prometheus.yaml From 0e94708fd879e2d3ec523c3f3d47093b16314bb1 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Sat, 27 Apr 2019 20:46:31 -0700 Subject: [PATCH 088/523] Update kube-state-metrics from v1.5.0 to v1.6.0-rc.2 * Collect metrics Ingress resources * Collects metrics about certificates.k8s.io certificatesigningrequests * https://github.com/kubernetes/kube-state-metrics/releases/tag/v1.6.0-rc.2 --- CHANGES.md | 1 + .../exporters/kube-state-metrics/cluster-role.yaml | 8 ++++++++ .../exporters/kube-state-metrics/deployment.yaml | 2 +- 3 files changed, 10 insertions(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index 348d70d4d..b59d13c85 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -7,6 +7,7 @@ Notable changes between versions. #### Addons * Update Prometheus from v2.8.1 to v2.9.2 + * Update kube-state-metrics from v1.5.0 to v1.6.0-rc.2 * Update Grafana from v6.1.3 to v6.1.4 ## v1.14.1 diff --git a/addons/prometheus/exporters/kube-state-metrics/cluster-role.yaml b/addons/prometheus/exporters/kube-state-metrics/cluster-role.yaml index 1b3f2cde8..6cd144b78 100644 --- a/addons/prometheus/exporters/kube-state-metrics/cluster-role.yaml +++ b/addons/prometheus/exporters/kube-state-metrics/cluster-role.yaml @@ -27,6 +27,7 @@ rules: - daemonsets - deployments - replicasets + - ingresses verbs: - list - watch @@ -62,3 +63,10 @@ rules: verbs: - list - watch +- apiGroups: + - certificates.k8s.io + resources: + - certificatesigningrequests + verbs: + - list + - watch diff --git a/addons/prometheus/exporters/kube-state-metrics/deployment.yaml b/addons/prometheus/exporters/kube-state-metrics/deployment.yaml index 598b0f574..ac23bb941 100644 --- a/addons/prometheus/exporters/kube-state-metrics/deployment.yaml +++ b/addons/prometheus/exporters/kube-state-metrics/deployment.yaml @@ -24,7 +24,7 @@ spec: serviceAccountName: kube-state-metrics containers: - name: kube-state-metrics - image: quay.io/coreos/kube-state-metrics:v1.5.0 + image: quay.io/coreos/kube-state-metrics:v1.6.0-rc.2 ports: - name: metrics containerPort: 8080 From ec5aef5c9254fffbd02f1251b37c7fa1361b87d3 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Sat, 27 Apr 2019 22:41:13 -0700 Subject: [PATCH 089/523] Refresh Prometheus rules and Grafana dashboards * Adds several network related alerts from upstream --- addons/grafana/dashboards-k8s-resources.yaml | 28 +- addons/grafana/dashboards-k8s.yaml | 362 +++++++++++++++++-- addons/prometheus/rules.yaml | 54 +++ 3 files changed, 400 insertions(+), 44 deletions(-) diff --git a/addons/grafana/dashboards-k8s-resources.yaml b/addons/grafana/dashboards-k8s-resources.yaml index eedcc7e46..f00af428b 100644 --- a/addons/grafana/dashboards-k8s-resources.yaml +++ b/addons/grafana/dashboards-k8s-resources.yaml @@ -1136,24 +1136,6 @@ data: "type": "number", "unit": "short" }, - { - "alias": "CPU Usage", - "colorMode": null, - "colors": [ - - ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "link": false, - "linkTooltip": "Drill down", - "linkUrl": "", - "pattern": "Value #C", - "thresholds": [ - - ], - "type": "number", - "unit": "short" - }, { "alias": "Memory Usage", "colorMode": null, @@ -1165,7 +1147,7 @@ data: "link": false, "linkTooltip": "Drill down", "linkUrl": "", - "pattern": "Value #D", + "pattern": "Value #C", "thresholds": [ ], @@ -1183,7 +1165,7 @@ data: "link": false, "linkTooltip": "Drill down", "linkUrl": "", - "pattern": "Value #E", + "pattern": "Value #D", "thresholds": [ ], @@ -1201,7 +1183,7 @@ data: "link": false, "linkTooltip": "Drill down", "linkUrl": "", - "pattern": "Value #F", + "pattern": "Value #E", "thresholds": [ ], @@ -1219,7 +1201,7 @@ data: "link": false, "linkTooltip": "Drill down", "linkUrl": "", - "pattern": "Value #G", + "pattern": "Value #F", "thresholds": [ ], @@ -1237,7 +1219,7 @@ data: "link": false, "linkTooltip": "Drill down", "linkUrl": "", - "pattern": "Value #H", + "pattern": "Value #G", "thresholds": [ ], diff --git a/addons/grafana/dashboards-k8s.yaml b/addons/grafana/dashboards-k8s.yaml index c8322aeb2..d83a49cfb 100644 --- a/addons/grafana/dashboards-k8s.yaml +++ b/addons/grafana/dashboards-k8s.yaml @@ -1995,6 +1995,13 @@ data: "intervalFactor": 2, "legendFormat": "load 15m", "refId": "C" + }, + { + "expr": "count(node_cpu_seconds_total{cluster=\"$cluster\", job=\"node-exporter\", instance=\"$instance\", mode=\"user\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "logical cores", + "refId": "D" } ], "thresholds": [ @@ -3293,7 +3300,7 @@ data: }, "id": 2, "legend": { - "alignAsTable": false, + "alignAsTable": true, "avg": true, "current": true, "max": true, @@ -3318,16 +3325,23 @@ data: ], "spaceLength": 10, - "span": 12, - "stack": false, + "span": 9, + "stack": true, "steppedLine": false, "targets": [ { - "expr": "(kubelet_volume_stats_capacity_bytes{cluster=\"$cluster\", job=\"kubelet\", persistentvolumeclaim=\"$volume\"} - kubelet_volume_stats_available_bytes{cluster=\"$cluster\", job=\"kubelet\", persistentvolumeclaim=\"$volume\"}) / kubelet_volume_stats_capacity_bytes{cluster=\"$cluster\", job=\"kubelet\", persistentvolumeclaim=\"$volume\"} * 100\n", + "expr": "(\n sum without(instance, node) (kubelet_volume_stats_capacity_bytes{cluster=\"$cluster\", job=\"kubelet\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"})\n -\n sum without(instance, node) (kubelet_volume_stats_available_bytes{cluster=\"$cluster\", job=\"kubelet\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"})\n)\n", "format": "time_series", "intervalFactor": 1, - "legendFormat": "{{ Usage }}", + "legendFormat": "Used Space", "refId": "A" + }, + { + "expr": "sum without(instance, node) (kubelet_volume_stats_available_bytes{cluster=\"$cluster\", job=\"kubelet\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"})\n", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Free Space", + "refId": "B" } ], "thresholds": [ @@ -3353,22 +3367,106 @@ data: }, "yaxes": [ { - "format": "percent", + "format": "bytes", "label": null, "logBase": 1, - "max": 100, + "max": null, "min": 0, "show": true }, { - "format": "percent", + "format": "bytes", "label": null, "logBase": 1, - "max": 100, + "max": null, "min": 0, "show": true } ] + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "datasource": "$datasource", + "format": "percent", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": true, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + + }, + "id": 3, + "interval": null, + "links": [ + + ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "span": 3, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "(\n kubelet_volume_stats_capacity_bytes{cluster=\"$cluster\", job=\"kubelet\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"}\n -\n kubelet_volume_stats_available_bytes{cluster=\"$cluster\", job=\"kubelet\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"}\n)\n/\nkubelet_volume_stats_capacity_bytes{cluster=\"$cluster\", job=\"kubelet\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"}\n* 100\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": "80, 90", + "title": "Volume Space Usage", + "tooltip": { + "shared": false + }, + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" } ], "repeat": null, @@ -3395,9 +3493,9 @@ data: "gridPos": { }, - "id": 3, + "id": 4, "legend": { - "alignAsTable": false, + "alignAsTable": true, "avg": true, "current": true, "max": true, @@ -3422,16 +3520,23 @@ data: ], "spaceLength": 10, - "span": 12, - "stack": false, + "span": 9, + "stack": true, "steppedLine": false, "targets": [ { - "expr": "kubelet_volume_stats_inodes_used{cluster=\"$cluster\", job=\"kubelet\", persistentvolumeclaim=\"$volume\"} / kubelet_volume_stats_inodes{cluster=\"$cluster\", job=\"kubelet\", persistentvolumeclaim=\"$volume\"} * 100\n", + "expr": "sum without(instance, node) (kubelet_volume_stats_inodes_used{cluster=\"$cluster\", job=\"kubelet\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"})\n", "format": "time_series", "intervalFactor": 1, - "legendFormat": "{{ Usage }}", + "legendFormat": "Used inodes", "refId": "A" + }, + { + "expr": "(\n sum without(instance, node) (kubelet_volume_stats_inodes{cluster=\"$cluster\", job=\"kubelet\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"})\n -\n sum without(instance, node) (kubelet_volume_stats_inodes_used{cluster=\"$cluster\", job=\"kubelet\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"})\n)\n", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": " Free inodes", + "refId": "B" } ], "thresholds": [ @@ -3457,22 +3562,106 @@ data: }, "yaxes": [ { - "format": "percent", + "format": "none", "label": null, "logBase": 1, - "max": 100, + "max": null, "min": 0, "show": true }, { - "format": "percent", + "format": "none", "label": null, "logBase": 1, - "max": 100, + "max": null, "min": 0, "show": true } ] + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "datasource": "$datasource", + "format": "percent", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": true, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + + }, + "id": 5, + "interval": null, + "links": [ + + ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "span": 3, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "kubelet_volume_stats_inodes_used{cluster=\"$cluster\", job=\"kubelet\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"}\n/\nkubelet_volume_stats_inodes{cluster=\"$cluster\", job=\"kubelet\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"}\n* 100\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": "80, 90", + "title": "Volume inodes Usage", + "tooltip": { + "shared": false + }, + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" } ], "repeat": null, @@ -3631,7 +3820,20 @@ data: ], "annotations": { "list": [ - + { + "builtIn": 1, + "datasource": "$datasource", + "enable": true, + "expr": "time() == BOOL timestamp(rate(kube_pod_container_status_restarts_total{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}[2m]) > 0)", + "hide": false, + "iconColor": "rgba(215, 44, 44, 1)", + "name": "Restarts", + "showIn": 0, + "tags": [ + "restart" + ], + "type": "rows" + } ] }, "editable": false, @@ -3711,6 +3913,13 @@ data: "intervalFactor": 2, "legendFormat": "Limit: {{ container }}", "refId": "C" + }, + { + "expr": "sum by(container_name) (container_memory_cache{job=\"kubernetes-cadvisor\", namespace=\"$namespace\", pod_name=~\"$pod\", container_name=~\"$container\", container_name!=\"POD\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Cache: {{ container_name }}", + "refId": "D" } ], "thresholds": [ @@ -3931,8 +4140,15 @@ data: "expr": "sort_desc(sum by (pod_name) (rate(container_network_receive_bytes_total{job=\"kubernetes-cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\", pod_name=\"$pod\"}[1m])))", "format": "time_series", "intervalFactor": 2, - "legendFormat": "{{ pod_name }}", + "legendFormat": "RX: {{ pod_name }}", "refId": "A" + }, + { + "expr": "sort_desc(sum by (pod_name) (rate(container_network_transmit_bytes_total{job=\"kubernetes-cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\", pod_name=\"$pod\"}[1m])))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "TX: {{ pod_name }}", + "refId": "B" } ], "thresholds": [ @@ -3983,6 +4199,110 @@ data: "title": "Dashboard Row", "titleSize": "h6", "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "gridPos": { + + }, + "id": 5, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "max by (container) (kube_pod_container_status_restarts_total{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\", container=~\"$container\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Restarts: {{ container }}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Total Restarts Per Container", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" } ], "schemaVersion": 14, diff --git a/addons/prometheus/rules.yaml b/addons/prometheus/rules.yaml index 1b2718167..2962028a1 100644 --- a/addons/prometheus/rules.yaml +++ b/addons/prometheus/rules.yaml @@ -992,6 +992,60 @@ data: } ] }, + { + "name": "node-time", + "rules": [ + { + "alert": "ClockSkewDetected", + "annotations": { + "message": "Clock skew detected on node-exporter {{ $labels.namespace }}/{{ $labels.pod }}. Ensure NTP is configured correctly on this host." + }, + "expr": "abs(node_timex_offset_seconds{job=\"node-exporter\"}) > 0.03\n", + "for": "2m", + "labels": { + "severity": "warning" + } + } + ] + }, + { + "name": "node-network", + "rules": [ + { + "alert": "NetworkReceiveErrors", + "annotations": { + "message": "Network interface \"{{ $labels.device }}\" showing receive errors on node-exporter {{ $labels.namespace }}/{{ $labels.pod }}\"" + }, + "expr": "rate(node_network_receive_errs_total{job=\"node-exporter\",device!~\"veth.+|tunl.+\"}[2m]) > 0\n", + "for": "2m", + "labels": { + "severity": "warning" + } + }, + { + "alert": "NetworkTransmitErrors", + "annotations": { + "message": "Network interface \"{{ $labels.device }}\" showing transmit errors on node-exporter {{ $labels.namespace }}/{{ $labels.pod }}\"" + }, + "expr": "rate(node_network_transmit_errs_total{job=\"node-exporter\",device!~\"veth.+|tunl.+\"}[2m]) > 0\n", + "for": "2m", + "labels": { + "severity": "warning" + } + }, + { + "alert": "NodeNetworkInterfaceFlapping", + "annotations": { + "message": "Network interface \"{{ $labels.device }}\" changing it's up status often on node-exporter {{ $labels.namespace }}/{{ $labels.pod }}\"" + }, + "expr": "changes(node_network_up{job=\"node-exporter\",device!~\"veth.+|tunl.+\"}[2m]) > 2\n", + "for": "2m", + "labels": { + "severity": "warning" + } + } + ] + }, { "name": "prometheus.rules", "rules": [ From 3a6979920c64a8afeac321803f53501ce0b042cf Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Sat, 27 Apr 2019 20:35:00 -0700 Subject: [PATCH 090/523] Update provider plugin versions in tutorial docs * Update terraform provider plugin version in docs to reflect the recommended current versions that are currently used --- docs/cl/aws.md | 2 +- docs/cl/azure.md | 2 +- docs/cl/digital-ocean.md | 2 +- docs/cl/google-cloud.md | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/cl/aws.md b/docs/cl/aws.md index b0ffc41cb..b84b06b9e 100644 --- a/docs/cl/aws.md +++ b/docs/cl/aws.md @@ -49,7 +49,7 @@ Configure the AWS provider to use your access key credentials in a `providers.tf ```tf provider "aws" { - version = "~> 2.6.0" + version = "~> 2.8.0" alias = "default" region = "eu-central-1" diff --git a/docs/cl/azure.md b/docs/cl/azure.md index 147e90202..965c94ef8 100644 --- a/docs/cl/azure.md +++ b/docs/cl/azure.md @@ -50,7 +50,7 @@ Configure the Azure provider in a `providers.tf` file. ```tf provider "azurerm" { - version = "~> 1.24.0" + version = "~> 1.27.1" alias = "default" } diff --git a/docs/cl/digital-ocean.md b/docs/cl/digital-ocean.md index 6801c3185..de965b454 100644 --- a/docs/cl/digital-ocean.md +++ b/docs/cl/digital-ocean.md @@ -50,7 +50,7 @@ Configure the DigitalOcean provider to use your token in a `providers.tf` file. ```tf provider "digitalocean" { - version = "~> 1.1.0" + version = "~> 1.2.0" token = "${chomp(file("~/.config/digital-ocean/token"))}" alias = "default" } diff --git a/docs/cl/google-cloud.md b/docs/cl/google-cloud.md index ab9bc2496..a84f14779 100644 --- a/docs/cl/google-cloud.md +++ b/docs/cl/google-cloud.md @@ -49,7 +49,7 @@ Configure the Google Cloud provider to use your service account key, project-id, ```tf provider "google" { - version = "~> 2.3.0" + version = "~> 2.5.1" alias = "default" credentials = "${file("~/.config/google-cloud/terraform.json")}" From a3c3aa12138563b322c277dfc3d93861d10b1aa7 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Sun, 28 Apr 2019 14:23:49 -0700 Subject: [PATCH 091/523] Update mkdocs-material from v4.1.2 to v4.2.0 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 61422abb4..3a7faec13 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ mkdocs==1.0.4 -mkdocs-material==4.1.2 +mkdocs-material==4.2.0 pygments==2.2.0 pymdown-extensions==5.0.0 From 253831aac36b24949b01ac3ca615bb7bba94ccf4 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Sat, 4 May 2019 10:46:07 -0700 Subject: [PATCH 092/523] Update links to Matchbox, terraform-provider-ct, etc. * Matchbox, terraform-provider-matchbox, and terraform-provider-ct have moved to the poseidon Github organization --- CHANGES.md | 12 ++++++------ docs/announce.md | 2 +- docs/atomic/bare-metal.md | 12 ++++++------ docs/cl/aws.md | 6 +++--- docs/cl/azure.md | 6 +++--- docs/cl/bare-metal.md | 12 ++++++------ docs/cl/digital-ocean.md | 6 +++--- docs/cl/google-cloud.md | 6 +++--- docs/topics/maintenance.md | 16 ++++++++-------- 9 files changed, 39 insertions(+), 39 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index b59d13c85..c009fff04 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -68,7 +68,7 @@ Notable changes between versions. * Reverse DNS lookups for service IPv4 addresses unchanged * Upgrade Calico from v3.5.2 to [v3.6.0](https://docs.projectcalico.org/v3.6/release-notes/) ([#430](https://github.com/poseidon/typhoon/pull/430)) * Change pod IPAM from `host-local` to `calico-ipam`. `pod_cidr` is still divided into `/24` subnets per node, but managed as `ippools` and `ipamblocks` -* Suggest updating [terraform-provider-ct](https://github.com/coreos/terraform-provider-ct) from v0.3.0 to [v0.3.1](https://github.com/coreos/terraform-provider-ct/releases/tag/v0.3.1) ([#434](https://github.com/poseidon/typhoon/pull/434)) +* Suggest updating [terraform-provider-ct](https://github.com/poseidon/terraform-provider-ct) from v0.3.0 to [v0.3.1](https://github.com/poseidon/terraform-provider-ct/releases/tag/v0.3.1) ([#434](https://github.com/poseidon/typhoon/pull/434)) * Announce: Fedora Atomic modules will be not be updated beyond Kubernetes v1.13.x ([#437](https://github.com/poseidon/typhoon/pull/437)) * Thank you Project Atomic team and users, please see the deprecation [notice](https://typhoon.psdn.io/announce/#march-27-2019) @@ -113,7 +113,7 @@ Notable changes between versions. #### Bare-Metal -* Recommend updating [terraform-provider-matchbox](https://github.com/coreos/terraform-provider-matchbox) plugin from v0.2.2 to [v0.2.3](https://github.com/coreos/terraform-provider-matchbox/releases/tag/v0.2.3) ([#402](https://github.com/poseidon/typhoon/pull/402)) +* Recommend updating [terraform-provider-matchbox](https://github.com/poseidon/terraform-provider-matchbox) plugin from v0.2.2 to [v0.2.3](https://github.com/poseidon/terraform-provider-matchbox/releases/tag/v0.2.3) ([#402](https://github.com/poseidon/typhoon/pull/402)) * Improve docs on using Ubiquiti EdgeOS with bare-metal clusters ([#413](https://github.com/poseidon/typhoon/pull/413)) #### Google Cloud @@ -625,15 +625,15 @@ Notable changes between versions. #### AWS -* [Require](https://typhoon.psdn.io/topics/maintenance/#terraform-provider-ct-v021) updating `terraform-provider-ct` plugin from v0.2.0 to [v0.2.1](https://github.com/coreos/terraform-provider-ct/releases/tag/v0.2.1) (action required!) +* [Require](https://typhoon.psdn.io/topics/maintenance/#terraform-provider-ct-v021) updating `terraform-provider-ct` plugin from v0.2.0 to [v0.2.1](https://github.com/poseidon/terraform-provider-ct/releases/tag/v0.2.1) (action required!) #### Digital Ocean -* [Require](https://typhoon.psdn.io/topics/maintenance/#terraform-provider-ct-v021) updating `terraform-provider-ct` plugin from v0.2.0 to [v0.2.1](https://github.com/coreos/terraform-provider-ct/releases/tag/v0.2.1) (action required!) +* [Require](https://typhoon.psdn.io/topics/maintenance/#terraform-provider-ct-v021) updating `terraform-provider-ct` plugin from v0.2.0 to [v0.2.1](https://github.com/poseidon/terraform-provider-ct/releases/tag/v0.2.1) (action required!) #### Google Cloud -* [Require](https://typhoon.psdn.io/topics/maintenance/#terraform-provider-ct-v021) updating `terraform-provider-ct` plugin from v0.2.0 to [v0.2.1](https://github.com/coreos/terraform-provider-ct/releases/tag/v0.2.1) (action required!) +* [Require](https://typhoon.psdn.io/topics/maintenance/#terraform-provider-ct-v021) updating `terraform-provider-ct` plugin from v0.2.0 to [v0.2.1](https://github.com/poseidon/terraform-provider-ct/releases/tag/v0.2.1) (action required!) * Relax `os_image` to optional. Default to "coreos-stable". #### Addons @@ -653,7 +653,7 @@ Notable changes between versions. * Upgrade etcd from v3.2.15 to v3.3.2 * Update Calico from v3.0.2 to v3.0.3 * Use kubernetes-incubator/bootkube v0.11.0 -* [Recommend](https://typhoon.psdn.io/topics/maintenance/#terraform-provider-ct-v021) updating `terraform-provider-ct` plugin from v0.2.0 to [v0.2.1](https://github.com/coreos/terraform-provider-ct/releases/tag/v0.2.1) (action recommended) +* [Recommend](https://typhoon.psdn.io/topics/maintenance/#terraform-provider-ct-v021) updating `terraform-provider-ct` plugin from v0.2.0 to [v0.2.1](https://github.com/poseidon/terraform-provider-ct/releases/tag/v0.2.1) (action recommended) #### AWS diff --git a/docs/announce.md b/docs/announce.md index 789ebf382..1c46d42ff 100644 --- a/docs/announce.md +++ b/docs/announce.md @@ -52,7 +52,7 @@ Get started with the [basics](https://typhoon.psdn.io/architecture/concepts/) or Heed the warnings. Typhoon for Fedora Atomic is still alpha. Container Linux continues to be the recommended flavor for production clusters. Atomic is not meant to detract from efforts on Container Linux or its derivatives. !!! tip - For bare-metal, you may continue to use your v0.7+ [Matchbox](https://github.com/coreos/matchbox) service and `terraform-provider-matchbox` plugin to provision both Container Linux and Fedora Atomic clusters. No changes needed. + For bare-metal, you may continue to use your v0.7+ [Matchbox](https://github.com/poseidon/matchbox) service and `terraform-provider-matchbox` plugin to provision both Container Linux and Fedora Atomic clusters. No changes needed. [^1]: Using `etcd`, `kubelet`, and `bootkube` as system containers required metadata files be added in [system-containers](https://github.com/poseidon/system-containers) diff --git a/docs/atomic/bare-metal.md b/docs/atomic/bare-metal.md index c83a6e3da..7c6be2bfe 100644 --- a/docs/atomic/bare-metal.md +++ b/docs/atomic/bare-metal.md @@ -5,7 +5,7 @@ In this tutorial, we'll network boot and provision a Kubernetes v1.14.1 cluster on bare-metal with Fedora Atomic. -First, we'll deploy a [Matchbox](https://github.com/coreos/matchbox) service and setup a network boot environment. Then, we'll declare a Kubernetes cluster using the Typhoon Terraform module and power on machines. On PXE boot, machines will install Fedora Atomic via kickstart, reboot into the disk install, and provision themselves as Kubernetes controllers or workers via cloud-init. +First, we'll deploy a [Matchbox](https://github.com/poseidon/matchbox) service and setup a network boot environment. Then, we'll declare a Kubernetes cluster using the Typhoon Terraform module and power on machines. On PXE boot, machines will install Fedora Atomic via kickstart, reboot into the disk install, and provision themselves as Kubernetes controllers or workers via cloud-init. Controllers are provisioned to run `etcd` and `kubelet` [system containers](http://www.projectatomic.io/blog/2016/09/intro-to-system-containers/). Workers run just a `kubelet` system container. A one-time [bootkube](https://github.com/kubernetes-incubator/bootkube) bootstrap schedules the `apiserver`, `scheduler`, `controller-manager`, and `coredns` on controllers and schedules `kube-proxy` and `calico` (or `flannel`) on every node. A generated `kubeconfig` provides `kubectl` access to the cluster. @@ -16,7 +16,7 @@ Controllers are provisioned to run `etcd` and `kubelet` [system containers](http * Matchbox v0.7+ deployment with API enabled * HTTP server for Fedora install assets and ostree repo * Matchbox credentials `client.crt`, `client.key`, `ca.crt` -* Terraform v0.11.x and [terraform-provider-matchbox](https://github.com/coreos/terraform-provider-matchbox) installed locally +* Terraform v0.11.x and [terraform-provider-matchbox](https://github.com/poseidon/terraform-provider-matchbox) installed locally ## Machines @@ -97,7 +97,7 @@ chain http://matchbox.foo:8080/boot.ipxe For networks with Ubiquiti Routers, you can [configure the router](/topics/hardware/#ubiquiti) itself to chainload machines to iPXE and Matchbox. -For a small lab, you may wish to checkout the [quay.io/coreos/dnsmasq](https://quay.io/repository/coreos/dnsmasq) container image and [copy-paste examples](https://github.com/coreos/matchbox/blob/master/Documentation/network-setup.md#coreosdnsmasq). +For a small lab, you may wish to checkout the [quay.io/poseidon/dnsmasq](https://quay.io/repository/poseidon/dnsmasq) container image and [copy-paste examples](https://github.com/poseidon/matchbox/blob/master/Documentation/network-setup.md#coreosdnsmasq). Read about the [many ways](https://coreos.com/matchbox/docs/latest/network-setup.html) to setup a compliant iPXE-enabled network. There is quite a bit of flexibility: @@ -163,7 +163,7 @@ curl http://example.com/fedora/28/ ``` !!! note - It is possible to use the Matchbox `/assets` [cache](https://github.com/coreos/matchbox/blob/master/Documentation/matchbox.md#assets) as an HTTP server. + It is possible to use the Matchbox `/assets` [cache](https://github.com/poseidon/matchbox/blob/master/Documentation/matchbox.md#assets) as an HTTP server. ## Terraform Setup @@ -174,10 +174,10 @@ $ terraform version Terraform v0.11.12 ``` -Add the [terraform-provider-matchbox](https://github.com/coreos/terraform-provider-matchbox) plugin binary for your system to `~/.terraform.d/plugins/`, noting the final name. +Add the [terraform-provider-matchbox](https://github.com/poseidon/terraform-provider-matchbox) plugin binary for your system to `~/.terraform.d/plugins/`, noting the final name. ```sh -wget https://github.com/coreos/terraform-provider-matchbox/releases/download/v0.2.3/terraform-provider-matchbox-v0.2.3-linux-amd64.tar.gz +wget https://github.com/poseidon/terraform-provider-matchbox/releases/download/v0.2.3/terraform-provider-matchbox-v0.2.3-linux-amd64.tar.gz tar xzf terraform-provider-matchbox-v0.2.3-linux-amd64.tar.gz mv terraform-provider-matchbox-v0.2.3-linux-amd64/terraform-provider-matchbox ~/.terraform.d/plugins/terraform-provider-matchbox_v0.2.3 ``` diff --git a/docs/cl/aws.md b/docs/cl/aws.md index b84b06b9e..076eb6122 100644 --- a/docs/cl/aws.md +++ b/docs/cl/aws.md @@ -10,7 +10,7 @@ Controllers are provisioned to run an `etcd-member` peer and a `kubelet` service * AWS Account and IAM credentials * AWS Route53 DNS Zone (registered Domain Name or delegated subdomain) -* Terraform v0.11.x and [terraform-provider-ct](https://github.com/coreos/terraform-provider-ct) installed locally +* Terraform v0.11.x and [terraform-provider-ct](https://github.com/poseidon/terraform-provider-ct) installed locally ## Terraform Setup @@ -21,10 +21,10 @@ $ terraform version Terraform v0.11.12 ``` -Add the [terraform-provider-ct](https://github.com/coreos/terraform-provider-ct) plugin binary for your system to `~/.terraform.d/plugins/`, noting the final name. +Add the [terraform-provider-ct](https://github.com/poseidon/terraform-provider-ct) plugin binary for your system to `~/.terraform.d/plugins/`, noting the final name. ```sh -wget https://github.com/coreos/terraform-provider-ct/releases/download/v0.3.1/terraform-provider-ct-v0.3.1-linux-amd64.tar.gz +wget https://github.com/poseidon/terraform-provider-ct/releases/download/v0.3.1/terraform-provider-ct-v0.3.1-linux-amd64.tar.gz tar xzf terraform-provider-ct-v0.3.1-linux-amd64.tar.gz mv terraform-provider-ct-v0.3.1-linux-amd64/terraform-provider-ct ~/.terraform.d/plugins/terraform-provider-ct_v0.3.1 ``` diff --git a/docs/cl/azure.md b/docs/cl/azure.md index 965c94ef8..0282a76dd 100644 --- a/docs/cl/azure.md +++ b/docs/cl/azure.md @@ -13,7 +13,7 @@ Controllers are provisioned to run an `etcd-member` peer and a `kubelet` service * Azure account * Azure DNS Zone (registered Domain Name or delegated subdomain) -* Terraform v0.11.x and [terraform-provider-ct](https://github.com/coreos/terraform-provider-ct) installed locally +* Terraform v0.11.x and [terraform-provider-ct](https://github.com/poseidon/terraform-provider-ct) installed locally ## Terraform Setup @@ -24,10 +24,10 @@ $ terraform version Terraform v0.11.12 ``` -Add the [terraform-provider-ct](https://github.com/coreos/terraform-provider-ct) plugin binary for your system to `~/.terraform.d/plugins/`, noting the final name. +Add the [terraform-provider-ct](https://github.com/poseidon/terraform-provider-ct) plugin binary for your system to `~/.terraform.d/plugins/`, noting the final name. ```sh -wget https://github.com/coreos/terraform-provider-ct/releases/download/v0.3.1/terraform-provider-ct-v0.3.1-linux-amd64.tar.gz +wget https://github.com/poseidon/terraform-provider-ct/releases/download/v0.3.1/terraform-provider-ct-v0.3.1-linux-amd64.tar.gz tar xzf terraform-provider-ct-v0.3.1-linux-amd64.tar.gz mv terraform-provider-ct-v0.3.1-linux-amd64/terraform-provider-ct ~/.terraform.d/plugins/terraform-provider-ct_v0.3.1 ``` diff --git a/docs/cl/bare-metal.md b/docs/cl/bare-metal.md index fcd869626..7ba447961 100644 --- a/docs/cl/bare-metal.md +++ b/docs/cl/bare-metal.md @@ -2,7 +2,7 @@ In this tutorial, we'll network boot and provision a Kubernetes v1.14.1 cluster on bare-metal with Container Linux. -First, we'll deploy a [Matchbox](https://github.com/coreos/matchbox) service and setup a network boot environment. Then, we'll declare a Kubernetes cluster using the Typhoon Terraform module and power on machines. On PXE boot, machines will install Container Linux to disk, reboot into the disk install, and provision themselves as Kubernetes controllers or workers via Ignition. +First, we'll deploy a [Matchbox](https://github.com/poseidon/matchbox) service and setup a network boot environment. Then, we'll declare a Kubernetes cluster using the Typhoon Terraform module and power on machines. On PXE boot, machines will install Container Linux to disk, reboot into the disk install, and provision themselves as Kubernetes controllers or workers via Ignition. Controllers are provisioned to run an `etcd-member` peer and a `kubelet` service. Workers run just a `kubelet` service. A one-time [bootkube](https://github.com/kubernetes-incubator/bootkube) bootstrap schedules the `apiserver`, `scheduler`, `controller-manager`, and `coredns` on controllers and schedules `kube-proxy` and `calico` (or `flannel`) on every node. A generated `kubeconfig` provides `kubectl` access to the cluster. @@ -12,7 +12,7 @@ Controllers are provisioned to run an `etcd-member` peer and a `kubelet` service * PXE-enabled [network boot](https://coreos.com/matchbox/docs/latest/network-setup.html) environment (with HTTPS support) * Matchbox v0.6+ deployment with API enabled * Matchbox credentials `client.crt`, `client.key`, `ca.crt` -* Terraform v0.11.x, [terraform-provider-matchbox](https://github.com/coreos/terraform-provider-matchbox), and [terraform-provider-ct](https://github.com/coreos/terraform-provider-ct) installed locally +* Terraform v0.11.x, [terraform-provider-matchbox](https://github.com/poseidon/terraform-provider-matchbox), and [terraform-provider-ct](https://github.com/poseidon/terraform-provider-ct) installed locally ## Machines @@ -114,18 +114,18 @@ $ terraform version Terraform v0.11.12 ``` -Add the [terraform-provider-matchbox](https://github.com/coreos/terraform-provider-matchbox) plugin binary for your system to `~/.terraform.d/plugins/`, noting the final name. +Add the [terraform-provider-matchbox](https://github.com/poseidon/terraform-provider-matchbox) plugin binary for your system to `~/.terraform.d/plugins/`, noting the final name. ```sh -wget https://github.com/coreos/terraform-provider-matchbox/releases/download/v0.2.3/terraform-provider-matchbox-v0.2.3-linux-amd64.tar.gz +wget https://github.com/poseidon/terraform-provider-matchbox/releases/download/v0.2.3/terraform-provider-matchbox-v0.2.3-linux-amd64.tar.gz tar xzf terraform-provider-matchbox-v0.2.3-linux-amd64.tar.gz mv terraform-provider-matchbox-v0.2.3-linux-amd64/terraform-provider-matchbox ~/.terraform.d/plugins/terraform-provider-matchbox_v0.2.3 ``` -Add the [terraform-provider-ct](https://github.com/coreos/terraform-provider-ct) plugin binary for your system to `~/.terraform.d/plugins/`, noting the final name. +Add the [terraform-provider-ct](https://github.com/poseidon/terraform-provider-ct) plugin binary for your system to `~/.terraform.d/plugins/`, noting the final name. ```sh -wget https://github.com/coreos/terraform-provider-ct/releases/download/v0.3.1/terraform-provider-ct-v0.3.1-linux-amd64.tar.gz +wget https://github.com/poseidon/terraform-provider-ct/releases/download/v0.3.1/terraform-provider-ct-v0.3.1-linux-amd64.tar.gz tar xzf terraform-provider-ct-v0.3.1-linux-amd64.tar.gz mv terraform-provider-ct-v0.3.1-linux-amd64/terraform-provider-ct ~/.terraform.d/plugins/terraform-provider-ct_v0.3.1 ``` diff --git a/docs/cl/digital-ocean.md b/docs/cl/digital-ocean.md index de965b454..37d46350a 100644 --- a/docs/cl/digital-ocean.md +++ b/docs/cl/digital-ocean.md @@ -10,7 +10,7 @@ Controllers are provisioned to run an `etcd-member` peer and a `kubelet` service * Digital Ocean Account and Token * Digital Ocean Domain (registered Domain Name or delegated subdomain) -* Terraform v0.11.x and [terraform-provider-ct](https://github.com/coreos/terraform-provider-ct) installed locally +* Terraform v0.11.x and [terraform-provider-ct](https://github.com/poseidon/terraform-provider-ct) installed locally ## Terraform Setup @@ -21,10 +21,10 @@ $ terraform version Terraform v0.11.12 ``` -Add the [terraform-provider-ct](https://github.com/coreos/terraform-provider-ct) plugin binary for your system to `~/.terraform.d/plugins/`, noting the final name. +Add the [terraform-provider-ct](https://github.com/poseidon/terraform-provider-ct) plugin binary for your system to `~/.terraform.d/plugins/`, noting the final name. ```sh -wget https://github.com/coreos/terraform-provider-ct/releases/download/v0.3.1/terraform-provider-ct-v0.3.1-linux-amd64.tar.gz +wget https://github.com/poseidon/terraform-provider-ct/releases/download/v0.3.1/terraform-provider-ct-v0.3.1-linux-amd64.tar.gz tar xzf terraform-provider-ct-v0.3.1-linux-amd64.tar.gz mv terraform-provider-ct-v0.3.1-linux-amd64/terraform-provider-ct ~/.terraform.d/plugins/terraform-provider-ct_v0.3.1 ``` diff --git a/docs/cl/google-cloud.md b/docs/cl/google-cloud.md index a84f14779..34bb5807e 100644 --- a/docs/cl/google-cloud.md +++ b/docs/cl/google-cloud.md @@ -10,7 +10,7 @@ Controllers are provisioned to run an `etcd-member` peer and a `kubelet` service * Google Cloud Account and Service Account * Google Cloud DNS Zone (registered Domain Name or delegated subdomain) -* Terraform v0.11.x and [terraform-provider-ct](https://github.com/coreos/terraform-provider-ct) installed locally +* Terraform v0.11.x and [terraform-provider-ct](https://github.com/poseidon/terraform-provider-ct) installed locally ## Terraform Setup @@ -21,10 +21,10 @@ $ terraform version Terraform v0.11.12 ``` -Add the [terraform-provider-ct](https://github.com/coreos/terraform-provider-ct) plugin binary for your system to `~/.terraform.d/plugins/`, noting the final name. +Add the [terraform-provider-ct](https://github.com/poseidon/terraform-provider-ct) plugin binary for your system to `~/.terraform.d/plugins/`, noting the final name. ```sh -wget https://github.com/coreos/terraform-provider-ct/releases/download/v0.3.1/terraform-provider-ct-v0.3.1-linux-amd64.tar.gz +wget https://github.com/poseidon/terraform-provider-ct/releases/download/v0.3.1/terraform-provider-ct-v0.3.1-linux-amd64.tar.gz tar xzf terraform-provider-ct-v0.3.1-linux-amd64.tar.gz mv terraform-provider-ct-v0.3.1-linux-amd64/terraform-provider-ct ~/.terraform.d/plugins/terraform-provider-ct_v0.3.1 ``` diff --git a/docs/topics/maintenance.md b/docs/topics/maintenance.md index 6cd0141e6..8a0378d10 100644 --- a/docs/topics/maintenance.md +++ b/docs/topics/maintenance.md @@ -45,7 +45,7 @@ Blue-green replacement provides some subtler benefits as well: ### Bare-Metal -Typhoon bare-metal clusters are provisioned by a PXE-enabled network boot environment and a [Matchbox](https://github.com/coreos/matchbox) service. To upgrade, re-provision machines into a new cluster. +Typhoon bare-metal clusters are provisioned by a PXE-enabled network boot environment and a [Matchbox](https://github.com/poseidon/matchbox) service. To upgrade, re-provision machines into a new cluster. Failover application workloads to another cluster (varies). @@ -145,18 +145,18 @@ Migrate to using the Terraform plugin directory. Move `~/.terraformrc` to a back mv ~/.terraformrc ~/.terraform-backup ``` -Add the [terraform-provider-ct](https://github.com/coreos/terraform-provider-ct) plugin binary for your system to `~/.terraform.d/plugins/`. Download the **same version** of `terraform-provider-ct` you were using with `~/.terraformrc`, updating only be done as a followup and is **only** safe for v1.12.2+ clusters! +Add the [terraform-provider-ct](https://github.com/poseidon/terraform-provider-ct) plugin binary for your system to `~/.terraform.d/plugins/`. Download the **same version** of `terraform-provider-ct` you were using with `~/.terraformrc`, updating only be done as a followup and is **only** safe for v1.12.2+ clusters! ```sh -wget https://github.com/coreos/terraform-provider-ct/releases/download/v0.2.1/terraform-provider-ct-v0.2.1-linux-amd64.tar.gz +wget https://github.com/poseidon/terraform-provider-ct/releases/download/v0.2.1/terraform-provider-ct-v0.2.1-linux-amd64.tar.gz tar xzf terraform-provider-ct-v0.2.1-linux-amd64.tar.gz mv terraform-provider-ct-v0.2.1-linux-amd64/terraform-provider-ct ~/.terraform.d/plugins/terraform-provider-ct_v0.2.1 ``` -If you use bare-metal, add the [terraform-provider-matchbox](https://github.com/coreos/terraform-provider-matchbox) plugin binary for your system to `~/.terraform.d/plugins/`, noting the versioned name. +If you use bare-metal, add the [terraform-provider-matchbox](https://github.com/poseidon/terraform-provider-matchbox) plugin binary for your system to `~/.terraform.d/plugins/`, noting the versioned name. ```sh -wget https://github.com/coreos/terraform-provider-matchbox/releases/download/v0.2.3/terraform-provider-matchbox-v0.2.3-linux-amd64.tar.gz +wget https://github.com/poseidon/terraform-provider-matchbox/releases/download/v0.2.3/terraform-provider-matchbox-v0.2.3-linux-amd64.tar.gz tar xzf terraform-provider-matchbox-v0.2.3-linux-amd64.tar.gz mv terraform-provider-matchbox-v0.2.3-linux-amd64/terraform-provider-matchbox ~/.terraform.d/plugins/terraform-provider-matchbox_v0.2.3 ``` @@ -195,14 +195,14 @@ $ terraform plan ### Upgrade terraform-provider-ct -The [terraform-provider-ct](https://github.com/coreos/terraform-provider-ct) plugin parses, validates, and converts Container Linux Configs into Ignition user-data for provisioning instances. Previously, updating the plugin re-provisioned controller nodes and was destructive to clusters. With Typhoon v1.12.2+, the plugin can be updated in-place and on apply, only workers will be replaced. +The [terraform-provider-ct](https://github.com/poseidon/terraform-provider-ct) plugin parses, validates, and converts Container Linux Configs into Ignition user-data for provisioning instances. Previously, updating the plugin re-provisioned controller nodes and was destructive to clusters. With Typhoon v1.12.2+, the plugin can be updated in-place and on apply, only workers will be replaced. First, [migrate](#terraform-plugins-directory) to the Terraform 3rd-party plugin directory to allow 3rd-party plugins to be defined and versioned independently (rather than globally). -Add the [terraform-provider-ct](https://github.com/coreos/terraform-provider-ct) plugin binary for your system to `~/.terraform.d/plugins/`, noting the final name. +Add the [terraform-provider-ct](https://github.com/poseidon/terraform-provider-ct) plugin binary for your system to `~/.terraform.d/plugins/`, noting the final name. ```sh -wget https://github.com/coreos/terraform-provider-ct/releases/download/v0.3.1/terraform-provider-ct-v0.3.1-linux-amd64.tar.gz +wget https://github.com/poseidon/terraform-provider-ct/releases/download/v0.3.1/terraform-provider-ct-v0.3.1-linux-amd64.tar.gz tar xzf terraform-provider-ct-v0.3.1-linux-amd64.tar.gz mv terraform-provider-ct-v0.3.1-linux-amd64/terraform-provider-ct ~/.terraform.d/plugins/terraform-provider-ct_v0.3.1 ``` From 6e9b2450fe5a713c26f78a874a91cab9bc38ad53 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Sat, 4 May 2019 11:14:37 -0700 Subject: [PATCH 093/523] Update Grafana from v6.1.4 to v6.1.6 * https://github.com/grafana/grafana/releases/tag/v6.1.6 --- CHANGES.md | 2 +- addons/grafana/deployment.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index c009fff04..9d3d5f0f8 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -8,7 +8,7 @@ Notable changes between versions. * Update Prometheus from v2.8.1 to v2.9.2 * Update kube-state-metrics from v1.5.0 to v1.6.0-rc.2 -* Update Grafana from v6.1.3 to v6.1.4 +* Update Grafana from v6.1.3 to v6.1.6 ## v1.14.1 diff --git a/addons/grafana/deployment.yaml b/addons/grafana/deployment.yaml index 809290fd4..2190134f6 100644 --- a/addons/grafana/deployment.yaml +++ b/addons/grafana/deployment.yaml @@ -23,7 +23,7 @@ spec: spec: containers: - name: grafana - image: grafana/grafana:6.1.4 + image: grafana/grafana:6.1.6 env: - name: GF_PATHS_CONFIG value: "/etc/grafana/custom.ini" From fd3c81d04de1eb5bcc6a59c2d330f8bf2650a3f2 Mon Sep 17 00:00:00 2001 From: Jordan Pittier Date: Sat, 4 May 2019 20:36:02 +0200 Subject: [PATCH 094/523] Remove create/update endpoints from nginx-ingress Role (#458) * nginx-ingress no longer requires endpoints create/update RBAC Role permissions * https://github.com/kubernetes/ingress-nginx/pull/1527 --- addons/nginx-ingress/aws/rbac/role.yaml | 2 -- addons/nginx-ingress/azure/rbac/role.yaml | 2 -- addons/nginx-ingress/bare-metal/rbac/role.yaml | 2 -- addons/nginx-ingress/digital-ocean/rbac/role.yaml | 2 -- addons/nginx-ingress/google-cloud/rbac/role.yaml | 2 -- 5 files changed, 10 deletions(-) diff --git a/addons/nginx-ingress/aws/rbac/role.yaml b/addons/nginx-ingress/aws/rbac/role.yaml index 84a6065f9..adbcb50cb 100644 --- a/addons/nginx-ingress/aws/rbac/role.yaml +++ b/addons/nginx-ingress/aws/rbac/role.yaml @@ -37,5 +37,3 @@ rules: - endpoints verbs: - get - - create - - update diff --git a/addons/nginx-ingress/azure/rbac/role.yaml b/addons/nginx-ingress/azure/rbac/role.yaml index 84a6065f9..adbcb50cb 100644 --- a/addons/nginx-ingress/azure/rbac/role.yaml +++ b/addons/nginx-ingress/azure/rbac/role.yaml @@ -37,5 +37,3 @@ rules: - endpoints verbs: - get - - create - - update diff --git a/addons/nginx-ingress/bare-metal/rbac/role.yaml b/addons/nginx-ingress/bare-metal/rbac/role.yaml index 84a6065f9..adbcb50cb 100644 --- a/addons/nginx-ingress/bare-metal/rbac/role.yaml +++ b/addons/nginx-ingress/bare-metal/rbac/role.yaml @@ -37,5 +37,3 @@ rules: - endpoints verbs: - get - - create - - update diff --git a/addons/nginx-ingress/digital-ocean/rbac/role.yaml b/addons/nginx-ingress/digital-ocean/rbac/role.yaml index 84a6065f9..adbcb50cb 100644 --- a/addons/nginx-ingress/digital-ocean/rbac/role.yaml +++ b/addons/nginx-ingress/digital-ocean/rbac/role.yaml @@ -37,5 +37,3 @@ rules: - endpoints verbs: - get - - create - - update diff --git a/addons/nginx-ingress/google-cloud/rbac/role.yaml b/addons/nginx-ingress/google-cloud/rbac/role.yaml index 84a6065f9..adbcb50cb 100644 --- a/addons/nginx-ingress/google-cloud/rbac/role.yaml +++ b/addons/nginx-ingress/google-cloud/rbac/role.yaml @@ -37,5 +37,3 @@ rules: - endpoints verbs: - get - - create - - update From ecbbdd905e8315400c75f92319b65f0a3559c63e Mon Sep 17 00:00:00 2001 From: Jordan Pittier Date: Sat, 4 May 2019 21:27:22 +0200 Subject: [PATCH 095/523] Use ./ prefix for inner/local worker pool modules * Terraform v0.11 encouraged use of a "./" prefix for local module references and Terraform v0.12 will require it * https://www.terraform.io/docs/modules/sources.html#local-paths Related: https://github.com/hashicorp/terraform/issues/19745 --- aws/container-linux/kubernetes/workers.tf | 2 +- aws/fedora-atomic/kubernetes/workers.tf | 2 +- azure/container-linux/kubernetes/workers.tf | 2 +- google-cloud/container-linux/kubernetes/workers.tf | 2 +- google-cloud/fedora-atomic/kubernetes/workers.tf | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/aws/container-linux/kubernetes/workers.tf b/aws/container-linux/kubernetes/workers.tf index 987d8d135..5065c0dbb 100644 --- a/aws/container-linux/kubernetes/workers.tf +++ b/aws/container-linux/kubernetes/workers.tf @@ -1,5 +1,5 @@ module "workers" { - source = "workers" + source = "./workers" name = "${var.cluster_name}" # AWS diff --git a/aws/fedora-atomic/kubernetes/workers.tf b/aws/fedora-atomic/kubernetes/workers.tf index 79273f53b..56da05678 100644 --- a/aws/fedora-atomic/kubernetes/workers.tf +++ b/aws/fedora-atomic/kubernetes/workers.tf @@ -1,5 +1,5 @@ module "workers" { - source = "workers" + source = "./workers" name = "${var.cluster_name}" # AWS diff --git a/azure/container-linux/kubernetes/workers.tf b/azure/container-linux/kubernetes/workers.tf index 1ff6246db..1abbc6571 100644 --- a/azure/container-linux/kubernetes/workers.tf +++ b/azure/container-linux/kubernetes/workers.tf @@ -1,5 +1,5 @@ module "workers" { - source = "workers" + source = "./workers" name = "${var.cluster_name}" # Azure diff --git a/google-cloud/container-linux/kubernetes/workers.tf b/google-cloud/container-linux/kubernetes/workers.tf index 565fbb7fc..b33607741 100644 --- a/google-cloud/container-linux/kubernetes/workers.tf +++ b/google-cloud/container-linux/kubernetes/workers.tf @@ -1,5 +1,5 @@ module "workers" { - source = "workers" + source = "./workers" name = "${var.cluster_name}" cluster_name = "${var.cluster_name}" diff --git a/google-cloud/fedora-atomic/kubernetes/workers.tf b/google-cloud/fedora-atomic/kubernetes/workers.tf index bc16d43ae..0bbd16b51 100644 --- a/google-cloud/fedora-atomic/kubernetes/workers.tf +++ b/google-cloud/fedora-atomic/kubernetes/workers.tf @@ -1,5 +1,5 @@ module "workers" { - source = "workers" + source = "./workers" name = "${var.cluster_name}" cluster_name = "${var.cluster_name}" From feb6192aac1d055b0fe296169ef31a0f57c1b8cc Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Sat, 4 May 2019 11:42:51 -0700 Subject: [PATCH 096/523] Update etcd from v3.3.12 to v3.3.13 on Container Linux * Skip updating etcd for Fedora Atomic clusters, now that Fedora Atomic has been deprecated --- CHANGES.md | 3 +++ aws/container-linux/kubernetes/cl/controller.yaml.tmpl | 2 +- azure/container-linux/kubernetes/cl/controller.yaml.tmpl | 2 +- bare-metal/container-linux/kubernetes/cl/controller.yaml.tmpl | 2 +- .../container-linux/kubernetes/cl/controller.yaml.tmpl | 2 +- .../container-linux/kubernetes/cl/controller.yaml.tmpl | 2 +- 6 files changed, 8 insertions(+), 5 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 9d3d5f0f8..d7e7e50af 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -3,12 +3,15 @@ Notable changes between versions. ## Latest + +* Update etcd from v3.3.12 to [v3.3.13](https://github.com/etcd-io/etcd/releases/tag/v3.3.13) #### Addons * Update Prometheus from v2.8.1 to v2.9.2 * Update kube-state-metrics from v1.5.0 to v1.6.0-rc.2 * Update Grafana from v6.1.3 to v6.1.6 +* Reduce nginx-ingress Role RBAC permissions ([#458](https://github.com/poseidon/typhoon/pull/458)) ## v1.14.1 diff --git a/aws/container-linux/kubernetes/cl/controller.yaml.tmpl b/aws/container-linux/kubernetes/cl/controller.yaml.tmpl index d6472d4ce..dd9d98fdb 100644 --- a/aws/container-linux/kubernetes/cl/controller.yaml.tmpl +++ b/aws/container-linux/kubernetes/cl/controller.yaml.tmpl @@ -7,7 +7,7 @@ systemd: - name: 40-etcd-cluster.conf contents: | [Service] - Environment="ETCD_IMAGE_TAG=v3.3.12" + Environment="ETCD_IMAGE_TAG=v3.3.13" Environment="ETCD_NAME=${etcd_name}" Environment="ETCD_ADVERTISE_CLIENT_URLS=https://${etcd_domain}:2379" Environment="ETCD_INITIAL_ADVERTISE_PEER_URLS=https://${etcd_domain}:2380" diff --git a/azure/container-linux/kubernetes/cl/controller.yaml.tmpl b/azure/container-linux/kubernetes/cl/controller.yaml.tmpl index d6472d4ce..dd9d98fdb 100644 --- a/azure/container-linux/kubernetes/cl/controller.yaml.tmpl +++ b/azure/container-linux/kubernetes/cl/controller.yaml.tmpl @@ -7,7 +7,7 @@ systemd: - name: 40-etcd-cluster.conf contents: | [Service] - Environment="ETCD_IMAGE_TAG=v3.3.12" + Environment="ETCD_IMAGE_TAG=v3.3.13" Environment="ETCD_NAME=${etcd_name}" Environment="ETCD_ADVERTISE_CLIENT_URLS=https://${etcd_domain}:2379" Environment="ETCD_INITIAL_ADVERTISE_PEER_URLS=https://${etcd_domain}:2380" diff --git a/bare-metal/container-linux/kubernetes/cl/controller.yaml.tmpl b/bare-metal/container-linux/kubernetes/cl/controller.yaml.tmpl index b8e29c37e..95b746a86 100644 --- a/bare-metal/container-linux/kubernetes/cl/controller.yaml.tmpl +++ b/bare-metal/container-linux/kubernetes/cl/controller.yaml.tmpl @@ -7,7 +7,7 @@ systemd: - name: 40-etcd-cluster.conf contents: | [Service] - Environment="ETCD_IMAGE_TAG=v3.3.12" + Environment="ETCD_IMAGE_TAG=v3.3.13" Environment="ETCD_NAME=${etcd_name}" Environment="ETCD_ADVERTISE_CLIENT_URLS=https://${domain_name}:2379" Environment="ETCD_INITIAL_ADVERTISE_PEER_URLS=https://${domain_name}:2380" diff --git a/digital-ocean/container-linux/kubernetes/cl/controller.yaml.tmpl b/digital-ocean/container-linux/kubernetes/cl/controller.yaml.tmpl index 84ac9c41f..5aa77d1d7 100644 --- a/digital-ocean/container-linux/kubernetes/cl/controller.yaml.tmpl +++ b/digital-ocean/container-linux/kubernetes/cl/controller.yaml.tmpl @@ -7,7 +7,7 @@ systemd: - name: 40-etcd-cluster.conf contents: | [Service] - Environment="ETCD_IMAGE_TAG=v3.3.12" + Environment="ETCD_IMAGE_TAG=v3.3.13" Environment="ETCD_NAME=${etcd_name}" Environment="ETCD_ADVERTISE_CLIENT_URLS=https://${etcd_domain}:2379" Environment="ETCD_INITIAL_ADVERTISE_PEER_URLS=https://${etcd_domain}:2380" diff --git a/google-cloud/container-linux/kubernetes/cl/controller.yaml.tmpl b/google-cloud/container-linux/kubernetes/cl/controller.yaml.tmpl index b7a09a55f..c7d8be081 100644 --- a/google-cloud/container-linux/kubernetes/cl/controller.yaml.tmpl +++ b/google-cloud/container-linux/kubernetes/cl/controller.yaml.tmpl @@ -7,7 +7,7 @@ systemd: - name: 40-etcd-cluster.conf contents: | [Service] - Environment="ETCD_IMAGE_TAG=v3.3.12" + Environment="ETCD_IMAGE_TAG=v3.3.13" Environment="ETCD_NAME=${etcd_name}" Environment="ETCD_ADVERTISE_CLIENT_URLS=https://${etcd_domain}:2379" Environment="ETCD_INITIAL_ADVERTISE_PEER_URLS=https://${etcd_domain}:2380" From 93f7a3508a70d18841418a1a2183f9e815719383 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Sat, 4 May 2019 13:01:29 -0700 Subject: [PATCH 097/523] Hide Fedora Atomic docs from site navigation * Remove Fedora Atomic tutorials and docs from the Typhoon site to make it more obvious the modules are deprecated * Continue to serve Fedora Atomic materials via direct link for some time --- mkdocs.yml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/mkdocs.yml b/mkdocs.yml index dc6be25fb..ba0fa18e3 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -59,11 +59,6 @@ nav: - 'Bare-Metal': 'cl/bare-metal.md' - 'Digital Ocean': 'cl/digital-ocean.md' - 'Google Cloud': 'cl/google-cloud.md' - - 'Fedora Atomic': - - 'AWS': 'atomic/aws.md' - - 'Bare-Metal': 'atomic/bare-metal.md' - - 'Digital Ocean': 'atomic/digital-ocean.md' - - 'Google Cloud': 'atomic/google-cloud.md' - 'Topics': - 'Maintenance': 'topics/maintenance.md' - 'Hardware': 'topics/hardware.md' From 09e02301118f3a597839e033663698e8edba0555 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Mon, 6 May 2019 00:44:15 -0700 Subject: [PATCH 098/523] Upgrade Calico from v3.6.1 to v3.7.0 * https://docs.projectcalico.org/v3.7/release-notes/ * https://github.com/poseidon/terraform-render-bootkube/pull/131 --- CHANGES.md | 1 + aws/container-linux/kubernetes/bootkube.tf | 2 +- aws/fedora-atomic/kubernetes/bootkube.tf | 2 +- azure/container-linux/kubernetes/bootkube.tf | 2 +- bare-metal/container-linux/kubernetes/bootkube.tf | 2 +- bare-metal/fedora-atomic/kubernetes/bootkube.tf | 2 +- digital-ocean/container-linux/kubernetes/bootkube.tf | 2 +- digital-ocean/fedora-atomic/kubernetes/bootkube.tf | 2 +- google-cloud/container-linux/kubernetes/bootkube.tf | 2 +- google-cloud/fedora-atomic/kubernetes/bootkube.tf | 2 +- 10 files changed, 10 insertions(+), 9 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index d7e7e50af..e46fd4c6c 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -5,6 +5,7 @@ Notable changes between versions. ## Latest * Update etcd from v3.3.12 to [v3.3.13](https://github.com/etcd-io/etcd/releases/tag/v3.3.13) +* Upgrade Calico from v3.6.1 to [v3.7.0](https://docs.projectcalico.org/v3.7/release-notes/) #### Addons diff --git a/aws/container-linux/kubernetes/bootkube.tf b/aws/container-linux/kubernetes/bootkube.tf index 605f57a5d..416ae51c8 100644 --- a/aws/container-linux/kubernetes/bootkube.tf +++ b/aws/container-linux/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=a80eed2b6ac489243a6454dc2f46b17eefa7d84d" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=b96d641f6d42cf5d9bf3ac36f557aa21cc157680" cluster_name = "${var.cluster_name}" api_servers = ["${format("%s.%s", var.cluster_name, var.dns_zone)}"] diff --git a/aws/fedora-atomic/kubernetes/bootkube.tf b/aws/fedora-atomic/kubernetes/bootkube.tf index 6d00cbeda..89ab7c15b 100644 --- a/aws/fedora-atomic/kubernetes/bootkube.tf +++ b/aws/fedora-atomic/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=a80eed2b6ac489243a6454dc2f46b17eefa7d84d" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=b96d641f6d42cf5d9bf3ac36f557aa21cc157680" cluster_name = "${var.cluster_name}" api_servers = ["${format("%s.%s", var.cluster_name, var.dns_zone)}"] diff --git a/azure/container-linux/kubernetes/bootkube.tf b/azure/container-linux/kubernetes/bootkube.tf index d9de6b92e..a23afa20d 100644 --- a/azure/container-linux/kubernetes/bootkube.tf +++ b/azure/container-linux/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=a80eed2b6ac489243a6454dc2f46b17eefa7d84d" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=b96d641f6d42cf5d9bf3ac36f557aa21cc157680" cluster_name = "${var.cluster_name}" api_servers = ["${format("%s.%s", var.cluster_name, var.dns_zone)}"] diff --git a/bare-metal/container-linux/kubernetes/bootkube.tf b/bare-metal/container-linux/kubernetes/bootkube.tf index 062ddcf8f..3222ca965 100644 --- a/bare-metal/container-linux/kubernetes/bootkube.tf +++ b/bare-metal/container-linux/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=a80eed2b6ac489243a6454dc2f46b17eefa7d84d" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=b96d641f6d42cf5d9bf3ac36f557aa21cc157680" cluster_name = "${var.cluster_name}" api_servers = ["${var.k8s_domain_name}"] diff --git a/bare-metal/fedora-atomic/kubernetes/bootkube.tf b/bare-metal/fedora-atomic/kubernetes/bootkube.tf index 934b39643..25eb150ae 100644 --- a/bare-metal/fedora-atomic/kubernetes/bootkube.tf +++ b/bare-metal/fedora-atomic/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=a80eed2b6ac489243a6454dc2f46b17eefa7d84d" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=b96d641f6d42cf5d9bf3ac36f557aa21cc157680" cluster_name = "${var.cluster_name}" api_servers = ["${var.k8s_domain_name}"] diff --git a/digital-ocean/container-linux/kubernetes/bootkube.tf b/digital-ocean/container-linux/kubernetes/bootkube.tf index 16dd9805c..df06e23c4 100644 --- a/digital-ocean/container-linux/kubernetes/bootkube.tf +++ b/digital-ocean/container-linux/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=a80eed2b6ac489243a6454dc2f46b17eefa7d84d" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=b96d641f6d42cf5d9bf3ac36f557aa21cc157680" cluster_name = "${var.cluster_name}" api_servers = ["${format("%s.%s", var.cluster_name, var.dns_zone)}"] diff --git a/digital-ocean/fedora-atomic/kubernetes/bootkube.tf b/digital-ocean/fedora-atomic/kubernetes/bootkube.tf index 6d37b646f..801ec6b4e 100644 --- a/digital-ocean/fedora-atomic/kubernetes/bootkube.tf +++ b/digital-ocean/fedora-atomic/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=a80eed2b6ac489243a6454dc2f46b17eefa7d84d" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=b96d641f6d42cf5d9bf3ac36f557aa21cc157680" cluster_name = "${var.cluster_name}" api_servers = ["${format("%s.%s", var.cluster_name, var.dns_zone)}"] diff --git a/google-cloud/container-linux/kubernetes/bootkube.tf b/google-cloud/container-linux/kubernetes/bootkube.tf index 9d8279a85..ca1718dab 100644 --- a/google-cloud/container-linux/kubernetes/bootkube.tf +++ b/google-cloud/container-linux/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=a80eed2b6ac489243a6454dc2f46b17eefa7d84d" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=b96d641f6d42cf5d9bf3ac36f557aa21cc157680" cluster_name = "${var.cluster_name}" api_servers = ["${format("%s.%s", var.cluster_name, var.dns_zone)}"] diff --git a/google-cloud/fedora-atomic/kubernetes/bootkube.tf b/google-cloud/fedora-atomic/kubernetes/bootkube.tf index 78469044d..88ab88127 100644 --- a/google-cloud/fedora-atomic/kubernetes/bootkube.tf +++ b/google-cloud/fedora-atomic/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=a80eed2b6ac489243a6454dc2f46b17eefa7d84d" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=b96d641f6d42cf5d9bf3ac36f557aa21cc157680" cluster_name = "${var.cluster_name}" api_servers = ["${format("%s.%s", var.cluster_name, var.dns_zone)}"] From 2d19ab84575233bc0489ae96569ade7d0c2cc295 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Mon, 6 May 2019 21:30:49 -0700 Subject: [PATCH 099/523] Update kube-state-metrics from v1.6.0-rc.2 to v1.6.0 * https://github.com/kubernetes/kube-state-metrics/releases/tag/v1.6.0 --- CHANGES.md | 2 +- addons/prometheus/exporters/kube-state-metrics/deployment.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index e46fd4c6c..3abf044c4 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -10,7 +10,7 @@ Notable changes between versions. #### Addons * Update Prometheus from v2.8.1 to v2.9.2 - * Update kube-state-metrics from v1.5.0 to v1.6.0-rc.2 + * Update kube-state-metrics from v1.5.0 to v1.6.0 * Update Grafana from v6.1.3 to v6.1.6 * Reduce nginx-ingress Role RBAC permissions ([#458](https://github.com/poseidon/typhoon/pull/458)) diff --git a/addons/prometheus/exporters/kube-state-metrics/deployment.yaml b/addons/prometheus/exporters/kube-state-metrics/deployment.yaml index ac23bb941..d2e0aff46 100644 --- a/addons/prometheus/exporters/kube-state-metrics/deployment.yaml +++ b/addons/prometheus/exporters/kube-state-metrics/deployment.yaml @@ -24,7 +24,7 @@ spec: serviceAccountName: kube-state-metrics containers: - name: kube-state-metrics - image: quay.io/coreos/kube-state-metrics:v1.6.0-rc.2 + image: quay.io/coreos/kube-state-metrics:v1.6.0 ports: - name: metrics containerPort: 8080 From af18296bc5ee8bf69745d236444df7a9d44c4050 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Mon, 6 May 2019 21:56:38 -0700 Subject: [PATCH 100/523] Change flannel port from 8472 to 4789 * Change flannel port from the kernel default 8472 to the IANA assigned VXLAN port 4789 * Update firewall rules or security groups for VXLAN * Why now? Calico now offers its own VXLAN backend so standardizing on the IANA port will simplify config * https://github.com/coreos/flannel/blob/master/Documentation/backends.md#vxlan --- CHANGES.md | 5 ++ aws/container-linux/kubernetes/bootkube.tf | 2 +- aws/container-linux/kubernetes/security.tf | 50 +++++++++++-------- aws/fedora-atomic/kubernetes/bootkube.tf | 2 +- aws/fedora-atomic/kubernetes/security.tf | 50 +++++++++++-------- azure/container-linux/kubernetes/bootkube.tf | 2 +- azure/container-linux/kubernetes/security.tf | 12 ++--- .../container-linux/kubernetes/bootkube.tf | 2 +- .../fedora-atomic/kubernetes/bootkube.tf | 2 +- .../container-linux/kubernetes/bootkube.tf | 2 +- .../container-linux/kubernetes/network.tf | 2 +- .../fedora-atomic/kubernetes/bootkube.tf | 2 +- .../container-linux/kubernetes/bootkube.tf | 2 +- .../container-linux/kubernetes/network.tf | 8 +-- .../fedora-atomic/kubernetes/bootkube.tf | 2 +- .../fedora-atomic/kubernetes/network.tf | 8 +-- 16 files changed, 87 insertions(+), 66 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 3abf044c4..96d2f099f 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -6,6 +6,11 @@ Notable changes between versions. * Update etcd from v3.3.12 to [v3.3.13](https://github.com/etcd-io/etcd/releases/tag/v3.3.13) * Upgrade Calico from v3.6.1 to [v3.7.0](https://docs.projectcalico.org/v3.7/release-notes/) +* Change flannel port from 8472 (kernel default) to 4789 (IANA VXLAN) + +#### AWS + +* Only set internal VXLAN rules when `networking` is flannel (default: calico) #### Addons diff --git a/aws/container-linux/kubernetes/bootkube.tf b/aws/container-linux/kubernetes/bootkube.tf index 416ae51c8..60e05daf3 100644 --- a/aws/container-linux/kubernetes/bootkube.tf +++ b/aws/container-linux/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=b96d641f6d42cf5d9bf3ac36f557aa21cc157680" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=fc7a6fb20abe41ccd40f3449ee8ee7bba7bd9932" cluster_name = "${var.cluster_name}" api_servers = ["${format("%s.%s", var.cluster_name, var.dns_zone)}"] diff --git a/aws/container-linux/kubernetes/security.tf b/aws/container-linux/kubernetes/security.tf index fc7b959af..7672a92e8 100644 --- a/aws/container-linux/kubernetes/security.tf +++ b/aws/container-linux/kubernetes/security.tf @@ -42,36 +42,40 @@ resource "aws_security_group_rule" "controller-etcd-metrics" { source_security_group_id = "${aws_security_group.worker.id}" } -resource "aws_security_group_rule" "controller-apiserver" { - security_group_id = "${aws_security_group.controller.id}" +resource "aws_security_group_rule" "controller-vxlan" { + count = "${var.networking == "flannel" ? 1 : 0}" - type = "ingress" - protocol = "tcp" - from_port = 6443 - to_port = 6443 - cidr_blocks = ["0.0.0.0/0"] -} - -resource "aws_security_group_rule" "controller-flannel" { security_group_id = "${aws_security_group.controller.id}" type = "ingress" protocol = "udp" - from_port = 8472 - to_port = 8472 + from_port = 4789 + to_port = 4789 source_security_group_id = "${aws_security_group.worker.id}" } -resource "aws_security_group_rule" "controller-flannel-self" { +resource "aws_security_group_rule" "controller-vxlan-self" { + count = "${var.networking == "flannel" ? 1 : 0}" + security_group_id = "${aws_security_group.controller.id}" type = "ingress" protocol = "udp" - from_port = 8472 - to_port = 8472 + from_port = 4789 + to_port = 4789 self = true } +resource "aws_security_group_rule" "controller-apiserver" { + security_group_id = "${aws_security_group.controller.id}" + + type = "ingress" + protocol = "tcp" + from_port = 6443 + to_port = 6443 + cidr_blocks = ["0.0.0.0/0"] +} + # Allow Prometheus to scrape node-exporter daemonset resource "aws_security_group_rule" "controller-node-exporter" { security_group_id = "${aws_security_group.controller.id}" @@ -216,23 +220,27 @@ resource "aws_security_group_rule" "worker-https" { cidr_blocks = ["0.0.0.0/0"] } -resource "aws_security_group_rule" "worker-flannel" { +resource "aws_security_group_rule" "worker-vxlan" { + count = "${var.networking == "flannel" ? 1 : 0}" + security_group_id = "${aws_security_group.worker.id}" type = "ingress" protocol = "udp" - from_port = 8472 - to_port = 8472 + from_port = 4789 + to_port = 4789 source_security_group_id = "${aws_security_group.controller.id}" } -resource "aws_security_group_rule" "worker-flannel-self" { +resource "aws_security_group_rule" "worker-vxlan-self" { + count = "${var.networking == "flannel" ? 1 : 0}" + security_group_id = "${aws_security_group.worker.id}" type = "ingress" protocol = "udp" - from_port = 8472 - to_port = 8472 + from_port = 4789 + to_port = 4789 self = true } diff --git a/aws/fedora-atomic/kubernetes/bootkube.tf b/aws/fedora-atomic/kubernetes/bootkube.tf index 89ab7c15b..5df842efd 100644 --- a/aws/fedora-atomic/kubernetes/bootkube.tf +++ b/aws/fedora-atomic/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=b96d641f6d42cf5d9bf3ac36f557aa21cc157680" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=fc7a6fb20abe41ccd40f3449ee8ee7bba7bd9932" cluster_name = "${var.cluster_name}" api_servers = ["${format("%s.%s", var.cluster_name, var.dns_zone)}"] diff --git a/aws/fedora-atomic/kubernetes/security.tf b/aws/fedora-atomic/kubernetes/security.tf index fc7b959af..7672a92e8 100644 --- a/aws/fedora-atomic/kubernetes/security.tf +++ b/aws/fedora-atomic/kubernetes/security.tf @@ -42,36 +42,40 @@ resource "aws_security_group_rule" "controller-etcd-metrics" { source_security_group_id = "${aws_security_group.worker.id}" } -resource "aws_security_group_rule" "controller-apiserver" { - security_group_id = "${aws_security_group.controller.id}" +resource "aws_security_group_rule" "controller-vxlan" { + count = "${var.networking == "flannel" ? 1 : 0}" - type = "ingress" - protocol = "tcp" - from_port = 6443 - to_port = 6443 - cidr_blocks = ["0.0.0.0/0"] -} - -resource "aws_security_group_rule" "controller-flannel" { security_group_id = "${aws_security_group.controller.id}" type = "ingress" protocol = "udp" - from_port = 8472 - to_port = 8472 + from_port = 4789 + to_port = 4789 source_security_group_id = "${aws_security_group.worker.id}" } -resource "aws_security_group_rule" "controller-flannel-self" { +resource "aws_security_group_rule" "controller-vxlan-self" { + count = "${var.networking == "flannel" ? 1 : 0}" + security_group_id = "${aws_security_group.controller.id}" type = "ingress" protocol = "udp" - from_port = 8472 - to_port = 8472 + from_port = 4789 + to_port = 4789 self = true } +resource "aws_security_group_rule" "controller-apiserver" { + security_group_id = "${aws_security_group.controller.id}" + + type = "ingress" + protocol = "tcp" + from_port = 6443 + to_port = 6443 + cidr_blocks = ["0.0.0.0/0"] +} + # Allow Prometheus to scrape node-exporter daemonset resource "aws_security_group_rule" "controller-node-exporter" { security_group_id = "${aws_security_group.controller.id}" @@ -216,23 +220,27 @@ resource "aws_security_group_rule" "worker-https" { cidr_blocks = ["0.0.0.0/0"] } -resource "aws_security_group_rule" "worker-flannel" { +resource "aws_security_group_rule" "worker-vxlan" { + count = "${var.networking == "flannel" ? 1 : 0}" + security_group_id = "${aws_security_group.worker.id}" type = "ingress" protocol = "udp" - from_port = 8472 - to_port = 8472 + from_port = 4789 + to_port = 4789 source_security_group_id = "${aws_security_group.controller.id}" } -resource "aws_security_group_rule" "worker-flannel-self" { +resource "aws_security_group_rule" "worker-vxlan-self" { + count = "${var.networking == "flannel" ? 1 : 0}" + security_group_id = "${aws_security_group.worker.id}" type = "ingress" protocol = "udp" - from_port = 8472 - to_port = 8472 + from_port = 4789 + to_port = 4789 self = true } diff --git a/azure/container-linux/kubernetes/bootkube.tf b/azure/container-linux/kubernetes/bootkube.tf index a23afa20d..884bbbba7 100644 --- a/azure/container-linux/kubernetes/bootkube.tf +++ b/azure/container-linux/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=b96d641f6d42cf5d9bf3ac36f557aa21cc157680" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=fc7a6fb20abe41ccd40f3449ee8ee7bba7bd9932" cluster_name = "${var.cluster_name}" api_servers = ["${format("%s.%s", var.cluster_name, var.dns_zone)}"] diff --git a/azure/container-linux/kubernetes/security.tf b/azure/container-linux/kubernetes/security.tf index 9f1d34638..138307099 100644 --- a/azure/container-linux/kubernetes/security.tf +++ b/azure/container-linux/kubernetes/security.tf @@ -68,17 +68,17 @@ resource "azurerm_network_security_rule" "controller-apiserver" { destination_address_prefix = "${azurerm_subnet.controller.address_prefix}" } -resource "azurerm_network_security_rule" "controller-flannel" { +resource "azurerm_network_security_rule" "controller-vxlan" { resource_group_name = "${azurerm_resource_group.cluster.name}" - name = "allow-flannel" + name = "allow-vxlan" network_security_group_name = "${azurerm_network_security_group.controller.name}" priority = "2020" access = "Allow" direction = "Inbound" protocol = "Udp" source_port_range = "*" - destination_port_range = "8472" + destination_port_range = "4789" source_address_prefixes = ["${azurerm_subnet.controller.address_prefix}", "${azurerm_subnet.worker.address_prefix}"] destination_address_prefix = "${azurerm_subnet.controller.address_prefix}" } @@ -204,17 +204,17 @@ resource "azurerm_network_security_rule" "worker-https" { destination_address_prefix = "${azurerm_subnet.worker.address_prefix}" } -resource "azurerm_network_security_rule" "worker-flannel" { +resource "azurerm_network_security_rule" "worker-vxlan" { resource_group_name = "${azurerm_resource_group.cluster.name}" - name = "allow-flannel" + name = "allow-vxlan" network_security_group_name = "${azurerm_network_security_group.worker.name}" priority = "2015" access = "Allow" direction = "Inbound" protocol = "Udp" source_port_range = "*" - destination_port_range = "8472" + destination_port_range = "4789" source_address_prefixes = ["${azurerm_subnet.controller.address_prefix}", "${azurerm_subnet.worker.address_prefix}"] destination_address_prefix = "${azurerm_subnet.worker.address_prefix}" } diff --git a/bare-metal/container-linux/kubernetes/bootkube.tf b/bare-metal/container-linux/kubernetes/bootkube.tf index 3222ca965..86ab9f04b 100644 --- a/bare-metal/container-linux/kubernetes/bootkube.tf +++ b/bare-metal/container-linux/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=b96d641f6d42cf5d9bf3ac36f557aa21cc157680" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=fc7a6fb20abe41ccd40f3449ee8ee7bba7bd9932" cluster_name = "${var.cluster_name}" api_servers = ["${var.k8s_domain_name}"] diff --git a/bare-metal/fedora-atomic/kubernetes/bootkube.tf b/bare-metal/fedora-atomic/kubernetes/bootkube.tf index 25eb150ae..f57c7e8cd 100644 --- a/bare-metal/fedora-atomic/kubernetes/bootkube.tf +++ b/bare-metal/fedora-atomic/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=b96d641f6d42cf5d9bf3ac36f557aa21cc157680" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=fc7a6fb20abe41ccd40f3449ee8ee7bba7bd9932" cluster_name = "${var.cluster_name}" api_servers = ["${var.k8s_domain_name}"] diff --git a/digital-ocean/container-linux/kubernetes/bootkube.tf b/digital-ocean/container-linux/kubernetes/bootkube.tf index df06e23c4..911c90fe9 100644 --- a/digital-ocean/container-linux/kubernetes/bootkube.tf +++ b/digital-ocean/container-linux/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=b96d641f6d42cf5d9bf3ac36f557aa21cc157680" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=fc7a6fb20abe41ccd40f3449ee8ee7bba7bd9932" cluster_name = "${var.cluster_name}" api_servers = ["${format("%s.%s", var.cluster_name, var.dns_zone)}"] diff --git a/digital-ocean/container-linux/kubernetes/network.tf b/digital-ocean/container-linux/kubernetes/network.tf index 52cf7fb12..76342f66c 100644 --- a/digital-ocean/container-linux/kubernetes/network.tf +++ b/digital-ocean/container-linux/kubernetes/network.tf @@ -12,7 +12,7 @@ resource "digitalocean_firewall" "rules" { }, { protocol = "udp" - port_range = "8472" + port_range = "4789" source_tags = ["${digitalocean_tag.controllers.name}", "${digitalocean_tag.workers.name}"] }, { diff --git a/digital-ocean/fedora-atomic/kubernetes/bootkube.tf b/digital-ocean/fedora-atomic/kubernetes/bootkube.tf index 801ec6b4e..fff4b5de8 100644 --- a/digital-ocean/fedora-atomic/kubernetes/bootkube.tf +++ b/digital-ocean/fedora-atomic/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=b96d641f6d42cf5d9bf3ac36f557aa21cc157680" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=fc7a6fb20abe41ccd40f3449ee8ee7bba7bd9932" cluster_name = "${var.cluster_name}" api_servers = ["${format("%s.%s", var.cluster_name, var.dns_zone)}"] diff --git a/google-cloud/container-linux/kubernetes/bootkube.tf b/google-cloud/container-linux/kubernetes/bootkube.tf index ca1718dab..1c1b79654 100644 --- a/google-cloud/container-linux/kubernetes/bootkube.tf +++ b/google-cloud/container-linux/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=b96d641f6d42cf5d9bf3ac36f557aa21cc157680" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=fc7a6fb20abe41ccd40f3449ee8ee7bba7bd9932" cluster_name = "${var.cluster_name}" api_servers = ["${format("%s.%s", var.cluster_name, var.dns_zone)}"] diff --git a/google-cloud/container-linux/kubernetes/network.tf b/google-cloud/container-linux/kubernetes/network.tf index 04496186e..e4d698cc3 100644 --- a/google-cloud/container-linux/kubernetes/network.tf +++ b/google-cloud/container-linux/kubernetes/network.tf @@ -78,16 +78,16 @@ resource "google_compute_firewall" "internal-bgp" { target_tags = ["${var.cluster_name}-controller", "${var.cluster_name}-worker"] } -# flannel -resource "google_compute_firewall" "internal-flannel" { +# flannel VXLAN +resource "google_compute_firewall" "internal-vxlan" { count = "${var.networking == "flannel" ? 1 : 0}" - name = "${var.cluster_name}-internal-flannel" + name = "${var.cluster_name}-internal-vxlan" network = "${google_compute_network.network.name}" allow { protocol = "udp" - ports = [8472] + ports = [4789] } source_tags = ["${var.cluster_name}-controller", "${var.cluster_name}-worker"] diff --git a/google-cloud/fedora-atomic/kubernetes/bootkube.tf b/google-cloud/fedora-atomic/kubernetes/bootkube.tf index 88ab88127..eafe60aab 100644 --- a/google-cloud/fedora-atomic/kubernetes/bootkube.tf +++ b/google-cloud/fedora-atomic/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=b96d641f6d42cf5d9bf3ac36f557aa21cc157680" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=fc7a6fb20abe41ccd40f3449ee8ee7bba7bd9932" cluster_name = "${var.cluster_name}" api_servers = ["${format("%s.%s", var.cluster_name, var.dns_zone)}"] diff --git a/google-cloud/fedora-atomic/kubernetes/network.tf b/google-cloud/fedora-atomic/kubernetes/network.tf index a507ce8ea..ecfe7ae7b 100644 --- a/google-cloud/fedora-atomic/kubernetes/network.tf +++ b/google-cloud/fedora-atomic/kubernetes/network.tf @@ -78,16 +78,16 @@ resource "google_compute_firewall" "internal-bgp" { target_tags = ["${var.cluster_name}-controller", "${var.cluster_name}-worker"] } -# flannel -resource "google_compute_firewall" "internal-flannel" { +# flannel VXLAN +resource "google_compute_firewall" "internal-vxlan" { count = "${var.networking == "flannel" ? 1 : 0}" - name = "${var.cluster_name}-internal-flannel" + name = "${var.cluster_name}-internal-vxlan" network = "${google_compute_network.network.name}" allow { protocol = "udp" - ports = [8472] + ports = [4789] } source_tags = ["${var.cluster_name}-controller", "${var.cluster_name}-worker"] From f62286b677f0ff7c13edb349b8315e95fd57f1e6 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Fri, 17 May 2019 12:29:46 +0200 Subject: [PATCH 101/523] Update Calico from v3.7.0 to v3.7.2 * https://docs.projectcalico.org/v3.7/release-notes/ --- CHANGES.md | 2 +- aws/container-linux/kubernetes/bootkube.tf | 2 +- aws/fedora-atomic/kubernetes/bootkube.tf | 2 +- azure/container-linux/kubernetes/bootkube.tf | 2 +- bare-metal/container-linux/kubernetes/bootkube.tf | 2 +- bare-metal/fedora-atomic/kubernetes/bootkube.tf | 2 +- digital-ocean/container-linux/kubernetes/bootkube.tf | 2 +- digital-ocean/fedora-atomic/kubernetes/bootkube.tf | 2 +- google-cloud/container-linux/kubernetes/bootkube.tf | 2 +- google-cloud/fedora-atomic/kubernetes/bootkube.tf | 2 +- 10 files changed, 10 insertions(+), 10 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 96d2f099f..7e662aadc 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -5,7 +5,7 @@ Notable changes between versions. ## Latest * Update etcd from v3.3.12 to [v3.3.13](https://github.com/etcd-io/etcd/releases/tag/v3.3.13) -* Upgrade Calico from v3.6.1 to [v3.7.0](https://docs.projectcalico.org/v3.7/release-notes/) +* Upgrade Calico from v3.6.1 to [v3.7.2](https://docs.projectcalico.org/v3.7/release-notes/) * Change flannel port from 8472 (kernel default) to 4789 (IANA VXLAN) #### AWS diff --git a/aws/container-linux/kubernetes/bootkube.tf b/aws/container-linux/kubernetes/bootkube.tf index 60e05daf3..a9775d8a3 100644 --- a/aws/container-linux/kubernetes/bootkube.tf +++ b/aws/container-linux/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=fc7a6fb20abe41ccd40f3449ee8ee7bba7bd9932" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=eca7c49fe17652e9f288c7cc4461dcaccc2c2661" cluster_name = "${var.cluster_name}" api_servers = ["${format("%s.%s", var.cluster_name, var.dns_zone)}"] diff --git a/aws/fedora-atomic/kubernetes/bootkube.tf b/aws/fedora-atomic/kubernetes/bootkube.tf index 5df842efd..ece0c213f 100644 --- a/aws/fedora-atomic/kubernetes/bootkube.tf +++ b/aws/fedora-atomic/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=fc7a6fb20abe41ccd40f3449ee8ee7bba7bd9932" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=eca7c49fe17652e9f288c7cc4461dcaccc2c2661" cluster_name = "${var.cluster_name}" api_servers = ["${format("%s.%s", var.cluster_name, var.dns_zone)}"] diff --git a/azure/container-linux/kubernetes/bootkube.tf b/azure/container-linux/kubernetes/bootkube.tf index 884bbbba7..d90ad5c77 100644 --- a/azure/container-linux/kubernetes/bootkube.tf +++ b/azure/container-linux/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=fc7a6fb20abe41ccd40f3449ee8ee7bba7bd9932" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=eca7c49fe17652e9f288c7cc4461dcaccc2c2661" cluster_name = "${var.cluster_name}" api_servers = ["${format("%s.%s", var.cluster_name, var.dns_zone)}"] diff --git a/bare-metal/container-linux/kubernetes/bootkube.tf b/bare-metal/container-linux/kubernetes/bootkube.tf index 86ab9f04b..7f2306315 100644 --- a/bare-metal/container-linux/kubernetes/bootkube.tf +++ b/bare-metal/container-linux/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=fc7a6fb20abe41ccd40f3449ee8ee7bba7bd9932" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=eca7c49fe17652e9f288c7cc4461dcaccc2c2661" cluster_name = "${var.cluster_name}" api_servers = ["${var.k8s_domain_name}"] diff --git a/bare-metal/fedora-atomic/kubernetes/bootkube.tf b/bare-metal/fedora-atomic/kubernetes/bootkube.tf index f57c7e8cd..8ae797846 100644 --- a/bare-metal/fedora-atomic/kubernetes/bootkube.tf +++ b/bare-metal/fedora-atomic/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=fc7a6fb20abe41ccd40f3449ee8ee7bba7bd9932" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=eca7c49fe17652e9f288c7cc4461dcaccc2c2661" cluster_name = "${var.cluster_name}" api_servers = ["${var.k8s_domain_name}"] diff --git a/digital-ocean/container-linux/kubernetes/bootkube.tf b/digital-ocean/container-linux/kubernetes/bootkube.tf index 911c90fe9..cb991001b 100644 --- a/digital-ocean/container-linux/kubernetes/bootkube.tf +++ b/digital-ocean/container-linux/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=fc7a6fb20abe41ccd40f3449ee8ee7bba7bd9932" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=eca7c49fe17652e9f288c7cc4461dcaccc2c2661" cluster_name = "${var.cluster_name}" api_servers = ["${format("%s.%s", var.cluster_name, var.dns_zone)}"] diff --git a/digital-ocean/fedora-atomic/kubernetes/bootkube.tf b/digital-ocean/fedora-atomic/kubernetes/bootkube.tf index fff4b5de8..02df3cbcb 100644 --- a/digital-ocean/fedora-atomic/kubernetes/bootkube.tf +++ b/digital-ocean/fedora-atomic/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=fc7a6fb20abe41ccd40f3449ee8ee7bba7bd9932" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=eca7c49fe17652e9f288c7cc4461dcaccc2c2661" cluster_name = "${var.cluster_name}" api_servers = ["${format("%s.%s", var.cluster_name, var.dns_zone)}"] diff --git a/google-cloud/container-linux/kubernetes/bootkube.tf b/google-cloud/container-linux/kubernetes/bootkube.tf index 1c1b79654..8fbf0e4e4 100644 --- a/google-cloud/container-linux/kubernetes/bootkube.tf +++ b/google-cloud/container-linux/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=fc7a6fb20abe41ccd40f3449ee8ee7bba7bd9932" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=eca7c49fe17652e9f288c7cc4461dcaccc2c2661" cluster_name = "${var.cluster_name}" api_servers = ["${format("%s.%s", var.cluster_name, var.dns_zone)}"] diff --git a/google-cloud/fedora-atomic/kubernetes/bootkube.tf b/google-cloud/fedora-atomic/kubernetes/bootkube.tf index eafe60aab..871cfdbb0 100644 --- a/google-cloud/fedora-atomic/kubernetes/bootkube.tf +++ b/google-cloud/fedora-atomic/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=fc7a6fb20abe41ccd40f3449ee8ee7bba7bd9932" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=eca7c49fe17652e9f288c7cc4461dcaccc2c2661" cluster_name = "${var.cluster_name}" api_servers = ["${format("%s.%s", var.cluster_name, var.dns_zone)}"] From 37ce722f9c32a259f19dcbc2369ef8c0e7dcf184 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Fri, 17 May 2019 12:52:16 +0200 Subject: [PATCH 102/523] Fix race condition in DigitalOcean cluster create * DigitalOcean clusters must secure copy a kubeconfig to worker nodes, but Terraform could decide to try copying before firewall rules have been added to allow SSH access. * Add an explicit dependency on adding firewall rules first --- CHANGES.md | 5 +++++ digital-ocean/container-linux/kubernetes/ssh.tf | 3 +++ digital-ocean/fedora-atomic/kubernetes/ssh.tf | 3 +++ 3 files changed, 11 insertions(+) diff --git a/CHANGES.md b/CHANGES.md index 7e662aadc..27e52f713 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -12,6 +12,11 @@ Notable changes between versions. * Only set internal VXLAN rules when `networking` is flannel (default: calico) +#### DigitalOcean + +* Add explicit ordering between firewall rule creation and secure copying Kubelet credentials ([#469](https://github.com/poseidon/typhoon/pull/469)) + * Fix race scenario if copies to nodes were before rule creation, blocking cluster creation + #### Addons * Update Prometheus from v2.8.1 to v2.9.2 diff --git a/digital-ocean/container-linux/kubernetes/ssh.tf b/digital-ocean/container-linux/kubernetes/ssh.tf index 5c5ed7749..98740a22e 100644 --- a/digital-ocean/container-linux/kubernetes/ssh.tf +++ b/digital-ocean/container-linux/kubernetes/ssh.tf @@ -1,6 +1,9 @@ # Secure copy etcd TLS assets and kubeconfig to controllers. Activates kubelet.service resource "null_resource" "copy-controller-secrets" { count = "${var.controller_count}" + depends_on = [ + "digitalocean_firewall.rules", + ] connection { type = "ssh" diff --git a/digital-ocean/fedora-atomic/kubernetes/ssh.tf b/digital-ocean/fedora-atomic/kubernetes/ssh.tf index 1d7e00025..156a4894b 100644 --- a/digital-ocean/fedora-atomic/kubernetes/ssh.tf +++ b/digital-ocean/fedora-atomic/kubernetes/ssh.tf @@ -1,6 +1,9 @@ # Secure copy etcd TLS assets and kubeconfig to controllers. Activates kubelet.service resource "null_resource" "copy-controller-secrets" { count = "${var.controller_count}" + depends_on = [ + "digitalocean_firewall.rules", + ] connection { type = "ssh" From da97bd4f129e3d457a13411b482850cf80ba46d9 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Fri, 17 May 2019 13:09:15 +0200 Subject: [PATCH 103/523] Update Kubernetes from v1.14.1 to v1.14.2 * https://github.com/kubernetes/kubernetes/blob/master/CHANGELOG-1.14.md#v1142 --- CHANGES.md | 1 + README.md | 10 +++++----- aws/container-linux/kubernetes/README.md | 2 +- aws/container-linux/kubernetes/bootkube.tf | 2 +- .../kubernetes/cl/controller.yaml.tmpl | 2 +- .../kubernetes/workers/cl/worker.yaml.tmpl | 4 ++-- aws/fedora-atomic/kubernetes/README.md | 2 +- aws/fedora-atomic/kubernetes/bootkube.tf | 2 +- azure/container-linux/kubernetes/README.md | 2 +- azure/container-linux/kubernetes/bootkube.tf | 2 +- .../kubernetes/cl/controller.yaml.tmpl | 2 +- .../kubernetes/workers/cl/worker.yaml.tmpl | 4 ++-- bare-metal/container-linux/kubernetes/README.md | 2 +- .../container-linux/kubernetes/bootkube.tf | 2 +- .../kubernetes/cl/controller.yaml.tmpl | 2 +- .../kubernetes/cl/worker.yaml.tmpl | 2 +- bare-metal/fedora-atomic/kubernetes/README.md | 2 +- bare-metal/fedora-atomic/kubernetes/bootkube.tf | 2 +- .../container-linux/kubernetes/README.md | 2 +- .../container-linux/kubernetes/bootkube.tf | 2 +- .../kubernetes/cl/controller.yaml.tmpl | 2 +- .../kubernetes/cl/worker.yaml.tmpl | 4 ++-- digital-ocean/fedora-atomic/kubernetes/README.md | 2 +- .../fedora-atomic/kubernetes/bootkube.tf | 2 +- docs/advanced/worker-pools.md | 16 ++++++++-------- docs/atomic/aws.md | 10 +++++----- docs/atomic/bare-metal.md | 10 +++++----- docs/atomic/digital-ocean.md | 10 +++++----- docs/atomic/google-cloud.md | 10 +++++----- docs/cl/aws.md | 10 +++++----- docs/cl/azure.md | 10 +++++----- docs/cl/bare-metal.md | 14 +++++++------- docs/cl/digital-ocean.md | 10 +++++----- docs/cl/google-cloud.md | 10 +++++----- docs/index.md | 10 +++++----- docs/topics/maintenance.md | 2 +- .../container-linux/kubernetes/README.md | 2 +- .../container-linux/kubernetes/bootkube.tf | 2 +- .../kubernetes/cl/controller.yaml.tmpl | 2 +- .../kubernetes/workers/cl/worker.yaml.tmpl | 4 ++-- google-cloud/fedora-atomic/kubernetes/README.md | 2 +- .../fedora-atomic/kubernetes/bootkube.tf | 2 +- 42 files changed, 99 insertions(+), 98 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 27e52f713..a7f5191a5 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -4,6 +4,7 @@ Notable changes between versions. ## Latest +* Kubernetes [v1.14.2](https://github.com/kubernetes/kubernetes/blob/master/CHANGELOG-1.14.md#v1142) * Update etcd from v3.3.12 to [v3.3.13](https://github.com/etcd-io/etcd/releases/tag/v3.3.13) * Upgrade Calico from v3.6.1 to [v3.7.2](https://docs.projectcalico.org/v3.7/release-notes/) * Change flannel port from 8472 (kernel default) to 4789 (IANA VXLAN) diff --git a/README.md b/README.md index 3bfbb4038..569ac2c5b 100644 --- a/README.md +++ b/README.md @@ -11,7 +11,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features -* Kubernetes v1.14.1 (upstream, via [kubernetes-incubator/bootkube](https://github.com/kubernetes-incubator/bootkube)) +* Kubernetes v1.14.2 (upstream, via [kubernetes-incubator/bootkube](https://github.com/kubernetes-incubator/bootkube)) * Single or multi-master, [Calico](https://www.projectcalico.org/) or [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/) * Advanced features like [worker pools](https://typhoon.psdn.io/advanced/worker-pools/), [preemptible](https://typhoon.psdn.io/cl/google-cloud/#preemption) workers, and [snippets](https://typhoon.psdn.io/advanced/customization/#container-linux) customization @@ -50,7 +50,7 @@ Define a Kubernetes cluster by using the Terraform module for your chosen platfo ```tf module "google-cloud-yavin" { - source = "git::https://github.com/poseidon/typhoon//google-cloud/container-linux/kubernetes?ref=v1.14.1" + source = "git::https://github.com/poseidon/typhoon//google-cloud/container-linux/kubernetes?ref=v1.14.2" providers = { google = "google.default" @@ -91,9 +91,9 @@ In 4-8 minutes (varies by platform), the cluster will be ready. This Google Clou $ export KUBECONFIG=/home/user/.secrets/clusters/yavin/auth/kubeconfig $ kubectl get nodes NAME ROLES STATUS AGE VERSION -yavin-controller-0.c.example-com.internal controller,master Ready 6m v1.14.1 -yavin-worker-jrbf.c.example-com.internal node Ready 5m v1.14.1 -yavin-worker-mzdm.c.example-com.internal node Ready 5m v1.14.1 +yavin-controller-0.c.example-com.internal controller,master Ready 6m v1.14.2 +yavin-worker-jrbf.c.example-com.internal node Ready 5m v1.14.2 +yavin-worker-mzdm.c.example-com.internal node Ready 5m v1.14.2 ``` List the pods. diff --git a/aws/container-linux/kubernetes/README.md b/aws/container-linux/kubernetes/README.md index 7a66e9b57..25911236e 100644 --- a/aws/container-linux/kubernetes/README.md +++ b/aws/container-linux/kubernetes/README.md @@ -11,7 +11,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features -* Kubernetes v1.14.1 (upstream, via [kubernetes-incubator/bootkube](https://github.com/kubernetes-incubator/bootkube)) +* Kubernetes v1.14.2 (upstream, via [kubernetes-incubator/bootkube](https://github.com/kubernetes-incubator/bootkube)) * Single or multi-master, [Calico](https://www.projectcalico.org/) or [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/) * Advanced features like [worker pools](https://typhoon.psdn.io/advanced/worker-pools/), [spot](https://typhoon.psdn.io/cl/aws/#spot) workers, and [snippets](https://typhoon.psdn.io/advanced/customization/#container-linux) customization diff --git a/aws/container-linux/kubernetes/bootkube.tf b/aws/container-linux/kubernetes/bootkube.tf index a9775d8a3..416634479 100644 --- a/aws/container-linux/kubernetes/bootkube.tf +++ b/aws/container-linux/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=eca7c49fe17652e9f288c7cc4461dcaccc2c2661" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=85571f6dae3522e2a7de01b7e0a3f7e3a9359641/" cluster_name = "${var.cluster_name}" api_servers = ["${format("%s.%s", var.cluster_name, var.dns_zone)}"] diff --git a/aws/container-linux/kubernetes/cl/controller.yaml.tmpl b/aws/container-linux/kubernetes/cl/controller.yaml.tmpl index dd9d98fdb..9b92d48d2 100644 --- a/aws/container-linux/kubernetes/cl/controller.yaml.tmpl +++ b/aws/container-linux/kubernetes/cl/controller.yaml.tmpl @@ -123,7 +123,7 @@ storage: contents: inline: | KUBELET_IMAGE_URL=docker://k8s.gcr.io/hyperkube - KUBELET_IMAGE_TAG=v1.14.1 + KUBELET_IMAGE_TAG=v1.14.2 - path: /etc/sysctl.d/max-user-watches.conf filesystem: root contents: diff --git a/aws/container-linux/kubernetes/workers/cl/worker.yaml.tmpl b/aws/container-linux/kubernetes/workers/cl/worker.yaml.tmpl index 84b816d9c..230533b6e 100644 --- a/aws/container-linux/kubernetes/workers/cl/worker.yaml.tmpl +++ b/aws/container-linux/kubernetes/workers/cl/worker.yaml.tmpl @@ -93,7 +93,7 @@ storage: contents: inline: | KUBELET_IMAGE_URL=docker://k8s.gcr.io/hyperkube - KUBELET_IMAGE_TAG=v1.14.1 + KUBELET_IMAGE_TAG=v1.14.2 - path: /etc/sysctl.d/max-user-watches.conf filesystem: root contents: @@ -111,7 +111,7 @@ storage: --volume config,kind=host,source=/etc/kubernetes \ --mount volume=config,target=/etc/kubernetes \ --insecure-options=image \ - docker://k8s.gcr.io/hyperkube:v1.14.1 \ + docker://k8s.gcr.io/hyperkube:v1.14.2 \ --net=host \ --dns=host \ --exec=/kubectl -- --kubeconfig=/etc/kubernetes/kubeconfig delete node $(hostname) diff --git a/aws/fedora-atomic/kubernetes/README.md b/aws/fedora-atomic/kubernetes/README.md index cdb64baa2..94df4efa5 100644 --- a/aws/fedora-atomic/kubernetes/README.md +++ b/aws/fedora-atomic/kubernetes/README.md @@ -11,7 +11,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features -* Kubernetes v1.14.1 (upstream, via [kubernetes-incubator/bootkube](https://github.com/kubernetes-incubator/bootkube)) +* Kubernetes v1.14.2 (upstream, via [kubernetes-incubator/bootkube](https://github.com/kubernetes-incubator/bootkube)) * Single or multi-master, [Calico](https://www.projectcalico.org/) or [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/) * Advanced features like [worker pools](https://typhoon.psdn.io/advanced/worker-pools/) and [spot](https://typhoon.psdn.io/cl/aws/#spot) workers diff --git a/aws/fedora-atomic/kubernetes/bootkube.tf b/aws/fedora-atomic/kubernetes/bootkube.tf index ece0c213f..5f448be36 100644 --- a/aws/fedora-atomic/kubernetes/bootkube.tf +++ b/aws/fedora-atomic/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=eca7c49fe17652e9f288c7cc4461dcaccc2c2661" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=85571f6dae3522e2a7de01b7e0a3f7e3a9359641/" cluster_name = "${var.cluster_name}" api_servers = ["${format("%s.%s", var.cluster_name, var.dns_zone)}"] diff --git a/azure/container-linux/kubernetes/README.md b/azure/container-linux/kubernetes/README.md index 819a70ae9..91b61894f 100644 --- a/azure/container-linux/kubernetes/README.md +++ b/azure/container-linux/kubernetes/README.md @@ -11,7 +11,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features -* Kubernetes v1.14.1 (upstream, via [kubernetes-incubator/bootkube](https://github.com/kubernetes-incubator/bootkube)) +* Kubernetes v1.14.2 (upstream, via [kubernetes-incubator/bootkube](https://github.com/kubernetes-incubator/bootkube)) * Single or multi-master, [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled * Advanced features like [worker pools](https://typhoon.psdn.io/advanced/worker-pools/), [low-priority](https://typhoon.psdn.io/cl/azure/#low-priority) workers, and [snippets](https://typhoon.psdn.io/advanced/customization/#container-linux) customization diff --git a/azure/container-linux/kubernetes/bootkube.tf b/azure/container-linux/kubernetes/bootkube.tf index d90ad5c77..8912a0b7a 100644 --- a/azure/container-linux/kubernetes/bootkube.tf +++ b/azure/container-linux/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=eca7c49fe17652e9f288c7cc4461dcaccc2c2661" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=85571f6dae3522e2a7de01b7e0a3f7e3a9359641/" cluster_name = "${var.cluster_name}" api_servers = ["${format("%s.%s", var.cluster_name, var.dns_zone)}"] diff --git a/azure/container-linux/kubernetes/cl/controller.yaml.tmpl b/azure/container-linux/kubernetes/cl/controller.yaml.tmpl index dd9d98fdb..9b92d48d2 100644 --- a/azure/container-linux/kubernetes/cl/controller.yaml.tmpl +++ b/azure/container-linux/kubernetes/cl/controller.yaml.tmpl @@ -123,7 +123,7 @@ storage: contents: inline: | KUBELET_IMAGE_URL=docker://k8s.gcr.io/hyperkube - KUBELET_IMAGE_TAG=v1.14.1 + KUBELET_IMAGE_TAG=v1.14.2 - path: /etc/sysctl.d/max-user-watches.conf filesystem: root contents: diff --git a/azure/container-linux/kubernetes/workers/cl/worker.yaml.tmpl b/azure/container-linux/kubernetes/workers/cl/worker.yaml.tmpl index aade6ef8b..c0e0155e6 100644 --- a/azure/container-linux/kubernetes/workers/cl/worker.yaml.tmpl +++ b/azure/container-linux/kubernetes/workers/cl/worker.yaml.tmpl @@ -93,7 +93,7 @@ storage: contents: inline: | KUBELET_IMAGE_URL=docker://k8s.gcr.io/hyperkube - KUBELET_IMAGE_TAG=v1.14.1 + KUBELET_IMAGE_TAG=v1.14.2 - path: /etc/sysctl.d/max-user-watches.conf filesystem: root contents: @@ -111,7 +111,7 @@ storage: --volume config,kind=host,source=/etc/kubernetes \ --mount volume=config,target=/etc/kubernetes \ --insecure-options=image \ - docker://k8s.gcr.io/hyperkube:v1.14.1 \ + docker://k8s.gcr.io/hyperkube:v1.14.2 \ --net=host \ --dns=host \ --exec=/kubectl -- --kubeconfig=/etc/kubernetes/kubeconfig delete node $(hostname | tr '[:upper:]' '[:lower:]') diff --git a/bare-metal/container-linux/kubernetes/README.md b/bare-metal/container-linux/kubernetes/README.md index 835ae8bb9..92205dbb0 100644 --- a/bare-metal/container-linux/kubernetes/README.md +++ b/bare-metal/container-linux/kubernetes/README.md @@ -11,7 +11,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features -* Kubernetes v1.14.1 (upstream, via [kubernetes-incubator/bootkube](https://github.com/kubernetes-incubator/bootkube)) +* Kubernetes v1.14.2 (upstream, via [kubernetes-incubator/bootkube](https://github.com/kubernetes-incubator/bootkube)) * Single or multi-master, [Calico](https://www.projectcalico.org/) or [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/) * Advanced features like [snippets](https://typhoon.psdn.io/advanced/customization/#container-linux) customization diff --git a/bare-metal/container-linux/kubernetes/bootkube.tf b/bare-metal/container-linux/kubernetes/bootkube.tf index 7f2306315..8ea8d5229 100644 --- a/bare-metal/container-linux/kubernetes/bootkube.tf +++ b/bare-metal/container-linux/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=eca7c49fe17652e9f288c7cc4461dcaccc2c2661" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=85571f6dae3522e2a7de01b7e0a3f7e3a9359641/" cluster_name = "${var.cluster_name}" api_servers = ["${var.k8s_domain_name}"] diff --git a/bare-metal/container-linux/kubernetes/cl/controller.yaml.tmpl b/bare-metal/container-linux/kubernetes/cl/controller.yaml.tmpl index 95b746a86..42cad1e9f 100644 --- a/bare-metal/container-linux/kubernetes/cl/controller.yaml.tmpl +++ b/bare-metal/container-linux/kubernetes/cl/controller.yaml.tmpl @@ -128,7 +128,7 @@ storage: contents: inline: | KUBELET_IMAGE_URL=docker://k8s.gcr.io/hyperkube - KUBELET_IMAGE_TAG=v1.14.1 + KUBELET_IMAGE_TAG=v1.14.2 - path: /etc/hostname filesystem: root mode: 0644 diff --git a/bare-metal/container-linux/kubernetes/cl/worker.yaml.tmpl b/bare-metal/container-linux/kubernetes/cl/worker.yaml.tmpl index f7a1b5fb0..6f13ff834 100644 --- a/bare-metal/container-linux/kubernetes/cl/worker.yaml.tmpl +++ b/bare-metal/container-linux/kubernetes/cl/worker.yaml.tmpl @@ -89,7 +89,7 @@ storage: contents: inline: | KUBELET_IMAGE_URL=docker://k8s.gcr.io/hyperkube - KUBELET_IMAGE_TAG=v1.14.1 + KUBELET_IMAGE_TAG=v1.14.2 - path: /etc/hostname filesystem: root mode: 0644 diff --git a/bare-metal/fedora-atomic/kubernetes/README.md b/bare-metal/fedora-atomic/kubernetes/README.md index 6ddd86098..69919066b 100644 --- a/bare-metal/fedora-atomic/kubernetes/README.md +++ b/bare-metal/fedora-atomic/kubernetes/README.md @@ -11,7 +11,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features -* Kubernetes v1.14.1 (upstream, via [kubernetes-incubator/bootkube](https://github.com/kubernetes-incubator/bootkube)) +* Kubernetes v1.14.2 (upstream, via [kubernetes-incubator/bootkube](https://github.com/kubernetes-incubator/bootkube)) * Single or multi-master, [Calico](https://www.projectcalico.org/) or [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/) * Ready for Ingress, Prometheus, Grafana, and other optional [addons](https://typhoon.psdn.io/addons/overview/) diff --git a/bare-metal/fedora-atomic/kubernetes/bootkube.tf b/bare-metal/fedora-atomic/kubernetes/bootkube.tf index 8ae797846..f99ec555d 100644 --- a/bare-metal/fedora-atomic/kubernetes/bootkube.tf +++ b/bare-metal/fedora-atomic/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=eca7c49fe17652e9f288c7cc4461dcaccc2c2661" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=85571f6dae3522e2a7de01b7e0a3f7e3a9359641/" cluster_name = "${var.cluster_name}" api_servers = ["${var.k8s_domain_name}"] diff --git a/digital-ocean/container-linux/kubernetes/README.md b/digital-ocean/container-linux/kubernetes/README.md index 13ec7d334..e682d5ad5 100644 --- a/digital-ocean/container-linux/kubernetes/README.md +++ b/digital-ocean/container-linux/kubernetes/README.md @@ -11,7 +11,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features -* Kubernetes v1.14.1 (upstream, via [kubernetes-incubator/bootkube](https://github.com/kubernetes-incubator/bootkube)) +* Kubernetes v1.14.2 (upstream, via [kubernetes-incubator/bootkube](https://github.com/kubernetes-incubator/bootkube)) * Single or multi-master, [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled * Advanced features like [snippets](https://typhoon.psdn.io/advanced/customization/#container-linux) customization diff --git a/digital-ocean/container-linux/kubernetes/bootkube.tf b/digital-ocean/container-linux/kubernetes/bootkube.tf index cb991001b..32f084247 100644 --- a/digital-ocean/container-linux/kubernetes/bootkube.tf +++ b/digital-ocean/container-linux/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=eca7c49fe17652e9f288c7cc4461dcaccc2c2661" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=85571f6dae3522e2a7de01b7e0a3f7e3a9359641/" cluster_name = "${var.cluster_name}" api_servers = ["${format("%s.%s", var.cluster_name, var.dns_zone)}"] diff --git a/digital-ocean/container-linux/kubernetes/cl/controller.yaml.tmpl b/digital-ocean/container-linux/kubernetes/cl/controller.yaml.tmpl index 5aa77d1d7..33ba69219 100644 --- a/digital-ocean/container-linux/kubernetes/cl/controller.yaml.tmpl +++ b/digital-ocean/container-linux/kubernetes/cl/controller.yaml.tmpl @@ -129,7 +129,7 @@ storage: contents: inline: | KUBELET_IMAGE_URL=docker://k8s.gcr.io/hyperkube - KUBELET_IMAGE_TAG=v1.14.1 + KUBELET_IMAGE_TAG=v1.14.2 - path: /etc/sysctl.d/max-user-watches.conf filesystem: root contents: diff --git a/digital-ocean/container-linux/kubernetes/cl/worker.yaml.tmpl b/digital-ocean/container-linux/kubernetes/cl/worker.yaml.tmpl index c724a6e9e..11b4e553d 100644 --- a/digital-ocean/container-linux/kubernetes/cl/worker.yaml.tmpl +++ b/digital-ocean/container-linux/kubernetes/cl/worker.yaml.tmpl @@ -99,7 +99,7 @@ storage: contents: inline: | KUBELET_IMAGE_URL=docker://k8s.gcr.io/hyperkube - KUBELET_IMAGE_TAG=v1.14.1 + KUBELET_IMAGE_TAG=v1.14.2 - path: /etc/sysctl.d/max-user-watches.conf filesystem: root contents: @@ -117,7 +117,7 @@ storage: --volume config,kind=host,source=/etc/kubernetes \ --mount volume=config,target=/etc/kubernetes \ --insecure-options=image \ - docker://k8s.gcr.io/hyperkube:v1.14.1 \ + docker://k8s.gcr.io/hyperkube:v1.14.2 \ --net=host \ --dns=host \ --exec=/kubectl -- --kubeconfig=/etc/kubernetes/kubeconfig delete node $(hostname) diff --git a/digital-ocean/fedora-atomic/kubernetes/README.md b/digital-ocean/fedora-atomic/kubernetes/README.md index 2ccbfa8e7..6203d59f9 100644 --- a/digital-ocean/fedora-atomic/kubernetes/README.md +++ b/digital-ocean/fedora-atomic/kubernetes/README.md @@ -11,7 +11,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features -* Kubernetes v1.14.1 (upstream, via [kubernetes-incubator/bootkube](https://github.com/kubernetes-incubator/bootkube)) +* Kubernetes v1.14.2 (upstream, via [kubernetes-incubator/bootkube](https://github.com/kubernetes-incubator/bootkube)) * Single or multi-master, [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled * Ready for Ingress, Prometheus, Grafana, and other optional [addons](https://typhoon.psdn.io/addons/overview/) diff --git a/digital-ocean/fedora-atomic/kubernetes/bootkube.tf b/digital-ocean/fedora-atomic/kubernetes/bootkube.tf index 02df3cbcb..55d5cef66 100644 --- a/digital-ocean/fedora-atomic/kubernetes/bootkube.tf +++ b/digital-ocean/fedora-atomic/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=eca7c49fe17652e9f288c7cc4461dcaccc2c2661" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=85571f6dae3522e2a7de01b7e0a3f7e3a9359641/" cluster_name = "${var.cluster_name}" api_servers = ["${format("%s.%s", var.cluster_name, var.dns_zone)}"] diff --git a/docs/advanced/worker-pools.md b/docs/advanced/worker-pools.md index 26b58f0a9..848819773 100644 --- a/docs/advanced/worker-pools.md +++ b/docs/advanced/worker-pools.md @@ -16,7 +16,7 @@ Create a cluster following the AWS [tutorial](../cl/aws.md#cluster). Define a wo ```tf module "tempest-worker-pool" { - source = "git::https://github.com/poseidon/typhoon//aws/container-linux/kubernetes/workers?ref=v1.14.1" + source = "git::https://github.com/poseidon/typhoon//aws/container-linux/kubernetes/workers?ref=v1.14.2" providers = { aws = "aws.default" @@ -82,7 +82,7 @@ Create a cluster following the Azure [tutorial](../cl/azure.md#cluster). Define ```tf module "ramius-worker-pool" { - source = "git::https://github.com/poseidon/typhoon//azure/container-linux/kubernetes/workers?ref=v1.14.1" + source = "git::https://github.com/poseidon/typhoon//azure/container-linux/kubernetes/workers?ref=v1.14.2" providers = { azurerm = "azurerm.default" @@ -152,7 +152,7 @@ Create a cluster following the Google Cloud [tutorial](../cl/google-cloud.md#clu ```tf module "yavin-worker-pool" { - source = "git::https://github.com/poseidon/typhoon//google-cloud/container-linux/kubernetes/workers?ref=v1.14.1" + source = "git::https://github.com/poseidon/typhoon//google-cloud/container-linux/kubernetes/workers?ref=v1.14.2" providers = { google = "google.default" @@ -187,11 +187,11 @@ Verify a managed instance group of workers joins the cluster within a few minute ``` $ kubectl get nodes NAME STATUS AGE VERSION -yavin-controller-0.c.example-com.internal Ready 6m v1.14.1 -yavin-worker-jrbf.c.example-com.internal Ready 5m v1.14.1 -yavin-worker-mzdm.c.example-com.internal Ready 5m v1.14.1 -yavin-16x-worker-jrbf.c.example-com.internal Ready 3m v1.14.1 -yavin-16x-worker-mzdm.c.example-com.internal Ready 3m v1.14.1 +yavin-controller-0.c.example-com.internal Ready 6m v1.14.2 +yavin-worker-jrbf.c.example-com.internal Ready 5m v1.14.2 +yavin-worker-mzdm.c.example-com.internal Ready 5m v1.14.2 +yavin-16x-worker-jrbf.c.example-com.internal Ready 3m v1.14.2 +yavin-16x-worker-mzdm.c.example-com.internal Ready 3m v1.14.2 ``` ### Variables diff --git a/docs/atomic/aws.md b/docs/atomic/aws.md index 7e1668bba..77ed0824b 100644 --- a/docs/atomic/aws.md +++ b/docs/atomic/aws.md @@ -3,7 +3,7 @@ !!! danger Typhoon for Fedora Atomic will not be updated much beyond Kubernetes v1.13. -In this tutorial, we'll create a Kubernetes v1.14.1 cluster on AWS with Fedora Atomic. +In this tutorial, we'll create a Kubernetes v1.14.2 cluster on AWS with Fedora Atomic. We'll declare a Kubernetes cluster using the Typhoon Terraform module. Then apply the changes to create a VPC, gateway, subnets, security groups, controller instances, worker auto-scaling group, network load balancer, and TLS assets. Instances are provisioned on first boot with cloud-init. @@ -83,7 +83,7 @@ Define a Kubernetes cluster using the module `aws/fedora-atomic/kubernetes`. ```tf module "aws-tempest" { - source = "git::https://github.com/poseidon/typhoon//aws/fedora-atomic/kubernetes?ref=v1.14.1" + source = "git::https://github.com/poseidon/typhoon//aws/fedora-atomic/kubernetes?ref=v1.14.2" providers = { aws = "aws.default" @@ -156,9 +156,9 @@ In 5-10 minutes, the Kubernetes cluster will be ready. $ export KUBECONFIG=/home/user/.secrets/clusters/tempest/auth/kubeconfig $ kubectl get nodes NAME STATUS ROLES AGE VERSION -ip-10-0-3-155 Ready controller,master 10m v1.14.1 -ip-10-0-26-65 Ready node 10m v1.14.1 -ip-10-0-41-21 Ready node 10m v1.14.1 +ip-10-0-3-155 Ready controller,master 10m v1.14.2 +ip-10-0-26-65 Ready node 10m v1.14.2 +ip-10-0-41-21 Ready node 10m v1.14.2 ``` List the pods. diff --git a/docs/atomic/bare-metal.md b/docs/atomic/bare-metal.md index 7c6be2bfe..e258ad079 100644 --- a/docs/atomic/bare-metal.md +++ b/docs/atomic/bare-metal.md @@ -3,7 +3,7 @@ !!! danger Typhoon for Fedora Atomic will not be updated much beyond Kubernetes v1.13. -In this tutorial, we'll network boot and provision a Kubernetes v1.14.1 cluster on bare-metal with Fedora Atomic. +In this tutorial, we'll network boot and provision a Kubernetes v1.14.2 cluster on bare-metal with Fedora Atomic. First, we'll deploy a [Matchbox](https://github.com/poseidon/matchbox) service and setup a network boot environment. Then, we'll declare a Kubernetes cluster using the Typhoon Terraform module and power on machines. On PXE boot, machines will install Fedora Atomic via kickstart, reboot into the disk install, and provision themselves as Kubernetes controllers or workers via cloud-init. @@ -228,7 +228,7 @@ Define a Kubernetes cluster using the module `bare-metal/fedora-atomic/kubernete ```tf module "bare-metal-mercury" { - source = "git::https://github.com/poseidon/typhoon//bare-metal/fedora-atomic/kubernetes?ref=v1.14.1" + source = "git::https://github.com/poseidon/typhoon//bare-metal/fedora-atomic/kubernetes?ref=v1.14.2" providers = { local = "local.default" @@ -354,9 +354,9 @@ bootkube[5]: Tearing down temporary bootstrap control plane... $ export KUBECONFIG=/home/user/.secrets/clusters/mercury/auth/kubeconfig $ kubectl get nodes NAME STATUS ROLES AGE VERSION -node1.example.com Ready controller,master 10m v1.14.1 -node2.example.com Ready node 10m v1.14.1 -node3.example.com Ready node 10m v1.14.1 +node1.example.com Ready controller,master 10m v1.14.2 +node2.example.com Ready node 10m v1.14.2 +node3.example.com Ready node 10m v1.14.2 ``` List the pods. diff --git a/docs/atomic/digital-ocean.md b/docs/atomic/digital-ocean.md index 30fc8723d..370bbd98b 100644 --- a/docs/atomic/digital-ocean.md +++ b/docs/atomic/digital-ocean.md @@ -3,7 +3,7 @@ !!! danger Typhoon for Fedora Atomic will not be updated much beyond Kubernetes v1.13. -In this tutorial, we'll create a Kubernetes v1.14.1 cluster on DigitalOcean with Fedora Atomic. +In this tutorial, we'll create a Kubernetes v1.14.2 cluster on DigitalOcean with Fedora Atomic. We'll declare a Kubernetes cluster using the Typhoon Terraform module. Then apply the changes to create controller droplets, worker droplets, DNS records, tags, and TLS assets. Instances are provisioned on first boot with cloud-init. @@ -77,7 +77,7 @@ Define a Kubernetes cluster using the module `digital-ocean/fedora-atomic/kubern ```tf module "digital-ocean-nemo" { - source = "git::https://github.com/poseidon/typhoon//digital-ocean/fedora-atomic/kubernetes?ref=v1.14.1" + source = "git::https://github.com/poseidon/typhoon//digital-ocean/fedora-atomic/kubernetes?ref=v1.14.2" providers = { digitalocean = "digitalocean.default" @@ -152,9 +152,9 @@ In 3-6 minutes, the Kubernetes cluster will be ready. $ export KUBECONFIG=/home/user/.secrets/clusters/nemo/auth/kubeconfig $ kubectl get nodes NAME STATUS ROLES AGE VERSION -10.132.110.130 Ready controller,master 10m v1.14.1 -10.132.115.81 Ready node 10m v1.14.1 -10.132.124.107 Ready node 10m v1.14.1 +10.132.110.130 Ready controller,master 10m v1.14.2 +10.132.115.81 Ready node 10m v1.14.2 +10.132.124.107 Ready node 10m v1.14.2 ``` List the pods. diff --git a/docs/atomic/google-cloud.md b/docs/atomic/google-cloud.md index 13a6eb0d0..891ef102d 100644 --- a/docs/atomic/google-cloud.md +++ b/docs/atomic/google-cloud.md @@ -3,7 +3,7 @@ !!! danger Typhoon for Fedora Atomic will not be updated much beyond Kubernetes v1.13. Fedora does not publish official images for Google Cloud so you must prepare them yourself. Expect rough edges and changes. -In this tutorial, we'll create a Kubernetes v1.14.1 cluster on Google Compute Engine with Fedora Atomic. +In this tutorial, we'll create a Kubernetes v1.14.2 cluster on Google Compute Engine with Fedora Atomic. We'll declare a Kubernetes cluster using the Typhoon Terraform module. Then apply the changes to create a network, firewall rules, health checks, controller instances, worker managed instance group, load balancers, and TLS assets. Instances are provisioned on first boot with cloud-init. @@ -121,7 +121,7 @@ Define a Kubernetes cluster using the module `google-cloud/fedora-atomic/kuberne ```tf module "google-cloud-yavin" { - source = "git::https://github.com/poseidon/typhoon//google-cloud/fedora-atomic/kubernetes?ref=v1.14.1" + source = "git::https://github.com/poseidon/typhoon//google-cloud/fedora-atomic/kubernetes?ref=v1.14.2" providers = { google = "google.default" @@ -197,9 +197,9 @@ In 5-10 minutes, the Kubernetes cluster will be ready. $ export KUBECONFIG=/home/user/.secrets/clusters/yavin/auth/kubeconfig $ kubectl get nodes NAME ROLES STATUS AGE VERSION -yavin-controller-0.c.example-com.internal controller,master Ready 6m v1.14.1 -yavin-worker-jrbf.c.example-com.internal node Ready 5m v1.14.1 -yavin-worker-mzdm.c.example-com.internal node Ready 5m v1.14.1 +yavin-controller-0.c.example-com.internal controller,master Ready 6m v1.14.2 +yavin-worker-jrbf.c.example-com.internal node Ready 5m v1.14.2 +yavin-worker-mzdm.c.example-com.internal node Ready 5m v1.14.2 ``` List the pods. diff --git a/docs/cl/aws.md b/docs/cl/aws.md index 076eb6122..e08292d0d 100644 --- a/docs/cl/aws.md +++ b/docs/cl/aws.md @@ -1,6 +1,6 @@ # AWS -In this tutorial, we'll create a Kubernetes v1.14.1 cluster on AWS with Container Linux. +In this tutorial, we'll create a Kubernetes v1.14.2 cluster on AWS with Container Linux. We'll declare a Kubernetes cluster using the Typhoon Terraform module. Then apply the changes to create a VPC, gateway, subnets, security groups, controller instances, worker auto-scaling group, network load balancer, and TLS assets. @@ -92,7 +92,7 @@ Define a Kubernetes cluster using the module `aws/container-linux/kubernetes`. ```tf module "aws-tempest" { - source = "git::https://github.com/poseidon/typhoon//aws/container-linux/kubernetes?ref=v1.14.1" + source = "git::https://github.com/poseidon/typhoon//aws/container-linux/kubernetes?ref=v1.14.2" providers = { aws = "aws.default" @@ -165,9 +165,9 @@ In 4-8 minutes, the Kubernetes cluster will be ready. $ export KUBECONFIG=/home/user/.secrets/clusters/tempest/auth/kubeconfig $ kubectl get nodes NAME STATUS ROLES AGE VERSION -ip-10-0-3-155 Ready controller,master 10m v1.14.1 -ip-10-0-26-65 Ready node 10m v1.14.1 -ip-10-0-41-21 Ready node 10m v1.14.1 +ip-10-0-3-155 Ready controller,master 10m v1.14.2 +ip-10-0-26-65 Ready node 10m v1.14.2 +ip-10-0-41-21 Ready node 10m v1.14.2 ``` List the pods. diff --git a/docs/cl/azure.md b/docs/cl/azure.md index 0282a76dd..8c0826d75 100644 --- a/docs/cl/azure.md +++ b/docs/cl/azure.md @@ -3,7 +3,7 @@ !!! danger Typhoon for Azure is alpha. For production, use AWS, Google Cloud, or bare-metal. As Azure matures, check [errata](https://github.com/poseidon/typhoon/wiki/Errata) for known shortcomings. -In this tutorial, we'll create a Kubernetes v1.14.1 cluster on Azure with Container Linux. +In this tutorial, we'll create a Kubernetes v1.14.2 cluster on Azure with Container Linux. We'll declare a Kubernetes cluster using the Typhoon Terraform module. Then apply the changes to create a resource group, virtual network, subnets, security groups, controller availability set, worker scale set, load balancer, and TLS assets. @@ -87,7 +87,7 @@ Define a Kubernetes cluster using the module `azure/container-linux/kubernetes`. ```tf module "azure-ramius" { - source = "git::https://github.com/poseidon/typhoon//azure/container-linux/kubernetes?ref=v1.14.1" + source = "git::https://github.com/poseidon/typhoon//azure/container-linux/kubernetes?ref=v1.14.2" providers = { azurerm = "azurerm.default" @@ -161,9 +161,9 @@ In 4-8 minutes, the Kubernetes cluster will be ready. $ export KUBECONFIG=/home/user/.secrets/clusters/ramius/auth/kubeconfig $ kubectl get nodes NAME STATUS ROLES AGE VERSION -ramius-controller-0 Ready controller,master 24m v1.14.1 -ramius-worker-000001 Ready node 25m v1.14.1 -ramius-worker-000002 Ready node 24m v1.14.1 +ramius-controller-0 Ready controller,master 24m v1.14.2 +ramius-worker-000001 Ready node 25m v1.14.2 +ramius-worker-000002 Ready node 24m v1.14.2 ``` List the pods. diff --git a/docs/cl/bare-metal.md b/docs/cl/bare-metal.md index 7ba447961..11d37eb89 100644 --- a/docs/cl/bare-metal.md +++ b/docs/cl/bare-metal.md @@ -1,6 +1,6 @@ # Bare-Metal -In this tutorial, we'll network boot and provision a Kubernetes v1.14.1 cluster on bare-metal with Container Linux. +In this tutorial, we'll network boot and provision a Kubernetes v1.14.2 cluster on bare-metal with Container Linux. First, we'll deploy a [Matchbox](https://github.com/poseidon/matchbox) service and setup a network boot environment. Then, we'll declare a Kubernetes cluster using the Typhoon Terraform module and power on machines. On PXE boot, machines will install Container Linux to disk, reboot into the disk install, and provision themselves as Kubernetes controllers or workers via Ignition. @@ -180,7 +180,7 @@ Define a Kubernetes cluster using the module `bare-metal/container-linux/kuberne ```tf module "bare-metal-mercury" { - source = "git::https://github.com/poseidon/typhoon//bare-metal/container-linux/kubernetes?ref=v1.14.1" + source = "git::https://github.com/poseidon/typhoon//bare-metal/container-linux/kubernetes?ref=v1.14.2" providers = { local = "local.default" @@ -292,9 +292,9 @@ Apply complete! Resources: 55 added, 0 changed, 0 destroyed. To watch the install to disk (until machines reboot from disk), SSH to port 2222. ``` -# before v1.14.1 +# before v1.14.2 $ ssh debug@node1.example.com -# after v1.14.1 +# after v1.14.2 $ ssh -p 2222 core@node1.example.com ``` @@ -319,9 +319,9 @@ bootkube[5]: Tearing down temporary bootstrap control plane... $ export KUBECONFIG=/home/user/.secrets/clusters/mercury/auth/kubeconfig $ kubectl get nodes NAME STATUS ROLES AGE VERSION -node1.example.com Ready controller,master 10m v1.14.1 -node2.example.com Ready node 10m v1.14.1 -node3.example.com Ready node 10m v1.14.1 +node1.example.com Ready controller,master 10m v1.14.2 +node2.example.com Ready node 10m v1.14.2 +node3.example.com Ready node 10m v1.14.2 ``` List the pods. diff --git a/docs/cl/digital-ocean.md b/docs/cl/digital-ocean.md index 37d46350a..3aab64ef5 100644 --- a/docs/cl/digital-ocean.md +++ b/docs/cl/digital-ocean.md @@ -1,6 +1,6 @@ # Digital Ocean -In this tutorial, we'll create a Kubernetes v1.14.1 cluster on DigitalOcean with Container Linux. +In this tutorial, we'll create a Kubernetes v1.14.2 cluster on DigitalOcean with Container Linux. We'll declare a Kubernetes cluster using the Typhoon Terraform module. Then apply the changes to create controller droplets, worker droplets, DNS records, tags, and TLS assets. @@ -86,7 +86,7 @@ Define a Kubernetes cluster using the module `digital-ocean/container-linux/kube ```tf module "digital-ocean-nemo" { - source = "git::https://github.com/poseidon/typhoon//digital-ocean/container-linux/kubernetes?ref=v1.14.1" + source = "git::https://github.com/poseidon/typhoon//digital-ocean/container-linux/kubernetes?ref=v1.14.2" providers = { digitalocean = "digitalocean.default" @@ -160,9 +160,9 @@ In 3-6 minutes, the Kubernetes cluster will be ready. $ export KUBECONFIG=/home/user/.secrets/clusters/nemo/auth/kubeconfig $ kubectl get nodes NAME STATUS ROLES AGE VERSION -10.132.110.130 Ready controller,master 10m v1.14.1 -10.132.115.81 Ready node 10m v1.14.1 -10.132.124.107 Ready node 10m v1.14.1 +10.132.110.130 Ready controller,master 10m v1.14.2 +10.132.115.81 Ready node 10m v1.14.2 +10.132.124.107 Ready node 10m v1.14.2 ``` List the pods. diff --git a/docs/cl/google-cloud.md b/docs/cl/google-cloud.md index 34bb5807e..b8959464b 100644 --- a/docs/cl/google-cloud.md +++ b/docs/cl/google-cloud.md @@ -1,6 +1,6 @@ # Google Cloud -In this tutorial, we'll create a Kubernetes v1.14.1 cluster on Google Compute Engine with Container Linux. +In this tutorial, we'll create a Kubernetes v1.14.2 cluster on Google Compute Engine with Container Linux. We'll declare a Kubernetes cluster using the Typhoon Terraform module. Then apply the changes to create a network, firewall rules, health checks, controller instances, worker managed instance group, load balancers, and TLS assets. @@ -93,7 +93,7 @@ Define a Kubernetes cluster using the module `google-cloud/container-linux/kuber ```tf module "google-cloud-yavin" { - source = "git::https://github.com/poseidon/typhoon//google-cloud/container-linux/kubernetes?ref=v1.14.1" + source = "git::https://github.com/poseidon/typhoon//google-cloud/container-linux/kubernetes?ref=v1.14.2" providers = { google = "google.default" @@ -168,9 +168,9 @@ In 4-8 minutes, the Kubernetes cluster will be ready. $ export KUBECONFIG=/home/user/.secrets/clusters/yavin/auth/kubeconfig $ kubectl get nodes NAME ROLES STATUS AGE VERSION -yavin-controller-0.c.example-com.internal controller,master Ready 6m v1.14.1 -yavin-worker-jrbf.c.example-com.internal node Ready 5m v1.14.1 -yavin-worker-mzdm.c.example-com.internal node Ready 5m v1.14.1 +yavin-controller-0.c.example-com.internal controller,master Ready 6m v1.14.2 +yavin-worker-jrbf.c.example-com.internal node Ready 5m v1.14.2 +yavin-worker-mzdm.c.example-com.internal node Ready 5m v1.14.2 ``` List the pods. diff --git a/docs/index.md b/docs/index.md index 1408af720..fa227594c 100644 --- a/docs/index.md +++ b/docs/index.md @@ -11,7 +11,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features -* Kubernetes v1.14.1 (upstream, via [kubernetes-incubator/bootkube](https://github.com/kubernetes-incubator/bootkube)) +* Kubernetes v1.14.2 (upstream, via [kubernetes-incubator/bootkube](https://github.com/kubernetes-incubator/bootkube)) * Single or multi-master, [Calico](https://www.projectcalico.org/) or [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/) * Advanced features like [worker pools](advanced/worker-pools/), [preemptible](cl/google-cloud/#preemption) workers, and [snippets](advanced/customization/#container-linux) customization @@ -49,7 +49,7 @@ Define a Kubernetes cluster by using the Terraform module for your chosen platfo ```tf module "google-cloud-yavin" { - source = "git::https://github.com/poseidon/typhoon//google-cloud/container-linux/kubernetes?ref=v1.14.1" + source = "git::https://github.com/poseidon/typhoon//google-cloud/container-linux/kubernetes?ref=v1.14.2" providers = { google = "google.default" @@ -90,9 +90,9 @@ In 4-8 minutes (varies by platform), the cluster will be ready. This Google Clou $ export KUBECONFIG=/home/user/.secrets/clusters/yavin/auth/kubeconfig $ kubectl get nodes NAME ROLES STATUS AGE VERSION -yavin-controller-0.c.example-com.internal controller,master Ready 6m v1.14.1 -yavin-worker-jrbf.c.example-com.internal node Ready 5m v1.14.1 -yavin-worker-mzdm.c.example-com.internal node Ready 5m v1.14.1 +yavin-controller-0.c.example-com.internal controller,master Ready 6m v1.14.2 +yavin-worker-jrbf.c.example-com.internal node Ready 5m v1.14.2 +yavin-worker-mzdm.c.example-com.internal node Ready 5m v1.14.2 ``` List the pods. diff --git a/docs/topics/maintenance.md b/docs/topics/maintenance.md index 8a0378d10..c3f794495 100644 --- a/docs/topics/maintenance.md +++ b/docs/topics/maintenance.md @@ -18,7 +18,7 @@ module "google-cloud-yavin" { } module "bare-metal-mercury" { - source = "git::https://github.com/poseidon/typhoon//bare-metal/container-linux/kubernetes?ref=v1.14.1" + source = "git::https://github.com/poseidon/typhoon//bare-metal/container-linux/kubernetes?ref=v1.14.2" ... } ``` diff --git a/google-cloud/container-linux/kubernetes/README.md b/google-cloud/container-linux/kubernetes/README.md index 40787c28c..5f84486d5 100644 --- a/google-cloud/container-linux/kubernetes/README.md +++ b/google-cloud/container-linux/kubernetes/README.md @@ -11,7 +11,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features -* Kubernetes v1.14.1 (upstream, via [kubernetes-incubator/bootkube](https://github.com/kubernetes-incubator/bootkube)) +* Kubernetes v1.14.2 (upstream, via [kubernetes-incubator/bootkube](https://github.com/kubernetes-incubator/bootkube)) * Single or multi-master, [Calico](https://www.projectcalico.org/) or [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/) * Advanced features like [worker pools](https://typhoon.psdn.io/advanced/worker-pools/), [preemptible](https://typhoon.psdn.io/cl/google-cloud/#preemption) workers, and [snippets](https://typhoon.psdn.io/advanced/customization/#container-linux) customization diff --git a/google-cloud/container-linux/kubernetes/bootkube.tf b/google-cloud/container-linux/kubernetes/bootkube.tf index 8fbf0e4e4..dd36b5126 100644 --- a/google-cloud/container-linux/kubernetes/bootkube.tf +++ b/google-cloud/container-linux/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=eca7c49fe17652e9f288c7cc4461dcaccc2c2661" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=85571f6dae3522e2a7de01b7e0a3f7e3a9359641/" cluster_name = "${var.cluster_name}" api_servers = ["${format("%s.%s", var.cluster_name, var.dns_zone)}"] diff --git a/google-cloud/container-linux/kubernetes/cl/controller.yaml.tmpl b/google-cloud/container-linux/kubernetes/cl/controller.yaml.tmpl index c7d8be081..53e3d229b 100644 --- a/google-cloud/container-linux/kubernetes/cl/controller.yaml.tmpl +++ b/google-cloud/container-linux/kubernetes/cl/controller.yaml.tmpl @@ -124,7 +124,7 @@ storage: contents: inline: | KUBELET_IMAGE_URL=docker://k8s.gcr.io/hyperkube - KUBELET_IMAGE_TAG=v1.14.1 + KUBELET_IMAGE_TAG=v1.14.2 - path: /etc/sysctl.d/max-user-watches.conf filesystem: root contents: diff --git a/google-cloud/container-linux/kubernetes/workers/cl/worker.yaml.tmpl b/google-cloud/container-linux/kubernetes/workers/cl/worker.yaml.tmpl index ced7def3e..38705ea8c 100644 --- a/google-cloud/container-linux/kubernetes/workers/cl/worker.yaml.tmpl +++ b/google-cloud/container-linux/kubernetes/workers/cl/worker.yaml.tmpl @@ -94,7 +94,7 @@ storage: contents: inline: | KUBELET_IMAGE_URL=docker://k8s.gcr.io/hyperkube - KUBELET_IMAGE_TAG=v1.14.1 + KUBELET_IMAGE_TAG=v1.14.2 - path: /etc/sysctl.d/max-user-watches.conf filesystem: root contents: @@ -112,7 +112,7 @@ storage: --volume config,kind=host,source=/etc/kubernetes \ --mount volume=config,target=/etc/kubernetes \ --insecure-options=image \ - docker://k8s.gcr.io/hyperkube:v1.14.1 \ + docker://k8s.gcr.io/hyperkube:v1.14.2 \ --net=host \ --dns=host \ --exec=/kubectl -- --kubeconfig=/etc/kubernetes/kubeconfig delete node $(hostname) diff --git a/google-cloud/fedora-atomic/kubernetes/README.md b/google-cloud/fedora-atomic/kubernetes/README.md index 25712fee1..a788d0803 100644 --- a/google-cloud/fedora-atomic/kubernetes/README.md +++ b/google-cloud/fedora-atomic/kubernetes/README.md @@ -11,7 +11,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features -* Kubernetes v1.14.1 (upstream, via [kubernetes-incubator/bootkube](https://github.com/kubernetes-incubator/bootkube)) +* Kubernetes v1.14.2 (upstream, via [kubernetes-incubator/bootkube](https://github.com/kubernetes-incubator/bootkube)) * Single or multi-master, [Calico](https://www.projectcalico.org/) or [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/) * Advanced features like [worker pools](https://typhoon.psdn.io/advanced/worker-pools/) and [preemptible](https://typhoon.psdn.io/cl/google-cloud/#preemption) workers diff --git a/google-cloud/fedora-atomic/kubernetes/bootkube.tf b/google-cloud/fedora-atomic/kubernetes/bootkube.tf index 871cfdbb0..c3a979f39 100644 --- a/google-cloud/fedora-atomic/kubernetes/bootkube.tf +++ b/google-cloud/fedora-atomic/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=eca7c49fe17652e9f288c7cc4461dcaccc2c2661" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=85571f6dae3522e2a7de01b7e0a3f7e3a9359641/" cluster_name = "${var.cluster_name}" api_servers = ["${format("%s.%s", var.cluster_name, var.dns_zone)}"] From 222a94247cfbcbd4704e828282888e2988385186 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Fri, 17 May 2019 20:01:30 +0200 Subject: [PATCH 104/523] Update node_exporter from v0.17.0 to v0.18.0 * https://github.com/prometheus/node_exporter/releases/tag/v0.18.0 --- CHANGES.md | 1 + addons/prometheus/exporters/node-exporter/daemonset.yaml | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index a7f5191a5..ef6121153 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -22,6 +22,7 @@ Notable changes between versions. * Update Prometheus from v2.8.1 to v2.9.2 * Update kube-state-metrics from v1.5.0 to v1.6.0 +* Update node-exporter from v0.17.0 to v0.18.0 * Update Grafana from v6.1.3 to v6.1.6 * Reduce nginx-ingress Role RBAC permissions ([#458](https://github.com/poseidon/typhoon/pull/458)) diff --git a/addons/prometheus/exporters/node-exporter/daemonset.yaml b/addons/prometheus/exporters/node-exporter/daemonset.yaml index 2b631c931..3e7d3c1e4 100644 --- a/addons/prometheus/exporters/node-exporter/daemonset.yaml +++ b/addons/prometheus/exporters/node-exporter/daemonset.yaml @@ -28,7 +28,7 @@ spec: hostPID: true containers: - name: node-exporter - image: quay.io/prometheus/node-exporter:v0.17.0 + image: quay.io/prometheus/node-exporter:v0.18.0 args: - --path.procfs=/host/proc - --path.sysfs=/host/sys From b9bab739ce69401285539fc24e7dc3d82c2b6b5e Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Sun, 19 May 2019 17:52:22 +0200 Subject: [PATCH 105/523] Update docs link for installing kubectl * Fix install kubectl link to refer to upstream docs. Link to coreos.com is now outdated and directed users to install kubectl v1.8.4 * https://github.com/poseidon/typhoon/issues/476 --- docs/atomic/aws.md | 2 +- docs/atomic/bare-metal.md | 2 +- docs/atomic/digital-ocean.md | 2 +- docs/atomic/google-cloud.md | 2 +- docs/cl/aws.md | 2 +- docs/cl/azure.md | 2 +- docs/cl/bare-metal.md | 2 +- docs/cl/digital-ocean.md | 2 +- docs/cl/google-cloud.md | 2 +- 9 files changed, 9 insertions(+), 9 deletions(-) diff --git a/docs/atomic/aws.md b/docs/atomic/aws.md index 77ed0824b..ffa57156d 100644 --- a/docs/atomic/aws.md +++ b/docs/atomic/aws.md @@ -150,7 +150,7 @@ In 5-10 minutes, the Kubernetes cluster will be ready. ## Verify -[Install kubectl](https://coreos.com/kubernetes/docs/latest/configure-kubectl.html) on your system. Use the generated `kubeconfig` credentials to access the Kubernetes cluster and list nodes. +[Install kubectl](https://kubernetes.io/docs/tasks/tools/install-kubectl/) on your system. Use the generated `kubeconfig` credentials to access the Kubernetes cluster and list nodes. ``` $ export KUBECONFIG=/home/user/.secrets/clusters/tempest/auth/kubeconfig diff --git a/docs/atomic/bare-metal.md b/docs/atomic/bare-metal.md index e258ad079..da2f6b597 100644 --- a/docs/atomic/bare-metal.md +++ b/docs/atomic/bare-metal.md @@ -348,7 +348,7 @@ bootkube[5]: Tearing down temporary bootstrap control plane... ## Verify -[Install kubectl](https://coreos.com/kubernetes/docs/latest/configure-kubectl.html) on your system. Use the generated `kubeconfig` credentials to access the Kubernetes cluster and list nodes. +[Install kubectl](https://kubernetes.io/docs/tasks/tools/install-kubectl/) on your system. Use the generated `kubeconfig` credentials to access the Kubernetes cluster and list nodes. ``` $ export KUBECONFIG=/home/user/.secrets/clusters/mercury/auth/kubeconfig diff --git a/docs/atomic/digital-ocean.md b/docs/atomic/digital-ocean.md index 370bbd98b..01a7911a5 100644 --- a/docs/atomic/digital-ocean.md +++ b/docs/atomic/digital-ocean.md @@ -146,7 +146,7 @@ In 3-6 minutes, the Kubernetes cluster will be ready. ## Verify -[Install kubectl](https://coreos.com/kubernetes/docs/latest/configure-kubectl.html) on your system. Use the generated `kubeconfig` credentials to access the Kubernetes cluster and list nodes. +[Install kubectl](https://kubernetes.io/docs/tasks/tools/install-kubectl/) on your system. Use the generated `kubeconfig` credentials to access the Kubernetes cluster and list nodes. ``` $ export KUBECONFIG=/home/user/.secrets/clusters/nemo/auth/kubeconfig diff --git a/docs/atomic/google-cloud.md b/docs/atomic/google-cloud.md index 891ef102d..7f62eb100 100644 --- a/docs/atomic/google-cloud.md +++ b/docs/atomic/google-cloud.md @@ -191,7 +191,7 @@ In 5-10 minutes, the Kubernetes cluster will be ready. ## Verify -[Install kubectl](https://coreos.com/kubernetes/docs/latest/configure-kubectl.html) on your system. Use the generated `kubeconfig` credentials to access the Kubernetes cluster and list nodes. +[Install kubectl](https://kubernetes.io/docs/tasks/tools/install-kubectl/) on your system. Use the generated `kubeconfig` credentials to access the Kubernetes cluster and list nodes. ``` $ export KUBECONFIG=/home/user/.secrets/clusters/yavin/auth/kubeconfig diff --git a/docs/cl/aws.md b/docs/cl/aws.md index e08292d0d..4dc947b0c 100644 --- a/docs/cl/aws.md +++ b/docs/cl/aws.md @@ -159,7 +159,7 @@ In 4-8 minutes, the Kubernetes cluster will be ready. ## Verify -[Install kubectl](https://coreos.com/kubernetes/docs/latest/configure-kubectl.html) on your system. Use the generated `kubeconfig` credentials to access the Kubernetes cluster and list nodes. +[Install kubectl](https://kubernetes.io/docs/tasks/tools/install-kubectl/) on your system. Use the generated `kubeconfig` credentials to access the Kubernetes cluster and list nodes. ``` $ export KUBECONFIG=/home/user/.secrets/clusters/tempest/auth/kubeconfig diff --git a/docs/cl/azure.md b/docs/cl/azure.md index 8c0826d75..2a12f80dc 100644 --- a/docs/cl/azure.md +++ b/docs/cl/azure.md @@ -155,7 +155,7 @@ In 4-8 minutes, the Kubernetes cluster will be ready. ## Verify -[Install kubectl](https://coreos.com/kubernetes/docs/latest/configure-kubectl.html) on your system. Use the generated `kubeconfig` credentials to access the Kubernetes cluster and list nodes. +[Install kubectl](https://kubernetes.io/docs/tasks/tools/install-kubectl/) on your system. Use the generated `kubeconfig` credentials to access the Kubernetes cluster and list nodes. ``` $ export KUBECONFIG=/home/user/.secrets/clusters/ramius/auth/kubeconfig diff --git a/docs/cl/bare-metal.md b/docs/cl/bare-metal.md index 11d37eb89..b60ecfc8d 100644 --- a/docs/cl/bare-metal.md +++ b/docs/cl/bare-metal.md @@ -313,7 +313,7 @@ bootkube[5]: Tearing down temporary bootstrap control plane... ## Verify -[Install kubectl](https://coreos.com/kubernetes/docs/latest/configure-kubectl.html) on your system. Use the generated `kubeconfig` credentials to access the Kubernetes cluster and list nodes. +[Install kubectl](https://kubernetes.io/docs/tasks/tools/install-kubectl/) on your system. Use the generated `kubeconfig` credentials to access the Kubernetes cluster and list nodes. ``` $ export KUBECONFIG=/home/user/.secrets/clusters/mercury/auth/kubeconfig diff --git a/docs/cl/digital-ocean.md b/docs/cl/digital-ocean.md index 3aab64ef5..90e26b4fa 100644 --- a/docs/cl/digital-ocean.md +++ b/docs/cl/digital-ocean.md @@ -154,7 +154,7 @@ In 3-6 minutes, the Kubernetes cluster will be ready. ## Verify -[Install kubectl](https://coreos.com/kubernetes/docs/latest/configure-kubectl.html) on your system. Use the generated `kubeconfig` credentials to access the Kubernetes cluster and list nodes. +[Install kubectl](https://kubernetes.io/docs/tasks/tools/install-kubectl/) on your system. Use the generated `kubeconfig` credentials to access the Kubernetes cluster and list nodes. ``` $ export KUBECONFIG=/home/user/.secrets/clusters/nemo/auth/kubeconfig diff --git a/docs/cl/google-cloud.md b/docs/cl/google-cloud.md index b8959464b..7c188dd44 100644 --- a/docs/cl/google-cloud.md +++ b/docs/cl/google-cloud.md @@ -162,7 +162,7 @@ In 4-8 minutes, the Kubernetes cluster will be ready. ## Verify -[Install kubectl](https://coreos.com/kubernetes/docs/latest/configure-kubectl.html) on your system. Use the generated `kubeconfig` credentials to access the Kubernetes cluster and list nodes. +[Install kubectl](https://kubernetes.io/docs/tasks/tools/install-kubectl/) on your system. Use the generated `kubeconfig` credentials to access the Kubernetes cluster and list nodes. ``` $ export KUBECONFIG=/home/user/.secrets/clusters/yavin/auth/kubeconfig From 147c21a4bd69fb92e763f99ea4bb5738bdd1af65 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Mon, 6 May 2019 00:38:23 -0700 Subject: [PATCH 106/523] Allow Calico networking on Azure and DigitalOcean * Introduce "calico" as a `networking` option on Azure and DigitalOcean using Calico's new VXLAN support (similar to flannel). Flannel remains the default on these platforms for now. * Historically, DigitalOcean and Azure only allowed Flannel as the CNI provider, since those platforms don't support IPIP traffic that was previously required for Calico. * Looking forward, its desireable for Calico to become the default across Typhoon clusters, since it provides NetworkPolicy and a consistent experience * No changes to AWS, GCP, or bare-metal where Calico remains the default CNI provider. On these platforms, IPIP mode will always be used, since its available and more performant than vxlan --- CHANGES.md | 13 +++++++++++-- azure/container-linux/kubernetes/bootkube.tf | 17 ++++++++++++----- azure/container-linux/kubernetes/variables.tf | 6 ++++++ .../container-linux/kubernetes/bootkube.tf | 17 +++++++++++------ digital-ocean/container-linux/kubernetes/ssh.tf | 1 + .../container-linux/kubernetes/variables.tf | 6 ++++++ digital-ocean/fedora-atomic/kubernetes/ssh.tf | 1 + docs/cl/azure.md | 1 + docs/cl/digital-ocean.md | 1 + docs/topics/performance.md | 13 ++++++------- 10 files changed, 56 insertions(+), 20 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index ef6121153..2c77ce71a 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -7,14 +7,23 @@ Notable changes between versions. * Kubernetes [v1.14.2](https://github.com/kubernetes/kubernetes/blob/master/CHANGELOG-1.14.md#v1142) * Update etcd from v3.3.12 to [v3.3.13](https://github.com/etcd-io/etcd/releases/tag/v3.3.13) * Upgrade Calico from v3.6.1 to [v3.7.2](https://docs.projectcalico.org/v3.7/release-notes/) -* Change flannel port from 8472 (kernel default) to 4789 (IANA VXLAN) +* Change flannel VXLAN port from 8472 (kernel default) to 4789 (IANA VXLAN) #### AWS -* Only set internal VXLAN rules when `networking` is flannel (default: calico) +* Only set internal VXLAN rules when `networking` is "flannel" (default: calico) + +#### Azure + +* Allow choosing Calico as the network provider (experimental) ([#472](https://github.com/poseidon/typhoon/pull/472)) + * Add a `networking` variable accepting "flannel" (default) or "calico" + * Use VXLAN encapsulation since Azure doesn't support IPIP #### DigitalOcean +* Allow choosing Calico as the network provider (experimental) ([#472](https://github.com/poseidon/typhoon/pull/472)) + * Add a `networking` variable accepting "flannel" (default) or "calico" + * Use VXLAN encapsulation since DigitalOcean doesn't support IPIP * Add explicit ordering between firewall rule creation and secure copying Kubelet credentials ([#469](https://github.com/poseidon/typhoon/pull/469)) * Fix race scenario if copies to nodes were before rule creation, blocking cluster creation diff --git a/azure/container-linux/kubernetes/bootkube.tf b/azure/container-linux/kubernetes/bootkube.tf index 8912a0b7a..0898fb1c1 100644 --- a/azure/container-linux/kubernetes/bootkube.tf +++ b/azure/container-linux/kubernetes/bootkube.tf @@ -2,11 +2,18 @@ module "bootkube" { source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=85571f6dae3522e2a7de01b7e0a3f7e3a9359641/" - cluster_name = "${var.cluster_name}" - api_servers = ["${format("%s.%s", var.cluster_name, var.dns_zone)}"] - etcd_servers = ["${formatlist("%s.%s", azurerm_dns_a_record.etcds.*.name, var.dns_zone)}"] - asset_dir = "${var.asset_dir}" - networking = "flannel" + cluster_name = "${var.cluster_name}" + api_servers = ["${format("%s.%s", var.cluster_name, var.dns_zone)}"] + etcd_servers = ["${formatlist("%s.%s", azurerm_dns_a_record.etcds.*.name, var.dns_zone)}"] + asset_dir = "${var.asset_dir}" + + networking = "${var.networking}" + + # only effective with Calico networking + # we should be able to use 1450 MTU, but in practice, 1410 was needed + network_encapsulation = "vxlan" + network_mtu = "1410" + pod_cidr = "${var.pod_cidr}" service_cidr = "${var.service_cidr}" cluster_domain_suffix = "${var.cluster_domain_suffix}" diff --git a/azure/container-linux/kubernetes/variables.tf b/azure/container-linux/kubernetes/variables.tf index b5e378dc6..90a14574f 100644 --- a/azure/container-linux/kubernetes/variables.tf +++ b/azure/container-linux/kubernetes/variables.tf @@ -88,6 +88,12 @@ variable "asset_dir" { type = "string" } +variable "networking" { + description = "Choice of networking provider (flannel or calico)" + type = "string" + default = "flannel" +} + variable "host_cidr" { description = "CIDR IPv4 range to assign to instances" type = "string" diff --git a/digital-ocean/container-linux/kubernetes/bootkube.tf b/digital-ocean/container-linux/kubernetes/bootkube.tf index 32f084247..f229de769 100644 --- a/digital-ocean/container-linux/kubernetes/bootkube.tf +++ b/digital-ocean/container-linux/kubernetes/bootkube.tf @@ -2,12 +2,17 @@ module "bootkube" { source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=85571f6dae3522e2a7de01b7e0a3f7e3a9359641/" - cluster_name = "${var.cluster_name}" - api_servers = ["${format("%s.%s", var.cluster_name, var.dns_zone)}"] - etcd_servers = "${digitalocean_record.etcds.*.fqdn}" - asset_dir = "${var.asset_dir}" - networking = "flannel" - network_mtu = 1440 + cluster_name = "${var.cluster_name}" + api_servers = ["${format("%s.%s", var.cluster_name, var.dns_zone)}"] + etcd_servers = "${digitalocean_record.etcds.*.fqdn}" + asset_dir = "${var.asset_dir}" + + networking = "${var.networking}" + + # only effective with Calico networking + network_encapsulation = "vxlan" + network_mtu = "1450" + pod_cidr = "${var.pod_cidr}" service_cidr = "${var.service_cidr}" cluster_domain_suffix = "${var.cluster_domain_suffix}" diff --git a/digital-ocean/container-linux/kubernetes/ssh.tf b/digital-ocean/container-linux/kubernetes/ssh.tf index 98740a22e..1816f5649 100644 --- a/digital-ocean/container-linux/kubernetes/ssh.tf +++ b/digital-ocean/container-linux/kubernetes/ssh.tf @@ -1,6 +1,7 @@ # Secure copy etcd TLS assets and kubeconfig to controllers. Activates kubelet.service resource "null_resource" "copy-controller-secrets" { count = "${var.controller_count}" + depends_on = [ "digitalocean_firewall.rules", ] diff --git a/digital-ocean/container-linux/kubernetes/variables.tf b/digital-ocean/container-linux/kubernetes/variables.tf index 9606fed08..bd8b41950 100644 --- a/digital-ocean/container-linux/kubernetes/variables.tf +++ b/digital-ocean/container-linux/kubernetes/variables.tf @@ -71,6 +71,12 @@ variable "asset_dir" { type = "string" } +variable "networking" { + description = "Choice of networking provider (flannel or calico)" + type = "string" + default = "flannel" +} + variable "pod_cidr" { description = "CIDR IPv4 range to assign Kubernetes pods" type = "string" diff --git a/digital-ocean/fedora-atomic/kubernetes/ssh.tf b/digital-ocean/fedora-atomic/kubernetes/ssh.tf index 156a4894b..77ae4abaf 100644 --- a/digital-ocean/fedora-atomic/kubernetes/ssh.tf +++ b/digital-ocean/fedora-atomic/kubernetes/ssh.tf @@ -1,6 +1,7 @@ # Secure copy etcd TLS assets and kubeconfig to controllers. Activates kubelet.service resource "null_resource" "copy-controller-secrets" { count = "${var.controller_count}" + depends_on = [ "digitalocean_firewall.rules", ] diff --git a/docs/cl/azure.md b/docs/cl/azure.md index 2a12f80dc..97d09d725 100644 --- a/docs/cl/azure.md +++ b/docs/cl/azure.md @@ -253,6 +253,7 @@ Reference the DNS zone with `"${azurerm_dns_zone.clusters.name}"` and its resour | worker_priority | Set priority to Low to use reduced cost surplus capacity, with the tradeoff that instances can be deallocated at any time | Regular | Low | | controller_clc_snippets | Controller Container Linux Config snippets | [] | [example](/advanced/customization/#usage) | | worker_clc_snippets | Worker Container Linux Config snippets | [] | [example](/advanced/customization/#usage) | +| networking | Choice of networking provider | "flannel" | "flannel" or "calico" (experimental) | | host_cidr | CIDR IPv4 range to assign to instances | "10.0.0.0/16" | "10.0.0.0/20" | | pod_cidr | CIDR IPv4 range to assign to Kubernetes pods | "10.2.0.0/16" | "10.22.0.0/16" | | service_cidr | CIDR IPv4 range to assign to Kubernetes services | "10.3.0.0/16" | "10.3.0.0/24" | diff --git a/docs/cl/digital-ocean.md b/docs/cl/digital-ocean.md index 90e26b4fa..faf7f0420 100644 --- a/docs/cl/digital-ocean.md +++ b/docs/cl/digital-ocean.md @@ -253,6 +253,7 @@ Digital Ocean requires the SSH public key be uploaded to your account, so you ma | image | Container Linux image for instances | "coreos-stable" | coreos-stable, coreos-beta, coreos-alpha | | controller_clc_snippets | Controller Container Linux Config snippets | [] | [example](/advanced/customization/) | | worker_clc_snippets | Worker Container Linux Config snippets | [] | [example](/advanced/customization/) | +| networking | Choice of networking provider | "flannel" | "flannel" or "calico" (experimental) | | pod_cidr | CIDR IPv4 range to assign to Kubernetes pods | "10.2.0.0/16" | "10.22.0.0/16" | | service_cidr | CIDR IPv4 range to assign to Kubernetes services | "10.3.0.0/16" | "10.3.0.0/24" | | cluster_domain_suffix | FQDN suffix for Kubernetes services answered by coredns. | "cluster.local" | "k8s.example.com" | diff --git a/docs/topics/performance.md b/docs/topics/performance.md index 20f32f42b..d7aa60c04 100644 --- a/docs/topics/performance.md +++ b/docs/topics/performance.md @@ -26,20 +26,19 @@ Network performance varies based on the platform and CNI plugin. `iperf` was use |----------------------------|-------:|-------------:|-------------:| | AWS (flannel) | 5 Gb/s | 4.94 Gb/s | 4.89 Gb/s | | AWS (calico, MTU 1480) | 5 Gb/s | 4.94 Gb/s | 4.42 Gb/s | -| AWS (calico, MTU 8981) | 5 Gb/s | 4.94 Gb/s | 4.75 Gb/s | -| Azure (flannel) | Varies | 749 Mb/s | 680 Mb/s | +| AWS (calico, MTU 8981) | 5 Gb/s | 4.94 Gb/s | 4.90 Gb/s | +| Azure (flannel) | Varies | 749 Mb/s | 650 Mb/s | +| Azure (calico) | Varies | 749 Mb/s | 650 Mb/s | | Bare-Metal (flannel) | 1 Gb/s | 940 Mb/s | 903 Mb/s | | Bare-Metal (calico) | 1 Gb/s | 940 Mb/s | 931 Mb/s | -| Bare-Metal (flannel, bond) | 3 Gb/s | 2.3 Gb/s | 1.17 Gb/s | -| Bare-Metal (calico, bond) | 3 Gb/s | 2.3 Gb/s | 1.17 Gb/s | -| Digital Ocean | 2 Gb/s | 1.97 Gb/s | 1.64 Gb/s | +| Digital Ocean (flannel) | Varies | 1.97 Gb/s | 1.20 Gb/s | +| Digital Ocean (calico) | Varies | 1.97 Gb/s | 1.20 Gb/s | | Google Cloud (flannel) | 2 Gb/s | 1.94 Gb/s | 1.76 Gb/s | | Google Cloud (calico) | 2 Gb/s | 1.94 Gb/s | 1.81 Gb/s | Notes: * Calico and Flannel have comparable performance. Platform and configuration differences dominate. -* AWS and Azure node bandwidth (i.e. upper bound) depends greatly on machine type +* Azure and DigitalOcean network performance can be quite variable or depend on machine type * Only [certain AWS EC2 instance types](http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/network_mtu.html#jumbo_frame_instances) allow jumbo frames. This is why the default MTU on AWS must be 1480. -* Neither CNI provider seems to be able to leverage bonded NICs well (bare-metal) From 5653ba38cf45bf8bbc64e896192937eea86988f0 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Mon, 20 May 2019 17:19:58 +0200 Subject: [PATCH 107/523] Update mkdocs-material from v4.2.0 to v4.3.0 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 3a7faec13..d8b45b69c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ mkdocs==1.0.4 -mkdocs-material==4.2.0 +mkdocs-material==4.3.0 pygments==2.2.0 pymdown-extensions==5.0.0 From bef9b991b79f1dc3604a6ec869bf17435a72f475 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Mon, 20 May 2019 18:29:56 +0200 Subject: [PATCH 108/523] Bump Terraform provider versions in docs * Bump Terraform provider versions to reflect the versions used by the maintainer --- CHANGES.md | 2 ++ docs/cl/aws.md | 2 +- docs/cl/digital-ocean.md | 2 +- 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 2c77ce71a..ef5c157a7 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -3,6 +3,8 @@ Notable changes between versions. ## Latest + +## v1.14.2 * Kubernetes [v1.14.2](https://github.com/kubernetes/kubernetes/blob/master/CHANGELOG-1.14.md#v1142) * Update etcd from v3.3.12 to [v3.3.13](https://github.com/etcd-io/etcd/releases/tag/v3.3.13) diff --git a/docs/cl/aws.md b/docs/cl/aws.md index 4dc947b0c..61d502c48 100644 --- a/docs/cl/aws.md +++ b/docs/cl/aws.md @@ -49,7 +49,7 @@ Configure the AWS provider to use your access key credentials in a `providers.tf ```tf provider "aws" { - version = "~> 2.8.0" + version = "~> 2.11.0" alias = "default" region = "eu-central-1" diff --git a/docs/cl/digital-ocean.md b/docs/cl/digital-ocean.md index faf7f0420..0a738c5b1 100644 --- a/docs/cl/digital-ocean.md +++ b/docs/cl/digital-ocean.md @@ -50,7 +50,7 @@ Configure the DigitalOcean provider to use your token in a `providers.tf` file. ```tf provider "digitalocean" { - version = "~> 1.2.0" + version = "~> 1.3.0" token = "${chomp(file("~/.config/digital-ocean/token"))}" alias = "default" } From 6e4cf65c4c74c9c65fc1fe2061f7a83d44ef9203 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Wed, 22 May 2019 18:24:42 +0200 Subject: [PATCH 109/523] Fix terraform-render-bootkube to remove trailing slash * Fix to remove a trailing slash that was erroneously introduced in the scripting that updated from v1.14.1 to v1.14.2 * Workaround before this fix was to re-run `terraform init` --- CHANGES.md | 2 ++ aws/container-linux/kubernetes/bootkube.tf | 2 +- aws/fedora-atomic/kubernetes/bootkube.tf | 2 +- azure/container-linux/kubernetes/bootkube.tf | 2 +- bare-metal/container-linux/kubernetes/bootkube.tf | 2 +- bare-metal/fedora-atomic/kubernetes/bootkube.tf | 2 +- digital-ocean/container-linux/kubernetes/bootkube.tf | 2 +- digital-ocean/fedora-atomic/kubernetes/bootkube.tf | 2 +- google-cloud/container-linux/kubernetes/bootkube.tf | 2 +- google-cloud/fedora-atomic/kubernetes/bootkube.tf | 2 +- 10 files changed, 11 insertions(+), 9 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index ef5c157a7..7b46ae58e 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -4,6 +4,8 @@ Notable changes between versions. ## Latest +* Fix trailing slash in terraform-render-bootkube version ([#479](https://github.com/poseidon/typhoon/pull/479)) + ## v1.14.2 * Kubernetes [v1.14.2](https://github.com/kubernetes/kubernetes/blob/master/CHANGELOG-1.14.md#v1142) diff --git a/aws/container-linux/kubernetes/bootkube.tf b/aws/container-linux/kubernetes/bootkube.tf index 416634479..70a6623aa 100644 --- a/aws/container-linux/kubernetes/bootkube.tf +++ b/aws/container-linux/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=85571f6dae3522e2a7de01b7e0a3f7e3a9359641/" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=85571f6dae3522e2a7de01b7e0a3f7e3a9359641" cluster_name = "${var.cluster_name}" api_servers = ["${format("%s.%s", var.cluster_name, var.dns_zone)}"] diff --git a/aws/fedora-atomic/kubernetes/bootkube.tf b/aws/fedora-atomic/kubernetes/bootkube.tf index 5f448be36..afb8bedf2 100644 --- a/aws/fedora-atomic/kubernetes/bootkube.tf +++ b/aws/fedora-atomic/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=85571f6dae3522e2a7de01b7e0a3f7e3a9359641/" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=85571f6dae3522e2a7de01b7e0a3f7e3a9359641" cluster_name = "${var.cluster_name}" api_servers = ["${format("%s.%s", var.cluster_name, var.dns_zone)}"] diff --git a/azure/container-linux/kubernetes/bootkube.tf b/azure/container-linux/kubernetes/bootkube.tf index 0898fb1c1..358b45efc 100644 --- a/azure/container-linux/kubernetes/bootkube.tf +++ b/azure/container-linux/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=85571f6dae3522e2a7de01b7e0a3f7e3a9359641/" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=85571f6dae3522e2a7de01b7e0a3f7e3a9359641" cluster_name = "${var.cluster_name}" api_servers = ["${format("%s.%s", var.cluster_name, var.dns_zone)}"] diff --git a/bare-metal/container-linux/kubernetes/bootkube.tf b/bare-metal/container-linux/kubernetes/bootkube.tf index 8ea8d5229..60d2bc9da 100644 --- a/bare-metal/container-linux/kubernetes/bootkube.tf +++ b/bare-metal/container-linux/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=85571f6dae3522e2a7de01b7e0a3f7e3a9359641/" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=85571f6dae3522e2a7de01b7e0a3f7e3a9359641" cluster_name = "${var.cluster_name}" api_servers = ["${var.k8s_domain_name}"] diff --git a/bare-metal/fedora-atomic/kubernetes/bootkube.tf b/bare-metal/fedora-atomic/kubernetes/bootkube.tf index f99ec555d..a020d3139 100644 --- a/bare-metal/fedora-atomic/kubernetes/bootkube.tf +++ b/bare-metal/fedora-atomic/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=85571f6dae3522e2a7de01b7e0a3f7e3a9359641/" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=85571f6dae3522e2a7de01b7e0a3f7e3a9359641" cluster_name = "${var.cluster_name}" api_servers = ["${var.k8s_domain_name}"] diff --git a/digital-ocean/container-linux/kubernetes/bootkube.tf b/digital-ocean/container-linux/kubernetes/bootkube.tf index f229de769..bf6a69653 100644 --- a/digital-ocean/container-linux/kubernetes/bootkube.tf +++ b/digital-ocean/container-linux/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=85571f6dae3522e2a7de01b7e0a3f7e3a9359641/" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=85571f6dae3522e2a7de01b7e0a3f7e3a9359641" cluster_name = "${var.cluster_name}" api_servers = ["${format("%s.%s", var.cluster_name, var.dns_zone)}"] diff --git a/digital-ocean/fedora-atomic/kubernetes/bootkube.tf b/digital-ocean/fedora-atomic/kubernetes/bootkube.tf index 55d5cef66..c821af674 100644 --- a/digital-ocean/fedora-atomic/kubernetes/bootkube.tf +++ b/digital-ocean/fedora-atomic/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=85571f6dae3522e2a7de01b7e0a3f7e3a9359641/" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=85571f6dae3522e2a7de01b7e0a3f7e3a9359641" cluster_name = "${var.cluster_name}" api_servers = ["${format("%s.%s", var.cluster_name, var.dns_zone)}"] diff --git a/google-cloud/container-linux/kubernetes/bootkube.tf b/google-cloud/container-linux/kubernetes/bootkube.tf index dd36b5126..e4d2b2b40 100644 --- a/google-cloud/container-linux/kubernetes/bootkube.tf +++ b/google-cloud/container-linux/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=85571f6dae3522e2a7de01b7e0a3f7e3a9359641/" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=85571f6dae3522e2a7de01b7e0a3f7e3a9359641" cluster_name = "${var.cluster_name}" api_servers = ["${format("%s.%s", var.cluster_name, var.dns_zone)}"] diff --git a/google-cloud/fedora-atomic/kubernetes/bootkube.tf b/google-cloud/fedora-atomic/kubernetes/bootkube.tf index c3a979f39..dbf5a82a8 100644 --- a/google-cloud/fedora-atomic/kubernetes/bootkube.tf +++ b/google-cloud/fedora-atomic/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=85571f6dae3522e2a7de01b7e0a3f7e3a9359641/" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=85571f6dae3522e2a7de01b7e0a3f7e3a9359641" cluster_name = "${var.cluster_name}" api_servers = ["${format("%s.%s", var.cluster_name, var.dns_zone)}"] From 221889cc9bb4c77719c719f059408eddb97f87b7 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Sun, 26 May 2019 21:58:28 -0700 Subject: [PATCH 110/523] Update Prometheus from v2.9.2 to v2.10.0 * https://github.com/prometheus/prometheus/releases/tag/v2.10.0 --- CHANGES.md | 4 ++++ addons/prometheus/deployment.yaml | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index 7b46ae58e..c3890ab22 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -6,6 +6,10 @@ Notable changes between versions. * Fix trailing slash in terraform-render-bootkube version ([#479](https://github.com/poseidon/typhoon/pull/479)) +#### Addons + +* Update Prometheus from v2.9.2 to v2.10.0 + ## v1.14.2 * Kubernetes [v1.14.2](https://github.com/kubernetes/kubernetes/blob/master/CHANGELOG-1.14.md#v1142) diff --git a/addons/prometheus/deployment.yaml b/addons/prometheus/deployment.yaml index 691c8df94..06abfa9e5 100644 --- a/addons/prometheus/deployment.yaml +++ b/addons/prometheus/deployment.yaml @@ -20,7 +20,7 @@ spec: serviceAccountName: prometheus containers: - name: prometheus - image: quay.io/prometheus/prometheus:v2.9.2 + image: quay.io/prometheus/prometheus:v2.10.0 args: - --web.listen-address=0.0.0.0:9090 - --config.file=/etc/prometheus/prometheus.yaml From 5d2684a04dd8076a76fd26a47cc24cf9d72b6b82 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Sun, 26 May 2019 22:00:47 -0700 Subject: [PATCH 111/523] Update Grafana from v6.1.6 to v6.2.0 * https://github.com/grafana/grafana/releases/tag/v6.2.0 --- CHANGES.md | 1 + addons/grafana/deployment.yaml | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index c3890ab22..e58cae3be 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -9,6 +9,7 @@ Notable changes between versions. #### Addons * Update Prometheus from v2.9.2 to v2.10.0 +* Update Grafana from v6.1.6 to v6.2.0 ## v1.14.2 diff --git a/addons/grafana/deployment.yaml b/addons/grafana/deployment.yaml index 2190134f6..7c139c351 100644 --- a/addons/grafana/deployment.yaml +++ b/addons/grafana/deployment.yaml @@ -23,7 +23,7 @@ spec: spec: containers: - name: grafana - image: grafana/grafana:6.1.6 + image: grafana/grafana:6.2.0 env: - name: GF_PATHS_CONFIG value: "/etc/grafana/custom.ini" From 0a835ee403914e45a2f5bf054b5e7a8321f6d7ec Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Sun, 26 May 2019 23:14:50 -0700 Subject: [PATCH 112/523] Replace deprecated `azurerm_autoscale_setting` * Fix Terraform provider azure warning about `azurerm_autoscale_setting` * Require terraform-provider-azure v1.22+ version that introduces the new `azurerm_monitor_autoscale_setting` resource * https://github.com/terraform-providers/terraform-provider-azurerm/blob/master/CHANGELOG.md#1220-february-11-2019 --- CHANGES.md | 5 +++++ azure/container-linux/kubernetes/require.tf | 2 +- azure/container-linux/kubernetes/workers/workers.tf | 2 +- 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index e58cae3be..572c4211d 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -6,6 +6,11 @@ Notable changes between versions. * Fix trailing slash in terraform-render-bootkube version ([#479](https://github.com/poseidon/typhoon/pull/479)) +#### Azure + +* Replace `azurerm_autoscale_setting` with `azurerm_monitor_autoscale_setting` ([#482](https://github.com/poseidon/typhoon/pull/482)) + * Require `terraform-provider-azurerm` v1.22+ (action required) + #### Addons * Update Prometheus from v2.9.2 to v2.10.0 diff --git a/azure/container-linux/kubernetes/require.tf b/azure/container-linux/kubernetes/require.tf index f85cf7b49..836a87d54 100644 --- a/azure/container-linux/kubernetes/require.tf +++ b/azure/container-linux/kubernetes/require.tf @@ -5,7 +5,7 @@ terraform { } provider "azurerm" { - version = "~> 1.21" + version = "~> 1.22" } provider "local" { diff --git a/azure/container-linux/kubernetes/workers/workers.tf b/azure/container-linux/kubernetes/workers/workers.tf index a929d5656..5d1773954 100644 --- a/azure/container-linux/kubernetes/workers/workers.tf +++ b/azure/container-linux/kubernetes/workers/workers.tf @@ -73,7 +73,7 @@ resource "azurerm_virtual_machine_scale_set" "workers" { } # Scale up or down to maintain desired number, tolerating deallocations. -resource "azurerm_autoscale_setting" "workers" { +resource "azurerm_monitor_autoscale_setting" "workers" { resource_group_name = "${var.resource_group_name}" name = "${var.name}-maintain-desired" From 2a71cba0e3a6a398d67050e327d3c0314976defa Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Mon, 27 May 2019 00:11:52 -0700 Subject: [PATCH 113/523] Update CoreDNS from v1.3.1 to v1.5.0 * Add `ready` plugin to improve readinessProbe * https://coredns.io/2019/04/06/coredns-1.5.0-release/ --- CHANGES.md | 2 ++ aws/container-linux/kubernetes/bootkube.tf | 2 +- aws/fedora-atomic/kubernetes/bootkube.tf | 2 +- azure/container-linux/kubernetes/bootkube.tf | 2 +- bare-metal/container-linux/kubernetes/bootkube.tf | 2 +- bare-metal/fedora-atomic/kubernetes/bootkube.tf | 2 +- digital-ocean/container-linux/kubernetes/bootkube.tf | 2 +- digital-ocean/fedora-atomic/kubernetes/bootkube.tf | 2 +- google-cloud/container-linux/kubernetes/bootkube.tf | 2 +- google-cloud/fedora-atomic/kubernetes/bootkube.tf | 2 +- 10 files changed, 11 insertions(+), 9 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 572c4211d..8fac50f00 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -4,6 +4,8 @@ Notable changes between versions. ## Latest +* Update CoreDNS from v1.3.1 to v1.5.0 + * Add `ready` plugin to improve readinessProbe * Fix trailing slash in terraform-render-bootkube version ([#479](https://github.com/poseidon/typhoon/pull/479)) #### Azure diff --git a/aws/container-linux/kubernetes/bootkube.tf b/aws/container-linux/kubernetes/bootkube.tf index 70a6623aa..7d11959b4 100644 --- a/aws/container-linux/kubernetes/bootkube.tf +++ b/aws/container-linux/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=85571f6dae3522e2a7de01b7e0a3f7e3a9359641" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=efd1cfd9bf2f56490c760e12c65f9d49ddddec43" cluster_name = "${var.cluster_name}" api_servers = ["${format("%s.%s", var.cluster_name, var.dns_zone)}"] diff --git a/aws/fedora-atomic/kubernetes/bootkube.tf b/aws/fedora-atomic/kubernetes/bootkube.tf index afb8bedf2..cf4e00315 100644 --- a/aws/fedora-atomic/kubernetes/bootkube.tf +++ b/aws/fedora-atomic/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=85571f6dae3522e2a7de01b7e0a3f7e3a9359641" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=efd1cfd9bf2f56490c760e12c65f9d49ddddec43" cluster_name = "${var.cluster_name}" api_servers = ["${format("%s.%s", var.cluster_name, var.dns_zone)}"] diff --git a/azure/container-linux/kubernetes/bootkube.tf b/azure/container-linux/kubernetes/bootkube.tf index 358b45efc..ee50b7b9c 100644 --- a/azure/container-linux/kubernetes/bootkube.tf +++ b/azure/container-linux/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=85571f6dae3522e2a7de01b7e0a3f7e3a9359641" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=efd1cfd9bf2f56490c760e12c65f9d49ddddec43" cluster_name = "${var.cluster_name}" api_servers = ["${format("%s.%s", var.cluster_name, var.dns_zone)}"] diff --git a/bare-metal/container-linux/kubernetes/bootkube.tf b/bare-metal/container-linux/kubernetes/bootkube.tf index 60d2bc9da..ca4d9b6bb 100644 --- a/bare-metal/container-linux/kubernetes/bootkube.tf +++ b/bare-metal/container-linux/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=85571f6dae3522e2a7de01b7e0a3f7e3a9359641" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=efd1cfd9bf2f56490c760e12c65f9d49ddddec43" cluster_name = "${var.cluster_name}" api_servers = ["${var.k8s_domain_name}"] diff --git a/bare-metal/fedora-atomic/kubernetes/bootkube.tf b/bare-metal/fedora-atomic/kubernetes/bootkube.tf index a020d3139..baddb56cd 100644 --- a/bare-metal/fedora-atomic/kubernetes/bootkube.tf +++ b/bare-metal/fedora-atomic/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=85571f6dae3522e2a7de01b7e0a3f7e3a9359641" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=efd1cfd9bf2f56490c760e12c65f9d49ddddec43" cluster_name = "${var.cluster_name}" api_servers = ["${var.k8s_domain_name}"] diff --git a/digital-ocean/container-linux/kubernetes/bootkube.tf b/digital-ocean/container-linux/kubernetes/bootkube.tf index bf6a69653..87fde3661 100644 --- a/digital-ocean/container-linux/kubernetes/bootkube.tf +++ b/digital-ocean/container-linux/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=85571f6dae3522e2a7de01b7e0a3f7e3a9359641" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=efd1cfd9bf2f56490c760e12c65f9d49ddddec43" cluster_name = "${var.cluster_name}" api_servers = ["${format("%s.%s", var.cluster_name, var.dns_zone)}"] diff --git a/digital-ocean/fedora-atomic/kubernetes/bootkube.tf b/digital-ocean/fedora-atomic/kubernetes/bootkube.tf index c821af674..92d280da7 100644 --- a/digital-ocean/fedora-atomic/kubernetes/bootkube.tf +++ b/digital-ocean/fedora-atomic/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=85571f6dae3522e2a7de01b7e0a3f7e3a9359641" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=efd1cfd9bf2f56490c760e12c65f9d49ddddec43" cluster_name = "${var.cluster_name}" api_servers = ["${format("%s.%s", var.cluster_name, var.dns_zone)}"] diff --git a/google-cloud/container-linux/kubernetes/bootkube.tf b/google-cloud/container-linux/kubernetes/bootkube.tf index e4d2b2b40..21d96211a 100644 --- a/google-cloud/container-linux/kubernetes/bootkube.tf +++ b/google-cloud/container-linux/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=85571f6dae3522e2a7de01b7e0a3f7e3a9359641" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=efd1cfd9bf2f56490c760e12c65f9d49ddddec43" cluster_name = "${var.cluster_name}" api_servers = ["${format("%s.%s", var.cluster_name, var.dns_zone)}"] diff --git a/google-cloud/fedora-atomic/kubernetes/bootkube.tf b/google-cloud/fedora-atomic/kubernetes/bootkube.tf index dbf5a82a8..1e74b4971 100644 --- a/google-cloud/fedora-atomic/kubernetes/bootkube.tf +++ b/google-cloud/fedora-atomic/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=85571f6dae3522e2a7de01b7e0a3f7e3a9359641" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=efd1cfd9bf2f56490c760e12c65f9d49ddddec43" cluster_name = "${var.cluster_name}" api_servers = ["${format("%s.%s", var.cluster_name, var.dns_zone)}"] From d9e71954772b21b370321cf55b27d751a02f0b1e Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Mon, 27 May 2019 12:25:00 -0700 Subject: [PATCH 114/523] Update Grafana from v2.6.0 to v2.6.1 --- CHANGES.md | 2 +- addons/grafana/deployment.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 8fac50f00..40a52f8ba 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -16,7 +16,7 @@ Notable changes between versions. #### Addons * Update Prometheus from v2.9.2 to v2.10.0 -* Update Grafana from v6.1.6 to v6.2.0 +* Update Grafana from v6.1.6 to v6.2.1 ## v1.14.2 diff --git a/addons/grafana/deployment.yaml b/addons/grafana/deployment.yaml index 7c139c351..2dd6a381f 100644 --- a/addons/grafana/deployment.yaml +++ b/addons/grafana/deployment.yaml @@ -23,7 +23,7 @@ spec: spec: containers: - name: grafana - image: grafana/grafana:6.2.0 + image: grafana/grafana:6.2.1 env: - name: GF_PATHS_CONFIG value: "/etc/grafana/custom.ini" From c565f9fd4779f33ba4bef8fe939b7b3f53d7cf43 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Mon, 27 May 2019 16:33:21 -0700 Subject: [PATCH 115/523] Rename worker pool modules' count variable to worker_count * This change affects users who use worker pools on AWS, GCP, or Azure with a Container Linux derivative * Rename worker pool modules' `count` variable to `worker_count`, because `count` will be a reserved variable name in Terraform v0.12 --- CHANGES.md | 12 ++++++++++++ aws/container-linux/kubernetes/workers.tf | 2 +- .../kubernetes/workers/variables.tf | 2 +- .../kubernetes/workers/workers.tf | 6 +++--- azure/container-linux/kubernetes/workers.tf | 2 +- .../kubernetes/workers/variables.tf | 2 +- .../kubernetes/workers/workers.tf | 8 ++++---- docs/advanced/worker-pools.md | 16 ++++++++-------- .../container-linux/kubernetes/workers.tf | 2 +- .../kubernetes/workers/variables.tf | 2 +- .../kubernetes/workers/workers.tf | 2 +- 11 files changed, 34 insertions(+), 22 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 40a52f8ba..841f136fa 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -8,10 +8,22 @@ Notable changes between versions. * Add `ready` plugin to improve readinessProbe * Fix trailing slash in terraform-render-bootkube version ([#479](https://github.com/poseidon/typhoon/pull/479)) +### AWS + +* Rename `worker` pool module `count` variable to `worker_count` ([#485](https://github.com/poseidon/typhoon/pull/485)) (action required) + * `count` will become a reserved variable name in Terraform v0.12 + #### Azure * Replace `azurerm_autoscale_setting` with `azurerm_monitor_autoscale_setting` ([#482](https://github.com/poseidon/typhoon/pull/482)) * Require `terraform-provider-azurerm` v1.22+ (action required) +* Rename `worker` pool module `count` variable to `worker_count` ([#485](https://github.com/poseidon/typhoon/pull/485)) (action required) + * `count` will become a reserved variable name in Terraform v0.12 + +### Google Cloud + +* Rename `worker` pool module `count` variable to `worker_count` ([#485](https://github.com/poseidon/typhoon/pull/485)) (action required) + * `count` will become a reserved variable name in Terraform v0.12 #### Addons diff --git a/aws/container-linux/kubernetes/workers.tf b/aws/container-linux/kubernetes/workers.tf index 5065c0dbb..f28b13a5b 100644 --- a/aws/container-linux/kubernetes/workers.tf +++ b/aws/container-linux/kubernetes/workers.tf @@ -6,7 +6,7 @@ module "workers" { vpc_id = "${aws_vpc.network.id}" subnet_ids = ["${aws_subnet.public.*.id}"] security_groups = ["${aws_security_group.worker.id}"] - count = "${var.worker_count}" + worker_count = "${var.worker_count}" instance_type = "${var.worker_type}" os_image = "${var.os_image}" disk_size = "${var.disk_size}" diff --git a/aws/container-linux/kubernetes/workers/variables.tf b/aws/container-linux/kubernetes/workers/variables.tf index 253b3dc94..9d10da36d 100644 --- a/aws/container-linux/kubernetes/workers/variables.tf +++ b/aws/container-linux/kubernetes/workers/variables.tf @@ -22,7 +22,7 @@ variable "security_groups" { # instances -variable "count" { +variable "worker_count" { type = "string" default = "1" description = "Number of instances" diff --git a/aws/container-linux/kubernetes/workers/workers.tf b/aws/container-linux/kubernetes/workers/workers.tf index ef21c8258..db5a454f8 100644 --- a/aws/container-linux/kubernetes/workers/workers.tf +++ b/aws/container-linux/kubernetes/workers/workers.tf @@ -3,9 +3,9 @@ resource "aws_autoscaling_group" "workers" { name = "${var.name}-worker ${aws_launch_configuration.worker.name}" # count - desired_capacity = "${var.count}" - min_size = "${var.count}" - max_size = "${var.count + 2}" + desired_capacity = "${var.worker_count}" + min_size = "${var.worker_count}" + max_size = "${var.worker_count + 2}" default_cooldown = 30 health_check_grace_period = 30 diff --git a/azure/container-linux/kubernetes/workers.tf b/azure/container-linux/kubernetes/workers.tf index 1abbc6571..c0d7c4f00 100644 --- a/azure/container-linux/kubernetes/workers.tf +++ b/azure/container-linux/kubernetes/workers.tf @@ -9,7 +9,7 @@ module "workers" { security_group_id = "${azurerm_network_security_group.worker.id}" backend_address_pool_id = "${azurerm_lb_backend_address_pool.worker.id}" - count = "${var.worker_count}" + worker_count = "${var.worker_count}" vm_type = "${var.worker_type}" os_image = "${var.os_image}" priority = "${var.worker_priority}" diff --git a/azure/container-linux/kubernetes/workers/variables.tf b/azure/container-linux/kubernetes/workers/variables.tf index 1b0bd6d75..6e77d1c6e 100644 --- a/azure/container-linux/kubernetes/workers/variables.tf +++ b/azure/container-linux/kubernetes/workers/variables.tf @@ -32,7 +32,7 @@ variable "backend_address_pool_id" { # instances -variable "count" { +variable "worker_count" { type = "string" default = "1" description = "Number of instances" diff --git a/azure/container-linux/kubernetes/workers/workers.tf b/azure/container-linux/kubernetes/workers/workers.tf index 5d1773954..40ec8a1d3 100644 --- a/azure/container-linux/kubernetes/workers/workers.tf +++ b/azure/container-linux/kubernetes/workers/workers.tf @@ -15,7 +15,7 @@ resource "azurerm_virtual_machine_scale_set" "workers" { sku { name = "${var.vm_type}" tier = "standard" - capacity = "${var.count}" + capacity = "${var.worker_count}" } # boot @@ -87,9 +87,9 @@ resource "azurerm_monitor_autoscale_setting" "workers" { name = "default" capacity { - minimum = "${var.count}" - default = "${var.count}" - maximum = "${var.count}" + minimum = "${var.worker_count}" + default = "${var.worker_count}" + maximum = "${var.worker_count}" } } } diff --git a/docs/advanced/worker-pools.md b/docs/advanced/worker-pools.md index 848819773..fe91e4831 100644 --- a/docs/advanced/worker-pools.md +++ b/docs/advanced/worker-pools.md @@ -33,7 +33,7 @@ module "tempest-worker-pool" { ssh_authorized_key = "${var.ssh_authorized_key}" # optional - count = 2 + worker_count = 2 instance_type = "m5.large" os_image = "coreos-beta" } @@ -66,7 +66,7 @@ The AWS internal `workers` module supports a number of [variables](https://githu | Name | Description | Default | Example | |:-----|:------------|:--------|:--------| -| count | Number of instances | 1 | 3 | +| worker_count | Number of instances | 1 | 3 | | instance_type | EC2 instance type | "t3.small" | "t3.medium" | | os_image | AMI channel for a Container Linux derivative | coreos-stable | coreos-stable, coreos-beta, coreos-alpha, flatcar-stable, flatcar-beta, flatcar-alpha | | disk_size | Size of the disk in GB | 40 | 100 | @@ -101,9 +101,9 @@ module "ramius-worker-pool" { ssh_authorized_key = "${var.ssh_authorized_key}" # optional - count = 2 - vm_type = "Standard_F4" - priority = "Low" + worker_count = 2 + vm_type = "Standard_F4" + priority = "Low" } ``` @@ -136,7 +136,7 @@ The Azure internal `workers` module supports a number of [variables](https://git | Name | Description | Default | Example | |:-----|:------------|:--------|:--------| -| count | Number of instances | 1 | 3 | +| worker_count | Number of instances | 1 | 3 | | vm_type | Machine type for instances | "Standard_F1" | See below | | os_image | Channel for a Container Linux derivative | coreos-stable | coreos-stable, coreos-beta, coreos-alpha | | priority | Set priority to Low to use reduced cost surplus capacity, with the tradeoff that instances can be deallocated at any time | Regular | Low | @@ -169,7 +169,7 @@ module "yavin-worker-pool" { ssh_authorized_key = "${var.ssh_authorized_key}" # optional - count = 2 + worker_count = 2 machine_type = "n1-standard-16" os_image = "coreos-beta" preemptible = true @@ -215,7 +215,7 @@ Check the list of regions [docs](https://cloud.google.com/compute/docs/regions-z | Name | Description | Default | Example | |:-----|:------------|:--------|:--------| -| count | Number of instances | 1 | 3 | +| worker_count | Number of instances | 1 | 3 | | machine_type | Compute instance machine type | "n1-standard-1" | See below | | os_image | Container Linux image for compute instances | "coreos-stable" | "coreos-alpha", "coreos-beta" | | disk_size | Size of the disk in GB | 40 | 100 | diff --git a/google-cloud/container-linux/kubernetes/workers.tf b/google-cloud/container-linux/kubernetes/workers.tf index b33607741..089524a09 100644 --- a/google-cloud/container-linux/kubernetes/workers.tf +++ b/google-cloud/container-linux/kubernetes/workers.tf @@ -6,7 +6,7 @@ module "workers" { # GCE region = "${var.region}" network = "${google_compute_network.network.name}" - count = "${var.worker_count}" + worker_count = "${var.worker_count}" machine_type = "${var.worker_type}" os_image = "${var.os_image}" disk_size = "${var.disk_size}" diff --git a/google-cloud/container-linux/kubernetes/workers/variables.tf b/google-cloud/container-linux/kubernetes/workers/variables.tf index c67f3343b..2b5a5e830 100644 --- a/google-cloud/container-linux/kubernetes/workers/variables.tf +++ b/google-cloud/container-linux/kubernetes/workers/variables.tf @@ -22,7 +22,7 @@ variable "network" { # instances -variable "count" { +variable "worker_count" { type = "string" default = "1" description = "Number of worker compute instances the instance group should manage" diff --git a/google-cloud/container-linux/kubernetes/workers/workers.tf b/google-cloud/container-linux/kubernetes/workers/workers.tf index 1dcf0cbc8..bc258c1a3 100644 --- a/google-cloud/container-linux/kubernetes/workers/workers.tf +++ b/google-cloud/container-linux/kubernetes/workers/workers.tf @@ -8,7 +8,7 @@ resource "google_compute_region_instance_group_manager" "workers" { instance_template = "${google_compute_instance_template.worker.self_link}" region = "${var.region}" - target_size = "${var.count}" + target_size = "${var.worker_count}" target_pools = ["${google_compute_target_pool.workers.self_link}"] named_port { From c6faa6b5b8027bc4b42be573866c377db3059ad1 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Fri, 31 May 2019 00:45:16 -0700 Subject: [PATCH 116/523] Recommend updating Terraform providers ct and matchbox * Recomment updating Terraform provider plugins `terraform-provider-ct` and `terraform-provider-matchbox` to prepare for the upcoming Terraform v0.12 migration * https://github.com/poseidon/terraform-provider-ct/releases/tag/v0.3.2 * https://github.com/poseidon/terraform-provider-matchbox/releases/tag/v0.3.0 --- CHANGES.md | 9 +++++++-- docs/cl/aws.md | 12 ++++++------ docs/cl/azure.md | 12 ++++++------ docs/cl/bare-metal.md | 18 +++++++++--------- docs/cl/digital-ocean.md | 10 +++++----- docs/cl/google-cloud.md | 12 ++++++------ 6 files changed, 39 insertions(+), 34 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 841f136fa..68eb37ef2 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -7,6 +7,7 @@ Notable changes between versions. * Update CoreDNS from v1.3.1 to v1.5.0 * Add `ready` plugin to improve readinessProbe * Fix trailing slash in terraform-render-bootkube version ([#479](https://github.com/poseidon/typhoon/pull/479)) +* Recommend updating `terraform-provider-ct` plugin from v0.3.1 to [v0.3.2](https://github.com/poseidon/terraform-provider-ct/releases/tag/v0.3.2) ([#487](https://github.com/poseidon/typhoon/pull/487)) ### AWS @@ -20,7 +21,11 @@ Notable changes between versions. * Rename `worker` pool module `count` variable to `worker_count` ([#485](https://github.com/poseidon/typhoon/pull/485)) (action required) * `count` will become a reserved variable name in Terraform v0.12 -### Google Cloud +#### Bare-Metal + +* Recommend updating `terraform-provider-matchbox` plugin from v0.2.3 to [v0.3.0](https://github.com/poseidon/terraform-provider-matchbox/releases/tag/v0.3.0) ([#487](https://github.com/poseidon/typhoon/pull/487)) + +#### Google Cloud * Rename `worker` pool module `count` variable to `worker_count` ([#485](https://github.com/poseidon/typhoon/pull/485)) (action required) * `count` will become a reserved variable name in Terraform v0.12 @@ -121,7 +126,7 @@ Notable changes between versions. * Reverse DNS lookups for service IPv4 addresses unchanged * Upgrade Calico from v3.5.2 to [v3.6.0](https://docs.projectcalico.org/v3.6/release-notes/) ([#430](https://github.com/poseidon/typhoon/pull/430)) * Change pod IPAM from `host-local` to `calico-ipam`. `pod_cidr` is still divided into `/24` subnets per node, but managed as `ippools` and `ipamblocks` -* Suggest updating [terraform-provider-ct](https://github.com/poseidon/terraform-provider-ct) from v0.3.0 to [v0.3.1](https://github.com/poseidon/terraform-provider-ct/releases/tag/v0.3.1) ([#434](https://github.com/poseidon/typhoon/pull/434)) +* Recommend updating [terraform-provider-ct](https://github.com/poseidon/terraform-provider-ct) from v0.3.0 to [v0.3.1](https://github.com/poseidon/terraform-provider-ct/releases/tag/v0.3.1) ([#434](https://github.com/poseidon/typhoon/pull/434)) * Announce: Fedora Atomic modules will be not be updated beyond Kubernetes v1.13.x ([#437](https://github.com/poseidon/typhoon/pull/437)) * Thank you Project Atomic team and users, please see the deprecation [notice](https://typhoon.psdn.io/announce/#march-27-2019) diff --git a/docs/cl/aws.md b/docs/cl/aws.md index 61d502c48..77475a04f 100644 --- a/docs/cl/aws.md +++ b/docs/cl/aws.md @@ -18,15 +18,15 @@ Install [Terraform](https://www.terraform.io/downloads.html) v0.11.x on your sys ```sh $ terraform version -Terraform v0.11.12 +Terraform v0.11.14 ``` Add the [terraform-provider-ct](https://github.com/poseidon/terraform-provider-ct) plugin binary for your system to `~/.terraform.d/plugins/`, noting the final name. ```sh -wget https://github.com/poseidon/terraform-provider-ct/releases/download/v0.3.1/terraform-provider-ct-v0.3.1-linux-amd64.tar.gz -tar xzf terraform-provider-ct-v0.3.1-linux-amd64.tar.gz -mv terraform-provider-ct-v0.3.1-linux-amd64/terraform-provider-ct ~/.terraform.d/plugins/terraform-provider-ct_v0.3.1 +wget https://github.com/poseidon/terraform-provider-ct/releases/download/v0.3.2/terraform-provider-ct-v0.3.2-linux-amd64.tar.gz +tar xzf terraform-provider-ct-v0.3.2-linux-amd64.tar.gz +mv terraform-provider-ct-v0.3.2-linux-amd64/terraform-provider-ct ~/.terraform.d/plugins/terraform-provider-ct_v0.3.2 ``` Read [concepts](/architecture/concepts/) to learn about Terraform, modules, and organizing resources. Change to your infrastructure repository (e.g. `infra`). @@ -49,7 +49,7 @@ Configure the AWS provider to use your access key credentials in a `providers.tf ```tf provider "aws" { - version = "~> 2.11.0" + version = "~> 2.12.0" alias = "default" region = "eu-central-1" @@ -57,7 +57,7 @@ provider "aws" { } provider "ct" { - version = "0.3.1" + version = "0.3.2" } provider "local" { diff --git a/docs/cl/azure.md b/docs/cl/azure.md index 97d09d725..d25eea5cd 100644 --- a/docs/cl/azure.md +++ b/docs/cl/azure.md @@ -21,15 +21,15 @@ Install [Terraform](https://www.terraform.io/downloads.html) v0.11.x on your sys ```sh $ terraform version -Terraform v0.11.12 +Terraform v0.11.14 ``` Add the [terraform-provider-ct](https://github.com/poseidon/terraform-provider-ct) plugin binary for your system to `~/.terraform.d/plugins/`, noting the final name. ```sh -wget https://github.com/poseidon/terraform-provider-ct/releases/download/v0.3.1/terraform-provider-ct-v0.3.1-linux-amd64.tar.gz -tar xzf terraform-provider-ct-v0.3.1-linux-amd64.tar.gz -mv terraform-provider-ct-v0.3.1-linux-amd64/terraform-provider-ct ~/.terraform.d/plugins/terraform-provider-ct_v0.3.1 +wget https://github.com/poseidon/terraform-provider-ct/releases/download/v0.3.2/terraform-provider-ct-v0.3.2-linux-amd64.tar.gz +tar xzf terraform-provider-ct-v0.3.2-linux-amd64.tar.gz +mv terraform-provider-ct-v0.3.2-linux-amd64/terraform-provider-ct ~/.terraform.d/plugins/terraform-provider-ct_v0.3.2 ``` Read [concepts](/architecture/concepts/) to learn about Terraform, modules, and organizing resources. Change to your infrastructure repository (e.g. `infra`). @@ -50,12 +50,12 @@ Configure the Azure provider in a `providers.tf` file. ```tf provider "azurerm" { - version = "~> 1.27.1" + version = "~> 1.29.0" alias = "default" } provider "ct" { - version = "0.3.1" + version = "0.3.2" } provider "local" { diff --git a/docs/cl/bare-metal.md b/docs/cl/bare-metal.md index b60ecfc8d..17112a889 100644 --- a/docs/cl/bare-metal.md +++ b/docs/cl/bare-metal.md @@ -111,23 +111,23 @@ Install [Terraform](https://www.terraform.io/downloads.html) v0.11.x on your sys ```sh $ terraform version -Terraform v0.11.12 +Terraform v0.11.14 ``` Add the [terraform-provider-matchbox](https://github.com/poseidon/terraform-provider-matchbox) plugin binary for your system to `~/.terraform.d/plugins/`, noting the final name. ```sh -wget https://github.com/poseidon/terraform-provider-matchbox/releases/download/v0.2.3/terraform-provider-matchbox-v0.2.3-linux-amd64.tar.gz -tar xzf terraform-provider-matchbox-v0.2.3-linux-amd64.tar.gz -mv terraform-provider-matchbox-v0.2.3-linux-amd64/terraform-provider-matchbox ~/.terraform.d/plugins/terraform-provider-matchbox_v0.2.3 +wget https://github.com/poseidon/terraform-provider-matchbox/releases/download/v0.3.0/terraform-provider-matchbox-v0.3.0-linux-amd64.tar.gz +tar xzf terraform-provider-matchbox-v0.3.0-linux-amd64.tar.gz +mv terraform-provider-matchbox-v0.3.0-linux-amd64/terraform-provider-matchbox ~/.terraform.d/plugins/terraform-provider-matchbox_v0.3.0 ``` Add the [terraform-provider-ct](https://github.com/poseidon/terraform-provider-ct) plugin binary for your system to `~/.terraform.d/plugins/`, noting the final name. ```sh -wget https://github.com/poseidon/terraform-provider-ct/releases/download/v0.3.1/terraform-provider-ct-v0.3.1-linux-amd64.tar.gz -tar xzf terraform-provider-ct-v0.3.1-linux-amd64.tar.gz -mv terraform-provider-ct-v0.3.1-linux-amd64/terraform-provider-ct ~/.terraform.d/plugins/terraform-provider-ct_v0.3.1 +wget https://github.com/poseidon/terraform-provider-ct/releases/download/v0.3.2/terraform-provider-ct-v0.3.2-linux-amd64.tar.gz +tar xzf terraform-provider-ct-v0.3.2-linux-amd64.tar.gz +mv terraform-provider-ct-v0.3.2-linux-amd64/terraform-provider-ct ~/.terraform.d/plugins/terraform-provider-ct_v0.3.2 ``` Read [concepts](/architecture/concepts/) to learn about Terraform, modules, and organizing resources. Change to your infrastructure repository (e.g. `infra`). @@ -142,7 +142,7 @@ Configure the Matchbox provider to use your Matchbox API endpoint and client cer ```tf provider "matchbox" { - version = "0.2.3" + version = "0.3.0" endpoint = "matchbox.example.com:8081" client_cert = "${file("~/.config/matchbox/client.crt")}" client_key = "${file("~/.config/matchbox/client.key")}" @@ -150,7 +150,7 @@ provider "matchbox" { } provider "ct" { - version = "0.3.1" + version = "0.3.2" } provider "local" { diff --git a/docs/cl/digital-ocean.md b/docs/cl/digital-ocean.md index 0a738c5b1..5b88b6be8 100644 --- a/docs/cl/digital-ocean.md +++ b/docs/cl/digital-ocean.md @@ -18,15 +18,15 @@ Install [Terraform](https://www.terraform.io/downloads.html) v0.11.x on your sys ```sh $ terraform version -Terraform v0.11.12 +Terraform v0.11.14 ``` Add the [terraform-provider-ct](https://github.com/poseidon/terraform-provider-ct) plugin binary for your system to `~/.terraform.d/plugins/`, noting the final name. ```sh -wget https://github.com/poseidon/terraform-provider-ct/releases/download/v0.3.1/terraform-provider-ct-v0.3.1-linux-amd64.tar.gz -tar xzf terraform-provider-ct-v0.3.1-linux-amd64.tar.gz -mv terraform-provider-ct-v0.3.1-linux-amd64/terraform-provider-ct ~/.terraform.d/plugins/terraform-provider-ct_v0.3.1 +wget https://github.com/poseidon/terraform-provider-ct/releases/download/v0.3.2/terraform-provider-ct-v0.3.2-linux-amd64.tar.gz +tar xzf terraform-provider-ct-v0.3.2-linux-amd64.tar.gz +mv terraform-provider-ct-v0.3.2-linux-amd64/terraform-provider-ct ~/.terraform.d/plugins/terraform-provider-ct_v0.3.2 ``` Read [concepts](/architecture/concepts/) to learn about Terraform, modules, and organizing resources. Change to your infrastructure repository (e.g. `infra`). @@ -56,7 +56,7 @@ provider "digitalocean" { } provider "ct" { - version = "0.3.1" + version = "0.3.2" } provider "local" { diff --git a/docs/cl/google-cloud.md b/docs/cl/google-cloud.md index 7c188dd44..197c6a8e2 100644 --- a/docs/cl/google-cloud.md +++ b/docs/cl/google-cloud.md @@ -18,15 +18,15 @@ Install [Terraform](https://www.terraform.io/downloads.html) v0.11.x on your sys ```sh $ terraform version -Terraform v0.11.12 +Terraform v0.11.14 ``` Add the [terraform-provider-ct](https://github.com/poseidon/terraform-provider-ct) plugin binary for your system to `~/.terraform.d/plugins/`, noting the final name. ```sh -wget https://github.com/poseidon/terraform-provider-ct/releases/download/v0.3.1/terraform-provider-ct-v0.3.1-linux-amd64.tar.gz -tar xzf terraform-provider-ct-v0.3.1-linux-amd64.tar.gz -mv terraform-provider-ct-v0.3.1-linux-amd64/terraform-provider-ct ~/.terraform.d/plugins/terraform-provider-ct_v0.3.1 +wget https://github.com/poseidon/terraform-provider-ct/releases/download/v0.3.2/terraform-provider-ct-v0.3.2-linux-amd64.tar.gz +tar xzf terraform-provider-ct-v0.3.2-linux-amd64.tar.gz +mv terraform-provider-ct-v0.3.2-linux-amd64/terraform-provider-ct ~/.terraform.d/plugins/terraform-provider-ct_v0.3.2 ``` Read [concepts](/architecture/concepts/) to learn about Terraform, modules, and organizing resources. Change to your infrastructure repository (e.g. `infra`). @@ -49,7 +49,7 @@ Configure the Google Cloud provider to use your service account key, project-id, ```tf provider "google" { - version = "~> 2.5.1" + version = "~> 2.7.0" alias = "default" credentials = "${file("~/.config/google-cloud/terraform.json")}" @@ -58,7 +58,7 @@ provider "google" { } provider "ct" { - version = "0.3.1" + version = "0.3.2" } provider "local" { From 0ccb2217b546607af6f9755d6a4d8feb4bed262e Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Fri, 31 May 2019 01:08:32 -0700 Subject: [PATCH 117/523] Update Kubernetes from v1.14.2 to v1.14.3 * https://github.com/kubernetes/kubernetes/blob/master/CHANGELOG-1.14.md#v1143 --- CHANGES.md | 3 +++ README.md | 10 +++++----- aws/container-linux/kubernetes/README.md | 2 +- aws/container-linux/kubernetes/bootkube.tf | 2 +- .../kubernetes/cl/controller.yaml.tmpl | 2 +- .../kubernetes/workers/cl/worker.yaml.tmpl | 4 ++-- aws/fedora-atomic/kubernetes/README.md | 2 +- aws/fedora-atomic/kubernetes/bootkube.tf | 2 +- azure/container-linux/kubernetes/README.md | 2 +- azure/container-linux/kubernetes/bootkube.tf | 2 +- .../kubernetes/cl/controller.yaml.tmpl | 2 +- .../kubernetes/workers/cl/worker.yaml.tmpl | 4 ++-- bare-metal/container-linux/kubernetes/README.md | 2 +- .../container-linux/kubernetes/bootkube.tf | 2 +- .../kubernetes/cl/controller.yaml.tmpl | 2 +- .../kubernetes/cl/worker.yaml.tmpl | 2 +- bare-metal/fedora-atomic/kubernetes/README.md | 2 +- bare-metal/fedora-atomic/kubernetes/bootkube.tf | 2 +- .../container-linux/kubernetes/README.md | 2 +- .../container-linux/kubernetes/bootkube.tf | 2 +- .../kubernetes/cl/controller.yaml.tmpl | 2 +- .../kubernetes/cl/worker.yaml.tmpl | 4 ++-- digital-ocean/fedora-atomic/kubernetes/README.md | 2 +- .../fedora-atomic/kubernetes/bootkube.tf | 2 +- docs/advanced/worker-pools.md | 16 ++++++++-------- docs/atomic/aws.md | 10 +++++----- docs/atomic/bare-metal.md | 10 +++++----- docs/atomic/digital-ocean.md | 10 +++++----- docs/atomic/google-cloud.md | 10 +++++----- docs/cl/aws.md | 10 +++++----- docs/cl/azure.md | 10 +++++----- docs/cl/bare-metal.md | 14 +++++++------- docs/cl/digital-ocean.md | 10 +++++----- docs/cl/google-cloud.md | 10 +++++----- docs/index.md | 10 +++++----- docs/topics/maintenance.md | 2 +- .../container-linux/kubernetes/README.md | 2 +- .../container-linux/kubernetes/bootkube.tf | 2 +- .../kubernetes/cl/controller.yaml.tmpl | 2 +- .../kubernetes/workers/cl/worker.yaml.tmpl | 4 ++-- google-cloud/fedora-atomic/kubernetes/README.md | 2 +- .../fedora-atomic/kubernetes/bootkube.tf | 2 +- 42 files changed, 101 insertions(+), 98 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 68eb37ef2..d04763289 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -4,6 +4,9 @@ Notable changes between versions. ## Latest +## v1.14.3 + +* Kubernetes [v1.14.3](https://github.com/kubernetes/kubernetes/blob/master/CHANGELOG-1.14.md#v1143) * Update CoreDNS from v1.3.1 to v1.5.0 * Add `ready` plugin to improve readinessProbe * Fix trailing slash in terraform-render-bootkube version ([#479](https://github.com/poseidon/typhoon/pull/479)) diff --git a/README.md b/README.md index 569ac2c5b..3ba0e6e31 100644 --- a/README.md +++ b/README.md @@ -11,7 +11,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features -* Kubernetes v1.14.2 (upstream, via [kubernetes-incubator/bootkube](https://github.com/kubernetes-incubator/bootkube)) +* Kubernetes v1.14.3 (upstream, via [kubernetes-incubator/bootkube](https://github.com/kubernetes-incubator/bootkube)) * Single or multi-master, [Calico](https://www.projectcalico.org/) or [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/) * Advanced features like [worker pools](https://typhoon.psdn.io/advanced/worker-pools/), [preemptible](https://typhoon.psdn.io/cl/google-cloud/#preemption) workers, and [snippets](https://typhoon.psdn.io/advanced/customization/#container-linux) customization @@ -50,7 +50,7 @@ Define a Kubernetes cluster by using the Terraform module for your chosen platfo ```tf module "google-cloud-yavin" { - source = "git::https://github.com/poseidon/typhoon//google-cloud/container-linux/kubernetes?ref=v1.14.2" + source = "git::https://github.com/poseidon/typhoon//google-cloud/container-linux/kubernetes?ref=v1.14.3" providers = { google = "google.default" @@ -91,9 +91,9 @@ In 4-8 minutes (varies by platform), the cluster will be ready. This Google Clou $ export KUBECONFIG=/home/user/.secrets/clusters/yavin/auth/kubeconfig $ kubectl get nodes NAME ROLES STATUS AGE VERSION -yavin-controller-0.c.example-com.internal controller,master Ready 6m v1.14.2 -yavin-worker-jrbf.c.example-com.internal node Ready 5m v1.14.2 -yavin-worker-mzdm.c.example-com.internal node Ready 5m v1.14.2 +yavin-controller-0.c.example-com.internal controller,master Ready 6m v1.14.3 +yavin-worker-jrbf.c.example-com.internal node Ready 5m v1.14.3 +yavin-worker-mzdm.c.example-com.internal node Ready 5m v1.14.3 ``` List the pods. diff --git a/aws/container-linux/kubernetes/README.md b/aws/container-linux/kubernetes/README.md index 25911236e..e89e6c18e 100644 --- a/aws/container-linux/kubernetes/README.md +++ b/aws/container-linux/kubernetes/README.md @@ -11,7 +11,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features -* Kubernetes v1.14.2 (upstream, via [kubernetes-incubator/bootkube](https://github.com/kubernetes-incubator/bootkube)) +* Kubernetes v1.14.3 (upstream, via [kubernetes-incubator/bootkube](https://github.com/kubernetes-incubator/bootkube)) * Single or multi-master, [Calico](https://www.projectcalico.org/) or [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/) * Advanced features like [worker pools](https://typhoon.psdn.io/advanced/worker-pools/), [spot](https://typhoon.psdn.io/cl/aws/#spot) workers, and [snippets](https://typhoon.psdn.io/advanced/customization/#container-linux) customization diff --git a/aws/container-linux/kubernetes/bootkube.tf b/aws/container-linux/kubernetes/bootkube.tf index 7d11959b4..0bb6b24fc 100644 --- a/aws/container-linux/kubernetes/bootkube.tf +++ b/aws/container-linux/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=efd1cfd9bf2f56490c760e12c65f9d49ddddec43" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=082921d67905417755609eebda7d39a7e26f7fdb" cluster_name = "${var.cluster_name}" api_servers = ["${format("%s.%s", var.cluster_name, var.dns_zone)}"] diff --git a/aws/container-linux/kubernetes/cl/controller.yaml.tmpl b/aws/container-linux/kubernetes/cl/controller.yaml.tmpl index 9b92d48d2..e83b16753 100644 --- a/aws/container-linux/kubernetes/cl/controller.yaml.tmpl +++ b/aws/container-linux/kubernetes/cl/controller.yaml.tmpl @@ -123,7 +123,7 @@ storage: contents: inline: | KUBELET_IMAGE_URL=docker://k8s.gcr.io/hyperkube - KUBELET_IMAGE_TAG=v1.14.2 + KUBELET_IMAGE_TAG=v1.14.3 - path: /etc/sysctl.d/max-user-watches.conf filesystem: root contents: diff --git a/aws/container-linux/kubernetes/workers/cl/worker.yaml.tmpl b/aws/container-linux/kubernetes/workers/cl/worker.yaml.tmpl index 230533b6e..9ed6ebfb9 100644 --- a/aws/container-linux/kubernetes/workers/cl/worker.yaml.tmpl +++ b/aws/container-linux/kubernetes/workers/cl/worker.yaml.tmpl @@ -93,7 +93,7 @@ storage: contents: inline: | KUBELET_IMAGE_URL=docker://k8s.gcr.io/hyperkube - KUBELET_IMAGE_TAG=v1.14.2 + KUBELET_IMAGE_TAG=v1.14.3 - path: /etc/sysctl.d/max-user-watches.conf filesystem: root contents: @@ -111,7 +111,7 @@ storage: --volume config,kind=host,source=/etc/kubernetes \ --mount volume=config,target=/etc/kubernetes \ --insecure-options=image \ - docker://k8s.gcr.io/hyperkube:v1.14.2 \ + docker://k8s.gcr.io/hyperkube:v1.14.3 \ --net=host \ --dns=host \ --exec=/kubectl -- --kubeconfig=/etc/kubernetes/kubeconfig delete node $(hostname) diff --git a/aws/fedora-atomic/kubernetes/README.md b/aws/fedora-atomic/kubernetes/README.md index 94df4efa5..8b24e6527 100644 --- a/aws/fedora-atomic/kubernetes/README.md +++ b/aws/fedora-atomic/kubernetes/README.md @@ -11,7 +11,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features -* Kubernetes v1.14.2 (upstream, via [kubernetes-incubator/bootkube](https://github.com/kubernetes-incubator/bootkube)) +* Kubernetes v1.14.3 (upstream, via [kubernetes-incubator/bootkube](https://github.com/kubernetes-incubator/bootkube)) * Single or multi-master, [Calico](https://www.projectcalico.org/) or [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/) * Advanced features like [worker pools](https://typhoon.psdn.io/advanced/worker-pools/) and [spot](https://typhoon.psdn.io/cl/aws/#spot) workers diff --git a/aws/fedora-atomic/kubernetes/bootkube.tf b/aws/fedora-atomic/kubernetes/bootkube.tf index cf4e00315..5258e3a5a 100644 --- a/aws/fedora-atomic/kubernetes/bootkube.tf +++ b/aws/fedora-atomic/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=efd1cfd9bf2f56490c760e12c65f9d49ddddec43" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=082921d67905417755609eebda7d39a7e26f7fdb" cluster_name = "${var.cluster_name}" api_servers = ["${format("%s.%s", var.cluster_name, var.dns_zone)}"] diff --git a/azure/container-linux/kubernetes/README.md b/azure/container-linux/kubernetes/README.md index 91b61894f..ab7e06637 100644 --- a/azure/container-linux/kubernetes/README.md +++ b/azure/container-linux/kubernetes/README.md @@ -11,7 +11,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features -* Kubernetes v1.14.2 (upstream, via [kubernetes-incubator/bootkube](https://github.com/kubernetes-incubator/bootkube)) +* Kubernetes v1.14.3 (upstream, via [kubernetes-incubator/bootkube](https://github.com/kubernetes-incubator/bootkube)) * Single or multi-master, [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled * Advanced features like [worker pools](https://typhoon.psdn.io/advanced/worker-pools/), [low-priority](https://typhoon.psdn.io/cl/azure/#low-priority) workers, and [snippets](https://typhoon.psdn.io/advanced/customization/#container-linux) customization diff --git a/azure/container-linux/kubernetes/bootkube.tf b/azure/container-linux/kubernetes/bootkube.tf index ee50b7b9c..a27896349 100644 --- a/azure/container-linux/kubernetes/bootkube.tf +++ b/azure/container-linux/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=efd1cfd9bf2f56490c760e12c65f9d49ddddec43" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=082921d67905417755609eebda7d39a7e26f7fdb" cluster_name = "${var.cluster_name}" api_servers = ["${format("%s.%s", var.cluster_name, var.dns_zone)}"] diff --git a/azure/container-linux/kubernetes/cl/controller.yaml.tmpl b/azure/container-linux/kubernetes/cl/controller.yaml.tmpl index 9b92d48d2..e83b16753 100644 --- a/azure/container-linux/kubernetes/cl/controller.yaml.tmpl +++ b/azure/container-linux/kubernetes/cl/controller.yaml.tmpl @@ -123,7 +123,7 @@ storage: contents: inline: | KUBELET_IMAGE_URL=docker://k8s.gcr.io/hyperkube - KUBELET_IMAGE_TAG=v1.14.2 + KUBELET_IMAGE_TAG=v1.14.3 - path: /etc/sysctl.d/max-user-watches.conf filesystem: root contents: diff --git a/azure/container-linux/kubernetes/workers/cl/worker.yaml.tmpl b/azure/container-linux/kubernetes/workers/cl/worker.yaml.tmpl index c0e0155e6..d2a417141 100644 --- a/azure/container-linux/kubernetes/workers/cl/worker.yaml.tmpl +++ b/azure/container-linux/kubernetes/workers/cl/worker.yaml.tmpl @@ -93,7 +93,7 @@ storage: contents: inline: | KUBELET_IMAGE_URL=docker://k8s.gcr.io/hyperkube - KUBELET_IMAGE_TAG=v1.14.2 + KUBELET_IMAGE_TAG=v1.14.3 - path: /etc/sysctl.d/max-user-watches.conf filesystem: root contents: @@ -111,7 +111,7 @@ storage: --volume config,kind=host,source=/etc/kubernetes \ --mount volume=config,target=/etc/kubernetes \ --insecure-options=image \ - docker://k8s.gcr.io/hyperkube:v1.14.2 \ + docker://k8s.gcr.io/hyperkube:v1.14.3 \ --net=host \ --dns=host \ --exec=/kubectl -- --kubeconfig=/etc/kubernetes/kubeconfig delete node $(hostname | tr '[:upper:]' '[:lower:]') diff --git a/bare-metal/container-linux/kubernetes/README.md b/bare-metal/container-linux/kubernetes/README.md index 92205dbb0..231a21a58 100644 --- a/bare-metal/container-linux/kubernetes/README.md +++ b/bare-metal/container-linux/kubernetes/README.md @@ -11,7 +11,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features -* Kubernetes v1.14.2 (upstream, via [kubernetes-incubator/bootkube](https://github.com/kubernetes-incubator/bootkube)) +* Kubernetes v1.14.3 (upstream, via [kubernetes-incubator/bootkube](https://github.com/kubernetes-incubator/bootkube)) * Single or multi-master, [Calico](https://www.projectcalico.org/) or [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/) * Advanced features like [snippets](https://typhoon.psdn.io/advanced/customization/#container-linux) customization diff --git a/bare-metal/container-linux/kubernetes/bootkube.tf b/bare-metal/container-linux/kubernetes/bootkube.tf index ca4d9b6bb..fb82739a0 100644 --- a/bare-metal/container-linux/kubernetes/bootkube.tf +++ b/bare-metal/container-linux/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=efd1cfd9bf2f56490c760e12c65f9d49ddddec43" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=082921d67905417755609eebda7d39a7e26f7fdb" cluster_name = "${var.cluster_name}" api_servers = ["${var.k8s_domain_name}"] diff --git a/bare-metal/container-linux/kubernetes/cl/controller.yaml.tmpl b/bare-metal/container-linux/kubernetes/cl/controller.yaml.tmpl index 42cad1e9f..827ab5197 100644 --- a/bare-metal/container-linux/kubernetes/cl/controller.yaml.tmpl +++ b/bare-metal/container-linux/kubernetes/cl/controller.yaml.tmpl @@ -128,7 +128,7 @@ storage: contents: inline: | KUBELET_IMAGE_URL=docker://k8s.gcr.io/hyperkube - KUBELET_IMAGE_TAG=v1.14.2 + KUBELET_IMAGE_TAG=v1.14.3 - path: /etc/hostname filesystem: root mode: 0644 diff --git a/bare-metal/container-linux/kubernetes/cl/worker.yaml.tmpl b/bare-metal/container-linux/kubernetes/cl/worker.yaml.tmpl index 6f13ff834..c2be52683 100644 --- a/bare-metal/container-linux/kubernetes/cl/worker.yaml.tmpl +++ b/bare-metal/container-linux/kubernetes/cl/worker.yaml.tmpl @@ -89,7 +89,7 @@ storage: contents: inline: | KUBELET_IMAGE_URL=docker://k8s.gcr.io/hyperkube - KUBELET_IMAGE_TAG=v1.14.2 + KUBELET_IMAGE_TAG=v1.14.3 - path: /etc/hostname filesystem: root mode: 0644 diff --git a/bare-metal/fedora-atomic/kubernetes/README.md b/bare-metal/fedora-atomic/kubernetes/README.md index 69919066b..ae164f079 100644 --- a/bare-metal/fedora-atomic/kubernetes/README.md +++ b/bare-metal/fedora-atomic/kubernetes/README.md @@ -11,7 +11,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features -* Kubernetes v1.14.2 (upstream, via [kubernetes-incubator/bootkube](https://github.com/kubernetes-incubator/bootkube)) +* Kubernetes v1.14.3 (upstream, via [kubernetes-incubator/bootkube](https://github.com/kubernetes-incubator/bootkube)) * Single or multi-master, [Calico](https://www.projectcalico.org/) or [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/) * Ready for Ingress, Prometheus, Grafana, and other optional [addons](https://typhoon.psdn.io/addons/overview/) diff --git a/bare-metal/fedora-atomic/kubernetes/bootkube.tf b/bare-metal/fedora-atomic/kubernetes/bootkube.tf index baddb56cd..45e638acc 100644 --- a/bare-metal/fedora-atomic/kubernetes/bootkube.tf +++ b/bare-metal/fedora-atomic/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=efd1cfd9bf2f56490c760e12c65f9d49ddddec43" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=082921d67905417755609eebda7d39a7e26f7fdb" cluster_name = "${var.cluster_name}" api_servers = ["${var.k8s_domain_name}"] diff --git a/digital-ocean/container-linux/kubernetes/README.md b/digital-ocean/container-linux/kubernetes/README.md index e682d5ad5..26a10f393 100644 --- a/digital-ocean/container-linux/kubernetes/README.md +++ b/digital-ocean/container-linux/kubernetes/README.md @@ -11,7 +11,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features -* Kubernetes v1.14.2 (upstream, via [kubernetes-incubator/bootkube](https://github.com/kubernetes-incubator/bootkube)) +* Kubernetes v1.14.3 (upstream, via [kubernetes-incubator/bootkube](https://github.com/kubernetes-incubator/bootkube)) * Single or multi-master, [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled * Advanced features like [snippets](https://typhoon.psdn.io/advanced/customization/#container-linux) customization diff --git a/digital-ocean/container-linux/kubernetes/bootkube.tf b/digital-ocean/container-linux/kubernetes/bootkube.tf index 87fde3661..1ec7a1e0a 100644 --- a/digital-ocean/container-linux/kubernetes/bootkube.tf +++ b/digital-ocean/container-linux/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=efd1cfd9bf2f56490c760e12c65f9d49ddddec43" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=082921d67905417755609eebda7d39a7e26f7fdb" cluster_name = "${var.cluster_name}" api_servers = ["${format("%s.%s", var.cluster_name, var.dns_zone)}"] diff --git a/digital-ocean/container-linux/kubernetes/cl/controller.yaml.tmpl b/digital-ocean/container-linux/kubernetes/cl/controller.yaml.tmpl index 33ba69219..6c2ff1aaa 100644 --- a/digital-ocean/container-linux/kubernetes/cl/controller.yaml.tmpl +++ b/digital-ocean/container-linux/kubernetes/cl/controller.yaml.tmpl @@ -129,7 +129,7 @@ storage: contents: inline: | KUBELET_IMAGE_URL=docker://k8s.gcr.io/hyperkube - KUBELET_IMAGE_TAG=v1.14.2 + KUBELET_IMAGE_TAG=v1.14.3 - path: /etc/sysctl.d/max-user-watches.conf filesystem: root contents: diff --git a/digital-ocean/container-linux/kubernetes/cl/worker.yaml.tmpl b/digital-ocean/container-linux/kubernetes/cl/worker.yaml.tmpl index 11b4e553d..6200eaae3 100644 --- a/digital-ocean/container-linux/kubernetes/cl/worker.yaml.tmpl +++ b/digital-ocean/container-linux/kubernetes/cl/worker.yaml.tmpl @@ -99,7 +99,7 @@ storage: contents: inline: | KUBELET_IMAGE_URL=docker://k8s.gcr.io/hyperkube - KUBELET_IMAGE_TAG=v1.14.2 + KUBELET_IMAGE_TAG=v1.14.3 - path: /etc/sysctl.d/max-user-watches.conf filesystem: root contents: @@ -117,7 +117,7 @@ storage: --volume config,kind=host,source=/etc/kubernetes \ --mount volume=config,target=/etc/kubernetes \ --insecure-options=image \ - docker://k8s.gcr.io/hyperkube:v1.14.2 \ + docker://k8s.gcr.io/hyperkube:v1.14.3 \ --net=host \ --dns=host \ --exec=/kubectl -- --kubeconfig=/etc/kubernetes/kubeconfig delete node $(hostname) diff --git a/digital-ocean/fedora-atomic/kubernetes/README.md b/digital-ocean/fedora-atomic/kubernetes/README.md index 6203d59f9..2520d28d1 100644 --- a/digital-ocean/fedora-atomic/kubernetes/README.md +++ b/digital-ocean/fedora-atomic/kubernetes/README.md @@ -11,7 +11,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features -* Kubernetes v1.14.2 (upstream, via [kubernetes-incubator/bootkube](https://github.com/kubernetes-incubator/bootkube)) +* Kubernetes v1.14.3 (upstream, via [kubernetes-incubator/bootkube](https://github.com/kubernetes-incubator/bootkube)) * Single or multi-master, [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled * Ready for Ingress, Prometheus, Grafana, and other optional [addons](https://typhoon.psdn.io/addons/overview/) diff --git a/digital-ocean/fedora-atomic/kubernetes/bootkube.tf b/digital-ocean/fedora-atomic/kubernetes/bootkube.tf index 92d280da7..3b98d7e6b 100644 --- a/digital-ocean/fedora-atomic/kubernetes/bootkube.tf +++ b/digital-ocean/fedora-atomic/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=efd1cfd9bf2f56490c760e12c65f9d49ddddec43" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=082921d67905417755609eebda7d39a7e26f7fdb" cluster_name = "${var.cluster_name}" api_servers = ["${format("%s.%s", var.cluster_name, var.dns_zone)}"] diff --git a/docs/advanced/worker-pools.md b/docs/advanced/worker-pools.md index fe91e4831..e3ac59818 100644 --- a/docs/advanced/worker-pools.md +++ b/docs/advanced/worker-pools.md @@ -16,7 +16,7 @@ Create a cluster following the AWS [tutorial](../cl/aws.md#cluster). Define a wo ```tf module "tempest-worker-pool" { - source = "git::https://github.com/poseidon/typhoon//aws/container-linux/kubernetes/workers?ref=v1.14.2" + source = "git::https://github.com/poseidon/typhoon//aws/container-linux/kubernetes/workers?ref=v1.14.3" providers = { aws = "aws.default" @@ -82,7 +82,7 @@ Create a cluster following the Azure [tutorial](../cl/azure.md#cluster). Define ```tf module "ramius-worker-pool" { - source = "git::https://github.com/poseidon/typhoon//azure/container-linux/kubernetes/workers?ref=v1.14.2" + source = "git::https://github.com/poseidon/typhoon//azure/container-linux/kubernetes/workers?ref=v1.14.3" providers = { azurerm = "azurerm.default" @@ -152,7 +152,7 @@ Create a cluster following the Google Cloud [tutorial](../cl/google-cloud.md#clu ```tf module "yavin-worker-pool" { - source = "git::https://github.com/poseidon/typhoon//google-cloud/container-linux/kubernetes/workers?ref=v1.14.2" + source = "git::https://github.com/poseidon/typhoon//google-cloud/container-linux/kubernetes/workers?ref=v1.14.3" providers = { google = "google.default" @@ -187,11 +187,11 @@ Verify a managed instance group of workers joins the cluster within a few minute ``` $ kubectl get nodes NAME STATUS AGE VERSION -yavin-controller-0.c.example-com.internal Ready 6m v1.14.2 -yavin-worker-jrbf.c.example-com.internal Ready 5m v1.14.2 -yavin-worker-mzdm.c.example-com.internal Ready 5m v1.14.2 -yavin-16x-worker-jrbf.c.example-com.internal Ready 3m v1.14.2 -yavin-16x-worker-mzdm.c.example-com.internal Ready 3m v1.14.2 +yavin-controller-0.c.example-com.internal Ready 6m v1.14.3 +yavin-worker-jrbf.c.example-com.internal Ready 5m v1.14.3 +yavin-worker-mzdm.c.example-com.internal Ready 5m v1.14.3 +yavin-16x-worker-jrbf.c.example-com.internal Ready 3m v1.14.3 +yavin-16x-worker-mzdm.c.example-com.internal Ready 3m v1.14.3 ``` ### Variables diff --git a/docs/atomic/aws.md b/docs/atomic/aws.md index ffa57156d..334a26ca1 100644 --- a/docs/atomic/aws.md +++ b/docs/atomic/aws.md @@ -3,7 +3,7 @@ !!! danger Typhoon for Fedora Atomic will not be updated much beyond Kubernetes v1.13. -In this tutorial, we'll create a Kubernetes v1.14.2 cluster on AWS with Fedora Atomic. +In this tutorial, we'll create a Kubernetes v1.14.3 cluster on AWS with Fedora Atomic. We'll declare a Kubernetes cluster using the Typhoon Terraform module. Then apply the changes to create a VPC, gateway, subnets, security groups, controller instances, worker auto-scaling group, network load balancer, and TLS assets. Instances are provisioned on first boot with cloud-init. @@ -83,7 +83,7 @@ Define a Kubernetes cluster using the module `aws/fedora-atomic/kubernetes`. ```tf module "aws-tempest" { - source = "git::https://github.com/poseidon/typhoon//aws/fedora-atomic/kubernetes?ref=v1.14.2" + source = "git::https://github.com/poseidon/typhoon//aws/fedora-atomic/kubernetes?ref=v1.14.3" providers = { aws = "aws.default" @@ -156,9 +156,9 @@ In 5-10 minutes, the Kubernetes cluster will be ready. $ export KUBECONFIG=/home/user/.secrets/clusters/tempest/auth/kubeconfig $ kubectl get nodes NAME STATUS ROLES AGE VERSION -ip-10-0-3-155 Ready controller,master 10m v1.14.2 -ip-10-0-26-65 Ready node 10m v1.14.2 -ip-10-0-41-21 Ready node 10m v1.14.2 +ip-10-0-3-155 Ready controller,master 10m v1.14.3 +ip-10-0-26-65 Ready node 10m v1.14.3 +ip-10-0-41-21 Ready node 10m v1.14.3 ``` List the pods. diff --git a/docs/atomic/bare-metal.md b/docs/atomic/bare-metal.md index da2f6b597..d7b9424ae 100644 --- a/docs/atomic/bare-metal.md +++ b/docs/atomic/bare-metal.md @@ -3,7 +3,7 @@ !!! danger Typhoon for Fedora Atomic will not be updated much beyond Kubernetes v1.13. -In this tutorial, we'll network boot and provision a Kubernetes v1.14.2 cluster on bare-metal with Fedora Atomic. +In this tutorial, we'll network boot and provision a Kubernetes v1.14.3 cluster on bare-metal with Fedora Atomic. First, we'll deploy a [Matchbox](https://github.com/poseidon/matchbox) service and setup a network boot environment. Then, we'll declare a Kubernetes cluster using the Typhoon Terraform module and power on machines. On PXE boot, machines will install Fedora Atomic via kickstart, reboot into the disk install, and provision themselves as Kubernetes controllers or workers via cloud-init. @@ -228,7 +228,7 @@ Define a Kubernetes cluster using the module `bare-metal/fedora-atomic/kubernete ```tf module "bare-metal-mercury" { - source = "git::https://github.com/poseidon/typhoon//bare-metal/fedora-atomic/kubernetes?ref=v1.14.2" + source = "git::https://github.com/poseidon/typhoon//bare-metal/fedora-atomic/kubernetes?ref=v1.14.3" providers = { local = "local.default" @@ -354,9 +354,9 @@ bootkube[5]: Tearing down temporary bootstrap control plane... $ export KUBECONFIG=/home/user/.secrets/clusters/mercury/auth/kubeconfig $ kubectl get nodes NAME STATUS ROLES AGE VERSION -node1.example.com Ready controller,master 10m v1.14.2 -node2.example.com Ready node 10m v1.14.2 -node3.example.com Ready node 10m v1.14.2 +node1.example.com Ready controller,master 10m v1.14.3 +node2.example.com Ready node 10m v1.14.3 +node3.example.com Ready node 10m v1.14.3 ``` List the pods. diff --git a/docs/atomic/digital-ocean.md b/docs/atomic/digital-ocean.md index 01a7911a5..0a4111bb1 100644 --- a/docs/atomic/digital-ocean.md +++ b/docs/atomic/digital-ocean.md @@ -3,7 +3,7 @@ !!! danger Typhoon for Fedora Atomic will not be updated much beyond Kubernetes v1.13. -In this tutorial, we'll create a Kubernetes v1.14.2 cluster on DigitalOcean with Fedora Atomic. +In this tutorial, we'll create a Kubernetes v1.14.3 cluster on DigitalOcean with Fedora Atomic. We'll declare a Kubernetes cluster using the Typhoon Terraform module. Then apply the changes to create controller droplets, worker droplets, DNS records, tags, and TLS assets. Instances are provisioned on first boot with cloud-init. @@ -77,7 +77,7 @@ Define a Kubernetes cluster using the module `digital-ocean/fedora-atomic/kubern ```tf module "digital-ocean-nemo" { - source = "git::https://github.com/poseidon/typhoon//digital-ocean/fedora-atomic/kubernetes?ref=v1.14.2" + source = "git::https://github.com/poseidon/typhoon//digital-ocean/fedora-atomic/kubernetes?ref=v1.14.3" providers = { digitalocean = "digitalocean.default" @@ -152,9 +152,9 @@ In 3-6 minutes, the Kubernetes cluster will be ready. $ export KUBECONFIG=/home/user/.secrets/clusters/nemo/auth/kubeconfig $ kubectl get nodes NAME STATUS ROLES AGE VERSION -10.132.110.130 Ready controller,master 10m v1.14.2 -10.132.115.81 Ready node 10m v1.14.2 -10.132.124.107 Ready node 10m v1.14.2 +10.132.110.130 Ready controller,master 10m v1.14.3 +10.132.115.81 Ready node 10m v1.14.3 +10.132.124.107 Ready node 10m v1.14.3 ``` List the pods. diff --git a/docs/atomic/google-cloud.md b/docs/atomic/google-cloud.md index 7f62eb100..1e4ae76cb 100644 --- a/docs/atomic/google-cloud.md +++ b/docs/atomic/google-cloud.md @@ -3,7 +3,7 @@ !!! danger Typhoon for Fedora Atomic will not be updated much beyond Kubernetes v1.13. Fedora does not publish official images for Google Cloud so you must prepare them yourself. Expect rough edges and changes. -In this tutorial, we'll create a Kubernetes v1.14.2 cluster on Google Compute Engine with Fedora Atomic. +In this tutorial, we'll create a Kubernetes v1.14.3 cluster on Google Compute Engine with Fedora Atomic. We'll declare a Kubernetes cluster using the Typhoon Terraform module. Then apply the changes to create a network, firewall rules, health checks, controller instances, worker managed instance group, load balancers, and TLS assets. Instances are provisioned on first boot with cloud-init. @@ -121,7 +121,7 @@ Define a Kubernetes cluster using the module `google-cloud/fedora-atomic/kuberne ```tf module "google-cloud-yavin" { - source = "git::https://github.com/poseidon/typhoon//google-cloud/fedora-atomic/kubernetes?ref=v1.14.2" + source = "git::https://github.com/poseidon/typhoon//google-cloud/fedora-atomic/kubernetes?ref=v1.14.3" providers = { google = "google.default" @@ -197,9 +197,9 @@ In 5-10 minutes, the Kubernetes cluster will be ready. $ export KUBECONFIG=/home/user/.secrets/clusters/yavin/auth/kubeconfig $ kubectl get nodes NAME ROLES STATUS AGE VERSION -yavin-controller-0.c.example-com.internal controller,master Ready 6m v1.14.2 -yavin-worker-jrbf.c.example-com.internal node Ready 5m v1.14.2 -yavin-worker-mzdm.c.example-com.internal node Ready 5m v1.14.2 +yavin-controller-0.c.example-com.internal controller,master Ready 6m v1.14.3 +yavin-worker-jrbf.c.example-com.internal node Ready 5m v1.14.3 +yavin-worker-mzdm.c.example-com.internal node Ready 5m v1.14.3 ``` List the pods. diff --git a/docs/cl/aws.md b/docs/cl/aws.md index 77475a04f..abd33b680 100644 --- a/docs/cl/aws.md +++ b/docs/cl/aws.md @@ -1,6 +1,6 @@ # AWS -In this tutorial, we'll create a Kubernetes v1.14.2 cluster on AWS with Container Linux. +In this tutorial, we'll create a Kubernetes v1.14.3 cluster on AWS with Container Linux. We'll declare a Kubernetes cluster using the Typhoon Terraform module. Then apply the changes to create a VPC, gateway, subnets, security groups, controller instances, worker auto-scaling group, network load balancer, and TLS assets. @@ -92,7 +92,7 @@ Define a Kubernetes cluster using the module `aws/container-linux/kubernetes`. ```tf module "aws-tempest" { - source = "git::https://github.com/poseidon/typhoon//aws/container-linux/kubernetes?ref=v1.14.2" + source = "git::https://github.com/poseidon/typhoon//aws/container-linux/kubernetes?ref=v1.14.3" providers = { aws = "aws.default" @@ -165,9 +165,9 @@ In 4-8 minutes, the Kubernetes cluster will be ready. $ export KUBECONFIG=/home/user/.secrets/clusters/tempest/auth/kubeconfig $ kubectl get nodes NAME STATUS ROLES AGE VERSION -ip-10-0-3-155 Ready controller,master 10m v1.14.2 -ip-10-0-26-65 Ready node 10m v1.14.2 -ip-10-0-41-21 Ready node 10m v1.14.2 +ip-10-0-3-155 Ready controller,master 10m v1.14.3 +ip-10-0-26-65 Ready node 10m v1.14.3 +ip-10-0-41-21 Ready node 10m v1.14.3 ``` List the pods. diff --git a/docs/cl/azure.md b/docs/cl/azure.md index d25eea5cd..c55442158 100644 --- a/docs/cl/azure.md +++ b/docs/cl/azure.md @@ -3,7 +3,7 @@ !!! danger Typhoon for Azure is alpha. For production, use AWS, Google Cloud, or bare-metal. As Azure matures, check [errata](https://github.com/poseidon/typhoon/wiki/Errata) for known shortcomings. -In this tutorial, we'll create a Kubernetes v1.14.2 cluster on Azure with Container Linux. +In this tutorial, we'll create a Kubernetes v1.14.3 cluster on Azure with Container Linux. We'll declare a Kubernetes cluster using the Typhoon Terraform module. Then apply the changes to create a resource group, virtual network, subnets, security groups, controller availability set, worker scale set, load balancer, and TLS assets. @@ -87,7 +87,7 @@ Define a Kubernetes cluster using the module `azure/container-linux/kubernetes`. ```tf module "azure-ramius" { - source = "git::https://github.com/poseidon/typhoon//azure/container-linux/kubernetes?ref=v1.14.2" + source = "git::https://github.com/poseidon/typhoon//azure/container-linux/kubernetes?ref=v1.14.3" providers = { azurerm = "azurerm.default" @@ -161,9 +161,9 @@ In 4-8 minutes, the Kubernetes cluster will be ready. $ export KUBECONFIG=/home/user/.secrets/clusters/ramius/auth/kubeconfig $ kubectl get nodes NAME STATUS ROLES AGE VERSION -ramius-controller-0 Ready controller,master 24m v1.14.2 -ramius-worker-000001 Ready node 25m v1.14.2 -ramius-worker-000002 Ready node 24m v1.14.2 +ramius-controller-0 Ready controller,master 24m v1.14.3 +ramius-worker-000001 Ready node 25m v1.14.3 +ramius-worker-000002 Ready node 24m v1.14.3 ``` List the pods. diff --git a/docs/cl/bare-metal.md b/docs/cl/bare-metal.md index 17112a889..3832a4173 100644 --- a/docs/cl/bare-metal.md +++ b/docs/cl/bare-metal.md @@ -1,6 +1,6 @@ # Bare-Metal -In this tutorial, we'll network boot and provision a Kubernetes v1.14.2 cluster on bare-metal with Container Linux. +In this tutorial, we'll network boot and provision a Kubernetes v1.14.3 cluster on bare-metal with Container Linux. First, we'll deploy a [Matchbox](https://github.com/poseidon/matchbox) service and setup a network boot environment. Then, we'll declare a Kubernetes cluster using the Typhoon Terraform module and power on machines. On PXE boot, machines will install Container Linux to disk, reboot into the disk install, and provision themselves as Kubernetes controllers or workers via Ignition. @@ -180,7 +180,7 @@ Define a Kubernetes cluster using the module `bare-metal/container-linux/kuberne ```tf module "bare-metal-mercury" { - source = "git::https://github.com/poseidon/typhoon//bare-metal/container-linux/kubernetes?ref=v1.14.2" + source = "git::https://github.com/poseidon/typhoon//bare-metal/container-linux/kubernetes?ref=v1.14.3" providers = { local = "local.default" @@ -292,9 +292,9 @@ Apply complete! Resources: 55 added, 0 changed, 0 destroyed. To watch the install to disk (until machines reboot from disk), SSH to port 2222. ``` -# before v1.14.2 +# before v1.14.3 $ ssh debug@node1.example.com -# after v1.14.2 +# after v1.14.3 $ ssh -p 2222 core@node1.example.com ``` @@ -319,9 +319,9 @@ bootkube[5]: Tearing down temporary bootstrap control plane... $ export KUBECONFIG=/home/user/.secrets/clusters/mercury/auth/kubeconfig $ kubectl get nodes NAME STATUS ROLES AGE VERSION -node1.example.com Ready controller,master 10m v1.14.2 -node2.example.com Ready node 10m v1.14.2 -node3.example.com Ready node 10m v1.14.2 +node1.example.com Ready controller,master 10m v1.14.3 +node2.example.com Ready node 10m v1.14.3 +node3.example.com Ready node 10m v1.14.3 ``` List the pods. diff --git a/docs/cl/digital-ocean.md b/docs/cl/digital-ocean.md index 5b88b6be8..9e0e6bb8e 100644 --- a/docs/cl/digital-ocean.md +++ b/docs/cl/digital-ocean.md @@ -1,6 +1,6 @@ # Digital Ocean -In this tutorial, we'll create a Kubernetes v1.14.2 cluster on DigitalOcean with Container Linux. +In this tutorial, we'll create a Kubernetes v1.14.3 cluster on DigitalOcean with Container Linux. We'll declare a Kubernetes cluster using the Typhoon Terraform module. Then apply the changes to create controller droplets, worker droplets, DNS records, tags, and TLS assets. @@ -86,7 +86,7 @@ Define a Kubernetes cluster using the module `digital-ocean/container-linux/kube ```tf module "digital-ocean-nemo" { - source = "git::https://github.com/poseidon/typhoon//digital-ocean/container-linux/kubernetes?ref=v1.14.2" + source = "git::https://github.com/poseidon/typhoon//digital-ocean/container-linux/kubernetes?ref=v1.14.3" providers = { digitalocean = "digitalocean.default" @@ -160,9 +160,9 @@ In 3-6 minutes, the Kubernetes cluster will be ready. $ export KUBECONFIG=/home/user/.secrets/clusters/nemo/auth/kubeconfig $ kubectl get nodes NAME STATUS ROLES AGE VERSION -10.132.110.130 Ready controller,master 10m v1.14.2 -10.132.115.81 Ready node 10m v1.14.2 -10.132.124.107 Ready node 10m v1.14.2 +10.132.110.130 Ready controller,master 10m v1.14.3 +10.132.115.81 Ready node 10m v1.14.3 +10.132.124.107 Ready node 10m v1.14.3 ``` List the pods. diff --git a/docs/cl/google-cloud.md b/docs/cl/google-cloud.md index 197c6a8e2..b1331fa4f 100644 --- a/docs/cl/google-cloud.md +++ b/docs/cl/google-cloud.md @@ -1,6 +1,6 @@ # Google Cloud -In this tutorial, we'll create a Kubernetes v1.14.2 cluster on Google Compute Engine with Container Linux. +In this tutorial, we'll create a Kubernetes v1.14.3 cluster on Google Compute Engine with Container Linux. We'll declare a Kubernetes cluster using the Typhoon Terraform module. Then apply the changes to create a network, firewall rules, health checks, controller instances, worker managed instance group, load balancers, and TLS assets. @@ -93,7 +93,7 @@ Define a Kubernetes cluster using the module `google-cloud/container-linux/kuber ```tf module "google-cloud-yavin" { - source = "git::https://github.com/poseidon/typhoon//google-cloud/container-linux/kubernetes?ref=v1.14.2" + source = "git::https://github.com/poseidon/typhoon//google-cloud/container-linux/kubernetes?ref=v1.14.3" providers = { google = "google.default" @@ -168,9 +168,9 @@ In 4-8 minutes, the Kubernetes cluster will be ready. $ export KUBECONFIG=/home/user/.secrets/clusters/yavin/auth/kubeconfig $ kubectl get nodes NAME ROLES STATUS AGE VERSION -yavin-controller-0.c.example-com.internal controller,master Ready 6m v1.14.2 -yavin-worker-jrbf.c.example-com.internal node Ready 5m v1.14.2 -yavin-worker-mzdm.c.example-com.internal node Ready 5m v1.14.2 +yavin-controller-0.c.example-com.internal controller,master Ready 6m v1.14.3 +yavin-worker-jrbf.c.example-com.internal node Ready 5m v1.14.3 +yavin-worker-mzdm.c.example-com.internal node Ready 5m v1.14.3 ``` List the pods. diff --git a/docs/index.md b/docs/index.md index fa227594c..17aca9529 100644 --- a/docs/index.md +++ b/docs/index.md @@ -11,7 +11,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features -* Kubernetes v1.14.2 (upstream, via [kubernetes-incubator/bootkube](https://github.com/kubernetes-incubator/bootkube)) +* Kubernetes v1.14.3 (upstream, via [kubernetes-incubator/bootkube](https://github.com/kubernetes-incubator/bootkube)) * Single or multi-master, [Calico](https://www.projectcalico.org/) or [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/) * Advanced features like [worker pools](advanced/worker-pools/), [preemptible](cl/google-cloud/#preemption) workers, and [snippets](advanced/customization/#container-linux) customization @@ -49,7 +49,7 @@ Define a Kubernetes cluster by using the Terraform module for your chosen platfo ```tf module "google-cloud-yavin" { - source = "git::https://github.com/poseidon/typhoon//google-cloud/container-linux/kubernetes?ref=v1.14.2" + source = "git::https://github.com/poseidon/typhoon//google-cloud/container-linux/kubernetes?ref=v1.14.3" providers = { google = "google.default" @@ -90,9 +90,9 @@ In 4-8 minutes (varies by platform), the cluster will be ready. This Google Clou $ export KUBECONFIG=/home/user/.secrets/clusters/yavin/auth/kubeconfig $ kubectl get nodes NAME ROLES STATUS AGE VERSION -yavin-controller-0.c.example-com.internal controller,master Ready 6m v1.14.2 -yavin-worker-jrbf.c.example-com.internal node Ready 5m v1.14.2 -yavin-worker-mzdm.c.example-com.internal node Ready 5m v1.14.2 +yavin-controller-0.c.example-com.internal controller,master Ready 6m v1.14.3 +yavin-worker-jrbf.c.example-com.internal node Ready 5m v1.14.3 +yavin-worker-mzdm.c.example-com.internal node Ready 5m v1.14.3 ``` List the pods. diff --git a/docs/topics/maintenance.md b/docs/topics/maintenance.md index c3f794495..ed99bd9cb 100644 --- a/docs/topics/maintenance.md +++ b/docs/topics/maintenance.md @@ -18,7 +18,7 @@ module "google-cloud-yavin" { } module "bare-metal-mercury" { - source = "git::https://github.com/poseidon/typhoon//bare-metal/container-linux/kubernetes?ref=v1.14.2" + source = "git::https://github.com/poseidon/typhoon//bare-metal/container-linux/kubernetes?ref=v1.14.3" ... } ``` diff --git a/google-cloud/container-linux/kubernetes/README.md b/google-cloud/container-linux/kubernetes/README.md index 5f84486d5..4c42ed351 100644 --- a/google-cloud/container-linux/kubernetes/README.md +++ b/google-cloud/container-linux/kubernetes/README.md @@ -11,7 +11,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features -* Kubernetes v1.14.2 (upstream, via [kubernetes-incubator/bootkube](https://github.com/kubernetes-incubator/bootkube)) +* Kubernetes v1.14.3 (upstream, via [kubernetes-incubator/bootkube](https://github.com/kubernetes-incubator/bootkube)) * Single or multi-master, [Calico](https://www.projectcalico.org/) or [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/) * Advanced features like [worker pools](https://typhoon.psdn.io/advanced/worker-pools/), [preemptible](https://typhoon.psdn.io/cl/google-cloud/#preemption) workers, and [snippets](https://typhoon.psdn.io/advanced/customization/#container-linux) customization diff --git a/google-cloud/container-linux/kubernetes/bootkube.tf b/google-cloud/container-linux/kubernetes/bootkube.tf index 21d96211a..d9872305c 100644 --- a/google-cloud/container-linux/kubernetes/bootkube.tf +++ b/google-cloud/container-linux/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=efd1cfd9bf2f56490c760e12c65f9d49ddddec43" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=082921d67905417755609eebda7d39a7e26f7fdb" cluster_name = "${var.cluster_name}" api_servers = ["${format("%s.%s", var.cluster_name, var.dns_zone)}"] diff --git a/google-cloud/container-linux/kubernetes/cl/controller.yaml.tmpl b/google-cloud/container-linux/kubernetes/cl/controller.yaml.tmpl index 53e3d229b..a2cca2433 100644 --- a/google-cloud/container-linux/kubernetes/cl/controller.yaml.tmpl +++ b/google-cloud/container-linux/kubernetes/cl/controller.yaml.tmpl @@ -124,7 +124,7 @@ storage: contents: inline: | KUBELET_IMAGE_URL=docker://k8s.gcr.io/hyperkube - KUBELET_IMAGE_TAG=v1.14.2 + KUBELET_IMAGE_TAG=v1.14.3 - path: /etc/sysctl.d/max-user-watches.conf filesystem: root contents: diff --git a/google-cloud/container-linux/kubernetes/workers/cl/worker.yaml.tmpl b/google-cloud/container-linux/kubernetes/workers/cl/worker.yaml.tmpl index 38705ea8c..7273df5d4 100644 --- a/google-cloud/container-linux/kubernetes/workers/cl/worker.yaml.tmpl +++ b/google-cloud/container-linux/kubernetes/workers/cl/worker.yaml.tmpl @@ -94,7 +94,7 @@ storage: contents: inline: | KUBELET_IMAGE_URL=docker://k8s.gcr.io/hyperkube - KUBELET_IMAGE_TAG=v1.14.2 + KUBELET_IMAGE_TAG=v1.14.3 - path: /etc/sysctl.d/max-user-watches.conf filesystem: root contents: @@ -112,7 +112,7 @@ storage: --volume config,kind=host,source=/etc/kubernetes \ --mount volume=config,target=/etc/kubernetes \ --insecure-options=image \ - docker://k8s.gcr.io/hyperkube:v1.14.2 \ + docker://k8s.gcr.io/hyperkube:v1.14.3 \ --net=host \ --dns=host \ --exec=/kubectl -- --kubeconfig=/etc/kubernetes/kubeconfig delete node $(hostname) diff --git a/google-cloud/fedora-atomic/kubernetes/README.md b/google-cloud/fedora-atomic/kubernetes/README.md index a788d0803..d225483b1 100644 --- a/google-cloud/fedora-atomic/kubernetes/README.md +++ b/google-cloud/fedora-atomic/kubernetes/README.md @@ -11,7 +11,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features -* Kubernetes v1.14.2 (upstream, via [kubernetes-incubator/bootkube](https://github.com/kubernetes-incubator/bootkube)) +* Kubernetes v1.14.3 (upstream, via [kubernetes-incubator/bootkube](https://github.com/kubernetes-incubator/bootkube)) * Single or multi-master, [Calico](https://www.projectcalico.org/) or [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/) * Advanced features like [worker pools](https://typhoon.psdn.io/advanced/worker-pools/) and [preemptible](https://typhoon.psdn.io/cl/google-cloud/#preemption) workers diff --git a/google-cloud/fedora-atomic/kubernetes/bootkube.tf b/google-cloud/fedora-atomic/kubernetes/bootkube.tf index 1e74b4971..86acda556 100644 --- a/google-cloud/fedora-atomic/kubernetes/bootkube.tf +++ b/google-cloud/fedora-atomic/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=efd1cfd9bf2f56490c760e12c65f9d49ddddec43" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=082921d67905417755609eebda7d39a7e26f7fdb" cluster_name = "${var.cluster_name}" api_servers = ["${format("%s.%s", var.cluster_name, var.dns_zone)}"] From 1366ae404b35b17e6c8eb6230e13f2702fcc39ad Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Mon, 27 May 2019 15:37:36 -0700 Subject: [PATCH 118/523] Migrate DigitalOcean module from Terraform v0.11 to v0.12 * Replace v0.11 bracket type hints with Terraform v0.12 list expressions * Use expression syntax instead of interpolated strings, where suggested * Update DigitalOcean tutorial documentation * Define Terraform and plugin version requirements in versions.tf * Require digitalocean ~> v1.3 to support Terraform v0.12 * Require ct ~> v0.3.2 to support Terraform v0.12 --- CHANGES.md | 6 + .../container-linux/kubernetes/bootkube.tf | 23 +-- .../container-linux/kubernetes/controllers.tf | 58 ++++--- .../container-linux/kubernetes/network.tf | 150 +++++++++--------- .../container-linux/kubernetes/outputs.tf | 19 +-- .../container-linux/kubernetes/require.tf | 25 --- .../container-linux/kubernetes/ssh.tf | 39 ++--- .../container-linux/kubernetes/variables.tf | 44 ++--- .../container-linux/kubernetes/versions.tf | 12 ++ .../container-linux/kubernetes/workers.tf | 37 ++--- docs/cl/digital-ocean.md | 39 +---- 11 files changed, 211 insertions(+), 241 deletions(-) delete mode 100644 digital-ocean/container-linux/kubernetes/require.tf create mode 100644 digital-ocean/container-linux/kubernetes/versions.tf diff --git a/CHANGES.md b/CHANGES.md index d04763289..6bb34544a 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -2,6 +2,12 @@ Notable changes between versions. +#### DigitalOcean + +* Migrate from Terraform v0.11 to v0.12.x (**action required!**) + * Require `terraform-provider-digitalocean` v1.3+ to support Terraform v0.12 + * Require `terraform-provider-ct` ~> v0.3.2+ to support Terraform v0.12 + ## Latest ## v1.14.3 diff --git a/digital-ocean/container-linux/kubernetes/bootkube.tf b/digital-ocean/container-linux/kubernetes/bootkube.tf index 1ec7a1e0a..31ffe924d 100644 --- a/digital-ocean/container-linux/kubernetes/bootkube.tf +++ b/digital-ocean/container-linux/kubernetes/bootkube.tf @@ -1,21 +1,22 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=082921d67905417755609eebda7d39a7e26f7fdb" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=0103bc06bb3f597455a765bf5d916f9b241cbea0" - cluster_name = "${var.cluster_name}" - api_servers = ["${format("%s.%s", var.cluster_name, var.dns_zone)}"] - etcd_servers = "${digitalocean_record.etcds.*.fqdn}" - asset_dir = "${var.asset_dir}" + cluster_name = var.cluster_name + api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] + etcd_servers = digitalocean_record.etcds.*.fqdn + asset_dir = var.asset_dir - networking = "${var.networking}" + networking = var.networking # only effective with Calico networking network_encapsulation = "vxlan" network_mtu = "1450" - pod_cidr = "${var.pod_cidr}" - service_cidr = "${var.service_cidr}" - cluster_domain_suffix = "${var.cluster_domain_suffix}" - enable_reporting = "${var.enable_reporting}" - enable_aggregation = "${var.enable_aggregation}" + pod_cidr = var.pod_cidr + service_cidr = var.service_cidr + cluster_domain_suffix = var.cluster_domain_suffix + enable_reporting = var.enable_reporting + enable_aggregation = var.enable_aggregation } + diff --git a/digital-ocean/container-linux/kubernetes/controllers.tf b/digital-ocean/container-linux/kubernetes/controllers.tf index 1f8f15c79..58458b4ba 100644 --- a/digital-ocean/container-linux/kubernetes/controllers.tf +++ b/digital-ocean/container-linux/kubernetes/controllers.tf @@ -1,25 +1,25 @@ # Controller Instance DNS records resource "digitalocean_record" "controllers" { - count = "${var.controller_count}" + count = var.controller_count # DNS zone where record should be created - domain = "${var.dns_zone}" + domain = var.dns_zone # DNS record (will be prepended to domain) - name = "${var.cluster_name}" + name = var.cluster_name type = "A" ttl = 300 # IPv4 addresses of controllers - value = "${element(digitalocean_droplet.controllers.*.ipv4_address, count.index)}" + value = element(digitalocean_droplet.controllers.*.ipv4_address, count.index) } # Discrete DNS records for each controller's private IPv4 for etcd usage resource "digitalocean_record" "etcds" { - count = "${var.controller_count}" + count = var.controller_count # DNS zone where record should be created - domain = "${var.dns_zone}" + domain = var.dns_zone # DNS record (will be prepended to domain) name = "${var.cluster_name}-etcd${count.index}" @@ -27,34 +27,32 @@ resource "digitalocean_record" "etcds" { ttl = 300 # private IPv4 address for etcd - value = "${element(digitalocean_droplet.controllers.*.ipv4_address_private, count.index)}" + value = element(digitalocean_droplet.controllers.*.ipv4_address_private, count.index) } # Controller droplet instances resource "digitalocean_droplet" "controllers" { - count = "${var.controller_count}" + count = var.controller_count name = "${var.cluster_name}-controller-${count.index}" - region = "${var.region}" + region = var.region - image = "${var.image}" - size = "${var.controller_type}" + image = var.image + size = var.controller_type # network ipv6 = true private_networking = true - user_data = "${element(data.ct_config.controller-ignitions.*.rendered, count.index)}" - ssh_keys = ["${var.ssh_fingerprints}"] + user_data = element(data.ct_config.controller-ignitions.*.rendered, count.index) + ssh_keys = var.ssh_fingerprints tags = [ - "${digitalocean_tag.controllers.id}", + digitalocean_tag.controllers.id, ] lifecycle { - ignore_changes = [ - "user_data", - ] + ignore_changes = [user_data] } } @@ -65,37 +63,37 @@ resource "digitalocean_tag" "controllers" { # Controller Ignition configs data "ct_config" "controller-ignitions" { - count = "${var.controller_count}" - content = "${element(data.template_file.controller-configs.*.rendered, count.index)}" + count = var.controller_count + content = element(data.template_file.controller-configs.*.rendered, count.index) pretty_print = false - snippets = ["${var.controller_clc_snippets}"] + snippets = var.controller_clc_snippets } # Controller Container Linux configs data "template_file" "controller-configs" { - count = "${var.controller_count}" + count = var.controller_count - template = "${file("${path.module}/cl/controller.yaml.tmpl")}" + template = file("${path.module}/cl/controller.yaml.tmpl") vars = { # Cannot use cyclic dependencies on controllers or their DNS records etcd_name = "etcd${count.index}" etcd_domain = "${var.cluster_name}-etcd${count.index}.${var.dns_zone}" - # etcd0=https://cluster-etcd0.example.com,etcd1=https://cluster-etcd1.example.com,... - etcd_initial_cluster = "${join(",", data.template_file.etcds.*.rendered)}" - cluster_dns_service_ip = "${cidrhost(var.service_cidr, 10)}" - cluster_domain_suffix = "${var.cluster_domain_suffix}" + etcd_initial_cluster = join(",", data.template_file.etcds.*.rendered) + cluster_dns_service_ip = cidrhost(var.service_cidr, 10) + cluster_domain_suffix = var.cluster_domain_suffix } } data "template_file" "etcds" { - count = "${var.controller_count}" + count = var.controller_count template = "etcd$${index}=https://$${cluster_name}-etcd$${index}.$${dns_zone}:2380" vars = { - index = "${count.index}" - cluster_name = "${var.cluster_name}" - dns_zone = "${var.dns_zone}" + index = count.index + cluster_name = var.cluster_name + dns_zone = var.dns_zone } } + diff --git a/digital-ocean/container-linux/kubernetes/network.tf b/digital-ocean/container-linux/kubernetes/network.tf index 76342f66c..b145cdf30 100644 --- a/digital-ocean/container-linux/kubernetes/network.tf +++ b/digital-ocean/container-linux/kubernetes/network.tf @@ -1,50 +1,51 @@ resource "digitalocean_firewall" "rules" { - name = "${var.cluster_name}" + name = var.cluster_name tags = ["${var.cluster_name}-controller", "${var.cluster_name}-worker"] # allow ssh, internal flannel, internal node-exporter, internal kubelet - inbound_rule = [ - { - protocol = "tcp" - port_range = "22" - source_addresses = ["0.0.0.0/0", "::/0"] - }, - { - protocol = "udp" - port_range = "4789" - source_tags = ["${digitalocean_tag.controllers.name}", "${digitalocean_tag.workers.name}"] - }, - { - protocol = "tcp" - port_range = "9100" - source_tags = ["${digitalocean_tag.workers.name}"] - }, - { - protocol = "tcp" - port_range = "10250" - source_tags = ["${digitalocean_tag.controllers.name}", "${digitalocean_tag.workers.name}"] - }, - ] + inbound_rule { + protocol = "tcp" + port_range = "22" + source_addresses = ["0.0.0.0/0", "::/0"] + } + + inbound_rule { + protocol = "udp" + port_range = "4789" + source_tags = [digitalocean_tag.controllers.name, digitalocean_tag.workers.name] + } + + inbound_rule { + protocol = "tcp" + port_range = "9100" + source_tags = [digitalocean_tag.workers.name] + } + + inbound_rule { + protocol = "tcp" + port_range = "10250" + source_tags = [digitalocean_tag.controllers.name, digitalocean_tag.workers.name] + } # allow all outbound traffic - outbound_rule = [ - { - protocol = "tcp" - port_range = "1-65535" - destination_addresses = ["0.0.0.0/0", "::/0"] - }, - { - protocol = "udp" - port_range = "1-65535" - destination_addresses = ["0.0.0.0/0", "::/0"] - }, - { - protocol = "icmp" - port_range = "1-65535" - destination_addresses = ["0.0.0.0/0", "::/0"] - }, - ] + outbound_rule { + protocol = "tcp" + port_range = "1-65535" + destination_addresses = ["0.0.0.0/0", "::/0"] + } + + outbound_rule { + protocol = "udp" + port_range = "1-65535" + destination_addresses = ["0.0.0.0/0", "::/0"] + } + + outbound_rule { + protocol = "icmp" + port_range = "1-65535" + destination_addresses = ["0.0.0.0/0", "::/0"] + } } resource "digitalocean_firewall" "controllers" { @@ -53,23 +54,23 @@ resource "digitalocean_firewall" "controllers" { tags = ["${var.cluster_name}-controller"] # etcd, kube-apiserver, kubelet - inbound_rule = [ - { - protocol = "tcp" - port_range = "2379-2380" - source_tags = ["${digitalocean_tag.controllers.name}"] - }, - { - protocol = "tcp" - port_range = "2381" - source_tags = ["${digitalocean_tag.workers.name}"] - }, - { - protocol = "tcp" - port_range = "6443" - source_addresses = ["0.0.0.0/0", "::/0"] - }, - ] + inbound_rule { + protocol = "tcp" + port_range = "2379-2380" + source_tags = [digitalocean_tag.controllers.name] + } + + inbound_rule { + protocol = "tcp" + port_range = "2381" + source_tags = [digitalocean_tag.workers.name] + } + + inbound_rule { + protocol = "tcp" + port_range = "6443" + source_addresses = ["0.0.0.0/0", "::/0"] + } } resource "digitalocean_firewall" "workers" { @@ -78,21 +79,22 @@ resource "digitalocean_firewall" "workers" { tags = ["${var.cluster_name}-worker"] # allow HTTP/HTTPS ingress - inbound_rule = [ - { - protocol = "tcp" - port_range = "80" - source_addresses = ["0.0.0.0/0", "::/0"] - }, - { - protocol = "tcp" - port_range = "443" - source_addresses = ["0.0.0.0/0", "::/0"] - }, - { - protocol = "tcp" - port_range = "10254" - source_addresses = ["0.0.0.0/0"] - }, - ] + inbound_rule { + protocol = "tcp" + port_range = "80" + source_addresses = ["0.0.0.0/0", "::/0"] + } + + inbound_rule { + protocol = "tcp" + port_range = "443" + source_addresses = ["0.0.0.0/0", "::/0"] + } + + inbound_rule { + protocol = "tcp" + port_range = "10254" + source_addresses = ["0.0.0.0/0"] + } } + diff --git a/digital-ocean/container-linux/kubernetes/outputs.tf b/digital-ocean/container-linux/kubernetes/outputs.tf index 15172d7f5..bf05b4a0d 100644 --- a/digital-ocean/container-linux/kubernetes/outputs.tf +++ b/digital-ocean/container-linux/kubernetes/outputs.tf @@ -1,40 +1,41 @@ output "kubeconfig-admin" { - value = "${module.bootkube.kubeconfig-admin}" + value = module.bootkube.kubeconfig-admin } output "controllers_dns" { - value = "${digitalocean_record.controllers.0.fqdn}" + value = digitalocean_record.controllers[0].fqdn } output "workers_dns" { # Multiple A and AAAA records with the same FQDN - value = "${digitalocean_record.workers-record-a.0.fqdn}" + value = digitalocean_record.workers-record-a[0].fqdn } output "controllers_ipv4" { - value = ["${digitalocean_droplet.controllers.*.ipv4_address}"] + value = [digitalocean_droplet.controllers.*.ipv4_address] } output "controllers_ipv6" { - value = ["${digitalocean_droplet.controllers.*.ipv6_address}"] + value = [digitalocean_droplet.controllers.*.ipv6_address] } output "workers_ipv4" { - value = ["${digitalocean_droplet.workers.*.ipv4_address}"] + value = [digitalocean_droplet.workers.*.ipv4_address] } output "workers_ipv6" { - value = ["${digitalocean_droplet.workers.*.ipv6_address}"] + value = [digitalocean_droplet.workers.*.ipv6_address] } # Outputs for custom firewalls output "controller_tag" { description = "Tag applied to controller droplets" - value = "${digitalocean_tag.controllers.name}" + value = digitalocean_tag.controllers.name } output "worker_tag" { description = "Tag applied to worker droplets" - value = "${digitalocean_tag.workers.name}" + value = digitalocean_tag.workers.name } + diff --git a/digital-ocean/container-linux/kubernetes/require.tf b/digital-ocean/container-linux/kubernetes/require.tf deleted file mode 100644 index 4651337d7..000000000 --- a/digital-ocean/container-linux/kubernetes/require.tf +++ /dev/null @@ -1,25 +0,0 @@ -# Terraform version and plugin versions - -terraform { - required_version = ">= 0.11.0" -} - -provider "digitalocean" { - version = "~> 1.0" -} - -provider "local" { - version = "~> 1.0" -} - -provider "null" { - version = "~> 1.0" -} - -provider "template" { - version = "~> 1.0" -} - -provider "tls" { - version = "~> 1.0" -} diff --git a/digital-ocean/container-linux/kubernetes/ssh.tf b/digital-ocean/container-linux/kubernetes/ssh.tf index 1816f5649..cc9385d27 100644 --- a/digital-ocean/container-linux/kubernetes/ssh.tf +++ b/digital-ocean/container-linux/kubernetes/ssh.tf @@ -1,55 +1,55 @@ # Secure copy etcd TLS assets and kubeconfig to controllers. Activates kubelet.service resource "null_resource" "copy-controller-secrets" { - count = "${var.controller_count}" + count = var.controller_count depends_on = [ - "digitalocean_firewall.rules", + digitalocean_firewall.rules ] connection { type = "ssh" - host = "${element(concat(digitalocean_droplet.controllers.*.ipv4_address), count.index)}" + host = element(digitalocean_droplet.controllers.*.ipv4_address, count.index) user = "core" timeout = "15m" } provisioner "file" { - content = "${module.bootkube.kubeconfig-kubelet}" + content = module.bootkube.kubeconfig-kubelet destination = "$HOME/kubeconfig" } provisioner "file" { - content = "${module.bootkube.etcd_ca_cert}" + content = module.bootkube.etcd_ca_cert destination = "$HOME/etcd-client-ca.crt" } provisioner "file" { - content = "${module.bootkube.etcd_client_cert}" + content = module.bootkube.etcd_client_cert destination = "$HOME/etcd-client.crt" } provisioner "file" { - content = "${module.bootkube.etcd_client_key}" + content = module.bootkube.etcd_client_key destination = "$HOME/etcd-client.key" } provisioner "file" { - content = "${module.bootkube.etcd_server_cert}" + content = module.bootkube.etcd_server_cert destination = "$HOME/etcd-server.crt" } provisioner "file" { - content = "${module.bootkube.etcd_server_key}" + content = module.bootkube.etcd_server_key destination = "$HOME/etcd-server.key" } provisioner "file" { - content = "${module.bootkube.etcd_peer_cert}" + content = module.bootkube.etcd_peer_cert destination = "$HOME/etcd-peer.crt" } provisioner "file" { - content = "${module.bootkube.etcd_peer_key}" + content = module.bootkube.etcd_peer_key destination = "$HOME/etcd-peer.key" } @@ -72,17 +72,17 @@ resource "null_resource" "copy-controller-secrets" { # Secure copy kubeconfig to all workers. Activates kubelet.service. resource "null_resource" "copy-worker-secrets" { - count = "${var.worker_count}" + count = var.worker_count connection { type = "ssh" - host = "${element(concat(digitalocean_droplet.workers.*.ipv4_address), count.index)}" + host = element(digitalocean_droplet.workers.*.ipv4_address, count.index) user = "core" timeout = "15m" } provisioner "file" { - content = "${module.bootkube.kubeconfig-kubelet}" + content = module.bootkube.kubeconfig-kubelet destination = "$HOME/kubeconfig" } @@ -97,20 +97,20 @@ resource "null_resource" "copy-worker-secrets" { # one-time self-hosted cluster bootstrapping. resource "null_resource" "bootkube-start" { depends_on = [ - "module.bootkube", - "null_resource.copy-controller-secrets", - "null_resource.copy-worker-secrets", + module.bootkube, + null_resource.copy-controller-secrets, + null_resource.copy-worker-secrets, ] connection { type = "ssh" - host = "${digitalocean_droplet.controllers.0.ipv4_address}" + host = digitalocean_droplet.controllers[0].ipv4_address user = "core" timeout = "15m" } provisioner "file" { - source = "${var.asset_dir}" + source = var.asset_dir destination = "$HOME/assets" } @@ -121,3 +121,4 @@ resource "null_resource" "bootkube-start" { ] } } + diff --git a/digital-ocean/container-linux/kubernetes/variables.tf b/digital-ocean/container-linux/kubernetes/variables.tf index bd8b41950..e1940120c 100644 --- a/digital-ocean/container-linux/kubernetes/variables.tf +++ b/digital-ocean/container-linux/kubernetes/variables.tf @@ -1,60 +1,60 @@ variable "cluster_name" { - type = "string" + type = string description = "Unique cluster name (prepended to dns_zone)" } # Digital Ocean variable "region" { - type = "string" + type = string description = "Digital Ocean region (e.g. nyc1, sfo2, fra1, tor1)" } variable "dns_zone" { - type = "string" + type = string description = "Digital Ocean domain (i.e. DNS zone) (e.g. do.example.com)" } # instances variable "controller_count" { - type = "string" + type = string default = "1" description = "Number of controllers (i.e. masters)" } variable "worker_count" { - type = "string" + type = string default = "1" description = "Number of workers" } variable "controller_type" { - type = "string" + type = string default = "s-2vcpu-2gb" description = "Droplet type for controllers (e.g. s-2vcpu-2gb, s-2vcpu-4gb, s-4vcpu-8gb)." } variable "worker_type" { - type = "string" + type = string default = "s-1vcpu-1gb" description = "Droplet type for workers (e.g. s-1vcpu-1gb, s-1vcpu-2gb, s-2vcpu-2gb)" } variable "image" { - type = "string" + type = string default = "coreos-stable" description = "Container Linux image for instances (e.g. coreos-stable)" } variable "controller_clc_snippets" { - type = "list" + type = list(string) description = "Controller Container Linux Config snippets" default = [] } variable "worker_clc_snippets" { - type = "list" + type = list(string) description = "Worker Container Linux Config snippets" default = [] } @@ -62,24 +62,24 @@ variable "worker_clc_snippets" { # configuration variable "ssh_fingerprints" { - type = "list" + type = list(string) description = "SSH public key fingerprints. (e.g. see `ssh-add -l -E md5`)" } variable "asset_dir" { description = "Path to a directory where generated assets should be placed (contains secrets)" - type = "string" + type = string } variable "networking" { description = "Choice of networking provider (flannel or calico)" - type = "string" + type = string default = "flannel" } variable "pod_cidr" { description = "CIDR IPv4 range to assign Kubernetes pods" - type = "string" + type = string default = "10.2.0.0/16" } @@ -89,24 +89,26 @@ CIDR IPv4 range to assign Kubernetes services. The 1st IP will be reserved for kube_apiserver, the 10th IP will be reserved for coredns. EOD - type = "string" + + type = string default = "10.3.0.0/16" } variable "cluster_domain_suffix" { description = "Queries for domains with the suffix will be answered by coredns. Default is cluster.local (e.g. foo.default.svc.cluster.local) " - type = "string" - default = "cluster.local" + type = string + default = "cluster.local" } variable "enable_reporting" { - type = "string" + type = string description = "Enable usage or analytics reporting to upstreams (Calico)" - default = "false" + default = "false" } variable "enable_aggregation" { description = "Enable the Kubernetes Aggregation Layer (defaults to false)" - type = "string" - default = "false" + type = string + default = "false" } + diff --git a/digital-ocean/container-linux/kubernetes/versions.tf b/digital-ocean/container-linux/kubernetes/versions.tf new file mode 100644 index 000000000..e10ec96d3 --- /dev/null +++ b/digital-ocean/container-linux/kubernetes/versions.tf @@ -0,0 +1,12 @@ +# Terraform version and plugin versions + +terraform { + required_version = "~> 0.12.0" + required_providers { + digitalocean = "~> 1.3" + ct = "~> 0.3.2" + template = "~> 2.1" + null = "~> 2.1" + } +} + diff --git a/digital-ocean/container-linux/kubernetes/workers.tf b/digital-ocean/container-linux/kubernetes/workers.tf index dfa969088..fb2fa02d0 100644 --- a/digital-ocean/container-linux/kubernetes/workers.tf +++ b/digital-ocean/container-linux/kubernetes/workers.tf @@ -1,47 +1,47 @@ # Worker DNS records resource "digitalocean_record" "workers-record-a" { - count = "${var.worker_count}" + count = var.worker_count # DNS zone where record should be created - domain = "${var.dns_zone}" + domain = var.dns_zone name = "${var.cluster_name}-workers" type = "A" ttl = 300 - value = "${element(digitalocean_droplet.workers.*.ipv4_address, count.index)}" + value = element(digitalocean_droplet.workers.*.ipv4_address, count.index) } resource "digitalocean_record" "workers-record-aaaa" { - count = "${var.worker_count}" + count = var.worker_count # DNS zone where record should be created - domain = "${var.dns_zone}" + domain = var.dns_zone name = "${var.cluster_name}-workers" type = "AAAA" ttl = 300 - value = "${element(digitalocean_droplet.workers.*.ipv6_address, count.index)}" + value = element(digitalocean_droplet.workers.*.ipv6_address, count.index) } # Worker droplet instances resource "digitalocean_droplet" "workers" { - count = "${var.worker_count}" + count = var.worker_count name = "${var.cluster_name}-worker-${count.index}" - region = "${var.region}" + region = var.region - image = "${var.image}" - size = "${var.worker_type}" + image = var.image + size = var.worker_type # network ipv6 = true private_networking = true - user_data = "${data.ct_config.worker-ignition.rendered}" - ssh_keys = ["${var.ssh_fingerprints}"] + user_data = data.ct_config.worker-ignition.rendered + ssh_keys = var.ssh_fingerprints tags = [ - "${digitalocean_tag.workers.id}", + digitalocean_tag.workers.id, ] lifecycle { @@ -56,17 +56,18 @@ resource "digitalocean_tag" "workers" { # Worker Ignition config data "ct_config" "worker-ignition" { - content = "${data.template_file.worker-config.rendered}" + content = data.template_file.worker-config.rendered pretty_print = false - snippets = ["${var.worker_clc_snippets}"] + snippets = var.worker_clc_snippets } # Worker Container Linux config data "template_file" "worker-config" { - template = "${file("${path.module}/cl/worker.yaml.tmpl")}" + template = file("${path.module}/cl/worker.yaml.tmpl") vars = { - cluster_dns_service_ip = "${cidrhost(var.service_cidr, 10)}" - cluster_domain_suffix = "${var.cluster_domain_suffix}" + cluster_dns_service_ip = cidrhost(var.service_cidr, 10) + cluster_domain_suffix = var.cluster_domain_suffix } } + diff --git a/docs/cl/digital-ocean.md b/docs/cl/digital-ocean.md index 9e0e6bb8e..afe0bb489 100644 --- a/docs/cl/digital-ocean.md +++ b/docs/cl/digital-ocean.md @@ -10,15 +10,15 @@ Controllers are provisioned to run an `etcd-member` peer and a `kubelet` service * Digital Ocean Account and Token * Digital Ocean Domain (registered Domain Name or delegated subdomain) -* Terraform v0.11.x and [terraform-provider-ct](https://github.com/poseidon/terraform-provider-ct) installed locally +* Terraform v0.12.x and [terraform-provider-ct](https://github.com/poseidon/terraform-provider-ct) installed locally ## Terraform Setup -Install [Terraform](https://www.terraform.io/downloads.html) v0.11.x on your system. +Install [Terraform](https://www.terraform.io/downloads.html) v0.12.x on your system. ```sh $ terraform version -Terraform v0.11.14 +Terraform v0.12.0 ``` Add the [terraform-provider-ct](https://github.com/poseidon/terraform-provider-ct) plugin binary for your system to `~/.terraform.d/plugins/`, noting the final name. @@ -50,34 +50,13 @@ Configure the DigitalOcean provider to use your token in a `providers.tf` file. ```tf provider "digitalocean" { - version = "~> 1.3.0" + version = "1.3.0" token = "${chomp(file("~/.config/digital-ocean/token"))}" - alias = "default" } provider "ct" { version = "0.3.2" } - -provider "local" { - version = "~> 1.0" - alias = "default" -} - -provider "null" { - version = "~> 1.0" - alias = "default" -} - -provider "template" { - version = "~> 1.0" - alias = "default" -} - -provider "tls" { - version = "~> 1.0" - alias = "default" -} ``` ## Cluster @@ -86,15 +65,7 @@ Define a Kubernetes cluster using the module `digital-ocean/container-linux/kube ```tf module "digital-ocean-nemo" { - source = "git::https://github.com/poseidon/typhoon//digital-ocean/container-linux/kubernetes?ref=v1.14.3" - - providers = { - digitalocean = "digitalocean.default" - local = "local.default" - null = "null.default" - template = "template.default" - tls = "tls.default" - } + source = "git::https://github.com/poseidon/typhoon//digital-ocean/container-linux/kubernetes?ref=v1.14.4" # Digital Ocean cluster_name = "nemo" From 2ba0181dbe0bb421fedcbaf7cb54340a4a081d14 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Mon, 27 May 2019 20:42:48 -0700 Subject: [PATCH 119/523] Migrate AWS module Terraform v0.11 to v0.12 * Replace v0.11 bracket type hints with Terraform v0.12 list expressions * Use expression syntax instead of interpolated strings, where suggested * Update AWS tutorial and worker pools documentation * Define Terraform and plugin version requirements in versions.tf * Require aws ~> 2.7 to support Terraform v0.12 * Require ct ~> 0.3.2 to support Terraform v0.12 --- CHANGES.md | 3 + aws/container-linux/kubernetes/ami.tf | 7 +- aws/container-linux/kubernetes/bootkube.tf | 25 ++-- aws/container-linux/kubernetes/controllers.tf | 64 +++++----- aws/container-linux/kubernetes/network.tf | 46 +++++--- aws/container-linux/kubernetes/nlb.tf | 31 ++--- aws/container-linux/kubernetes/outputs.tf | 21 ++-- aws/container-linux/kubernetes/require.tf | 25 ---- aws/container-linux/kubernetes/security.tf | 109 +++++++++--------- aws/container-linux/kubernetes/ssh.tf | 31 ++--- aws/container-linux/kubernetes/variables.tf | 58 +++++----- aws/container-linux/kubernetes/versions.tf | 11 ++ aws/container-linux/kubernetes/workers.tf | 31 ++--- aws/container-linux/kubernetes/workers/ami.tf | 7 +- .../kubernetes/workers/ingress.tf | 5 +- .../kubernetes/workers/outputs.tf | 5 +- .../kubernetes/workers/variables.tf | 38 +++--- .../kubernetes/workers/versions.tf | 4 + .../kubernetes/workers/workers.tf | 65 ++++++----- docs/advanced/worker-pools.md | 26 ++--- docs/cl/aws.md | 38 +----- 21 files changed, 321 insertions(+), 329 deletions(-) delete mode 100644 aws/container-linux/kubernetes/require.tf create mode 100644 aws/container-linux/kubernetes/versions.tf create mode 100644 aws/container-linux/kubernetes/workers/versions.tf diff --git a/CHANGES.md b/CHANGES.md index 6bb34544a..fb739fca9 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -20,6 +20,9 @@ Notable changes between versions. ### AWS +* Migrate from Terraform v0.11 to v0.12.x (**action required!**) + * Require `terraform-provider-aws` v2.7+ to support Terraform v0.12 + * Require `terraform-provider-ct` v0.3.2+ to support Terraform v0.12 * Rename `worker` pool module `count` variable to `worker_count` ([#485](https://github.com/poseidon/typhoon/pull/485)) (action required) * `count` will become a reserved variable name in Terraform v0.12 diff --git a/aws/container-linux/kubernetes/ami.tf b/aws/container-linux/kubernetes/ami.tf index c2b8efbb0..cde47bdfc 100644 --- a/aws/container-linux/kubernetes/ami.tf +++ b/aws/container-linux/kubernetes/ami.tf @@ -2,10 +2,10 @@ locals { # Pick a CoreOS Container Linux derivative # coreos-stable -> Container Linux AMI # flatcar-stable -> Flatcar Linux AMI - ami_id = "${local.flavor == "flatcar" ? data.aws_ami.flatcar.image_id : data.aws_ami.coreos.image_id}" + ami_id = local.flavor == "flatcar" ? data.aws_ami.flatcar.image_id : data.aws_ami.coreos.image_id - flavor = "${element(split("-", var.os_image), 0)}" - channel = "${element(split("-", var.os_image), 1)}" + flavor = element(split("-", var.os_image), 0) + channel = element(split("-", var.os_image), 1) } data "aws_ami" "coreos" { @@ -47,3 +47,4 @@ data "aws_ami" "flatcar" { values = ["Flatcar-${local.channel}-*"] } } + diff --git a/aws/container-linux/kubernetes/bootkube.tf b/aws/container-linux/kubernetes/bootkube.tf index 0bb6b24fc..9d7f10ef5 100644 --- a/aws/container-linux/kubernetes/bootkube.tf +++ b/aws/container-linux/kubernetes/bootkube.tf @@ -1,16 +1,17 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=082921d67905417755609eebda7d39a7e26f7fdb" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=0103bc06bb3f597455a765bf5d916f9b241cbea0" - cluster_name = "${var.cluster_name}" - api_servers = ["${format("%s.%s", var.cluster_name, var.dns_zone)}"] - etcd_servers = ["${aws_route53_record.etcds.*.fqdn}"] - asset_dir = "${var.asset_dir}" - networking = "${var.networking}" - network_mtu = "${var.network_mtu}" - pod_cidr = "${var.pod_cidr}" - service_cidr = "${var.service_cidr}" - cluster_domain_suffix = "${var.cluster_domain_suffix}" - enable_reporting = "${var.enable_reporting}" - enable_aggregation = "${var.enable_aggregation}" + cluster_name = var.cluster_name + api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] + etcd_servers = aws_route53_record.etcds.*.fqdn + asset_dir = var.asset_dir + networking = var.networking + network_mtu = var.network_mtu + pod_cidr = var.pod_cidr + service_cidr = var.service_cidr + cluster_domain_suffix = var.cluster_domain_suffix + enable_reporting = var.enable_reporting + enable_aggregation = var.enable_aggregation } + diff --git a/aws/container-linux/kubernetes/controllers.tf b/aws/container-linux/kubernetes/controllers.tf index 7cac72c08..98e7d002a 100644 --- a/aws/container-linux/kubernetes/controllers.tf +++ b/aws/container-linux/kubernetes/controllers.tf @@ -1,87 +1,89 @@ # Discrete DNS records for each controller's private IPv4 for etcd usage resource "aws_route53_record" "etcds" { - count = "${var.controller_count}" + count = var.controller_count # DNS Zone where record should be created - zone_id = "${var.dns_zone_id}" + zone_id = var.dns_zone_id - name = "${format("%s-etcd%d.%s.", var.cluster_name, count.index, var.dns_zone)}" + name = format("%s-etcd%d.%s.", var.cluster_name, count.index, var.dns_zone) type = "A" ttl = 300 # private IPv4 address for etcd - records = ["${element(aws_instance.controllers.*.private_ip, count.index)}"] + records = [element(aws_instance.controllers.*.private_ip, count.index)] } # Controller instances resource "aws_instance" "controllers" { - count = "${var.controller_count}" + count = var.controller_count tags = { Name = "${var.cluster_name}-controller-${count.index}" } - instance_type = "${var.controller_type}" + instance_type = var.controller_type - ami = "${local.ami_id}" - user_data = "${element(data.ct_config.controller-ignitions.*.rendered, count.index)}" + ami = local.ami_id + user_data = element(data.ct_config.controller-ignitions.*.rendered, count.index) # storage root_block_device { - volume_type = "${var.disk_type}" - volume_size = "${var.disk_size}" - iops = "${var.disk_iops}" + volume_type = var.disk_type + volume_size = var.disk_size + iops = var.disk_iops } # network associate_public_ip_address = true - subnet_id = "${element(aws_subnet.public.*.id, count.index)}" - vpc_security_group_ids = ["${aws_security_group.controller.id}"] + subnet_id = element(aws_subnet.public.*.id, count.index) + vpc_security_group_ids = [aws_security_group.controller.id] lifecycle { ignore_changes = [ - "ami", - "user_data", + ami, + user_data, ] } } # Controller Ignition configs data "ct_config" "controller-ignitions" { - count = "${var.controller_count}" - content = "${element(data.template_file.controller-configs.*.rendered, count.index)}" + count = var.controller_count + content = element( + data.template_file.controller-configs.*.rendered, + count.index, + ) pretty_print = false - snippets = ["${var.controller_clc_snippets}"] + snippets = var.controller_clc_snippets } # Controller Container Linux configs data "template_file" "controller-configs" { - count = "${var.controller_count}" + count = var.controller_count - template = "${file("${path.module}/cl/controller.yaml.tmpl")}" + template = file("${path.module}/cl/controller.yaml.tmpl") vars = { # Cannot use cyclic dependencies on controllers or their DNS records etcd_name = "etcd${count.index}" etcd_domain = "${var.cluster_name}-etcd${count.index}.${var.dns_zone}" - # etcd0=https://cluster-etcd0.example.com,etcd1=https://cluster-etcd1.example.com,... - etcd_initial_cluster = "${join(",", data.template_file.etcds.*.rendered)}" - - kubeconfig = "${indent(10, module.bootkube.kubeconfig-kubelet)}" - ssh_authorized_key = "${var.ssh_authorized_key}" - cluster_dns_service_ip = "${cidrhost(var.service_cidr, 10)}" - cluster_domain_suffix = "${var.cluster_domain_suffix}" + etcd_initial_cluster = join(",", data.template_file.etcds.*.rendered) + kubeconfig = indent(10, module.bootkube.kubeconfig-kubelet) + ssh_authorized_key = var.ssh_authorized_key + cluster_dns_service_ip = cidrhost(var.service_cidr, 10) + cluster_domain_suffix = var.cluster_domain_suffix } } data "template_file" "etcds" { - count = "${var.controller_count}" + count = var.controller_count template = "etcd$${index}=https://$${cluster_name}-etcd$${index}.$${dns_zone}:2380" vars = { - index = "${count.index}" - cluster_name = "${var.cluster_name}" - dns_zone = "${var.dns_zone}" + index = count.index + cluster_name = var.cluster_name + dns_zone = var.dns_zone } } + diff --git a/aws/container-linux/kubernetes/network.tf b/aws/container-linux/kubernetes/network.tf index 1be5073ba..ae159ba52 100644 --- a/aws/container-linux/kubernetes/network.tf +++ b/aws/container-linux/kubernetes/network.tf @@ -1,57 +1,67 @@ -data "aws_availability_zones" "all" {} +data "aws_availability_zones" "all" { +} # Network VPC, gateway, and routes resource "aws_vpc" "network" { - cidr_block = "${var.host_cidr}" + cidr_block = var.host_cidr assign_generated_ipv6_cidr_block = true enable_dns_support = true enable_dns_hostnames = true - tags = "${map("Name", "${var.cluster_name}")}" + tags = { + "Name" = var.cluster_name + } } resource "aws_internet_gateway" "gateway" { - vpc_id = "${aws_vpc.network.id}" + vpc_id = aws_vpc.network.id - tags = "${map("Name", "${var.cluster_name}")}" + tags = { + "Name" = var.cluster_name + } } resource "aws_route_table" "default" { - vpc_id = "${aws_vpc.network.id}" + vpc_id = aws_vpc.network.id route { cidr_block = "0.0.0.0/0" - gateway_id = "${aws_internet_gateway.gateway.id}" + gateway_id = aws_internet_gateway.gateway.id } route { ipv6_cidr_block = "::/0" - gateway_id = "${aws_internet_gateway.gateway.id}" + gateway_id = aws_internet_gateway.gateway.id } - tags = "${map("Name", "${var.cluster_name}")}" + tags = { + "Name" = var.cluster_name + } } # Subnets (one per availability zone) resource "aws_subnet" "public" { - count = "${length(data.aws_availability_zones.all.names)}" + count = length(data.aws_availability_zones.all.names) - vpc_id = "${aws_vpc.network.id}" - availability_zone = "${data.aws_availability_zones.all.names[count.index]}" + vpc_id = aws_vpc.network.id + availability_zone = data.aws_availability_zones.all.names[count.index] - cidr_block = "${cidrsubnet(var.host_cidr, 4, count.index)}" - ipv6_cidr_block = "${cidrsubnet(aws_vpc.network.ipv6_cidr_block, 8, count.index)}" + cidr_block = cidrsubnet(var.host_cidr, 4, count.index) + ipv6_cidr_block = cidrsubnet(aws_vpc.network.ipv6_cidr_block, 8, count.index) map_public_ip_on_launch = true assign_ipv6_address_on_creation = true - tags = "${map("Name", "${var.cluster_name}-public-${count.index}")}" + tags = { + "Name" = "${var.cluster_name}-public-${count.index}" + } } resource "aws_route_table_association" "public" { - count = "${length(data.aws_availability_zones.all.names)}" + count = length(data.aws_availability_zones.all.names) - route_table_id = "${aws_route_table.default.id}" - subnet_id = "${element(aws_subnet.public.*.id, count.index)}" + route_table_id = aws_route_table.default.id + subnet_id = element(aws_subnet.public.*.id, count.index) } + diff --git a/aws/container-linux/kubernetes/nlb.tf b/aws/container-linux/kubernetes/nlb.tf index ddcc52f5b..08458a97e 100644 --- a/aws/container-linux/kubernetes/nlb.tf +++ b/aws/container-linux/kubernetes/nlb.tf @@ -1,14 +1,14 @@ # Network Load Balancer DNS Record resource "aws_route53_record" "apiserver" { - zone_id = "${var.dns_zone_id}" + zone_id = var.dns_zone_id - name = "${format("%s.%s.", var.cluster_name, var.dns_zone)}" + name = format("%s.%s.", var.cluster_name, var.dns_zone) type = "A" # AWS recommends their special "alias" records for NLBs alias { - name = "${aws_lb.nlb.dns_name}" - zone_id = "${aws_lb.nlb.zone_id}" + name = aws_lb.nlb.dns_name + zone_id = aws_lb.nlb.zone_id evaluate_target_health = true } } @@ -19,51 +19,51 @@ resource "aws_lb" "nlb" { load_balancer_type = "network" internal = false - subnets = ["${aws_subnet.public.*.id}"] + subnets = aws_subnet.public.*.id enable_cross_zone_load_balancing = true } # Forward TCP apiserver traffic to controllers resource "aws_lb_listener" "apiserver-https" { - load_balancer_arn = "${aws_lb.nlb.arn}" + load_balancer_arn = aws_lb.nlb.arn protocol = "TCP" port = "6443" default_action { type = "forward" - target_group_arn = "${aws_lb_target_group.controllers.arn}" + target_group_arn = aws_lb_target_group.controllers.arn } } # Forward HTTP ingress traffic to workers resource "aws_lb_listener" "ingress-http" { - load_balancer_arn = "${aws_lb.nlb.arn}" + load_balancer_arn = aws_lb.nlb.arn protocol = "TCP" port = 80 default_action { type = "forward" - target_group_arn = "${module.workers.target_group_http}" + target_group_arn = module.workers.target_group_http } } # Forward HTTPS ingress traffic to workers resource "aws_lb_listener" "ingress-https" { - load_balancer_arn = "${aws_lb.nlb.arn}" + load_balancer_arn = aws_lb.nlb.arn protocol = "TCP" port = 443 default_action { type = "forward" - target_group_arn = "${module.workers.target_group_https}" + target_group_arn = module.workers.target_group_https } } # Target group of controllers resource "aws_lb_target_group" "controllers" { name = "${var.cluster_name}-controllers" - vpc_id = "${aws_vpc.network.id}" + vpc_id = aws_vpc.network.id target_type = "instance" protocol = "TCP" @@ -85,9 +85,10 @@ resource "aws_lb_target_group" "controllers" { # Attach controller instances to apiserver NLB resource "aws_lb_target_group_attachment" "controllers" { - count = "${var.controller_count}" + count = var.controller_count - target_group_arn = "${aws_lb_target_group.controllers.arn}" - target_id = "${element(aws_instance.controllers.*.id, count.index)}" + target_group_arn = aws_lb_target_group.controllers.arn + target_id = element(aws_instance.controllers.*.id, count.index) port = 6443 } + diff --git a/aws/container-linux/kubernetes/outputs.tf b/aws/container-linux/kubernetes/outputs.tf index ac66d0fd1..471c3300e 100644 --- a/aws/container-linux/kubernetes/outputs.tf +++ b/aws/container-linux/kubernetes/outputs.tf @@ -1,53 +1,54 @@ output "kubeconfig-admin" { - value = "${module.bootkube.kubeconfig-admin}" + value = module.bootkube.kubeconfig-admin } # Outputs for Kubernetes Ingress output "ingress_dns_name" { - value = "${aws_lb.nlb.dns_name}" + value = aws_lb.nlb.dns_name description = "DNS name of the network load balancer for distributing traffic to Ingress controllers" } output "ingress_zone_id" { - value = "${aws_lb.nlb.zone_id}" + value = aws_lb.nlb.zone_id description = "Route53 zone id of the network load balancer DNS name that can be used in Route53 alias records" } # Outputs for worker pools output "vpc_id" { - value = "${aws_vpc.network.id}" + value = aws_vpc.network.id description = "ID of the VPC for creating worker instances" } output "subnet_ids" { - value = ["${aws_subnet.public.*.id}"] + value = aws_subnet.public.*.id description = "List of subnet IDs for creating worker instances" } output "worker_security_groups" { - value = ["${aws_security_group.worker.id}"] + value = [aws_security_group.worker.id] description = "List of worker security group IDs" } output "kubeconfig" { - value = "${module.bootkube.kubeconfig-kubelet}" + value = module.bootkube.kubeconfig-kubelet } # Outputs for custom load balancing output "nlb_id" { description = "ARN of the Network Load Balancer" - value = "${aws_lb.nlb.id}" + value = aws_lb.nlb.id } output "worker_target_group_http" { description = "ARN of a target group of workers for HTTP traffic" - value = "${module.workers.target_group_http}" + value = module.workers.target_group_http } output "worker_target_group_https" { description = "ARN of a target group of workers for HTTPS traffic" - value = "${module.workers.target_group_https}" + value = module.workers.target_group_https } + diff --git a/aws/container-linux/kubernetes/require.tf b/aws/container-linux/kubernetes/require.tf deleted file mode 100644 index 68f475d62..000000000 --- a/aws/container-linux/kubernetes/require.tf +++ /dev/null @@ -1,25 +0,0 @@ -# Terraform version and plugin versions - -terraform { - required_version = ">= 0.11.0" -} - -provider "aws" { - version = ">= 1.13, < 3.0" -} - -provider "local" { - version = "~> 1.0" -} - -provider "null" { - version = "~> 1.0" -} - -provider "template" { - version = "~> 1.0" -} - -provider "tls" { - version = "~> 1.0" -} diff --git a/aws/container-linux/kubernetes/security.tf b/aws/container-linux/kubernetes/security.tf index 7672a92e8..aa2f84cb8 100644 --- a/aws/container-linux/kubernetes/security.tf +++ b/aws/container-linux/kubernetes/security.tf @@ -6,13 +6,15 @@ resource "aws_security_group" "controller" { name = "${var.cluster_name}-controller" description = "${var.cluster_name} controller security group" - vpc_id = "${aws_vpc.network.id}" + vpc_id = aws_vpc.network.id - tags = "${map("Name", "${var.cluster_name}-controller")}" + tags = { + "Name" = "${var.cluster_name}-controller" + } } resource "aws_security_group_rule" "controller-ssh" { - security_group_id = "${aws_security_group.controller.id}" + security_group_id = aws_security_group.controller.id type = "ingress" protocol = "tcp" @@ -22,7 +24,7 @@ resource "aws_security_group_rule" "controller-ssh" { } resource "aws_security_group_rule" "controller-etcd" { - security_group_id = "${aws_security_group.controller.id}" + security_group_id = aws_security_group.controller.id type = "ingress" protocol = "tcp" @@ -33,31 +35,31 @@ resource "aws_security_group_rule" "controller-etcd" { # Allow Prometheus to scrape etcd metrics resource "aws_security_group_rule" "controller-etcd-metrics" { - security_group_id = "${aws_security_group.controller.id}" + security_group_id = aws_security_group.controller.id type = "ingress" protocol = "tcp" from_port = 2381 to_port = 2381 - source_security_group_id = "${aws_security_group.worker.id}" + source_security_group_id = aws_security_group.worker.id } resource "aws_security_group_rule" "controller-vxlan" { - count = "${var.networking == "flannel" ? 1 : 0}" + count = var.networking == "flannel" ? 1 : 0 - security_group_id = "${aws_security_group.controller.id}" + security_group_id = aws_security_group.controller.id type = "ingress" protocol = "udp" from_port = 4789 to_port = 4789 - source_security_group_id = "${aws_security_group.worker.id}" + source_security_group_id = aws_security_group.worker.id } resource "aws_security_group_rule" "controller-vxlan-self" { - count = "${var.networking == "flannel" ? 1 : 0}" + count = var.networking == "flannel" ? 1 : 0 - security_group_id = "${aws_security_group.controller.id}" + security_group_id = aws_security_group.controller.id type = "ingress" protocol = "udp" @@ -67,7 +69,7 @@ resource "aws_security_group_rule" "controller-vxlan-self" { } resource "aws_security_group_rule" "controller-apiserver" { - security_group_id = "${aws_security_group.controller.id}" + security_group_id = aws_security_group.controller.id type = "ingress" protocol = "tcp" @@ -78,28 +80,28 @@ resource "aws_security_group_rule" "controller-apiserver" { # Allow Prometheus to scrape node-exporter daemonset resource "aws_security_group_rule" "controller-node-exporter" { - security_group_id = "${aws_security_group.controller.id}" + security_group_id = aws_security_group.controller.id type = "ingress" protocol = "tcp" from_port = 9100 to_port = 9100 - source_security_group_id = "${aws_security_group.worker.id}" + source_security_group_id = aws_security_group.worker.id } # Allow apiserver to access kubelets for exec, log, port-forward resource "aws_security_group_rule" "controller-kubelet" { - security_group_id = "${aws_security_group.controller.id}" + security_group_id = aws_security_group.controller.id type = "ingress" protocol = "tcp" from_port = 10250 to_port = 10250 - source_security_group_id = "${aws_security_group.worker.id}" + source_security_group_id = aws_security_group.worker.id } resource "aws_security_group_rule" "controller-kubelet-self" { - security_group_id = "${aws_security_group.controller.id}" + security_group_id = aws_security_group.controller.id type = "ingress" protocol = "tcp" @@ -109,17 +111,17 @@ resource "aws_security_group_rule" "controller-kubelet-self" { } resource "aws_security_group_rule" "controller-bgp" { - security_group_id = "${aws_security_group.controller.id}" + security_group_id = aws_security_group.controller.id type = "ingress" protocol = "tcp" from_port = 179 to_port = 179 - source_security_group_id = "${aws_security_group.worker.id}" + source_security_group_id = aws_security_group.worker.id } resource "aws_security_group_rule" "controller-bgp-self" { - security_group_id = "${aws_security_group.controller.id}" + security_group_id = aws_security_group.controller.id type = "ingress" protocol = "tcp" @@ -129,17 +131,17 @@ resource "aws_security_group_rule" "controller-bgp-self" { } resource "aws_security_group_rule" "controller-ipip" { - security_group_id = "${aws_security_group.controller.id}" + security_group_id = aws_security_group.controller.id type = "ingress" protocol = 4 from_port = 0 to_port = 0 - source_security_group_id = "${aws_security_group.worker.id}" + source_security_group_id = aws_security_group.worker.id } resource "aws_security_group_rule" "controller-ipip-self" { - security_group_id = "${aws_security_group.controller.id}" + security_group_id = aws_security_group.controller.id type = "ingress" protocol = 4 @@ -149,17 +151,17 @@ resource "aws_security_group_rule" "controller-ipip-self" { } resource "aws_security_group_rule" "controller-ipip-legacy" { - security_group_id = "${aws_security_group.controller.id}" + security_group_id = aws_security_group.controller.id type = "ingress" protocol = 94 from_port = 0 to_port = 0 - source_security_group_id = "${aws_security_group.worker.id}" + source_security_group_id = aws_security_group.worker.id } resource "aws_security_group_rule" "controller-ipip-legacy-self" { - security_group_id = "${aws_security_group.controller.id}" + security_group_id = aws_security_group.controller.id type = "ingress" protocol = 94 @@ -169,7 +171,7 @@ resource "aws_security_group_rule" "controller-ipip-legacy-self" { } resource "aws_security_group_rule" "controller-egress" { - security_group_id = "${aws_security_group.controller.id}" + security_group_id = aws_security_group.controller.id type = "egress" protocol = "-1" @@ -185,13 +187,15 @@ resource "aws_security_group" "worker" { name = "${var.cluster_name}-worker" description = "${var.cluster_name} worker security group" - vpc_id = "${aws_vpc.network.id}" + vpc_id = aws_vpc.network.id - tags = "${map("Name", "${var.cluster_name}-worker")}" + tags = { + "Name" = "${var.cluster_name}-worker" + } } resource "aws_security_group_rule" "worker-ssh" { - security_group_id = "${aws_security_group.worker.id}" + security_group_id = aws_security_group.worker.id type = "ingress" protocol = "tcp" @@ -201,7 +205,7 @@ resource "aws_security_group_rule" "worker-ssh" { } resource "aws_security_group_rule" "worker-http" { - security_group_id = "${aws_security_group.worker.id}" + security_group_id = aws_security_group.worker.id type = "ingress" protocol = "tcp" @@ -211,7 +215,7 @@ resource "aws_security_group_rule" "worker-http" { } resource "aws_security_group_rule" "worker-https" { - security_group_id = "${aws_security_group.worker.id}" + security_group_id = aws_security_group.worker.id type = "ingress" protocol = "tcp" @@ -221,21 +225,21 @@ resource "aws_security_group_rule" "worker-https" { } resource "aws_security_group_rule" "worker-vxlan" { - count = "${var.networking == "flannel" ? 1 : 0}" + count = var.networking == "flannel" ? 1 : 0 - security_group_id = "${aws_security_group.worker.id}" + security_group_id = aws_security_group.worker.id type = "ingress" protocol = "udp" from_port = 4789 to_port = 4789 - source_security_group_id = "${aws_security_group.controller.id}" + source_security_group_id = aws_security_group.controller.id } resource "aws_security_group_rule" "worker-vxlan-self" { - count = "${var.networking == "flannel" ? 1 : 0}" + count = var.networking == "flannel" ? 1 : 0 - security_group_id = "${aws_security_group.worker.id}" + security_group_id = aws_security_group.worker.id type = "ingress" protocol = "udp" @@ -246,7 +250,7 @@ resource "aws_security_group_rule" "worker-vxlan-self" { # Allow Prometheus to scrape node-exporter daemonset resource "aws_security_group_rule" "worker-node-exporter" { - security_group_id = "${aws_security_group.worker.id}" + security_group_id = aws_security_group.worker.id type = "ingress" protocol = "tcp" @@ -256,7 +260,7 @@ resource "aws_security_group_rule" "worker-node-exporter" { } resource "aws_security_group_rule" "ingress-health" { - security_group_id = "${aws_security_group.worker.id}" + security_group_id = aws_security_group.worker.id type = "ingress" protocol = "tcp" @@ -267,18 +271,18 @@ resource "aws_security_group_rule" "ingress-health" { # Allow apiserver to access kubelets for exec, log, port-forward resource "aws_security_group_rule" "worker-kubelet" { - security_group_id = "${aws_security_group.worker.id}" + security_group_id = aws_security_group.worker.id type = "ingress" protocol = "tcp" from_port = 10250 to_port = 10250 - source_security_group_id = "${aws_security_group.controller.id}" + source_security_group_id = aws_security_group.controller.id } # Allow Prometheus to scrape kubelet metrics resource "aws_security_group_rule" "worker-kubelet-self" { - security_group_id = "${aws_security_group.worker.id}" + security_group_id = aws_security_group.worker.id type = "ingress" protocol = "tcp" @@ -288,17 +292,17 @@ resource "aws_security_group_rule" "worker-kubelet-self" { } resource "aws_security_group_rule" "worker-bgp" { - security_group_id = "${aws_security_group.worker.id}" + security_group_id = aws_security_group.worker.id type = "ingress" protocol = "tcp" from_port = 179 to_port = 179 - source_security_group_id = "${aws_security_group.controller.id}" + source_security_group_id = aws_security_group.controller.id } resource "aws_security_group_rule" "worker-bgp-self" { - security_group_id = "${aws_security_group.worker.id}" + security_group_id = aws_security_group.worker.id type = "ingress" protocol = "tcp" @@ -308,17 +312,17 @@ resource "aws_security_group_rule" "worker-bgp-self" { } resource "aws_security_group_rule" "worker-ipip" { - security_group_id = "${aws_security_group.worker.id}" + security_group_id = aws_security_group.worker.id type = "ingress" protocol = 4 from_port = 0 to_port = 0 - source_security_group_id = "${aws_security_group.controller.id}" + source_security_group_id = aws_security_group.controller.id } resource "aws_security_group_rule" "worker-ipip-self" { - security_group_id = "${aws_security_group.worker.id}" + security_group_id = aws_security_group.worker.id type = "ingress" protocol = 4 @@ -328,17 +332,17 @@ resource "aws_security_group_rule" "worker-ipip-self" { } resource "aws_security_group_rule" "worker-ipip-legacy" { - security_group_id = "${aws_security_group.worker.id}" + security_group_id = aws_security_group.worker.id type = "ingress" protocol = 94 from_port = 0 to_port = 0 - source_security_group_id = "${aws_security_group.controller.id}" + source_security_group_id = aws_security_group.controller.id } resource "aws_security_group_rule" "worker-ipip-legacy-self" { - security_group_id = "${aws_security_group.worker.id}" + security_group_id = aws_security_group.worker.id type = "ingress" protocol = 94 @@ -348,7 +352,7 @@ resource "aws_security_group_rule" "worker-ipip-legacy-self" { } resource "aws_security_group_rule" "worker-egress" { - security_group_id = "${aws_security_group.worker.id}" + security_group_id = aws_security_group.worker.id type = "egress" protocol = "-1" @@ -357,3 +361,4 @@ resource "aws_security_group_rule" "worker-egress" { cidr_blocks = ["0.0.0.0/0"] ipv6_cidr_blocks = ["::/0"] } + diff --git a/aws/container-linux/kubernetes/ssh.tf b/aws/container-linux/kubernetes/ssh.tf index 111bbeaa4..22b7a5d98 100644 --- a/aws/container-linux/kubernetes/ssh.tf +++ b/aws/container-linux/kubernetes/ssh.tf @@ -1,46 +1,46 @@ # Secure copy etcd TLS assets to controllers. resource "null_resource" "copy-controller-secrets" { - count = "${var.controller_count}" + count = var.controller_count connection { type = "ssh" - host = "${element(aws_instance.controllers.*.public_ip, count.index)}" + host = element(aws_instance.controllers.*.public_ip, count.index) user = "core" timeout = "15m" } provisioner "file" { - content = "${module.bootkube.etcd_ca_cert}" + content = module.bootkube.etcd_ca_cert destination = "$HOME/etcd-client-ca.crt" } provisioner "file" { - content = "${module.bootkube.etcd_client_cert}" + content = module.bootkube.etcd_client_cert destination = "$HOME/etcd-client.crt" } provisioner "file" { - content = "${module.bootkube.etcd_client_key}" + content = module.bootkube.etcd_client_key destination = "$HOME/etcd-client.key" } provisioner "file" { - content = "${module.bootkube.etcd_server_cert}" + content = module.bootkube.etcd_server_cert destination = "$HOME/etcd-server.crt" } provisioner "file" { - content = "${module.bootkube.etcd_server_key}" + content = module.bootkube.etcd_server_key destination = "$HOME/etcd-server.key" } provisioner "file" { - content = "${module.bootkube.etcd_peer_cert}" + content = module.bootkube.etcd_peer_cert destination = "$HOME/etcd-peer.crt" } provisioner "file" { - content = "${module.bootkube.etcd_peer_key}" + content = module.bootkube.etcd_peer_key destination = "$HOME/etcd-peer.key" } @@ -64,21 +64,21 @@ resource "null_resource" "copy-controller-secrets" { # one-time self-hosted cluster bootstrapping. resource "null_resource" "bootkube-start" { depends_on = [ - "module.bootkube", - "module.workers", - "aws_route53_record.apiserver", - "null_resource.copy-controller-secrets", + module.bootkube, + module.workers, + aws_route53_record.apiserver, + null_resource.copy-controller-secrets, ] connection { type = "ssh" - host = "${aws_instance.controllers.0.public_ip}" + host = aws_instance.controllers[0].public_ip user = "core" timeout = "15m" } provisioner "file" { - source = "${var.asset_dir}" + source = var.asset_dir destination = "$HOME/assets" } @@ -89,3 +89,4 @@ resource "null_resource" "bootkube-start" { ] } } + diff --git a/aws/container-linux/kubernetes/variables.tf b/aws/container-linux/kubernetes/variables.tf index 33be7faac..42d032823 100644 --- a/aws/container-linux/kubernetes/variables.tf +++ b/aws/container-linux/kubernetes/variables.tf @@ -1,90 +1,90 @@ variable "cluster_name" { - type = "string" + type = string description = "Unique cluster name (prepended to dns_zone)" } # AWS variable "dns_zone" { - type = "string" + type = string description = "AWS Route53 DNS Zone (e.g. aws.example.com)" } variable "dns_zone_id" { - type = "string" + type = string description = "AWS Route53 DNS Zone ID (e.g. Z3PAABBCFAKEC0)" } # instances variable "controller_count" { - type = "string" + type = string default = "1" description = "Number of controllers (i.e. masters)" } variable "worker_count" { - type = "string" + type = string default = "1" description = "Number of workers" } variable "controller_type" { - type = "string" + type = string default = "t3.small" description = "EC2 instance type for controllers" } variable "worker_type" { - type = "string" + type = string default = "t3.small" description = "EC2 instance type for workers" } variable "os_image" { - type = "string" + type = string default = "coreos-stable" description = "AMI channel for a Container Linux derivative (coreos-stable, coreos-beta, coreos-alpha, flatcar-stable, flatcar-beta, flatcar-alpha)" } variable "disk_size" { - type = "string" + type = string default = "40" description = "Size of the EBS volume in GB" } variable "disk_type" { - type = "string" + type = string default = "gp2" description = "Type of the EBS volume (e.g. standard, gp2, io1)" } variable "disk_iops" { - type = "string" + type = string default = "0" description = "IOPS of the EBS volume (e.g. 100)" } variable "worker_price" { - type = "string" + type = string default = "" description = "Spot price in USD for autoscaling group spot instances. Leave as default empty string for autoscaling group to use on-demand instances. Note, switching in-place from spot to on-demand is not possible: https://github.com/terraform-providers/terraform-provider-aws/issues/4320" } variable "worker_target_groups" { - type = "list" + type = list(string) description = "Additional target group ARNs to which worker instances should be added" default = [] } variable "controller_clc_snippets" { - type = "list" + type = list(string) description = "Controller Container Linux Config snippets" default = [] } variable "worker_clc_snippets" { - type = "list" + type = list(string) description = "Worker Container Linux Config snippets" default = [] } @@ -92,36 +92,36 @@ variable "worker_clc_snippets" { # configuration variable "ssh_authorized_key" { - type = "string" + type = string description = "SSH public key for user 'core'" } variable "asset_dir" { description = "Path to a directory where generated assets should be placed (contains secrets)" - type = "string" + type = string } variable "networking" { description = "Choice of networking provider (calico or flannel)" - type = "string" + type = string default = "calico" } variable "network_mtu" { description = "CNI interface MTU (applies to calico only). Use 8981 if using instances types with Jumbo frames." - type = "string" + type = string default = "1480" } variable "host_cidr" { description = "CIDR IPv4 range to assign to EC2 nodes" - type = "string" + type = string default = "10.0.0.0/16" } variable "pod_cidr" { description = "CIDR IPv4 range to assign Kubernetes pods" - type = "string" + type = string default = "10.2.0.0/16" } @@ -131,24 +131,26 @@ CIDR IPv4 range to assign Kubernetes services. The 1st IP will be reserved for kube_apiserver, the 10th IP will be reserved for coredns. EOD - type = "string" + + type = string default = "10.3.0.0/16" } variable "cluster_domain_suffix" { description = "Queries for domains with the suffix will be answered by coredns. Default is cluster.local (e.g. foo.default.svc.cluster.local) " - type = "string" - default = "cluster.local" + type = string + default = "cluster.local" } variable "enable_reporting" { - type = "string" + type = string description = "Enable usage or analytics reporting to upstreams (Calico)" - default = "false" + default = "false" } variable "enable_aggregation" { description = "Enable the Kubernetes Aggregation Layer (defaults to false)" - type = "string" - default = "false" + type = string + default = "false" } + diff --git a/aws/container-linux/kubernetes/versions.tf b/aws/container-linux/kubernetes/versions.tf new file mode 100644 index 000000000..19d087c68 --- /dev/null +++ b/aws/container-linux/kubernetes/versions.tf @@ -0,0 +1,11 @@ +# Terraform version and plugin versions + +terraform { + required_version = "~> 0.12.0" + required_providers { + aws = "~> 2.7" + ct = "~> 0.3.2" + template = "~> 2.1" + null = "~> 2.1" + } +} diff --git a/aws/container-linux/kubernetes/workers.tf b/aws/container-linux/kubernetes/workers.tf index f28b13a5b..5b3b787d2 100644 --- a/aws/container-linux/kubernetes/workers.tf +++ b/aws/container-linux/kubernetes/workers.tf @@ -1,22 +1,23 @@ module "workers" { source = "./workers" - name = "${var.cluster_name}" + name = var.cluster_name # AWS - vpc_id = "${aws_vpc.network.id}" - subnet_ids = ["${aws_subnet.public.*.id}"] - security_groups = ["${aws_security_group.worker.id}"] - worker_count = "${var.worker_count}" - instance_type = "${var.worker_type}" - os_image = "${var.os_image}" - disk_size = "${var.disk_size}" - spot_price = "${var.worker_price}" - target_groups = ["${var.worker_target_groups}"] + vpc_id = aws_vpc.network.id + subnet_ids = aws_subnet.public.*.id + security_groups = [aws_security_group.worker.id] + worker_count = var.worker_count + instance_type = var.worker_type + os_image = var.os_image + disk_size = var.disk_size + spot_price = var.worker_price + target_groups = var.worker_target_groups # configuration - kubeconfig = "${module.bootkube.kubeconfig-kubelet}" - ssh_authorized_key = "${var.ssh_authorized_key}" - service_cidr = "${var.service_cidr}" - cluster_domain_suffix = "${var.cluster_domain_suffix}" - clc_snippets = "${var.worker_clc_snippets}" + kubeconfig = module.bootkube.kubeconfig-kubelet + ssh_authorized_key = var.ssh_authorized_key + service_cidr = var.service_cidr + cluster_domain_suffix = var.cluster_domain_suffix + clc_snippets = var.worker_clc_snippets } + diff --git a/aws/container-linux/kubernetes/workers/ami.tf b/aws/container-linux/kubernetes/workers/ami.tf index c2b8efbb0..cde47bdfc 100644 --- a/aws/container-linux/kubernetes/workers/ami.tf +++ b/aws/container-linux/kubernetes/workers/ami.tf @@ -2,10 +2,10 @@ locals { # Pick a CoreOS Container Linux derivative # coreos-stable -> Container Linux AMI # flatcar-stable -> Flatcar Linux AMI - ami_id = "${local.flavor == "flatcar" ? data.aws_ami.flatcar.image_id : data.aws_ami.coreos.image_id}" + ami_id = local.flavor == "flatcar" ? data.aws_ami.flatcar.image_id : data.aws_ami.coreos.image_id - flavor = "${element(split("-", var.os_image), 0)}" - channel = "${element(split("-", var.os_image), 1)}" + flavor = element(split("-", var.os_image), 0) + channel = element(split("-", var.os_image), 1) } data "aws_ami" "coreos" { @@ -47,3 +47,4 @@ data "aws_ami" "flatcar" { values = ["Flatcar-${local.channel}-*"] } } + diff --git a/aws/container-linux/kubernetes/workers/ingress.tf b/aws/container-linux/kubernetes/workers/ingress.tf index bdb7362fb..6b25152f0 100644 --- a/aws/container-linux/kubernetes/workers/ingress.tf +++ b/aws/container-linux/kubernetes/workers/ingress.tf @@ -2,7 +2,7 @@ resource "aws_lb_target_group" "workers-http" { name = "${var.name}-workers-http" - vpc_id = "${var.vpc_id}" + vpc_id = var.vpc_id target_type = "instance" protocol = "TCP" @@ -25,7 +25,7 @@ resource "aws_lb_target_group" "workers-http" { resource "aws_lb_target_group" "workers-https" { name = "${var.name}-workers-https" - vpc_id = "${var.vpc_id}" + vpc_id = var.vpc_id target_type = "instance" protocol = "TCP" @@ -45,3 +45,4 @@ resource "aws_lb_target_group" "workers-https" { interval = 10 } } + diff --git a/aws/container-linux/kubernetes/workers/outputs.tf b/aws/container-linux/kubernetes/workers/outputs.tf index 19552266a..22f378855 100644 --- a/aws/container-linux/kubernetes/workers/outputs.tf +++ b/aws/container-linux/kubernetes/workers/outputs.tf @@ -1,9 +1,10 @@ output "target_group_http" { description = "ARN of a target group of workers for HTTP traffic" - value = "${aws_lb_target_group.workers-http.arn}" + value = aws_lb_target_group.workers-http.arn } output "target_group_https" { description = "ARN of a target group of workers for HTTPS traffic" - value = "${aws_lb_target_group.workers-https.arn}" + value = aws_lb_target_group.workers-https.arn } + diff --git a/aws/container-linux/kubernetes/workers/variables.tf b/aws/container-linux/kubernetes/workers/variables.tf index 9d10da36d..67b5e061f 100644 --- a/aws/container-linux/kubernetes/workers/variables.tf +++ b/aws/container-linux/kubernetes/workers/variables.tf @@ -1,77 +1,77 @@ variable "name" { - type = "string" + type = string description = "Unique name for the worker pool" } # AWS variable "vpc_id" { - type = "string" + type = string description = "Must be set to `vpc_id` output by cluster" } variable "subnet_ids" { - type = "list" + type = list(string) description = "Must be set to `subnet_ids` output by cluster" } variable "security_groups" { - type = "list" + type = list(string) description = "Must be set to `worker_security_groups` output by cluster" } # instances variable "worker_count" { - type = "string" + type = string default = "1" description = "Number of instances" } variable "instance_type" { - type = "string" + type = string default = "t3.small" description = "EC2 instance type" } variable "os_image" { - type = "string" + type = string default = "coreos-stable" description = "AMI channel for a Container Linux derivative (coreos-stable, coreos-beta, coreos-alpha, flatcar-stable, flatcar-beta, flatcar-alpha)" } variable "disk_size" { - type = "string" + type = string default = "40" description = "Size of the EBS volume in GB" } variable "disk_type" { - type = "string" + type = string default = "gp2" description = "Type of the EBS volume (e.g. standard, gp2, io1)" } variable "disk_iops" { - type = "string" + type = string default = "0" description = "IOPS of the EBS volume (required for io1)" } variable "spot_price" { - type = "string" + type = string default = "" description = "Spot price in USD for autoscaling group spot instances. Leave as default empty string for autoscaling group to use on-demand instances. Note, switching in-place from spot to on-demand is not possible: https://github.com/terraform-providers/terraform-provider-aws/issues/4320" } variable "target_groups" { - type = "list" + type = list(string) description = "Additional target group ARNs to which instances should be added" default = [] } variable "clc_snippets" { - type = "list" + type = list(string) description = "Container Linux Config snippets" default = [] } @@ -79,12 +79,12 @@ variable "clc_snippets" { # configuration variable "kubeconfig" { - type = "string" + type = string description = "Must be set to `kubeconfig` output by cluster" } variable "ssh_authorized_key" { - type = "string" + type = string description = "SSH public key for user 'core'" } @@ -94,12 +94,14 @@ CIDR IPv4 range to assign Kubernetes services. The 1st IP will be reserved for kube_apiserver, the 10th IP will be reserved for coredns. EOD - type = "string" + + type = string default = "10.3.0.0/16" } variable "cluster_domain_suffix" { description = "Queries for domains with the suffix will be answered by coredns. Default is cluster.local (e.g. foo.default.svc.cluster.local) " - type = "string" - default = "cluster.local" + type = string + default = "cluster.local" } + diff --git a/aws/container-linux/kubernetes/workers/versions.tf b/aws/container-linux/kubernetes/workers/versions.tf new file mode 100644 index 000000000..ac97c6ac8 --- /dev/null +++ b/aws/container-linux/kubernetes/workers/versions.tf @@ -0,0 +1,4 @@ + +terraform { + required_version = ">= 0.12" +} diff --git a/aws/container-linux/kubernetes/workers/workers.tf b/aws/container-linux/kubernetes/workers/workers.tf index db5a454f8..26f9eab37 100644 --- a/aws/container-linux/kubernetes/workers/workers.tf +++ b/aws/container-linux/kubernetes/workers/workers.tf @@ -3,24 +3,24 @@ resource "aws_autoscaling_group" "workers" { name = "${var.name}-worker ${aws_launch_configuration.worker.name}" # count - desired_capacity = "${var.worker_count}" - min_size = "${var.worker_count}" - max_size = "${var.worker_count + 2}" + desired_capacity = var.worker_count + min_size = var.worker_count + max_size = var.worker_count + 2 default_cooldown = 30 health_check_grace_period = 30 # network - vpc_zone_identifier = ["${var.subnet_ids}"] + vpc_zone_identifier = var.subnet_ids # template - launch_configuration = "${aws_launch_configuration.worker.name}" + launch_configuration = aws_launch_configuration.worker.name # target groups to which instances should be added - target_group_arns = [ - "${aws_lb_target_group.workers-http.id}", - "${aws_lb_target_group.workers-https.id}", - "${var.target_groups}", - ] + target_group_arns = flatten([ + aws_lb_target_group.workers-http.id, + aws_lb_target_group.workers-https.id, + var.target_groups, + ]) lifecycle { # override the default destroy and replace update behavior @@ -33,54 +33,57 @@ resource "aws_autoscaling_group" "workers" { # used. Disable wait to avoid issues and align with other clouds. wait_for_capacity_timeout = "0" - tags = [{ - key = "Name" - value = "${var.name}-worker" - propagate_at_launch = true - }] + tags = [ + { + key = "Name" + value = "${var.name}-worker" + propagate_at_launch = true + }, + ] } # Worker template resource "aws_launch_configuration" "worker" { - image_id = "${local.ami_id}" - instance_type = "${var.instance_type}" - spot_price = "${var.spot_price}" + image_id = local.ami_id + instance_type = var.instance_type + spot_price = var.spot_price enable_monitoring = false - user_data = "${data.ct_config.worker-ignition.rendered}" + user_data = data.ct_config.worker-ignition.rendered # storage root_block_device { - volume_type = "${var.disk_type}" - volume_size = "${var.disk_size}" - iops = "${var.disk_iops}" + volume_type = var.disk_type + volume_size = var.disk_size + iops = var.disk_iops } # network - security_groups = ["${var.security_groups}"] + security_groups = var.security_groups lifecycle { // Override the default destroy and replace update behavior create_before_destroy = true - ignore_changes = ["image_id"] + ignore_changes = [image_id] } } # Worker Ignition config data "ct_config" "worker-ignition" { - content = "${data.template_file.worker-config.rendered}" + content = data.template_file.worker-config.rendered pretty_print = false - snippets = ["${var.clc_snippets}"] + snippets = var.clc_snippets } # Worker Container Linux config data "template_file" "worker-config" { - template = "${file("${path.module}/cl/worker.yaml.tmpl")}" + template = file("${path.module}/cl/worker.yaml.tmpl") vars = { - kubeconfig = "${indent(10, var.kubeconfig)}" - ssh_authorized_key = "${var.ssh_authorized_key}" - cluster_dns_service_ip = "${cidrhost(var.service_cidr, 10)}" - cluster_domain_suffix = "${var.cluster_domain_suffix}" + kubeconfig = indent(10, var.kubeconfig) + ssh_authorized_key = var.ssh_authorized_key + cluster_dns_service_ip = cidrhost(var.service_cidr, 10) + cluster_domain_suffix = var.cluster_domain_suffix } } + diff --git a/docs/advanced/worker-pools.md b/docs/advanced/worker-pools.md index e3ac59818..7935a5de5 100644 --- a/docs/advanced/worker-pools.md +++ b/docs/advanced/worker-pools.md @@ -16,21 +16,17 @@ Create a cluster following the AWS [tutorial](../cl/aws.md#cluster). Define a wo ```tf module "tempest-worker-pool" { - source = "git::https://github.com/poseidon/typhoon//aws/container-linux/kubernetes/workers?ref=v1.14.3" - - providers = { - aws = "aws.default" - } + source = "git::https://github.com/poseidon/typhoon//aws/container-linux/kubernetes/workers?ref=v1.14.4" # AWS - vpc_id = "${module.aws-tempest.vpc_id}" - subnet_ids = "${module.aws-tempest.subnet_ids}" - security_groups = "${module.aws-tempest.worker_security_groups}" + vpc_id = module.aws-tempest.vpc_id + subnet_ids = module.aws-tempest.subnet_ids + security_groups = module.aws-tempest.worker_security_groups # configuration - name = "tempest-worker-pool" - kubeconfig = "${module.aws-tempest.kubeconfig}" - ssh_authorized_key = "${var.ssh_authorized_key}" + name = "tempest-pool" + kubeconfig = module.aws-tempest.kubeconfig + ssh_authorized_key = var.ssh_authorized_key # optional worker_count = 2 @@ -56,10 +52,10 @@ The AWS internal `workers` module supports a number of [variables](https://githu | Name | Description | Example | |:-----|:------------|:--------| | name | Unique name (distinct from cluster name) | "tempest-m5s" | -| vpc_id | Must be set to `vpc_id` output by cluster | "${module.cluster.vpc_id}" | -| subnet_ids | Must be set to `subnet_ids` output by cluster | "${module.cluster.subnet_ids}" | -| security_groups | Must be set to `worker_security_groups` output by cluster | "${module.cluster.worker_security_groups}" | -| kubeconfig | Must be set to `kubeconfig` output by cluster | "${module.cluster.kubeconfig}" | +| vpc_id | Must be set to `vpc_id` output by cluster | module.cluster.vpc_id | +| subnet_ids | Must be set to `subnet_ids` output by cluster | module.cluster.subnet_ids | +| security_groups | Must be set to `worker_security_groups` output by cluster | module.cluster.worker_security_groups | +| kubeconfig | Must be set to `kubeconfig` output by cluster | module.cluster.kubeconfig | | ssh_authorized_key | SSH public key for user 'core' | "ssh-rsa AAAAB3NZ..." | #### Optional diff --git a/docs/cl/aws.md b/docs/cl/aws.md index abd33b680..7dbfffc4c 100644 --- a/docs/cl/aws.md +++ b/docs/cl/aws.md @@ -10,15 +10,15 @@ Controllers are provisioned to run an `etcd-member` peer and a `kubelet` service * AWS Account and IAM credentials * AWS Route53 DNS Zone (registered Domain Name or delegated subdomain) -* Terraform v0.11.x and [terraform-provider-ct](https://github.com/poseidon/terraform-provider-ct) installed locally +* Terraform v0.12.x and [terraform-provider-ct](https://github.com/poseidon/terraform-provider-ct) installed locally ## Terraform Setup -Install [Terraform](https://www.terraform.io/downloads.html) v0.11.x on your system. +Install [Terraform](https://www.terraform.io/downloads.html) v0.12.x on your system. ```sh $ terraform version -Terraform v0.11.14 +Terraform v0.12.0 ``` Add the [terraform-provider-ct](https://github.com/poseidon/terraform-provider-ct) plugin binary for your system to `~/.terraform.d/plugins/`, noting the final name. @@ -49,9 +49,7 @@ Configure the AWS provider to use your access key credentials in a `providers.tf ```tf provider "aws" { - version = "~> 2.12.0" - alias = "default" - + version = "2.12.0" region = "eu-central-1" shared_credentials_file = "/home/user/.config/aws/credentials" } @@ -59,26 +57,6 @@ provider "aws" { provider "ct" { version = "0.3.2" } - -provider "local" { - version = "~> 1.0" - alias = "default" -} - -provider "null" { - version = "~> 1.0" - alias = "default" -} - -provider "template" { - version = "~> 1.0" - alias = "default" -} - -provider "tls" { - version = "~> 1.0" - alias = "default" -} ``` Additional configuration options are described in the `aws` provider [docs](https://www.terraform.io/docs/providers/aws/). @@ -94,14 +72,6 @@ Define a Kubernetes cluster using the module `aws/container-linux/kubernetes`. module "aws-tempest" { source = "git::https://github.com/poseidon/typhoon//aws/container-linux/kubernetes?ref=v1.14.3" - providers = { - aws = "aws.default" - local = "local.default" - null = "null.default" - template = "template.default" - tls = "tls.default" - } - # AWS cluster_name = "tempest" dns_zone = "aws.example.com" From d6d9e6c4b9fa5731b5561f70b9675d2a1a5b0631 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Mon, 27 May 2019 21:14:25 -0700 Subject: [PATCH 120/523] Migrate Google Cloud module Terraform v0.11 to v0.12 * Replace v0.11 bracket type hints with Terraform v0.12 list expressions * Use expression syntax instead of interpolated strings, where suggested * Update Google Cloud tutorial and worker pools documentation * Define Terraform and plugin version requirements in versions.tf * Require google ~> 2.5 to support Terraform v0.12 * Require ct ~> 0.3.2 to support Terraform v0.12 --- CHANGES.md | 5 +- README.md | 10 +-- docs/advanced/worker-pools.md | 14 ++-- docs/cl/google-cloud.md | 42 ++--------- docs/index.md | 10 +-- .../container-linux/kubernetes/apiserver.tf | 41 ++++++----- .../container-linux/kubernetes/bootkube.tf | 23 ++++--- .../container-linux/kubernetes/controllers.tf | 69 ++++++++++--------- .../container-linux/kubernetes/ingress.tf | 31 +++++---- .../container-linux/kubernetes/network.tf | 32 ++++----- .../container-linux/kubernetes/outputs.tf | 17 ++--- .../container-linux/kubernetes/require.tf | 25 ------- .../container-linux/kubernetes/ssh.tf | 31 +++++---- .../container-linux/kubernetes/variables.tf | 50 +++++++------- .../container-linux/kubernetes/versions.tf | 11 +++ .../container-linux/kubernetes/workers.tf | 29 ++++---- .../kubernetes/workers/outputs.tf | 5 +- .../kubernetes/workers/target_pool.tf | 5 +- .../kubernetes/workers/variables.tf | 42 +++++------ .../kubernetes/workers/versions.tf | 4 ++ .../kubernetes/workers/workers.tf | 44 ++++++------ 21 files changed, 249 insertions(+), 291 deletions(-) delete mode 100644 google-cloud/container-linux/kubernetes/require.tf create mode 100644 google-cloud/container-linux/kubernetes/versions.tf create mode 100644 google-cloud/container-linux/kubernetes/workers/versions.tf diff --git a/CHANGES.md b/CHANGES.md index fb739fca9..b99410fff 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -39,8 +39,11 @@ Notable changes between versions. #### Google Cloud +* Migrate from Terraform v0.11 to v0.12.x (**action required!**) + * Require `terraform-provider-google` v2.5+ to support Terraform v0.12 + * Require `terraform-provider-ct` v0.3.2+ to support Terraform v0.12 * Rename `worker` pool module `count` variable to `worker_count` ([#485](https://github.com/poseidon/typhoon/pull/485)) (action required) - * `count` will become a reserved variable name in Terraform v0.12 + * `count` is a reserved variable in Terraform v0.12 #### Addons diff --git a/README.md b/README.md index 3ba0e6e31..266f961cd 100644 --- a/README.md +++ b/README.md @@ -50,15 +50,7 @@ Define a Kubernetes cluster by using the Terraform module for your chosen platfo ```tf module "google-cloud-yavin" { - source = "git::https://github.com/poseidon/typhoon//google-cloud/container-linux/kubernetes?ref=v1.14.3" - - providers = { - google = "google.default" - local = "local.default" - null = "null.default" - template = "template.default" - tls = "tls.default" - } + source = "git::https://github.com/poseidon/typhoon//google-cloud/container-linux/kubernetes?ref=v1.14.4" # Google Cloud cluster_name = "yavin" diff --git a/docs/advanced/worker-pools.md b/docs/advanced/worker-pools.md index 7935a5de5..584af00ae 100644 --- a/docs/advanced/worker-pools.md +++ b/docs/advanced/worker-pools.md @@ -150,19 +150,15 @@ Create a cluster following the Google Cloud [tutorial](../cl/google-cloud.md#clu module "yavin-worker-pool" { source = "git::https://github.com/poseidon/typhoon//google-cloud/container-linux/kubernetes/workers?ref=v1.14.3" - providers = { - google = "google.default" - } - # Google Cloud region = "europe-west2" - network = "${module.google-cloud-yavin.network_name}" + network = module.google-cloud-yavin.network_name cluster_name = "yavin" # configuration name = "yavin-16x" - kubeconfig = "${module.google-cloud-yavin.kubeconfig}" - ssh_authorized_key = "${var.ssh_authorized_key}" + kubeconfig = module.google-cloud-yavin.kubeconfig + ssh_authorized_key = var.ssh_authorized_key # optional worker_count = 2 @@ -200,9 +196,9 @@ The Google Cloud internal `workers` module supports a number of [variables](http |:-----|:------------|:--------| | name | Unique name (distinct from cluster name) | "yavin-16x" | | region | Region for the worker pool instances. May differ from the cluster's region | "europe-west2" | -| network | Must be set to `network_name` output by cluster | "${module.cluster.network_name}" | +| network | Must be set to `network_name` output by cluster | module.cluster.network_name | | cluster_name | Must be set to `cluster_name` of cluster | "yavin" | -| kubeconfig | Must be set to `kubeconfig` output by cluster | "${module.cluster.kubeconfig}" | +| kubeconfig | Must be set to `kubeconfig` output by cluster | module.cluster.kubeconfig | | ssh_authorized_key | SSH public key for user 'core' | "ssh-rsa AAAAB3NZ..." | Check the list of regions [docs](https://cloud.google.com/compute/docs/regions-zones/regions-zones) or with `gcloud compute regions list`. diff --git a/docs/cl/google-cloud.md b/docs/cl/google-cloud.md index b1331fa4f..4edc7aeb8 100644 --- a/docs/cl/google-cloud.md +++ b/docs/cl/google-cloud.md @@ -10,15 +10,15 @@ Controllers are provisioned to run an `etcd-member` peer and a `kubelet` service * Google Cloud Account and Service Account * Google Cloud DNS Zone (registered Domain Name or delegated subdomain) -* Terraform v0.11.x and [terraform-provider-ct](https://github.com/poseidon/terraform-provider-ct) installed locally +* Terraform v0.12.x and [terraform-provider-ct](https://github.com/poseidon/terraform-provider-ct) installed locally ## Terraform Setup -Install [Terraform](https://www.terraform.io/downloads.html) v0.11.x on your system. +Install [Terraform](https://www.terraform.io/downloads.html) v0.12.x on your system. ```sh $ terraform version -Terraform v0.11.14 +Terraform v0.12.0 ``` Add the [terraform-provider-ct](https://github.com/poseidon/terraform-provider-ct) plugin binary for your system to `~/.terraform.d/plugins/`, noting the final name. @@ -49,37 +49,15 @@ Configure the Google Cloud provider to use your service account key, project-id, ```tf provider "google" { - version = "~> 2.7.0" - alias = "default" - - credentials = "${file("~/.config/google-cloud/terraform.json")}" + version = "2.7.0" project = "project-id" region = "us-central1" + credentials = "${file("~/.config/google-cloud/terraform.json")}" } provider "ct" { version = "0.3.2" } - -provider "local" { - version = "~> 1.0" - alias = "default" -} - -provider "null" { - version = "~> 1.0" - alias = "default" -} - -provider "template" { - version = "~> 1.0" - alias = "default" -} - -provider "tls" { - version = "~> 1.0" - alias = "default" -} ``` Additional configuration options are described in the `google` provider [docs](https://www.terraform.io/docs/providers/google/index.html). @@ -93,15 +71,7 @@ Define a Kubernetes cluster using the module `google-cloud/container-linux/kuber ```tf module "google-cloud-yavin" { - source = "git::https://github.com/poseidon/typhoon//google-cloud/container-linux/kubernetes?ref=v1.14.3" - - providers = { - google = "google.default" - local = "local.default" - null = "null.default" - template = "template.default" - tls = "tls.default" - } + source = "git::https://github.com/poseidon/typhoon//google-cloud/container-linux/kubernetes?ref=v1.14.4" # Google Cloud cluster_name = "yavin" diff --git a/docs/index.md b/docs/index.md index 17aca9529..c7bbe0448 100644 --- a/docs/index.md +++ b/docs/index.md @@ -49,15 +49,7 @@ Define a Kubernetes cluster by using the Terraform module for your chosen platfo ```tf module "google-cloud-yavin" { - source = "git::https://github.com/poseidon/typhoon//google-cloud/container-linux/kubernetes?ref=v1.14.3" - - providers = { - google = "google.default" - local = "local.default" - null = "null.default" - template = "template.default" - tls = "tls.default" - } + source = "git::https://github.com/poseidon/typhoon//google-cloud/container-linux/kubernetes?ref=v1.14.4" # Google Cloud cluster_name = "yavin" diff --git a/google-cloud/container-linux/kubernetes/apiserver.tf b/google-cloud/container-linux/kubernetes/apiserver.tf index 883feb244..c6080f9ce 100644 --- a/google-cloud/container-linux/kubernetes/apiserver.tf +++ b/google-cloud/container-linux/kubernetes/apiserver.tf @@ -1,15 +1,15 @@ # TCP Proxy load balancer DNS record resource "google_dns_record_set" "apiserver" { # DNS Zone name where record should be created - managed_zone = "${var.dns_zone_name}" + managed_zone = var.dns_zone_name # DNS record - name = "${format("%s.%s.", var.cluster_name, var.dns_zone)}" + name = format("%s.%s.", var.cluster_name, var.dns_zone) type = "A" ttl = 300 # IPv4 address of apiserver TCP Proxy load balancer - rrdatas = ["${google_compute_global_address.apiserver-ipv4.address}"] + rrdatas = [google_compute_global_address.apiserver-ipv4.address] } # Static IPv4 address for the TCP Proxy Load Balancer @@ -21,17 +21,17 @@ resource "google_compute_global_address" "apiserver-ipv4" { # Forward IPv4 TCP traffic to the TCP proxy load balancer resource "google_compute_global_forwarding_rule" "apiserver" { name = "${var.cluster_name}-apiserver" - ip_address = "${google_compute_global_address.apiserver-ipv4.address}" + ip_address = google_compute_global_address.apiserver-ipv4.address ip_protocol = "TCP" port_range = "443" - target = "${google_compute_target_tcp_proxy.apiserver.self_link}" + target = google_compute_target_tcp_proxy.apiserver.self_link } # Global TCP Proxy Load Balancer for apiservers resource "google_compute_target_tcp_proxy" "apiserver" { name = "${var.cluster_name}-apiserver" description = "Distribute TCP load across ${var.cluster_name} controllers" - backend_service = "${google_compute_backend_service.apiserver.self_link}" + backend_service = google_compute_backend_service.apiserver.self_link } # Global backend service backed by unmanaged instance groups @@ -46,26 +46,30 @@ resource "google_compute_backend_service" "apiserver" { # controller(s) spread across zonal instance groups backend { - group = "${google_compute_instance_group.controllers.0.self_link}" + group = google_compute_instance_group.controllers[0].self_link } backend { - group = "${google_compute_instance_group.controllers.1.self_link}" + group = google_compute_instance_group.controllers[1].self_link } backend { - group = "${google_compute_instance_group.controllers.2.self_link}" + group = google_compute_instance_group.controllers[2].self_link } - health_checks = ["${google_compute_health_check.apiserver.self_link}"] + health_checks = [google_compute_health_check.apiserver.self_link] } # Instance group of heterogeneous (unmanged) controller instances resource "google_compute_instance_group" "controllers" { - count = "${length(local.zones)}" + count = length(local.zones) - name = "${format("%s-controllers-%s", var.cluster_name, element(local.zones, count.index))}" - zone = "${element(local.zones, count.index)}" + name = format( + "%s-controllers-%s", + var.cluster_name, + element(local.zones, count.index), + ) + zone = element(local.zones, count.index) named_port { name = "apiserver" @@ -73,11 +77,11 @@ resource "google_compute_instance_group" "controllers" { } # add instances in the zone into the instance group - instances = [ - "${matchkeys(google_compute_instance.controllers.*.self_link, - google_compute_instance.controllers.*.zone, - list(element(local.zones, count.index)))}", - ] + instances = matchkeys( + google_compute_instance.controllers.*.self_link, + google_compute_instance.controllers.*.zone, + [element(local.zones, count.index)], + ) } # TCP health check for apiserver @@ -95,3 +99,4 @@ resource "google_compute_health_check" "apiserver" { port = "443" } } + diff --git a/google-cloud/container-linux/kubernetes/bootkube.tf b/google-cloud/container-linux/kubernetes/bootkube.tf index d9872305c..997356b30 100644 --- a/google-cloud/container-linux/kubernetes/bootkube.tf +++ b/google-cloud/container-linux/kubernetes/bootkube.tf @@ -1,19 +1,20 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=082921d67905417755609eebda7d39a7e26f7fdb" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=0103bc06bb3f597455a765bf5d916f9b241cbea0" - cluster_name = "${var.cluster_name}" - api_servers = ["${format("%s.%s", var.cluster_name, var.dns_zone)}"] - etcd_servers = ["${google_dns_record_set.etcds.*.name}"] - asset_dir = "${var.asset_dir}" - networking = "${var.networking}" + cluster_name = var.cluster_name + api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] + etcd_servers = google_dns_record_set.etcds.*.name + asset_dir = var.asset_dir + networking = var.networking network_mtu = 1440 - pod_cidr = "${var.pod_cidr}" - service_cidr = "${var.service_cidr}" - cluster_domain_suffix = "${var.cluster_domain_suffix}" - enable_reporting = "${var.enable_reporting}" - enable_aggregation = "${var.enable_aggregation}" + pod_cidr = var.pod_cidr + service_cidr = var.service_cidr + cluster_domain_suffix = var.cluster_domain_suffix + enable_reporting = var.enable_reporting + enable_aggregation = var.enable_aggregation // temporary apiserver_port = 443 } + diff --git a/google-cloud/container-linux/kubernetes/controllers.tf b/google-cloud/container-linux/kubernetes/controllers.tf index f61face6d..2ba068c7b 100644 --- a/google-cloud/container-linux/kubernetes/controllers.tf +++ b/google-cloud/container-linux/kubernetes/controllers.tf @@ -1,106 +1,107 @@ # Discrete DNS records for each controller's private IPv4 for etcd usage resource "google_dns_record_set" "etcds" { - count = "${var.controller_count}" + count = var.controller_count # DNS Zone name where record should be created - managed_zone = "${var.dns_zone_name}" + managed_zone = var.dns_zone_name # DNS record - name = "${format("%s-etcd%d.%s.", var.cluster_name, count.index, var.dns_zone)}" + name = format("%s-etcd%d.%s.", var.cluster_name, count.index, var.dns_zone) type = "A" ttl = 300 # private IPv4 address for etcd - rrdatas = ["${element(google_compute_instance.controllers.*.network_interface.0.network_ip, count.index)}"] + rrdatas = [element(google_compute_instance.controllers.*.network_interface.0.network_ip, count.index)] } # Zones in the region data "google_compute_zones" "all" { - region = "${var.region}" + region = var.region } locals { # TCP proxy load balancers require a fixed number of zonal backends. Spread # controllers over up to 3 zones, since all GCP regions have at least 3. - zones = "${slice(data.google_compute_zones.all.names, 0, 3)}" + zones = slice(data.google_compute_zones.all.names, 0, 3) - controllers_ipv4_public = ["${google_compute_instance.controllers.*.network_interface.0.access_config.0.nat_ip}"] + controllers_ipv4_public = google_compute_instance.controllers.*.network_interface.0.access_config.0.nat_ip } # Controller instances resource "google_compute_instance" "controllers" { - count = "${var.controller_count}" + count = var.controller_count name = "${var.cluster_name}-controller-${count.index}" - zone = "${element(local.zones, count.index)}" - machine_type = "${var.controller_type}" + zone = element(local.zones, count.index) + machine_type = var.controller_type metadata = { - user-data = "${element(data.ct_config.controller-ignitions.*.rendered, count.index)}" + user-data = element(data.ct_config.controller-ignitions.*.rendered, count.index) } boot_disk { auto_delete = true initialize_params { - image = "${var.os_image}" - size = "${var.disk_size}" + image = var.os_image + size = var.disk_size } } network_interface { - network = "${google_compute_network.network.name}" + network = google_compute_network.network.name # Ephemeral external IP - access_config = {} + access_config { + } } can_ip_forward = true tags = ["${var.cluster_name}-controller"] lifecycle { - ignore_changes = [ - "metadata", - ] + ignore_changes = [metadata] } } # Controller Ignition configs data "ct_config" "controller-ignitions" { - count = "${var.controller_count}" - content = "${element(data.template_file.controller-configs.*.rendered, count.index)}" + count = var.controller_count + content = element( + data.template_file.controller-configs.*.rendered, + count.index, + ) pretty_print = false - snippets = ["${var.controller_clc_snippets}"] + snippets = var.controller_clc_snippets } # Controller Container Linux configs data "template_file" "controller-configs" { - count = "${var.controller_count}" + count = var.controller_count - template = "${file("${path.module}/cl/controller.yaml.tmpl")}" + template = file("${path.module}/cl/controller.yaml.tmpl") vars = { # Cannot use cyclic dependencies on controllers or their DNS records etcd_name = "etcd${count.index}" etcd_domain = "${var.cluster_name}-etcd${count.index}.${var.dns_zone}" - # etcd0=https://cluster-etcd0.example.com,etcd1=https://cluster-etcd1.example.com,... - etcd_initial_cluster = "${join(",", data.template_file.etcds.*.rendered)}" - - kubeconfig = "${indent(10, module.bootkube.kubeconfig-kubelet)}" - ssh_authorized_key = "${var.ssh_authorized_key}" - cluster_dns_service_ip = "${cidrhost(var.service_cidr, 10)}" - cluster_domain_suffix = "${var.cluster_domain_suffix}" + etcd_initial_cluster = join(",", data.template_file.etcds.*.rendered) + kubeconfig = indent(10, module.bootkube.kubeconfig-kubelet) + ssh_authorized_key = var.ssh_authorized_key + cluster_dns_service_ip = cidrhost(var.service_cidr, 10) + cluster_domain_suffix = var.cluster_domain_suffix } } data "template_file" "etcds" { - count = "${var.controller_count}" + count = var.controller_count template = "etcd$${index}=https://$${cluster_name}-etcd$${index}.$${dns_zone}:2380" vars = { - index = "${count.index}" - cluster_name = "${var.cluster_name}" - dns_zone = "${var.dns_zone}" + index = count.index + cluster_name = var.cluster_name + dns_zone = var.dns_zone } } + diff --git a/google-cloud/container-linux/kubernetes/ingress.tf b/google-cloud/container-linux/kubernetes/ingress.tf index de84765e3..71b57a082 100644 --- a/google-cloud/container-linux/kubernetes/ingress.tf +++ b/google-cloud/container-linux/kubernetes/ingress.tf @@ -14,52 +14,52 @@ resource "google_compute_global_address" "ingress-ipv6" { # Google Cloud does not allow TCP proxies for port 80. Must use HTTP proxy. resource "google_compute_global_forwarding_rule" "ingress-http-ipv4" { name = "${var.cluster_name}-ingress-http-ipv4" - ip_address = "${google_compute_global_address.ingress-ipv4.address}" + ip_address = google_compute_global_address.ingress-ipv4.address ip_protocol = "TCP" port_range = "80" - target = "${google_compute_target_http_proxy.ingress-http.self_link}" + target = google_compute_target_http_proxy.ingress-http.self_link } # Forward IPv4 TCP traffic to the TCP proxy load balancer resource "google_compute_global_forwarding_rule" "ingress-https-ipv4" { name = "${var.cluster_name}-ingress-https-ipv4" - ip_address = "${google_compute_global_address.ingress-ipv4.address}" + ip_address = google_compute_global_address.ingress-ipv4.address ip_protocol = "TCP" port_range = "443" - target = "${google_compute_target_tcp_proxy.ingress-https.self_link}" + target = google_compute_target_tcp_proxy.ingress-https.self_link } # Forward IPv6 TCP traffic to the HTTP proxy load balancer # Google Cloud does not allow TCP proxies for port 80. Must use HTTP proxy. resource "google_compute_global_forwarding_rule" "ingress-http-ipv6" { name = "${var.cluster_name}-ingress-http-ipv6" - ip_address = "${google_compute_global_address.ingress-ipv6.address}" + ip_address = google_compute_global_address.ingress-ipv6.address ip_protocol = "TCP" port_range = "80" - target = "${google_compute_target_http_proxy.ingress-http.self_link}" + target = google_compute_target_http_proxy.ingress-http.self_link } # Forward IPv6 TCP traffic to the TCP proxy load balancer resource "google_compute_global_forwarding_rule" "ingress-https-ipv6" { name = "${var.cluster_name}-ingress-https-ipv6" - ip_address = "${google_compute_global_address.ingress-ipv6.address}" + ip_address = google_compute_global_address.ingress-ipv6.address ip_protocol = "TCP" port_range = "443" - target = "${google_compute_target_tcp_proxy.ingress-https.self_link}" + target = google_compute_target_tcp_proxy.ingress-https.self_link } # HTTP proxy load balancer for ingress controllers resource "google_compute_target_http_proxy" "ingress-http" { name = "${var.cluster_name}-ingress-http" description = "Distribute HTTP load across ${var.cluster_name} workers" - url_map = "${google_compute_url_map.ingress-http.self_link}" + url_map = google_compute_url_map.ingress-http.self_link } # TCP proxy load balancer for ingress controllers resource "google_compute_target_tcp_proxy" "ingress-https" { name = "${var.cluster_name}-ingress-https" description = "Distribute HTTPS load across ${var.cluster_name} workers" - backend_service = "${google_compute_backend_service.ingress-https.self_link}" + backend_service = google_compute_backend_service.ingress-https.self_link } # HTTP URL Map (required) @@ -67,7 +67,7 @@ resource "google_compute_url_map" "ingress-http" { name = "${var.cluster_name}-ingress-http" # Do not add host/path rules for applications here. Use Ingress resources. - default_service = "${google_compute_backend_service.ingress-http.self_link}" + default_service = google_compute_backend_service.ingress-http.self_link } # Backend service backed by managed instance group of workers @@ -81,10 +81,10 @@ resource "google_compute_backend_service" "ingress-http" { timeout_sec = "60" backend { - group = "${module.workers.instance_group}" + group = module.workers.instance_group } - health_checks = ["${google_compute_health_check.ingress.self_link}"] + health_checks = [google_compute_health_check.ingress.self_link] } # Backend service backed by managed instance group of workers @@ -98,10 +98,10 @@ resource "google_compute_backend_service" "ingress-https" { timeout_sec = "60" backend { - group = "${module.workers.instance_group}" + group = module.workers.instance_group } - health_checks = ["${google_compute_health_check.ingress.self_link}"] + health_checks = [google_compute_health_check.ingress.self_link] } # Ingress HTTP Health Check @@ -120,3 +120,4 @@ resource "google_compute_health_check" "ingress" { request_path = "/healthz" } } + diff --git a/google-cloud/container-linux/kubernetes/network.tf b/google-cloud/container-linux/kubernetes/network.tf index e4d698cc3..4f9d78a84 100644 --- a/google-cloud/container-linux/kubernetes/network.tf +++ b/google-cloud/container-linux/kubernetes/network.tf @@ -1,12 +1,12 @@ resource "google_compute_network" "network" { - name = "${var.cluster_name}" + name = var.cluster_name description = "Network for the ${var.cluster_name} cluster" auto_create_subnetworks = true } resource "google_compute_firewall" "allow-ssh" { name = "${var.cluster_name}-allow-ssh" - network = "${google_compute_network.network.name}" + network = google_compute_network.network.name allow { protocol = "tcp" @@ -19,7 +19,7 @@ resource "google_compute_firewall" "allow-ssh" { resource "google_compute_firewall" "internal-etcd" { name = "${var.cluster_name}-internal-etcd" - network = "${google_compute_network.network.name}" + network = google_compute_network.network.name allow { protocol = "tcp" @@ -33,7 +33,7 @@ resource "google_compute_firewall" "internal-etcd" { # Allow Prometheus to scrape etcd metrics resource "google_compute_firewall" "internal-etcd-metrics" { name = "${var.cluster_name}-internal-etcd-metrics" - network = "${google_compute_network.network.name}" + network = google_compute_network.network.name allow { protocol = "tcp" @@ -46,7 +46,7 @@ resource "google_compute_firewall" "internal-etcd-metrics" { resource "google_compute_firewall" "allow-apiserver" { name = "${var.cluster_name}-allow-apiserver" - network = "${google_compute_network.network.name}" + network = google_compute_network.network.name allow { protocol = "tcp" @@ -60,10 +60,10 @@ resource "google_compute_firewall" "allow-apiserver" { # BGP and IPIP # https://docs.projectcalico.org/latest/reference/public-cloud/gce resource "google_compute_firewall" "internal-bgp" { - count = "${var.networking != "flannel" ? 1 : 0}" + count = var.networking != "flannel" ? 1 : 0 name = "${var.cluster_name}-internal-bgp" - network = "${google_compute_network.network.name}" + network = google_compute_network.network.name allow { protocol = "tcp" @@ -80,10 +80,10 @@ resource "google_compute_firewall" "internal-bgp" { # flannel VXLAN resource "google_compute_firewall" "internal-vxlan" { - count = "${var.networking == "flannel" ? 1 : 0}" + count = var.networking == "flannel" ? 1 : 0 name = "${var.cluster_name}-internal-vxlan" - network = "${google_compute_network.network.name}" + network = google_compute_network.network.name allow { protocol = "udp" @@ -97,7 +97,7 @@ resource "google_compute_firewall" "internal-vxlan" { # Allow Prometheus to scrape node-exporter daemonset resource "google_compute_firewall" "internal-node-exporter" { name = "${var.cluster_name}-internal-node-exporter" - network = "${google_compute_network.network.name}" + network = google_compute_network.network.name allow { protocol = "tcp" @@ -111,7 +111,7 @@ resource "google_compute_firewall" "internal-node-exporter" { # Allow apiserver to access kubelets for exec, log, port-forward resource "google_compute_firewall" "internal-kubelet" { name = "${var.cluster_name}-internal-kubelet" - network = "${google_compute_network.network.name}" + network = google_compute_network.network.name allow { protocol = "tcp" @@ -127,7 +127,7 @@ resource "google_compute_firewall" "internal-kubelet" { resource "google_compute_firewall" "allow-ingress" { name = "${var.cluster_name}-allow-ingress" - network = "${google_compute_network.network.name}" + network = google_compute_network.network.name allow { protocol = "tcp" @@ -140,7 +140,7 @@ resource "google_compute_firewall" "allow-ingress" { resource "google_compute_firewall" "google-ingress-health-checks" { name = "${var.cluster_name}-ingress-health" - network = "${google_compute_network.network.name}" + network = google_compute_network.network.name allow { protocol = "tcp" @@ -149,17 +149,13 @@ resource "google_compute_firewall" "google-ingress-health-checks" { # https://cloud.google.com/load-balancing/docs/health-check-concepts#method source_ranges = [ - # Global LB health checks "35.191.0.0/16", - "130.211.0.0/22", - - # Region LB health checks "35.191.0.0/16", - "209.85.152.0/22", "209.85.204.0/22", ] target_tags = ["${var.cluster_name}-worker"] } + diff --git a/google-cloud/container-linux/kubernetes/outputs.tf b/google-cloud/container-linux/kubernetes/outputs.tf index aefdd9b81..5c6c889bd 100644 --- a/google-cloud/container-linux/kubernetes/outputs.tf +++ b/google-cloud/container-linux/kubernetes/outputs.tf @@ -1,43 +1,44 @@ output "kubeconfig-admin" { - value = "${module.bootkube.kubeconfig-admin}" + value = module.bootkube.kubeconfig-admin } # Outputs for Kubernetes Ingress output "ingress_static_ipv4" { description = "Global IPv4 address for proxy load balancing to the nearest Ingress controller" - value = "${google_compute_global_address.ingress-ipv4.address}" + value = google_compute_global_address.ingress-ipv4.address } output "ingress_static_ipv6" { description = "Global IPv6 address for proxy load balancing to the nearest Ingress controller" - value = "${google_compute_global_address.ingress-ipv6.address}" + value = google_compute_global_address.ingress-ipv6.address } # Outputs for worker pools output "network_name" { - value = "${google_compute_network.network.name}" + value = google_compute_network.network.name } output "kubeconfig" { - value = "${module.bootkube.kubeconfig-kubelet}" + value = module.bootkube.kubeconfig-kubelet } # Outputs for custom firewalling output "network_self_link" { - value = "${google_compute_network.network.self_link}" + value = google_compute_network.network.self_link } # Outputs for custom load balancing output "worker_instance_group" { description = "Worker managed instance group full URL" - value = "${module.workers.instance_group}" + value = module.workers.instance_group } output "worker_target_pool" { description = "Worker target pool self link" - value = "${module.workers.target_pool}" + value = module.workers.target_pool } + diff --git a/google-cloud/container-linux/kubernetes/require.tf b/google-cloud/container-linux/kubernetes/require.tf deleted file mode 100644 index d1f7f4d54..000000000 --- a/google-cloud/container-linux/kubernetes/require.tf +++ /dev/null @@ -1,25 +0,0 @@ -# Terraform version and plugin versions - -terraform { - required_version = ">= 0.11.0" -} - -provider "google" { - version = ">= 1.19, < 3.0" -} - -provider "local" { - version = "~> 1.0" -} - -provider "null" { - version = "~> 1.0" -} - -provider "template" { - version = "~> 1.0" -} - -provider "tls" { - version = "~> 1.0" -} diff --git a/google-cloud/container-linux/kubernetes/ssh.tf b/google-cloud/container-linux/kubernetes/ssh.tf index bba9b9fce..ba298ea39 100644 --- a/google-cloud/container-linux/kubernetes/ssh.tf +++ b/google-cloud/container-linux/kubernetes/ssh.tf @@ -1,46 +1,46 @@ # Secure copy etcd TLS assets to controllers. resource "null_resource" "copy-controller-secrets" { - count = "${var.controller_count}" + count = var.controller_count connection { type = "ssh" - host = "${element(local.controllers_ipv4_public, count.index)}" + host = element(local.controllers_ipv4_public, count.index) user = "core" timeout = "15m" } provisioner "file" { - content = "${module.bootkube.etcd_ca_cert}" + content = module.bootkube.etcd_ca_cert destination = "$HOME/etcd-client-ca.crt" } provisioner "file" { - content = "${module.bootkube.etcd_client_cert}" + content = module.bootkube.etcd_client_cert destination = "$HOME/etcd-client.crt" } provisioner "file" { - content = "${module.bootkube.etcd_client_key}" + content = module.bootkube.etcd_client_key destination = "$HOME/etcd-client.key" } provisioner "file" { - content = "${module.bootkube.etcd_server_cert}" + content = module.bootkube.etcd_server_cert destination = "$HOME/etcd-server.crt" } provisioner "file" { - content = "${module.bootkube.etcd_server_key}" + content = module.bootkube.etcd_server_key destination = "$HOME/etcd-server.key" } provisioner "file" { - content = "${module.bootkube.etcd_peer_cert}" + content = module.bootkube.etcd_peer_cert destination = "$HOME/etcd-peer.crt" } provisioner "file" { - content = "${module.bootkube.etcd_peer_key}" + content = module.bootkube.etcd_peer_key destination = "$HOME/etcd-peer.key" } @@ -64,21 +64,21 @@ resource "null_resource" "copy-controller-secrets" { # one-time self-hosted cluster bootstrapping. resource "null_resource" "bootkube-start" { depends_on = [ - "module.bootkube", - "module.workers", - "google_dns_record_set.apiserver", - "null_resource.copy-controller-secrets", + module.bootkube, + module.workers, + google_dns_record_set.apiserver, + null_resource.copy-controller-secrets, ] connection { type = "ssh" - host = "${element(local.controllers_ipv4_public, 0)}" + host = element(local.controllers_ipv4_public, 0) user = "core" timeout = "15m" } provisioner "file" { - source = "${var.asset_dir}" + source = var.asset_dir destination = "$HOME/assets" } @@ -89,3 +89,4 @@ resource "null_resource" "bootkube-start" { ] } } + diff --git a/google-cloud/container-linux/kubernetes/variables.tf b/google-cloud/container-linux/kubernetes/variables.tf index 971dc3140..874deecb8 100644 --- a/google-cloud/container-linux/kubernetes/variables.tf +++ b/google-cloud/container-linux/kubernetes/variables.tf @@ -1,77 +1,77 @@ variable "cluster_name" { - type = "string" + type = string description = "Unique cluster name (prepended to dns_zone)" } # Google Cloud variable "region" { - type = "string" + type = string description = "Google Cloud Region (e.g. us-central1, see `gcloud compute regions list`)" } variable "dns_zone" { - type = "string" + type = string description = "Google Cloud DNS Zone (e.g. google-cloud.example.com)" } variable "dns_zone_name" { - type = "string" + type = string description = "Google Cloud DNS Zone name (e.g. example-zone)" } # instances variable "controller_count" { - type = "string" + type = string default = "1" description = "Number of controllers (i.e. masters)" } variable "worker_count" { - type = "string" + type = string default = "1" description = "Number of workers" } variable "controller_type" { - type = "string" + type = string default = "n1-standard-1" description = "Machine type for controllers (see `gcloud compute machine-types list`)" } variable "worker_type" { - type = "string" + type = string default = "n1-standard-1" description = "Machine type for controllers (see `gcloud compute machine-types list`)" } variable "os_image" { - type = "string" + type = string default = "coreos-stable" description = "Container Linux image for compute instances (e.g. coreos-stable)" } variable "disk_size" { - type = "string" + type = string default = "40" description = "Size of the disk in GB" } variable "worker_preemptible" { - type = "string" + type = string default = "false" description = "If enabled, Compute Engine will terminate workers randomly within 24 hours" } variable "controller_clc_snippets" { - type = "list" + type = list(string) description = "Controller Container Linux Config snippets" default = [] } variable "worker_clc_snippets" { - type = "list" + type = list(string) description = "Worker Container Linux Config snippets" default = [] } @@ -79,24 +79,24 @@ variable "worker_clc_snippets" { # configuration variable "ssh_authorized_key" { - type = "string" + type = string description = "SSH public key for user 'core'" } variable "asset_dir" { description = "Path to a directory where generated assets should be placed (contains secrets)" - type = "string" + type = string } variable "networking" { description = "Choice of networking provider (flannel or calico)" - type = "string" + type = string default = "calico" } variable "pod_cidr" { description = "CIDR IPv4 range to assign Kubernetes pods" - type = "string" + type = string default = "10.2.0.0/16" } @@ -106,24 +106,26 @@ CIDR IPv4 range to assign Kubernetes services. The 1st IP will be reserved for kube_apiserver, the 10th IP will be reserved for coredns. EOD - type = "string" + + type = string default = "10.3.0.0/16" } variable "cluster_domain_suffix" { description = "Queries for domains with the suffix will be answered by coredns. Default is cluster.local (e.g. foo.default.svc.cluster.local) " - type = "string" - default = "cluster.local" + type = string + default = "cluster.local" } variable "enable_reporting" { - type = "string" + type = string description = "Enable usage or analytics reporting to upstreams (Calico)" - default = "false" + default = "false" } variable "enable_aggregation" { description = "Enable the Kubernetes Aggregation Layer (defaults to false)" - type = "string" - default = "false" + type = string + default = "false" } + diff --git a/google-cloud/container-linux/kubernetes/versions.tf b/google-cloud/container-linux/kubernetes/versions.tf new file mode 100644 index 000000000..9689ce3b2 --- /dev/null +++ b/google-cloud/container-linux/kubernetes/versions.tf @@ -0,0 +1,11 @@ +# Terraform version and plugin versions + +terraform { + required_version = "~> 0.12.0" + required_providers { + google = "~> 2.5" + ct = "~> 0.3.2" + template = "~> 2.1" + null = "~> 2.1" + } +} diff --git a/google-cloud/container-linux/kubernetes/workers.tf b/google-cloud/container-linux/kubernetes/workers.tf index 089524a09..0c76ae712 100644 --- a/google-cloud/container-linux/kubernetes/workers.tf +++ b/google-cloud/container-linux/kubernetes/workers.tf @@ -1,21 +1,22 @@ module "workers" { source = "./workers" - name = "${var.cluster_name}" - cluster_name = "${var.cluster_name}" + name = var.cluster_name + cluster_name = var.cluster_name # GCE - region = "${var.region}" - network = "${google_compute_network.network.name}" - worker_count = "${var.worker_count}" - machine_type = "${var.worker_type}" - os_image = "${var.os_image}" - disk_size = "${var.disk_size}" - preemptible = "${var.worker_preemptible}" + region = var.region + network = google_compute_network.network.name + worker_count = var.worker_count + machine_type = var.worker_type + os_image = var.os_image + disk_size = var.disk_size + preemptible = var.worker_preemptible # configuration - kubeconfig = "${module.bootkube.kubeconfig-kubelet}" - ssh_authorized_key = "${var.ssh_authorized_key}" - service_cidr = "${var.service_cidr}" - cluster_domain_suffix = "${var.cluster_domain_suffix}" - clc_snippets = "${var.worker_clc_snippets}" + kubeconfig = module.bootkube.kubeconfig-kubelet + ssh_authorized_key = var.ssh_authorized_key + service_cidr = var.service_cidr + cluster_domain_suffix = var.cluster_domain_suffix + clc_snippets = var.worker_clc_snippets } + diff --git a/google-cloud/container-linux/kubernetes/workers/outputs.tf b/google-cloud/container-linux/kubernetes/workers/outputs.tf index 0817d675d..b73c42af7 100644 --- a/google-cloud/container-linux/kubernetes/workers/outputs.tf +++ b/google-cloud/container-linux/kubernetes/workers/outputs.tf @@ -2,12 +2,13 @@ output "instance_group" { description = "Worker managed instance group full URL" - value = "${google_compute_region_instance_group_manager.workers.instance_group}" + value = google_compute_region_instance_group_manager.workers.instance_group } # Outputs for regional load balancing output "target_pool" { description = "Worker target pool self link" - value = "${google_compute_target_pool.workers.self_link}" + value = google_compute_target_pool.workers.self_link } + diff --git a/google-cloud/container-linux/kubernetes/workers/target_pool.tf b/google-cloud/container-linux/kubernetes/workers/target_pool.tf index 9197d9781..a07855bfc 100644 --- a/google-cloud/container-linux/kubernetes/workers/target_pool.tf +++ b/google-cloud/container-linux/kubernetes/workers/target_pool.tf @@ -1,11 +1,11 @@ # Target pool for TCP/UDP load balancing resource "google_compute_target_pool" "workers" { name = "${var.name}-worker-pool" - region = "${var.region}" + region = var.region session_affinity = "NONE" health_checks = [ - "${google_compute_http_health_check.workers.name}", + google_compute_http_health_check.workers.name, ] } @@ -20,3 +20,4 @@ resource "google_compute_http_health_check" "workers" { port = 10254 request_path = "/healthz" } + diff --git a/google-cloud/container-linux/kubernetes/workers/variables.tf b/google-cloud/container-linux/kubernetes/workers/variables.tf index 2b5a5e830..a6c6aba33 100644 --- a/google-cloud/container-linux/kubernetes/workers/variables.tf +++ b/google-cloud/container-linux/kubernetes/workers/variables.tf @@ -1,53 +1,53 @@ variable "name" { - type = "string" + type = string description = "Unique name for the worker pool" } variable "cluster_name" { - type = "string" + type = string description = "Must be set to `cluster_name of cluster`" } # Google Cloud variable "region" { - type = "string" + type = string description = "Must be set to `region` of cluster" } variable "network" { - type = "string" + type = string description = "Must be set to `network_name` output by cluster" } # instances variable "worker_count" { - type = "string" + type = string default = "1" description = "Number of worker compute instances the instance group should manage" } variable "machine_type" { - type = "string" + type = string default = "n1-standard-1" description = "Machine type for compute instances (e.g. gcloud compute machine-types list)" } variable "os_image" { - type = "string" + type = string default = "coreos-stable" description = "Container Linux image for compute instanges (e.g. gcloud compute images list)" } variable "disk_size" { - type = "string" + type = string default = "40" description = "Size of the disk in GB" } variable "preemptible" { - type = "string" + type = string default = "false" description = "If enabled, Compute Engine will terminate instances randomly within 24 hours" } @@ -55,12 +55,12 @@ variable "preemptible" { # configuration variable "kubeconfig" { - type = "string" + type = string description = "Must be set to `kubeconfig` output by cluster" } variable "ssh_authorized_key" { - type = "string" + type = string description = "SSH public key for user 'core'" } @@ -70,32 +70,34 @@ CIDR IPv4 range to assign Kubernetes services. The 1st IP will be reserved for kube_apiserver, the 10th IP will be reserved for coredns. EOD - type = "string" + + type = string default = "10.3.0.0/16" } variable "cluster_domain_suffix" { description = "Queries for domains with the suffix will be answered by coredns. Default is cluster.local (e.g. foo.default.svc.cluster.local) " - type = "string" - default = "cluster.local" + type = string + default = "cluster.local" } variable "clc_snippets" { - type = "list" + type = list(string) description = "Container Linux Config snippets" - default = [] + default = [] } # unofficial, undocumented, unsupported, temporary variable "accelerator_type" { - type = "string" - default = "" + type = string + default = "" description = "Google Compute Engine accelerator type (e.g. nvidia-tesla-k80, see gcloud compute accelerator-types list)" } variable "accelerator_count" { - type = "string" - default = "0" + type = string + default = "0" description = "Number of compute engine accelerators" } + diff --git a/google-cloud/container-linux/kubernetes/workers/versions.tf b/google-cloud/container-linux/kubernetes/workers/versions.tf new file mode 100644 index 000000000..ac97c6ac8 --- /dev/null +++ b/google-cloud/container-linux/kubernetes/workers/versions.tf @@ -0,0 +1,4 @@ + +terraform { + required_version = ">= 0.12" +} diff --git a/google-cloud/container-linux/kubernetes/workers/workers.tf b/google-cloud/container-linux/kubernetes/workers/workers.tf index bc258c1a3..18b938f74 100644 --- a/google-cloud/container-linux/kubernetes/workers/workers.tf +++ b/google-cloud/container-linux/kubernetes/workers/workers.tf @@ -5,11 +5,11 @@ resource "google_compute_region_instance_group_manager" "workers" { # instance name prefix for instances in the group base_instance_name = "${var.name}-worker" - instance_template = "${google_compute_instance_template.worker.self_link}" - region = "${var.region}" + instance_template = google_compute_instance_template.worker.self_link + region = var.region - target_size = "${var.worker_count}" - target_pools = ["${google_compute_target_pool.workers.self_link}"] + target_size = var.worker_count + target_pools = [google_compute_target_pool.workers.self_link] named_port { name = "http" @@ -26,37 +26,38 @@ resource "google_compute_region_instance_group_manager" "workers" { resource "google_compute_instance_template" "worker" { name_prefix = "${var.name}-worker-" description = "Worker Instance template" - machine_type = "${var.machine_type}" + machine_type = var.machine_type metadata = { - user-data = "${data.ct_config.worker-ignition.rendered}" + user-data = data.ct_config.worker-ignition.rendered } scheduling { - automatic_restart = "${var.preemptible ? false : true}" - preemptible = "${var.preemptible}" + automatic_restart = var.preemptible ? false : true + preemptible = var.preemptible } disk { auto_delete = true boot = true - source_image = "${var.os_image}" - disk_size_gb = "${var.disk_size}" + source_image = var.os_image + disk_size_gb = var.disk_size } network_interface { - network = "${var.network}" + network = var.network # Ephemeral external IP - access_config = {} + access_config { + } } can_ip_forward = true tags = ["worker", "${var.cluster_name}-worker", "${var.name}-worker"] guest_accelerator { - count = "${var.accelerator_count}" - type = "${var.accelerator_type}" + count = var.accelerator_count + type = var.accelerator_type } lifecycle { @@ -67,19 +68,20 @@ resource "google_compute_instance_template" "worker" { # Worker Ignition config data "ct_config" "worker-ignition" { - content = "${data.template_file.worker-config.rendered}" + content = data.template_file.worker-config.rendered pretty_print = false - snippets = ["${var.clc_snippets}"] + snippets = var.clc_snippets } # Worker Container Linux config data "template_file" "worker-config" { - template = "${file("${path.module}/cl/worker.yaml.tmpl")}" + template = file("${path.module}/cl/worker.yaml.tmpl") vars = { - kubeconfig = "${indent(10, var.kubeconfig)}" - ssh_authorized_key = "${var.ssh_authorized_key}" - cluster_dns_service_ip = "${cidrhost(var.service_cidr, 10)}" - cluster_domain_suffix = "${var.cluster_domain_suffix}" + kubeconfig = indent(10, var.kubeconfig) + ssh_authorized_key = var.ssh_authorized_key + cluster_dns_service_ip = cidrhost(var.service_cidr, 10) + cluster_domain_suffix = var.cluster_domain_suffix } } + From 189487ecaab4f40f1418e98d95d4aecf365f7d4d Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Mon, 27 May 2019 21:43:08 -0700 Subject: [PATCH 121/523] Migrate Azure module Terraform v0.11 to v0.12 * Replace v0.11 bracket type hints with Terraform v0.12 list expressions * Use expression syntax instead of interpolated strings, where suggested * Update Azure tutorial and worker pools documentation * Define Terraform and plugin version requirements in versions.tf * Require azurerm ~> 1.27 to support Terraform v0.12 * Require ct ~> 0.3.2 to support Terraform v0.12 --- CHANGES.md | 29 +++-- azure/container-linux/kubernetes/bootkube.tf | 23 ++-- .../container-linux/kubernetes/controllers.tf | 99 +++++++------- azure/container-linux/kubernetes/lb.tf | 67 +++++----- azure/container-linux/kubernetes/network.tf | 25 ++-- azure/container-linux/kubernetes/outputs.tf | 23 ++-- azure/container-linux/kubernetes/require.tf | 25 ---- azure/container-linux/kubernetes/security.tf | 121 +++++++++--------- azure/container-linux/kubernetes/ssh.tf | 35 +++-- azure/container-linux/kubernetes/variables.tf | 52 ++++---- azure/container-linux/kubernetes/versions.tf | 12 ++ azure/container-linux/kubernetes/workers.tf | 31 ++--- .../kubernetes/workers/variables.tf | 34 ++--- .../kubernetes/workers/versions.tf | 4 + .../kubernetes/workers/workers.tf | 51 ++++---- docs/advanced/worker-pools.md | 30 ++--- docs/cl/azure.md | 39 +----- 17 files changed, 342 insertions(+), 358 deletions(-) delete mode 100644 azure/container-linux/kubernetes/require.tf create mode 100644 azure/container-linux/kubernetes/versions.tf create mode 100644 azure/container-linux/kubernetes/workers/versions.tf diff --git a/CHANGES.md b/CHANGES.md index b99410fff..24ae8114b 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -2,13 +2,31 @@ Notable changes between versions. +#### AWS + +* Migrate from Terraform v0.11 to v0.12.x (**action required!**) + * Require `terraform-provider-aws` v2.7+ to support Terraform v0.12 + * Require `terraform-provider-ct` v0.3.2+ to support Terraform v0.12 + +#### Azure + +* Migrate from Terraform v0.11 to v0.12.x (**action required!**) + * Require `terraform-provider-azurerm` v1.27+ to support Terraform v0.12 + * Require `terraform-provider-ct` v0.3.2+ to support Terraform v0.12 + #### DigitalOcean * Migrate from Terraform v0.11 to v0.12.x (**action required!**) * Require `terraform-provider-digitalocean` v1.3+ to support Terraform v0.12 * Require `terraform-provider-ct` ~> v0.3.2+ to support Terraform v0.12 -## Latest +#### Google Cloud + +* Migrate from Terraform v0.11 to v0.12.x (**action required!**) + * Require `terraform-provider-google` v2.5+ to support Terraform v0.12 + * Require `terraform-provider-ct` v0.3.2+ to support Terraform v0.12 + +## v1.14.3 ## v1.14.3 @@ -18,18 +36,14 @@ Notable changes between versions. * Fix trailing slash in terraform-render-bootkube version ([#479](https://github.com/poseidon/typhoon/pull/479)) * Recommend updating `terraform-provider-ct` plugin from v0.3.1 to [v0.3.2](https://github.com/poseidon/terraform-provider-ct/releases/tag/v0.3.2) ([#487](https://github.com/poseidon/typhoon/pull/487)) -### AWS +#### AWS -* Migrate from Terraform v0.11 to v0.12.x (**action required!**) - * Require `terraform-provider-aws` v2.7+ to support Terraform v0.12 - * Require `terraform-provider-ct` v0.3.2+ to support Terraform v0.12 * Rename `worker` pool module `count` variable to `worker_count` ([#485](https://github.com/poseidon/typhoon/pull/485)) (action required) * `count` will become a reserved variable name in Terraform v0.12 #### Azure * Replace `azurerm_autoscale_setting` with `azurerm_monitor_autoscale_setting` ([#482](https://github.com/poseidon/typhoon/pull/482)) - * Require `terraform-provider-azurerm` v1.22+ (action required) * Rename `worker` pool module `count` variable to `worker_count` ([#485](https://github.com/poseidon/typhoon/pull/485)) (action required) * `count` will become a reserved variable name in Terraform v0.12 @@ -39,9 +53,6 @@ Notable changes between versions. #### Google Cloud -* Migrate from Terraform v0.11 to v0.12.x (**action required!**) - * Require `terraform-provider-google` v2.5+ to support Terraform v0.12 - * Require `terraform-provider-ct` v0.3.2+ to support Terraform v0.12 * Rename `worker` pool module `count` variable to `worker_count` ([#485](https://github.com/poseidon/typhoon/pull/485)) (action required) * `count` is a reserved variable in Terraform v0.12 diff --git a/azure/container-linux/kubernetes/bootkube.tf b/azure/container-linux/kubernetes/bootkube.tf index a27896349..958e0170e 100644 --- a/azure/container-linux/kubernetes/bootkube.tf +++ b/azure/container-linux/kubernetes/bootkube.tf @@ -1,22 +1,23 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=082921d67905417755609eebda7d39a7e26f7fdb" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=0103bc06bb3f597455a765bf5d916f9b241cbea0" - cluster_name = "${var.cluster_name}" - api_servers = ["${format("%s.%s", var.cluster_name, var.dns_zone)}"] - etcd_servers = ["${formatlist("%s.%s", azurerm_dns_a_record.etcds.*.name, var.dns_zone)}"] - asset_dir = "${var.asset_dir}" + cluster_name = var.cluster_name + api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] + etcd_servers = formatlist("%s.%s", azurerm_dns_a_record.etcds.*.name, var.dns_zone) + asset_dir = var.asset_dir - networking = "${var.networking}" + networking = var.networking # only effective with Calico networking # we should be able to use 1450 MTU, but in practice, 1410 was needed network_encapsulation = "vxlan" network_mtu = "1410" - pod_cidr = "${var.pod_cidr}" - service_cidr = "${var.service_cidr}" - cluster_domain_suffix = "${var.cluster_domain_suffix}" - enable_reporting = "${var.enable_reporting}" - enable_aggregation = "${var.enable_aggregation}" + pod_cidr = var.pod_cidr + service_cidr = var.service_cidr + cluster_domain_suffix = var.cluster_domain_suffix + enable_reporting = var.enable_reporting + enable_aggregation = var.enable_aggregation } + diff --git a/azure/container-linux/kubernetes/controllers.tf b/azure/container-linux/kubernetes/controllers.tf index 63a8fb17f..2739eb6ea 100644 --- a/azure/container-linux/kubernetes/controllers.tf +++ b/azure/container-linux/kubernetes/controllers.tf @@ -1,31 +1,34 @@ # Discrete DNS records for each controller's private IPv4 for etcd usage resource "azurerm_dns_a_record" "etcds" { - count = "${var.controller_count}" - resource_group_name = "${var.dns_zone_group}" + count = var.controller_count + resource_group_name = var.dns_zone_group # DNS Zone name where record should be created - zone_name = "${var.dns_zone}" + zone_name = var.dns_zone # DNS record - name = "${format("%s-etcd%d", var.cluster_name, count.index)}" + name = format("%s-etcd%d", var.cluster_name, count.index) ttl = 300 # private IPv4 address for etcd - records = ["${element(azurerm_network_interface.controllers.*.private_ip_address, count.index)}"] + records = [element( + azurerm_network_interface.controllers.*.private_ip_address, + count.index, + )] } locals { # Channel for a Container Linux derivative # coreos-stable -> Container Linux Stable - channel = "${element(split("-", var.os_image), 1)}" + channel = element(split("-", var.os_image), 1) } # Controller availability set to spread controllers resource "azurerm_availability_set" "controllers" { - resource_group_name = "${azurerm_resource_group.cluster.name}" + resource_group_name = azurerm_resource_group.cluster.name name = "${var.cluster_name}-controllers" - location = "${var.region}" + location = var.region platform_fault_domain_count = 2 platform_update_domain_count = 4 managed = true @@ -33,19 +36,19 @@ resource "azurerm_availability_set" "controllers" { # Controller instances resource "azurerm_virtual_machine" "controllers" { - count = "${var.controller_count}" - resource_group_name = "${azurerm_resource_group.cluster.name}" + count = var.controller_count + resource_group_name = azurerm_resource_group.cluster.name name = "${var.cluster_name}-controller-${count.index}" - location = "${var.region}" - availability_set_id = "${azurerm_availability_set.controllers.id}" - vm_size = "${var.controller_type}" + location = var.region + availability_set_id = azurerm_availability_set.controllers.id + vm_size = var.controller_type # boot storage_image_reference { publisher = "CoreOS" offer = "CoreOS" - sku = "${local.channel}" + sku = local.channel version = "latest" } @@ -54,18 +57,18 @@ resource "azurerm_virtual_machine" "controllers" { name = "${var.cluster_name}-controller-${count.index}" create_option = "FromImage" caching = "ReadWrite" - disk_size_gb = "${var.disk_size}" + disk_size_gb = var.disk_size os_type = "Linux" managed_disk_type = "Premium_LRS" } # network - network_interface_ids = ["${element(azurerm_network_interface.controllers.*.id, count.index)}"] + network_interface_ids = [element(azurerm_network_interface.controllers.*.id, count.index)] os_profile { computer_name = "${var.cluster_name}-controller-${count.index}" admin_username = "core" - custom_data = "${element(data.ct_config.controller-ignitions.*.rendered, count.index)}" + custom_data = element(data.ct_config.controller-ignitions.*.rendered, count.index) } # Azure mandates setting an ssh_key, even though Ignition custom_data handles it too @@ -74,7 +77,7 @@ resource "azurerm_virtual_machine" "controllers" { ssh_keys { path = "/home/core/.ssh/authorized_keys" - key_data = "${var.ssh_authorized_key}" + key_data = var.ssh_authorized_key } } @@ -84,85 +87,87 @@ resource "azurerm_virtual_machine" "controllers" { lifecycle { ignore_changes = [ - "storage_os_disk", - "os_profile", + storage_os_disk, + os_profile, ] } } # Controller NICs with public and private IPv4 resource "azurerm_network_interface" "controllers" { - count = "${var.controller_count}" - resource_group_name = "${azurerm_resource_group.cluster.name}" + count = var.controller_count + resource_group_name = azurerm_resource_group.cluster.name name = "${var.cluster_name}-controller-${count.index}" - location = "${azurerm_resource_group.cluster.location}" - network_security_group_id = "${azurerm_network_security_group.controller.id}" + location = azurerm_resource_group.cluster.location + network_security_group_id = azurerm_network_security_group.controller.id ip_configuration { name = "ip0" - subnet_id = "${azurerm_subnet.controller.id}" + subnet_id = azurerm_subnet.controller.id private_ip_address_allocation = "dynamic" # public IPv4 - public_ip_address_id = "${element(azurerm_public_ip.controllers.*.id, count.index)}" + public_ip_address_id = element(azurerm_public_ip.controllers.*.id, count.index) } } # Add controller NICs to the controller backend address pool resource "azurerm_network_interface_backend_address_pool_association" "controllers" { - network_interface_id = "${azurerm_network_interface.controllers.id}" + network_interface_id = azurerm_network_interface.controllers[0].id ip_configuration_name = "ip0" - backend_address_pool_id = "${azurerm_lb_backend_address_pool.controller.id}" + backend_address_pool_id = azurerm_lb_backend_address_pool.controller.id } # Controller public IPv4 addresses resource "azurerm_public_ip" "controllers" { - count = "${var.controller_count}" - resource_group_name = "${azurerm_resource_group.cluster.name}" + count = var.controller_count + resource_group_name = azurerm_resource_group.cluster.name name = "${var.cluster_name}-controller-${count.index}" - location = "${azurerm_resource_group.cluster.location}" + location = azurerm_resource_group.cluster.location sku = "Standard" allocation_method = "Static" } # Controller Ignition configs data "ct_config" "controller-ignitions" { - count = "${var.controller_count}" - content = "${element(data.template_file.controller-configs.*.rendered, count.index)}" + count = var.controller_count + content = element( + data.template_file.controller-configs.*.rendered, + count.index, + ) pretty_print = false - snippets = ["${var.controller_clc_snippets}"] + snippets = var.controller_clc_snippets } # Controller Container Linux configs data "template_file" "controller-configs" { - count = "${var.controller_count}" + count = var.controller_count - template = "${file("${path.module}/cl/controller.yaml.tmpl")}" + template = file("${path.module}/cl/controller.yaml.tmpl") vars = { # Cannot use cyclic dependencies on controllers or their DNS records etcd_name = "etcd${count.index}" etcd_domain = "${var.cluster_name}-etcd${count.index}.${var.dns_zone}" - # etcd0=https://cluster-etcd0.example.com,etcd1=https://cluster-etcd1.example.com,... - etcd_initial_cluster = "${join(",", data.template_file.etcds.*.rendered)}" - - kubeconfig = "${indent(10, module.bootkube.kubeconfig-kubelet)}" - ssh_authorized_key = "${var.ssh_authorized_key}" - cluster_dns_service_ip = "${cidrhost(var.service_cidr, 10)}" - cluster_domain_suffix = "${var.cluster_domain_suffix}" + etcd_initial_cluster = join(",", data.template_file.etcds.*.rendered) + kubeconfig = indent(10, module.bootkube.kubeconfig-kubelet) + ssh_authorized_key = var.ssh_authorized_key + cluster_dns_service_ip = cidrhost(var.service_cidr, 10) + cluster_domain_suffix = var.cluster_domain_suffix } } data "template_file" "etcds" { - count = "${var.controller_count}" + count = var.controller_count template = "etcd$${index}=https://$${cluster_name}-etcd$${index}.$${dns_zone}:2380" vars = { - index = "${count.index}" - cluster_name = "${var.cluster_name}" - dns_zone = "${var.dns_zone}" + index = count.index + cluster_name = var.cluster_name + dns_zone = var.dns_zone } } + diff --git a/azure/container-linux/kubernetes/lb.tf b/azure/container-linux/kubernetes/lb.tf index d010937fc..6de81b22f 100644 --- a/azure/container-linux/kubernetes/lb.tf +++ b/azure/container-linux/kubernetes/lb.tf @@ -1,123 +1,123 @@ # DNS record for the apiserver load balancer resource "azurerm_dns_a_record" "apiserver" { - resource_group_name = "${var.dns_zone_group}" + resource_group_name = var.dns_zone_group # DNS Zone name where record should be created - zone_name = "${var.dns_zone}" + zone_name = var.dns_zone # DNS record - name = "${var.cluster_name}" + name = var.cluster_name ttl = 300 # IPv4 address of apiserver load balancer - records = ["${azurerm_public_ip.apiserver-ipv4.ip_address}"] + records = [azurerm_public_ip.apiserver-ipv4.ip_address] } # Static IPv4 address for the apiserver frontend resource "azurerm_public_ip" "apiserver-ipv4" { - resource_group_name = "${azurerm_resource_group.cluster.name}" + resource_group_name = azurerm_resource_group.cluster.name name = "${var.cluster_name}-apiserver-ipv4" - location = "${var.region}" + location = var.region sku = "Standard" allocation_method = "Static" } # Static IPv4 address for the ingress frontend resource "azurerm_public_ip" "ingress-ipv4" { - resource_group_name = "${azurerm_resource_group.cluster.name}" + resource_group_name = azurerm_resource_group.cluster.name name = "${var.cluster_name}-ingress-ipv4" - location = "${var.region}" + location = var.region sku = "Standard" allocation_method = "Static" } # Network Load Balancer for apiservers and ingress resource "azurerm_lb" "cluster" { - resource_group_name = "${azurerm_resource_group.cluster.name}" + resource_group_name = azurerm_resource_group.cluster.name - name = "${var.cluster_name}" - location = "${var.region}" + name = var.cluster_name + location = var.region sku = "Standard" frontend_ip_configuration { name = "apiserver" - public_ip_address_id = "${azurerm_public_ip.apiserver-ipv4.id}" + public_ip_address_id = azurerm_public_ip.apiserver-ipv4.id } frontend_ip_configuration { name = "ingress" - public_ip_address_id = "${azurerm_public_ip.ingress-ipv4.id}" + public_ip_address_id = azurerm_public_ip.ingress-ipv4.id } } resource "azurerm_lb_rule" "apiserver" { - resource_group_name = "${azurerm_resource_group.cluster.name}" + resource_group_name = azurerm_resource_group.cluster.name name = "apiserver" - loadbalancer_id = "${azurerm_lb.cluster.id}" + loadbalancer_id = azurerm_lb.cluster.id frontend_ip_configuration_name = "apiserver" protocol = "Tcp" frontend_port = 6443 backend_port = 6443 - backend_address_pool_id = "${azurerm_lb_backend_address_pool.controller.id}" - probe_id = "${azurerm_lb_probe.apiserver.id}" + backend_address_pool_id = azurerm_lb_backend_address_pool.controller.id + probe_id = azurerm_lb_probe.apiserver.id } resource "azurerm_lb_rule" "ingress-http" { - resource_group_name = "${azurerm_resource_group.cluster.name}" + resource_group_name = azurerm_resource_group.cluster.name name = "ingress-http" - loadbalancer_id = "${azurerm_lb.cluster.id}" + loadbalancer_id = azurerm_lb.cluster.id frontend_ip_configuration_name = "ingress" protocol = "Tcp" frontend_port = 80 backend_port = 80 - backend_address_pool_id = "${azurerm_lb_backend_address_pool.worker.id}" - probe_id = "${azurerm_lb_probe.ingress.id}" + backend_address_pool_id = azurerm_lb_backend_address_pool.worker.id + probe_id = azurerm_lb_probe.ingress.id } resource "azurerm_lb_rule" "ingress-https" { - resource_group_name = "${azurerm_resource_group.cluster.name}" + resource_group_name = azurerm_resource_group.cluster.name name = "ingress-https" - loadbalancer_id = "${azurerm_lb.cluster.id}" + loadbalancer_id = azurerm_lb.cluster.id frontend_ip_configuration_name = "ingress" protocol = "Tcp" frontend_port = 443 backend_port = 443 - backend_address_pool_id = "${azurerm_lb_backend_address_pool.worker.id}" - probe_id = "${azurerm_lb_probe.ingress.id}" + backend_address_pool_id = azurerm_lb_backend_address_pool.worker.id + probe_id = azurerm_lb_probe.ingress.id } # Address pool of controllers resource "azurerm_lb_backend_address_pool" "controller" { - resource_group_name = "${azurerm_resource_group.cluster.name}" + resource_group_name = azurerm_resource_group.cluster.name name = "controller" - loadbalancer_id = "${azurerm_lb.cluster.id}" + loadbalancer_id = azurerm_lb.cluster.id } # Address pool of workers resource "azurerm_lb_backend_address_pool" "worker" { - resource_group_name = "${azurerm_resource_group.cluster.name}" + resource_group_name = azurerm_resource_group.cluster.name name = "worker" - loadbalancer_id = "${azurerm_lb.cluster.id}" + loadbalancer_id = azurerm_lb.cluster.id } # Health checks / probes # TCP health check for apiserver resource "azurerm_lb_probe" "apiserver" { - resource_group_name = "${azurerm_resource_group.cluster.name}" + resource_group_name = azurerm_resource_group.cluster.name name = "apiserver" - loadbalancer_id = "${azurerm_lb.cluster.id}" + loadbalancer_id = azurerm_lb.cluster.id protocol = "Tcp" port = 6443 @@ -129,10 +129,10 @@ resource "azurerm_lb_probe" "apiserver" { # HTTP health check for ingress resource "azurerm_lb_probe" "ingress" { - resource_group_name = "${azurerm_resource_group.cluster.name}" + resource_group_name = azurerm_resource_group.cluster.name name = "ingress" - loadbalancer_id = "${azurerm_lb.cluster.id}" + loadbalancer_id = azurerm_lb.cluster.id protocol = "Http" port = 10254 request_path = "/healthz" @@ -142,3 +142,4 @@ resource "azurerm_lb_probe" "ingress" { interval_in_seconds = 5 } + diff --git a/azure/container-linux/kubernetes/network.tf b/azure/container-linux/kubernetes/network.tf index da67a3ed0..aa9157b0b 100644 --- a/azure/container-linux/kubernetes/network.tf +++ b/azure/container-linux/kubernetes/network.tf @@ -1,15 +1,15 @@ # Organize cluster into a resource group resource "azurerm_resource_group" "cluster" { - name = "${var.cluster_name}" - location = "${var.region}" + name = var.cluster_name + location = var.region } resource "azurerm_virtual_network" "network" { - resource_group_name = "${azurerm_resource_group.cluster.name}" + resource_group_name = azurerm_resource_group.cluster.name - name = "${var.cluster_name}" - location = "${azurerm_resource_group.cluster.location}" - address_space = ["${var.host_cidr}"] + name = var.cluster_name + location = azurerm_resource_group.cluster.location + address_space = [var.host_cidr] } # Subnets - separate subnets for controller and workers because Azure @@ -17,17 +17,18 @@ resource "azurerm_virtual_network" "network" { # tags like GCP or security group membership like AWS resource "azurerm_subnet" "controller" { - resource_group_name = "${azurerm_resource_group.cluster.name}" + resource_group_name = azurerm_resource_group.cluster.name name = "controller" - virtual_network_name = "${azurerm_virtual_network.network.name}" - address_prefix = "${cidrsubnet(var.host_cidr, 1, 0)}" + virtual_network_name = azurerm_virtual_network.network.name + address_prefix = cidrsubnet(var.host_cidr, 1, 0) } resource "azurerm_subnet" "worker" { - resource_group_name = "${azurerm_resource_group.cluster.name}" + resource_group_name = azurerm_resource_group.cluster.name name = "worker" - virtual_network_name = "${azurerm_virtual_network.network.name}" - address_prefix = "${cidrsubnet(var.host_cidr, 1, 1)}" + virtual_network_name = azurerm_virtual_network.network.name + address_prefix = cidrsubnet(var.host_cidr, 1, 1) } + diff --git a/azure/container-linux/kubernetes/outputs.tf b/azure/container-linux/kubernetes/outputs.tf index bfd2a1439..79a936a80 100644 --- a/azure/container-linux/kubernetes/outputs.tf +++ b/azure/container-linux/kubernetes/outputs.tf @@ -1,55 +1,56 @@ output "kubeconfig-admin" { - value = "${module.bootkube.kubeconfig-admin}" + value = module.bootkube.kubeconfig-admin } # Outputs for Kubernetes Ingress output "ingress_static_ipv4" { - value = "${azurerm_public_ip.ingress-ipv4.ip_address}" + value = azurerm_public_ip.ingress-ipv4.ip_address description = "IPv4 address of the load balancer for distributing traffic to Ingress controllers" } # Outputs for worker pools output "region" { - value = "${azurerm_resource_group.cluster.location}" + value = azurerm_resource_group.cluster.location } output "resource_group_name" { - value = "${azurerm_resource_group.cluster.name}" + value = azurerm_resource_group.cluster.name } output "subnet_id" { - value = "${azurerm_subnet.worker.id}" + value = azurerm_subnet.worker.id } output "security_group_id" { - value = "${azurerm_network_security_group.worker.id}" + value = azurerm_network_security_group.worker.id } output "kubeconfig" { - value = "${module.bootkube.kubeconfig-kubelet}" + value = module.bootkube.kubeconfig-kubelet } # Outputs for custom firewalling output "worker_security_group_name" { - value = "${azurerm_network_security_group.worker.name}" + value = azurerm_network_security_group.worker.name } output "worker_address_prefix" { description = "Worker network subnet CIDR address (for source/destination)" - value = "${azurerm_subnet.worker.address_prefix}" + value = azurerm_subnet.worker.address_prefix } # Outputs for custom load balancing output "loadbalancer_id" { description = "ID of the cluster load balancer" - value = "${azurerm_lb.cluster.id}" + value = azurerm_lb.cluster.id } output "backend_address_pool_id" { description = "ID of the worker backend address pool" - value = "${azurerm_lb_backend_address_pool.worker.id}" + value = azurerm_lb_backend_address_pool.worker.id } + diff --git a/azure/container-linux/kubernetes/require.tf b/azure/container-linux/kubernetes/require.tf deleted file mode 100644 index 836a87d54..000000000 --- a/azure/container-linux/kubernetes/require.tf +++ /dev/null @@ -1,25 +0,0 @@ -# Terraform version and plugin versions - -terraform { - required_version = ">= 0.11.0" -} - -provider "azurerm" { - version = "~> 1.22" -} - -provider "local" { - version = "~> 1.0" -} - -provider "null" { - version = "~> 1.0" -} - -provider "template" { - version = "~> 1.0" -} - -provider "tls" { - version = "~> 1.0" -} diff --git a/azure/container-linux/kubernetes/security.tf b/azure/container-linux/kubernetes/security.tf index 138307099..b9fd1c65f 100644 --- a/azure/container-linux/kubernetes/security.tf +++ b/azure/container-linux/kubernetes/security.tf @@ -1,17 +1,17 @@ # Controller security group resource "azurerm_network_security_group" "controller" { - resource_group_name = "${azurerm_resource_group.cluster.name}" + resource_group_name = azurerm_resource_group.cluster.name name = "${var.cluster_name}-controller" - location = "${azurerm_resource_group.cluster.location}" + location = azurerm_resource_group.cluster.location } resource "azurerm_network_security_rule" "controller-ssh" { - resource_group_name = "${azurerm_resource_group.cluster.name}" + resource_group_name = azurerm_resource_group.cluster.name name = "allow-ssh" - network_security_group_name = "${azurerm_network_security_group.controller.name}" + network_security_group_name = azurerm_network_security_group.controller.name priority = "2000" access = "Allow" direction = "Inbound" @@ -19,45 +19,45 @@ resource "azurerm_network_security_rule" "controller-ssh" { source_port_range = "*" destination_port_range = "22" source_address_prefix = "*" - destination_address_prefix = "${azurerm_subnet.controller.address_prefix}" + destination_address_prefix = azurerm_subnet.controller.address_prefix } resource "azurerm_network_security_rule" "controller-etcd" { - resource_group_name = "${azurerm_resource_group.cluster.name}" + resource_group_name = azurerm_resource_group.cluster.name name = "allow-etcd" - network_security_group_name = "${azurerm_network_security_group.controller.name}" + network_security_group_name = azurerm_network_security_group.controller.name priority = "2005" access = "Allow" direction = "Inbound" protocol = "Tcp" source_port_range = "*" destination_port_range = "2379-2380" - source_address_prefix = "${azurerm_subnet.controller.address_prefix}" - destination_address_prefix = "${azurerm_subnet.controller.address_prefix}" + source_address_prefix = azurerm_subnet.controller.address_prefix + destination_address_prefix = azurerm_subnet.controller.address_prefix } # Allow Prometheus to scrape etcd metrics resource "azurerm_network_security_rule" "controller-etcd-metrics" { - resource_group_name = "${azurerm_resource_group.cluster.name}" + resource_group_name = azurerm_resource_group.cluster.name name = "allow-etcd-metrics" - network_security_group_name = "${azurerm_network_security_group.controller.name}" + network_security_group_name = azurerm_network_security_group.controller.name priority = "2010" access = "Allow" direction = "Inbound" protocol = "Tcp" source_port_range = "*" destination_port_range = "2381" - source_address_prefix = "${azurerm_subnet.worker.address_prefix}" - destination_address_prefix = "${azurerm_subnet.controller.address_prefix}" + source_address_prefix = azurerm_subnet.worker.address_prefix + destination_address_prefix = azurerm_subnet.controller.address_prefix } resource "azurerm_network_security_rule" "controller-apiserver" { - resource_group_name = "${azurerm_resource_group.cluster.name}" + resource_group_name = azurerm_resource_group.cluster.name name = "allow-apiserver" - network_security_group_name = "${azurerm_network_security_group.controller.name}" + network_security_group_name = azurerm_network_security_group.controller.name priority = "2015" access = "Allow" direction = "Inbound" @@ -65,46 +65,46 @@ resource "azurerm_network_security_rule" "controller-apiserver" { source_port_range = "*" destination_port_range = "6443" source_address_prefix = "*" - destination_address_prefix = "${azurerm_subnet.controller.address_prefix}" + destination_address_prefix = azurerm_subnet.controller.address_prefix } resource "azurerm_network_security_rule" "controller-vxlan" { - resource_group_name = "${azurerm_resource_group.cluster.name}" + resource_group_name = azurerm_resource_group.cluster.name name = "allow-vxlan" - network_security_group_name = "${azurerm_network_security_group.controller.name}" + network_security_group_name = azurerm_network_security_group.controller.name priority = "2020" access = "Allow" direction = "Inbound" protocol = "Udp" source_port_range = "*" destination_port_range = "4789" - source_address_prefixes = ["${azurerm_subnet.controller.address_prefix}", "${azurerm_subnet.worker.address_prefix}"] - destination_address_prefix = "${azurerm_subnet.controller.address_prefix}" + source_address_prefixes = [azurerm_subnet.controller.address_prefix, azurerm_subnet.worker.address_prefix] + destination_address_prefix = azurerm_subnet.controller.address_prefix } # Allow Prometheus to scrape node-exporter daemonset resource "azurerm_network_security_rule" "controller-node-exporter" { - resource_group_name = "${azurerm_resource_group.cluster.name}" + resource_group_name = azurerm_resource_group.cluster.name name = "allow-node-exporter" - network_security_group_name = "${azurerm_network_security_group.controller.name}" + network_security_group_name = azurerm_network_security_group.controller.name priority = "2025" access = "Allow" direction = "Inbound" protocol = "Tcp" source_port_range = "*" destination_port_range = "9100" - source_address_prefix = "${azurerm_subnet.worker.address_prefix}" - destination_address_prefix = "${azurerm_subnet.controller.address_prefix}" + source_address_prefix = azurerm_subnet.worker.address_prefix + destination_address_prefix = azurerm_subnet.controller.address_prefix } # Allow apiserver to access kubelet's for exec, log, port-forward resource "azurerm_network_security_rule" "controller-kubelet" { - resource_group_name = "${azurerm_resource_group.cluster.name}" + resource_group_name = azurerm_resource_group.cluster.name name = "allow-kubelet" - network_security_group_name = "${azurerm_network_security_group.controller.name}" + network_security_group_name = azurerm_network_security_group.controller.name priority = "2030" access = "Allow" direction = "Inbound" @@ -113,18 +113,18 @@ resource "azurerm_network_security_rule" "controller-kubelet" { destination_port_range = "10250" # allow Prometheus to scrape kubelet metrics too - source_address_prefixes = ["${azurerm_subnet.controller.address_prefix}", "${azurerm_subnet.worker.address_prefix}"] - destination_address_prefix = "${azurerm_subnet.controller.address_prefix}" + source_address_prefixes = [azurerm_subnet.controller.address_prefix, azurerm_subnet.worker.address_prefix] + destination_address_prefix = azurerm_subnet.controller.address_prefix } # Override Azure AllowVNetInBound and AllowAzureLoadBalancerInBound # https://docs.microsoft.com/en-us/azure/virtual-network/security-overview#default-security-rules resource "azurerm_network_security_rule" "controller-allow-loadblancer" { - resource_group_name = "${azurerm_resource_group.cluster.name}" + resource_group_name = azurerm_resource_group.cluster.name name = "allow-loadbalancer" - network_security_group_name = "${azurerm_network_security_group.controller.name}" + network_security_group_name = azurerm_network_security_group.controller.name priority = "3000" access = "Allow" direction = "Inbound" @@ -136,10 +136,10 @@ resource "azurerm_network_security_rule" "controller-allow-loadblancer" { } resource "azurerm_network_security_rule" "controller-deny-all" { - resource_group_name = "${azurerm_resource_group.cluster.name}" + resource_group_name = azurerm_resource_group.cluster.name name = "deny-all" - network_security_group_name = "${azurerm_network_security_group.controller.name}" + network_security_group_name = azurerm_network_security_group.controller.name priority = "3005" access = "Deny" direction = "Inbound" @@ -153,32 +153,32 @@ resource "azurerm_network_security_rule" "controller-deny-all" { # Worker security group resource "azurerm_network_security_group" "worker" { - resource_group_name = "${azurerm_resource_group.cluster.name}" + resource_group_name = azurerm_resource_group.cluster.name name = "${var.cluster_name}-worker" - location = "${azurerm_resource_group.cluster.location}" + location = azurerm_resource_group.cluster.location } resource "azurerm_network_security_rule" "worker-ssh" { - resource_group_name = "${azurerm_resource_group.cluster.name}" + resource_group_name = azurerm_resource_group.cluster.name name = "allow-ssh" - network_security_group_name = "${azurerm_network_security_group.worker.name}" + network_security_group_name = azurerm_network_security_group.worker.name priority = "2000" access = "Allow" direction = "Inbound" protocol = "Tcp" source_port_range = "*" destination_port_range = "22" - source_address_prefix = "${azurerm_subnet.controller.address_prefix}" - destination_address_prefix = "${azurerm_subnet.worker.address_prefix}" + source_address_prefix = azurerm_subnet.controller.address_prefix + destination_address_prefix = azurerm_subnet.worker.address_prefix } resource "azurerm_network_security_rule" "worker-http" { - resource_group_name = "${azurerm_resource_group.cluster.name}" + resource_group_name = azurerm_resource_group.cluster.name name = "allow-http" - network_security_group_name = "${azurerm_network_security_group.worker.name}" + network_security_group_name = azurerm_network_security_group.worker.name priority = "2005" access = "Allow" direction = "Inbound" @@ -186,14 +186,14 @@ resource "azurerm_network_security_rule" "worker-http" { source_port_range = "*" destination_port_range = "80" source_address_prefix = "*" - destination_address_prefix = "${azurerm_subnet.worker.address_prefix}" + destination_address_prefix = azurerm_subnet.worker.address_prefix } resource "azurerm_network_security_rule" "worker-https" { - resource_group_name = "${azurerm_resource_group.cluster.name}" + resource_group_name = azurerm_resource_group.cluster.name name = "allow-https" - network_security_group_name = "${azurerm_network_security_group.worker.name}" + network_security_group_name = azurerm_network_security_group.worker.name priority = "2010" access = "Allow" direction = "Inbound" @@ -201,46 +201,46 @@ resource "azurerm_network_security_rule" "worker-https" { source_port_range = "*" destination_port_range = "443" source_address_prefix = "*" - destination_address_prefix = "${azurerm_subnet.worker.address_prefix}" + destination_address_prefix = azurerm_subnet.worker.address_prefix } resource "azurerm_network_security_rule" "worker-vxlan" { - resource_group_name = "${azurerm_resource_group.cluster.name}" + resource_group_name = azurerm_resource_group.cluster.name name = "allow-vxlan" - network_security_group_name = "${azurerm_network_security_group.worker.name}" + network_security_group_name = azurerm_network_security_group.worker.name priority = "2015" access = "Allow" direction = "Inbound" protocol = "Udp" source_port_range = "*" destination_port_range = "4789" - source_address_prefixes = ["${azurerm_subnet.controller.address_prefix}", "${azurerm_subnet.worker.address_prefix}"] - destination_address_prefix = "${azurerm_subnet.worker.address_prefix}" + source_address_prefixes = [azurerm_subnet.controller.address_prefix, azurerm_subnet.worker.address_prefix] + destination_address_prefix = azurerm_subnet.worker.address_prefix } # Allow Prometheus to scrape node-exporter daemonset resource "azurerm_network_security_rule" "worker-node-exporter" { - resource_group_name = "${azurerm_resource_group.cluster.name}" + resource_group_name = azurerm_resource_group.cluster.name name = "allow-node-exporter" - network_security_group_name = "${azurerm_network_security_group.worker.name}" + network_security_group_name = azurerm_network_security_group.worker.name priority = "2020" access = "Allow" direction = "Inbound" protocol = "Tcp" source_port_range = "*" destination_port_range = "9100" - source_address_prefix = "${azurerm_subnet.worker.address_prefix}" - destination_address_prefix = "${azurerm_subnet.worker.address_prefix}" + source_address_prefix = azurerm_subnet.worker.address_prefix + destination_address_prefix = azurerm_subnet.worker.address_prefix } # Allow apiserver to access kubelet's for exec, log, port-forward resource "azurerm_network_security_rule" "worker-kubelet" { - resource_group_name = "${azurerm_resource_group.cluster.name}" + resource_group_name = azurerm_resource_group.cluster.name name = "allow-kubelet" - network_security_group_name = "${azurerm_network_security_group.worker.name}" + network_security_group_name = azurerm_network_security_group.worker.name priority = "2025" access = "Allow" direction = "Inbound" @@ -249,18 +249,18 @@ resource "azurerm_network_security_rule" "worker-kubelet" { destination_port_range = "10250" # allow Prometheus to scrape kubelet metrics too - source_address_prefixes = ["${azurerm_subnet.controller.address_prefix}", "${azurerm_subnet.worker.address_prefix}"] - destination_address_prefix = "${azurerm_subnet.worker.address_prefix}" + source_address_prefixes = [azurerm_subnet.controller.address_prefix, azurerm_subnet.worker.address_prefix] + destination_address_prefix = azurerm_subnet.worker.address_prefix } # Override Azure AllowVNetInBound and AllowAzureLoadBalancerInBound # https://docs.microsoft.com/en-us/azure/virtual-network/security-overview#default-security-rules resource "azurerm_network_security_rule" "worker-allow-loadblancer" { - resource_group_name = "${azurerm_resource_group.cluster.name}" + resource_group_name = azurerm_resource_group.cluster.name name = "allow-loadbalancer" - network_security_group_name = "${azurerm_network_security_group.worker.name}" + network_security_group_name = azurerm_network_security_group.worker.name priority = "3000" access = "Allow" direction = "Inbound" @@ -272,10 +272,10 @@ resource "azurerm_network_security_rule" "worker-allow-loadblancer" { } resource "azurerm_network_security_rule" "worker-deny-all" { - resource_group_name = "${azurerm_resource_group.cluster.name}" + resource_group_name = azurerm_resource_group.cluster.name name = "deny-all" - network_security_group_name = "${azurerm_network_security_group.worker.name}" + network_security_group_name = azurerm_network_security_group.worker.name priority = "3005" access = "Deny" direction = "Inbound" @@ -285,3 +285,4 @@ resource "azurerm_network_security_rule" "worker-deny-all" { source_address_prefix = "*" destination_address_prefix = "*" } + diff --git a/azure/container-linux/kubernetes/ssh.tf b/azure/container-linux/kubernetes/ssh.tf index 5c8aef26f..85a5d7b46 100644 --- a/azure/container-linux/kubernetes/ssh.tf +++ b/azure/container-linux/kubernetes/ssh.tf @@ -1,50 +1,48 @@ # Secure copy etcd TLS assets to controllers. resource "null_resource" "copy-controller-secrets" { - count = "${var.controller_count}" + count = var.controller_count - depends_on = [ - "azurerm_virtual_machine.controllers", - ] + depends_on = [azurerm_virtual_machine.controllers] connection { type = "ssh" - host = "${element(azurerm_public_ip.controllers.*.ip_address, count.index)}" + host = element(azurerm_public_ip.controllers.*.ip_address, count.index) user = "core" timeout = "15m" } provisioner "file" { - content = "${module.bootkube.etcd_ca_cert}" + content = module.bootkube.etcd_ca_cert destination = "$HOME/etcd-client-ca.crt" } provisioner "file" { - content = "${module.bootkube.etcd_client_cert}" + content = module.bootkube.etcd_client_cert destination = "$HOME/etcd-client.crt" } provisioner "file" { - content = "${module.bootkube.etcd_client_key}" + content = module.bootkube.etcd_client_key destination = "$HOME/etcd-client.key" } provisioner "file" { - content = "${module.bootkube.etcd_server_cert}" + content = module.bootkube.etcd_server_cert destination = "$HOME/etcd-server.crt" } provisioner "file" { - content = "${module.bootkube.etcd_server_key}" + content = module.bootkube.etcd_server_key destination = "$HOME/etcd-server.key" } provisioner "file" { - content = "${module.bootkube.etcd_peer_cert}" + content = module.bootkube.etcd_peer_cert destination = "$HOME/etcd-peer.crt" } provisioner "file" { - content = "${module.bootkube.etcd_peer_key}" + content = module.bootkube.etcd_peer_key destination = "$HOME/etcd-peer.key" } @@ -68,21 +66,21 @@ resource "null_resource" "copy-controller-secrets" { # one-time self-hosted cluster bootstrapping. resource "null_resource" "bootkube-start" { depends_on = [ - "module.bootkube", - "module.workers", - "azurerm_dns_a_record.apiserver", - "null_resource.copy-controller-secrets", + module.bootkube, + module.workers, + azurerm_dns_a_record.apiserver, + null_resource.copy-controller-secrets, ] connection { type = "ssh" - host = "${element(azurerm_public_ip.controllers.*.ip_address, 0)}" + host = element(azurerm_public_ip.controllers.*.ip_address, 0) user = "core" timeout = "15m" } provisioner "file" { - source = "${var.asset_dir}" + source = var.asset_dir destination = "$HOME/assets" } @@ -93,3 +91,4 @@ resource "null_resource" "bootkube-start" { ] } } + diff --git a/azure/container-linux/kubernetes/variables.tf b/azure/container-linux/kubernetes/variables.tf index 90a14574f..5ff72471a 100644 --- a/azure/container-linux/kubernetes/variables.tf +++ b/azure/container-linux/kubernetes/variables.tf @@ -1,77 +1,77 @@ variable "cluster_name" { - type = "string" + type = string description = "Unique cluster name (prepended to dns_zone)" } # Azure variable "region" { - type = "string" + type = string description = "Azure Region (e.g. centralus , see `az account list-locations --output table`)" } variable "dns_zone" { - type = "string" + type = string description = "Azure DNS Zone (e.g. azure.example.com)" } variable "dns_zone_group" { - type = "string" + type = string description = "Resource group where the Azure DNS Zone resides (e.g. global)" } # instances variable "controller_count" { - type = "string" + type = string default = "1" description = "Number of controllers (i.e. masters)" } variable "worker_count" { - type = "string" + type = string default = "1" description = "Number of workers" } variable "controller_type" { - type = "string" + type = string default = "Standard_DS1_v2" description = "Machine type for controllers (see `az vm list-skus --location centralus`)" } variable "worker_type" { - type = "string" + type = string default = "Standard_F1" description = "Machine type for workers (see `az vm list-skus --location centralus`)" } variable "os_image" { - type = "string" + type = string default = "coreos-stable" description = "Channel for a Container Linux derivative (coreos-stable, coreos-beta, coreos-alpha)" } variable "disk_size" { - type = "string" + type = string default = "40" description = "Size of the disk in GB" } variable "worker_priority" { - type = "string" + type = string default = "Regular" description = "Set worker priority to Low to use reduced cost surplus capacity, with the tradeoff that instances can be deallocated at any time." } variable "controller_clc_snippets" { - type = "list" + type = list(string) description = "Controller Container Linux Config snippets" default = [] } variable "worker_clc_snippets" { - type = "list" + type = list(string) description = "Worker Container Linux Config snippets" default = [] } @@ -79,30 +79,30 @@ variable "worker_clc_snippets" { # configuration variable "ssh_authorized_key" { - type = "string" + type = string description = "SSH public key for user 'core'" } variable "asset_dir" { description = "Path to a directory where generated assets should be placed (contains secrets)" - type = "string" + type = string } variable "networking" { description = "Choice of networking provider (flannel or calico)" - type = "string" + type = string default = "flannel" } variable "host_cidr" { description = "CIDR IPv4 range to assign to instances" - type = "string" + type = string default = "10.0.0.0/16" } variable "pod_cidr" { description = "CIDR IPv4 range to assign Kubernetes pods" - type = "string" + type = string default = "10.2.0.0/16" } @@ -112,24 +112,26 @@ CIDR IPv4 range to assign Kubernetes services. The 1st IP will be reserved for kube_apiserver, the 10th IP will be reserved for coredns. EOD - type = "string" + + type = string default = "10.3.0.0/16" } variable "cluster_domain_suffix" { description = "Queries for domains with the suffix will be answered by coredns. Default is cluster.local (e.g. foo.default.svc.cluster.local) " - type = "string" - default = "cluster.local" + type = string + default = "cluster.local" } variable "enable_reporting" { - type = "string" + type = string description = "Enable usage or analytics reporting to upstreams (Calico)" - default = "false" + default = "false" } variable "enable_aggregation" { description = "Enable the Kubernetes Aggregation Layer (defaults to false)" - type = "string" - default = "false" + type = string + default = "false" } + diff --git a/azure/container-linux/kubernetes/versions.tf b/azure/container-linux/kubernetes/versions.tf new file mode 100644 index 000000000..e4e0a55ba --- /dev/null +++ b/azure/container-linux/kubernetes/versions.tf @@ -0,0 +1,12 @@ +# Terraform version and plugin versions + +terraform { + required_version = "~> 0.12.0" + required_providers { + azurerm = "~> 1.27" + ct = "~> 0.3.2" + template = "~> 2.1" + null = "~> 2.1" + } +} + diff --git a/azure/container-linux/kubernetes/workers.tf b/azure/container-linux/kubernetes/workers.tf index c0d7c4f00..0f40da907 100644 --- a/azure/container-linux/kubernetes/workers.tf +++ b/azure/container-linux/kubernetes/workers.tf @@ -1,23 +1,24 @@ module "workers" { source = "./workers" - name = "${var.cluster_name}" + name = var.cluster_name # Azure - resource_group_name = "${azurerm_resource_group.cluster.name}" - region = "${azurerm_resource_group.cluster.location}" - subnet_id = "${azurerm_subnet.worker.id}" - security_group_id = "${azurerm_network_security_group.worker.id}" - backend_address_pool_id = "${azurerm_lb_backend_address_pool.worker.id}" + resource_group_name = azurerm_resource_group.cluster.name + region = azurerm_resource_group.cluster.location + subnet_id = azurerm_subnet.worker.id + security_group_id = azurerm_network_security_group.worker.id + backend_address_pool_id = azurerm_lb_backend_address_pool.worker.id - worker_count = "${var.worker_count}" - vm_type = "${var.worker_type}" - os_image = "${var.os_image}" - priority = "${var.worker_priority}" + worker_count = var.worker_count + vm_type = var.worker_type + os_image = var.os_image + priority = var.worker_priority # configuration - kubeconfig = "${module.bootkube.kubeconfig-kubelet}" - ssh_authorized_key = "${var.ssh_authorized_key}" - service_cidr = "${var.service_cidr}" - cluster_domain_suffix = "${var.cluster_domain_suffix}" - clc_snippets = "${var.worker_clc_snippets}" + kubeconfig = module.bootkube.kubeconfig-kubelet + ssh_authorized_key = var.ssh_authorized_key + service_cidr = var.service_cidr + cluster_domain_suffix = var.cluster_domain_suffix + clc_snippets = var.worker_clc_snippets } + diff --git a/azure/container-linux/kubernetes/workers/variables.tf b/azure/container-linux/kubernetes/workers/variables.tf index 6e77d1c6e..a430a94a6 100644 --- a/azure/container-linux/kubernetes/workers/variables.tf +++ b/azure/container-linux/kubernetes/workers/variables.tf @@ -1,63 +1,63 @@ variable "name" { - type = "string" + type = string description = "Unique name for the worker pool" } # Azure variable "region" { - type = "string" + type = string description = "Must be set to the Azure Region of cluster" } variable "resource_group_name" { - type = "string" + type = string description = "Must be set to the resource group name of cluster" } variable "subnet_id" { - type = "string" + type = string description = "Must be set to the `worker_subnet_id` output by cluster" } variable "security_group_id" { - type = "string" + type = string description = "Must be set to the `worker_security_group_id` output by cluster" } variable "backend_address_pool_id" { - type = "string" + type = string description = "Must be set to the `worker_backend_address_pool_id` output by cluster" } # instances variable "worker_count" { - type = "string" + type = string default = "1" description = "Number of instances" } variable "vm_type" { - type = "string" + type = string default = "Standard_F1" description = "Machine type for instances (see `az vm list-skus --location centralus`)" } variable "os_image" { - type = "string" + type = string default = "coreos-stable" description = "Channel for a Container Linux derivative (coreos-stable, coreos-beta, coreos-alpha)" } variable "priority" { - type = "string" + type = string default = "Regular" description = "Set priority to Low to use reduced cost surplus capacity, with the tradeoff that instances can be evicted at any time." } variable "clc_snippets" { - type = "list" + type = list(string) description = "Container Linux Config snippets" default = [] } @@ -65,12 +65,12 @@ variable "clc_snippets" { # configuration variable "kubeconfig" { - type = "string" + type = string description = "Must be set to `kubeconfig` output by cluster" } variable "ssh_authorized_key" { - type = "string" + type = string description = "SSH public key for user 'core'" } @@ -80,12 +80,14 @@ CIDR IPv4 range to assign Kubernetes services. The 1st IP will be reserved for kube_apiserver, the 10th IP will be reserved for coredns. EOD - type = "string" + + type = string default = "10.3.0.0/16" } variable "cluster_domain_suffix" { description = "Queries for domains with the suffix will be answered by coredns. Default is cluster.local (e.g. foo.default.svc.cluster.local) " - type = "string" - default = "cluster.local" + type = string + default = "cluster.local" } + diff --git a/azure/container-linux/kubernetes/workers/versions.tf b/azure/container-linux/kubernetes/workers/versions.tf new file mode 100644 index 000000000..ac97c6ac8 --- /dev/null +++ b/azure/container-linux/kubernetes/workers/versions.tf @@ -0,0 +1,4 @@ + +terraform { + required_version = ">= 0.12" +} diff --git a/azure/container-linux/kubernetes/workers/workers.tf b/azure/container-linux/kubernetes/workers/workers.tf index 40ec8a1d3..01e35f7f7 100644 --- a/azure/container-linux/kubernetes/workers/workers.tf +++ b/azure/container-linux/kubernetes/workers/workers.tf @@ -1,28 +1,28 @@ locals { # Channel for a Container Linux derivative # coreos-stable -> Container Linux Stable - channel = "${element(split("-", var.os_image), 1)}" + channel = element(split("-", var.os_image), 1) } # Workers scale set resource "azurerm_virtual_machine_scale_set" "workers" { - resource_group_name = "${var.resource_group_name}" + resource_group_name = var.resource_group_name name = "${var.name}-workers" - location = "${var.region}" + location = var.region single_placement_group = false sku { - name = "${var.vm_type}" + name = var.vm_type tier = "standard" - capacity = "${var.worker_count}" + capacity = var.worker_count } # boot storage_profile_image_reference { publisher = "CoreOS" offer = "CoreOS" - sku = "${local.channel}" + sku = local.channel version = "latest" } @@ -37,7 +37,7 @@ resource "azurerm_virtual_machine_scale_set" "workers" { os_profile { computer_name_prefix = "${var.name}-worker-" admin_username = "core" - custom_data = "${data.ct_config.worker-ignition.rendered}" + custom_data = data.ct_config.worker-ignition.rendered } # Azure mandates setting an ssh_key, even though Ignition custom_data handles it too @@ -46,7 +46,7 @@ resource "azurerm_virtual_machine_scale_set" "workers" { ssh_keys { path = "/home/core/.ssh/authorized_keys" - key_data = "${var.ssh_authorized_key}" + key_data = var.ssh_authorized_key } } @@ -54,61 +54,62 @@ resource "azurerm_virtual_machine_scale_set" "workers" { network_profile { name = "nic0" primary = true - network_security_group_id = "${var.security_group_id}" + network_security_group_id = var.security_group_id ip_configuration { name = "ip0" primary = true - subnet_id = "${var.subnet_id}" + subnet_id = var.subnet_id # backend address pool to which the NIC should be added - load_balancer_backend_address_pool_ids = ["${var.backend_address_pool_id}"] + load_balancer_backend_address_pool_ids = [var.backend_address_pool_id] } } # lifecycle upgrade_policy_mode = "Manual" - priority = "${var.priority}" + priority = var.priority eviction_policy = "Delete" } # Scale up or down to maintain desired number, tolerating deallocations. resource "azurerm_monitor_autoscale_setting" "workers" { - resource_group_name = "${var.resource_group_name}" + resource_group_name = var.resource_group_name name = "${var.name}-maintain-desired" - location = "${var.region}" + location = var.region # autoscale enabled = true - target_resource_id = "${azurerm_virtual_machine_scale_set.workers.id}" + target_resource_id = azurerm_virtual_machine_scale_set.workers.id profile { name = "default" capacity { - minimum = "${var.worker_count}" - default = "${var.worker_count}" - maximum = "${var.worker_count}" + minimum = var.worker_count + default = var.worker_count + maximum = var.worker_count } } } # Worker Ignition configs data "ct_config" "worker-ignition" { - content = "${data.template_file.worker-config.rendered}" + content = data.template_file.worker-config.rendered pretty_print = false - snippets = ["${var.clc_snippets}"] + snippets = var.clc_snippets } # Worker Container Linux configs data "template_file" "worker-config" { - template = "${file("${path.module}/cl/worker.yaml.tmpl")}" + template = file("${path.module}/cl/worker.yaml.tmpl") vars = { - kubeconfig = "${indent(10, var.kubeconfig)}" - ssh_authorized_key = "${var.ssh_authorized_key}" - cluster_dns_service_ip = "${cidrhost(var.service_cidr, 10)}" - cluster_domain_suffix = "${var.cluster_domain_suffix}" + kubeconfig = indent(10, var.kubeconfig) + ssh_authorized_key = var.ssh_authorized_key + cluster_dns_service_ip = cidrhost(var.service_cidr, 10) + cluster_domain_suffix = var.cluster_domain_suffix } } + diff --git a/docs/advanced/worker-pools.md b/docs/advanced/worker-pools.md index 584af00ae..719c7bb68 100644 --- a/docs/advanced/worker-pools.md +++ b/docs/advanced/worker-pools.md @@ -80,21 +80,17 @@ Create a cluster following the Azure [tutorial](../cl/azure.md#cluster). Define module "ramius-worker-pool" { source = "git::https://github.com/poseidon/typhoon//azure/container-linux/kubernetes/workers?ref=v1.14.3" - providers = { - azurerm = "azurerm.default" - } - # Azure - region = "${module.azure-ramius.region}" - resource_group_name = "${module.azure-ramius.resource_group_name}" - subnet_id = "${module.azure-ramius.subnet_id}" - security_group_id = "${module.azure-ramius.security_group_id}" - backend_address_pool_id = "${module.azure-ramius.backend_address_pool_id}" + region = module.azure-ramius.region + resource_group_name = module.azure-ramius.resource_group_name + subnet_id = module.azure-ramius.subnet_id + security_group_id = module.azure-ramius.security_group_id + backend_address_pool_id = module.azure-ramius.backend_address_pool_id # configuration name = "ramius-low-priority" - kubeconfig = "${module.azure-ramius.kubeconfig}" - ssh_authorized_key = "${var.ssh_authorized_key}" + kubeconfig = module.azure-ramius.kubeconfig + ssh_authorized_key = var.ssh_authorized_key # optional worker_count = 2 @@ -120,12 +116,12 @@ The Azure internal `workers` module supports a number of [variables](https://git | Name | Description | Example | |:-----|:------------|:--------| | name | Unique name (distinct from cluster name) | "ramius-f4" | -| region | Must be set to `region` output by cluster | "${module.cluster.region}" | -| resource_group_name | Must be set to `resource_group_name` output by cluster | "${module.cluster.resource_group_name}" | -| subnet_id | Must be set to `subnet_id` output by cluster | "${module.cluster.subnet_id}" | -| security_group_id | Must be set to `security_group_id` output by cluster | "${module.cluster.security_group_id}" | -| backend_address_pool_id | Must be set to `backend_address_pool_id` output by cluster | "${module.cluster.backend_address_pool_id}" | -| kubeconfig | Must be set to `kubeconfig` output by cluster | "${module.cluster.kubeconfig}" | +| region | Must be set to `region` output by cluster | module.cluster.region | +| resource_group_name | Must be set to `resource_group_name` output by cluster | module.cluster.resource_group_name | +| subnet_id | Must be set to `subnet_id` output by cluster | module.cluster.subnet_id | +| security_group_id | Must be set to `security_group_id` output by cluster | module.cluster.security_group_id | +| backend_address_pool_id | Must be set to `backend_address_pool_id` output by cluster | module.cluster.backend_address_pool_id | +| kubeconfig | Must be set to `kubeconfig` output by cluster | module.cluster.kubeconfig | | ssh_authorized_key | SSH public key for user 'core' | "ssh-rsa AAAAB3NZ..." | #### Optional diff --git a/docs/cl/azure.md b/docs/cl/azure.md index c55442158..b7f17a26e 100644 --- a/docs/cl/azure.md +++ b/docs/cl/azure.md @@ -13,15 +13,15 @@ Controllers are provisioned to run an `etcd-member` peer and a `kubelet` service * Azure account * Azure DNS Zone (registered Domain Name or delegated subdomain) -* Terraform v0.11.x and [terraform-provider-ct](https://github.com/poseidon/terraform-provider-ct) installed locally +* Terraform v0.12.x and [terraform-provider-ct](https://github.com/poseidon/terraform-provider-ct) installed locally ## Terraform Setup -Install [Terraform](https://www.terraform.io/downloads.html) v0.11.x on your system. +Install [Terraform](https://www.terraform.io/downloads.html) v0.12.x on your system. ```sh $ terraform version -Terraform v0.11.14 +Terraform v0.12.0 ``` Add the [terraform-provider-ct](https://github.com/poseidon/terraform-provider-ct) plugin binary for your system to `~/.terraform.d/plugins/`, noting the final name. @@ -50,33 +50,12 @@ Configure the Azure provider in a `providers.tf` file. ```tf provider "azurerm" { - version = "~> 1.29.0" - alias = "default" + version = "1.29.0" } provider "ct" { version = "0.3.2" } - -provider "local" { - version = "~> 1.0" - alias = "default" -} - -provider "null" { - version = "~> 1.0" - alias = "default" -} - -provider "template" { - version = "~> 1.0" - alias = "default" -} - -provider "tls" { - version = "~> 1.0" - alias = "default" -} ``` Additional configuration options are described in the `azurerm` provider [docs](https://www.terraform.io/docs/providers/azurerm/). @@ -87,15 +66,7 @@ Define a Kubernetes cluster using the module `azure/container-linux/kubernetes`. ```tf module "azure-ramius" { - source = "git::https://github.com/poseidon/typhoon//azure/container-linux/kubernetes?ref=v1.14.3" - - providers = { - azurerm = "azurerm.default" - local = "local.default" - null = "null.default" - template = "template.default" - tls = "tls.default" - } + source = "git::https://github.com/poseidon/typhoon//azure/container-linux/kubernetes?ref=v1.14.4" # Azure cluster_name = "ramius" From 28506df9c7e0ee3b8dcba8fc5e15524b80cefe69 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Mon, 27 May 2019 22:55:16 -0700 Subject: [PATCH 122/523] Avoid unneeded rotations of Regular priority virtual machine scale sets * Azure only allows `eviction_policy` to be set for Low priority VMs. Supporting Low priority VMs meant when Regular VMs were used, each `terraform apply` rolled workers, to set eviction_policy to null. * Terraform v0.12 nullable variables fix the issue and plan does not produce a diff --- CHANGES.md | 3 +++ azure/container-linux/kubernetes/workers/workers.tf | 3 ++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index 24ae8114b..8f4e32629 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -13,6 +13,9 @@ Notable changes between versions. * Migrate from Terraform v0.11 to v0.12.x (**action required!**) * Require `terraform-provider-azurerm` v1.27+ to support Terraform v0.12 * Require `terraform-provider-ct` v0.3.2+ to support Terraform v0.12 +* Avoid unneeded rotations of Regular priority virtual machine scale sets + * Azure only allows `eviction_policy` to be set for Low priority VMs. Supporting Low priority VMs meant when Regular VMs were used, each `terraform apply` rolled workers, to set eviction_policy to null. + * Terraform v0.12 nullable variables fix the issue and plan does not produce a diff. #### DigitalOcean diff --git a/azure/container-linux/kubernetes/workers/workers.tf b/azure/container-linux/kubernetes/workers/workers.tf index 01e35f7f7..9e32c4b68 100644 --- a/azure/container-linux/kubernetes/workers/workers.tf +++ b/azure/container-linux/kubernetes/workers/workers.tf @@ -68,8 +68,9 @@ resource "azurerm_virtual_machine_scale_set" "workers" { # lifecycle upgrade_policy_mode = "Manual" + # eviction policy may only be set when priority is Low priority = var.priority - eviction_policy = "Delete" + eviction_policy = var.priority == "Low" ? "Delete" : null } # Scale up or down to maintain desired number, tolerating deallocations. From db36959178240c9a8f27e27b639d7178f238b5df Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Tue, 28 May 2019 19:19:23 -0700 Subject: [PATCH 123/523] Migrate bare-metal module Terraform v0.11 to v0.12 * Replace v0.11 bracket type hints with Terraform v0.12 list expressions * Use expression syntax instead of interpolated strings, where suggested * Update bare-metal tutorial * Define `clc_snippets` type constraint map(list(string)) * Define Terraform and plugin version requirements in versions.tf * Require matchbox ~> 0.3.0 to support Terraform v0.12 * Require ct ~> 0.3.2 to support Terraform v0.12 --- CHANGES.md | 27 +-- .../container-linux/kubernetes/bootkube.tf | 27 +-- .../kubernetes/cl/controller.yaml.tmpl | 2 +- .../container-linux/kubernetes/groups.tf | 26 +-- .../container-linux/kubernetes/outputs.tf | 3 +- .../container-linux/kubernetes/profiles.tf | 157 +++++++++--------- .../container-linux/kubernetes/require.tf | 21 --- bare-metal/container-linux/kubernetes/ssh.tf | 47 +++--- .../container-linux/kubernetes/variables.tf | 72 ++++---- .../container-linux/kubernetes/versions.tf | 12 ++ docs/cl/bare-metal.md | 33 +--- 11 files changed, 199 insertions(+), 228 deletions(-) delete mode 100644 bare-metal/container-linux/kubernetes/require.tf create mode 100644 bare-metal/container-linux/kubernetes/versions.tf diff --git a/CHANGES.md b/CHANGES.md index 8f4e32629..eb0ca7c9d 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -2,32 +2,33 @@ Notable changes between versions. -#### AWS +## Latest * Migrate from Terraform v0.11 to v0.12.x (**action required!**) - * Require `terraform-provider-aws` v2.7+ to support Terraform v0.12 - * Require `terraform-provider-ct` v0.3.2+ to support Terraform v0.12 + * Require `terraform-provider-ct` v0.3.2+ to support Terraform v0.12 (action required) + +#### AWS + +* Require `terraform-provider-aws` v2.7+ to support Terraform v0.12 (action required) #### Azure -* Migrate from Terraform v0.11 to v0.12.x (**action required!**) - * Require `terraform-provider-azurerm` v1.27+ to support Terraform v0.12 - * Require `terraform-provider-ct` v0.3.2+ to support Terraform v0.12 +* Require `terraform-provider-azurerm` v1.27+ to support Terraform v0.12 (action required) * Avoid unneeded rotations of Regular priority virtual machine scale sets * Azure only allows `eviction_policy` to be set for Low priority VMs. Supporting Low priority VMs meant when Regular VMs were used, each `terraform apply` rolled workers, to set eviction_policy to null. - * Terraform v0.12 nullable variables fix the issue and plan does not produce a diff. + * Terraform v0.12 nullable variables fix the issue so plan does not produce a diff. + +#### Bare-Metal + +* Require `terraform-provider-matchbox` v0.3.0+ to support Terraform v0.12 (action required) #### DigitalOcean -* Migrate from Terraform v0.11 to v0.12.x (**action required!**) - * Require `terraform-provider-digitalocean` v1.3+ to support Terraform v0.12 - * Require `terraform-provider-ct` ~> v0.3.2+ to support Terraform v0.12 +* Require `terraform-provider-digitalocean` v1.3+ to support Terraform v0.12 (action required) #### Google Cloud -* Migrate from Terraform v0.11 to v0.12.x (**action required!**) - * Require `terraform-provider-google` v2.5+ to support Terraform v0.12 - * Require `terraform-provider-ct` v0.3.2+ to support Terraform v0.12 +* Require `terraform-provider-google` v2.5+ to support Terraform v0.12 (action required) ## v1.14.3 diff --git a/bare-metal/container-linux/kubernetes/bootkube.tf b/bare-metal/container-linux/kubernetes/bootkube.tf index fb82739a0..927226ac5 100644 --- a/bare-metal/container-linux/kubernetes/bootkube.tf +++ b/bare-metal/container-linux/kubernetes/bootkube.tf @@ -1,17 +1,18 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=082921d67905417755609eebda7d39a7e26f7fdb" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=0103bc06bb3f597455a765bf5d916f9b241cbea0" - cluster_name = "${var.cluster_name}" - api_servers = ["${var.k8s_domain_name}"] - etcd_servers = ["${var.controller_domains}"] - asset_dir = "${var.asset_dir}" - networking = "${var.networking}" - network_mtu = "${var.network_mtu}" - network_ip_autodetection_method = "${var.network_ip_autodetection_method}" - pod_cidr = "${var.pod_cidr}" - service_cidr = "${var.service_cidr}" - cluster_domain_suffix = "${var.cluster_domain_suffix}" - enable_reporting = "${var.enable_reporting}" - enable_aggregation = "${var.enable_aggregation}" + cluster_name = var.cluster_name + api_servers = [var.k8s_domain_name] + etcd_servers = var.controller_domains + asset_dir = var.asset_dir + networking = var.networking + network_mtu = var.network_mtu + network_ip_autodetection_method = var.network_ip_autodetection_method + pod_cidr = var.pod_cidr + service_cidr = var.service_cidr + cluster_domain_suffix = var.cluster_domain_suffix + enable_reporting = var.enable_reporting + enable_aggregation = var.enable_aggregation } + diff --git a/bare-metal/container-linux/kubernetes/cl/controller.yaml.tmpl b/bare-metal/container-linux/kubernetes/cl/controller.yaml.tmpl index 827ab5197..13befbc99 100644 --- a/bare-metal/container-linux/kubernetes/cl/controller.yaml.tmpl +++ b/bare-metal/container-linux/kubernetes/cl/controller.yaml.tmpl @@ -160,7 +160,7 @@ storage: --mount volume=assets,target=/assets \ --volume bootstrap,kind=host,source=/etc/kubernetes \ --mount volume=bootstrap,target=/etc/kubernetes \ - $$RKT_OPTS \ + $${RKT_OPTS} \ quay.io/coreos/bootkube:v0.14.0 \ --net=host \ --dns=host \ diff --git a/bare-metal/container-linux/kubernetes/groups.tf b/bare-metal/container-linux/kubernetes/groups.tf index 728ddc4b6..2cc410d90 100644 --- a/bare-metal/container-linux/kubernetes/groups.tf +++ b/bare-metal/container-linux/kubernetes/groups.tf @@ -1,33 +1,35 @@ resource "matchbox_group" "install" { - count = "${length(var.controller_names) + length(var.worker_names)}" + count = length(var.controller_names) + length(var.worker_names) - name = "${format("install-%s", element(concat(var.controller_names, var.worker_names), count.index))}" + name = format("install-%s", element(concat(var.controller_names, var.worker_names), count.index)) - profile = "${local.flavor == "flatcar" ? var.cached_install == "true" ? element(matchbox_profile.cached-flatcar-linux-install.*.name, count.index) : element(matchbox_profile.flatcar-install.*.name, count.index) : var.cached_install == "true" ? element(matchbox_profile.cached-container-linux-install.*.name, count.index) : element(matchbox_profile.container-linux-install.*.name, count.index)}" + # pick one of 4 Matchbox profiles (Container Linux or Flatcar, cached or non-cached) + profile = local.flavor == "flatcar" ? var.cached_install == "true" ? element(matchbox_profile.cached-flatcar-linux-install.*.name, count.index) : element(matchbox_profile.flatcar-install.*.name, count.index) : var.cached_install == "true" ? element(matchbox_profile.cached-container-linux-install.*.name, count.index) : element(matchbox_profile.container-linux-install.*.name, count.index) selector = { - mac = "${element(concat(var.controller_macs, var.worker_macs), count.index)}" + mac = element(concat(var.controller_macs, var.worker_macs), count.index) } } resource "matchbox_group" "controller" { - count = "${length(var.controller_names)}" - name = "${format("%s-%s", var.cluster_name, element(var.controller_names, count.index))}" - profile = "${element(matchbox_profile.controllers.*.name, count.index)}" + count = length(var.controller_names) + name = format("%s-%s", var.cluster_name, element(var.controller_names, count.index)) + profile = element(matchbox_profile.controllers.*.name, count.index) selector = { - mac = "${element(var.controller_macs, count.index)}" + mac = element(var.controller_macs, count.index) os = "installed" } } resource "matchbox_group" "worker" { - count = "${length(var.worker_names)}" - name = "${format("%s-%s", var.cluster_name, element(var.worker_names, count.index))}" - profile = "${element(matchbox_profile.workers.*.name, count.index)}" + count = length(var.worker_names) + name = format("%s-%s", var.cluster_name, element(var.worker_names, count.index)) + profile = element(matchbox_profile.workers.*.name, count.index) selector = { - mac = "${element(var.worker_macs, count.index)}" + mac = element(var.worker_macs, count.index) os = "installed" } } + diff --git a/bare-metal/container-linux/kubernetes/outputs.tf b/bare-metal/container-linux/kubernetes/outputs.tf index a0977ea39..1fd43af63 100644 --- a/bare-metal/container-linux/kubernetes/outputs.tf +++ b/bare-metal/container-linux/kubernetes/outputs.tf @@ -1,3 +1,4 @@ output "kubeconfig-admin" { - value = "${module.bootkube.kubeconfig-admin}" + value = module.bootkube.kubeconfig-admin } + diff --git a/bare-metal/container-linux/kubernetes/profiles.tf b/bare-metal/container-linux/kubernetes/profiles.tf index 7d0d481cc..20f8ec2fa 100644 --- a/bare-metal/container-linux/kubernetes/profiles.tf +++ b/bare-metal/container-linux/kubernetes/profiles.tf @@ -1,15 +1,15 @@ locals { # coreos-stable -> coreos flavor, stable channel # flatcar-stable -> flatcar flavor, stable channel - flavor = "${element(split("-", var.os_channel), 0)}" + flavor = element(split("-", var.os_channel), 0) - channel = "${element(split("-", var.os_channel), 1)}" + channel = element(split("-", var.os_channel), 1) } // Container Linux Install profile (from release.core-os.net) resource "matchbox_profile" "container-linux-install" { - count = "${length(var.controller_names) + length(var.worker_names)}" - name = "${format("%s-container-linux-install-%s", var.cluster_name, element(concat(var.controller_names, var.worker_names), count.index))}" + count = length(var.controller_names) + length(var.worker_names) + name = format("%s-container-linux-install-%s", var.cluster_name, element(concat(var.controller_names, var.worker_names), count.index)) kernel = "${var.download_protocol}://${local.channel}.release.core-os.net/amd64-usr/${var.os_version}/coreos_production_pxe.vmlinuz" @@ -17,32 +17,31 @@ resource "matchbox_profile" "container-linux-install" { "${var.download_protocol}://${local.channel}.release.core-os.net/amd64-usr/${var.os_version}/coreos_production_pxe_image.cpio.gz", ] - args = [ + args = flatten([ "initrd=coreos_production_pxe_image.cpio.gz", "coreos.config.url=${var.matchbox_http_endpoint}/ignition?uuid=$${uuid}&mac=$${mac:hexhyp}", "coreos.first_boot=yes", "console=tty0", "console=ttyS0", - "${var.kernel_args}", - ] + var.kernel_args, + ]) - container_linux_config = "${element(data.template_file.container-linux-install-configs.*.rendered, count.index)}" + container_linux_config = element(data.template_file.container-linux-install-configs.*.rendered, count.index) } data "template_file" "container-linux-install-configs" { - count = "${length(var.controller_names) + length(var.worker_names)}" + count = length(var.controller_names) + length(var.worker_names) - template = "${file("${path.module}/cl/install.yaml.tmpl")}" + template = file("${path.module}/cl/install.yaml.tmpl") vars = { - os_flavor = "${local.flavor}" - os_channel = "${local.channel}" - os_version = "${var.os_version}" - ignition_endpoint = "${format("%s/ignition", var.matchbox_http_endpoint)}" - install_disk = "${var.install_disk}" - container_linux_oem = "${var.container_linux_oem}" - ssh_authorized_key = "${var.ssh_authorized_key}" - + os_flavor = local.flavor + os_channel = local.channel + os_version = var.os_version + ignition_endpoint = format("%s/ignition", var.matchbox_http_endpoint) + install_disk = var.install_disk + container_linux_oem = var.container_linux_oem + ssh_authorized_key = var.ssh_authorized_key # only cached-container-linux profile adds -b baseurl baseurl_flag = "" } @@ -51,8 +50,8 @@ data "template_file" "container-linux-install-configs" { // Container Linux Install profile (from matchbox /assets cache) // Note: Admin must have downloaded os_version into matchbox assets/coreos. resource "matchbox_profile" "cached-container-linux-install" { - count = "${length(var.controller_names) + length(var.worker_names)}" - name = "${format("%s-cached-container-linux-install-%s", var.cluster_name, element(concat(var.controller_names, var.worker_names), count.index))}" + count = length(var.controller_names) + length(var.worker_names) + name = format("%s-cached-container-linux-install-%s", var.cluster_name, element(concat(var.controller_names, var.worker_names), count.index)) kernel = "/assets/coreos/${var.os_version}/coreos_production_pxe.vmlinuz" @@ -60,32 +59,31 @@ resource "matchbox_profile" "cached-container-linux-install" { "/assets/coreos/${var.os_version}/coreos_production_pxe_image.cpio.gz", ] - args = [ + args = flatten([ "initrd=coreos_production_pxe_image.cpio.gz", "coreos.config.url=${var.matchbox_http_endpoint}/ignition?uuid=$${uuid}&mac=$${mac:hexhyp}", "coreos.first_boot=yes", "console=tty0", "console=ttyS0", - "${var.kernel_args}", - ] + var.kernel_args, + ]) - container_linux_config = "${element(data.template_file.cached-container-linux-install-configs.*.rendered, count.index)}" + container_linux_config = element(data.template_file.cached-container-linux-install-configs.*.rendered, count.index) } data "template_file" "cached-container-linux-install-configs" { - count = "${length(var.controller_names) + length(var.worker_names)}" + count = length(var.controller_names) + length(var.worker_names) - template = "${file("${path.module}/cl/install.yaml.tmpl")}" + template = file("${path.module}/cl/install.yaml.tmpl") vars = { - os_flavor = "${local.flavor}" - os_channel = "${local.channel}" - os_version = "${var.os_version}" - ignition_endpoint = "${format("%s/ignition", var.matchbox_http_endpoint)}" - install_disk = "${var.install_disk}" - container_linux_oem = "${var.container_linux_oem}" - ssh_authorized_key = "${var.ssh_authorized_key}" - + os_flavor = local.flavor + os_channel = local.channel + os_version = var.os_version + ignition_endpoint = format("%s/ignition", var.matchbox_http_endpoint) + install_disk = var.install_disk + container_linux_oem = var.container_linux_oem + ssh_authorized_key = var.ssh_authorized_key # profile uses -b baseurl to install from matchbox cache baseurl_flag = "-b ${var.matchbox_http_endpoint}/assets/${local.flavor}" } @@ -93,8 +91,8 @@ data "template_file" "cached-container-linux-install-configs" { // Flatcar Linux install profile (from release.flatcar-linux.net) resource "matchbox_profile" "flatcar-install" { - count = "${length(var.controller_names) + length(var.worker_names)}" - name = "${format("%s-flatcar-install-%s", var.cluster_name, element(concat(var.controller_names, var.worker_names), count.index))}" + count = length(var.controller_names) + length(var.worker_names) + name = format("%s-flatcar-install-%s", var.cluster_name, element(concat(var.controller_names, var.worker_names), count.index)) kernel = "${var.download_protocol}://${local.channel}.release.flatcar-linux.net/amd64-usr/${var.os_version}/flatcar_production_pxe.vmlinuz" @@ -102,23 +100,23 @@ resource "matchbox_profile" "flatcar-install" { "${var.download_protocol}://${local.channel}.release.flatcar-linux.net/amd64-usr/${var.os_version}/flatcar_production_pxe_image.cpio.gz", ] - args = [ + args = flatten([ "initrd=flatcar_production_pxe_image.cpio.gz", "flatcar.config.url=${var.matchbox_http_endpoint}/ignition?uuid=$${uuid}&mac=$${mac:hexhyp}", "flatcar.first_boot=yes", "console=tty0", "console=ttyS0", - "${var.kernel_args}", - ] + var.kernel_args, + ]) - container_linux_config = "${element(data.template_file.container-linux-install-configs.*.rendered, count.index)}" + container_linux_config = element(data.template_file.container-linux-install-configs.*.rendered, count.index) } // Flatcar Linux Install profile (from matchbox /assets cache) // Note: Admin must have downloaded os_version into matchbox assets/flatcar. resource "matchbox_profile" "cached-flatcar-linux-install" { - count = "${length(var.controller_names) + length(var.worker_names)}" - name = "${format("%s-cached-flatcar-linux-install-%s", var.cluster_name, element(concat(var.controller_names, var.worker_names), count.index))}" + count = length(var.controller_names) + length(var.worker_names) + name = format("%s-cached-flatcar-linux-install-%s", var.cluster_name, element(concat(var.controller_names, var.worker_names), count.index)) kernel = "/assets/flatcar/${var.os_version}/flatcar_production_pxe.vmlinuz" @@ -126,90 +124,91 @@ resource "matchbox_profile" "cached-flatcar-linux-install" { "/assets/flatcar/${var.os_version}/flatcar_production_pxe_image.cpio.gz", ] - args = [ + args = flatten([ "initrd=flatcar_production_pxe_image.cpio.gz", "flatcar.config.url=${var.matchbox_http_endpoint}/ignition?uuid=$${uuid}&mac=$${mac:hexhyp}", "flatcar.first_boot=yes", "console=tty0", "console=ttyS0", - "${var.kernel_args}", - ] + var.kernel_args, + ]) - container_linux_config = "${element(data.template_file.cached-container-linux-install-configs.*.rendered, count.index)}" + container_linux_config = element(data.template_file.cached-container-linux-install-configs.*.rendered, count.index) } // Kubernetes Controller profiles resource "matchbox_profile" "controllers" { - count = "${length(var.controller_names)}" - name = "${format("%s-controller-%s", var.cluster_name, element(var.controller_names, count.index))}" - raw_ignition = "${element(data.ct_config.controller-ignitions.*.rendered, count.index)}" + count = length(var.controller_names) + name = format("%s-controller-%s", var.cluster_name, element(var.controller_names, count.index)) + raw_ignition = element(data.ct_config.controller-ignitions.*.rendered, count.index) } data "ct_config" "controller-ignitions" { - count = "${length(var.controller_names)}" - content = "${element(data.template_file.controller-configs.*.rendered, count.index)}" + count = length(var.controller_names) + content = element(data.template_file.controller-configs.*.rendered, count.index) pretty_print = false - - # Must use direct lookup. Cannot use lookup(map, key) since it only works for flat maps - snippets = ["${local.clc_map[element(var.controller_names, count.index)]}"] + snippets = local.clc_map[element(var.controller_names, count.index)] } data "template_file" "controller-configs" { - count = "${length(var.controller_names)}" + count = length(var.controller_names) - template = "${file("${path.module}/cl/controller.yaml.tmpl")}" + template = file("${path.module}/cl/controller.yaml.tmpl") vars = { - domain_name = "${element(var.controller_domains, count.index)}" - etcd_name = "${element(var.controller_names, count.index)}" - etcd_initial_cluster = "${join(",", formatlist("%s=https://%s:2380", var.controller_names, var.controller_domains))}" - cluster_dns_service_ip = "${module.bootkube.cluster_dns_service_ip}" - cluster_domain_suffix = "${var.cluster_domain_suffix}" - ssh_authorized_key = "${var.ssh_authorized_key}" + domain_name = element(var.controller_domains, count.index) + etcd_name = element(var.controller_names, count.index) + etcd_initial_cluster = join(",", formatlist("%s=https://%s:2380", var.controller_names, var.controller_domains)) + cluster_dns_service_ip = module.bootkube.cluster_dns_service_ip + cluster_domain_suffix = var.cluster_domain_suffix + ssh_authorized_key = var.ssh_authorized_key } } // Kubernetes Worker profiles resource "matchbox_profile" "workers" { - count = "${length(var.worker_names)}" - name = "${format("%s-worker-%s", var.cluster_name, element(var.worker_names, count.index))}" - raw_ignition = "${element(data.ct_config.worker-ignitions.*.rendered, count.index)}" + count = length(var.worker_names) + name = format("%s-worker-%s", var.cluster_name, element(var.worker_names, count.index)) + raw_ignition = element(data.ct_config.worker-ignitions.*.rendered, count.index) } data "ct_config" "worker-ignitions" { - count = "${length(var.worker_names)}" - content = "${element(data.template_file.worker-configs.*.rendered, count.index)}" + count = length(var.worker_names) + content = element(data.template_file.worker-configs.*.rendered, count.index) pretty_print = false - - # Must use direct lookup. Cannot use lookup(map, key) since it only works for flat maps - snippets = ["${local.clc_map[element(var.worker_names, count.index)]}"] + snippets = local.clc_map[element(var.worker_names, count.index)] } data "template_file" "worker-configs" { - count = "${length(var.worker_names)}" + count = length(var.worker_names) - template = "${file("${path.module}/cl/worker.yaml.tmpl")}" + template = file("${path.module}/cl/worker.yaml.tmpl") vars = { - domain_name = "${element(var.worker_domains, count.index)}" - cluster_dns_service_ip = "${module.bootkube.cluster_dns_service_ip}" - cluster_domain_suffix = "${var.cluster_domain_suffix}" - ssh_authorized_key = "${var.ssh_authorized_key}" + domain_name = element(var.worker_domains, count.index) + cluster_dns_service_ip = module.bootkube.cluster_dns_service_ip + cluster_domain_suffix = var.cluster_domain_suffix + ssh_authorized_key = var.ssh_authorized_key } } locals { # Hack to workaround https://github.com/hashicorp/terraform/issues/17251 + # Still an issue in Terraform v0.12 https://github.com/hashicorp/terraform/issues/20572 # Default Container Linux config snippets map every node names to list("\n") so # all lookups succeed - clc_defaults = "${zipmap(concat(var.controller_names, var.worker_names), chunklist(data.template_file.clc-default-snippets.*.rendered, 1))}" + clc_defaults = zipmap( + concat(var.controller_names, var.worker_names), + chunklist(data.template_file.clc-default-snippets.*.rendered, 1), + ) # Union of the default and user specific snippets, later overrides prior. - clc_map = "${merge(local.clc_defaults, var.clc_snippets)}" + clc_map = merge(local.clc_defaults, var.clc_snippets) } // Horrible hack to generate a Terraform list of node count length data "template_file" "clc-default-snippets" { - count = "${length(var.controller_names) + length(var.worker_names)}" + count = length(var.controller_names) + length(var.worker_names) template = "\n" } + diff --git a/bare-metal/container-linux/kubernetes/require.tf b/bare-metal/container-linux/kubernetes/require.tf deleted file mode 100644 index a6435bec6..000000000 --- a/bare-metal/container-linux/kubernetes/require.tf +++ /dev/null @@ -1,21 +0,0 @@ -# Terraform version and plugin versions - -terraform { - required_version = ">= 0.11.0" -} - -provider "local" { - version = "~> 1.0" -} - -provider "null" { - version = "~> 1.0" -} - -provider "template" { - version = "~> 1.0" -} - -provider "tls" { - version = "~> 1.0" -} diff --git a/bare-metal/container-linux/kubernetes/ssh.tf b/bare-metal/container-linux/kubernetes/ssh.tf index 661295f4e..72f3c58fd 100644 --- a/bare-metal/container-linux/kubernetes/ssh.tf +++ b/bare-metal/container-linux/kubernetes/ssh.tf @@ -1,59 +1,59 @@ # Secure copy etcd TLS assets and kubeconfig to controllers. Activates kubelet.service resource "null_resource" "copy-controller-secrets" { - count = "${length(var.controller_names)}" + count = length(var.controller_names) # Without depends_on, remote-exec could start and wait for machines before # matchbox groups are written, causing a deadlock. depends_on = [ - "matchbox_group.install", - "matchbox_group.controller", - "matchbox_group.worker", + matchbox_group.install, + matchbox_group.controller, + matchbox_group.worker, ] connection { type = "ssh" - host = "${element(var.controller_domains, count.index)}" + host = element(var.controller_domains, count.index) user = "core" timeout = "60m" } provisioner "file" { - content = "${module.bootkube.kubeconfig-kubelet}" + content = module.bootkube.kubeconfig-kubelet destination = "$HOME/kubeconfig" } provisioner "file" { - content = "${module.bootkube.etcd_ca_cert}" + content = module.bootkube.etcd_ca_cert destination = "$HOME/etcd-client-ca.crt" } provisioner "file" { - content = "${module.bootkube.etcd_client_cert}" + content = module.bootkube.etcd_client_cert destination = "$HOME/etcd-client.crt" } provisioner "file" { - content = "${module.bootkube.etcd_client_key}" + content = module.bootkube.etcd_client_key destination = "$HOME/etcd-client.key" } provisioner "file" { - content = "${module.bootkube.etcd_server_cert}" + content = module.bootkube.etcd_server_cert destination = "$HOME/etcd-server.crt" } provisioner "file" { - content = "${module.bootkube.etcd_server_key}" + content = module.bootkube.etcd_server_key destination = "$HOME/etcd-server.key" } provisioner "file" { - content = "${module.bootkube.etcd_peer_cert}" + content = module.bootkube.etcd_peer_cert destination = "$HOME/etcd-peer.crt" } provisioner "file" { - content = "${module.bootkube.etcd_peer_key}" + content = module.bootkube.etcd_peer_key destination = "$HOME/etcd-peer.key" } @@ -76,25 +76,25 @@ resource "null_resource" "copy-controller-secrets" { # Secure copy kubeconfig to all workers. Activates kubelet.service resource "null_resource" "copy-worker-secrets" { - count = "${length(var.worker_names)}" + count = length(var.worker_names) # Without depends_on, remote-exec could start and wait for machines before # matchbox groups are written, causing a deadlock. depends_on = [ - "matchbox_group.install", - "matchbox_group.controller", - "matchbox_group.worker", + matchbox_group.install, + matchbox_group.controller, + matchbox_group.worker, ] connection { type = "ssh" - host = "${element(var.worker_domains, count.index)}" + host = element(var.worker_domains, count.index) user = "core" timeout = "60m" } provisioner "file" { - content = "${module.bootkube.kubeconfig-kubelet}" + content = module.bootkube.kubeconfig-kubelet destination = "$HOME/kubeconfig" } @@ -112,19 +112,19 @@ resource "null_resource" "bootkube-start" { # Terraform only does one task at a time, so it would try to bootstrap # while no Kubelets are running. depends_on = [ - "null_resource.copy-controller-secrets", - "null_resource.copy-worker-secrets", + null_resource.copy-controller-secrets, + null_resource.copy-worker-secrets, ] connection { type = "ssh" - host = "${element(var.controller_domains, 0)}" + host = element(var.controller_domains, 0) user = "core" timeout = "15m" } provisioner "file" { - source = "${var.asset_dir}" + source = var.asset_dir destination = "$HOME/assets" } @@ -135,3 +135,4 @@ resource "null_resource" "bootkube-start" { ] } } + diff --git a/bare-metal/container-linux/kubernetes/variables.tf b/bare-metal/container-linux/kubernetes/variables.tf index 788ecd4aa..18e7d009c 100644 --- a/bare-metal/container-linux/kubernetes/variables.tf +++ b/bare-metal/container-linux/kubernetes/variables.tf @@ -1,22 +1,22 @@ variable "cluster_name" { - type = "string" + type = string description = "Unique cluster name" } # bare-metal variable "matchbox_http_endpoint" { - type = "string" + type = string description = "Matchbox HTTP read-only endpoint (e.g. http://matchbox.example.com:8080)" } variable "os_channel" { - type = "string" + type = string description = "Channel for a Container Linux derivative (coreos-stable, coreos-beta, coreos-alpha, flatcar-stable, flatcar-beta, flatcar-alpha)" } variable "os_version" { - type = "string" + type = string description = "Version for a Container Linux derivative to PXE and install (coreos-stable, coreos-beta, coreos-alpha, flatcar-stable, flatcar-beta, flatcar-alpha)" } @@ -24,37 +24,37 @@ variable "os_version" { # Terraform's crude "type system" does not properly support lists of maps so we do this. variable "controller_names" { - type = "list" + type = list(string) description = "Ordered list of controller names (e.g. [node1])" } variable "controller_macs" { - type = "list" + type = list(string) description = "Ordered list of controller identifying MAC addresses (e.g. [52:54:00:a1:9c:ae])" } variable "controller_domains" { - type = "list" + type = list(string) description = "Ordered list of controller FQDNs (e.g. [node1.example.com])" } variable "worker_names" { - type = "list" + type = list(string) description = "Ordered list of worker names (e.g. [node2, node3])" } variable "worker_macs" { - type = "list" + type = list(string) description = "Ordered list of worker identifying MAC addresses (e.g. [52:54:00:b2:2f:86, 52:54:00:c3:61:77])" } variable "worker_domains" { - type = "list" + type = list(string) description = "Ordered list of worker FQDNs (e.g. [node2.example.com, node3.example.com])" } variable "clc_snippets" { - type = "map" + type = map(list(string)) description = "Map from machine names to lists of Container Linux Config snippets" default = {} } @@ -63,40 +63,40 @@ variable "clc_snippets" { variable "k8s_domain_name" { description = "Controller DNS name which resolves to a controller instance. Workers and kubeconfig's will communicate with this endpoint (e.g. cluster.example.com)" - type = "string" + type = string } variable "ssh_authorized_key" { - type = "string" + type = string description = "SSH public key for user 'core'" } variable "asset_dir" { description = "Path to a directory where generated assets should be placed (contains secrets)" - type = "string" + type = string } variable "networking" { description = "Choice of networking provider (flannel or calico)" - type = "string" + type = string default = "calico" } variable "network_mtu" { description = "CNI interface MTU (applies to calico only)" - type = "string" + type = string default = "1480" } variable "network_ip_autodetection_method" { description = "Method to autodetect the host IPv4 address (applies to calico only)" - type = "string" + type = string default = "first-found" } variable "pod_cidr" { description = "CIDR IPv4 range to assign Kubernetes pods" - type = "string" + type = string default = "10.2.0.0/16" } @@ -106,7 +106,8 @@ CIDR IPv4 range to assign Kubernetes services. The 1st IP will be reserved for kube_apiserver, the 10th IP will be reserved for coredns. EOD - type = "string" + + type = string default = "10.3.0.0/16" } @@ -114,48 +115,49 @@ EOD variable "cluster_domain_suffix" { description = "Queries for domains with the suffix will be answered by coredns. Default is cluster.local (e.g. foo.default.svc.cluster.local) " - type = "string" - default = "cluster.local" + type = string + default = "cluster.local" } variable "download_protocol" { - type = "string" - default = "https" + type = string + default = "https" description = "Protocol iPXE should use to download the kernel and initrd. Defaults to https, which requires iPXE compiled with crypto support. Unused if cached_install is true." } variable "cached_install" { - type = "string" - default = "false" + type = string + default = "false" description = "Whether Container Linux should PXE boot and install from matchbox /assets cache. Note that the admin must have downloaded the os_version into matchbox assets." } variable "install_disk" { - type = "string" - default = "/dev/sda" + type = string + default = "/dev/sda" description = "Disk device to which the install profiles should install Container Linux (e.g. /dev/sda)" } variable "container_linux_oem" { - type = "string" - default = "" + type = string + default = "" description = "DEPRECATED: Specify an OEM image id to use as base for the installation (e.g. ami, vmware_raw, xen) or leave blank for the default image" } variable "kernel_args" { description = "Additional kernel arguments to provide at PXE boot." - type = "list" - default = [] + type = list(string) + default = [] } variable "enable_reporting" { - type = "string" + type = string description = "Enable usage or analytics reporting to upstreams (Calico)" - default = "false" + default = "false" } variable "enable_aggregation" { description = "Enable the Kubernetes Aggregation Layer (defaults to false)" - type = "string" - default = "false" + type = string + default = "false" } + diff --git a/bare-metal/container-linux/kubernetes/versions.tf b/bare-metal/container-linux/kubernetes/versions.tf new file mode 100644 index 000000000..166ee67de --- /dev/null +++ b/bare-metal/container-linux/kubernetes/versions.tf @@ -0,0 +1,12 @@ +# Terraform version and plugin versions + +terraform { + required_version = "~> 0.12.0" + required_providers { + matchbox = "~> 0.3.0" + ct = "~> 0.3.2" + template = "~> 2.1" + null = "~> 2.1" + } +} + diff --git a/docs/cl/bare-metal.md b/docs/cl/bare-metal.md index 3832a4173..992851d0e 100644 --- a/docs/cl/bare-metal.md +++ b/docs/cl/bare-metal.md @@ -12,7 +12,7 @@ Controllers are provisioned to run an `etcd-member` peer and a `kubelet` service * PXE-enabled [network boot](https://coreos.com/matchbox/docs/latest/network-setup.html) environment (with HTTPS support) * Matchbox v0.6+ deployment with API enabled * Matchbox credentials `client.crt`, `client.key`, `ca.crt` -* Terraform v0.11.x, [terraform-provider-matchbox](https://github.com/poseidon/terraform-provider-matchbox), and [terraform-provider-ct](https://github.com/poseidon/terraform-provider-ct) installed locally +* Terraform v0.12.x, [terraform-provider-matchbox](https://github.com/poseidon/terraform-provider-matchbox), and [terraform-provider-ct](https://github.com/poseidon/terraform-provider-ct) installed locally ## Machines @@ -107,11 +107,11 @@ Read about the [many ways](https://coreos.com/matchbox/docs/latest/network-setup ## Terraform Setup -Install [Terraform](https://www.terraform.io/downloads.html) v0.11.x on your system. +Install [Terraform](https://www.terraform.io/downloads.html) v0.12.x on your system. ```sh $ terraform version -Terraform v0.11.14 +Terraform v0.12.0 ``` Add the [terraform-provider-matchbox](https://github.com/poseidon/terraform-provider-matchbox) plugin binary for your system to `~/.terraform.d/plugins/`, noting the final name. @@ -152,26 +152,6 @@ provider "matchbox" { provider "ct" { version = "0.3.2" } - -provider "local" { - version = "~> 1.0" - alias = "default" -} - -provider "null" { - version = "~> 1.0" - alias = "default" -} - -provider "template" { - version = "~> 1.0" - alias = "default" -} - -provider "tls" { - version = "~> 1.0" - alias = "default" -} ``` ## Cluster @@ -182,13 +162,6 @@ Define a Kubernetes cluster using the module `bare-metal/container-linux/kuberne module "bare-metal-mercury" { source = "git::https://github.com/poseidon/typhoon//bare-metal/container-linux/kubernetes?ref=v1.14.3" - providers = { - local = "local.default" - null = "null.default" - template = "template.default" - tls = "tls.default" - } - # bare-metal cluster_name = "mercury" matchbox_http_endpoint = "http://matchbox.example.com" From 3276bf587850218b8f967978a4bf2b05d5f440a2 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Fri, 31 May 2019 00:27:25 -0700 Subject: [PATCH 124/523] Add migration instructions from Terraform v0.11 to v0.12 * Provide Terraform v0.11 to v0.12 migration guide. Show an in-place strategy and a move resources strategy * Describe in-place modifying an existing cluster and providers, using the Terraform helper to edit syntax, and checking the plan produces a zero diff * Describe replacing existing clusters by creating a new config directory for use with Terraform v0.12 only and moving resources one by one * Provide some limited advise on migrating non-Typhoon resources --- CHANGES.md | 3 +- docs/topics/maintenance.md | 158 +++++++++++++++++++++++++++++++++++++ 2 files changed, 160 insertions(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index eb0ca7c9d..46a0e57c6 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -5,7 +5,8 @@ Notable changes between versions. ## Latest * Migrate from Terraform v0.11 to v0.12.x (**action required!**) - * Require `terraform-provider-ct` v0.3.2+ to support Terraform v0.12 (action required) + * [Migration](https://typhoon.psdn.io/topics/maintenance/#terraform-v012x) instructions for Terraform v0.12 +* Require `terraform-provider-ct` v0.3.2+ to support Terraform v0.12 (action required) #### AWS diff --git a/docs/topics/maintenance.md b/docs/topics/maintenance.md index ed99bd9cb..df9906cd6 100644 --- a/docs/topics/maintenance.md +++ b/docs/topics/maintenance.md @@ -271,3 +271,161 @@ Expect downtime. Google Cloud creates a new worker template and edits the worker instance group instantly. Manually terminate workers and replacement workers will use the user-data. +## Terraform v0.12.x + +Terraform [v0.12](https://www.hashicorp.com/blog/announcing-terraform-0-12) introduces major changes to the provider plugin protocol and HCL language (first-class expressions, formal list and map types, nullable variables, variable constraints, and short-circuting ternary operators). + +Typhoon modules have been adapted for Terraform v0.12. Provider plugins requirements now enforce v0.12 compatibility. However, some HCL language changes were breaking. List [type hint](https://www.terraform.io/upgrade-guides/0-12.html#referring-to-list-variables) workarounds in v0.11 now have new meaning. We cannot offer both v0.11 and v0.12 compatibility at the same time. Upgrading Terraform to v0.12 is neccessary. + +| Typhoon Release | Terraform version | +|-------------------|---------------------| +| v1.14.4 - ? | v0.12.x | +| v1.10.3 - v1.14.3 | v0.11.x | +| v1.9.2 - v1.10.2 | v0.10.4+ or v0.11.x | +| v1.7.3 - v1.9.1 | v0.10.x | +| v1.6.4 - v1.7.2 | v0.9.x | + +### New users + +New users can start with Terraform v0.12.x and follow the docs for Typhoon v1.14.4+ without issue. + +### Existing users + +Migrate from Terraform v0.11 to v0.12 either **in-place** (easier, riskier) or by **moving resources** (safer, tedious). + +Install [Terraform](https://www.terraform.io/downloads.html) v0.12.x on your system alongside Terraform v0.11.x. + +```shell +sudo ln -sf ~/Downloads/terraform-0.12.0/terraform /usr/local/bin/terraform12 +``` + +!!! note + For example, `terraform` may refer Terraform v0.11.14, while `terraform12` is symlinked to Terraform v0.12.0. Once migration is complete, Terraform v0.11.x can be deleted and `terraform12` renamed. + +#### In-place + +For existing Typhoon v1.14.2 or v1.14.3 clusters, edit the Typhoon `ref` to the first SHA that introduced Terraform v0.12 support (aim is to minimize the diff). For example: + +```tf + module "bare-metal-mercury" { +- source = "git::https://github.com/poseidon/typhoon//bare-metal/container-linux/kubernetes?ref=v1.14.3" ++ source = "git::https://github.com/poseidon/typhoon//bare-metal/container-linux/kubernetes?ref=v1.14.4" + ... +``` + +Typhoon clusters no longer require the `providers` block (unless you actually need to pass an [aliased provider](https://www.terraform.io/docs/configuration/providers.html#alias-multiple-provider-instances)). A regression in Terraform v0.11 made it neccessary to explicitly pass aliased providers in order for Typhoon to continue to enforce constraints (see [terraform#16824](https://github.com/hashicorp/terraform/issues/16824)). Terraform v0.12 resolves this issue. + +```tf + module "bare-metal-mercury" { + source = "git::https://github.com/poseidon/typhoon//bare-metal/container-linux/kubernetes?ref=v1.14.4" + +- providers = { +- local = "local.default" +- null = "null.default" +- template = "template.default" +- tls = "tls.default" +- } +``` + +Provider constrains ensure suitable plugin versions are used. Install new versions of `terraform-provider-ct` and `terraform-provider-matchbox` (bare-metal only) according to tutorial docs. Remove the now uneccessary `local`, `null`, `template`, and `tls` blocks in `providers.tf`. + +```tf + provider "matchbox" { +- version = "0.2.3" ++ version = "0.3.0" + endpoint = "matchbox.example.com:8081" + client_cert = "${file("~/.config/matchbox/client.crt")}" + client_key = "${file("~/.config/matchbox/client.key")}" + } + + provider "ct" { +- version = "0.3.2" ++ version = "0.3.3" + } +- +-provider "local" { +- version = "~> 1.0" +- alias = "default" +-} +- +-provider "null" { +- version = "~> 1.0" +- alias = "default" +-} +- +-provider "template" { +- version = "~> 1.0" +- alias = "default" +-} +- +-provider "tls" { +- version = "~> 1.0" +- alias = "default" +-} +``` + +Within the Terraform config directory (i.e. working directory), initialize to fetch suitable provider plugins. + +```shell +terraform12 init # using Terraform v0.12 binary, not v0.11 +``` + +Use the Terraform v0.12 upgrade subcommand to convert v0.11 syntax to v0.12. This _will_ edit resource definitions in `*.tf` files in the working directory. Start from a clean version control state. Inspect the changes. Resolve any "TODO" items. + +```shell +terraform12 0.12upgrade +git diff +``` + +Finally, plan. + +```shell +terraform12 plan +``` + +Verify no changes are proposed and commit changes to version control. You've migrated to Terraform v0.12! Use the Terraform v0.12 binary going forward. + +!!! note + It is known that plan may propose re-creating `template_dir` resources. This is harmless. + +!!! error + If plan produced errors, seek to address them (they may be in non-Typhoon resources). If plan proposed a diff, you'll need to evaluate whether that's expected and safe to apply. In-place edits between Typhoon releases aren't supported (favoring blue/green replacement). The larger the version skew, the greater the risk. Use good judgement. If in doubt, abandon the generated changes, delete `.terraform` as [suggested](https://www.terraform.io/upgrade-guides/0-12.html#upgrading-to-terraform-0-12), and try the move resources approach. + +#### Moving Resources + +Alternately, continue maintaining existing clusters using Terraform v0.11.x and existing Terraform configuration directory(ies). Create new Terraform directory(ies) and move resources there to be managed with Terraform v0.12. This approach allows resources to be migrated incrementally and ensures existing resources can always be managed (e.g. emergency patches). + +Create a new Terraform [config directory](/architecture/concepts#organize) for *new* resources. + +```shell +mkdir infra2 +tree . +├── infraA <- existing Terraform v0.11.x configs +└── infraB <- new Terraform v0.12.x configs +``` + +Define Typhoon clusters in the new config directory using Terraform v0.12 syntax. Follow the Typhoon v1.14.4+ docs (e.g. use `terraform12` in the `infraB` dir). See [AWS](/cl/aws), [Azure](/cl/azure), [Bare-Metal](/cl/bare-metal), [Digital Ocean](/cl/digital-ocean), or [Google-Cloud](/cl/google-cloud)) to create new clusters. Follow the usual [upgrade](/topics/maintenance/#upgrades) process to apply workloads and shift traffic. Later, switch back to the old config directory and deprovision clusters with Terraform v0.11. + +```shell +terraform12 init +terraform12 plan +terraform12 apply +``` + +Your Terraform configuration directory likely defines resources other than just Typhoon modules (e.g. application DNS records, firewall rules, etc.). While such migrations are outside Typhoon's scope, you'll probably want to move existing resource definitions into your new Terraform configuration directory. Use Terraform v0.12 to import the resource into the state associated with the new config directory (to avoid trying to recreate a resource that exists). Then with Terraform v0.11 in the old directory, remove the resource from the state (to avoid trying to delete the resource). Verify neither `plan` produces a diff. + +```sh +# move google_dns_record_set.some-app from infraA to infraB +cd infraA +terraform state list +terraform state show google_dns_record_set.some-app + +cd ../infraB +terraform12 import google_dns_record_set.some-app SOMEID +terraform12 plan + +cd ../infraA +terraform state rm google_dns_record_set.some-app +terraform plan +``` + From da3f2b5d95871aa430917b77ddf070c01683fbc4 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Thu, 6 Jun 2019 23:31:29 -0700 Subject: [PATCH 125/523] Adjust README example and Terraform version in docs * Delay changing README example. Its prominent display on github.com may lead to new users copying it, even though it corresponds to an "in between releases" state and v1.14.4 doesn't exist yet * Leave docs tutorials the same, they can reflect master --- CHANGES.md | 2 -- README.md | 10 +++++++++- docs/cl/aws.md | 2 +- docs/cl/azure.md | 2 +- docs/cl/bare-metal.md | 2 +- docs/cl/digital-ocean.md | 2 +- docs/cl/google-cloud.md | 2 +- docs/topics/maintenance.md | 2 +- 8 files changed, 15 insertions(+), 9 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 46a0e57c6..c57c4cc70 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -33,8 +33,6 @@ Notable changes between versions. ## v1.14.3 -## v1.14.3 - * Kubernetes [v1.14.3](https://github.com/kubernetes/kubernetes/blob/master/CHANGELOG-1.14.md#v1143) * Update CoreDNS from v1.3.1 to v1.5.0 * Add `ready` plugin to improve readinessProbe diff --git a/README.md b/README.md index 266f961cd..69b3ef8af 100644 --- a/README.md +++ b/README.md @@ -50,7 +50,15 @@ Define a Kubernetes cluster by using the Terraform module for your chosen platfo ```tf module "google-cloud-yavin" { - source = "git::https://github.com/poseidon/typhoon//google-cloud/container-linux/kubernetes?ref=v1.14.4" + source = "git::https://github.com/poseidon/typhoon//google-cloud/container-linux/kubernetes?ref=v1.14.3" + + providers = { + google = "google.default" + local = "local.default" + null = "null.default" + template = "template.default" + tls = "tls.default" + } # Google Cloud cluster_name = "yavin" diff --git a/docs/cl/aws.md b/docs/cl/aws.md index 7dbfffc4c..2fbf01f7f 100644 --- a/docs/cl/aws.md +++ b/docs/cl/aws.md @@ -18,7 +18,7 @@ Install [Terraform](https://www.terraform.io/downloads.html) v0.12.x on your sys ```sh $ terraform version -Terraform v0.12.0 +Terraform v0.12.1 ``` Add the [terraform-provider-ct](https://github.com/poseidon/terraform-provider-ct) plugin binary for your system to `~/.terraform.d/plugins/`, noting the final name. diff --git a/docs/cl/azure.md b/docs/cl/azure.md index b7f17a26e..7b86eed6b 100644 --- a/docs/cl/azure.md +++ b/docs/cl/azure.md @@ -21,7 +21,7 @@ Install [Terraform](https://www.terraform.io/downloads.html) v0.12.x on your sys ```sh $ terraform version -Terraform v0.12.0 +Terraform v0.12.1 ``` Add the [terraform-provider-ct](https://github.com/poseidon/terraform-provider-ct) plugin binary for your system to `~/.terraform.d/plugins/`, noting the final name. diff --git a/docs/cl/bare-metal.md b/docs/cl/bare-metal.md index 992851d0e..d4068890c 100644 --- a/docs/cl/bare-metal.md +++ b/docs/cl/bare-metal.md @@ -111,7 +111,7 @@ Install [Terraform](https://www.terraform.io/downloads.html) v0.12.x on your sys ```sh $ terraform version -Terraform v0.12.0 +Terraform v0.12.1 ``` Add the [terraform-provider-matchbox](https://github.com/poseidon/terraform-provider-matchbox) plugin binary for your system to `~/.terraform.d/plugins/`, noting the final name. diff --git a/docs/cl/digital-ocean.md b/docs/cl/digital-ocean.md index afe0bb489..4b1217a27 100644 --- a/docs/cl/digital-ocean.md +++ b/docs/cl/digital-ocean.md @@ -18,7 +18,7 @@ Install [Terraform](https://www.terraform.io/downloads.html) v0.12.x on your sys ```sh $ terraform version -Terraform v0.12.0 +Terraform v0.12.1 ``` Add the [terraform-provider-ct](https://github.com/poseidon/terraform-provider-ct) plugin binary for your system to `~/.terraform.d/plugins/`, noting the final name. diff --git a/docs/cl/google-cloud.md b/docs/cl/google-cloud.md index 4edc7aeb8..4cbdf93f8 100644 --- a/docs/cl/google-cloud.md +++ b/docs/cl/google-cloud.md @@ -18,7 +18,7 @@ Install [Terraform](https://www.terraform.io/downloads.html) v0.12.x on your sys ```sh $ terraform version -Terraform v0.12.0 +Terraform v0.12.1 ``` Add the [terraform-provider-ct](https://github.com/poseidon/terraform-provider-ct) plugin binary for your system to `~/.terraform.d/plugins/`, noting the final name. diff --git a/docs/topics/maintenance.md b/docs/topics/maintenance.md index df9906cd6..9652338c7 100644 --- a/docs/topics/maintenance.md +++ b/docs/topics/maintenance.md @@ -300,7 +300,7 @@ sudo ln -sf ~/Downloads/terraform-0.12.0/terraform /usr/local/bin/terraform12 ``` !!! note - For example, `terraform` may refer Terraform v0.11.14, while `terraform12` is symlinked to Terraform v0.12.0. Once migration is complete, Terraform v0.11.x can be deleted and `terraform12` renamed. + For example, `terraform` may refer Terraform v0.11.14, while `terraform12` is symlinked to Terraform v0.12.1. Once migration is complete, Terraform v0.11.x can be deleted and `terraform12` renamed. #### In-place From 5303e32e38650c89d10f9dd057aef7613d9ea678 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Sat, 1 Jun 2019 13:49:56 -0700 Subject: [PATCH 126/523] Change DO worker_type default from s-1vcpu-1gb to s-1vcpu-2gb * On DigitalOcean, `s-1vcpu-1gb` worker nodes have 1GB of RAM, which is too small as a default, even for most cost constrained developers --- CHANGES.md | 1 + digital-ocean/container-linux/kubernetes/variables.tf | 4 ++-- docs/cl/digital-ocean.md | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index c57c4cc70..c09c63001 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -26,6 +26,7 @@ Notable changes between versions. #### DigitalOcean * Require `terraform-provider-digitalocean` v1.3+ to support Terraform v0.12 (action required) +* Change the default `worker_type` from `s-1vcpu1-1gb` to `s-1vcpu-2gb` #### Google Cloud diff --git a/digital-ocean/container-linux/kubernetes/variables.tf b/digital-ocean/container-linux/kubernetes/variables.tf index e1940120c..fbd6df519 100644 --- a/digital-ocean/container-linux/kubernetes/variables.tf +++ b/digital-ocean/container-linux/kubernetes/variables.tf @@ -37,8 +37,8 @@ variable "controller_type" { variable "worker_type" { type = string - default = "s-1vcpu-1gb" - description = "Droplet type for workers (e.g. s-1vcpu-1gb, s-1vcpu-2gb, s-2vcpu-2gb)" + default = "s-1vcpu-2gb" + description = "Droplet type for workers (e.g. s-1vcpu-2gb, s-2vcpu-2gb)" } variable "image" { diff --git a/docs/cl/digital-ocean.md b/docs/cl/digital-ocean.md index 4b1217a27..9ac44a4bc 100644 --- a/docs/cl/digital-ocean.md +++ b/docs/cl/digital-ocean.md @@ -220,7 +220,7 @@ Digital Ocean requires the SSH public key be uploaded to your account, so you ma | controller_count | Number of controllers (i.e. masters) | 1 | 1 | | worker_count | Number of workers | 1 | 3 | | controller_type | Droplet type for controllers | s-2vcpu-2gb | s-2vcpu-2gb, s-2vcpu-4gb, s-4vcpu-8gb, ... | -| worker_type | Droplet type for workers | s-1vcpu-1gb | s-1vcpu-1gb, s-1vcpu-2gb, s-2vcpu-2gb, ... | +| worker_type | Droplet type for workers | s-1vcpu-2gb | s-1vcpu-2gb, s-2vcpu-2gb, ... | | image | Container Linux image for instances | "coreos-stable" | coreos-stable, coreos-beta, coreos-alpha | | controller_clc_snippets | Controller Container Linux Config snippets | [] | [example](/advanced/customization/) | | worker_clc_snippets | Worker Container Linux Config snippets | [] | [example](/advanced/customization/) | From d44947727240d26b02f7f0a6136db77b9f43fb86 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Fri, 7 Jun 2019 00:07:54 -0700 Subject: [PATCH 127/523] Update Grafana from v6.2.1 to v6.2.2 * https://github.com/grafana/grafana/releases/tag/v6.2.2 --- CHANGES.md | 4 ++++ addons/grafana/deployment.yaml | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index c09c63001..5e15b2a3a 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -32,6 +32,10 @@ Notable changes between versions. * Require `terraform-provider-google` v2.5+ to support Terraform v0.12 (action required) +#### Addons + +* Update Grafana from v6.2.1 to v6.2.2 + ## v1.14.3 * Kubernetes [v1.14.3](https://github.com/kubernetes/kubernetes/blob/master/CHANGELOG-1.14.md#v1143) diff --git a/addons/grafana/deployment.yaml b/addons/grafana/deployment.yaml index 2dd6a381f..4279fc6be 100644 --- a/addons/grafana/deployment.yaml +++ b/addons/grafana/deployment.yaml @@ -23,7 +23,7 @@ spec: spec: containers: - name: grafana - image: grafana/grafana:6.2.1 + image: grafana/grafana:6.2.2 env: - name: GF_PATHS_CONFIG value: "/etc/grafana/custom.ini" From f5960e227dd6ca37032bc9ea3c7c3c58014033a7 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Fri, 7 Jun 2019 00:14:54 -0700 Subject: [PATCH 128/523] Update addon-resizer base image to distroless * Rel: https://github.com/kubernetes/kubernetes/pull/78397 --- addons/prometheus/exporters/kube-state-metrics/deployment.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/addons/prometheus/exporters/kube-state-metrics/deployment.yaml b/addons/prometheus/exporters/kube-state-metrics/deployment.yaml index d2e0aff46..d53afb964 100644 --- a/addons/prometheus/exporters/kube-state-metrics/deployment.yaml +++ b/addons/prometheus/exporters/kube-state-metrics/deployment.yaml @@ -35,7 +35,7 @@ spec: initialDelaySeconds: 5 timeoutSeconds: 5 - name: addon-resizer - image: k8s.gcr.io/addon-resizer:1.8.4 + image: k8s.gcr.io/addon-resizer:1.8.5 resources: limits: cpu: 100m From cc4f7e09ab587edaddf72eef296c8973ce17fee3 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Fri, 7 Jun 2019 02:08:28 -0700 Subject: [PATCH 129/523] Update node-exporter from v0.18.0 to v0.18.1 * https://github.com/prometheus/node_exporter/releases/tag/v0.18.1 --- CHANGES.md | 1 + addons/prometheus/exporters/node-exporter/daemonset.yaml | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index 5e15b2a3a..dccead084 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -35,6 +35,7 @@ Notable changes between versions. #### Addons * Update Grafana from v6.2.1 to v6.2.2 +* Update node-exporter from v0.18.0 to v0.18.1 ## v1.14.3 diff --git a/addons/prometheus/exporters/node-exporter/daemonset.yaml b/addons/prometheus/exporters/node-exporter/daemonset.yaml index 3e7d3c1e4..a8e584217 100644 --- a/addons/prometheus/exporters/node-exporter/daemonset.yaml +++ b/addons/prometheus/exporters/node-exporter/daemonset.yaml @@ -28,7 +28,7 @@ spec: hostPID: true containers: - name: node-exporter - image: quay.io/prometheus/node-exporter:v0.18.0 + image: quay.io/prometheus/node-exporter:v0.18.1 args: - --path.procfs=/host/proc - --path.sysfs=/host/sys From e7dda155f38615da41d98c9b221bb9d104307d01 Mon Sep 17 00:00:00 2001 From: Johannes Liebermann Date: Wed, 12 Jun 2019 04:59:42 +0200 Subject: [PATCH 130/523] Fix typo in maintenance docs (#494) s/circuting/circuiting/ --- docs/topics/maintenance.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/topics/maintenance.md b/docs/topics/maintenance.md index 9652338c7..fe4119df2 100644 --- a/docs/topics/maintenance.md +++ b/docs/topics/maintenance.md @@ -273,7 +273,7 @@ Google Cloud creates a new worker template and edits the worker instance group i ## Terraform v0.12.x -Terraform [v0.12](https://www.hashicorp.com/blog/announcing-terraform-0-12) introduces major changes to the provider plugin protocol and HCL language (first-class expressions, formal list and map types, nullable variables, variable constraints, and short-circuting ternary operators). +Terraform [v0.12](https://www.hashicorp.com/blog/announcing-terraform-0-12) introduces major changes to the provider plugin protocol and HCL language (first-class expressions, formal list and map types, nullable variables, variable constraints, and short-circuiting ternary operators). Typhoon modules have been adapted for Terraform v0.12. Provider plugins requirements now enforce v0.12 compatibility. However, some HCL language changes were breaking. List [type hint](https://www.terraform.io/upgrade-guides/0-12.html#referring-to-list-variables) workarounds in v0.11 now have new meaning. We cannot offer both v0.11 and v0.12 compatibility at the same time. Upgrading Terraform to v0.12 is neccessary. From b168db139b2a4f6dc66ec6d030dac60cc72fc27e Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Thu, 13 Jun 2019 23:52:00 -0700 Subject: [PATCH 131/523] Add tweaks to Terraform v0.12 migration docs * Provide an exact SHA early migrators might use to perform an in-place upgrade to Terraform v0.12 --- docs/architecture/operating-systems.md | 2 +- docs/topics/maintenance.md | 12 ++++++------ 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/docs/architecture/operating-systems.md b/docs/architecture/operating-systems.md index 1da4cde87..4c57898ac 100644 --- a/docs/architecture/operating-systems.md +++ b/docs/architecture/operating-systems.md @@ -1,6 +1,6 @@ # Operating Systems -Typhoon supports [Container Linux](https://coreos.com/why/) and Fedora [Atomic](https://www.projectatomic.io/) 28. These two operating systems were chosen because they offer: +Typhoon supports [Container Linux](https://coreos.com/why/) and Fedora [Atomic](https://www.projectatomic.io/) 28 (deprecated). These two operating systems were chosen because they offer: * Minimalism and focus on clustered operation * Automated and atomic operating system upgrades diff --git a/docs/topics/maintenance.md b/docs/topics/maintenance.md index fe4119df2..3e138a77e 100644 --- a/docs/topics/maintenance.md +++ b/docs/topics/maintenance.md @@ -273,9 +273,9 @@ Google Cloud creates a new worker template and edits the worker instance group i ## Terraform v0.12.x -Terraform [v0.12](https://www.hashicorp.com/blog/announcing-terraform-0-12) introduces major changes to the provider plugin protocol and HCL language (first-class expressions, formal list and map types, nullable variables, variable constraints, and short-circuiting ternary operators). +Terraform [v0.12](https://www.hashicorp.com/blog/announcing-terraform-0-12) introduced major changes to the provider plugin protocol and HCL language (first-class expressions, formal list and map types, nullable variables, variable constraints, and short-circuiting ternary operators). -Typhoon modules have been adapted for Terraform v0.12. Provider plugins requirements now enforce v0.12 compatibility. However, some HCL language changes were breaking. List [type hint](https://www.terraform.io/upgrade-guides/0-12.html#referring-to-list-variables) workarounds in v0.11 now have new meaning. We cannot offer both v0.11 and v0.12 compatibility at the same time. Upgrading Terraform to v0.12 is neccessary. +Typhoon modules have been adapted for Terraform v0.12. Provider plugins requirements now enforce v0.12 compatibility. However, some HCL language changes were breaking (list [type hint](https://www.terraform.io/upgrade-guides/0-12.html#referring-to-list-variables) workarounds in v0.11 now have new meaning). Typhoon cannot offer both v0.11 and v0.12 compatibility in the same release. Upcoming releases require upgrading Terraform to v0.12. | Typhoon Release | Terraform version | |-------------------|---------------------| @@ -304,7 +304,7 @@ sudo ln -sf ~/Downloads/terraform-0.12.0/terraform /usr/local/bin/terraform12 #### In-place -For existing Typhoon v1.14.2 or v1.14.3 clusters, edit the Typhoon `ref` to the first SHA that introduced Terraform v0.12 support (aim is to minimize the diff). For example: +For existing Typhoon v1.14.2 or v1.14.3 clusters, edit the Typhoon `ref` to the `v1.14.4` release (if published) or the first SHA that introduced Terraform v0.12 support (`3276bf587850218b8f967978a4bf2b05d5f440a2`). The aim is to minimize the diff. For example: ```tf module "bare-metal-mercury" { @@ -313,7 +313,7 @@ For existing Typhoon v1.14.2 or v1.14.3 clusters, edit the Typhoon `ref` to the ... ``` -Typhoon clusters no longer require the `providers` block (unless you actually need to pass an [aliased provider](https://www.terraform.io/docs/configuration/providers.html#alias-multiple-provider-instances)). A regression in Terraform v0.11 made it neccessary to explicitly pass aliased providers in order for Typhoon to continue to enforce constraints (see [terraform#16824](https://github.com/hashicorp/terraform/issues/16824)). Terraform v0.12 resolves this issue. +With Terraform v0.12, Typhoon clusters no longer require the `providers` block (unless you actually need to pass an [aliased provider](https://www.terraform.io/docs/configuration/providers.html#alias-multiple-provider-instances)). A regression in Terraform v0.11 made it neccessary to explicitly pass aliased providers in order for Typhoon to continue to enforce constraints (see [terraform#16824](https://github.com/hashicorp/terraform/issues/16824)). Terraform v0.12 resolves this issue. ```tf module "bare-metal-mercury" { @@ -327,7 +327,7 @@ Typhoon clusters no longer require the `providers` block (unless you actually ne - } ``` -Provider constrains ensure suitable plugin versions are used. Install new versions of `terraform-provider-ct` and `terraform-provider-matchbox` (bare-metal only) according to tutorial docs. Remove the now uneccessary `local`, `null`, `template`, and `tls` blocks in `providers.tf`. +Provider constrains ensure suitable plugin versions are used. Install new versions of `terraform-provider-ct` (v0.3.2+) and `terraform-provider-matchbox` (bare-metal only, v0.3.0+) according to the [changelog](https://github.com/poseidon/typhoon/blob/master/CHANGES.md#v1144) or tutorial docs. The `local`, `null`, `template`, and `tls` blocks in `providers.tf` are no longer needed. ```tf provider "matchbox" { @@ -383,7 +383,7 @@ Finally, plan. terraform12 plan ``` -Verify no changes are proposed and commit changes to version control. You've migrated to Terraform v0.12! Use the Terraform v0.12 binary going forward. +Verify no changes are proposed and commit changes to version control. You've migrated to Terraform v0.12! Repeat for other config directories. Use the Terraform v0.12 binary going forward. !!! note It is known that plan may propose re-creating `template_dir` resources. This is harmless. From 21fb632e90ca61ceeb49a518b1f49629c67d009b Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Thu, 13 Jun 2019 23:54:20 -0700 Subject: [PATCH 132/523] Update Calico from v3.7.2 to v3.7.3 * https://docs.projectcalico.org/v3.7/release-notes/ --- CHANGES.md | 1 + aws/container-linux/kubernetes/bootkube.tf | 2 +- aws/fedora-atomic/kubernetes/bootkube.tf | 2 +- azure/container-linux/kubernetes/bootkube.tf | 2 +- bare-metal/container-linux/kubernetes/bootkube.tf | 2 +- bare-metal/fedora-atomic/kubernetes/bootkube.tf | 2 +- digital-ocean/container-linux/kubernetes/bootkube.tf | 2 +- digital-ocean/fedora-atomic/kubernetes/bootkube.tf | 2 +- google-cloud/container-linux/kubernetes/bootkube.tf | 2 +- google-cloud/fedora-atomic/kubernetes/bootkube.tf | 2 +- 10 files changed, 10 insertions(+), 9 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index dccead084..a176a768f 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -7,6 +7,7 @@ Notable changes between versions. * Migrate from Terraform v0.11 to v0.12.x (**action required!**) * [Migration](https://typhoon.psdn.io/topics/maintenance/#terraform-v012x) instructions for Terraform v0.12 * Require `terraform-provider-ct` v0.3.2+ to support Terraform v0.12 (action required) +* Update Calico from v3.7.2 to [v3.7.3](https://docs.projectcalico.org/v3.7/release-notes/) #### AWS diff --git a/aws/container-linux/kubernetes/bootkube.tf b/aws/container-linux/kubernetes/bootkube.tf index 9d7f10ef5..4631b06eb 100644 --- a/aws/container-linux/kubernetes/bootkube.tf +++ b/aws/container-linux/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=0103bc06bb3f597455a765bf5d916f9b241cbea0" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=89c3ab4e2742c4e15865f0599c85cdac1ec88a6c" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/aws/fedora-atomic/kubernetes/bootkube.tf b/aws/fedora-atomic/kubernetes/bootkube.tf index 5258e3a5a..69192d653 100644 --- a/aws/fedora-atomic/kubernetes/bootkube.tf +++ b/aws/fedora-atomic/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=082921d67905417755609eebda7d39a7e26f7fdb" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=89c3ab4e2742c4e15865f0599c85cdac1ec88a6c" cluster_name = "${var.cluster_name}" api_servers = ["${format("%s.%s", var.cluster_name, var.dns_zone)}"] diff --git a/azure/container-linux/kubernetes/bootkube.tf b/azure/container-linux/kubernetes/bootkube.tf index 958e0170e..c02f045be 100644 --- a/azure/container-linux/kubernetes/bootkube.tf +++ b/azure/container-linux/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=0103bc06bb3f597455a765bf5d916f9b241cbea0" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=89c3ab4e2742c4e15865f0599c85cdac1ec88a6c" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/bare-metal/container-linux/kubernetes/bootkube.tf b/bare-metal/container-linux/kubernetes/bootkube.tf index 927226ac5..7ef2a2b62 100644 --- a/bare-metal/container-linux/kubernetes/bootkube.tf +++ b/bare-metal/container-linux/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=0103bc06bb3f597455a765bf5d916f9b241cbea0" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=89c3ab4e2742c4e15865f0599c85cdac1ec88a6c" cluster_name = var.cluster_name api_servers = [var.k8s_domain_name] diff --git a/bare-metal/fedora-atomic/kubernetes/bootkube.tf b/bare-metal/fedora-atomic/kubernetes/bootkube.tf index 45e638acc..bf8974e90 100644 --- a/bare-metal/fedora-atomic/kubernetes/bootkube.tf +++ b/bare-metal/fedora-atomic/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=082921d67905417755609eebda7d39a7e26f7fdb" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=89c3ab4e2742c4e15865f0599c85cdac1ec88a6c" cluster_name = "${var.cluster_name}" api_servers = ["${var.k8s_domain_name}"] diff --git a/digital-ocean/container-linux/kubernetes/bootkube.tf b/digital-ocean/container-linux/kubernetes/bootkube.tf index 31ffe924d..01a178158 100644 --- a/digital-ocean/container-linux/kubernetes/bootkube.tf +++ b/digital-ocean/container-linux/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=0103bc06bb3f597455a765bf5d916f9b241cbea0" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=89c3ab4e2742c4e15865f0599c85cdac1ec88a6c" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/digital-ocean/fedora-atomic/kubernetes/bootkube.tf b/digital-ocean/fedora-atomic/kubernetes/bootkube.tf index 3b98d7e6b..dba07edda 100644 --- a/digital-ocean/fedora-atomic/kubernetes/bootkube.tf +++ b/digital-ocean/fedora-atomic/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=082921d67905417755609eebda7d39a7e26f7fdb" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=89c3ab4e2742c4e15865f0599c85cdac1ec88a6c" cluster_name = "${var.cluster_name}" api_servers = ["${format("%s.%s", var.cluster_name, var.dns_zone)}"] diff --git a/google-cloud/container-linux/kubernetes/bootkube.tf b/google-cloud/container-linux/kubernetes/bootkube.tf index 997356b30..f0f17266d 100644 --- a/google-cloud/container-linux/kubernetes/bootkube.tf +++ b/google-cloud/container-linux/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=0103bc06bb3f597455a765bf5d916f9b241cbea0" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=89c3ab4e2742c4e15865f0599c85cdac1ec88a6c" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/google-cloud/fedora-atomic/kubernetes/bootkube.tf b/google-cloud/fedora-atomic/kubernetes/bootkube.tf index 86acda556..3089ce339 100644 --- a/google-cloud/fedora-atomic/kubernetes/bootkube.tf +++ b/google-cloud/fedora-atomic/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=082921d67905417755609eebda7d39a7e26f7fdb" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=89c3ab4e2742c4e15865f0599c85cdac1ec88a6c" cluster_name = "${var.cluster_name}" api_servers = ["${format("%s.%s", var.cluster_name, var.dns_zone)}"] From ce7bff0066703aac70bccd85c73ca52b6e1b78d0 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Sun, 16 Jun 2019 12:28:37 -0700 Subject: [PATCH 133/523] Update mkdocs-material from v4.3.0 to v4.4.0 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index d8b45b69c..f655243f5 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ mkdocs==1.0.4 -mkdocs-material==4.3.0 +mkdocs-material==4.4.0 pygments==2.2.0 pymdown-extensions==5.0.0 From 4ad69efc4333ee8a27851349b0334de0b072c578 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Wed, 19 Jun 2019 21:51:54 -0700 Subject: [PATCH 134/523] Update Grafana from v6.2.2 to v6.2.4 * https://github.com/grafana/grafana/releases/tag/v6.2.4 --- CHANGES.md | 2 +- addons/grafana/deployment.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index a176a768f..a4f1748c7 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -35,7 +35,7 @@ Notable changes between versions. #### Addons -* Update Grafana from v6.2.1 to v6.2.2 +* Update Grafana from v6.2.1 to v6.2.4 * Update node-exporter from v0.18.0 to v0.18.1 ## v1.14.3 diff --git a/addons/grafana/deployment.yaml b/addons/grafana/deployment.yaml index 4279fc6be..62773f4e4 100644 --- a/addons/grafana/deployment.yaml +++ b/addons/grafana/deployment.yaml @@ -23,7 +23,7 @@ spec: spec: containers: - name: grafana - image: grafana/grafana:6.2.2 + image: grafana/grafana:6.2.4 env: - name: GF_PATHS_CONFIG value: "/etc/grafana/custom.ini" From 3d5be86aaec32f75d940ebf2db7aaf5750ecd3fa Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Wed, 19 Jun 2019 21:58:43 -0700 Subject: [PATCH 135/523] Update provider plugin versions in tutorial docs * Update Terraform provider plugin versions in docs to reflect the recommended versions that we actively use --- docs/cl/aws.md | 4 ++-- docs/cl/azure.md | 4 ++-- docs/cl/bare-metal.md | 2 +- docs/cl/digital-ocean.md | 4 ++-- docs/cl/google-cloud.md | 4 ++-- 5 files changed, 9 insertions(+), 9 deletions(-) diff --git a/docs/cl/aws.md b/docs/cl/aws.md index 2fbf01f7f..857a72b73 100644 --- a/docs/cl/aws.md +++ b/docs/cl/aws.md @@ -18,7 +18,7 @@ Install [Terraform](https://www.terraform.io/downloads.html) v0.12.x on your sys ```sh $ terraform version -Terraform v0.12.1 +Terraform v0.12.2 ``` Add the [terraform-provider-ct](https://github.com/poseidon/terraform-provider-ct) plugin binary for your system to `~/.terraform.d/plugins/`, noting the final name. @@ -49,7 +49,7 @@ Configure the AWS provider to use your access key credentials in a `providers.tf ```tf provider "aws" { - version = "2.12.0" + version = "2.15.0" region = "eu-central-1" shared_credentials_file = "/home/user/.config/aws/credentials" } diff --git a/docs/cl/azure.md b/docs/cl/azure.md index 7b86eed6b..fba536965 100644 --- a/docs/cl/azure.md +++ b/docs/cl/azure.md @@ -21,7 +21,7 @@ Install [Terraform](https://www.terraform.io/downloads.html) v0.12.x on your sys ```sh $ terraform version -Terraform v0.12.1 +Terraform v0.12.2 ``` Add the [terraform-provider-ct](https://github.com/poseidon/terraform-provider-ct) plugin binary for your system to `~/.terraform.d/plugins/`, noting the final name. @@ -50,7 +50,7 @@ Configure the Azure provider in a `providers.tf` file. ```tf provider "azurerm" { - version = "1.29.0" + version = "1.30.1" } provider "ct" { diff --git a/docs/cl/bare-metal.md b/docs/cl/bare-metal.md index d4068890c..b9ad5e78c 100644 --- a/docs/cl/bare-metal.md +++ b/docs/cl/bare-metal.md @@ -111,7 +111,7 @@ Install [Terraform](https://www.terraform.io/downloads.html) v0.12.x on your sys ```sh $ terraform version -Terraform v0.12.1 +Terraform v0.12.2 ``` Add the [terraform-provider-matchbox](https://github.com/poseidon/terraform-provider-matchbox) plugin binary for your system to `~/.terraform.d/plugins/`, noting the final name. diff --git a/docs/cl/digital-ocean.md b/docs/cl/digital-ocean.md index 9ac44a4bc..0eb0a2ab0 100644 --- a/docs/cl/digital-ocean.md +++ b/docs/cl/digital-ocean.md @@ -18,7 +18,7 @@ Install [Terraform](https://www.terraform.io/downloads.html) v0.12.x on your sys ```sh $ terraform version -Terraform v0.12.1 +Terraform v0.12.2 ``` Add the [terraform-provider-ct](https://github.com/poseidon/terraform-provider-ct) plugin binary for your system to `~/.terraform.d/plugins/`, noting the final name. @@ -50,7 +50,7 @@ Configure the DigitalOcean provider to use your token in a `providers.tf` file. ```tf provider "digitalocean" { - version = "1.3.0" + version = "1.4.0" token = "${chomp(file("~/.config/digital-ocean/token"))}" } diff --git a/docs/cl/google-cloud.md b/docs/cl/google-cloud.md index 4cbdf93f8..adead5558 100644 --- a/docs/cl/google-cloud.md +++ b/docs/cl/google-cloud.md @@ -18,7 +18,7 @@ Install [Terraform](https://www.terraform.io/downloads.html) v0.12.x on your sys ```sh $ terraform version -Terraform v0.12.1 +Terraform v0.12.2 ``` Add the [terraform-provider-ct](https://github.com/poseidon/terraform-provider-ct) plugin binary for your system to `~/.terraform.d/plugins/`, noting the final name. @@ -49,7 +49,7 @@ Configure the Google Cloud provider to use your service account key, project-id, ```tf provider "google" { - version = "2.7.0" + version = "2.9.0" project = "project-id" region = "us-central1" credentials = "${file("~/.config/google-cloud/terraform.json")}" From d35c1cb9fb9f30aff88a40ca370b872da2eeabe0 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Wed, 19 Jun 2019 22:11:11 -0700 Subject: [PATCH 136/523] Fix advanced customization docs for Terraform v0.12 * Use Terraform v0.12 syntax in the Container Linux Config snippet customization docs --- docs/advanced/customization.md | 16 +++++++--------- docs/advanced/worker-pools.md | 2 +- 2 files changed, 8 insertions(+), 10 deletions(-) diff --git a/docs/advanced/customization.md b/docs/advanced/customization.md index 4bc86eec1..b653d9fb9 100644 --- a/docs/advanced/customization.md +++ b/docs/advanced/customization.md @@ -78,12 +78,12 @@ module "digital-ocean-nemo" { controller_count = 1 worker_count = 2 controller_clc_snippets = [ - "${file("./custom-files")}", - "${file("./custom-units")}", + file("./custom-files"), + file("./custom-units"), ] worker_clc_snippets = [ - "${file("./custom-files")}", - "${file("./custom-units")}", + file("./custom-files"), + file("./custom-units")", ] ... } @@ -100,12 +100,10 @@ module "bare-metal-mercury" { "node3", ] clc_snippets = { - "node2" = [ - "${file("./units/hello.yaml")}" - ] + "node2" = [file("./units/hello.yaml")] "node3" = [ - "${file("./units/world.yaml")}", - "${file("./units/hello.yaml")}", + file("./units/world.yaml"), + file("./units/hello.yaml"), ] } ... diff --git a/docs/advanced/worker-pools.md b/docs/advanced/worker-pools.md index 719c7bb68..13d1c39eb 100644 --- a/docs/advanced/worker-pools.md +++ b/docs/advanced/worker-pools.md @@ -16,7 +16,7 @@ Create a cluster following the AWS [tutorial](../cl/aws.md#cluster). Define a wo ```tf module "tempest-worker-pool" { - source = "git::https://github.com/poseidon/typhoon//aws/container-linux/kubernetes/workers?ref=v1.14.4" + source = "git::https://github.com/poseidon/typhoon//aws/container-linux/kubernetes/workers?ref=v1.14.3" # AWS vpc_id = module.aws-tempest.vpc_id From 405015f52c7e3e4d1d25e9227ba4b16177da8fc5 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Wed, 19 Jun 2019 22:21:58 -0700 Subject: [PATCH 137/523] Remove Fedora Atomic documentation * Typhoon for Fedora Atomic was deprecated in March 2019 * https://typhoon.psdn.io/announce/#march-27-2019 --- README.md | 9 - azure/container-linux/kubernetes/README.md | 4 +- .../container-linux/kubernetes/README.md | 4 +- docs/advanced/customization.md | 4 - docs/advanced/worker-pools.md | 2 - docs/atomic/aws.md | 243 ---------- docs/atomic/bare-metal.md | 419 ------------------ docs/atomic/digital-ocean.md | 250 ----------- docs/atomic/google-cloud.md | 285 ------------ docs/index.md | 9 - docs/topics/faq.md | 7 +- docs/topics/security.md | 4 +- 12 files changed, 6 insertions(+), 1234 deletions(-) delete mode 100644 docs/atomic/aws.md delete mode 100644 docs/atomic/bare-metal.md delete mode 100644 docs/atomic/digital-ocean.md delete mode 100644 docs/atomic/google-cloud.md diff --git a/README.md b/README.md index 69b3ef8af..e6ab04f01 100644 --- a/README.md +++ b/README.md @@ -29,15 +29,6 @@ Typhoon provides a Terraform Module for each supported operating system and plat | Digital Ocean | Container Linux | [digital-ocean/container-linux/kubernetes](digital-ocean/container-linux/kubernetes) | beta | | Google Cloud | Container Linux | [google-cloud/container-linux/kubernetes](google-cloud/container-linux/kubernetes) | stable | -Fedora Atomic support is alpha and will evolve as Fedora Atomic is replaced by Fedora CoreOS. - -| Platform | Operating System | Terraform Module | Status | -|---------------|------------------|------------------|--------| -| AWS | Fedora Atomic | [aws/fedora-atomic/kubernetes](aws/fedora-atomic/kubernetes) | deprecated | -| Bare-Metal | Fedora Atomic | [bare-metal/fedora-atomic/kubernetes](bare-metal/fedora-atomic/kubernetes) | deprecated | -| Digital Ocean | Fedora Atomic | [digital-ocean/fedora-atomic/kubernetes](digital-ocean/fedora-atomic/kubernetes) | deprecated | -| Google Cloud | Fedora Atomic | [google-cloud/fedora-atomic/kubernetes](google-cloud/fedora-atomic/kubernetes) | deprecated | - ## Documentation * [Docs](https://typhoon.psdn.io) diff --git a/azure/container-linux/kubernetes/README.md b/azure/container-linux/kubernetes/README.md index ab7e06637..3ca37356a 100644 --- a/azure/container-linux/kubernetes/README.md +++ b/azure/container-linux/kubernetes/README.md @@ -12,8 +12,8 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features * Kubernetes v1.14.3 (upstream, via [kubernetes-incubator/bootkube](https://github.com/kubernetes-incubator/bootkube)) -* Single or multi-master, [flannel](https://github.com/coreos/flannel) networking -* On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled +* Single or multi-master, [Calico](https://www.projectcalico.org/) or [flannel](https://github.com/coreos/flannel) networking +* On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/) * Advanced features like [worker pools](https://typhoon.psdn.io/advanced/worker-pools/), [low-priority](https://typhoon.psdn.io/cl/azure/#low-priority) workers, and [snippets](https://typhoon.psdn.io/advanced/customization/#container-linux) customization * Ready for Ingress, Prometheus, Grafana, and other optional [addons](https://typhoon.psdn.io/addons/overview/) diff --git a/digital-ocean/container-linux/kubernetes/README.md b/digital-ocean/container-linux/kubernetes/README.md index 26a10f393..5e07e4d93 100644 --- a/digital-ocean/container-linux/kubernetes/README.md +++ b/digital-ocean/container-linux/kubernetes/README.md @@ -12,8 +12,8 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features * Kubernetes v1.14.3 (upstream, via [kubernetes-incubator/bootkube](https://github.com/kubernetes-incubator/bootkube)) -* Single or multi-master, [flannel](https://github.com/coreos/flannel) networking -* On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled +* Single or multi-master, [Calico](https://www.projectcalico.org/) or [flannel](https://github.com/coreos/flannel) networking +* On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/) * Advanced features like [snippets](https://typhoon.psdn.io/advanced/customization/#container-linux) customization * Ready for Ingress, Prometheus, Grafana, CSI, and other [addons](https://typhoon.psdn.io/addons/overview/) diff --git a/docs/advanced/customization.md b/docs/advanced/customization.md index b653d9fb9..e4ff1a6aa 100644 --- a/docs/advanced/customization.md +++ b/docs/advanced/customization.md @@ -136,10 +136,6 @@ Container Linux Configs (and the CoreOS Ignition system) create immutable infras !!! danger Destroying and recreating controller instances is destructive! etcd runs on controller instances and stores data there. Do not modify controller snippets. See [blue/green](/topics/maintenance/#upgrades) clusters. -### Fedora Atomic - -Cloud-Init and kickstart (bare-metal only) declare how a Fedora Atomic instance should be provisioned. Customizing these declarations in ways beyond the provided Terraform variables is unsupported. - ## Architecture Typhoon chooses variables to expose with purpose. If you must customize clusters in ways that aren't supported by input variables, fork Typhoon and maintain a repository with customizations. Reference the repository by changing the username. diff --git a/docs/advanced/worker-pools.md b/docs/advanced/worker-pools.md index 13d1c39eb..7a8170b16 100644 --- a/docs/advanced/worker-pools.md +++ b/docs/advanced/worker-pools.md @@ -5,10 +5,8 @@ Typhoon AWS, Azure, and Google Cloud allow additional groups of workers to be de Internal Terraform Modules: * `aws/container-linux/kubernetes/workers` -* `aws/fedora-atomic/kubernetes/workers` * `azure/container-linux/kubernetes/workers` * `google-cloud/container-linux/kubernetes/workers` -* `google-cloud/fedora-atomic/kubernetes/workers` ## AWS diff --git a/docs/atomic/aws.md b/docs/atomic/aws.md deleted file mode 100644 index 334a26ca1..000000000 --- a/docs/atomic/aws.md +++ /dev/null @@ -1,243 +0,0 @@ -# AWS - -!!! danger - Typhoon for Fedora Atomic will not be updated much beyond Kubernetes v1.13. - -In this tutorial, we'll create a Kubernetes v1.14.3 cluster on AWS with Fedora Atomic. - -We'll declare a Kubernetes cluster using the Typhoon Terraform module. Then apply the changes to create a VPC, gateway, subnets, security groups, controller instances, worker auto-scaling group, network load balancer, and TLS assets. Instances are provisioned on first boot with cloud-init. - -Controllers are provisioned to run an `etcd` peer and a `kubelet` service. Workers run just a `kubelet` service. A one-time [bootkube](https://github.com/kubernetes-incubator/bootkube) bootstrap schedules the `apiserver`, `scheduler`, `controller-manager`, and `coredns` on controllers and schedules `kube-proxy` and `calico` (or `flannel`) on every node. A generated `kubeconfig` provides `kubectl` access to the cluster. - -## Requirements - -* AWS Account and IAM credentials -* AWS Route53 DNS Zone (registered Domain Name or delegated subdomain) -* Terraform v0.11.x installed locally - -## Terraform Setup - -Install [Terraform](https://www.terraform.io/downloads.html) v0.11.x on your system. - -```sh -$ terraform version -Terraform v0.11.12 -``` - -Read [concepts](/architecture/concepts/) to learn about Terraform, modules, and organizing resources. Change to your infrastructure repository (e.g. `infra`). - -``` -cd infra/clusters -``` - -## Provider - -Login to your AWS IAM dashboard and find your IAM user. Select "Security Credentials" and create an access key. Save the id and secret to a file that can be referenced in configs. - -``` -[default] -aws_access_key_id = xxx -aws_secret_access_key = yyy -``` - -Configure the AWS provider to use your access key credentials in a `providers.tf` file. - -```tf -provider "aws" { - version = "~> 2.3.0" - alias = "default" - - region = "eu-central-1" - shared_credentials_file = "/home/user/.config/aws/credentials" -} - -provider "local" { - version = "~> 1.0" - alias = "default" -} - -provider "null" { - version = "~> 1.0" - alias = "default" -} - -provider "template" { - version = "~> 1.0" - alias = "default" -} - -provider "tls" { - version = "~> 1.0" - alias = "default" -} -``` - -Additional configuration options are described in the `aws` provider [docs](https://www.terraform.io/docs/providers/aws/). - -!!! tip - Regions are listed in [docs](http://docs.aws.amazon.com/general/latest/gr/rande.html#ec2_region) or with `aws ec2 describe-regions`. - -## Cluster - -Define a Kubernetes cluster using the module `aws/fedora-atomic/kubernetes`. - -```tf -module "aws-tempest" { - source = "git::https://github.com/poseidon/typhoon//aws/fedora-atomic/kubernetes?ref=v1.14.3" - - providers = { - aws = "aws.default" - local = "local.default" - null = "null.default" - template = "template.default" - tls = "tls.default" - } - - # AWS - cluster_name = "tempest" - dns_zone = "aws.example.com" - dns_zone_id = "Z3PAABBCFAKEC0" - - # configuration - ssh_authorized_key = "ssh-rsa AAAAB3Nz..." - asset_dir = "/home/user/.secrets/clusters/tempest" - - # optional - worker_count = 2 - worker_type = "t2.medium" -} -``` - -Reference the [variables docs](#variables) or the [variables.tf](https://github.com/poseidon/typhoon/blob/master/aws/fedora-atomic/kubernetes/variables.tf) source. - -## ssh-agent - -Initial bootstrapping requires `bootkube.service` be started on one controller node. Terraform uses `ssh-agent` to automate this step. Add your SSH private key to `ssh-agent`. - -```sh -ssh-add ~/.ssh/id_rsa -ssh-add -L -``` - -## Apply - -Initialize the config directory if this is the first use with Terraform. - -```sh -terraform init -``` - -Plan the resources to be created. - -```sh -$ terraform plan -Plan: 106 to add, 0 to change, 0 to destroy. -``` - -Apply the changes to create the cluster. - -```sh -$ terraform apply -... -module.aws-tempest.null_resource.bootkube-start: Still creating... (4m50s elapsed) -module.aws-tempest.null_resource.bootkube-start: Still creating... (5m0s elapsed) -module.aws-tempest.null_resource.bootkube-start: Creation complete after 11m8s (ID: 3961816482286168143) - -Apply complete! Resources: 106 added, 0 changed, 0 destroyed. -``` - -In 5-10 minutes, the Kubernetes cluster will be ready. - -## Verify - -[Install kubectl](https://kubernetes.io/docs/tasks/tools/install-kubectl/) on your system. Use the generated `kubeconfig` credentials to access the Kubernetes cluster and list nodes. - -``` -$ export KUBECONFIG=/home/user/.secrets/clusters/tempest/auth/kubeconfig -$ kubectl get nodes -NAME STATUS ROLES AGE VERSION -ip-10-0-3-155 Ready controller,master 10m v1.14.3 -ip-10-0-26-65 Ready node 10m v1.14.3 -ip-10-0-41-21 Ready node 10m v1.14.3 -``` - -List the pods. - -``` -$ kubectl get pods --all-namespaces -NAMESPACE NAME READY STATUS RESTARTS AGE -kube-system calico-node-1m5bf 2/2 Running 0 34m -kube-system calico-node-7jmr1 2/2 Running 0 34m -kube-system calico-node-bknc8 2/2 Running 0 34m -kube-system coredns-1187388186-wx1lg 1/1 Running 0 34m -kube-system coredns-1187388186-qjnvp 1/1 Running 0 34m -kube-system kube-apiserver-4mjbk 1/1 Running 0 34m -kube-system kube-controller-manager-3597210155-j2jbt 1/1 Running 1 34m -kube-system kube-controller-manager-3597210155-j7g7x 1/1 Running 0 34m -kube-system kube-proxy-14wxv 1/1 Running 0 34m -kube-system kube-proxy-9vxh2 1/1 Running 0 34m -kube-system kube-proxy-sbbsh 1/1 Running 0 34m -kube-system kube-scheduler-3359497473-5plhf 1/1 Running 0 34m -kube-system kube-scheduler-3359497473-r7zg7 1/1 Running 1 34m -kube-system pod-checkpointer-4kxtl 1/1 Running 0 34m -kube-system pod-checkpointer-4kxtl-ip-10-0-3-155 1/1 Running 0 33m -``` - -## Going Further - -Learn about [maintenance](/topics/maintenance/) and [addons](/addons/overview/). - -## Variables - -Check the [variables.tf](https://github.com/poseidon/typhoon/blob/master/aws/fedora-atomic/kubernetes/variables.tf) source. - -### Required - -| Name | Description | Example | -|:-----|:------------|:--------| -| cluster_name | Unique cluster name (prepended to dns_zone) | "tempest" | -| dns_zone | AWS Route53 DNS zone | "aws.example.com" | -| dns_zone_id | AWS Route53 DNS zone id | "Z3PAABBCFAKEC0" | -| ssh_authorized_key | SSH public key for user 'fedora' | "ssh-rsa AAAAB3NZ..." | -| asset_dir | Path to a directory where generated assets should be placed (contains secrets) | "/home/user/.secrets/clusters/tempest" | - -#### DNS Zone - -Clusters create a DNS A record `${cluster_name}.${dns_zone}` to resolve a network load balancer backed by controller instances. This FQDN is used by workers and `kubectl` to access the apiserver(s). In this example, the cluster's apiserver would be accessible at `tempest.aws.example.com`. - -You'll need a registered domain name or delegated subdomain on AWS Route53. You can set this up once and create many clusters with unique names. - -```tf -resource "aws_route53_zone" "zone-for-clusters" { - name = "aws.example.com." -} -``` - -Reference the DNS zone id with `"${aws_route53_zone.zone-for-clusters.zone_id}"`. - -!!! tip "" - If you have an existing domain name with a zone file elsewhere, just delegate a subdomain that can be managed on Route53 (e.g. aws.mydomain.com) and [update nameservers](http://docs.aws.amazon.com/Route53/latest/DeveloperGuide/SOA-NSrecords.html). - -### Optional - -| Name | Description | Default | Example | -|:-----|:------------|:--------|:--------| -| controller_count | Number of controllers (i.e. masters) | 1 | 1 | -| worker_count | Number of workers | 1 | 3 | -| controller_type | EC2 instance type for controllers | "t3.small" | See below | -| worker_type | EC2 instance type for workers | "t3.small" | See below | -| disk_size | Size of the EBS volume in GB | "40" | "100" | -| disk_type | Type of the EBS volume | "gp2" | standard, gp2, io1 | -| disk_iops | IOPS of the EBS volume | "0" (i.e. auto) | "400" | -| worker_price | Spot price in USD for workers. Leave as default empty string for regular on-demand instances | "" | "0.10" | -| networking | Choice of networking provider | "calico" | "calico" or "flannel" | -| network_mtu | CNI interface MTU (calico only) | 1480 | 8981 | -| host_cidr | CIDR IPv4 range to assign to EC2 instances | "10.0.0.0/16" | "10.1.0.0/16" | -| pod_cidr | CIDR IPv4 range to assign to Kubernetes pods | "10.2.0.0/16" | "10.22.0.0/16" | -| service_cidr | CIDR IPv4 range to assign to Kubernetes services | "10.3.0.0/16" | "10.3.0.0/24" | -| cluster_domain_suffix | FQDN suffix for Kubernetes services answered by coredns. | "cluster.local" | "k8s.example.com" | - -Check the list of valid [instance types](https://aws.amazon.com/ec2/instance-types/). - -!!! warning - Do not choose a `controller_type` smaller than `t2.small`. Smaller instances are not sufficient for running a controller. diff --git a/docs/atomic/bare-metal.md b/docs/atomic/bare-metal.md deleted file mode 100644 index d7b9424ae..000000000 --- a/docs/atomic/bare-metal.md +++ /dev/null @@ -1,419 +0,0 @@ -# Bare-Metal - -!!! danger - Typhoon for Fedora Atomic will not be updated much beyond Kubernetes v1.13. - -In this tutorial, we'll network boot and provision a Kubernetes v1.14.3 cluster on bare-metal with Fedora Atomic. - -First, we'll deploy a [Matchbox](https://github.com/poseidon/matchbox) service and setup a network boot environment. Then, we'll declare a Kubernetes cluster using the Typhoon Terraform module and power on machines. On PXE boot, machines will install Fedora Atomic via kickstart, reboot into the disk install, and provision themselves as Kubernetes controllers or workers via cloud-init. - -Controllers are provisioned to run `etcd` and `kubelet` [system containers](http://www.projectatomic.io/blog/2016/09/intro-to-system-containers/). Workers run just a `kubelet` system container. A one-time [bootkube](https://github.com/kubernetes-incubator/bootkube) bootstrap schedules the `apiserver`, `scheduler`, `controller-manager`, and `coredns` on controllers and schedules `kube-proxy` and `calico` (or `flannel`) on every node. A generated `kubeconfig` provides `kubectl` access to the cluster. - -## Requirements - -* Machines with 2GB RAM, 30GB disk, PXE-enabled NIC, IPMI -* PXE-enabled [network boot](https://coreos.com/matchbox/docs/latest/network-setup.html) environment -* Matchbox v0.7+ deployment with API enabled -* HTTP server for Fedora install assets and ostree repo -* Matchbox credentials `client.crt`, `client.key`, `ca.crt` -* Terraform v0.11.x and [terraform-provider-matchbox](https://github.com/poseidon/terraform-provider-matchbox) installed locally - -## Machines - -Collect a MAC address from each machine. For machines with multiple PXE-enabled NICs, pick one of the MAC addresses. MAC addresses will be used to match machines to profiles during network boot. - -* 52:54:00:a1:9c:ae (node1) -* 52:54:00:b2:2f:86 (node2) -* 52:54:00:c3:61:77 (node3) - -Configure each machine to boot from the disk through IPMI or the BIOS menu. - -``` -ipmitool -H node1 -U USER -P PASS chassis bootdev disk options=persistent -``` - -During provisioning, you'll explicitly set the boot device to `pxe` for the next boot only. Machines will install (overwrite) the operating system to disk on PXE boot and reboot into the disk install. - -!!! tip "" - Ask your hardware vendor to provide MACs and preconfigure IPMI, if possible. With it, you can rack new servers, `terraform apply` with new info, and power on machines that network boot and provision into clusters. - -## DNS - -Create a DNS A (or AAAA) record for each node's default interface. Create a record that resolves to each controller node (or re-use the node record if there's one controller). - -* node1.example.com (node1) -* node2.example.com (node2) -* node3.example.com (node3) -* myk8s.example.com (node1) - -Cluster nodes will be configured to refer to the control plane and themselves by these fully qualified names and they'll be used in generated TLS certificates. - -## Matchbox - -Matchbox is an open-source app that matches network-booted bare-metal machines (based on labels like MAC, UUID, etc.) to profiles to automate cluster provisioning. - -Install Matchbox on a Kubernetes cluster or dedicated server. - -* Installing on [Kubernetes](https://coreos.com/matchbox/docs/latest/deployment.html#kubernetes) (recommended) -* Installing on a [server](https://coreos.com/matchbox/docs/latest/deployment.html#download) - -!!! tip - Deploy Matchbox as service that can be accessed by all of your bare-metal machines globally. This provides a single endpoint to use Terraform to manage bare-metal clusters at different sites. Typhoon will never include secrets in provisioning user-data so you may even deploy matchbox publicly. - -Matchbox provides a TLS client-authenticated API that clients, like Terraform, can use to manage machine matching and profiles. Think of it like a cloud provider API, but for creating bare-metal instances. - -[Generate TLS](https://coreos.com/matchbox/docs/latest/deployment.html#generate-tls-certificates) client credentials. Save the `ca.crt`, `client.crt`, and `client.key` where they can be referenced in Terraform configs. - -```sh -mv ca.crt client.crt client.key ~/.config/matchbox/ -``` - -Verify the matchbox read-only HTTP endpoints are accessible (port is configurable). - -```sh -$ curl http://matchbox.example.com:8080 -matchbox -``` - -Verify your TLS client certificate and key can be used to access the Matchbox API (port is configurable). - -```sh -$ openssl s_client -connect matchbox.example.com:8081 \ - -CAfile ~/.config/matchbox/ca.crt \ - -cert ~/.config/matchbox/client.crt \ - -key ~/.config/matchbox/client.key -``` - -## PXE Environment - -Create a iPXE-enabled network boot environment. Configure PXE clients to chainload [iPXE](http://ipxe.org/cmd) and instruct iPXE clients to chainload from your Matchbox service's `/boot.ipxe` endpoint. - -For networks already supporting iPXE clients, you can add a `default.ipxe` config. - -```ini -# /var/www/html/ipxe/default.ipxe -chain http://matchbox.foo:8080/boot.ipxe -``` - -For networks with Ubiquiti Routers, you can [configure the router](/topics/hardware/#ubiquiti) itself to chainload machines to iPXE and Matchbox. - -For a small lab, you may wish to checkout the [quay.io/poseidon/dnsmasq](https://quay.io/repository/poseidon/dnsmasq) container image and [copy-paste examples](https://github.com/poseidon/matchbox/blob/master/Documentation/network-setup.md#coreosdnsmasq). - -Read about the [many ways](https://coreos.com/matchbox/docs/latest/network-setup.html) to setup a compliant iPXE-enabled network. There is quite a bit of flexibility: - -* Continue using existing DHCP, TFTP, or DNS services -* Configure specific machines, subnets, or architectures to chainload from Matchbox -* Place Matchbox behind a menu entry (timeout and default to Matchbox) - -!!! note "" - TFTP chainloading to modern boot firmware, like iPXE, avoids issues with old NICs and allows faster transfer protocols like HTTP to be used. - -## Atomic Assets - -Fedora Atomic network installations require a local mirror of assets. Configure an HTTP server to serve the Atomic install tree and ostree repo. - -``` -sudo dnf install -y httpd -sudo firewall-cmd --permenant --add-port=80/tcp -sudo systemctl enable httpd --now -``` - -Download the [Fedora Atomic](https://getfedora.org/en/atomic/download/) ISO which contains install files and add them to the serve directory. - -``` -sudo mount -o loop,ro Fedora-AtomicHost-ostree-*.iso /mnt -sudo mkdir -p /var/www/html/fedora/28 -sudo cp -av /mnt/* /var/www/html/fedora/28/ -sudo umount /mnt -``` - -Checkout the [fedora-atomic](https://pagure.io/fedora-atomic) ostree manifest repo. - -``` -git clone https://pagure.io/fedora-atomic.git && cd fedora-atomic -git checkout f28 -``` - -Compose an ostree repo from RPM sources. - -``` -mkdir repo -ostree init --repo=repo --mode=archive -sudo dnf install rpm-ostree -sudo rpm-ostree compose tree --repo=repo fedora-atomic-host.json -``` - -Serve the ostree `repo` as well. - -``` -sudo cp -r repo /var/www/html/fedora/28/ -tree /var/www/html/fedora/28/ -├── images -│   ├── pxeboot -│      ├── initrd.img -│      └── vmlinuz -├── isolinux/ -├── repo/ -``` - -Verify `vmlinuz`, `initrd.img`, and `repo` are accessible from the HTTP server (i.e. `atomic_assets_endpoint`). - -``` -curl http://example.com/fedora/28/ -``` - -!!! note - It is possible to use the Matchbox `/assets` [cache](https://github.com/poseidon/matchbox/blob/master/Documentation/matchbox.md#assets) as an HTTP server. - -## Terraform Setup - -Install [Terraform](https://www.terraform.io/downloads.html) v0.11.x on your system. - -```sh -$ terraform version -Terraform v0.11.12 -``` - -Add the [terraform-provider-matchbox](https://github.com/poseidon/terraform-provider-matchbox) plugin binary for your system to `~/.terraform.d/plugins/`, noting the final name. - -```sh -wget https://github.com/poseidon/terraform-provider-matchbox/releases/download/v0.2.3/terraform-provider-matchbox-v0.2.3-linux-amd64.tar.gz -tar xzf terraform-provider-matchbox-v0.2.3-linux-amd64.tar.gz -mv terraform-provider-matchbox-v0.2.3-linux-amd64/terraform-provider-matchbox ~/.terraform.d/plugins/terraform-provider-matchbox_v0.2.3 -``` - -Read [concepts](/architecture/concepts/) to learn about Terraform, modules, and organizing resources. Change to your infrastructure repository (e.g. `infra`). - -``` -cd infra/clusters -``` - -## Provider - -Configure the Matchbox provider to use your Matchbox API endpoint and client certificate in a `providers.tf` file. - -```tf -provider "matchbox" { - version = "0.2.3" - endpoint = "matchbox.example.com:8081" - client_cert = "${file("~/.config/matchbox/client.crt")}" - client_key = "${file("~/.config/matchbox/client.key")}" - ca = "${file("~/.config/matchbox/ca.crt")}" -} - -provider "local" { - version = "~> 1.0" - alias = "default" -} - -provider "null" { - version = "~> 1.0" - alias = "default" -} - -provider "template" { - version = "~> 1.0" - alias = "default" -} - -provider "tls" { - version = "~> 1.0" - alias = "default" -} -``` - -## Cluster - -Define a Kubernetes cluster using the module `bare-metal/fedora-atomic/kubernetes`. - -```tf -module "bare-metal-mercury" { - source = "git::https://github.com/poseidon/typhoon//bare-metal/fedora-atomic/kubernetes?ref=v1.14.3" - - providers = { - local = "local.default" - null = "null.default" - template = "template.default" - tls = "tls.default" - } - - # bare-metal - cluster_name = "mercury" - matchbox_http_endpoint = "http://matchbox.example.com" - atomic_assets_endpoint = "http://example.com/fedora/28" - - # configuration - k8s_domain_name = "node1.example.com" - ssh_authorized_key = "ssh-rsa AAAAB3Nz..." - asset_dir = "/home/user/.secrets/clusters/mercury" - - # machines - controller_names = ["node1"] - controller_macs = ["52:54:00:a1:9c:ae"] - controller_domains = ["node1.example.com"] - worker_names = [ - "node2", - "node3", - ] - worker_macs = [ - "52:54:00:b2:2f:86", - "52:54:00:c3:61:77", - ] - worker_domains = [ - "node2.example.com", - "node3.example.com", - ] -} -``` - -Reference the [variables docs](#variables) or the [variables.tf](https://github.com/poseidon/typhoon/blob/master/bare-metal/fedora-atomic/kubernetes/variables.tf) source. - -## ssh-agent - -Initial bootstrapping requires `bootkube.service` be started on one controller node. Terraform uses `ssh-agent` to automate this step. Add your SSH private key to `ssh-agent`. - -```sh -ssh-add ~/.ssh/id_rsa -ssh-add -L -``` - -## Apply - -Initialize the config directory if this is the first use with Terraform. - -```sh -terraform init -``` - -Plan the resources to be created. - -```sh -$ terraform plan -Plan: 58 to add, 0 to change, 0 to destroy. -``` - -Apply the changes. Terraform will generate bootkube assets to `asset_dir` and create Matchbox profiles (e.g. controller, worker) and matching rules via the Matchbox API. - -```sh -$ terraform apply -module.bare-metal-mercury.null_resource.copy-kubeconfig.0: Provisioning with 'file'... -module.bare-metal-mercury.null_resource.copy-etcd-secrets.0: Provisioning with 'file'... -module.bare-metal-mercury.null_resource.copy-kubeconfig.0: Still creating... (10s elapsed) -module.bare-metal-mercury.null_resource.copy-etcd-secrets.0: Still creating... (10s elapsed) -... -``` - -Apply will then loop until it can successfully copy credentials to each machine and start the one-time Kubernetes bootstrap service. Proceed to the next step while this loops. - -### Power - -Power on each machine with the boot device set to `pxe` for the next boot only. - -```sh -ipmitool -H node1.example.com -U USER -P PASS chassis bootdev pxe -ipmitool -H node1.example.com -U USER -P PASS power on -``` - -Machines will network boot, install Fedora Atomic to disk via kickstart, reboot into the disk install, and provision themselves as controllers or workers via cloud-init. - -!!! tip "" - If this is the first test of your PXE-enabled network boot environment, watch the SOL console of a machine to spot any misconfigurations. - -### Bootstrap - -Wait for the `bootkube-start` step to finish bootstrapping the Kubernetes control plane. This may take 5-15 minutes depending on your network. - -``` -module.bare-metal-mercury.null_resource.bootkube-start: Still creating... (6m10s elapsed) -module.bare-metal-mercury.null_resource.bootkube-start: Still creating... (6m20s elapsed) -module.bare-metal-mercury.null_resource.bootkube-start: Still creating... (6m30s elapsed) -module.bare-metal-mercury.null_resource.bootkube-start: Still creating... (6m40s elapsed) -module.bare-metal-mercury.null_resource.bootkube-start: Creation complete (ID: 5441741360626669024) - -Apply complete! Resources: 58 added, 0 changed, 0 destroyed. -``` - -To watch the bootstrap process in detail, SSH to the first controller and journal the logs. - -``` -$ ssh fedora@node1.example.com -$ journalctl -f -u bootkube -bootkube[5]: Pod Status: pod-checkpointer Running -bootkube[5]: Pod Status: kube-apiserver Running -bootkube[5]: Pod Status: kube-scheduler Running -bootkube[5]: Pod Status: kube-controller-manager Running -bootkube[5]: All self-hosted control plane components successfully started -bootkube[5]: Tearing down temporary bootstrap control plane... -``` - -## Verify - -[Install kubectl](https://kubernetes.io/docs/tasks/tools/install-kubectl/) on your system. Use the generated `kubeconfig` credentials to access the Kubernetes cluster and list nodes. - -``` -$ export KUBECONFIG=/home/user/.secrets/clusters/mercury/auth/kubeconfig -$ kubectl get nodes -NAME STATUS ROLES AGE VERSION -node1.example.com Ready controller,master 10m v1.14.3 -node2.example.com Ready node 10m v1.14.3 -node3.example.com Ready node 10m v1.14.3 -``` - -List the pods. - -``` -$ kubectl get pods --all-namespaces -NAMESPACE NAME READY STATUS RESTARTS AGE -kube-system calico-node-6qp7f 2/2 Running 1 11m -kube-system calico-node-gnjrm 2/2 Running 0 11m -kube-system calico-node-llbgt 2/2 Running 0 11m -kube-system coredns-1187388186-dj3pd 1/1 Running 0 11m -kube-system coredns-1187388186-mx9rt 1/1 Running 0 11m -kube-system kube-apiserver-7336w 1/1 Running 0 11m -kube-system kube-controller-manager-3271970485-b9chx 1/1 Running 0 11m -kube-system kube-controller-manager-3271970485-v30js 1/1 Running 1 11m -kube-system kube-proxy-50sd4 1/1 Running 0 11m -kube-system kube-proxy-bczhp 1/1 Running 0 11m -kube-system kube-proxy-mp2fw 1/1 Running 0 11m -kube-system kube-scheduler-3895335239-fd3l7 1/1 Running 1 11m -kube-system kube-scheduler-3895335239-hfjv0 1/1 Running 0 11m -kube-system pod-checkpointer-wf65d 1/1 Running 0 11m -kube-system pod-checkpointer-wf65d-node1.example.com 1/1 Running 0 11m -``` - -## Going Further - -Learn about [maintenance](/topics/maintenance/) and [addons](/addons/overview/). - -## Variables - -Check the [variables.tf](https://github.com/poseidon/typhoon/blob/master/bare-metal/fedora-atomic/kubernetes/variables.tf) source. - -### Required - -| Name | Description | Example | -|:-----|:------------|:--------| -| cluster_name | Unique cluster name | mercury | -| matchbox_http_endpoint | Matchbox HTTP read-only endpoint | "http://matchbox.example.com:port" | -| atomic_assets_endpoint | HTTP endpoint serving the Fedora Atomic vmlinuz, initrd.img, and ostree repo | "http://example.com/fedora/28" | -| k8s_domain_name | FQDN resolving to the controller(s) nodes. Workers and kubectl will communicate with this endpoint | "myk8s.example.com" | -| ssh_authorized_key | SSH public key for user 'fedora' | "ssh-rsa AAAAB3Nz..." | -| asset_dir | Path to a directory where generated assets should be placed (contains secrets) | "/home/user/.secrets/clusters/mercury" | -| controller_names | Ordered list of controller short names | ["node1"] | -| controller_macs | Ordered list of controller identifying MAC addresses | ["52:54:00:a1:9c:ae"] | -| controller_domains | Ordered list of controller FQDNs | ["node1.example.com"] | -| worker_names | Ordered list of worker short names | ["node2", "node3"] | -| worker_macs | Ordered list of worker identifying MAC addresses | ["52:54:00:b2:2f:86", "52:54:00:c3:61:77"] | -| worker_domains | Ordered list of worker FQDNs | ["node2.example.com", "node3.example.com"] | - -### Optional - -| Name | Description | Default | Example | -|:-----|:------------|:--------|:--------| -| networking | Choice of networking provider | "calico" | "calico" or "flannel" | -| network_mtu | CNI interface MTU (calico-only) | 1480 | - | -| pod_cidr | CIDR IPv4 range to assign to Kubernetes pods | "10.2.0.0/16" | "10.22.0.0/16" | -| service_cidr | CIDR IPv4 range to assign to Kubernetes services | "10.3.0.0/16" | "10.3.0.0/24" | -| cluster_domain_suffix | FQDN suffix for Kubernetes services answered by coredns. | "cluster.local" | "k8s.example.com" | -| kernel_args | Additional kernel args to provide at PXE boot | [] | "kvm-intel.nested=1" | - diff --git a/docs/atomic/digital-ocean.md b/docs/atomic/digital-ocean.md deleted file mode 100644 index 0a4111bb1..000000000 --- a/docs/atomic/digital-ocean.md +++ /dev/null @@ -1,250 +0,0 @@ -# Digital Ocean - -!!! danger - Typhoon for Fedora Atomic will not be updated much beyond Kubernetes v1.13. - -In this tutorial, we'll create a Kubernetes v1.14.3 cluster on DigitalOcean with Fedora Atomic. - -We'll declare a Kubernetes cluster using the Typhoon Terraform module. Then apply the changes to create controller droplets, worker droplets, DNS records, tags, and TLS assets. Instances are provisioned on first boot with cloud-init. - -Controllers are provisioned to run an `etcd` peer and a `kubelet` service. Workers run just a `kubelet` service. A one-time [bootkube](https://github.com/kubernetes-incubator/bootkube) bootstrap schedules the `apiserver`, `scheduler`, `controller-manager`, and `coredns` on controllers and schedules `kube-proxy` and `flannel` on every node. A generated `kubeconfig` provides `kubectl` access to the cluster. - -## Requirements - -* Digital Ocean Account and Token -* Digital Ocean Domain (registered Domain Name or delegated subdomain) -* Terraform v0.11.x installed locally - -## Terraform Setup - -Install [Terraform](https://www.terraform.io/downloads.html) v0.11.x on your system. - -```sh -$ terraform version -Terraform v0.11.12 -``` - -Read [concepts](/architecture/concepts/) to learn about Terraform, modules, and organizing resources. Change to your infrastructure repository (e.g. `infra`). - -``` -cd infra/clusters -``` - -## Provider - -Login to [DigitalOcean](https://cloud.digitalocean.com) or create an [account](https://cloud.digitalocean.com/registrations/new), if you don't have one. - -Generate a Personal Access Token with read/write scope from the [API tab](https://cloud.digitalocean.com/settings/api/tokens). Write the token to a file that can be referenced in configs. - -```sh -mkdir -p ~/.config/digital-ocean -echo "TOKEN" > ~/.config/digital-ocean/token -``` - -Configure the DigitalOcean provider to use your token in a `providers.tf` file. - -```tf -provider "digitalocean" { - version = "~> 1.1.0" - token = "${chomp(file("~/.config/digital-ocean/token"))}" - alias = "default" -} - -provider "local" { - version = "~> 1.0" - alias = "default" -} - -provider "null" { - version = "~> 1.0" - alias = "default" -} - -provider "template" { - version = "~> 1.0" - alias = "default" -} - -provider "tls" { - version = "~> 1.0" - alias = "default" -} -``` - -## Cluster - -Define a Kubernetes cluster using the module `digital-ocean/fedora-atomic/kubernetes`. - -```tf -module "digital-ocean-nemo" { - source = "git::https://github.com/poseidon/typhoon//digital-ocean/fedora-atomic/kubernetes?ref=v1.14.3" - - providers = { - digitalocean = "digitalocean.default" - local = "local.default" - null = "null.default" - template = "template.default" - tls = "tls.default" - } - - # Digital Ocean - cluster_name = "nemo" - region = "nyc3" - dns_zone = "digital-ocean.example.com" - - # configuration - ssh_authorized_key = "ssh-rsa AAAAB3Nz..." - ssh_fingerprints = ["d7:9d:79:ae:56:32:73:79:95:88:e3:a2:ab:5d:45:e7"] - asset_dir = "/home/user/.secrets/clusters/nemo" - - # optional - worker_count = 2 - worker_type = "s-1vcpu-1gb" -} -``` - -Reference the [variables docs](#variables) or the [variables.tf](https://github.com/poseidon/typhoon/blob/master/digital-ocean/fedora-atomic/kubernetes/variables.tf) source. - -## ssh-agent - -Initial bootstrapping requires `bootkube.service` be started on one controller node. Terraform uses `ssh-agent` to automate this step. Add your SSH private key to `ssh-agent`. - -```sh -ssh-add ~/.ssh/id_rsa -ssh-add -L -``` - -## Apply - -Initialize the config directory if this is the first use with Terraform. - -```sh -terraform init -``` - -Plan the resources to be created. - -```sh -$ terraform plan -Plan: 54 to add, 0 to change, 0 to destroy. -``` - -Apply the changes to create the cluster. - -```sh -$ terraform apply -module.digital-ocean-nemo.null_resource.bootkube-start: Still creating... (30s elapsed) -module.digital-ocean-nemo.null_resource.bootkube-start: Provisioning with 'remote-exec'... -... -module.digital-ocean-nemo.null_resource.bootkube-start: Still creating... (6m20s elapsed) -module.digital-ocean-nemo.null_resource.bootkube-start: Creation complete (ID: 7599298447329218468) - -Apply complete! Resources: 54 added, 0 changed, 0 destroyed. -``` - -In 3-6 minutes, the Kubernetes cluster will be ready. - -## Verify - -[Install kubectl](https://kubernetes.io/docs/tasks/tools/install-kubectl/) on your system. Use the generated `kubeconfig` credentials to access the Kubernetes cluster and list nodes. - -``` -$ export KUBECONFIG=/home/user/.secrets/clusters/nemo/auth/kubeconfig -$ kubectl get nodes -NAME STATUS ROLES AGE VERSION -10.132.110.130 Ready controller,master 10m v1.14.3 -10.132.115.81 Ready node 10m v1.14.3 -10.132.124.107 Ready node 10m v1.14.3 -``` - -List the pods. - -``` -NAMESPACE NAME READY STATUS RESTARTS AGE -kube-system coredns-1187388186-ld1j7 1/1 Running 0 11m -kube-system coredns-1187388186-rdhf7 1/1 Running 0 11m -kube-system flannel-1cq1v 2/2 Running 0 11m -kube-system flannel-hq9t0 2/2 Running 1 11m -kube-system flannel-v0g9w 2/2 Running 0 11m -kube-system kube-apiserver-n10qr 1/1 Running 0 11m -kube-system kube-controller-manager-3271970485-37gtw 1/1 Running 1 11m -kube-system kube-controller-manager-3271970485-p52t5 1/1 Running 0 11m -kube-system kube-proxy-6kxjf 1/1 Running 0 11m -kube-system kube-proxy-fh3td 1/1 Running 0 11m -kube-system kube-proxy-k35rc 1/1 Running 0 11m -kube-system kube-scheduler-3895335239-2bc4c 1/1 Running 0 11m -kube-system kube-scheduler-3895335239-b7q47 1/1 Running 1 11m -kube-system pod-checkpointer-pr1lq 1/1 Running 0 11m -kube-system pod-checkpointer-pr1lq-10.132.115.81 1/1 Running 0 10m -``` - -## Going Further - -Learn about [maintenance](/topics/maintenance/) and [addons](/addons/overview/). - -## Variables - -Check the [variables.tf](https://github.com/poseidon/typhoon/blob/master/digital-ocean/fedora-atomic/kubernetes/variables.tf) source. - -### Required - -| Name | Description | Example | -|:-----|:------------|:--------| -| cluster_name | Unique cluster name (prepended to dns_zone) | nemo | -| region | Digital Ocean region | nyc1, sfo2, fra1, tor1 | -| dns_zone | Digital Ocean domain (i.e. DNS zone) | do.example.com | -| ssh_authorized_key | SSH public key for user 'fedora' | "ssh-rsa AAAAB3NZ..." | -| ssh_fingerprints | SSH public key fingerprints | ["d7:9d..."] | -| asset_dir | Path to a directory where generated assets should be placed (contains secrets) | /home/user/.secrets/nemo | - -#### DNS Zone - -Clusters create DNS A records `${cluster_name}.${dns_zone}` to resolve to controller droplets (round robin). This FQDN is used by workers and `kubectl` to access the apiserver(s). In this example, the cluster's apiserver would be accessible at `nemo.do.example.com`. - -You'll need a registered domain name or delegated subdomain in Digital Ocean Domains (i.e. DNS zones). You can set this up once and create many clusters with unique names. - -```tf -# Declare a DigitalOcean record to also create a zone file -resource "digitalocean_domain" "zone-for-clusters" { - name = "do.example.com" - ip_address = "8.8.8.8" -} -``` - -!!! tip "" - If you have an existing domain name with a zone file elsewhere, just delegate a subdomain that can be managed on DigitalOcean (e.g. do.mydomain.com) and [update nameservers](https://www.digitalocean.com/community/tutorials/how-to-set-up-a-host-name-with-digitalocean). - -#### SSH Fingerprints - -DigitalOcean droplets are created with your SSH public key "fingerprint" (i.e. MD5 hash) to allow access. If your SSH public key is at `~/.ssh/id_rsa`, find the fingerprint with, - -```bash -ssh-keygen -E md5 -lf ~/.ssh/id_rsa.pub | awk '{print $2}' -MD5:d7:9d:79:ae:56:32:73:79:95:88:e3:a2:ab:5d:45:e7 -``` - -If you use `ssh-agent` (e.g. Yubikey for SSH), find the fingerprint with, - -``` -ssh-add -l -E md5 -2048 MD5:d7:9d:79:ae:56:32:73:79:95:88:e3:a2:ab:5d:45:e7 cardno:000603633110 (RSA) -``` - -Digital Ocean requires the SSH public key be uploaded to your account, so you may also find the fingerprint under Settings -> Security. Finally, if you don't have an SSH key, [create one now](https://help.github.com/articles/generating-a-new-ssh-key-and-adding-it-to-the-ssh-agent/). - -### Optional - -| Name | Description | Default | Example | -|:-----|:------------|:--------|:--------| -| controller_count | Number of controllers (i.e. masters) | 1 | 1 | -| worker_count | Number of workers | 1 | 3 | -| controller_type | Droplet type for controllers | s-2vcpu-2gb | s-2vcpu-2gb, s-2vcpu-4gb, s-4vcpu-8gb, ... | -| worker_type | Droplet type for workers | s-1vcpu-1gb | s-1vcpu-1gb, s-1vcpu-2gb, s-2vcpu-2gb, ... | -| pod_cidr | CIDR IPv4 range to assign to Kubernetes pods | "10.2.0.0/16" | "10.22.0.0/16" | -| service_cidr | CIDR IPv4 range to assign to Kubernetes services | "10.3.0.0/16" | "10.3.0.0/24" | -| cluster_domain_suffix | FQDN suffix for Kubernetes services answered by coredns. | "cluster.local" | "k8s.example.com" | - -Check the list of valid [droplet types](https://developers.digitalocean.com/documentation/changelog/api-v2/new-size-slugs-for-droplet-plan-changes/) or use `doctl compute size list`. - -!!! warning - Do not choose a `controller_type` smaller than 2GB. Smaller droplets are not sufficient for running a controller and bootstrapping will fail. diff --git a/docs/atomic/google-cloud.md b/docs/atomic/google-cloud.md deleted file mode 100644 index 1e4ae76cb..000000000 --- a/docs/atomic/google-cloud.md +++ /dev/null @@ -1,285 +0,0 @@ -# Google Cloud - -!!! danger - Typhoon for Fedora Atomic will not be updated much beyond Kubernetes v1.13. Fedora does not publish official images for Google Cloud so you must prepare them yourself. Expect rough edges and changes. - -In this tutorial, we'll create a Kubernetes v1.14.3 cluster on Google Compute Engine with Fedora Atomic. - -We'll declare a Kubernetes cluster using the Typhoon Terraform module. Then apply the changes to create a network, firewall rules, health checks, controller instances, worker managed instance group, load balancers, and TLS assets. Instances are provisioned on first boot with cloud-init. - -Controllers are provisioned to run an `etcd` peer and a `kubelet` service. Workers run just a `kubelet` service. A one-time [bootkube](https://github.com/kubernetes-incubator/bootkube) bootstrap schedules the `apiserver`, `scheduler`, `controller-manager`, and `coredns` on controllers and schedules `kube-proxy` and `calico` (or `flannel`) on every node. A generated `kubeconfig` provides `kubectl` access to the cluster. - -## Requirements - -* Google Cloud Account and Service Account -* Google Cloud DNS Zone (registered main Name or delegated subdomain) -* Terraform v0.11.x installed locally -* `gcloud` and `gsutil` for uploading a disk image to Google Cloud (temporary) - -## Terraform Setup - -Install [Terraform](https://www.terraform.io/downloads.html) v0.11.x on your system. - -```sh -$ terraform version -Terraform v0.11.12 -``` - -Read [concepts](/architecture/concepts/) to learn about Terraform, modules, and organizing resources. Change to your infrastructure repository (e.g. `infra`). - -``` -cd infra/clusters -``` - -## Provider - -Login to your Google Console [API Manager](https://console.cloud.google.com/apis/dashboard) and select a project, or [signup](https://cloud.google.com/free/) if you don't have an account. - -Select "Credentials" and create a service account key. Choose the "Compute Engine Admin" and "DNS Administrator" roles and save the JSON private key to a file that can be referenced in configs. - -```sh -mv ~/Downloads/project-id-43048204.json ~/.config/google-cloud/terraform.json -``` - -Configure the Google Cloud provider to use your service account key, project-id, and region in a `providers.tf` file. - -```tf -provider "google" { - version = "~> 2.2.0" - alias = "default" - - credentials = "${file("~/.config/google-cloud/terraform.json")}" - project = "project-id" - region = "us-central1" -} - -provider "local" { - version = "~> 1.0" - alias = "default" -} - -provider "null" { - version = "~> 1.0" - alias = "default" -} - -provider "template" { - version = "~> 1.0" - alias = "default" -} - -provider "tls" { - version = "~> 1.0" - alias = "default" -} -``` - -Additional configuration options are described in the `google` provider [docs](https://www.terraform.io/docs/providers/google/index.html). - -!!! tip - Regions are listed in [docs](https://cloud.google.com/compute/docs/regions-zones/regions-zones) or with `gcloud compute regions list`. A project may container multiple clusters across different regions. - -## Atomic Image - -Project Atomic does not publish official Fedora Atomic images to Google Cloud. However, Google Cloud allows [custom boot images](https://cloud.google.com/compute/docs/images/import-existing-image) to be uploaded to a bucket and imported into your project. - -Download the Fedora Atomic 28 [raw image](https://getfedora.org/en/atomic/download/) and decompress the file. - -``` -xz -d Fedora-AtomicHost-28-20180528.0.x86_64.raw.xz -``` - -!!! warning - Download the exact dated version shown in docs. Fedora has no official Atomic images for Google Cloud. We've verified specific versions and found others to have problems. - -Rename the image `disk.raw`. Gzip compress and tar the image. - -``` -mv Fedora-AtomicHost-28-20180528.0.x86_64.raw disk.raw -tar cvzf fedora-atomic-28.tar.gz disk.raw -``` - -List available storage buckets and upload the tar.gz. - -``` -gsutil list -gsutil cp fedora-atomic-28.tar.gz gs://BUCKET_NAME -``` - -Create a Google Compute Engine image from the bucket file. - -``` -gcloud compute images list -gcloud compute images create fedora-atomic-28 --source-uri gs://BUCKET/fedora-atomic-28.tar.gz -``` - -Note your project id and the image name for setting `os_image` later (e.g. proj-id/fedora-atomic-28). - -## Cluster - -Define a Kubernetes cluster using the module `google-cloud/fedora-atomic/kubernetes`. - -```tf -module "google-cloud-yavin" { - source = "git::https://github.com/poseidon/typhoon//google-cloud/fedora-atomic/kubernetes?ref=v1.14.3" - - providers = { - google = "google.default" - local = "local.default" - null = "null.default" - template = "template.default" - tls = "tls.default" - } - - # Google Cloud - cluster_name = "yavin" - region = "us-central1" - dns_zone = "example.com" - dns_zone_name = "example-zone" - - # configuration - ssh_authorized_key = "ssh-rsa AAAAB3Nz..." - asset_dir = "/home/user/.secrets/clusters/yavin" - os_image = "MY-PROJECT_ID/fedora-atomic-28" - - # optional - worker_count = 2 -} -``` - -Reference the [variables docs](#variables) or the [variables.tf](https://github.com/poseidon/typhoon/blob/master/google-cloud/fedora-atomic/kubernetes/variables.tf) source. - -## ssh-agent - -Initial bootstrapping requires `bootkube.service` be started on one controller node. Terraform uses `ssh-agent` to automate this step. Add your SSH private key to `ssh-agent`. - -```sh -ssh-add ~/.ssh/id_rsa -ssh-add -L -``` - -## Apply - -Initialize the config directory if this is the first use with Terraform. - -```sh -terraform init -``` - -Plan the resources to be created. - -```sh -$ terraform plan -Plan: 73 to add, 0 to change, 0 to destroy. -``` - -Apply the changes to create the cluster. - -```sh -$ terraform apply -module.google-cloud-yavin.null_resource.bootkube-start: Still creating... (10s elapsed) -... - -module.google-cloud-yavin.null_resource.bootkube-start: Still creating... (5m30s elapsed) -module.google-cloud-yavin.null_resource.bootkube-start: Still creating... (5m40s elapsed) -module.google-cloud-yavin.null_resource.bootkube-start: Creation complete (ID: 5768638456220583358) - -Apply complete! Resources: 73 added, 0 changed, 0 destroyed. -``` - -In 5-10 minutes, the Kubernetes cluster will be ready. - -## Verify - -[Install kubectl](https://kubernetes.io/docs/tasks/tools/install-kubectl/) on your system. Use the generated `kubeconfig` credentials to access the Kubernetes cluster and list nodes. - -``` -$ export KUBECONFIG=/home/user/.secrets/clusters/yavin/auth/kubeconfig -$ kubectl get nodes -NAME ROLES STATUS AGE VERSION -yavin-controller-0.c.example-com.internal controller,master Ready 6m v1.14.3 -yavin-worker-jrbf.c.example-com.internal node Ready 5m v1.14.3 -yavin-worker-mzdm.c.example-com.internal node Ready 5m v1.14.3 -``` - -List the pods. - -``` -$ kubectl get pods --all-namespaces -NAMESPACE NAME READY STATUS RESTARTS AGE -kube-system calico-node-1cs8z 2/2 Running 0 6m -kube-system calico-node-d1l5b 2/2 Running 0 6m -kube-system calico-node-sp9ps 2/2 Running 0 6m -kube-system coredns-1187388186-dkh3o 1/1 Running 0 6m -kube-system coredns-1187388186-zj5dl 1/1 Running 0 6m -kube-system kube-apiserver-zppls 1/1 Running 0 6m -kube-system kube-controller-manager-3271970485-gh9kt 1/1 Running 0 6m -kube-system kube-controller-manager-3271970485-h90v8 1/1 Running 1 6m -kube-system kube-proxy-117v6 1/1 Running 0 6m -kube-system kube-proxy-9886n 1/1 Running 0 6m -kube-system kube-proxy-njn47 1/1 Running 0 6m -kube-system kube-scheduler-3895335239-5x87r 1/1 Running 0 6m -kube-system kube-scheduler-3895335239-bzrrt 1/1 Running 1 6m -kube-system pod-checkpointer-l6lrt 1/1 Running 0 6m -``` - -## Going Further - -Learn about [maintenance](/topics/maintenance/) and [addons](/addons/overview/). - -## Variables - -Check the [variables.tf](https://github.com/poseidon/typhoon/blob/master/google-cloud/fedora-atomic/kubernetes/variables.tf) source. - -### Required - -| Name | Description | Example | -|:-----|:------------|:--------| -| cluster_name | Unique cluster name (prepended to dns_zone) | "yavin" | -| region | Google Cloud region | "us-central1" | -| dns_zone | Google Cloud DNS zone | "google-cloud.example.com" | -| dns_zone_name | Google Cloud DNS zone name | "example-zone" | -| os_image | Custom uploaded Fedora Atomic image | "PROJECT-ID/fedora-atomic-28" | -| ssh_authorized_key | SSH public key for user 'fedora' | "ssh-rsa AAAAB3NZ..." | -| asset_dir | Path to a directory where generated assets should be placed (contains secrets) | "/home/user/.secrets/clusters/yavin" | - -Check the list of valid [regions](https://cloud.google.com/compute/docs/regions-zones/regions-zones). - -#### DNS Zone - -Clusters create a DNS A record `${cluster_name}.${dns_zone}` to resolve a network load balancer backed by controller instances. This FQDN is used by workers and `kubectl` to access the apiserver(s). In this example, the cluster's apiserver would be accessible at `yavin.google-cloud.example.com`. - -You'll need a registered domain name or delegated subdomain on Google Cloud DNS. You can set this up once and create many clusters with unique names. - -```tf -resource "google_dns_managed_zone" "zone-for-clusters" { - dns_name = "google-cloud.example.com." - name = "example-zone" - description = "Production DNS zone" -} -``` - -!!! tip "" - If you have an existing domain name with a zone file elsewhere, just delegate a subdomain that can be managed on Google Cloud (e.g. google-cloud.mydomain.com) and [update nameservers](https://cloud.google.com/dns/update-name-servers). - -### Optional - -| Name | Description | Default | Example | -|:-----|:------------|:--------|:--------| -| controller_count | Number of controllers (i.e. masters) | 1 | 3 | -| worker_count | Number of workers | 1 | 3 | -| controller_type | Machine type for controllers | "n1-standard-1" | See below | -| worker_type | Machine type for workers | "n1-standard-1" | See below | -| disk_size | Size of the disk in GB | 40 | 100 | -| worker_preemptible | If enabled, Compute Engine will terminate workers randomly within 24 hours | false | true | -| networking | Choice of networking provider | "calico" | "calico" or "flannel" | -| pod_cidr | CIDR IPv4 range to assign to Kubernetes pods | "10.2.0.0/16" | "10.22.0.0/16" | -| service_cidr | CIDR IPv4 range to assign to Kubernetes services | "10.3.0.0/16" | "10.3.0.0/24" | -| cluster_domain_suffix | FQDN suffix for Kubernetes services answered by coredns. | "cluster.local" | "k8s.example.com" | - -Check the list of valid [machine types](https://cloud.google.com/compute/docs/machine-types). - -#### Preemption - -Add `worker_preemeptible = "true"` to allow worker nodes to be [preempted](https://cloud.google.com/compute/docs/instances/preemptible) at random, but pay [significantly](https://cloud.google.com/compute/pricing) less. Clusters tolerate stopping instances fairly well (reschedules pods, but cannot drain) and preemption provides a nice reward for running fault-tolerant cluster systems.` - diff --git a/docs/index.md b/docs/index.md index c7bbe0448..593faf289 100644 --- a/docs/index.md +++ b/docs/index.md @@ -29,15 +29,6 @@ Typhoon provides a Terraform Module for each supported operating system and plat | Digital Ocean | Container Linux | [digital-ocean/container-linux/kubernetes](cl/digital-ocean.md) | beta | | Google Cloud | Container Linux | [google-cloud/container-linux/kubernetes](cl/google-cloud.md) | stable | -Fedora Atomic support is alpha and will evolve as Fedora Atomic is replaced by Fedora CoreOS. - -| Platform | Operating System | Terraform Module | Status | -|---------------|------------------|------------------|--------| -| AWS | Fedora Atomic | [aws/fedora-atomic/kubernetes](atomic/aws.md) | deprecated | -| Bare-Metal | Fedora Atomic | [bare-metal/fedora-atomic/kubernetes](atomic/bare-metal.md) | deprecated | -| Digital Ocean | Fedora Atomic | [digital-ocean/fedora-atomic/kubernetes](atomic/digital-ocean.md) | deprecated | -| Google Cloud | Fedora Atomic | [google-cloud/fedora-atomic/kubernetes](atomic/google-cloud.md) | deprecated | - ## Documentation * Architecture [concepts](architecture/concepts.md) and [operating-systems](architecture/operating-systems.md) diff --git a/docs/topics/faq.md b/docs/topics/faq.md index 72d99c751..1a8eef263 100644 --- a/docs/topics/faq.md +++ b/docs/topics/faq.md @@ -8,18 +8,13 @@ Formats rise and evolve. Typhoon may choose to adapt the format over time (with ## Operating Systems -Typhoon supports Container Linux and Fedora Atomic 28. These two operating systems were chosen because they offer: +Typhoon supports Container Linux and the Flatcar Linux derivative. These operating systems were chosen because they offer: * Minimalism and focus on clustered operation * Automated and atomic operating system upgrades * Declarative and immutable configuration * Optimization for containerized applications -Together, they diversify Typhoon to support a range of container technologies. - -* Container Linux: Gentoo core, rkt-fly, docker -* Fedora Atomic: RHEL core, rpm-ostree, system containers (i.e. runc), CRI-O - ## Get Help Ask questions on the IRC #typhoon channel on [freenode.net](http://freenode.net/). diff --git a/docs/topics/security.md b/docs/topics/security.md index 3a3411f3d..7eb0cdec4 100644 --- a/docs/topics/security.md +++ b/docs/topics/security.md @@ -42,9 +42,7 @@ Typhoon limits exposure to many security threats, but it is not a silver bullet. ## OpenPGP Signing -Typhoon uses upstream container images and binaries. We do not distribute artifacts of our own, except where required for system container images ([etcd](https://quay.io/repository/poseidon/etcd), [kubelet](https://quay.io/repository/poseidon/kubelet), [bootkube](https://quay.io/repository/poseidon/bootkube)) for Fedora Atomic only. - -If you find artifacts claiming to be from Typhoon, please send a note. +Typhoon uses upstream container images and binaries. We do not distribute artifacts of our own. If you find artifacts claiming to be from Typhoon, please send a note. ## Disclosures From 331ebd90f6a0347973efe6907c1e353ab8b9c1e1 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Fri, 21 Jun 2019 10:03:21 -0700 Subject: [PATCH 138/523] Acknowledge DigitalOcean providing credits for test clusters (#500) * [DigitalOcean](https://www.digitalocean.com/) kindly provides credits to support Typhoon test clusters. Many thanks! --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index e6ab04f01..37e91628a 100644 --- a/README.md +++ b/README.md @@ -136,3 +136,5 @@ Typhoon clusters will contain only [free](https://www.debian.org/intro/free) com ## Donations Typhoon does not accept money donations. Instead, we encourage you to donate to one of [these organizations](https://github.com/poseidon/typhoon/wiki/Donations) to show your appreciation. + +* [DigitalOcean](https://www.digitalocean.com/) kindly provides credits to support Typhoon test clusters. From 5c4486f57be4801a93e232b45b016f70806a042e Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Tue, 11 Jun 2019 22:13:41 -0700 Subject: [PATCH 139/523] Allow using Flatcar Linux Edge on bare-metal and AWS * On AWS, use Flatcar Linux Edge by setting `os_image` to "flatcar-edge" * On bare-metal, Flatcar Linux Edge by setting `os_channel` to "flatcar-edge" --- CHANGES.md | 2 ++ aws/container-linux/kubernetes/ami.tf | 4 ++-- aws/container-linux/kubernetes/variables.tf | 2 +- aws/container-linux/kubernetes/workers/ami.tf | 4 ++-- aws/container-linux/kubernetes/workers/variables.tf | 2 +- bare-metal/container-linux/kubernetes/variables.tf | 4 ++-- docs/cl/aws.md | 2 +- docs/cl/bare-metal.md | 2 +- docs/index.md | 4 ++-- 9 files changed, 14 insertions(+), 12 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index a4f1748c7..6d12b3429 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -12,6 +12,7 @@ Notable changes between versions. #### AWS * Require `terraform-provider-aws` v2.7+ to support Terraform v0.12 (action required) +* Allow using Flatcar Linux Edge by setting `os_image` to "flatcar-edge" #### Azure @@ -23,6 +24,7 @@ Notable changes between versions. #### Bare-Metal * Require `terraform-provider-matchbox` v0.3.0+ to support Terraform v0.12 (action required) +* Allow using Flatcar Linux Edge by setting `os_channel` to "flatcar-edge" #### DigitalOcean diff --git a/aws/container-linux/kubernetes/ami.tf b/aws/container-linux/kubernetes/ami.tf index cde47bdfc..b5124b4be 100644 --- a/aws/container-linux/kubernetes/ami.tf +++ b/aws/container-linux/kubernetes/ami.tf @@ -24,7 +24,7 @@ data "aws_ami" "coreos" { filter { name = "name" - values = ["CoreOS-${local.channel}-*"] + values = ["CoreOS-${local.flavor == "coreos" ? local.channel : "stable"}-*"] } } @@ -44,7 +44,7 @@ data "aws_ami" "flatcar" { filter { name = "name" - values = ["Flatcar-${local.channel}-*"] + values = ["Flatcar-${local.flavor == "flatcar" ? local.channel : "stable"}-*"] } } diff --git a/aws/container-linux/kubernetes/variables.tf b/aws/container-linux/kubernetes/variables.tf index 42d032823..7005e63b6 100644 --- a/aws/container-linux/kubernetes/variables.tf +++ b/aws/container-linux/kubernetes/variables.tf @@ -44,7 +44,7 @@ variable "worker_type" { variable "os_image" { type = string default = "coreos-stable" - description = "AMI channel for a Container Linux derivative (coreos-stable, coreos-beta, coreos-alpha, flatcar-stable, flatcar-beta, flatcar-alpha)" + description = "AMI channel for a Container Linux derivative (coreos-stable, coreos-beta, coreos-alpha, flatcar-stable, flatcar-beta, flatcar-alpha, flatcar-edge)" } variable "disk_size" { diff --git a/aws/container-linux/kubernetes/workers/ami.tf b/aws/container-linux/kubernetes/workers/ami.tf index cde47bdfc..b5124b4be 100644 --- a/aws/container-linux/kubernetes/workers/ami.tf +++ b/aws/container-linux/kubernetes/workers/ami.tf @@ -24,7 +24,7 @@ data "aws_ami" "coreos" { filter { name = "name" - values = ["CoreOS-${local.channel}-*"] + values = ["CoreOS-${local.flavor == "coreos" ? local.channel : "stable"}-*"] } } @@ -44,7 +44,7 @@ data "aws_ami" "flatcar" { filter { name = "name" - values = ["Flatcar-${local.channel}-*"] + values = ["Flatcar-${local.flavor == "flatcar" ? local.channel : "stable"}-*"] } } diff --git a/aws/container-linux/kubernetes/workers/variables.tf b/aws/container-linux/kubernetes/workers/variables.tf index 67b5e061f..e5c5b6082 100644 --- a/aws/container-linux/kubernetes/workers/variables.tf +++ b/aws/container-linux/kubernetes/workers/variables.tf @@ -37,7 +37,7 @@ variable "instance_type" { variable "os_image" { type = string default = "coreos-stable" - description = "AMI channel for a Container Linux derivative (coreos-stable, coreos-beta, coreos-alpha, flatcar-stable, flatcar-beta, flatcar-alpha)" + description = "AMI channel for a Container Linux derivative (coreos-stable, coreos-beta, coreos-alpha, flatcar-stable, flatcar-beta, flatcar-alpha, flatcar-edge)" } variable "disk_size" { diff --git a/bare-metal/container-linux/kubernetes/variables.tf b/bare-metal/container-linux/kubernetes/variables.tf index 18e7d009c..a95a808b3 100644 --- a/bare-metal/container-linux/kubernetes/variables.tf +++ b/bare-metal/container-linux/kubernetes/variables.tf @@ -12,12 +12,12 @@ variable "matchbox_http_endpoint" { variable "os_channel" { type = string - description = "Channel for a Container Linux derivative (coreos-stable, coreos-beta, coreos-alpha, flatcar-stable, flatcar-beta, flatcar-alpha)" + description = "Channel for a Container Linux derivative (coreos-stable, coreos-beta, coreos-alpha, flatcar-stable, flatcar-beta, flatcar-alpha, flatcar-edge)" } variable "os_version" { type = string - description = "Version for a Container Linux derivative to PXE and install (coreos-stable, coreos-beta, coreos-alpha, flatcar-stable, flatcar-beta, flatcar-alpha)" + description = "Version for a Container Linux derivative to PXE and install (e.g. 2079.5.1)" } # machines diff --git a/docs/cl/aws.md b/docs/cl/aws.md index 857a72b73..2c35c1477 100644 --- a/docs/cl/aws.md +++ b/docs/cl/aws.md @@ -208,7 +208,7 @@ Reference the DNS zone id with `"${aws_route53_zone.zone-for-clusters.zone_id}"` | worker_count | Number of workers | 1 | 3 | | controller_type | EC2 instance type for controllers | "t3.small" | See below | | worker_type | EC2 instance type for workers | "t3.small" | See below | -| os_image | AMI channel for a Container Linux derivative | coreos-stable | coreos-stable, coreos-beta, coreos-alpha, flatcar-stable, flatcar-beta, flatcar-alpha | +| os_image | AMI channel for a Container Linux derivative | coreos-stable | coreos-stable, coreos-beta, coreos-alpha, flatcar-stable, flatcar-beta, flatcar-alpha, flatcar-edge | | disk_size | Size of the EBS volume in GB | "40" | "100" | | disk_type | Type of the EBS volume | "gp2" | standard, gp2, io1 | | disk_iops | IOPS of the EBS volume | "0" (i.e. auto) | "400" | diff --git a/docs/cl/bare-metal.md b/docs/cl/bare-metal.md index b9ad5e78c..5ebe1f5f3 100644 --- a/docs/cl/bare-metal.md +++ b/docs/cl/bare-metal.md @@ -336,7 +336,7 @@ Check the [variables.tf](https://github.com/poseidon/typhoon/blob/master/bare-me |:-----|:------------|:--------| | cluster_name | Unique cluster name | mercury | | matchbox_http_endpoint | Matchbox HTTP read-only endpoint | http://matchbox.example.com:port | -| os_channel | Channel for a Container Linux derivative | coreos-stable, coreos-beta, coreos-alpha, flatcar-stable, flatcar-beta, flatcar-alpha | +| os_channel | Channel for a Container Linux derivative | coreos-stable, coreos-beta, coreos-alpha, flatcar-stable, flatcar-beta, flatcar-alpha, flatcar-edge | | os_version | Version for a Container Linux derivative to PXE and install | 1632.3.0 | | k8s_domain_name | FQDN resolving to the controller(s) nodes. Workers and kubectl will communicate with this endpoint | "myk8s.example.com" | | ssh_authorized_key | SSH public key for user 'core' | "ssh-rsa AAAAB3Nz..." | diff --git a/docs/index.md b/docs/index.md index 593faf289..ecbaf87c7 100644 --- a/docs/index.md +++ b/docs/index.md @@ -23,9 +23,9 @@ Typhoon provides a Terraform Module for each supported operating system and plat | Platform | Operating System | Terraform Module | Status | |---------------|------------------|------------------|--------| -| AWS | Container Linux | [aws/container-linux/kubernetes](cl/aws.md) | stable | +| AWS | Container Linux / Flatcar Linux | [aws/container-linux/kubernetes](cl/aws.md) | stable | | Azure | Container Linux | [azure/container-linux/kubernetes](cl/azure.md) | alpha | -| Bare-Metal | Container Linux | [bare-metal/container-linux/kubernetes](cl/bare-metal.md) | stable | +| Bare-Metal | Container Linux / Flatcar Linux | [bare-metal/container-linux/kubernetes](cl/bare-metal.md) | stable | | Digital Ocean | Container Linux | [digital-ocean/container-linux/kubernetes](cl/digital-ocean.md) | beta | | Google Cloud | Container Linux | [google-cloud/container-linux/kubernetes](cl/google-cloud.md) | stable | From 79d910821d6c64dfb6e256033337a8f3c4e0ff36 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Tue, 11 Jun 2019 23:24:01 -0700 Subject: [PATCH 140/523] Configure Kubelet cgroup-driver for Flatcar Linux Edge * For Container Linux or Flatcar Linux alpha/beta/stable, continue using the `cgroupfs` driver * For Fedora Atomic, continue using the `systemd` driver * For Flatcar Linux Edge, use the `systemd` driver --- aws/container-linux/kubernetes/cl/controller.yaml.tmpl | 2 ++ aws/container-linux/kubernetes/controllers.tf | 1 + aws/container-linux/kubernetes/workers/cl/worker.yaml.tmpl | 2 ++ aws/container-linux/kubernetes/workers/workers.tf | 1 + .../container-linux/kubernetes/cl/controller.yaml.tmpl | 2 ++ bare-metal/container-linux/kubernetes/cl/worker.yaml.tmpl | 2 ++ bare-metal/container-linux/kubernetes/profiles.tf | 2 ++ docs/architecture/operating-systems.md | 6 +++--- 8 files changed, 15 insertions(+), 3 deletions(-) diff --git a/aws/container-linux/kubernetes/cl/controller.yaml.tmpl b/aws/container-linux/kubernetes/cl/controller.yaml.tmpl index e83b16753..7b7b26306 100644 --- a/aws/container-linux/kubernetes/cl/controller.yaml.tmpl +++ b/aws/container-linux/kubernetes/cl/controller.yaml.tmpl @@ -63,6 +63,7 @@ systemd: --volume var-log,kind=host,source=/var/log \ --mount volume=var-log,target=/var/log \ --insecure-options=image" + Environment=KUBELET_CGROUP_DRIVER=${cgroup_driver} ExecStartPre=/bin/mkdir -p /opt/cni/bin ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d @@ -77,6 +78,7 @@ systemd: --anonymous-auth=false \ --authentication-token-webhook \ --authorization-mode=Webhook \ + --cgroup-driver=$${KUBELET_CGROUP_DRIVER} \ --client-ca-file=/etc/kubernetes/ca.crt \ --cluster_dns=${cluster_dns_service_ip} \ --cluster_domain=${cluster_domain_suffix} \ diff --git a/aws/container-linux/kubernetes/controllers.tf b/aws/container-linux/kubernetes/controllers.tf index 98e7d002a..c0553eb9f 100644 --- a/aws/container-linux/kubernetes/controllers.tf +++ b/aws/container-linux/kubernetes/controllers.tf @@ -69,6 +69,7 @@ data "template_file" "controller-configs" { etcd_domain = "${var.cluster_name}-etcd${count.index}.${var.dns_zone}" # etcd0=https://cluster-etcd0.example.com,etcd1=https://cluster-etcd1.example.com,... etcd_initial_cluster = join(",", data.template_file.etcds.*.rendered) + cgroup_driver = local.flavor == "flatcar" && local.channel == "edge" ? "systemd" : "cgroupfs" kubeconfig = indent(10, module.bootkube.kubeconfig-kubelet) ssh_authorized_key = var.ssh_authorized_key cluster_dns_service_ip = cidrhost(var.service_cidr, 10) diff --git a/aws/container-linux/kubernetes/workers/cl/worker.yaml.tmpl b/aws/container-linux/kubernetes/workers/cl/worker.yaml.tmpl index 9ed6ebfb9..f618476f2 100644 --- a/aws/container-linux/kubernetes/workers/cl/worker.yaml.tmpl +++ b/aws/container-linux/kubernetes/workers/cl/worker.yaml.tmpl @@ -38,6 +38,7 @@ systemd: --volume var-log,kind=host,source=/var/log \ --mount volume=var-log,target=/var/log \ --insecure-options=image" + Environment=KUBELET_CGROUP_DRIVER=${cgroup_driver} ExecStartPre=/bin/mkdir -p /opt/cni/bin ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d @@ -50,6 +51,7 @@ systemd: --anonymous-auth=false \ --authentication-token-webhook \ --authorization-mode=Webhook \ + --cgroup-driver=$${KUBELET_CGROUP_DRIVER} \ --client-ca-file=/etc/kubernetes/ca.crt \ --cluster_dns=${cluster_dns_service_ip} \ --cluster_domain=${cluster_domain_suffix} \ diff --git a/aws/container-linux/kubernetes/workers/workers.tf b/aws/container-linux/kubernetes/workers/workers.tf index 26f9eab37..d470b5407 100644 --- a/aws/container-linux/kubernetes/workers/workers.tf +++ b/aws/container-linux/kubernetes/workers/workers.tf @@ -84,6 +84,7 @@ data "template_file" "worker-config" { ssh_authorized_key = var.ssh_authorized_key cluster_dns_service_ip = cidrhost(var.service_cidr, 10) cluster_domain_suffix = var.cluster_domain_suffix + cgroup_driver = local.flavor == "flatcar" && local.channel == "edge" ? "systemd" : "cgroupfs" } } diff --git a/bare-metal/container-linux/kubernetes/cl/controller.yaml.tmpl b/bare-metal/container-linux/kubernetes/cl/controller.yaml.tmpl index 13befbc99..b6b2939ef 100644 --- a/bare-metal/container-linux/kubernetes/cl/controller.yaml.tmpl +++ b/bare-metal/container-linux/kubernetes/cl/controller.yaml.tmpl @@ -75,6 +75,7 @@ systemd: --volume iscsiadm,kind=host,source=/usr/sbin/iscsiadm \ --mount volume=iscsiadm,target=/sbin/iscsiadm \ --insecure-options=image" + Environment=KUBELET_CGROUP_DRIVER=${cgroup_driver} ExecStartPre=/bin/mkdir -p /opt/cni/bin ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d @@ -89,6 +90,7 @@ systemd: --anonymous-auth=false \ --authentication-token-webhook \ --authorization-mode=Webhook \ + --cgroup-driver=$${KUBELET_CGROUP_DRIVER} \ --client-ca-file=/etc/kubernetes/ca.crt \ --cluster_dns=${cluster_dns_service_ip} \ --cluster_domain=${cluster_domain_suffix} \ diff --git a/bare-metal/container-linux/kubernetes/cl/worker.yaml.tmpl b/bare-metal/container-linux/kubernetes/cl/worker.yaml.tmpl index c2be52683..750f328e6 100644 --- a/bare-metal/container-linux/kubernetes/cl/worker.yaml.tmpl +++ b/bare-metal/container-linux/kubernetes/cl/worker.yaml.tmpl @@ -50,6 +50,7 @@ systemd: --volume iscsiadm,kind=host,source=/usr/sbin/iscsiadm \ --mount volume=iscsiadm,target=/sbin/iscsiadm \ --insecure-options=image" + Environment=KUBELET_CGROUP_DRIVER=${cgroup_driver} ExecStartPre=/bin/mkdir -p /opt/cni/bin ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d @@ -62,6 +63,7 @@ systemd: --anonymous-auth=false \ --authentication-token-webhook \ --authorization-mode=Webhook \ + --cgroup-driver=$${KUBELET_CGROUP_DRIVER} \ --client-ca-file=/etc/kubernetes/ca.crt \ --cluster_dns=${cluster_dns_service_ip} \ --cluster_domain=${cluster_domain_suffix} \ diff --git a/bare-metal/container-linux/kubernetes/profiles.tf b/bare-metal/container-linux/kubernetes/profiles.tf index 20f8ec2fa..400ef6ed3 100644 --- a/bare-metal/container-linux/kubernetes/profiles.tf +++ b/bare-metal/container-linux/kubernetes/profiles.tf @@ -159,6 +159,7 @@ data "template_file" "controller-configs" { domain_name = element(var.controller_domains, count.index) etcd_name = element(var.controller_names, count.index) etcd_initial_cluster = join(",", formatlist("%s=https://%s:2380", var.controller_names, var.controller_domains)) + cgroup_driver = var.os_channel == "flatcar-edge" ? "systemd" : "cgroupfs" cluster_dns_service_ip = module.bootkube.cluster_dns_service_ip cluster_domain_suffix = var.cluster_domain_suffix ssh_authorized_key = var.ssh_authorized_key @@ -186,6 +187,7 @@ data "template_file" "worker-configs" { vars = { domain_name = element(var.worker_domains, count.index) + cgroup_driver = var.os_channel == "flatcar-edge" ? "systemd" : "cgroupfs" cluster_dns_service_ip = module.bootkube.cluster_dns_service_ip cluster_domain_suffix = var.cluster_domain_suffix ssh_authorized_key = var.ssh_authorized_key diff --git a/docs/architecture/operating-systems.md b/docs/architecture/operating-systems.md index 4c57898ac..804f70c1b 100644 --- a/docs/architecture/operating-systems.md +++ b/docs/architecture/operating-systems.md @@ -14,12 +14,12 @@ Together, they diversify Typhoon to support a range of container technologies. ## Host Properties -| Property | Container Linux | Fedora Atomic | -|-------------------|-----------------|---------------| +| Property | Container Linux / Flatcar Linux | Fedora Atomic | +|-------------------|-----------------|---------------|---------------| | host spec (bare-metal) | Container Linux Config | kickstart, cloud-init | | host spec (cloud) | Container Linux Config | cloud-init | | container runtime | docker | docker (CRIO planned) | -| cgroup driver | cgroupfs | systemd | +| cgroup driver | cgroupfs (except Flatcar edge) | systemd | | logging driver | json-file | journald | | storage driver | overlay2 | overlay2 | From 408e60075ae1aff68d19be668451998c255a2754 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Wed, 19 Jun 2019 21:29:43 -0700 Subject: [PATCH 141/523] Update Kubernetes from v1.14.3 to v1.15.0 * https://github.com/kubernetes/kubernetes/blob/master/CHANGELOG-1.15.md#v1150 * Remove docs referring to possible v1.14.4 release --- CHANGES.md | 3 +++ README.md | 16 ++++++++-------- aws/container-linux/kubernetes/README.md | 2 +- aws/container-linux/kubernetes/bootkube.tf | 2 +- .../kubernetes/cl/controller.yaml.tmpl | 2 +- .../kubernetes/workers/cl/worker.yaml.tmpl | 4 ++-- aws/fedora-atomic/kubernetes/README.md | 2 +- aws/fedora-atomic/kubernetes/bootkube.tf | 2 +- azure/container-linux/kubernetes/README.md | 2 +- azure/container-linux/kubernetes/bootkube.tf | 2 +- .../kubernetes/cl/controller.yaml.tmpl | 2 +- .../kubernetes/workers/cl/worker.yaml.tmpl | 4 ++-- bare-metal/container-linux/kubernetes/README.md | 2 +- .../container-linux/kubernetes/bootkube.tf | 2 +- .../kubernetes/cl/controller.yaml.tmpl | 2 +- .../kubernetes/cl/worker.yaml.tmpl | 2 +- bare-metal/fedora-atomic/kubernetes/README.md | 2 +- bare-metal/fedora-atomic/kubernetes/bootkube.tf | 2 +- .../container-linux/kubernetes/README.md | 2 +- .../container-linux/kubernetes/bootkube.tf | 2 +- .../kubernetes/cl/controller.yaml.tmpl | 2 +- .../kubernetes/cl/worker.yaml.tmpl | 4 ++-- digital-ocean/fedora-atomic/kubernetes/README.md | 2 +- .../fedora-atomic/kubernetes/bootkube.tf | 2 +- docs/advanced/worker-pools.md | 14 +++++++------- docs/cl/aws.md | 10 +++++----- docs/cl/azure.md | 10 +++++----- docs/cl/bare-metal.md | 14 +++++++------- docs/cl/digital-ocean.md | 11 +++++------ docs/cl/google-cloud.md | 10 +++++----- docs/index.md | 12 ++++++------ docs/topics/maintenance.md | 16 ++++++++-------- .../container-linux/kubernetes/README.md | 2 +- .../container-linux/kubernetes/bootkube.tf | 2 +- .../kubernetes/cl/controller.yaml.tmpl | 2 +- .../kubernetes/workers/cl/worker.yaml.tmpl | 4 ++-- google-cloud/fedora-atomic/kubernetes/README.md | 2 +- .../fedora-atomic/kubernetes/bootkube.tf | 2 +- 38 files changed, 91 insertions(+), 89 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 6d12b3429..6126a61af 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -4,6 +4,9 @@ Notable changes between versions. ## Latest +## v1.15.0 + +* Kubernetes [v1.15.0](https://github.com/kubernetes/kubernetes/blob/master/CHANGELOG-1.15.md#v1150) * Migrate from Terraform v0.11 to v0.12.x (**action required!**) * [Migration](https://typhoon.psdn.io/topics/maintenance/#terraform-v012x) instructions for Terraform v0.12 * Require `terraform-provider-ct` v0.3.2+ to support Terraform v0.12 (action required) diff --git a/README.md b/README.md index 37e91628a..8cdf3aa68 100644 --- a/README.md +++ b/README.md @@ -11,7 +11,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features -* Kubernetes v1.14.3 (upstream, via [kubernetes-incubator/bootkube](https://github.com/kubernetes-incubator/bootkube)) +* Kubernetes v1.15.0 (upstream, via [kubernetes-incubator/bootkube](https://github.com/kubernetes-incubator/bootkube)) * Single or multi-master, [Calico](https://www.projectcalico.org/) or [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/) * Advanced features like [worker pools](https://typhoon.psdn.io/advanced/worker-pools/), [preemptible](https://typhoon.psdn.io/cl/google-cloud/#preemption) workers, and [snippets](https://typhoon.psdn.io/advanced/customization/#container-linux) customization @@ -19,13 +19,13 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Modules -Typhoon provides a Terraform Module for each supported operating system and platform. Container Linux is a mature and reliable choice. Also, Kinvolk's Flatcar Linux fork is selectable on AWS and bare-metal. +Typhoon provides a Terraform Module for each supported operating system and platform. | Platform | Operating System | Terraform Module | Status | |---------------|------------------|------------------|--------| -| AWS | Container Linux | [aws/container-linux/kubernetes](aws/container-linux/kubernetes) | stable | +| AWS | Container Linux / Flatcar Linux | [aws/container-linux/kubernetes](aws/container-linux/kubernetes) | stable | | Azure | Container Linux | [azure/container-linux/kubernetes](cl/azure.md) | alpha | -| Bare-Metal | Container Linux | [bare-metal/container-linux/kubernetes](bare-metal/container-linux/kubernetes) | stable | +| Bare-Metal | Container Linux / Flatcar Linux | [bare-metal/container-linux/kubernetes](bare-metal/container-linux/kubernetes) | stable | | Digital Ocean | Container Linux | [digital-ocean/container-linux/kubernetes](digital-ocean/container-linux/kubernetes) | beta | | Google Cloud | Container Linux | [google-cloud/container-linux/kubernetes](google-cloud/container-linux/kubernetes) | stable | @@ -41,7 +41,7 @@ Define a Kubernetes cluster by using the Terraform module for your chosen platfo ```tf module "google-cloud-yavin" { - source = "git::https://github.com/poseidon/typhoon//google-cloud/container-linux/kubernetes?ref=v1.14.3" + source = "git::https://github.com/poseidon/typhoon//google-cloud/container-linux/kubernetes?ref=v1.15.0" providers = { google = "google.default" @@ -82,9 +82,9 @@ In 4-8 minutes (varies by platform), the cluster will be ready. This Google Clou $ export KUBECONFIG=/home/user/.secrets/clusters/yavin/auth/kubeconfig $ kubectl get nodes NAME ROLES STATUS AGE VERSION -yavin-controller-0.c.example-com.internal controller,master Ready 6m v1.14.3 -yavin-worker-jrbf.c.example-com.internal node Ready 5m v1.14.3 -yavin-worker-mzdm.c.example-com.internal node Ready 5m v1.14.3 +yavin-controller-0.c.example-com.internal controller,master Ready 6m v1.15.0 +yavin-worker-jrbf.c.example-com.internal node Ready 5m v1.15.0 +yavin-worker-mzdm.c.example-com.internal node Ready 5m v1.15.0 ``` List the pods. diff --git a/aws/container-linux/kubernetes/README.md b/aws/container-linux/kubernetes/README.md index e89e6c18e..ce8ee9a37 100644 --- a/aws/container-linux/kubernetes/README.md +++ b/aws/container-linux/kubernetes/README.md @@ -11,7 +11,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features -* Kubernetes v1.14.3 (upstream, via [kubernetes-incubator/bootkube](https://github.com/kubernetes-incubator/bootkube)) +* Kubernetes v1.15.0 (upstream, via [kubernetes-incubator/bootkube](https://github.com/kubernetes-incubator/bootkube)) * Single or multi-master, [Calico](https://www.projectcalico.org/) or [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/) * Advanced features like [worker pools](https://typhoon.psdn.io/advanced/worker-pools/), [spot](https://typhoon.psdn.io/cl/aws/#spot) workers, and [snippets](https://typhoon.psdn.io/advanced/customization/#container-linux) customization diff --git a/aws/container-linux/kubernetes/bootkube.tf b/aws/container-linux/kubernetes/bootkube.tf index 4631b06eb..68c4458a0 100644 --- a/aws/container-linux/kubernetes/bootkube.tf +++ b/aws/container-linux/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=89c3ab4e2742c4e15865f0599c85cdac1ec88a6c" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=62df9ad69cc0da35f47d40fa981370c4503ad581" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/aws/container-linux/kubernetes/cl/controller.yaml.tmpl b/aws/container-linux/kubernetes/cl/controller.yaml.tmpl index 7b7b26306..5005bf15e 100644 --- a/aws/container-linux/kubernetes/cl/controller.yaml.tmpl +++ b/aws/container-linux/kubernetes/cl/controller.yaml.tmpl @@ -125,7 +125,7 @@ storage: contents: inline: | KUBELET_IMAGE_URL=docker://k8s.gcr.io/hyperkube - KUBELET_IMAGE_TAG=v1.14.3 + KUBELET_IMAGE_TAG=v1.15.0 - path: /etc/sysctl.d/max-user-watches.conf filesystem: root contents: diff --git a/aws/container-linux/kubernetes/workers/cl/worker.yaml.tmpl b/aws/container-linux/kubernetes/workers/cl/worker.yaml.tmpl index f618476f2..3f4495200 100644 --- a/aws/container-linux/kubernetes/workers/cl/worker.yaml.tmpl +++ b/aws/container-linux/kubernetes/workers/cl/worker.yaml.tmpl @@ -95,7 +95,7 @@ storage: contents: inline: | KUBELET_IMAGE_URL=docker://k8s.gcr.io/hyperkube - KUBELET_IMAGE_TAG=v1.14.3 + KUBELET_IMAGE_TAG=v1.15.0 - path: /etc/sysctl.d/max-user-watches.conf filesystem: root contents: @@ -113,7 +113,7 @@ storage: --volume config,kind=host,source=/etc/kubernetes \ --mount volume=config,target=/etc/kubernetes \ --insecure-options=image \ - docker://k8s.gcr.io/hyperkube:v1.14.3 \ + docker://k8s.gcr.io/hyperkube:v1.15.0 \ --net=host \ --dns=host \ --exec=/kubectl -- --kubeconfig=/etc/kubernetes/kubeconfig delete node $(hostname) diff --git a/aws/fedora-atomic/kubernetes/README.md b/aws/fedora-atomic/kubernetes/README.md index 8b24e6527..82abe4318 100644 --- a/aws/fedora-atomic/kubernetes/README.md +++ b/aws/fedora-atomic/kubernetes/README.md @@ -11,7 +11,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features -* Kubernetes v1.14.3 (upstream, via [kubernetes-incubator/bootkube](https://github.com/kubernetes-incubator/bootkube)) +* Kubernetes v1.15.0 (upstream, via [kubernetes-incubator/bootkube](https://github.com/kubernetes-incubator/bootkube)) * Single or multi-master, [Calico](https://www.projectcalico.org/) or [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/) * Advanced features like [worker pools](https://typhoon.psdn.io/advanced/worker-pools/) and [spot](https://typhoon.psdn.io/cl/aws/#spot) workers diff --git a/aws/fedora-atomic/kubernetes/bootkube.tf b/aws/fedora-atomic/kubernetes/bootkube.tf index 69192d653..630894d32 100644 --- a/aws/fedora-atomic/kubernetes/bootkube.tf +++ b/aws/fedora-atomic/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=89c3ab4e2742c4e15865f0599c85cdac1ec88a6c" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=62df9ad69cc0da35f47d40fa981370c4503ad581" cluster_name = "${var.cluster_name}" api_servers = ["${format("%s.%s", var.cluster_name, var.dns_zone)}"] diff --git a/azure/container-linux/kubernetes/README.md b/azure/container-linux/kubernetes/README.md index 3ca37356a..2cc47ccd9 100644 --- a/azure/container-linux/kubernetes/README.md +++ b/azure/container-linux/kubernetes/README.md @@ -11,7 +11,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features -* Kubernetes v1.14.3 (upstream, via [kubernetes-incubator/bootkube](https://github.com/kubernetes-incubator/bootkube)) +* Kubernetes v1.15.0 (upstream, via [kubernetes-incubator/bootkube](https://github.com/kubernetes-incubator/bootkube)) * Single or multi-master, [Calico](https://www.projectcalico.org/) or [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/) * Advanced features like [worker pools](https://typhoon.psdn.io/advanced/worker-pools/), [low-priority](https://typhoon.psdn.io/cl/azure/#low-priority) workers, and [snippets](https://typhoon.psdn.io/advanced/customization/#container-linux) customization diff --git a/azure/container-linux/kubernetes/bootkube.tf b/azure/container-linux/kubernetes/bootkube.tf index c02f045be..75b7ef131 100644 --- a/azure/container-linux/kubernetes/bootkube.tf +++ b/azure/container-linux/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=89c3ab4e2742c4e15865f0599c85cdac1ec88a6c" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=62df9ad69cc0da35f47d40fa981370c4503ad581" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/azure/container-linux/kubernetes/cl/controller.yaml.tmpl b/azure/container-linux/kubernetes/cl/controller.yaml.tmpl index e83b16753..34755c18b 100644 --- a/azure/container-linux/kubernetes/cl/controller.yaml.tmpl +++ b/azure/container-linux/kubernetes/cl/controller.yaml.tmpl @@ -123,7 +123,7 @@ storage: contents: inline: | KUBELET_IMAGE_URL=docker://k8s.gcr.io/hyperkube - KUBELET_IMAGE_TAG=v1.14.3 + KUBELET_IMAGE_TAG=v1.15.0 - path: /etc/sysctl.d/max-user-watches.conf filesystem: root contents: diff --git a/azure/container-linux/kubernetes/workers/cl/worker.yaml.tmpl b/azure/container-linux/kubernetes/workers/cl/worker.yaml.tmpl index d2a417141..ec3000399 100644 --- a/azure/container-linux/kubernetes/workers/cl/worker.yaml.tmpl +++ b/azure/container-linux/kubernetes/workers/cl/worker.yaml.tmpl @@ -93,7 +93,7 @@ storage: contents: inline: | KUBELET_IMAGE_URL=docker://k8s.gcr.io/hyperkube - KUBELET_IMAGE_TAG=v1.14.3 + KUBELET_IMAGE_TAG=v1.15.0 - path: /etc/sysctl.d/max-user-watches.conf filesystem: root contents: @@ -111,7 +111,7 @@ storage: --volume config,kind=host,source=/etc/kubernetes \ --mount volume=config,target=/etc/kubernetes \ --insecure-options=image \ - docker://k8s.gcr.io/hyperkube:v1.14.3 \ + docker://k8s.gcr.io/hyperkube:v1.15.0 \ --net=host \ --dns=host \ --exec=/kubectl -- --kubeconfig=/etc/kubernetes/kubeconfig delete node $(hostname | tr '[:upper:]' '[:lower:]') diff --git a/bare-metal/container-linux/kubernetes/README.md b/bare-metal/container-linux/kubernetes/README.md index 231a21a58..4342252a7 100644 --- a/bare-metal/container-linux/kubernetes/README.md +++ b/bare-metal/container-linux/kubernetes/README.md @@ -11,7 +11,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features -* Kubernetes v1.14.3 (upstream, via [kubernetes-incubator/bootkube](https://github.com/kubernetes-incubator/bootkube)) +* Kubernetes v1.15.0 (upstream, via [kubernetes-incubator/bootkube](https://github.com/kubernetes-incubator/bootkube)) * Single or multi-master, [Calico](https://www.projectcalico.org/) or [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/) * Advanced features like [snippets](https://typhoon.psdn.io/advanced/customization/#container-linux) customization diff --git a/bare-metal/container-linux/kubernetes/bootkube.tf b/bare-metal/container-linux/kubernetes/bootkube.tf index 7ef2a2b62..1cc9c16f9 100644 --- a/bare-metal/container-linux/kubernetes/bootkube.tf +++ b/bare-metal/container-linux/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=89c3ab4e2742c4e15865f0599c85cdac1ec88a6c" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=62df9ad69cc0da35f47d40fa981370c4503ad581" cluster_name = var.cluster_name api_servers = [var.k8s_domain_name] diff --git a/bare-metal/container-linux/kubernetes/cl/controller.yaml.tmpl b/bare-metal/container-linux/kubernetes/cl/controller.yaml.tmpl index b6b2939ef..15a1316d2 100644 --- a/bare-metal/container-linux/kubernetes/cl/controller.yaml.tmpl +++ b/bare-metal/container-linux/kubernetes/cl/controller.yaml.tmpl @@ -130,7 +130,7 @@ storage: contents: inline: | KUBELET_IMAGE_URL=docker://k8s.gcr.io/hyperkube - KUBELET_IMAGE_TAG=v1.14.3 + KUBELET_IMAGE_TAG=v1.15.0 - path: /etc/hostname filesystem: root mode: 0644 diff --git a/bare-metal/container-linux/kubernetes/cl/worker.yaml.tmpl b/bare-metal/container-linux/kubernetes/cl/worker.yaml.tmpl index 750f328e6..24270cfdc 100644 --- a/bare-metal/container-linux/kubernetes/cl/worker.yaml.tmpl +++ b/bare-metal/container-linux/kubernetes/cl/worker.yaml.tmpl @@ -91,7 +91,7 @@ storage: contents: inline: | KUBELET_IMAGE_URL=docker://k8s.gcr.io/hyperkube - KUBELET_IMAGE_TAG=v1.14.3 + KUBELET_IMAGE_TAG=v1.15.0 - path: /etc/hostname filesystem: root mode: 0644 diff --git a/bare-metal/fedora-atomic/kubernetes/README.md b/bare-metal/fedora-atomic/kubernetes/README.md index ae164f079..16d10a7ca 100644 --- a/bare-metal/fedora-atomic/kubernetes/README.md +++ b/bare-metal/fedora-atomic/kubernetes/README.md @@ -11,7 +11,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features -* Kubernetes v1.14.3 (upstream, via [kubernetes-incubator/bootkube](https://github.com/kubernetes-incubator/bootkube)) +* Kubernetes v1.15.0 (upstream, via [kubernetes-incubator/bootkube](https://github.com/kubernetes-incubator/bootkube)) * Single or multi-master, [Calico](https://www.projectcalico.org/) or [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/) * Ready for Ingress, Prometheus, Grafana, and other optional [addons](https://typhoon.psdn.io/addons/overview/) diff --git a/bare-metal/fedora-atomic/kubernetes/bootkube.tf b/bare-metal/fedora-atomic/kubernetes/bootkube.tf index bf8974e90..63def25d5 100644 --- a/bare-metal/fedora-atomic/kubernetes/bootkube.tf +++ b/bare-metal/fedora-atomic/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=89c3ab4e2742c4e15865f0599c85cdac1ec88a6c" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=62df9ad69cc0da35f47d40fa981370c4503ad581" cluster_name = "${var.cluster_name}" api_servers = ["${var.k8s_domain_name}"] diff --git a/digital-ocean/container-linux/kubernetes/README.md b/digital-ocean/container-linux/kubernetes/README.md index 5e07e4d93..01ad9f01b 100644 --- a/digital-ocean/container-linux/kubernetes/README.md +++ b/digital-ocean/container-linux/kubernetes/README.md @@ -11,7 +11,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features -* Kubernetes v1.14.3 (upstream, via [kubernetes-incubator/bootkube](https://github.com/kubernetes-incubator/bootkube)) +* Kubernetes v1.15.0 (upstream, via [kubernetes-incubator/bootkube](https://github.com/kubernetes-incubator/bootkube)) * Single or multi-master, [Calico](https://www.projectcalico.org/) or [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/) * Advanced features like [snippets](https://typhoon.psdn.io/advanced/customization/#container-linux) customization diff --git a/digital-ocean/container-linux/kubernetes/bootkube.tf b/digital-ocean/container-linux/kubernetes/bootkube.tf index 01a178158..7ed6253c0 100644 --- a/digital-ocean/container-linux/kubernetes/bootkube.tf +++ b/digital-ocean/container-linux/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=89c3ab4e2742c4e15865f0599c85cdac1ec88a6c" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=62df9ad69cc0da35f47d40fa981370c4503ad581" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/digital-ocean/container-linux/kubernetes/cl/controller.yaml.tmpl b/digital-ocean/container-linux/kubernetes/cl/controller.yaml.tmpl index 6c2ff1aaa..5891e4029 100644 --- a/digital-ocean/container-linux/kubernetes/cl/controller.yaml.tmpl +++ b/digital-ocean/container-linux/kubernetes/cl/controller.yaml.tmpl @@ -129,7 +129,7 @@ storage: contents: inline: | KUBELET_IMAGE_URL=docker://k8s.gcr.io/hyperkube - KUBELET_IMAGE_TAG=v1.14.3 + KUBELET_IMAGE_TAG=v1.15.0 - path: /etc/sysctl.d/max-user-watches.conf filesystem: root contents: diff --git a/digital-ocean/container-linux/kubernetes/cl/worker.yaml.tmpl b/digital-ocean/container-linux/kubernetes/cl/worker.yaml.tmpl index 6200eaae3..cca36471d 100644 --- a/digital-ocean/container-linux/kubernetes/cl/worker.yaml.tmpl +++ b/digital-ocean/container-linux/kubernetes/cl/worker.yaml.tmpl @@ -99,7 +99,7 @@ storage: contents: inline: | KUBELET_IMAGE_URL=docker://k8s.gcr.io/hyperkube - KUBELET_IMAGE_TAG=v1.14.3 + KUBELET_IMAGE_TAG=v1.15.0 - path: /etc/sysctl.d/max-user-watches.conf filesystem: root contents: @@ -117,7 +117,7 @@ storage: --volume config,kind=host,source=/etc/kubernetes \ --mount volume=config,target=/etc/kubernetes \ --insecure-options=image \ - docker://k8s.gcr.io/hyperkube:v1.14.3 \ + docker://k8s.gcr.io/hyperkube:v1.15.0 \ --net=host \ --dns=host \ --exec=/kubectl -- --kubeconfig=/etc/kubernetes/kubeconfig delete node $(hostname) diff --git a/digital-ocean/fedora-atomic/kubernetes/README.md b/digital-ocean/fedora-atomic/kubernetes/README.md index 2520d28d1..74b38a7cf 100644 --- a/digital-ocean/fedora-atomic/kubernetes/README.md +++ b/digital-ocean/fedora-atomic/kubernetes/README.md @@ -11,7 +11,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features -* Kubernetes v1.14.3 (upstream, via [kubernetes-incubator/bootkube](https://github.com/kubernetes-incubator/bootkube)) +* Kubernetes v1.15.0 (upstream, via [kubernetes-incubator/bootkube](https://github.com/kubernetes-incubator/bootkube)) * Single or multi-master, [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled * Ready for Ingress, Prometheus, Grafana, and other optional [addons](https://typhoon.psdn.io/addons/overview/) diff --git a/digital-ocean/fedora-atomic/kubernetes/bootkube.tf b/digital-ocean/fedora-atomic/kubernetes/bootkube.tf index dba07edda..b8297cdd1 100644 --- a/digital-ocean/fedora-atomic/kubernetes/bootkube.tf +++ b/digital-ocean/fedora-atomic/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=89c3ab4e2742c4e15865f0599c85cdac1ec88a6c" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=62df9ad69cc0da35f47d40fa981370c4503ad581" cluster_name = "${var.cluster_name}" api_servers = ["${format("%s.%s", var.cluster_name, var.dns_zone)}"] diff --git a/docs/advanced/worker-pools.md b/docs/advanced/worker-pools.md index 7a8170b16..0e5d758a9 100644 --- a/docs/advanced/worker-pools.md +++ b/docs/advanced/worker-pools.md @@ -76,7 +76,7 @@ Create a cluster following the Azure [tutorial](../cl/azure.md#cluster). Define ```tf module "ramius-worker-pool" { - source = "git::https://github.com/poseidon/typhoon//azure/container-linux/kubernetes/workers?ref=v1.14.3" + source = "git::https://github.com/poseidon/typhoon//azure/container-linux/kubernetes/workers?ref=v1.15.0" # Azure region = module.azure-ramius.region @@ -142,7 +142,7 @@ Create a cluster following the Google Cloud [tutorial](../cl/google-cloud.md#clu ```tf module "yavin-worker-pool" { - source = "git::https://github.com/poseidon/typhoon//google-cloud/container-linux/kubernetes/workers?ref=v1.14.3" + source = "git::https://github.com/poseidon/typhoon//google-cloud/container-linux/kubernetes/workers?ref=v1.15.0" # Google Cloud region = "europe-west2" @@ -173,11 +173,11 @@ Verify a managed instance group of workers joins the cluster within a few minute ``` $ kubectl get nodes NAME STATUS AGE VERSION -yavin-controller-0.c.example-com.internal Ready 6m v1.14.3 -yavin-worker-jrbf.c.example-com.internal Ready 5m v1.14.3 -yavin-worker-mzdm.c.example-com.internal Ready 5m v1.14.3 -yavin-16x-worker-jrbf.c.example-com.internal Ready 3m v1.14.3 -yavin-16x-worker-mzdm.c.example-com.internal Ready 3m v1.14.3 +yavin-controller-0.c.example-com.internal Ready 6m v1.15.0 +yavin-worker-jrbf.c.example-com.internal Ready 5m v1.15.0 +yavin-worker-mzdm.c.example-com.internal Ready 5m v1.15.0 +yavin-16x-worker-jrbf.c.example-com.internal Ready 3m v1.15.0 +yavin-16x-worker-mzdm.c.example-com.internal Ready 3m v1.15.0 ``` ### Variables diff --git a/docs/cl/aws.md b/docs/cl/aws.md index 2c35c1477..3d09ff6d9 100644 --- a/docs/cl/aws.md +++ b/docs/cl/aws.md @@ -1,6 +1,6 @@ # AWS -In this tutorial, we'll create a Kubernetes v1.14.3 cluster on AWS with Container Linux. +In this tutorial, we'll create a Kubernetes v1.15.0 cluster on AWS with Container Linux. We'll declare a Kubernetes cluster using the Typhoon Terraform module. Then apply the changes to create a VPC, gateway, subnets, security groups, controller instances, worker auto-scaling group, network load balancer, and TLS assets. @@ -70,7 +70,7 @@ Define a Kubernetes cluster using the module `aws/container-linux/kubernetes`. ```tf module "aws-tempest" { - source = "git::https://github.com/poseidon/typhoon//aws/container-linux/kubernetes?ref=v1.14.3" + source = "git::https://github.com/poseidon/typhoon//aws/container-linux/kubernetes?ref=v1.15.0" # AWS cluster_name = "tempest" @@ -135,9 +135,9 @@ In 4-8 minutes, the Kubernetes cluster will be ready. $ export KUBECONFIG=/home/user/.secrets/clusters/tempest/auth/kubeconfig $ kubectl get nodes NAME STATUS ROLES AGE VERSION -ip-10-0-3-155 Ready controller,master 10m v1.14.3 -ip-10-0-26-65 Ready node 10m v1.14.3 -ip-10-0-41-21 Ready node 10m v1.14.3 +ip-10-0-3-155 Ready controller,master 10m v1.15.0 +ip-10-0-26-65 Ready node 10m v1.15.0 +ip-10-0-41-21 Ready node 10m v1.15.0 ``` List the pods. diff --git a/docs/cl/azure.md b/docs/cl/azure.md index fba536965..ec9931afe 100644 --- a/docs/cl/azure.md +++ b/docs/cl/azure.md @@ -3,7 +3,7 @@ !!! danger Typhoon for Azure is alpha. For production, use AWS, Google Cloud, or bare-metal. As Azure matures, check [errata](https://github.com/poseidon/typhoon/wiki/Errata) for known shortcomings. -In this tutorial, we'll create a Kubernetes v1.14.3 cluster on Azure with Container Linux. +In this tutorial, we'll create a Kubernetes v1.15.0 cluster on Azure with Container Linux. We'll declare a Kubernetes cluster using the Typhoon Terraform module. Then apply the changes to create a resource group, virtual network, subnets, security groups, controller availability set, worker scale set, load balancer, and TLS assets. @@ -66,7 +66,7 @@ Define a Kubernetes cluster using the module `azure/container-linux/kubernetes`. ```tf module "azure-ramius" { - source = "git::https://github.com/poseidon/typhoon//azure/container-linux/kubernetes?ref=v1.14.4" + source = "git::https://github.com/poseidon/typhoon//azure/container-linux/kubernetes?ref=v1.15.0" # Azure cluster_name = "ramius" @@ -132,9 +132,9 @@ In 4-8 minutes, the Kubernetes cluster will be ready. $ export KUBECONFIG=/home/user/.secrets/clusters/ramius/auth/kubeconfig $ kubectl get nodes NAME STATUS ROLES AGE VERSION -ramius-controller-0 Ready controller,master 24m v1.14.3 -ramius-worker-000001 Ready node 25m v1.14.3 -ramius-worker-000002 Ready node 24m v1.14.3 +ramius-controller-0 Ready controller,master 24m v1.15.0 +ramius-worker-000001 Ready node 25m v1.15.0 +ramius-worker-000002 Ready node 24m v1.15.0 ``` List the pods. diff --git a/docs/cl/bare-metal.md b/docs/cl/bare-metal.md index 5ebe1f5f3..2941e15fe 100644 --- a/docs/cl/bare-metal.md +++ b/docs/cl/bare-metal.md @@ -1,6 +1,6 @@ # Bare-Metal -In this tutorial, we'll network boot and provision a Kubernetes v1.14.3 cluster on bare-metal with Container Linux. +In this tutorial, we'll network boot and provision a Kubernetes v1.15.0 cluster on bare-metal with Container Linux. First, we'll deploy a [Matchbox](https://github.com/poseidon/matchbox) service and setup a network boot environment. Then, we'll declare a Kubernetes cluster using the Typhoon Terraform module and power on machines. On PXE boot, machines will install Container Linux to disk, reboot into the disk install, and provision themselves as Kubernetes controllers or workers via Ignition. @@ -160,7 +160,7 @@ Define a Kubernetes cluster using the module `bare-metal/container-linux/kuberne ```tf module "bare-metal-mercury" { - source = "git::https://github.com/poseidon/typhoon//bare-metal/container-linux/kubernetes?ref=v1.14.3" + source = "git::https://github.com/poseidon/typhoon//bare-metal/container-linux/kubernetes?ref=v1.15.0" # bare-metal cluster_name = "mercury" @@ -265,9 +265,9 @@ Apply complete! Resources: 55 added, 0 changed, 0 destroyed. To watch the install to disk (until machines reboot from disk), SSH to port 2222. ``` -# before v1.14.3 +# before v1.15.0 $ ssh debug@node1.example.com -# after v1.14.3 +# after v1.15.0 $ ssh -p 2222 core@node1.example.com ``` @@ -292,9 +292,9 @@ bootkube[5]: Tearing down temporary bootstrap control plane... $ export KUBECONFIG=/home/user/.secrets/clusters/mercury/auth/kubeconfig $ kubectl get nodes NAME STATUS ROLES AGE VERSION -node1.example.com Ready controller,master 10m v1.14.3 -node2.example.com Ready node 10m v1.14.3 -node3.example.com Ready node 10m v1.14.3 +node1.example.com Ready controller,master 10m v1.15.0 +node2.example.com Ready node 10m v1.15.0 +node3.example.com Ready node 10m v1.15.0 ``` List the pods. diff --git a/docs/cl/digital-ocean.md b/docs/cl/digital-ocean.md index 0eb0a2ab0..fa66e077b 100644 --- a/docs/cl/digital-ocean.md +++ b/docs/cl/digital-ocean.md @@ -1,6 +1,6 @@ # Digital Ocean -In this tutorial, we'll create a Kubernetes v1.14.3 cluster on DigitalOcean with Container Linux. +In this tutorial, we'll create a Kubernetes v1.15.0 cluster on DigitalOcean with Container Linux. We'll declare a Kubernetes cluster using the Typhoon Terraform module. Then apply the changes to create controller droplets, worker droplets, DNS records, tags, and TLS assets. @@ -65,7 +65,7 @@ Define a Kubernetes cluster using the module `digital-ocean/container-linux/kube ```tf module "digital-ocean-nemo" { - source = "git::https://github.com/poseidon/typhoon//digital-ocean/container-linux/kubernetes?ref=v1.14.4" + source = "git::https://github.com/poseidon/typhoon//digital-ocean/container-linux/kubernetes?ref=v1.15.0" # Digital Ocean cluster_name = "nemo" @@ -78,7 +78,6 @@ module "digital-ocean-nemo" { # optional worker_count = 2 - worker_type = "s-1vcpu-1gb" } ``` @@ -131,9 +130,9 @@ In 3-6 minutes, the Kubernetes cluster will be ready. $ export KUBECONFIG=/home/user/.secrets/clusters/nemo/auth/kubeconfig $ kubectl get nodes NAME STATUS ROLES AGE VERSION -10.132.110.130 Ready controller,master 10m v1.14.3 -10.132.115.81 Ready node 10m v1.14.3 -10.132.124.107 Ready node 10m v1.14.3 +10.132.110.130 Ready controller,master 10m v1.15.0 +10.132.115.81 Ready node 10m v1.15.0 +10.132.124.107 Ready node 10m v1.15.0 ``` List the pods. diff --git a/docs/cl/google-cloud.md b/docs/cl/google-cloud.md index adead5558..cb562a673 100644 --- a/docs/cl/google-cloud.md +++ b/docs/cl/google-cloud.md @@ -1,6 +1,6 @@ # Google Cloud -In this tutorial, we'll create a Kubernetes v1.14.3 cluster on Google Compute Engine with Container Linux. +In this tutorial, we'll create a Kubernetes v1.15.0 cluster on Google Compute Engine with Container Linux. We'll declare a Kubernetes cluster using the Typhoon Terraform module. Then apply the changes to create a network, firewall rules, health checks, controller instances, worker managed instance group, load balancers, and TLS assets. @@ -71,7 +71,7 @@ Define a Kubernetes cluster using the module `google-cloud/container-linux/kuber ```tf module "google-cloud-yavin" { - source = "git::https://github.com/poseidon/typhoon//google-cloud/container-linux/kubernetes?ref=v1.14.4" + source = "git::https://github.com/poseidon/typhoon//google-cloud/container-linux/kubernetes?ref=v1.15.0" # Google Cloud cluster_name = "yavin" @@ -138,9 +138,9 @@ In 4-8 minutes, the Kubernetes cluster will be ready. $ export KUBECONFIG=/home/user/.secrets/clusters/yavin/auth/kubeconfig $ kubectl get nodes NAME ROLES STATUS AGE VERSION -yavin-controller-0.c.example-com.internal controller,master Ready 6m v1.14.3 -yavin-worker-jrbf.c.example-com.internal node Ready 5m v1.14.3 -yavin-worker-mzdm.c.example-com.internal node Ready 5m v1.14.3 +yavin-controller-0.c.example-com.internal controller,master Ready 6m v1.15.0 +yavin-worker-jrbf.c.example-com.internal node Ready 5m v1.15.0 +yavin-worker-mzdm.c.example-com.internal node Ready 5m v1.15.0 ``` List the pods. diff --git a/docs/index.md b/docs/index.md index ecbaf87c7..a9153d6b6 100644 --- a/docs/index.md +++ b/docs/index.md @@ -11,7 +11,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features -* Kubernetes v1.14.3 (upstream, via [kubernetes-incubator/bootkube](https://github.com/kubernetes-incubator/bootkube)) +* Kubernetes v1.15.0 (upstream, via [kubernetes-incubator/bootkube](https://github.com/kubernetes-incubator/bootkube)) * Single or multi-master, [Calico](https://www.projectcalico.org/) or [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/) * Advanced features like [worker pools](advanced/worker-pools/), [preemptible](cl/google-cloud/#preemption) workers, and [snippets](advanced/customization/#container-linux) customization @@ -19,7 +19,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Modules -Typhoon provides a Terraform Module for each supported operating system and platform. Container Linux is a mature and reliable choice. Also, Kinvolk's Flatcar Linux fork is selectable on AWS and bare-metal. +Typhoon provides a Terraform Module for each supported operating system and platform. | Platform | Operating System | Terraform Module | Status | |---------------|------------------|------------------|--------| @@ -40,7 +40,7 @@ Define a Kubernetes cluster by using the Terraform module for your chosen platfo ```tf module "google-cloud-yavin" { - source = "git::https://github.com/poseidon/typhoon//google-cloud/container-linux/kubernetes?ref=v1.14.4" + source = "git::https://github.com/poseidon/typhoon//google-cloud/container-linux/kubernetes?ref=v1.15.0" # Google Cloud cluster_name = "yavin" @@ -73,9 +73,9 @@ In 4-8 minutes (varies by platform), the cluster will be ready. This Google Clou $ export KUBECONFIG=/home/user/.secrets/clusters/yavin/auth/kubeconfig $ kubectl get nodes NAME ROLES STATUS AGE VERSION -yavin-controller-0.c.example-com.internal controller,master Ready 6m v1.14.3 -yavin-worker-jrbf.c.example-com.internal node Ready 5m v1.14.3 -yavin-worker-mzdm.c.example-com.internal node Ready 5m v1.14.3 +yavin-controller-0.c.example-com.internal controller,master Ready 6m v1.15.0 +yavin-worker-jrbf.c.example-com.internal node Ready 5m v1.15.0 +yavin-worker-mzdm.c.example-com.internal node Ready 5m v1.15.0 ``` List the pods. diff --git a/docs/topics/maintenance.md b/docs/topics/maintenance.md index 3e138a77e..0d1237b25 100644 --- a/docs/topics/maintenance.md +++ b/docs/topics/maintenance.md @@ -18,7 +18,7 @@ module "google-cloud-yavin" { } module "bare-metal-mercury" { - source = "git::https://github.com/poseidon/typhoon//bare-metal/container-linux/kubernetes?ref=v1.14.3" + source = "git::https://github.com/poseidon/typhoon//bare-metal/container-linux/kubernetes?ref=v1.15.0" ... } ``` @@ -279,15 +279,15 @@ Typhoon modules have been adapted for Terraform v0.12. Provider plugins requirem | Typhoon Release | Terraform version | |-------------------|---------------------| -| v1.14.4 - ? | v0.12.x | -| v1.10.3 - v1.14.3 | v0.11.x | +| v1.15.0 - ? | v0.12.x | +| v1.10.3 - v1.15.0 | v0.11.x | | v1.9.2 - v1.10.2 | v0.10.4+ or v0.11.x | | v1.7.3 - v1.9.1 | v0.10.x | | v1.6.4 - v1.7.2 | v0.9.x | ### New users -New users can start with Terraform v0.12.x and follow the docs for Typhoon v1.14.4+ without issue. +New users can start with Terraform v0.12.x and follow the docs for Typhoon v1.15.0+ without issue. ### Existing users @@ -304,12 +304,12 @@ sudo ln -sf ~/Downloads/terraform-0.12.0/terraform /usr/local/bin/terraform12 #### In-place -For existing Typhoon v1.14.2 or v1.14.3 clusters, edit the Typhoon `ref` to the `v1.14.4` release (if published) or the first SHA that introduced Terraform v0.12 support (`3276bf587850218b8f967978a4bf2b05d5f440a2`). The aim is to minimize the diff. For example: +For existing Typhoon v1.14.2 or v1.14.3 clusters, edit the Typhoon `ref` to first SHA that introduced Terraform v0.12 support (`3276bf587850218b8f967978a4bf2b05d5f440a2`). The aim is to minimize the diff and convert to using Terraform v0.12.x. For example: ```tf module "bare-metal-mercury" { - source = "git::https://github.com/poseidon/typhoon//bare-metal/container-linux/kubernetes?ref=v1.14.3" -+ source = "git::https://github.com/poseidon/typhoon//bare-metal/container-linux/kubernetes?ref=v1.14.4" ++ source = "git::https://github.com/poseidon/typhoon//bare-metal/container-linux/kubernetes?ref=3276bf587850218b8f967978a4bf2b05d5f440a2" ... ``` @@ -317,7 +317,7 @@ With Terraform v0.12, Typhoon clusters no longer require the `providers` block ( ```tf module "bare-metal-mercury" { - source = "git::https://github.com/poseidon/typhoon//bare-metal/container-linux/kubernetes?ref=v1.14.4" + source = "git::https://github.com/poseidon/typhoon//bare-metal/container-linux/kubernetes?ref=3276bf587850218b8f967978a4bf2b05d5f440a2" - providers = { - local = "local.default" @@ -404,7 +404,7 @@ tree . └── infraB <- new Terraform v0.12.x configs ``` -Define Typhoon clusters in the new config directory using Terraform v0.12 syntax. Follow the Typhoon v1.14.4+ docs (e.g. use `terraform12` in the `infraB` dir). See [AWS](/cl/aws), [Azure](/cl/azure), [Bare-Metal](/cl/bare-metal), [Digital Ocean](/cl/digital-ocean), or [Google-Cloud](/cl/google-cloud)) to create new clusters. Follow the usual [upgrade](/topics/maintenance/#upgrades) process to apply workloads and shift traffic. Later, switch back to the old config directory and deprovision clusters with Terraform v0.11. +Define Typhoon clusters in the new config directory using Terraform v0.12 syntax. Follow the Typhoon v1.15.0+ docs (e.g. use `terraform12` in the `infraB` dir). See [AWS](/cl/aws), [Azure](/cl/azure), [Bare-Metal](/cl/bare-metal), [Digital Ocean](/cl/digital-ocean), or [Google-Cloud](/cl/google-cloud)) to create new clusters. Follow the usual [upgrade](/topics/maintenance/#upgrades) process to apply workloads and shift traffic. Later, switch back to the old config directory and deprovision clusters with Terraform v0.11. ```shell terraform12 init diff --git a/google-cloud/container-linux/kubernetes/README.md b/google-cloud/container-linux/kubernetes/README.md index 4c42ed351..8aa176fa4 100644 --- a/google-cloud/container-linux/kubernetes/README.md +++ b/google-cloud/container-linux/kubernetes/README.md @@ -11,7 +11,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features -* Kubernetes v1.14.3 (upstream, via [kubernetes-incubator/bootkube](https://github.com/kubernetes-incubator/bootkube)) +* Kubernetes v1.15.0 (upstream, via [kubernetes-incubator/bootkube](https://github.com/kubernetes-incubator/bootkube)) * Single or multi-master, [Calico](https://www.projectcalico.org/) or [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/) * Advanced features like [worker pools](https://typhoon.psdn.io/advanced/worker-pools/), [preemptible](https://typhoon.psdn.io/cl/google-cloud/#preemption) workers, and [snippets](https://typhoon.psdn.io/advanced/customization/#container-linux) customization diff --git a/google-cloud/container-linux/kubernetes/bootkube.tf b/google-cloud/container-linux/kubernetes/bootkube.tf index f0f17266d..0bab5fc25 100644 --- a/google-cloud/container-linux/kubernetes/bootkube.tf +++ b/google-cloud/container-linux/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=89c3ab4e2742c4e15865f0599c85cdac1ec88a6c" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=62df9ad69cc0da35f47d40fa981370c4503ad581" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/google-cloud/container-linux/kubernetes/cl/controller.yaml.tmpl b/google-cloud/container-linux/kubernetes/cl/controller.yaml.tmpl index a2cca2433..64ec62929 100644 --- a/google-cloud/container-linux/kubernetes/cl/controller.yaml.tmpl +++ b/google-cloud/container-linux/kubernetes/cl/controller.yaml.tmpl @@ -124,7 +124,7 @@ storage: contents: inline: | KUBELET_IMAGE_URL=docker://k8s.gcr.io/hyperkube - KUBELET_IMAGE_TAG=v1.14.3 + KUBELET_IMAGE_TAG=v1.15.0 - path: /etc/sysctl.d/max-user-watches.conf filesystem: root contents: diff --git a/google-cloud/container-linux/kubernetes/workers/cl/worker.yaml.tmpl b/google-cloud/container-linux/kubernetes/workers/cl/worker.yaml.tmpl index 7273df5d4..a34cca82f 100644 --- a/google-cloud/container-linux/kubernetes/workers/cl/worker.yaml.tmpl +++ b/google-cloud/container-linux/kubernetes/workers/cl/worker.yaml.tmpl @@ -94,7 +94,7 @@ storage: contents: inline: | KUBELET_IMAGE_URL=docker://k8s.gcr.io/hyperkube - KUBELET_IMAGE_TAG=v1.14.3 + KUBELET_IMAGE_TAG=v1.15.0 - path: /etc/sysctl.d/max-user-watches.conf filesystem: root contents: @@ -112,7 +112,7 @@ storage: --volume config,kind=host,source=/etc/kubernetes \ --mount volume=config,target=/etc/kubernetes \ --insecure-options=image \ - docker://k8s.gcr.io/hyperkube:v1.14.3 \ + docker://k8s.gcr.io/hyperkube:v1.15.0 \ --net=host \ --dns=host \ --exec=/kubectl -- --kubeconfig=/etc/kubernetes/kubeconfig delete node $(hostname) diff --git a/google-cloud/fedora-atomic/kubernetes/README.md b/google-cloud/fedora-atomic/kubernetes/README.md index d225483b1..1d95569a9 100644 --- a/google-cloud/fedora-atomic/kubernetes/README.md +++ b/google-cloud/fedora-atomic/kubernetes/README.md @@ -11,7 +11,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features -* Kubernetes v1.14.3 (upstream, via [kubernetes-incubator/bootkube](https://github.com/kubernetes-incubator/bootkube)) +* Kubernetes v1.15.0 (upstream, via [kubernetes-incubator/bootkube](https://github.com/kubernetes-incubator/bootkube)) * Single or multi-master, [Calico](https://www.projectcalico.org/) or [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/) * Advanced features like [worker pools](https://typhoon.psdn.io/advanced/worker-pools/) and [preemptible](https://typhoon.psdn.io/cl/google-cloud/#preemption) workers diff --git a/google-cloud/fedora-atomic/kubernetes/bootkube.tf b/google-cloud/fedora-atomic/kubernetes/bootkube.tf index 3089ce339..cbd1f104a 100644 --- a/google-cloud/fedora-atomic/kubernetes/bootkube.tf +++ b/google-cloud/fedora-atomic/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=89c3ab4e2742c4e15865f0599c85cdac1ec88a6c" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=62df9ad69cc0da35f47d40fa981370c4503ad581" cluster_name = "${var.cluster_name}" api_servers = ["${format("%s.%s", var.cluster_name, var.dns_zone)}"] From ca18fab5f0eff887b4d92c6f62b547fb70278aad Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Sun, 23 Jun 2019 13:34:33 -0700 Subject: [PATCH 142/523] Remove providers block, unused with Terraform v0.12 * Fix inconsistency btw README and the docs --- README.md | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/README.md b/README.md index 8cdf3aa68..ae2f8470a 100644 --- a/README.md +++ b/README.md @@ -43,14 +43,6 @@ Define a Kubernetes cluster by using the Terraform module for your chosen platfo module "google-cloud-yavin" { source = "git::https://github.com/poseidon/typhoon//google-cloud/container-linux/kubernetes?ref=v1.15.0" - providers = { - google = "google.default" - local = "local.default" - null = "null.default" - template = "template.default" - tls = "tls.default" - } - # Google Cloud cluster_name = "yavin" region = "us-central1" @@ -63,6 +55,7 @@ module "google-cloud-yavin" { # optional worker_count = 2 + worker_preemptible = true } ``` From fff7cc035d815c6db76e2104b6ffb145b4668578 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Sun, 23 Jun 2019 13:38:49 -0700 Subject: [PATCH 143/523] Remove Fedora Atomic modules * Typhoon for Fedora Atomic was deprecated in March 2019 * https://typhoon.psdn.io/announce/#march-27-2019 --- .github/ISSUE_TEMPLATE.md | 2 +- CHANGES.md | 1 + aws/fedora-atomic/kubernetes/LICENSE | 23 -- aws/fedora-atomic/kubernetes/README.md | 23 -- aws/fedora-atomic/kubernetes/ami.tf | 19 - aws/fedora-atomic/kubernetes/bootkube.tf | 18 - .../kubernetes/cloudinit/controller.yaml.tmpl | 93 ----- aws/fedora-atomic/kubernetes/controllers.tf | 79 ---- aws/fedora-atomic/kubernetes/network.tf | 57 --- aws/fedora-atomic/kubernetes/nlb.tf | 93 ----- aws/fedora-atomic/kubernetes/outputs.tf | 48 --- aws/fedora-atomic/kubernetes/require.tf | 25 -- aws/fedora-atomic/kubernetes/security.tf | 359 ------------------ aws/fedora-atomic/kubernetes/ssh.tf | 89 ----- aws/fedora-atomic/kubernetes/variables.tf | 124 ------ aws/fedora-atomic/kubernetes/workers.tf | 19 - aws/fedora-atomic/kubernetes/workers/ami.tf | 19 - .../workers/cloudinit/worker.yaml.tmpl | 66 ---- .../kubernetes/workers/ingress.tf | 47 --- .../kubernetes/workers/outputs.tf | 9 - .../kubernetes/workers/variables.tf | 87 ----- .../kubernetes/workers/workers.tf | 78 ---- aws/ignore/.gitkeep | 0 bare-metal/fedora-atomic/kubernetes/LICENSE | 23 -- bare-metal/fedora-atomic/kubernetes/README.md | 22 -- .../fedora-atomic/kubernetes/bootkube.tf | 18 - .../kubernetes/cloudinit/controller.yaml.tmpl | 100 ----- .../kubernetes/cloudinit/worker.yaml.tmpl | 73 ---- bare-metal/fedora-atomic/kubernetes/groups.tf | 37 -- .../kickstart/fedora-atomic.ks.tmpl | 36 -- .../fedora-atomic/kubernetes/outputs.tf | 3 - .../fedora-atomic/kubernetes/profiles.tf | 87 ----- .../fedora-atomic/kubernetes/require.tf | 21 - bare-metal/fedora-atomic/kubernetes/ssh.tf | 136 ------- .../fedora-atomic/kubernetes/variables.tf | 118 ------ bare-metal/ignore/.gitkeep | 0 .../fedora-atomic/kubernetes/LICENSE | 23 -- .../fedora-atomic/kubernetes/README.md | 22 -- .../fedora-atomic/kubernetes/bootkube.tf | 18 - .../kubernetes/cloudinit/controller.yaml.tmpl | 107 ------ .../kubernetes/cloudinit/worker.yaml.tmpl | 80 ---- .../fedora-atomic/kubernetes/controllers.tf | 95 ----- .../fedora-atomic/kubernetes/network.tf | 58 --- .../fedora-atomic/kubernetes/outputs.tf | 28 -- .../fedora-atomic/kubernetes/require.tf | 25 -- digital-ocean/fedora-atomic/kubernetes/ssh.tf | 121 ------ .../fedora-atomic/kubernetes/variables.tf | 93 ----- .../fedora-atomic/kubernetes/workers.tf | 66 ---- digital-ocean/ignore/.gitkeep | 0 google-cloud/fedora-atomic/kubernetes/LICENSE | 23 -- .../fedora-atomic/kubernetes/README.md | 23 -- .../fedora-atomic/kubernetes/apiserver.tf | 97 ----- .../fedora-atomic/kubernetes/bootkube.tf | 21 - .../kubernetes/cloudinit/controller.yaml.tmpl | 93 ----- .../fedora-atomic/kubernetes/controllers.tf | 98 ----- .../fedora-atomic/kubernetes/ingress.tf | 122 ------ .../fedora-atomic/kubernetes/network.tf | 153 -------- .../fedora-atomic/kubernetes/outputs.tf | 38 -- .../fedora-atomic/kubernetes/require.tf | 25 -- google-cloud/fedora-atomic/kubernetes/ssh.tf | 89 ----- .../fedora-atomic/kubernetes/variables.tf | 110 ------ .../fedora-atomic/kubernetes/workers.tf | 20 - .../workers/cloudinit/worker.yaml.tmpl | 66 ---- .../kubernetes/workers/outputs.tf | 4 - .../kubernetes/workers/variables.tf | 94 ----- .../kubernetes/workers/workers.tf | 77 ---- google-cloud/ignore/.gitkeep | 0 67 files changed, 2 insertions(+), 3939 deletions(-) delete mode 100644 aws/fedora-atomic/kubernetes/LICENSE delete mode 100644 aws/fedora-atomic/kubernetes/README.md delete mode 100644 aws/fedora-atomic/kubernetes/ami.tf delete mode 100644 aws/fedora-atomic/kubernetes/bootkube.tf delete mode 100644 aws/fedora-atomic/kubernetes/cloudinit/controller.yaml.tmpl delete mode 100644 aws/fedora-atomic/kubernetes/controllers.tf delete mode 100644 aws/fedora-atomic/kubernetes/network.tf delete mode 100644 aws/fedora-atomic/kubernetes/nlb.tf delete mode 100644 aws/fedora-atomic/kubernetes/outputs.tf delete mode 100644 aws/fedora-atomic/kubernetes/require.tf delete mode 100644 aws/fedora-atomic/kubernetes/security.tf delete mode 100644 aws/fedora-atomic/kubernetes/ssh.tf delete mode 100644 aws/fedora-atomic/kubernetes/variables.tf delete mode 100644 aws/fedora-atomic/kubernetes/workers.tf delete mode 100644 aws/fedora-atomic/kubernetes/workers/ami.tf delete mode 100644 aws/fedora-atomic/kubernetes/workers/cloudinit/worker.yaml.tmpl delete mode 100644 aws/fedora-atomic/kubernetes/workers/ingress.tf delete mode 100644 aws/fedora-atomic/kubernetes/workers/outputs.tf delete mode 100644 aws/fedora-atomic/kubernetes/workers/variables.tf delete mode 100644 aws/fedora-atomic/kubernetes/workers/workers.tf create mode 100644 aws/ignore/.gitkeep delete mode 100644 bare-metal/fedora-atomic/kubernetes/LICENSE delete mode 100644 bare-metal/fedora-atomic/kubernetes/README.md delete mode 100644 bare-metal/fedora-atomic/kubernetes/bootkube.tf delete mode 100644 bare-metal/fedora-atomic/kubernetes/cloudinit/controller.yaml.tmpl delete mode 100644 bare-metal/fedora-atomic/kubernetes/cloudinit/worker.yaml.tmpl delete mode 100644 bare-metal/fedora-atomic/kubernetes/groups.tf delete mode 100644 bare-metal/fedora-atomic/kubernetes/kickstart/fedora-atomic.ks.tmpl delete mode 100644 bare-metal/fedora-atomic/kubernetes/outputs.tf delete mode 100644 bare-metal/fedora-atomic/kubernetes/profiles.tf delete mode 100644 bare-metal/fedora-atomic/kubernetes/require.tf delete mode 100644 bare-metal/fedora-atomic/kubernetes/ssh.tf delete mode 100644 bare-metal/fedora-atomic/kubernetes/variables.tf create mode 100644 bare-metal/ignore/.gitkeep delete mode 100644 digital-ocean/fedora-atomic/kubernetes/LICENSE delete mode 100644 digital-ocean/fedora-atomic/kubernetes/README.md delete mode 100644 digital-ocean/fedora-atomic/kubernetes/bootkube.tf delete mode 100644 digital-ocean/fedora-atomic/kubernetes/cloudinit/controller.yaml.tmpl delete mode 100644 digital-ocean/fedora-atomic/kubernetes/cloudinit/worker.yaml.tmpl delete mode 100644 digital-ocean/fedora-atomic/kubernetes/controllers.tf delete mode 100644 digital-ocean/fedora-atomic/kubernetes/network.tf delete mode 100644 digital-ocean/fedora-atomic/kubernetes/outputs.tf delete mode 100644 digital-ocean/fedora-atomic/kubernetes/require.tf delete mode 100644 digital-ocean/fedora-atomic/kubernetes/ssh.tf delete mode 100644 digital-ocean/fedora-atomic/kubernetes/variables.tf delete mode 100644 digital-ocean/fedora-atomic/kubernetes/workers.tf create mode 100644 digital-ocean/ignore/.gitkeep delete mode 100644 google-cloud/fedora-atomic/kubernetes/LICENSE delete mode 100644 google-cloud/fedora-atomic/kubernetes/README.md delete mode 100644 google-cloud/fedora-atomic/kubernetes/apiserver.tf delete mode 100644 google-cloud/fedora-atomic/kubernetes/bootkube.tf delete mode 100644 google-cloud/fedora-atomic/kubernetes/cloudinit/controller.yaml.tmpl delete mode 100644 google-cloud/fedora-atomic/kubernetes/controllers.tf delete mode 100644 google-cloud/fedora-atomic/kubernetes/ingress.tf delete mode 100644 google-cloud/fedora-atomic/kubernetes/network.tf delete mode 100644 google-cloud/fedora-atomic/kubernetes/outputs.tf delete mode 100644 google-cloud/fedora-atomic/kubernetes/require.tf delete mode 100644 google-cloud/fedora-atomic/kubernetes/ssh.tf delete mode 100644 google-cloud/fedora-atomic/kubernetes/variables.tf delete mode 100644 google-cloud/fedora-atomic/kubernetes/workers.tf delete mode 100644 google-cloud/fedora-atomic/kubernetes/workers/cloudinit/worker.yaml.tmpl delete mode 100644 google-cloud/fedora-atomic/kubernetes/workers/outputs.tf delete mode 100644 google-cloud/fedora-atomic/kubernetes/workers/variables.tf delete mode 100644 google-cloud/fedora-atomic/kubernetes/workers/workers.tf create mode 100644 google-cloud/ignore/.gitkeep diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md index 16225c6c7..6796f5b2a 100644 --- a/.github/ISSUE_TEMPLATE.md +++ b/.github/ISSUE_TEMPLATE.md @@ -5,7 +5,7 @@ ### Environment * Platform: aws, azure, bare-metal, google-cloud, digital-ocean -* OS: container-linux, flatcar-linux, or fedora-atomic +* OS: container-linux, flatcar-linux * Release: Typhoon version or Git SHA (reporting latest is **not** helpful) * Terraform: `terraform version` (reporting latest is **not** helpful) * Plugins: Provider plugin versions (reporting latest is **not** helpful) diff --git a/CHANGES.md b/CHANGES.md index 6126a61af..19c0ba3db 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -11,6 +11,7 @@ Notable changes between versions. * [Migration](https://typhoon.psdn.io/topics/maintenance/#terraform-v012x) instructions for Terraform v0.12 * Require `terraform-provider-ct` v0.3.2+ to support Terraform v0.12 (action required) * Update Calico from v3.7.2 to [v3.7.3](https://docs.projectcalico.org/v3.7/release-notes/) +* Remove Fedora Atomic modules (deprecated in March) ([#501](https://github.com/poseidon/typhoon/pull/501)) #### AWS diff --git a/aws/fedora-atomic/kubernetes/LICENSE b/aws/fedora-atomic/kubernetes/LICENSE deleted file mode 100644 index bd9a5eea2..000000000 --- a/aws/fedora-atomic/kubernetes/LICENSE +++ /dev/null @@ -1,23 +0,0 @@ -The MIT License (MIT) - -Copyright (c) 2017 Typhoon Authors -Copyright (c) 2017 Dalton Hubble - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in -all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -THE SOFTWARE. - diff --git a/aws/fedora-atomic/kubernetes/README.md b/aws/fedora-atomic/kubernetes/README.md deleted file mode 100644 index 82abe4318..000000000 --- a/aws/fedora-atomic/kubernetes/README.md +++ /dev/null @@ -1,23 +0,0 @@ -# Typhoon - -Typhoon is a minimal and free Kubernetes distribution. - -* Minimal, stable base Kubernetes distribution -* Declarative infrastructure and configuration -* Free (freedom and cost) and privacy-respecting -* Practical for labs, datacenters, and clouds - -Typhoon distributes upstream Kubernetes, architectural conventions, and cluster addons, much like a GNU/Linux distribution provides the Linux kernel and userspace components. - -## Features - -* Kubernetes v1.15.0 (upstream, via [kubernetes-incubator/bootkube](https://github.com/kubernetes-incubator/bootkube)) -* Single or multi-master, [Calico](https://www.projectcalico.org/) or [flannel](https://github.com/coreos/flannel) networking -* On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/) -* Advanced features like [worker pools](https://typhoon.psdn.io/advanced/worker-pools/) and [spot](https://typhoon.psdn.io/cl/aws/#spot) workers -* Ready for Ingress, Prometheus, Grafana, and other optional [addons](https://typhoon.psdn.io/addons/overview/) - -## Docs - -Please see the [official docs](https://typhoon.psdn.io) and the AWS [tutorial](https://typhoon.psdn.io/cl/aws/). - diff --git a/aws/fedora-atomic/kubernetes/ami.tf b/aws/fedora-atomic/kubernetes/ami.tf deleted file mode 100644 index 7573749c5..000000000 --- a/aws/fedora-atomic/kubernetes/ami.tf +++ /dev/null @@ -1,19 +0,0 @@ -data "aws_ami" "fedora" { - most_recent = true - owners = ["125523088429"] - - filter { - name = "architecture" - values = ["x86_64"] - } - - filter { - name = "virtualization-type" - values = ["hvm"] - } - - filter { - name = "name" - values = ["Fedora-AtomicHost-28-20180625.1.x86_64-*-gp2-*"] - } -} diff --git a/aws/fedora-atomic/kubernetes/bootkube.tf b/aws/fedora-atomic/kubernetes/bootkube.tf deleted file mode 100644 index 630894d32..000000000 --- a/aws/fedora-atomic/kubernetes/bootkube.tf +++ /dev/null @@ -1,18 +0,0 @@ -# Self-hosted Kubernetes assets (kubeconfig, manifests) -module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=62df9ad69cc0da35f47d40fa981370c4503ad581" - - cluster_name = "${var.cluster_name}" - api_servers = ["${format("%s.%s", var.cluster_name, var.dns_zone)}"] - etcd_servers = ["${aws_route53_record.etcds.*.fqdn}"] - asset_dir = "${var.asset_dir}" - networking = "${var.networking}" - network_mtu = "${var.network_mtu}" - pod_cidr = "${var.pod_cidr}" - service_cidr = "${var.service_cidr}" - cluster_domain_suffix = "${var.cluster_domain_suffix}" - enable_reporting = "${var.enable_reporting}" - - # Fedora - trusted_certs_dir = "/etc/pki/tls/certs" -} diff --git a/aws/fedora-atomic/kubernetes/cloudinit/controller.yaml.tmpl b/aws/fedora-atomic/kubernetes/cloudinit/controller.yaml.tmpl deleted file mode 100644 index 64c1058e7..000000000 --- a/aws/fedora-atomic/kubernetes/cloudinit/controller.yaml.tmpl +++ /dev/null @@ -1,93 +0,0 @@ -#cloud-config -write_files: - - path: /etc/etcd/etcd.conf - content: | - ETCD_NAME=${etcd_name} - ETCD_DATA_DIR=/var/lib/etcd - ETCD_ADVERTISE_CLIENT_URLS=https://${etcd_domain}:2379 - ETCD_INITIAL_ADVERTISE_PEER_URLS=https://${etcd_domain}:2380 - ETCD_LISTEN_CLIENT_URLS=https://0.0.0.0:2379 - ETCD_LISTEN_PEER_URLS=https://0.0.0.0:2380 - ETCD_LISTEN_METRICS_URLS=http://0.0.0.0:2381 - ETCD_INITIAL_CLUSTER=${etcd_initial_cluster} - ETCD_STRICT_RECONFIG_CHECK=true - ETCD_TRUSTED_CA_FILE=/etc/ssl/certs/etcd/server-ca.crt - ETCD_CERT_FILE=/etc/ssl/certs/etcd/server.crt - ETCD_KEY_FILE=/etc/ssl/certs/etcd/server.key - ETCD_CLIENT_CERT_AUTH=true - ETCD_PEER_TRUSTED_CA_FILE=/etc/ssl/certs/etcd/peer-ca.crt - ETCD_PEER_CERT_FILE=/etc/ssl/certs/etcd/peer.crt - ETCD_PEER_KEY_FILE=/etc/ssl/certs/etcd/peer.key - ETCD_PEER_CLIENT_CERT_AUTH=true - - path: /etc/systemd/system/kubelet.service.d/10-typhoon.conf - content: | - [Unit] - Wants=rpc-statd.service - [Service] - ExecStartPre=/bin/mkdir -p /opt/cni/bin - ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests - ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d - ExecStartPre=/bin/mkdir -p /etc/kubernetes/checkpoint-secrets - ExecStartPre=/bin/mkdir -p /etc/kubernetes/inactive-manifests - ExecStartPre=/bin/mkdir -p /var/lib/cni - ExecStartPre=/bin/mkdir -p /var/lib/kubelet/volumeplugins - ExecStartPre=/usr/bin/bash -c "grep 'certificate-authority-data' /etc/kubernetes/kubeconfig | awk '{print $2}' | base64 -d > /etc/kubernetes/ca.crt" - Restart=always - RestartSec=10 - - path: /etc/kubernetes/kubelet.conf - content: | - ARGS="--anonymous-auth=false \ - --authentication-token-webhook \ - --authorization-mode=Webhook \ - --client-ca-file=/etc/kubernetes/ca.crt \ - --cluster_dns=${cluster_dns_service_ip} \ - --cluster_domain=${cluster_domain_suffix} \ - --cni-conf-dir=/etc/kubernetes/cni/net.d \ - --exit-on-lock-contention \ - --kubeconfig=/etc/kubernetes/kubeconfig \ - --lock-file=/var/run/lock/kubelet.lock \ - --network-plugin=cni \ - --node-labels=node-role.kubernetes.io/master \ - --node-labels=node-role.kubernetes.io/controller="true" \ - --pod-manifest-path=/etc/kubernetes/manifests \ - --read-only-port=0 \ - --register-with-taints=node-role.kubernetes.io/master=:NoSchedule \ - --volume-plugin-dir=/var/lib/kubelet/volumeplugins" - - path: /etc/kubernetes/kubeconfig - permissions: '0644' - content: | - ${kubeconfig} - - path: /var/lib/bootkube/.keep - - path: /etc/NetworkManager/conf.d/typhoon.conf - content: | - [main] - plugins=keyfile - [keyfile] - unmanaged-devices=interface-name:cali*;interface-name:tunl* - - path: /etc/selinux/config - owner: root:root - permissions: '0644' - content: | - SELINUX=permissive - SELINUXTYPE=targeted -bootcmd: - - [setenforce, Permissive] - - [systemctl, disable, firewalld, --now] - # https://github.com/kubernetes/kubernetes/issues/60869 - - [modprobe, ip_vs] -runcmd: - - [systemctl, daemon-reload] - - [systemctl, restart, NetworkManager] - - "atomic install --system --name=etcd quay.io/poseidon/etcd:v3.3.12" - - "atomic install --system --name=kubelet quay.io/poseidon/kubelet:v1.14.1" - - "atomic install --system --name=bootkube quay.io/poseidon/bootkube:v0.14.0" - - [systemctl, start, --no-block, etcd.service] - - [systemctl, start, --no-block, kubelet.service] -users: - - default - - name: fedora - gecos: Fedora Admin - sudo: ALL=(ALL) NOPASSWD:ALL - groups: wheel,adm,systemd-journal,docker - ssh-authorized-keys: - - "${ssh_authorized_key}" diff --git a/aws/fedora-atomic/kubernetes/controllers.tf b/aws/fedora-atomic/kubernetes/controllers.tf deleted file mode 100644 index 26ba3ca36..000000000 --- a/aws/fedora-atomic/kubernetes/controllers.tf +++ /dev/null @@ -1,79 +0,0 @@ -# Discrete DNS records for each controller's private IPv4 for etcd usage -resource "aws_route53_record" "etcds" { - count = "${var.controller_count}" - - # DNS Zone where record should be created - zone_id = "${var.dns_zone_id}" - - name = "${format("%s-etcd%d.%s.", var.cluster_name, count.index, var.dns_zone)}" - type = "A" - ttl = 300 - - # private IPv4 address for etcd - records = ["${element(aws_instance.controllers.*.private_ip, count.index)}"] -} - -# Controller instances -resource "aws_instance" "controllers" { - count = "${var.controller_count}" - - tags = { - Name = "${var.cluster_name}-controller-${count.index}" - } - - instance_type = "${var.controller_type}" - - ami = "${data.aws_ami.fedora.image_id}" - user_data = "${element(data.template_file.controller-cloudinit.*.rendered, count.index)}" - - # storage - root_block_device { - volume_type = "${var.disk_type}" - volume_size = "${var.disk_size}" - iops = "${var.disk_iops}" - } - - # network - associate_public_ip_address = true - subnet_id = "${element(aws_subnet.public.*.id, count.index)}" - vpc_security_group_ids = ["${aws_security_group.controller.id}"] - - lifecycle { - ignore_changes = [ - "ami", - "user_data", - ] - } -} - -# Controller Cloud-Init -data "template_file" "controller-cloudinit" { - count = "${var.controller_count}" - - template = "${file("${path.module}/cloudinit/controller.yaml.tmpl")}" - - vars = { - # Cannot use cyclic dependencies on controllers or their DNS records - etcd_name = "etcd${count.index}" - etcd_domain = "${var.cluster_name}-etcd${count.index}.${var.dns_zone}" - - # etcd0=https://cluster-etcd0.example.com,etcd1=https://cluster-etcd1.example.com,... - etcd_initial_cluster = "${join(",", data.template_file.etcds.*.rendered)}" - - kubeconfig = "${indent(6, module.bootkube.kubeconfig-kubelet)}" - ssh_authorized_key = "${var.ssh_authorized_key}" - cluster_dns_service_ip = "${cidrhost(var.service_cidr, 10)}" - cluster_domain_suffix = "${var.cluster_domain_suffix}" - } -} - -data "template_file" "etcds" { - count = "${var.controller_count}" - template = "etcd$${index}=https://$${cluster_name}-etcd$${index}.$${dns_zone}:2380" - - vars = { - index = "${count.index}" - cluster_name = "${var.cluster_name}" - dns_zone = "${var.dns_zone}" - } -} diff --git a/aws/fedora-atomic/kubernetes/network.tf b/aws/fedora-atomic/kubernetes/network.tf deleted file mode 100644 index 1be5073ba..000000000 --- a/aws/fedora-atomic/kubernetes/network.tf +++ /dev/null @@ -1,57 +0,0 @@ -data "aws_availability_zones" "all" {} - -# Network VPC, gateway, and routes - -resource "aws_vpc" "network" { - cidr_block = "${var.host_cidr}" - assign_generated_ipv6_cidr_block = true - enable_dns_support = true - enable_dns_hostnames = true - - tags = "${map("Name", "${var.cluster_name}")}" -} - -resource "aws_internet_gateway" "gateway" { - vpc_id = "${aws_vpc.network.id}" - - tags = "${map("Name", "${var.cluster_name}")}" -} - -resource "aws_route_table" "default" { - vpc_id = "${aws_vpc.network.id}" - - route { - cidr_block = "0.0.0.0/0" - gateway_id = "${aws_internet_gateway.gateway.id}" - } - - route { - ipv6_cidr_block = "::/0" - gateway_id = "${aws_internet_gateway.gateway.id}" - } - - tags = "${map("Name", "${var.cluster_name}")}" -} - -# Subnets (one per availability zone) - -resource "aws_subnet" "public" { - count = "${length(data.aws_availability_zones.all.names)}" - - vpc_id = "${aws_vpc.network.id}" - availability_zone = "${data.aws_availability_zones.all.names[count.index]}" - - cidr_block = "${cidrsubnet(var.host_cidr, 4, count.index)}" - ipv6_cidr_block = "${cidrsubnet(aws_vpc.network.ipv6_cidr_block, 8, count.index)}" - map_public_ip_on_launch = true - assign_ipv6_address_on_creation = true - - tags = "${map("Name", "${var.cluster_name}-public-${count.index}")}" -} - -resource "aws_route_table_association" "public" { - count = "${length(data.aws_availability_zones.all.names)}" - - route_table_id = "${aws_route_table.default.id}" - subnet_id = "${element(aws_subnet.public.*.id, count.index)}" -} diff --git a/aws/fedora-atomic/kubernetes/nlb.tf b/aws/fedora-atomic/kubernetes/nlb.tf deleted file mode 100644 index ddcc52f5b..000000000 --- a/aws/fedora-atomic/kubernetes/nlb.tf +++ /dev/null @@ -1,93 +0,0 @@ -# Network Load Balancer DNS Record -resource "aws_route53_record" "apiserver" { - zone_id = "${var.dns_zone_id}" - - name = "${format("%s.%s.", var.cluster_name, var.dns_zone)}" - type = "A" - - # AWS recommends their special "alias" records for NLBs - alias { - name = "${aws_lb.nlb.dns_name}" - zone_id = "${aws_lb.nlb.zone_id}" - evaluate_target_health = true - } -} - -# Network Load Balancer for apiservers and ingress -resource "aws_lb" "nlb" { - name = "${var.cluster_name}-nlb" - load_balancer_type = "network" - internal = false - - subnets = ["${aws_subnet.public.*.id}"] - - enable_cross_zone_load_balancing = true -} - -# Forward TCP apiserver traffic to controllers -resource "aws_lb_listener" "apiserver-https" { - load_balancer_arn = "${aws_lb.nlb.arn}" - protocol = "TCP" - port = "6443" - - default_action { - type = "forward" - target_group_arn = "${aws_lb_target_group.controllers.arn}" - } -} - -# Forward HTTP ingress traffic to workers -resource "aws_lb_listener" "ingress-http" { - load_balancer_arn = "${aws_lb.nlb.arn}" - protocol = "TCP" - port = 80 - - default_action { - type = "forward" - target_group_arn = "${module.workers.target_group_http}" - } -} - -# Forward HTTPS ingress traffic to workers -resource "aws_lb_listener" "ingress-https" { - load_balancer_arn = "${aws_lb.nlb.arn}" - protocol = "TCP" - port = 443 - - default_action { - type = "forward" - target_group_arn = "${module.workers.target_group_https}" - } -} - -# Target group of controllers -resource "aws_lb_target_group" "controllers" { - name = "${var.cluster_name}-controllers" - vpc_id = "${aws_vpc.network.id}" - target_type = "instance" - - protocol = "TCP" - port = 6443 - - # TCP health check for apiserver - health_check { - protocol = "TCP" - port = 6443 - - # NLBs required to use same healthy and unhealthy thresholds - healthy_threshold = 3 - unhealthy_threshold = 3 - - # Interval between health checks required to be 10 or 30 - interval = 10 - } -} - -# Attach controller instances to apiserver NLB -resource "aws_lb_target_group_attachment" "controllers" { - count = "${var.controller_count}" - - target_group_arn = "${aws_lb_target_group.controllers.arn}" - target_id = "${element(aws_instance.controllers.*.id, count.index)}" - port = 6443 -} diff --git a/aws/fedora-atomic/kubernetes/outputs.tf b/aws/fedora-atomic/kubernetes/outputs.tf deleted file mode 100644 index d042a3d4b..000000000 --- a/aws/fedora-atomic/kubernetes/outputs.tf +++ /dev/null @@ -1,48 +0,0 @@ -output "kubeconfig-admin" { - value = "${module.bootkube.kubeconfig-admin}" -} - -# Outputs for Kubernetes Ingress - -output "ingress_dns_name" { - value = "${aws_lb.nlb.dns_name}" - description = "DNS name of the network load balancer for distributing traffic to Ingress controllers" -} - -output "ingress_zone_id" { - value = "${aws_lb.nlb.zone_id}" - description = "Route53 zone id of the network load balancer DNS name that can be used in Route53 alias records" -} - -# Outputs for worker pools - -output "vpc_id" { - value = "${aws_vpc.network.id}" - description = "ID of the VPC for creating worker instances" -} - -output "subnet_ids" { - value = ["${aws_subnet.public.*.id}"] - description = "List of subnet IDs for creating worker instances" -} - -output "worker_security_groups" { - value = ["${aws_security_group.worker.id}"] - description = "List of worker security group IDs" -} - -output "kubeconfig" { - value = "${module.bootkube.kubeconfig-kubelet}" -} - -# Outputs for custom load balancing - -output "worker_target_group_http" { - description = "ARN of a target group of workers for HTTP traffic" - value = "${module.workers.target_group_http}" -} - -output "worker_target_group_https" { - description = "ARN of a target group of workers for HTTPS traffic" - value = "${module.workers.target_group_https}" -} diff --git a/aws/fedora-atomic/kubernetes/require.tf b/aws/fedora-atomic/kubernetes/require.tf deleted file mode 100644 index 68f475d62..000000000 --- a/aws/fedora-atomic/kubernetes/require.tf +++ /dev/null @@ -1,25 +0,0 @@ -# Terraform version and plugin versions - -terraform { - required_version = ">= 0.11.0" -} - -provider "aws" { - version = ">= 1.13, < 3.0" -} - -provider "local" { - version = "~> 1.0" -} - -provider "null" { - version = "~> 1.0" -} - -provider "template" { - version = "~> 1.0" -} - -provider "tls" { - version = "~> 1.0" -} diff --git a/aws/fedora-atomic/kubernetes/security.tf b/aws/fedora-atomic/kubernetes/security.tf deleted file mode 100644 index 7672a92e8..000000000 --- a/aws/fedora-atomic/kubernetes/security.tf +++ /dev/null @@ -1,359 +0,0 @@ -# Security Groups (instance firewalls) - -# Controller security group - -resource "aws_security_group" "controller" { - name = "${var.cluster_name}-controller" - description = "${var.cluster_name} controller security group" - - vpc_id = "${aws_vpc.network.id}" - - tags = "${map("Name", "${var.cluster_name}-controller")}" -} - -resource "aws_security_group_rule" "controller-ssh" { - security_group_id = "${aws_security_group.controller.id}" - - type = "ingress" - protocol = "tcp" - from_port = 22 - to_port = 22 - cidr_blocks = ["0.0.0.0/0"] -} - -resource "aws_security_group_rule" "controller-etcd" { - security_group_id = "${aws_security_group.controller.id}" - - type = "ingress" - protocol = "tcp" - from_port = 2379 - to_port = 2380 - self = true -} - -# Allow Prometheus to scrape etcd metrics -resource "aws_security_group_rule" "controller-etcd-metrics" { - security_group_id = "${aws_security_group.controller.id}" - - type = "ingress" - protocol = "tcp" - from_port = 2381 - to_port = 2381 - source_security_group_id = "${aws_security_group.worker.id}" -} - -resource "aws_security_group_rule" "controller-vxlan" { - count = "${var.networking == "flannel" ? 1 : 0}" - - security_group_id = "${aws_security_group.controller.id}" - - type = "ingress" - protocol = "udp" - from_port = 4789 - to_port = 4789 - source_security_group_id = "${aws_security_group.worker.id}" -} - -resource "aws_security_group_rule" "controller-vxlan-self" { - count = "${var.networking == "flannel" ? 1 : 0}" - - security_group_id = "${aws_security_group.controller.id}" - - type = "ingress" - protocol = "udp" - from_port = 4789 - to_port = 4789 - self = true -} - -resource "aws_security_group_rule" "controller-apiserver" { - security_group_id = "${aws_security_group.controller.id}" - - type = "ingress" - protocol = "tcp" - from_port = 6443 - to_port = 6443 - cidr_blocks = ["0.0.0.0/0"] -} - -# Allow Prometheus to scrape node-exporter daemonset -resource "aws_security_group_rule" "controller-node-exporter" { - security_group_id = "${aws_security_group.controller.id}" - - type = "ingress" - protocol = "tcp" - from_port = 9100 - to_port = 9100 - source_security_group_id = "${aws_security_group.worker.id}" -} - -# Allow apiserver to access kubelets for exec, log, port-forward -resource "aws_security_group_rule" "controller-kubelet" { - security_group_id = "${aws_security_group.controller.id}" - - type = "ingress" - protocol = "tcp" - from_port = 10250 - to_port = 10250 - source_security_group_id = "${aws_security_group.worker.id}" -} - -resource "aws_security_group_rule" "controller-kubelet-self" { - security_group_id = "${aws_security_group.controller.id}" - - type = "ingress" - protocol = "tcp" - from_port = 10250 - to_port = 10250 - self = true -} - -resource "aws_security_group_rule" "controller-bgp" { - security_group_id = "${aws_security_group.controller.id}" - - type = "ingress" - protocol = "tcp" - from_port = 179 - to_port = 179 - source_security_group_id = "${aws_security_group.worker.id}" -} - -resource "aws_security_group_rule" "controller-bgp-self" { - security_group_id = "${aws_security_group.controller.id}" - - type = "ingress" - protocol = "tcp" - from_port = 179 - to_port = 179 - self = true -} - -resource "aws_security_group_rule" "controller-ipip" { - security_group_id = "${aws_security_group.controller.id}" - - type = "ingress" - protocol = 4 - from_port = 0 - to_port = 0 - source_security_group_id = "${aws_security_group.worker.id}" -} - -resource "aws_security_group_rule" "controller-ipip-self" { - security_group_id = "${aws_security_group.controller.id}" - - type = "ingress" - protocol = 4 - from_port = 0 - to_port = 0 - self = true -} - -resource "aws_security_group_rule" "controller-ipip-legacy" { - security_group_id = "${aws_security_group.controller.id}" - - type = "ingress" - protocol = 94 - from_port = 0 - to_port = 0 - source_security_group_id = "${aws_security_group.worker.id}" -} - -resource "aws_security_group_rule" "controller-ipip-legacy-self" { - security_group_id = "${aws_security_group.controller.id}" - - type = "ingress" - protocol = 94 - from_port = 0 - to_port = 0 - self = true -} - -resource "aws_security_group_rule" "controller-egress" { - security_group_id = "${aws_security_group.controller.id}" - - type = "egress" - protocol = "-1" - from_port = 0 - to_port = 0 - cidr_blocks = ["0.0.0.0/0"] - ipv6_cidr_blocks = ["::/0"] -} - -# Worker security group - -resource "aws_security_group" "worker" { - name = "${var.cluster_name}-worker" - description = "${var.cluster_name} worker security group" - - vpc_id = "${aws_vpc.network.id}" - - tags = "${map("Name", "${var.cluster_name}-worker")}" -} - -resource "aws_security_group_rule" "worker-ssh" { - security_group_id = "${aws_security_group.worker.id}" - - type = "ingress" - protocol = "tcp" - from_port = 22 - to_port = 22 - cidr_blocks = ["0.0.0.0/0"] -} - -resource "aws_security_group_rule" "worker-http" { - security_group_id = "${aws_security_group.worker.id}" - - type = "ingress" - protocol = "tcp" - from_port = 80 - to_port = 80 - cidr_blocks = ["0.0.0.0/0"] -} - -resource "aws_security_group_rule" "worker-https" { - security_group_id = "${aws_security_group.worker.id}" - - type = "ingress" - protocol = "tcp" - from_port = 443 - to_port = 443 - cidr_blocks = ["0.0.0.0/0"] -} - -resource "aws_security_group_rule" "worker-vxlan" { - count = "${var.networking == "flannel" ? 1 : 0}" - - security_group_id = "${aws_security_group.worker.id}" - - type = "ingress" - protocol = "udp" - from_port = 4789 - to_port = 4789 - source_security_group_id = "${aws_security_group.controller.id}" -} - -resource "aws_security_group_rule" "worker-vxlan-self" { - count = "${var.networking == "flannel" ? 1 : 0}" - - security_group_id = "${aws_security_group.worker.id}" - - type = "ingress" - protocol = "udp" - from_port = 4789 - to_port = 4789 - self = true -} - -# Allow Prometheus to scrape node-exporter daemonset -resource "aws_security_group_rule" "worker-node-exporter" { - security_group_id = "${aws_security_group.worker.id}" - - type = "ingress" - protocol = "tcp" - from_port = 9100 - to_port = 9100 - self = true -} - -resource "aws_security_group_rule" "ingress-health" { - security_group_id = "${aws_security_group.worker.id}" - - type = "ingress" - protocol = "tcp" - from_port = 10254 - to_port = 10254 - cidr_blocks = ["0.0.0.0/0"] -} - -# Allow apiserver to access kubelets for exec, log, port-forward -resource "aws_security_group_rule" "worker-kubelet" { - security_group_id = "${aws_security_group.worker.id}" - - type = "ingress" - protocol = "tcp" - from_port = 10250 - to_port = 10250 - source_security_group_id = "${aws_security_group.controller.id}" -} - -# Allow Prometheus to scrape kubelet metrics -resource "aws_security_group_rule" "worker-kubelet-self" { - security_group_id = "${aws_security_group.worker.id}" - - type = "ingress" - protocol = "tcp" - from_port = 10250 - to_port = 10250 - self = true -} - -resource "aws_security_group_rule" "worker-bgp" { - security_group_id = "${aws_security_group.worker.id}" - - type = "ingress" - protocol = "tcp" - from_port = 179 - to_port = 179 - source_security_group_id = "${aws_security_group.controller.id}" -} - -resource "aws_security_group_rule" "worker-bgp-self" { - security_group_id = "${aws_security_group.worker.id}" - - type = "ingress" - protocol = "tcp" - from_port = 179 - to_port = 179 - self = true -} - -resource "aws_security_group_rule" "worker-ipip" { - security_group_id = "${aws_security_group.worker.id}" - - type = "ingress" - protocol = 4 - from_port = 0 - to_port = 0 - source_security_group_id = "${aws_security_group.controller.id}" -} - -resource "aws_security_group_rule" "worker-ipip-self" { - security_group_id = "${aws_security_group.worker.id}" - - type = "ingress" - protocol = 4 - from_port = 0 - to_port = 0 - self = true -} - -resource "aws_security_group_rule" "worker-ipip-legacy" { - security_group_id = "${aws_security_group.worker.id}" - - type = "ingress" - protocol = 94 - from_port = 0 - to_port = 0 - source_security_group_id = "${aws_security_group.controller.id}" -} - -resource "aws_security_group_rule" "worker-ipip-legacy-self" { - security_group_id = "${aws_security_group.worker.id}" - - type = "ingress" - protocol = 94 - from_port = 0 - to_port = 0 - self = true -} - -resource "aws_security_group_rule" "worker-egress" { - security_group_id = "${aws_security_group.worker.id}" - - type = "egress" - protocol = "-1" - from_port = 0 - to_port = 0 - cidr_blocks = ["0.0.0.0/0"] - ipv6_cidr_blocks = ["::/0"] -} diff --git a/aws/fedora-atomic/kubernetes/ssh.tf b/aws/fedora-atomic/kubernetes/ssh.tf deleted file mode 100644 index c72a09e4d..000000000 --- a/aws/fedora-atomic/kubernetes/ssh.tf +++ /dev/null @@ -1,89 +0,0 @@ -# Secure copy etcd TLS assets to controllers. -resource "null_resource" "copy-controller-secrets" { - count = "${var.controller_count}" - - connection { - type = "ssh" - host = "${element(aws_instance.controllers.*.public_ip, count.index)}" - user = "fedora" - timeout = "15m" - } - - provisioner "file" { - content = "${module.bootkube.etcd_ca_cert}" - destination = "$HOME/etcd-client-ca.crt" - } - - provisioner "file" { - content = "${module.bootkube.etcd_client_cert}" - destination = "$HOME/etcd-client.crt" - } - - provisioner "file" { - content = "${module.bootkube.etcd_client_key}" - destination = "$HOME/etcd-client.key" - } - - provisioner "file" { - content = "${module.bootkube.etcd_server_cert}" - destination = "$HOME/etcd-server.crt" - } - - provisioner "file" { - content = "${module.bootkube.etcd_server_key}" - destination = "$HOME/etcd-server.key" - } - - provisioner "file" { - content = "${module.bootkube.etcd_peer_cert}" - destination = "$HOME/etcd-peer.crt" - } - - provisioner "file" { - content = "${module.bootkube.etcd_peer_key}" - destination = "$HOME/etcd-peer.key" - } - - provisioner "remote-exec" { - inline = [ - "sudo mkdir -p /etc/ssl/etcd/etcd", - "sudo mv etcd-client* /etc/ssl/etcd/", - "sudo cp /etc/ssl/etcd/etcd-client-ca.crt /etc/ssl/etcd/etcd/server-ca.crt", - "sudo mv etcd-server.crt /etc/ssl/etcd/etcd/server.crt", - "sudo mv etcd-server.key /etc/ssl/etcd/etcd/server.key", - "sudo cp /etc/ssl/etcd/etcd-client-ca.crt /etc/ssl/etcd/etcd/peer-ca.crt", - "sudo mv etcd-peer.crt /etc/ssl/etcd/etcd/peer.crt", - "sudo mv etcd-peer.key /etc/ssl/etcd/etcd/peer.key", - ] - } -} - -# Secure copy bootkube assets to ONE controller and start bootkube to perform -# one-time self-hosted cluster bootstrapping. -resource "null_resource" "bootkube-start" { - depends_on = [ - "null_resource.copy-controller-secrets", - "module.workers", - "aws_route53_record.apiserver", - ] - - connection { - type = "ssh" - host = "${aws_instance.controllers.0.public_ip}" - user = "fedora" - timeout = "15m" - } - - provisioner "file" { - source = "${var.asset_dir}" - destination = "$HOME/assets" - } - - provisioner "remote-exec" { - inline = [ - "while [ ! -f /var/lib/cloud/instance/boot-finished ]; do sleep 4; done", - "sudo mv $HOME/assets /var/lib/bootkube", - "sudo systemctl start bootkube", - ] - } -} diff --git a/aws/fedora-atomic/kubernetes/variables.tf b/aws/fedora-atomic/kubernetes/variables.tf deleted file mode 100644 index 538923f65..000000000 --- a/aws/fedora-atomic/kubernetes/variables.tf +++ /dev/null @@ -1,124 +0,0 @@ -variable "cluster_name" { - type = "string" - description = "Unique cluster name (prepended to dns_zone)" -} - -# AWS - -variable "dns_zone" { - type = "string" - description = "AWS DNS Zone (e.g. aws.example.com)" -} - -variable "dns_zone_id" { - type = "string" - description = "AWS DNS Zone ID (e.g. Z3PAABBCFAKEC0)" -} - -# instances - -variable "controller_count" { - type = "string" - default = "1" - description = "Number of controllers (i.e. masters)" -} - -variable "worker_count" { - type = "string" - default = "1" - description = "Number of workers" -} - -variable "controller_type" { - type = "string" - default = "t3.small" - description = "EC2 instance type for controllers" -} - -variable "worker_type" { - type = "string" - default = "t3.small" - description = "EC2 instance type for workers" -} - -variable "disk_size" { - type = "string" - default = "40" - description = "Size of the EBS volume in GB" -} - -variable "disk_type" { - type = "string" - default = "gp2" - description = "Type of the EBS volume (e.g. standard, gp2, io1)" -} - -variable "disk_iops" { - type = "string" - default = "0" - description = "IOPS of the EBS volume (e.g. 100)" -} - -variable "worker_price" { - type = "string" - default = "" - description = "Spot price in USD for autoscaling group spot instances. Leave as default empty string for autoscaling group to use on-demand instances. Note, switching in-place from spot to on-demand is not possible: https://github.com/terraform-providers/terraform-provider-aws/issues/4320" -} - -# configuration - -variable "ssh_authorized_key" { - type = "string" - description = "SSH public key for user 'fedora'" -} - -variable "asset_dir" { - description = "Path to a directory where generated assets should be placed (contains secrets)" - type = "string" -} - -variable "networking" { - description = "Choice of networking provider (calico or flannel)" - type = "string" - default = "calico" -} - -variable "network_mtu" { - description = "CNI interface MTU (applies to calico only). Use 8981 if using instances types with Jumbo frames." - type = "string" - default = "1480" -} - -variable "host_cidr" { - description = "CIDR IPv4 range to assign to EC2 nodes" - type = "string" - default = "10.0.0.0/16" -} - -variable "pod_cidr" { - description = "CIDR IPv4 range to assign Kubernetes pods" - type = "string" - default = "10.2.0.0/16" -} - -variable "service_cidr" { - description = < /etc/kubernetes/ca.crt" - Restart=always - RestartSec=10 - - path: /etc/kubernetes/kubelet.conf - content: | - ARGS="--anonymous-auth=false \ - --authentication-token-webhook \ - --authorization-mode=Webhook \ - --client-ca-file=/etc/kubernetes/ca.crt \ - --cluster_dns=${cluster_dns_service_ip} \ - --cluster_domain=${cluster_domain_suffix} \ - --cni-conf-dir=/etc/kubernetes/cni/net.d \ - --exit-on-lock-contention \ - --kubeconfig=/etc/kubernetes/kubeconfig \ - --lock-file=/var/run/lock/kubelet.lock \ - --network-plugin=cni \ - --node-labels=node-role.kubernetes.io/node \ - --pod-manifest-path=/etc/kubernetes/manifests \ - --read-only-port=0 \ - --volume-plugin-dir=/var/lib/kubelet/volumeplugins" - - path: /etc/kubernetes/kubeconfig - permissions: '0644' - content: | - ${kubeconfig} - - path: /etc/NetworkManager/conf.d/typhoon.conf - content: | - [main] - plugins=keyfile - [keyfile] - unmanaged-devices=interface-name:cali*;interface-name:tunl* - - path: /etc/selinux/config - owner: root:root - permissions: '0644' - content: | - SELINUX=permissive - SELINUXTYPE=targeted -bootcmd: - - [setenforce, Permissive] - - [systemctl, disable, firewalld, --now] - # https://github.com/kubernetes/kubernetes/issues/60869 - - [modprobe, ip_vs] -runcmd: - - [systemctl, daemon-reload] - - [systemctl, restart, NetworkManager] - - "atomic install --system --name=kubelet quay.io/poseidon/kubelet:v1.14.1" - - [systemctl, start, --no-block, kubelet.service] -users: - - default - - name: fedora - gecos: Fedora Admin - sudo: ALL=(ALL) NOPASSWD:ALL - groups: wheel,adm,systemd-journal,docker - ssh-authorized-keys: - - "${ssh_authorized_key}" diff --git a/aws/fedora-atomic/kubernetes/workers/ingress.tf b/aws/fedora-atomic/kubernetes/workers/ingress.tf deleted file mode 100644 index bdb7362fb..000000000 --- a/aws/fedora-atomic/kubernetes/workers/ingress.tf +++ /dev/null @@ -1,47 +0,0 @@ -# Target groups of instances for use with load balancers - -resource "aws_lb_target_group" "workers-http" { - name = "${var.name}-workers-http" - vpc_id = "${var.vpc_id}" - target_type = "instance" - - protocol = "TCP" - port = 80 - - # HTTP health check for ingress - health_check { - protocol = "HTTP" - port = 10254 - path = "/healthz" - - # NLBs required to use same healthy and unhealthy thresholds - healthy_threshold = 3 - unhealthy_threshold = 3 - - # Interval between health checks required to be 10 or 30 - interval = 10 - } -} - -resource "aws_lb_target_group" "workers-https" { - name = "${var.name}-workers-https" - vpc_id = "${var.vpc_id}" - target_type = "instance" - - protocol = "TCP" - port = 443 - - # HTTP health check for ingress - health_check { - protocol = "HTTP" - port = 10254 - path = "/healthz" - - # NLBs required to use same healthy and unhealthy thresholds - healthy_threshold = 3 - unhealthy_threshold = 3 - - # Interval between health checks required to be 10 or 30 - interval = 10 - } -} diff --git a/aws/fedora-atomic/kubernetes/workers/outputs.tf b/aws/fedora-atomic/kubernetes/workers/outputs.tf deleted file mode 100644 index 19552266a..000000000 --- a/aws/fedora-atomic/kubernetes/workers/outputs.tf +++ /dev/null @@ -1,9 +0,0 @@ -output "target_group_http" { - description = "ARN of a target group of workers for HTTP traffic" - value = "${aws_lb_target_group.workers-http.arn}" -} - -output "target_group_https" { - description = "ARN of a target group of workers for HTTPS traffic" - value = "${aws_lb_target_group.workers-https.arn}" -} diff --git a/aws/fedora-atomic/kubernetes/workers/variables.tf b/aws/fedora-atomic/kubernetes/workers/variables.tf deleted file mode 100644 index 7552e2971..000000000 --- a/aws/fedora-atomic/kubernetes/workers/variables.tf +++ /dev/null @@ -1,87 +0,0 @@ -variable "name" { - type = "string" - description = "Unique name for the worker pool" -} - -# AWS - -variable "vpc_id" { - type = "string" - description = "Must be set to `vpc_id` output by cluster" -} - -variable "subnet_ids" { - type = "list" - description = "Must be set to `subnet_ids` output by cluster" -} - -variable "security_groups" { - type = "list" - description = "Must be set to `worker_security_groups` output by cluster" -} - -# instances - -variable "count" { - type = "string" - default = "1" - description = "Number of instances" -} - -variable "instance_type" { - type = "string" - default = "t3.small" - description = "EC2 instance type" -} - -variable "disk_size" { - type = "string" - default = "40" - description = "Size of the EBS volume in GB" -} - -variable "disk_type" { - type = "string" - default = "gp2" - description = "Type of the EBS volume (e.g. standard, gp2, io1)" -} - -variable "disk_iops" { - type = "string" - default = "0" - description = "IOPS of the EBS volume (required for io1)" -} - -variable "spot_price" { - type = "string" - default = "" - description = "Spot price in USD for autoscaling group spot instances. Leave as default empty string for autoscaling group to use on-demand instances. Note, switching in-place from spot to on-demand is not possible: https://github.com/terraform-providers/terraform-provider-aws/issues/4320" -} - -# configuration - -variable "kubeconfig" { - type = "string" - description = "Must be set to `kubeconfig` output by cluster" -} - -variable "ssh_authorized_key" { - type = "string" - description = "SSH public key for user 'fedora'" -} - -variable "service_cidr" { - description = < - -Typhoon is a minimal and free Kubernetes distribution. - -* Minimal, stable base Kubernetes distribution -* Declarative infrastructure and configuration -* Free (freedom and cost) and privacy-respecting -* Practical for labs, datacenters, and clouds - -Typhoon distributes upstream Kubernetes, architectural conventions, and cluster addons, much like a GNU/Linux distribution provides the Linux kernel and userspace components. - -## Features - -* Kubernetes v1.15.0 (upstream, via [kubernetes-incubator/bootkube](https://github.com/kubernetes-incubator/bootkube)) -* Single or multi-master, [Calico](https://www.projectcalico.org/) or [flannel](https://github.com/coreos/flannel) networking -* On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/) -* Ready for Ingress, Prometheus, Grafana, and other optional [addons](https://typhoon.psdn.io/addons/overview/) - -## Docs - -Please see the [official docs](https://typhoon.psdn.io) and the bare-metal [tutorial](https://typhoon.psdn.io/cl/bare-metal/). - diff --git a/bare-metal/fedora-atomic/kubernetes/bootkube.tf b/bare-metal/fedora-atomic/kubernetes/bootkube.tf deleted file mode 100644 index 63def25d5..000000000 --- a/bare-metal/fedora-atomic/kubernetes/bootkube.tf +++ /dev/null @@ -1,18 +0,0 @@ -# Self-hosted Kubernetes assets (kubeconfig, manifests) -module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=62df9ad69cc0da35f47d40fa981370c4503ad581" - - cluster_name = "${var.cluster_name}" - api_servers = ["${var.k8s_domain_name}"] - etcd_servers = ["${var.controller_domains}"] - asset_dir = "${var.asset_dir}" - networking = "${var.networking}" - network_mtu = "${var.network_mtu}" - pod_cidr = "${var.pod_cidr}" - service_cidr = "${var.service_cidr}" - cluster_domain_suffix = "${var.cluster_domain_suffix}" - enable_reporting = "${var.enable_reporting}" - - # Fedora - trusted_certs_dir = "/etc/pki/tls/certs" -} diff --git a/bare-metal/fedora-atomic/kubernetes/cloudinit/controller.yaml.tmpl b/bare-metal/fedora-atomic/kubernetes/cloudinit/controller.yaml.tmpl deleted file mode 100644 index a43d58fd2..000000000 --- a/bare-metal/fedora-atomic/kubernetes/cloudinit/controller.yaml.tmpl +++ /dev/null @@ -1,100 +0,0 @@ -#cloud-config -write_files: - - path: /etc/etcd/etcd.conf - content: | - ETCD_NAME=${etcd_name} - ETCD_DATA_DIR=/var/lib/etcd - ETCD_ADVERTISE_CLIENT_URLS=https://${domain_name}:2379 - ETCD_INITIAL_ADVERTISE_PEER_URLS=https://${domain_name}:2380 - ETCD_LISTEN_CLIENT_URLS=https://0.0.0.0:2379 - ETCD_LISTEN_PEER_URLS=https://0.0.0.0:2380 - ETCD_LISTEN_METRICS_URLS=http://0.0.0.0:2381 - ETCD_INITIAL_CLUSTER=${etcd_initial_cluster} - ETCD_STRICT_RECONFIG_CHECK=true - ETCD_TRUSTED_CA_FILE=/etc/ssl/certs/etcd/server-ca.crt - ETCD_CERT_FILE=/etc/ssl/certs/etcd/server.crt - ETCD_KEY_FILE=/etc/ssl/certs/etcd/server.key - ETCD_CLIENT_CERT_AUTH=true - ETCD_PEER_TRUSTED_CA_FILE=/etc/ssl/certs/etcd/peer-ca.crt - ETCD_PEER_CERT_FILE=/etc/ssl/certs/etcd/peer.crt - ETCD_PEER_KEY_FILE=/etc/ssl/certs/etcd/peer.key - ETCD_PEER_CLIENT_CERT_AUTH=true - - path: /etc/systemd/system/kubelet.service.d/10-typhoon.conf - content: | - [Unit] - Wants=rpc-statd.service - [Service] - ExecStartPre=/bin/mkdir -p /opt/cni/bin - ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests - ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d - ExecStartPre=/bin/mkdir -p /etc/kubernetes/checkpoint-secrets - ExecStartPre=/bin/mkdir -p /etc/kubernetes/inactive-manifests - ExecStartPre=/bin/mkdir -p /var/lib/cni - ExecStartPre=/bin/mkdir -p /var/lib/kubelet/volumeplugins - ExecStartPre=/usr/bin/bash -c "grep 'certificate-authority-data' /etc/kubernetes/kubeconfig | awk '{print $2}' | base64 -d > /etc/kubernetes/ca.crt" - Restart=always - RestartSec=10 - - path: /etc/kubernetes/kubelet.conf - content: | - ARGS="--anonymous-auth=false \ - --authentication-token-webhook \ - --authorization-mode=Webhook \ - --client-ca-file=/etc/kubernetes/ca.crt \ - --cluster_dns=${cluster_dns_service_ip} \ - --cluster_domain=${cluster_domain_suffix} \ - --cni-conf-dir=/etc/kubernetes/cni/net.d \ - --exit-on-lock-contention \ - --hostname-override=${domain_name} \ - --kubeconfig=/etc/kubernetes/kubeconfig \ - --lock-file=/var/run/lock/kubelet.lock \ - --network-plugin=cni \ - --node-labels=node-role.kubernetes.io/master \ - --node-labels=node-role.kubernetes.io/controller="true" \ - --pod-manifest-path=/etc/kubernetes/manifests \ - --read-only-port=0 \ - --register-with-taints=node-role.kubernetes.io/master=:NoSchedule \ - --volume-plugin-dir=/var/lib/kubelet/volumeplugins" - - path: /etc/systemd/system/kubelet.path - content: | - [Unit] - Description=Watch for kubeconfig - [Path] - PathExists=/etc/kubernetes/kubeconfig - [Install] - WantedBy=multi-user.target - - path: /var/lib/bootkube/.keep - - path: /etc/NetworkManager/conf.d/typhoon.conf - content: | - [main] - plugins=keyfile - [keyfile] - unmanaged-devices=interface-name:cali*;interface-name:tunl* - - path: /etc/selinux/config - owner: root:root - permissions: '0644' - content: | - SELINUX=permissive - SELINUXTYPE=targeted -bootcmd: - - [setenforce, Permissive] - - [systemctl, disable, firewalld, --now] - # https://github.com/kubernetes/kubernetes/issues/60869 - - [modprobe, ip_vs] -runcmd: - - [systemctl, daemon-reload] - - [systemctl, restart, NetworkManager] - - [hostnamectl, set-hostname, ${domain_name}] - - "atomic install --system --name=etcd quay.io/poseidon/etcd:v3.3.12" - - "atomic install --system --name=kubelet quay.io/poseidon/kubelet:v1.14.1" - - "atomic install --system --name=bootkube quay.io/poseidon/bootkube:v0.14.0" - - [systemctl, start, --no-block, etcd.service] - - [systemctl, enable, kubelet.path] - - [systemctl, start, --no-block, kubelet.path] -users: - - default - - name: fedora - gecos: Fedora Admin - sudo: ALL=(ALL) NOPASSWD:ALL - groups: wheel,adm,systemd-journal,docker - ssh-authorized-keys: - - "${ssh_authorized_key}" diff --git a/bare-metal/fedora-atomic/kubernetes/cloudinit/worker.yaml.tmpl b/bare-metal/fedora-atomic/kubernetes/cloudinit/worker.yaml.tmpl deleted file mode 100644 index 2d4c5cf01..000000000 --- a/bare-metal/fedora-atomic/kubernetes/cloudinit/worker.yaml.tmpl +++ /dev/null @@ -1,73 +0,0 @@ -#cloud-config -write_files: - - path: /etc/systemd/system/kubelet.service.d/10-typhoon.conf - content: | - [Unit] - Wants=rpc-statd.service - [Service] - ExecStartPre=/bin/mkdir -p /opt/cni/bin - ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests - ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d - ExecStartPre=/bin/mkdir -p /var/lib/cni - ExecStartPre=/bin/mkdir -p /var/lib/kubelet/volumeplugins - ExecStartPre=/usr/bin/bash -c "grep 'certificate-authority-data' /etc/kubernetes/kubeconfig | awk '{print $2}' | base64 -d > /etc/kubernetes/ca.crt" - Restart=always - RestartSec=10 - - path: /etc/kubernetes/kubelet.conf - content: | - ARGS="--anonymous-auth=false \ - --authentication-token-webhook \ - --authorization-mode=Webhook \ - --client-ca-file=/etc/kubernetes/ca.crt \ - --cluster_dns=${cluster_dns_service_ip} \ - --cluster_domain=${cluster_domain_suffix} \ - --cni-conf-dir=/etc/kubernetes/cni/net.d \ - --exit-on-lock-contention \ - --hostname-override=${domain_name} \ - --kubeconfig=/etc/kubernetes/kubeconfig \ - --lock-file=/var/run/lock/kubelet.lock \ - --network-plugin=cni \ - --node-labels=node-role.kubernetes.io/node \ - --pod-manifest-path=/etc/kubernetes/manifests \ - --read-only-port=0 \ - --volume-plugin-dir=/var/lib/kubelet/volumeplugins" - - path: /etc/systemd/system/kubelet.path - content: | - [Unit] - Description=Watch for kubeconfig - [Path] - PathExists=/etc/kubernetes/kubeconfig - [Install] - WantedBy=multi-user.target - - path: /etc/NetworkManager/conf.d/typhoon.conf - content: | - [main] - plugins=keyfile - [keyfile] - unmanaged-devices=interface-name:cali*;interface-name:tunl* - - path: /etc/selinux/config - owner: root:root - permissions: '0644' - content: | - SELINUX=permissive - SELINUXTYPE=targeted -bootcmd: - - [setenforce, Permissive] - - [systemctl, disable, firewalld, --now] - # https://github.com/kubernetes/kubernetes/issues/60869 - - [modprobe, ip_vs] -runcmd: - - [systemctl, daemon-reload] - - [systemctl, restart, NetworkManager] - - [hostnamectl, set-hostname, ${domain_name}] - - "atomic install --system --name=kubelet quay.io/poseidon/kubelet:v1.14.1" - - [systemctl, enable, kubelet.path] - - [systemctl, start, --no-block, kubelet.path] -users: - - default - - name: fedora - gecos: Fedora Admin - sudo: ALL=(ALL) NOPASSWD:ALL - groups: wheel,adm,systemd-journal,docker - ssh-authorized-keys: - - "${ssh_authorized_key}" diff --git a/bare-metal/fedora-atomic/kubernetes/groups.tf b/bare-metal/fedora-atomic/kubernetes/groups.tf deleted file mode 100644 index 200fd3b96..000000000 --- a/bare-metal/fedora-atomic/kubernetes/groups.tf +++ /dev/null @@ -1,37 +0,0 @@ -// Install Fedora to disk -resource "matchbox_group" "install" { - count = "${length(var.controller_names) + length(var.worker_names)}" - - name = "${format("fedora-install-%s", element(concat(var.controller_names, var.worker_names), count.index))}" - profile = "${element(matchbox_profile.cached-fedora-install.*.name, count.index)}" - - selector = { - mac = "${element(concat(var.controller_macs, var.worker_macs), count.index)}" - } - - metadata = { - ssh_authorized_key = "${var.ssh_authorized_key}" - } -} - -resource "matchbox_group" "controller" { - count = "${length(var.controller_names)}" - name = "${format("%s-%s", var.cluster_name, element(var.controller_names, count.index))}" - profile = "${element(matchbox_profile.controllers.*.name, count.index)}" - - selector = { - mac = "${element(var.controller_macs, count.index)}" - os = "installed" - } -} - -resource "matchbox_group" "worker" { - count = "${length(var.worker_names)}" - name = "${format("%s-%s", var.cluster_name, element(var.worker_names, count.index))}" - profile = "${element(matchbox_profile.workers.*.name, count.index)}" - - selector = { - mac = "${element(var.worker_macs, count.index)}" - os = "installed" - } -} diff --git a/bare-metal/fedora-atomic/kubernetes/kickstart/fedora-atomic.ks.tmpl b/bare-metal/fedora-atomic/kubernetes/kickstart/fedora-atomic.ks.tmpl deleted file mode 100644 index 66e80a7cc..000000000 --- a/bare-metal/fedora-atomic/kubernetes/kickstart/fedora-atomic.ks.tmpl +++ /dev/null @@ -1,36 +0,0 @@ -# required -lang en_US.UTF-8 -keyboard us -timezone --utc Etc/UTC - -# wipe disks -zerombr -clearpart --all --initlabel - -# locked root and temporary user -rootpw --lock --iscrypted locked -user --name=none - -# config -autopart --type=lvm --noswap -network --bootproto=dhcp --device=link --activate --onboot=on -bootloader --timeout=1 --append="ds=nocloud\;seedfrom=/var/cloud-init/" -services --enabled=cloud-init,cloud-init-local,cloud-config,cloud-final - -ostreesetup --osname="fedora-atomic" --remote="fedora-atomic" --url="${atomic_assets_endpoint}/repo" --ref=fedora/28/x86_64/atomic-host --nogpg - -reboot - -%post --erroronfail -mkdir /var/cloud-init -curl --retry 10 "${matchbox_http_endpoint}/generic?mac=${mac}&os=installed" -o /var/cloud-init/user-data -echo "instance-id: iid-local01" > /var/cloud-init/meta-data - -rm -f /etc/ostree/remotes.d/fedora-atomic.conf -ostree remote add fedora-atomic https://dl.fedoraproject.org/atomic/repo/ --set=gpgkeypath=/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-28-primary - -# lock root user -passwd -l root -# remove temporary user -userdel -r none -%end diff --git a/bare-metal/fedora-atomic/kubernetes/outputs.tf b/bare-metal/fedora-atomic/kubernetes/outputs.tf deleted file mode 100644 index a0977ea39..000000000 --- a/bare-metal/fedora-atomic/kubernetes/outputs.tf +++ /dev/null @@ -1,3 +0,0 @@ -output "kubeconfig-admin" { - value = "${module.bootkube.kubeconfig-admin}" -} diff --git a/bare-metal/fedora-atomic/kubernetes/profiles.tf b/bare-metal/fedora-atomic/kubernetes/profiles.tf deleted file mode 100644 index 1f8b1c5ea..000000000 --- a/bare-metal/fedora-atomic/kubernetes/profiles.tf +++ /dev/null @@ -1,87 +0,0 @@ -locals { - default_assets_endpoint = "${var.matchbox_http_endpoint}/assets/fedora/28" - atomic_assets_endpoint = "${var.atomic_assets_endpoint != "" ? var.atomic_assets_endpoint : local.default_assets_endpoint}" -} - -// Cached Fedora Install profile (from matchbox /assets cache) -// Note: Admin must have downloaded Fedora kernel, initrd, and repo into -// matchbox assets. -resource "matchbox_profile" "cached-fedora-install" { - count = "${length(var.controller_names) + length(var.worker_names)}" - name = "${format("%s-cached-fedora-install-%s", var.cluster_name, element(concat(var.controller_names, var.worker_names), count.index))}" - - kernel = "${local.atomic_assets_endpoint}/images/pxeboot/vmlinuz" - - initrd = [ - "${local.atomic_assets_endpoint}/images/pxeboot/initrd.img", - ] - - args = [ - "initrd=initrd.img", - "inst.repo=${local.atomic_assets_endpoint}", - "inst.ks=${var.matchbox_http_endpoint}/generic?mac=${element(concat(var.controller_macs, var.worker_macs), count.index)}", - "inst.text", - "${var.kernel_args}", - ] - - # kickstart - generic_config = "${element(data.template_file.install-kickstarts.*.rendered, count.index)}" -} - -data "template_file" "install-kickstarts" { - count = "${length(var.controller_names) + length(var.worker_names)}" - - template = "${file("${path.module}/kickstart/fedora-atomic.ks.tmpl")}" - - vars = { - matchbox_http_endpoint = "${var.matchbox_http_endpoint}" - atomic_assets_endpoint = "${local.atomic_assets_endpoint}" - mac = "${element(concat(var.controller_macs, var.worker_macs), count.index)}" - } -} - -// Kubernetes Controller profiles -resource "matchbox_profile" "controllers" { - count = "${length(var.controller_names)}" - name = "${format("%s-controller-%s", var.cluster_name, element(var.controller_names, count.index))}" - - # cloud-init - generic_config = "${element(data.template_file.controller-configs.*.rendered, count.index)}" -} - -data "template_file" "controller-configs" { - count = "${length(var.controller_names)}" - - template = "${file("${path.module}/cloudinit/controller.yaml.tmpl")}" - - vars = { - domain_name = "${element(var.controller_domains, count.index)}" - etcd_name = "${element(var.controller_names, count.index)}" - etcd_initial_cluster = "${join(",", formatlist("%s=https://%s:2380", var.controller_names, var.controller_domains))}" - cluster_dns_service_ip = "${module.bootkube.cluster_dns_service_ip}" - cluster_domain_suffix = "${var.cluster_domain_suffix}" - ssh_authorized_key = "${var.ssh_authorized_key}" - } -} - -// Kubernetes Worker profiles -resource "matchbox_profile" "workers" { - count = "${length(var.worker_names)}" - name = "${format("%s-worker-%s", var.cluster_name, element(var.worker_names, count.index))}" - - # cloud-init - generic_config = "${element(data.template_file.worker-configs.*.rendered, count.index)}" -} - -data "template_file" "worker-configs" { - count = "${length(var.worker_names)}" - - template = "${file("${path.module}/cloudinit/worker.yaml.tmpl")}" - - vars = { - domain_name = "${element(var.worker_domains, count.index)}" - cluster_dns_service_ip = "${module.bootkube.cluster_dns_service_ip}" - cluster_domain_suffix = "${var.cluster_domain_suffix}" - ssh_authorized_key = "${var.ssh_authorized_key}" - } -} diff --git a/bare-metal/fedora-atomic/kubernetes/require.tf b/bare-metal/fedora-atomic/kubernetes/require.tf deleted file mode 100644 index a6435bec6..000000000 --- a/bare-metal/fedora-atomic/kubernetes/require.tf +++ /dev/null @@ -1,21 +0,0 @@ -# Terraform version and plugin versions - -terraform { - required_version = ">= 0.11.0" -} - -provider "local" { - version = "~> 1.0" -} - -provider "null" { - version = "~> 1.0" -} - -provider "template" { - version = "~> 1.0" -} - -provider "tls" { - version = "~> 1.0" -} diff --git a/bare-metal/fedora-atomic/kubernetes/ssh.tf b/bare-metal/fedora-atomic/kubernetes/ssh.tf deleted file mode 100644 index 71ccf6e8e..000000000 --- a/bare-metal/fedora-atomic/kubernetes/ssh.tf +++ /dev/null @@ -1,136 +0,0 @@ -# Secure copy etcd TLS assets and kubeconfig to controllers. Activates kubelet.service -resource "null_resource" "copy-controller-secrets" { - count = "${length(var.controller_names)}" - - # Without depends_on, remote-exec could start and wait for machines before - # matchbox groups are written, causing a deadlock. - depends_on = [ - "matchbox_group.install", - "matchbox_group.controller", - "matchbox_group.worker", - ] - - connection { - type = "ssh" - host = "${element(var.controller_domains, count.index)}" - user = "fedora" - timeout = "60m" - } - - provisioner "file" { - content = "${module.bootkube.kubeconfig-kubelet}" - destination = "$HOME/kubeconfig" - } - - provisioner "file" { - content = "${module.bootkube.etcd_ca_cert}" - destination = "$HOME/etcd-client-ca.crt" - } - - provisioner "file" { - content = "${module.bootkube.etcd_client_cert}" - destination = "$HOME/etcd-client.crt" - } - - provisioner "file" { - content = "${module.bootkube.etcd_client_key}" - destination = "$HOME/etcd-client.key" - } - - provisioner "file" { - content = "${module.bootkube.etcd_server_cert}" - destination = "$HOME/etcd-server.crt" - } - - provisioner "file" { - content = "${module.bootkube.etcd_server_key}" - destination = "$HOME/etcd-server.key" - } - - provisioner "file" { - content = "${module.bootkube.etcd_peer_cert}" - destination = "$HOME/etcd-peer.crt" - } - - provisioner "file" { - content = "${module.bootkube.etcd_peer_key}" - destination = "$HOME/etcd-peer.key" - } - - provisioner "remote-exec" { - inline = [ - "sudo mkdir -p /etc/ssl/etcd/etcd", - "sudo mv etcd-client* /etc/ssl/etcd/", - "sudo cp /etc/ssl/etcd/etcd-client-ca.crt /etc/ssl/etcd/etcd/server-ca.crt", - "sudo mv etcd-server.crt /etc/ssl/etcd/etcd/server.crt", - "sudo mv etcd-server.key /etc/ssl/etcd/etcd/server.key", - "sudo cp /etc/ssl/etcd/etcd-client-ca.crt /etc/ssl/etcd/etcd/peer-ca.crt", - "sudo mv etcd-peer.crt /etc/ssl/etcd/etcd/peer.crt", - "sudo mv etcd-peer.key /etc/ssl/etcd/etcd/peer.key", - "sudo mv $HOME/kubeconfig /etc/kubernetes/kubeconfig", - ] - } -} - -# Secure copy kubeconfig to all workers. Activates kubelet.service -resource "null_resource" "copy-worker-secrets" { - count = "${length(var.worker_names)}" - - # Without depends_on, remote-exec could start and wait for machines before - # matchbox groups are written, causing a deadlock. - depends_on = [ - "matchbox_group.install", - "matchbox_group.controller", - "matchbox_group.worker", - ] - - connection { - type = "ssh" - host = "${element(var.worker_domains, count.index)}" - user = "fedora" - timeout = "60m" - } - - provisioner "file" { - content = "${module.bootkube.kubeconfig-kubelet}" - destination = "$HOME/kubeconfig" - } - - provisioner "remote-exec" { - inline = [ - "sudo mv $HOME/kubeconfig /etc/kubernetes/kubeconfig", - ] - } -} - -# Secure copy bootkube assets to ONE controller and start bootkube to perform -# one-time self-hosted cluster bootstrapping. -resource "null_resource" "bootkube-start" { - # Without depends_on, this remote-exec may start before the kubeconfig copy. - # Terraform only does one task at a time, so it would try to bootstrap - # while no Kubelets are running. - depends_on = [ - "null_resource.copy-controller-secrets", - "null_resource.copy-worker-secrets", - ] - - connection { - type = "ssh" - host = "${element(var.controller_domains, 0)}" - user = "fedora" - timeout = "15m" - } - - provisioner "file" { - source = "${var.asset_dir}" - destination = "$HOME/assets" - } - - provisioner "remote-exec" { - inline = [ - "while [ ! -f /var/lib/cloud/instance/boot-finished ]; do sleep 4; done", - "sudo mv $HOME/assets /var/lib/bootkube", - "sudo systemctl start bootkube", - ] - } -} diff --git a/bare-metal/fedora-atomic/kubernetes/variables.tf b/bare-metal/fedora-atomic/kubernetes/variables.tf deleted file mode 100644 index 141bddefc..000000000 --- a/bare-metal/fedora-atomic/kubernetes/variables.tf +++ /dev/null @@ -1,118 +0,0 @@ -variable "cluster_name" { - type = "string" - description = "Unique cluster name" -} - -# bare-metal - -variable "matchbox_http_endpoint" { - type = "string" - description = "Matchbox HTTP read-only endpoint (e.g. http://matchbox.example.com:8080)" -} - -variable "atomic_assets_endpoint" { - type = "string" - default = "" - - description = < - -Typhoon is a minimal and free Kubernetes distribution. - -* Minimal, stable base Kubernetes distribution -* Declarative infrastructure and configuration -* Free (freedom and cost) and privacy-respecting -* Practical for labs, datacenters, and clouds - -Typhoon distributes upstream Kubernetes, architectural conventions, and cluster addons, much like a GNU/Linux distribution provides the Linux kernel and userspace components. - -## Features - -* Kubernetes v1.15.0 (upstream, via [kubernetes-incubator/bootkube](https://github.com/kubernetes-incubator/bootkube)) -* Single or multi-master, [flannel](https://github.com/coreos/flannel) networking -* On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled -* Ready for Ingress, Prometheus, Grafana, and other optional [addons](https://typhoon.psdn.io/addons/overview/) - -## Docs - -Please see the [official docs](https://typhoon.psdn.io) and the Digital Ocean [tutorial](https://typhoon.psdn.io/cl/digital-ocean/). - diff --git a/digital-ocean/fedora-atomic/kubernetes/bootkube.tf b/digital-ocean/fedora-atomic/kubernetes/bootkube.tf deleted file mode 100644 index b8297cdd1..000000000 --- a/digital-ocean/fedora-atomic/kubernetes/bootkube.tf +++ /dev/null @@ -1,18 +0,0 @@ -# Self-hosted Kubernetes assets (kubeconfig, manifests) -module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=62df9ad69cc0da35f47d40fa981370c4503ad581" - - cluster_name = "${var.cluster_name}" - api_servers = ["${format("%s.%s", var.cluster_name, var.dns_zone)}"] - etcd_servers = "${digitalocean_record.etcds.*.fqdn}" - asset_dir = "${var.asset_dir}" - networking = "flannel" - network_mtu = 1440 - pod_cidr = "${var.pod_cidr}" - service_cidr = "${var.service_cidr}" - cluster_domain_suffix = "${var.cluster_domain_suffix}" - enable_reporting = "${var.enable_reporting}" - - # Fedora - trusted_certs_dir = "/etc/pki/tls/certs" -} diff --git a/digital-ocean/fedora-atomic/kubernetes/cloudinit/controller.yaml.tmpl b/digital-ocean/fedora-atomic/kubernetes/cloudinit/controller.yaml.tmpl deleted file mode 100644 index e03312815..000000000 --- a/digital-ocean/fedora-atomic/kubernetes/cloudinit/controller.yaml.tmpl +++ /dev/null @@ -1,107 +0,0 @@ -#cloud-config -write_files: - - path: /etc/etcd/etcd.conf - content: | - ETCD_NAME=${etcd_name} - ETCD_DATA_DIR=/var/lib/etcd - ETCD_ADVERTISE_CLIENT_URLS=https://${etcd_domain}:2379 - ETCD_INITIAL_ADVERTISE_PEER_URLS=https://${etcd_domain}:2380 - ETCD_LISTEN_CLIENT_URLS=https://0.0.0.0:2379 - ETCD_LISTEN_PEER_URLS=https://0.0.0.0:2380 - ETCD_LISTEN_METRICS_URLS=http://0.0.0.0:2381 - ETCD_INITIAL_CLUSTER=${etcd_initial_cluster} - ETCD_STRICT_RECONFIG_CHECK=true - ETCD_TRUSTED_CA_FILE=/etc/ssl/certs/etcd/server-ca.crt - ETCD_CERT_FILE=/etc/ssl/certs/etcd/server.crt - ETCD_KEY_FILE=/etc/ssl/certs/etcd/server.key - ETCD_CLIENT_CERT_AUTH=true - ETCD_PEER_TRUSTED_CA_FILE=/etc/ssl/certs/etcd/peer-ca.crt - ETCD_PEER_CERT_FILE=/etc/ssl/certs/etcd/peer.crt - ETCD_PEER_KEY_FILE=/etc/ssl/certs/etcd/peer.key - ETCD_PEER_CLIENT_CERT_AUTH=true - - path: /etc/systemd/system/cloud-metadata.service - content: | - [Unit] - Description=Cloud metadata agent - [Service] - Type=oneshot - Environment=OUTPUT=/run/metadata/cloud - ExecStart=/usr/bin/mkdir -p /run/metadata - ExecStart=/usr/bin/bash -c 'echo "HOSTNAME_OVERRIDE=$(curl\ - --url http://169.254.169.254/metadata/v1/interfaces/private/0/ipv4/address\ - --retry 10)" > $${OUTPUT}' - [Install] - WantedBy=multi-user.target - - path: /etc/systemd/system/kubelet.service.d/10-typhoon.conf - content: | - [Unit] - Requires=cloud-metadata.service - After=cloud-metadata.service - Wants=rpc-statd.service - [Service] - ExecStartPre=/bin/mkdir -p /opt/cni/bin - ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests - ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d - ExecStartPre=/bin/mkdir -p /etc/kubernetes/checkpoint-secrets - ExecStartPre=/bin/mkdir -p /etc/kubernetes/inactive-manifests - ExecStartPre=/bin/mkdir -p /var/lib/cni - ExecStartPre=/bin/mkdir -p /var/lib/kubelet/volumeplugins - ExecStartPre=/usr/bin/bash -c "grep 'certificate-authority-data' /etc/kubernetes/kubeconfig | awk '{print $2}' | base64 -d > /etc/kubernetes/ca.crt" - Restart=always - RestartSec=10 - - path: /etc/kubernetes/kubelet.conf - content: | - ARGS="--anonymous-auth=false \ - --authentication-token-webhook \ - --authorization-mode=Webhook \ - --client-ca-file=/etc/kubernetes/ca.crt \ - --cluster_dns=${cluster_dns_service_ip} \ - --cluster_domain=${cluster_domain_suffix} \ - --cni-conf-dir=/etc/kubernetes/cni/net.d \ - --exit-on-lock-contention \ - --kubeconfig=/etc/kubernetes/kubeconfig \ - --lock-file=/var/run/lock/kubelet.lock \ - --network-plugin=cni \ - --node-labels=node-role.kubernetes.io/master \ - --node-labels=node-role.kubernetes.io/controller="true" \ - --pod-manifest-path=/etc/kubernetes/manifests \ - --read-only-port=0 \ - --register-with-taints=node-role.kubernetes.io/master=:NoSchedule \ - --volume-plugin-dir=/var/lib/kubelet/volumeplugins" - - path: /etc/systemd/system/kubelet.path - content: | - [Unit] - Description=Watch for kubeconfig - [Path] - PathExists=/etc/kubernetes/kubeconfig - [Install] - WantedBy=multi-user.target - - path: /var/lib/bootkube/.keep - - path: /etc/selinux/config - owner: root:root - permissions: '0644' - content: | - SELINUX=permissive - SELINUXTYPE=targeted -bootcmd: - - [setenforce, Permissive] - - [systemctl, disable, firewalld, --now] - # https://github.com/kubernetes/kubernetes/issues/60869 - - [modprobe, ip_vs] -runcmd: - - [systemctl, daemon-reload] - - "atomic install --system --name=etcd quay.io/poseidon/etcd:v3.3.12" - - "atomic install --system --name=kubelet quay.io/poseidon/kubelet:v1.14.1" - - "atomic install --system --name=bootkube quay.io/poseidon/bootkube:v0.14.0" - - [systemctl, start, --no-block, etcd.service] - - [systemctl, enable, cloud-metadata.service] - - [systemctl, enable, kubelet.path] - - [systemctl, start, --no-block, kubelet.path] -users: - - default - - name: fedora - gecos: Fedora Admin - sudo: ALL=(ALL) NOPASSWD:ALL - groups: wheel,adm,systemd-journal,docker - ssh-authorized-keys: - - "${ssh_authorized_key}" diff --git a/digital-ocean/fedora-atomic/kubernetes/cloudinit/worker.yaml.tmpl b/digital-ocean/fedora-atomic/kubernetes/cloudinit/worker.yaml.tmpl deleted file mode 100644 index 0bc273bab..000000000 --- a/digital-ocean/fedora-atomic/kubernetes/cloudinit/worker.yaml.tmpl +++ /dev/null @@ -1,80 +0,0 @@ -#cloud-config -write_files: - - path: /etc/systemd/system/cloud-metadata.service - content: | - [Unit] - Description=Cloud metadata agent - [Service] - Type=oneshot - Environment=OUTPUT=/run/metadata/cloud - ExecStart=/usr/bin/mkdir -p /run/metadata - ExecStart=/usr/bin/bash -c 'echo "HOSTNAME_OVERRIDE=$(curl\ - --url http://169.254.169.254/metadata/v1/interfaces/private/0/ipv4/address\ - --retry 10)" > $${OUTPUT}' - [Install] - WantedBy=multi-user.target - - path: /etc/systemd/system/kubelet.service.d/10-typhoon.conf - content: | - [Unit] - Requires=cloud-metadata.service - After=cloud-metadata.service - Wants=rpc-statd.service - [Service] - ExecStartPre=/bin/mkdir -p /opt/cni/bin - ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests - ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d - ExecStartPre=/bin/mkdir -p /var/lib/cni - ExecStartPre=/bin/mkdir -p /var/lib/kubelet/volumeplugins - ExecStartPre=/usr/bin/bash -c "grep 'certificate-authority-data' /etc/kubernetes/kubeconfig | awk '{print $2}' | base64 -d > /etc/kubernetes/ca.crt" - Restart=always - RestartSec=10 - - path: /etc/kubernetes/kubelet.conf - content: | - ARGS="--anonymous-auth=false \ - --authentication-token-webhook \ - --authorization-mode=Webhook \ - --client-ca-file=/etc/kubernetes/ca.crt \ - --cluster_dns=${cluster_dns_service_ip} \ - --cluster_domain=${cluster_domain_suffix} \ - --cni-conf-dir=/etc/kubernetes/cni/net.d \ - --exit-on-lock-contention \ - --kubeconfig=/etc/kubernetes/kubeconfig \ - --lock-file=/var/run/lock/kubelet.lock \ - --network-plugin=cni \ - --node-labels=node-role.kubernetes.io/node \ - --pod-manifest-path=/etc/kubernetes/manifests \ - --read-only-port=0 \ - --volume-plugin-dir=/var/lib/kubelet/volumeplugins" - - path: /etc/systemd/system/kubelet.path - content: | - [Unit] - Description=Watch for kubeconfig - [Path] - PathExists=/etc/kubernetes/kubeconfig - [Install] - WantedBy=multi-user.target - - path: /etc/selinux/config - owner: root:root - permissions: '0644' - content: | - SELINUX=permissive - SELINUXTYPE=targeted -bootcmd: - - [setenforce, Permissive] - - [systemctl, disable, firewalld, --now] - # https://github.com/kubernetes/kubernetes/issues/60869 - - [modprobe, ip_vs] -runcmd: - - [systemctl, daemon-reload] - - [systemctl, enable, cloud-metadata.service] - - "atomic install --system --name=kubelet quay.io/poseidon/kubelet:v1.14.1" - - [systemctl, enable, kubelet.path] - - [systemctl, start, --no-block, kubelet.path] -users: - - default - - name: fedora - gecos: Fedora Admin - sudo: ALL=(ALL) NOPASSWD:ALL - groups: wheel,adm,systemd-journal,docker - ssh-authorized-keys: - - "${ssh_authorized_key}" diff --git a/digital-ocean/fedora-atomic/kubernetes/controllers.tf b/digital-ocean/fedora-atomic/kubernetes/controllers.tf deleted file mode 100644 index deaa87b17..000000000 --- a/digital-ocean/fedora-atomic/kubernetes/controllers.tf +++ /dev/null @@ -1,95 +0,0 @@ -# Controller Instance DNS records -resource "digitalocean_record" "controllers" { - count = "${var.controller_count}" - - # DNS zone where record should be created - domain = "${var.dns_zone}" - - # DNS record (will be prepended to domain) - name = "${var.cluster_name}" - type = "A" - ttl = 300 - - # IPv4 addresses of controllers - value = "${element(digitalocean_droplet.controllers.*.ipv4_address, count.index)}" -} - -# Discrete DNS records for each controller's private IPv4 for etcd usage -resource "digitalocean_record" "etcds" { - count = "${var.controller_count}" - - # DNS zone where record should be created - domain = "${var.dns_zone}" - - # DNS record (will be prepended to domain) - name = "${var.cluster_name}-etcd${count.index}" - type = "A" - ttl = 300 - - # private IPv4 address for etcd - value = "${element(digitalocean_droplet.controllers.*.ipv4_address_private, count.index)}" -} - -# Controller droplet instances -resource "digitalocean_droplet" "controllers" { - count = "${var.controller_count}" - - name = "${var.cluster_name}-controller-${count.index}" - region = "${var.region}" - - image = "${var.image}" - size = "${var.controller_type}" - - # network - ipv6 = true - private_networking = true - - user_data = "${element(data.template_file.controller-cloudinit.*.rendered, count.index)}" - ssh_keys = ["${var.ssh_fingerprints}"] - - tags = [ - "${digitalocean_tag.controllers.id}", - ] - - lifecycle { - ignore_changes = [ - "user_data", - ] - } -} - -# Tag to label controllers -resource "digitalocean_tag" "controllers" { - name = "${var.cluster_name}-controller" -} - -# Controller Cloud-Init -data "template_file" "controller-cloudinit" { - count = "${var.controller_count}" - - template = "${file("${path.module}/cloudinit/controller.yaml.tmpl")}" - - vars = { - # Cannot use cyclic dependencies on controllers or their DNS records - etcd_name = "etcd${count.index}" - etcd_domain = "${var.cluster_name}-etcd${count.index}.${var.dns_zone}" - - # etcd0=https://cluster-etcd0.example.com,etcd1=https://cluster-etcd1.example.com,... - etcd_initial_cluster = "${join(",", data.template_file.etcds.*.rendered)}" - - ssh_authorized_key = "${var.ssh_authorized_key}" - cluster_dns_service_ip = "${cidrhost(var.service_cidr, 10)}" - cluster_domain_suffix = "${var.cluster_domain_suffix}" - } -} - -data "template_file" "etcds" { - count = "${var.controller_count}" - template = "etcd$${index}=https://$${cluster_name}-etcd$${index}.$${dns_zone}:2380" - - vars = { - index = "${count.index}" - cluster_name = "${var.cluster_name}" - dns_zone = "${var.dns_zone}" - } -} diff --git a/digital-ocean/fedora-atomic/kubernetes/network.tf b/digital-ocean/fedora-atomic/kubernetes/network.tf deleted file mode 100644 index 312d7966e..000000000 --- a/digital-ocean/fedora-atomic/kubernetes/network.tf +++ /dev/null @@ -1,58 +0,0 @@ -resource "digitalocean_firewall" "rules" { - name = "${var.cluster_name}" - - tags = ["${var.cluster_name}-controller", "${var.cluster_name}-worker"] - - # allow ssh, apiserver, http/https ingress, and peer-to-peer traffic - inbound_rule = [ - { - protocol = "tcp" - port_range = "22" - source_addresses = ["0.0.0.0/0", "::/0"] - }, - { - protocol = "tcp" - port_range = "80" - source_addresses = ["0.0.0.0/0", "::/0"] - }, - { - protocol = "tcp" - port_range = "443" - source_addresses = ["0.0.0.0/0", "::/0"] - }, - { - protocol = "tcp" - port_range = "6443" - source_addresses = ["0.0.0.0/0", "::/0"] - }, - { - protocol = "udp" - port_range = "1-65535" - source_tags = ["${digitalocean_tag.controllers.name}", "${digitalocean_tag.workers.name}"] - }, - { - protocol = "tcp" - port_range = "1-65535" - source_tags = ["${digitalocean_tag.controllers.name}", "${digitalocean_tag.workers.name}"] - }, - ] - - # allow all outbound traffic - outbound_rule = [ - { - protocol = "tcp" - port_range = "1-65535" - destination_addresses = ["0.0.0.0/0", "::/0"] - }, - { - protocol = "udp" - port_range = "1-65535" - destination_addresses = ["0.0.0.0/0", "::/0"] - }, - { - protocol = "icmp" - port_range = "1-65535" - destination_addresses = ["0.0.0.0/0", "::/0"] - }, - ] -} diff --git a/digital-ocean/fedora-atomic/kubernetes/outputs.tf b/digital-ocean/fedora-atomic/kubernetes/outputs.tf deleted file mode 100644 index 86a4c76ac..000000000 --- a/digital-ocean/fedora-atomic/kubernetes/outputs.tf +++ /dev/null @@ -1,28 +0,0 @@ -output "kubeconfig-admin" { - value = "${module.bootkube.kubeconfig-admin}" -} - -output "controllers_dns" { - value = "${digitalocean_record.controllers.0.fqdn}" -} - -output "workers_dns" { - # Multiple A and AAAA records with the same FQDN - value = "${digitalocean_record.workers-record-a.0.fqdn}" -} - -output "controllers_ipv4" { - value = ["${digitalocean_droplet.controllers.*.ipv4_address}"] -} - -output "controllers_ipv6" { - value = ["${digitalocean_droplet.controllers.*.ipv6_address}"] -} - -output "workers_ipv4" { - value = ["${digitalocean_droplet.workers.*.ipv4_address}"] -} - -output "workers_ipv6" { - value = ["${digitalocean_droplet.workers.*.ipv6_address}"] -} diff --git a/digital-ocean/fedora-atomic/kubernetes/require.tf b/digital-ocean/fedora-atomic/kubernetes/require.tf deleted file mode 100644 index 4651337d7..000000000 --- a/digital-ocean/fedora-atomic/kubernetes/require.tf +++ /dev/null @@ -1,25 +0,0 @@ -# Terraform version and plugin versions - -terraform { - required_version = ">= 0.11.0" -} - -provider "digitalocean" { - version = "~> 1.0" -} - -provider "local" { - version = "~> 1.0" -} - -provider "null" { - version = "~> 1.0" -} - -provider "template" { - version = "~> 1.0" -} - -provider "tls" { - version = "~> 1.0" -} diff --git a/digital-ocean/fedora-atomic/kubernetes/ssh.tf b/digital-ocean/fedora-atomic/kubernetes/ssh.tf deleted file mode 100644 index 77ae4abaf..000000000 --- a/digital-ocean/fedora-atomic/kubernetes/ssh.tf +++ /dev/null @@ -1,121 +0,0 @@ -# Secure copy etcd TLS assets and kubeconfig to controllers. Activates kubelet.service -resource "null_resource" "copy-controller-secrets" { - count = "${var.controller_count}" - - depends_on = [ - "digitalocean_firewall.rules", - ] - - connection { - type = "ssh" - host = "${element(concat(digitalocean_droplet.controllers.*.ipv4_address), count.index)}" - user = "fedora" - timeout = "15m" - } - - provisioner "file" { - content = "${module.bootkube.kubeconfig-kubelet}" - destination = "$HOME/kubeconfig" - } - - provisioner "file" { - content = "${module.bootkube.etcd_ca_cert}" - destination = "$HOME/etcd-client-ca.crt" - } - - provisioner "file" { - content = "${module.bootkube.etcd_client_cert}" - destination = "$HOME/etcd-client.crt" - } - - provisioner "file" { - content = "${module.bootkube.etcd_client_key}" - destination = "$HOME/etcd-client.key" - } - - provisioner "file" { - content = "${module.bootkube.etcd_server_cert}" - destination = "$HOME/etcd-server.crt" - } - - provisioner "file" { - content = "${module.bootkube.etcd_server_key}" - destination = "$HOME/etcd-server.key" - } - - provisioner "file" { - content = "${module.bootkube.etcd_peer_cert}" - destination = "$HOME/etcd-peer.crt" - } - - provisioner "file" { - content = "${module.bootkube.etcd_peer_key}" - destination = "$HOME/etcd-peer.key" - } - - provisioner "remote-exec" { - inline = [ - "sudo mkdir -p /etc/ssl/etcd/etcd", - "sudo mv etcd-client* /etc/ssl/etcd/", - "sudo cp /etc/ssl/etcd/etcd-client-ca.crt /etc/ssl/etcd/etcd/server-ca.crt", - "sudo mv etcd-server.crt /etc/ssl/etcd/etcd/server.crt", - "sudo mv etcd-server.key /etc/ssl/etcd/etcd/server.key", - "sudo cp /etc/ssl/etcd/etcd-client-ca.crt /etc/ssl/etcd/etcd/peer-ca.crt", - "sudo mv etcd-peer.crt /etc/ssl/etcd/etcd/peer.crt", - "sudo mv etcd-peer.key /etc/ssl/etcd/etcd/peer.key", - "sudo mv $HOME/kubeconfig /etc/kubernetes/kubeconfig", - ] - } -} - -# Secure copy kubeconfig to all workers. Activates kubelet.service. -resource "null_resource" "copy-worker-secrets" { - count = "${var.worker_count}" - - connection { - type = "ssh" - host = "${element(concat(digitalocean_droplet.workers.*.ipv4_address), count.index)}" - user = "fedora" - timeout = "15m" - } - - provisioner "file" { - content = "${module.bootkube.kubeconfig-kubelet}" - destination = "$HOME/kubeconfig" - } - - provisioner "remote-exec" { - inline = [ - "sudo mv $HOME/kubeconfig /etc/kubernetes/kubeconfig", - ] - } -} - -# Secure copy bootkube assets to ONE controller and start bootkube to perform -# one-time self-hosted cluster bootstrapping. -resource "null_resource" "bootkube-start" { - depends_on = [ - "null_resource.copy-controller-secrets", - "null_resource.copy-worker-secrets", - ] - - connection { - type = "ssh" - host = "${digitalocean_droplet.controllers.0.ipv4_address}" - user = "fedora" - timeout = "15m" - } - - provisioner "file" { - source = "${var.asset_dir}" - destination = "$HOME/assets" - } - - provisioner "remote-exec" { - inline = [ - "while [ ! -f /var/lib/cloud/instance/boot-finished ]; do sleep 4; done", - "sudo mv $HOME/assets /var/lib/bootkube", - "sudo systemctl start bootkube", - ] - } -} diff --git a/digital-ocean/fedora-atomic/kubernetes/variables.tf b/digital-ocean/fedora-atomic/kubernetes/variables.tf deleted file mode 100644 index accbb0968..000000000 --- a/digital-ocean/fedora-atomic/kubernetes/variables.tf +++ /dev/null @@ -1,93 +0,0 @@ -variable "cluster_name" { - type = "string" - description = "Unique cluster name (prepended to dns_zone)" -} - -# Digital Ocean - -variable "region" { - type = "string" - description = "Digital Ocean region (e.g. nyc1, sfo2, fra1, tor1)" -} - -variable "dns_zone" { - type = "string" - description = "Digital Ocean domain (i.e. DNS zone) (e.g. do.example.com)" -} - -# instances - -variable "controller_count" { - type = "string" - default = "1" - description = "Number of controllers (i.e. masters)" -} - -variable "worker_count" { - type = "string" - default = "1" - description = "Number of workers" -} - -variable "controller_type" { - type = "string" - default = "s-2vcpu-2gb" - description = "Droplet type for controllers (e.g. s-2vcpu-2gb, s-2vcpu-4gb, s-4vcpu-8gb)" -} - -variable "worker_type" { - type = "string" - default = "s-1vcpu-1gb" - description = "Droplet type for workers (e.g. s-1vcpu-1gb, s-1vcpu-2gb, s-2vcpu-2gb)" -} - -variable "image" { - type = "string" - default = "fedora-28-x64-atomic" - description = "OS image from which to initialize the disk (e.g. fedora-28-x64-atomic)" -} - -# configuration - -variable "ssh_authorized_key" { - type = "string" - description = "SSH public key for user 'fedora'" -} - -variable "ssh_fingerprints" { - type = "list" - description = "SSH public key fingerprints. (e.g. see `ssh-add -l -E md5`)" -} - -variable "asset_dir" { - description = "Path to a directory where generated assets should be placed (contains secrets)" - type = "string" -} - -variable "pod_cidr" { - description = "CIDR IPv4 range to assign Kubernetes pods" - type = "string" - default = "10.2.0.0/16" -} - -variable "service_cidr" { - description = < - -Typhoon is a minimal and free Kubernetes distribution. - -* Minimal, stable base Kubernetes distribution -* Declarative infrastructure and configuration -* Free (freedom and cost) and privacy-respecting -* Practical for labs, datacenters, and clouds - -Typhoon distributes upstream Kubernetes, architectural conventions, and cluster addons, much like a GNU/Linux distribution provides the Linux kernel and userspace components. - -## Features - -* Kubernetes v1.15.0 (upstream, via [kubernetes-incubator/bootkube](https://github.com/kubernetes-incubator/bootkube)) -* Single or multi-master, [Calico](https://www.projectcalico.org/) or [flannel](https://github.com/coreos/flannel) networking -* On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/) -* Advanced features like [worker pools](https://typhoon.psdn.io/advanced/worker-pools/) and [preemptible](https://typhoon.psdn.io/cl/google-cloud/#preemption) workers -* Ready for Ingress, Prometheus, Grafana, and other optional [addons](https://typhoon.psdn.io/addons/overview/) - -## Docs - -Please see the [official docs](https://typhoon.psdn.io) and the Google Cloud [tutorial](https://typhoon.psdn.io/cl/google-cloud/). - diff --git a/google-cloud/fedora-atomic/kubernetes/apiserver.tf b/google-cloud/fedora-atomic/kubernetes/apiserver.tf deleted file mode 100644 index b56a4e7b4..000000000 --- a/google-cloud/fedora-atomic/kubernetes/apiserver.tf +++ /dev/null @@ -1,97 +0,0 @@ -# TCP Proxy load balancer DNS record -resource "google_dns_record_set" "apiserver" { - # DNS Zone name where record should be created - managed_zone = "${var.dns_zone_name}" - - # DNS record - name = "${format("%s.%s.", var.cluster_name, var.dns_zone)}" - type = "A" - ttl = 300 - - # IPv4 address of apiserver TCP Proxy load balancer - rrdatas = ["${google_compute_global_address.apiserver-ipv4.address}"] -} - -# Static IPv4 address for the TCP Proxy Load Balancer -resource "google_compute_global_address" "apiserver-ipv4" { - name = "${var.cluster_name}-apiserver-ip" - ip_version = "IPV4" -} - -# Forward IPv4 TCP traffic to the TCP proxy load balancer -resource "google_compute_global_forwarding_rule" "apiserver" { - name = "${var.cluster_name}-apiserver" - ip_address = "${google_compute_global_address.apiserver-ipv4.address}" - ip_protocol = "TCP" - port_range = "443" - target = "${google_compute_target_tcp_proxy.apiserver.self_link}" -} - -# TCP Proxy Load Balancer for apiservers -resource "google_compute_target_tcp_proxy" "apiserver" { - name = "${var.cluster_name}-apiserver" - description = "Distribute TCP load across ${var.cluster_name} controllers" - backend_service = "${google_compute_backend_service.apiserver.self_link}" -} - -# Backend service backed by unmanaged instance groups -resource "google_compute_backend_service" "apiserver" { - name = "${var.cluster_name}-apiserver" - description = "${var.cluster_name} apiserver service" - - protocol = "TCP" - port_name = "apiserver" - session_affinity = "NONE" - timeout_sec = "300" - - # controller(s) spread across zonal instance groups - backend { - group = "${google_compute_instance_group.controllers.0.self_link}" - } - - backend { - group = "${google_compute_instance_group.controllers.1.self_link}" - } - - backend { - group = "${google_compute_instance_group.controllers.2.self_link}" - } - - health_checks = ["${google_compute_health_check.apiserver.self_link}"] -} - -# Instance group of heterogeneous (unmanged) controller instances -resource "google_compute_instance_group" "controllers" { - count = "${length(local.zones)}" - - name = "${format("%s-controllers-%s", var.cluster_name, element(local.zones, count.index))}" - zone = "${element(local.zones, count.index)}" - - named_port { - name = "apiserver" - port = "443" - } - - # add instances in the zone into the instance group - instances = [ - "${matchkeys(google_compute_instance.controllers.*.self_link, - google_compute_instance.controllers.*.zone, - list(element(local.zones, count.index)))}", - ] -} - -# TCP health check for apiserver -resource "google_compute_health_check" "apiserver" { - name = "${var.cluster_name}-apiserver-tcp-health" - description = "TCP health check for kube-apiserver" - - timeout_sec = 5 - check_interval_sec = 5 - - healthy_threshold = 1 - unhealthy_threshold = 3 - - tcp_health_check { - port = "443" - } -} diff --git a/google-cloud/fedora-atomic/kubernetes/bootkube.tf b/google-cloud/fedora-atomic/kubernetes/bootkube.tf deleted file mode 100644 index cbd1f104a..000000000 --- a/google-cloud/fedora-atomic/kubernetes/bootkube.tf +++ /dev/null @@ -1,21 +0,0 @@ -# Self-hosted Kubernetes assets (kubeconfig, manifests) -module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=62df9ad69cc0da35f47d40fa981370c4503ad581" - - cluster_name = "${var.cluster_name}" - api_servers = ["${format("%s.%s", var.cluster_name, var.dns_zone)}"] - etcd_servers = ["${google_dns_record_set.etcds.*.name}"] - asset_dir = "${var.asset_dir}" - networking = "${var.networking}" - network_mtu = 1440 - pod_cidr = "${var.pod_cidr}" - service_cidr = "${var.service_cidr}" - cluster_domain_suffix = "${var.cluster_domain_suffix}" - enable_reporting = "${var.enable_reporting}" - - # Fedora - trusted_certs_dir = "/etc/pki/tls/certs" - - // temporary - apiserver_port = 443 -} diff --git a/google-cloud/fedora-atomic/kubernetes/cloudinit/controller.yaml.tmpl b/google-cloud/fedora-atomic/kubernetes/cloudinit/controller.yaml.tmpl deleted file mode 100644 index 64c1058e7..000000000 --- a/google-cloud/fedora-atomic/kubernetes/cloudinit/controller.yaml.tmpl +++ /dev/null @@ -1,93 +0,0 @@ -#cloud-config -write_files: - - path: /etc/etcd/etcd.conf - content: | - ETCD_NAME=${etcd_name} - ETCD_DATA_DIR=/var/lib/etcd - ETCD_ADVERTISE_CLIENT_URLS=https://${etcd_domain}:2379 - ETCD_INITIAL_ADVERTISE_PEER_URLS=https://${etcd_domain}:2380 - ETCD_LISTEN_CLIENT_URLS=https://0.0.0.0:2379 - ETCD_LISTEN_PEER_URLS=https://0.0.0.0:2380 - ETCD_LISTEN_METRICS_URLS=http://0.0.0.0:2381 - ETCD_INITIAL_CLUSTER=${etcd_initial_cluster} - ETCD_STRICT_RECONFIG_CHECK=true - ETCD_TRUSTED_CA_FILE=/etc/ssl/certs/etcd/server-ca.crt - ETCD_CERT_FILE=/etc/ssl/certs/etcd/server.crt - ETCD_KEY_FILE=/etc/ssl/certs/etcd/server.key - ETCD_CLIENT_CERT_AUTH=true - ETCD_PEER_TRUSTED_CA_FILE=/etc/ssl/certs/etcd/peer-ca.crt - ETCD_PEER_CERT_FILE=/etc/ssl/certs/etcd/peer.crt - ETCD_PEER_KEY_FILE=/etc/ssl/certs/etcd/peer.key - ETCD_PEER_CLIENT_CERT_AUTH=true - - path: /etc/systemd/system/kubelet.service.d/10-typhoon.conf - content: | - [Unit] - Wants=rpc-statd.service - [Service] - ExecStartPre=/bin/mkdir -p /opt/cni/bin - ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests - ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d - ExecStartPre=/bin/mkdir -p /etc/kubernetes/checkpoint-secrets - ExecStartPre=/bin/mkdir -p /etc/kubernetes/inactive-manifests - ExecStartPre=/bin/mkdir -p /var/lib/cni - ExecStartPre=/bin/mkdir -p /var/lib/kubelet/volumeplugins - ExecStartPre=/usr/bin/bash -c "grep 'certificate-authority-data' /etc/kubernetes/kubeconfig | awk '{print $2}' | base64 -d > /etc/kubernetes/ca.crt" - Restart=always - RestartSec=10 - - path: /etc/kubernetes/kubelet.conf - content: | - ARGS="--anonymous-auth=false \ - --authentication-token-webhook \ - --authorization-mode=Webhook \ - --client-ca-file=/etc/kubernetes/ca.crt \ - --cluster_dns=${cluster_dns_service_ip} \ - --cluster_domain=${cluster_domain_suffix} \ - --cni-conf-dir=/etc/kubernetes/cni/net.d \ - --exit-on-lock-contention \ - --kubeconfig=/etc/kubernetes/kubeconfig \ - --lock-file=/var/run/lock/kubelet.lock \ - --network-plugin=cni \ - --node-labels=node-role.kubernetes.io/master \ - --node-labels=node-role.kubernetes.io/controller="true" \ - --pod-manifest-path=/etc/kubernetes/manifests \ - --read-only-port=0 \ - --register-with-taints=node-role.kubernetes.io/master=:NoSchedule \ - --volume-plugin-dir=/var/lib/kubelet/volumeplugins" - - path: /etc/kubernetes/kubeconfig - permissions: '0644' - content: | - ${kubeconfig} - - path: /var/lib/bootkube/.keep - - path: /etc/NetworkManager/conf.d/typhoon.conf - content: | - [main] - plugins=keyfile - [keyfile] - unmanaged-devices=interface-name:cali*;interface-name:tunl* - - path: /etc/selinux/config - owner: root:root - permissions: '0644' - content: | - SELINUX=permissive - SELINUXTYPE=targeted -bootcmd: - - [setenforce, Permissive] - - [systemctl, disable, firewalld, --now] - # https://github.com/kubernetes/kubernetes/issues/60869 - - [modprobe, ip_vs] -runcmd: - - [systemctl, daemon-reload] - - [systemctl, restart, NetworkManager] - - "atomic install --system --name=etcd quay.io/poseidon/etcd:v3.3.12" - - "atomic install --system --name=kubelet quay.io/poseidon/kubelet:v1.14.1" - - "atomic install --system --name=bootkube quay.io/poseidon/bootkube:v0.14.0" - - [systemctl, start, --no-block, etcd.service] - - [systemctl, start, --no-block, kubelet.service] -users: - - default - - name: fedora - gecos: Fedora Admin - sudo: ALL=(ALL) NOPASSWD:ALL - groups: wheel,adm,systemd-journal,docker - ssh-authorized-keys: - - "${ssh_authorized_key}" diff --git a/google-cloud/fedora-atomic/kubernetes/controllers.tf b/google-cloud/fedora-atomic/kubernetes/controllers.tf deleted file mode 100644 index 5ccba436b..000000000 --- a/google-cloud/fedora-atomic/kubernetes/controllers.tf +++ /dev/null @@ -1,98 +0,0 @@ -# Discrete DNS records for each controller's private IPv4 for etcd usage -resource "google_dns_record_set" "etcds" { - count = "${var.controller_count}" - - # DNS Zone name where record should be created - managed_zone = "${var.dns_zone_name}" - - # DNS record - name = "${format("%s-etcd%d.%s.", var.cluster_name, count.index, var.dns_zone)}" - type = "A" - ttl = 300 - - # private IPv4 address for etcd - rrdatas = ["${element(google_compute_instance.controllers.*.network_interface.0.network_ip, count.index)}"] -} - -# Zones in the region -data "google_compute_zones" "all" { - region = "${var.region}" -} - -locals { - # TCP proxy load balancers require a fixed number of zonal backends. Spread - # controllers over up to 3 zones, since all GCP regions have at least 3. - zones = "${slice(data.google_compute_zones.all.names, 0, 3)}" - - controllers_ipv4_public = ["${google_compute_instance.controllers.*.network_interface.0.access_config.0.nat_ip}"] -} - -# Controller instances -resource "google_compute_instance" "controllers" { - count = "${var.controller_count}" - - name = "${var.cluster_name}-controller-${count.index}" - zone = "${element(local.zones, count.index)}" - machine_type = "${var.controller_type}" - - metadata = { - user-data = "${element(data.template_file.controller-cloudinit.*.rendered, count.index)}" - } - - boot_disk { - auto_delete = true - - initialize_params { - image = "${var.os_image}" - size = "${var.disk_size}" - } - } - - network_interface { - network = "${google_compute_network.network.name}" - - # Ephemeral external IP - access_config = {} - } - - can_ip_forward = true - tags = ["${var.cluster_name}-controller"] - - lifecycle { - ignore_changes = [ - "metadata", - ] - } -} - -# Controller Cloud-Init -data "template_file" "controller-cloudinit" { - count = "${var.controller_count}" - - template = "${file("${path.module}/cloudinit/controller.yaml.tmpl")}" - - vars = { - # Cannot use cyclic dependencies on controllers or their DNS records - etcd_name = "etcd${count.index}" - etcd_domain = "${var.cluster_name}-etcd${count.index}.${var.dns_zone}" - - # etcd0=https://cluster-etcd0.example.com,etcd1=https://cluster-etcd1.example.com,... - etcd_initial_cluster = "${join(",", data.template_file.etcds.*.rendered)}" - - kubeconfig = "${indent(6, module.bootkube.kubeconfig-kubelet)}" - ssh_authorized_key = "${var.ssh_authorized_key}" - cluster_dns_service_ip = "${cidrhost(var.service_cidr, 10)}" - cluster_domain_suffix = "${var.cluster_domain_suffix}" - } -} - -data "template_file" "etcds" { - count = "${var.controller_count}" - template = "etcd$${index}=https://$${cluster_name}-etcd$${index}.$${dns_zone}:2380" - - vars = { - index = "${count.index}" - cluster_name = "${var.cluster_name}" - dns_zone = "${var.dns_zone}" - } -} diff --git a/google-cloud/fedora-atomic/kubernetes/ingress.tf b/google-cloud/fedora-atomic/kubernetes/ingress.tf deleted file mode 100644 index de84765e3..000000000 --- a/google-cloud/fedora-atomic/kubernetes/ingress.tf +++ /dev/null @@ -1,122 +0,0 @@ -# Static IPv4 address for Ingress Load Balancing -resource "google_compute_global_address" "ingress-ipv4" { - name = "${var.cluster_name}-ingress-ipv4" - ip_version = "IPV4" -} - -# Static IPv6 address for Ingress Load Balancing -resource "google_compute_global_address" "ingress-ipv6" { - name = "${var.cluster_name}-ingress-ipv6" - ip_version = "IPV6" -} - -# Forward IPv4 TCP traffic to the HTTP proxy load balancer -# Google Cloud does not allow TCP proxies for port 80. Must use HTTP proxy. -resource "google_compute_global_forwarding_rule" "ingress-http-ipv4" { - name = "${var.cluster_name}-ingress-http-ipv4" - ip_address = "${google_compute_global_address.ingress-ipv4.address}" - ip_protocol = "TCP" - port_range = "80" - target = "${google_compute_target_http_proxy.ingress-http.self_link}" -} - -# Forward IPv4 TCP traffic to the TCP proxy load balancer -resource "google_compute_global_forwarding_rule" "ingress-https-ipv4" { - name = "${var.cluster_name}-ingress-https-ipv4" - ip_address = "${google_compute_global_address.ingress-ipv4.address}" - ip_protocol = "TCP" - port_range = "443" - target = "${google_compute_target_tcp_proxy.ingress-https.self_link}" -} - -# Forward IPv6 TCP traffic to the HTTP proxy load balancer -# Google Cloud does not allow TCP proxies for port 80. Must use HTTP proxy. -resource "google_compute_global_forwarding_rule" "ingress-http-ipv6" { - name = "${var.cluster_name}-ingress-http-ipv6" - ip_address = "${google_compute_global_address.ingress-ipv6.address}" - ip_protocol = "TCP" - port_range = "80" - target = "${google_compute_target_http_proxy.ingress-http.self_link}" -} - -# Forward IPv6 TCP traffic to the TCP proxy load balancer -resource "google_compute_global_forwarding_rule" "ingress-https-ipv6" { - name = "${var.cluster_name}-ingress-https-ipv6" - ip_address = "${google_compute_global_address.ingress-ipv6.address}" - ip_protocol = "TCP" - port_range = "443" - target = "${google_compute_target_tcp_proxy.ingress-https.self_link}" -} - -# HTTP proxy load balancer for ingress controllers -resource "google_compute_target_http_proxy" "ingress-http" { - name = "${var.cluster_name}-ingress-http" - description = "Distribute HTTP load across ${var.cluster_name} workers" - url_map = "${google_compute_url_map.ingress-http.self_link}" -} - -# TCP proxy load balancer for ingress controllers -resource "google_compute_target_tcp_proxy" "ingress-https" { - name = "${var.cluster_name}-ingress-https" - description = "Distribute HTTPS load across ${var.cluster_name} workers" - backend_service = "${google_compute_backend_service.ingress-https.self_link}" -} - -# HTTP URL Map (required) -resource "google_compute_url_map" "ingress-http" { - name = "${var.cluster_name}-ingress-http" - - # Do not add host/path rules for applications here. Use Ingress resources. - default_service = "${google_compute_backend_service.ingress-http.self_link}" -} - -# Backend service backed by managed instance group of workers -resource "google_compute_backend_service" "ingress-http" { - name = "${var.cluster_name}-ingress-http" - description = "${var.cluster_name} ingress service" - - protocol = "HTTP" - port_name = "http" - session_affinity = "NONE" - timeout_sec = "60" - - backend { - group = "${module.workers.instance_group}" - } - - health_checks = ["${google_compute_health_check.ingress.self_link}"] -} - -# Backend service backed by managed instance group of workers -resource "google_compute_backend_service" "ingress-https" { - name = "${var.cluster_name}-ingress-https" - description = "${var.cluster_name} ingress service" - - protocol = "TCP" - port_name = "https" - session_affinity = "NONE" - timeout_sec = "60" - - backend { - group = "${module.workers.instance_group}" - } - - health_checks = ["${google_compute_health_check.ingress.self_link}"] -} - -# Ingress HTTP Health Check -resource "google_compute_health_check" "ingress" { - name = "${var.cluster_name}-ingress-health" - description = "Health check for Ingress controller" - - timeout_sec = 5 - check_interval_sec = 5 - - healthy_threshold = 2 - unhealthy_threshold = 4 - - http_health_check { - port = 10254 - request_path = "/healthz" - } -} diff --git a/google-cloud/fedora-atomic/kubernetes/network.tf b/google-cloud/fedora-atomic/kubernetes/network.tf deleted file mode 100644 index ecfe7ae7b..000000000 --- a/google-cloud/fedora-atomic/kubernetes/network.tf +++ /dev/null @@ -1,153 +0,0 @@ -resource "google_compute_network" "network" { - name = "${var.cluster_name}" - description = "Network for the ${var.cluster_name} cluster" - auto_create_subnetworks = true -} - -resource "google_compute_firewall" "allow-ssh" { - name = "${var.cluster_name}-allow-ssh" - network = "${google_compute_network.network.name}" - - allow { - protocol = "tcp" - ports = [22] - } - - source_ranges = ["0.0.0.0/0"] - target_tags = ["${var.cluster_name}-controller", "${var.cluster_name}-worker"] -} - -resource "google_compute_firewall" "internal-etcd" { - name = "${var.cluster_name}-internal-etcd" - network = "${google_compute_network.network.name}" - - allow { - protocol = "tcp" - ports = [2379, 2380] - } - - source_tags = ["${var.cluster_name}-controller"] - target_tags = ["${var.cluster_name}-controller"] -} - -# Allow Prometheus to scrape etcd metrics -resource "google_compute_firewall" "internal-etcd-metrics" { - name = "${var.cluster_name}-internal-etcd-metrics" - network = "${google_compute_network.network.name}" - - allow { - protocol = "tcp" - ports = [2381] - } - - source_tags = ["${var.cluster_name}-worker"] - target_tags = ["${var.cluster_name}-controller"] -} - -resource "google_compute_firewall" "allow-apiserver" { - name = "${var.cluster_name}-allow-apiserver" - network = "${google_compute_network.network.name}" - - allow { - protocol = "tcp" - ports = [443] - } - - source_ranges = ["0.0.0.0/0"] - target_tags = ["${var.cluster_name}-controller"] -} - -# BGP and IPIP -# https://docs.projectcalico.org/latest/reference/public-cloud/gce -resource "google_compute_firewall" "internal-bgp" { - count = "${var.networking != "flannel" ? 1 : 0}" - - name = "${var.cluster_name}-internal-bgp" - network = "${google_compute_network.network.name}" - - allow { - protocol = "tcp" - ports = ["179"] - } - - allow { - protocol = "ipip" - } - - source_tags = ["${var.cluster_name}-controller", "${var.cluster_name}-worker"] - target_tags = ["${var.cluster_name}-controller", "${var.cluster_name}-worker"] -} - -# flannel VXLAN -resource "google_compute_firewall" "internal-vxlan" { - count = "${var.networking == "flannel" ? 1 : 0}" - - name = "${var.cluster_name}-internal-vxlan" - network = "${google_compute_network.network.name}" - - allow { - protocol = "udp" - ports = [4789] - } - - source_tags = ["${var.cluster_name}-controller", "${var.cluster_name}-worker"] - target_tags = ["${var.cluster_name}-controller", "${var.cluster_name}-worker"] -} - -# Allow Prometheus to scrape node-exporter daemonset -resource "google_compute_firewall" "internal-node-exporter" { - name = "${var.cluster_name}-internal-node-exporter" - network = "${google_compute_network.network.name}" - - allow { - protocol = "tcp" - ports = [9100] - } - - source_tags = ["${var.cluster_name}-worker"] - target_tags = ["${var.cluster_name}-controller", "${var.cluster_name}-worker"] -} - -# Allow apiserver to access kubelets for exec, log, port-forward -resource "google_compute_firewall" "internal-kubelet" { - name = "${var.cluster_name}-internal-kubelet" - network = "${google_compute_network.network.name}" - - allow { - protocol = "tcp" - ports = [10250] - } - - # allow Prometheus to scrape kubelet metrics too - source_tags = ["${var.cluster_name}-controller", "${var.cluster_name}-worker"] - target_tags = ["${var.cluster_name}-controller", "${var.cluster_name}-worker"] -} - -# Workers - -resource "google_compute_firewall" "allow-ingress" { - name = "${var.cluster_name}-allow-ingress" - network = "${google_compute_network.network.name}" - - allow { - protocol = "tcp" - ports = [80, 443] - } - - source_ranges = ["0.0.0.0/0"] - target_tags = ["${var.cluster_name}-worker"] -} - -resource "google_compute_firewall" "google-health-checks" { - name = "${var.cluster_name}-google-health-checks" - network = "${google_compute_network.network.name}" - - allow { - protocol = "tcp" - ports = [10254] - } - - # https://cloud.google.com/compute/docs/load-balancing/tcp-ssl/tcp-proxy#health-checking - source_ranges = ["130.211.0.0/22", "35.191.0.0/16"] - target_tags = ["${var.cluster_name}-worker"] -} diff --git a/google-cloud/fedora-atomic/kubernetes/outputs.tf b/google-cloud/fedora-atomic/kubernetes/outputs.tf deleted file mode 100644 index f1a09ece4..000000000 --- a/google-cloud/fedora-atomic/kubernetes/outputs.tf +++ /dev/null @@ -1,38 +0,0 @@ -output "kubeconfig-admin" { - value = "${module.bootkube.kubeconfig-admin}" -} - -# Outputs for Kubernetes Ingress - -output "ingress_static_ipv4" { - description = "Global IPv4 address for proxy load balancing to the nearest Ingress controller" - value = "${google_compute_global_address.ingress-ipv4.address}" -} - -output "ingress_static_ipv6" { - description = "Global IPv6 address for proxy load balancing to the nearest Ingress controller" - value = "${google_compute_global_address.ingress-ipv6.address}" -} - -# Outputs for worker pools - -output "network_name" { - value = "${google_compute_network.network.name}" -} - -output "kubeconfig" { - value = "${module.bootkube.kubeconfig-kubelet}" -} - -# Outputs for custom firewalling - -output "network_self_link" { - value = "${google_compute_network.network.self_link}" -} - -# Outputs for custom load balancing - -output "worker_instance_group" { - description = "Full URL of the worker managed instance group" - value = "${module.workers.instance_group}" -} diff --git a/google-cloud/fedora-atomic/kubernetes/require.tf b/google-cloud/fedora-atomic/kubernetes/require.tf deleted file mode 100644 index d1f7f4d54..000000000 --- a/google-cloud/fedora-atomic/kubernetes/require.tf +++ /dev/null @@ -1,25 +0,0 @@ -# Terraform version and plugin versions - -terraform { - required_version = ">= 0.11.0" -} - -provider "google" { - version = ">= 1.19, < 3.0" -} - -provider "local" { - version = "~> 1.0" -} - -provider "null" { - version = "~> 1.0" -} - -provider "template" { - version = "~> 1.0" -} - -provider "tls" { - version = "~> 1.0" -} diff --git a/google-cloud/fedora-atomic/kubernetes/ssh.tf b/google-cloud/fedora-atomic/kubernetes/ssh.tf deleted file mode 100644 index 54517eaff..000000000 --- a/google-cloud/fedora-atomic/kubernetes/ssh.tf +++ /dev/null @@ -1,89 +0,0 @@ -# Secure copy etcd TLS assets to controllers. -resource "null_resource" "copy-controller-secrets" { - count = "${var.controller_count}" - - connection { - type = "ssh" - host = "${element(local.controllers_ipv4_public, count.index)}" - user = "fedora" - timeout = "15m" - } - - provisioner "file" { - content = "${module.bootkube.etcd_ca_cert}" - destination = "$HOME/etcd-client-ca.crt" - } - - provisioner "file" { - content = "${module.bootkube.etcd_client_cert}" - destination = "$HOME/etcd-client.crt" - } - - provisioner "file" { - content = "${module.bootkube.etcd_client_key}" - destination = "$HOME/etcd-client.key" - } - - provisioner "file" { - content = "${module.bootkube.etcd_server_cert}" - destination = "$HOME/etcd-server.crt" - } - - provisioner "file" { - content = "${module.bootkube.etcd_server_key}" - destination = "$HOME/etcd-server.key" - } - - provisioner "file" { - content = "${module.bootkube.etcd_peer_cert}" - destination = "$HOME/etcd-peer.crt" - } - - provisioner "file" { - content = "${module.bootkube.etcd_peer_key}" - destination = "$HOME/etcd-peer.key" - } - - provisioner "remote-exec" { - inline = [ - "sudo mkdir -p /etc/ssl/etcd/etcd", - "sudo mv etcd-client* /etc/ssl/etcd/", - "sudo cp /etc/ssl/etcd/etcd-client-ca.crt /etc/ssl/etcd/etcd/server-ca.crt", - "sudo mv etcd-server.crt /etc/ssl/etcd/etcd/server.crt", - "sudo mv etcd-server.key /etc/ssl/etcd/etcd/server.key", - "sudo cp /etc/ssl/etcd/etcd-client-ca.crt /etc/ssl/etcd/etcd/peer-ca.crt", - "sudo mv etcd-peer.crt /etc/ssl/etcd/etcd/peer.crt", - "sudo mv etcd-peer.key /etc/ssl/etcd/etcd/peer.key", - ] - } -} - -# Secure copy bootkube assets to ONE controller and start bootkube to perform -# one-time self-hosted cluster bootstrapping. -resource "null_resource" "bootkube-start" { - depends_on = [ - "null_resource.copy-controller-secrets", - "module.workers", - "google_dns_record_set.apiserver", - ] - - connection { - type = "ssh" - host = "${element(local.controllers_ipv4_public, 0)}" - user = "fedora" - timeout = "15m" - } - - provisioner "file" { - source = "${var.asset_dir}" - destination = "$HOME/assets" - } - - provisioner "remote-exec" { - inline = [ - "while [ ! -f /var/lib/cloud/instance/boot-finished ]; do sleep 4; done", - "sudo mv $HOME/assets /var/lib/bootkube", - "sudo systemctl start bootkube", - ] - } -} diff --git a/google-cloud/fedora-atomic/kubernetes/variables.tf b/google-cloud/fedora-atomic/kubernetes/variables.tf deleted file mode 100644 index 4866b1c0b..000000000 --- a/google-cloud/fedora-atomic/kubernetes/variables.tf +++ /dev/null @@ -1,110 +0,0 @@ -variable "cluster_name" { - type = "string" - description = "Unique cluster name (prepended to dns_zone)" -} - -# Google Cloud - -variable "region" { - type = "string" - description = "Google Cloud Region (e.g. us-central1, see `gcloud compute regions list`)" -} - -variable "dns_zone" { - type = "string" - description = "Google Cloud DNS Zone (e.g. google-cloud.example.com)" -} - -variable "dns_zone_name" { - type = "string" - description = "Google Cloud DNS Zone name (e.g. example-zone)" -} - -# instances - -variable "controller_count" { - type = "string" - default = "1" - description = "Number of controllers (i.e. masters)" -} - -variable "worker_count" { - type = "string" - default = "1" - description = "Number of workers" -} - -variable "controller_type" { - type = "string" - default = "n1-standard-1" - description = "Machine type for controllers (see `gcloud compute machine-types list`)" -} - -variable "worker_type" { - type = "string" - default = "n1-standard-1" - description = "Machine type for controllers (see `gcloud compute machine-types list`)" -} - -variable "os_image" { - type = "string" - description = "Custom Fedora Atomic image" -} - -variable "disk_size" { - type = "string" - default = "40" - description = "Size of the disk in GB" -} - -variable "worker_preemptible" { - type = "string" - default = "false" - description = "If enabled, Compute Engine will terminate workers randomly within 24 hours" -} - -# configuration - -variable "ssh_authorized_key" { - type = "string" - description = "SSH public key for user 'fedora'" -} - -variable "asset_dir" { - description = "Path to a directory where generated assets should be placed (contains secrets)" - type = "string" -} - -variable "networking" { - description = "Choice of networking provider (flannel or calico)" - type = "string" - default = "calico" -} - -variable "pod_cidr" { - description = "CIDR IPv4 range to assign Kubernetes pods" - type = "string" - default = "10.2.0.0/16" -} - -variable "service_cidr" { - description = < /etc/kubernetes/ca.crt" - Restart=always - RestartSec=10 - - path: /etc/kubernetes/kubelet.conf - content: | - ARGS="--anonymous-auth=false \ - --authentication-token-webhook \ - --authorization-mode=Webhook \ - --client-ca-file=/etc/kubernetes/ca.crt \ - --cluster_dns=${cluster_dns_service_ip} \ - --cluster_domain=${cluster_domain_suffix} \ - --cni-conf-dir=/etc/kubernetes/cni/net.d \ - --exit-on-lock-contention \ - --kubeconfig=/etc/kubernetes/kubeconfig \ - --lock-file=/var/run/lock/kubelet.lock \ - --network-plugin=cni \ - --node-labels=node-role.kubernetes.io/node \ - --pod-manifest-path=/etc/kubernetes/manifests \ - --read-only-port=0 \ - --volume-plugin-dir=/var/lib/kubelet/volumeplugins" - - path: /etc/kubernetes/kubeconfig - permissions: '0644' - content: | - ${kubeconfig} - - path: /etc/NetworkManager/conf.d/typhoon.conf - content: | - [main] - plugins=keyfile - [keyfile] - unmanaged-devices=interface-name:cali*;interface-name:tunl* - - path: /etc/selinux/config - owner: root:root - permissions: '0644' - content: | - SELINUX=permissive - SELINUXTYPE=targeted -bootcmd: - - [setenforce, Permissive] - - [systemctl, disable, firewalld, --now] - # https://github.com/kubernetes/kubernetes/issues/60869 - - [modprobe, ip_vs] -runcmd: - - [systemctl, daemon-reload] - - [systemctl, restart, NetworkManager] - - "atomic install --system --name=kubelet quay.io/poseidon/kubelet:v1.14.1" - - [systemctl, start, --no-block, kubelet.service] -users: - - default - - name: fedora - gecos: Fedora Admin - sudo: ALL=(ALL) NOPASSWD:ALL - groups: wheel,adm,systemd-journal,docker - ssh-authorized-keys: - - "${ssh_authorized_key}" diff --git a/google-cloud/fedora-atomic/kubernetes/workers/outputs.tf b/google-cloud/fedora-atomic/kubernetes/workers/outputs.tf deleted file mode 100644 index 4521d7923..000000000 --- a/google-cloud/fedora-atomic/kubernetes/workers/outputs.tf +++ /dev/null @@ -1,4 +0,0 @@ -output "instance_group" { - description = "Full URL of the worker managed instance group" - value = "${google_compute_region_instance_group_manager.workers.instance_group}" -} diff --git a/google-cloud/fedora-atomic/kubernetes/workers/variables.tf b/google-cloud/fedora-atomic/kubernetes/workers/variables.tf deleted file mode 100644 index 913afb73e..000000000 --- a/google-cloud/fedora-atomic/kubernetes/workers/variables.tf +++ /dev/null @@ -1,94 +0,0 @@ -variable "name" { - type = "string" - description = "Unique name for the worker pool" -} - -variable "cluster_name" { - type = "string" - description = "Must be set to `cluster_name of cluster`" -} - -# Google Cloud - -variable "region" { - type = "string" - description = "Must be set to `region` of cluster" -} - -variable "network" { - type = "string" - description = "Must be set to `network_name` output by cluster" -} - -# instances - -variable "count" { - type = "string" - default = "1" - description = "Number of worker compute instances the instance group should manage" -} - -variable "machine_type" { - type = "string" - default = "n1-standard-1" - description = "Machine type for compute instances (e.g. gcloud compute machine-types list)" -} - -variable "os_image" { - type = "string" - description = "Custom Fedora Atomic image" -} - -variable "disk_size" { - type = "string" - default = "40" - description = "Size of the disk in GB" -} - -variable "preemptible" { - type = "string" - default = "false" - description = "If enabled, Compute Engine will terminate instances randomly within 24 hours" -} - -# configuration - -variable "kubeconfig" { - type = "string" - description = "Must be set to `kubeconfig` output by cluster" -} - -variable "ssh_authorized_key" { - type = "string" - description = "SSH public key for user 'fedora'" -} - -variable "service_cidr" { - description = < Date: Sat, 29 Jun 2019 22:20:03 +0200 Subject: [PATCH 144/523] Fix README link to Azure module (#502) --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index ae2f8470a..794fba238 100644 --- a/README.md +++ b/README.md @@ -24,7 +24,7 @@ Typhoon provides a Terraform Module for each supported operating system and plat | Platform | Operating System | Terraform Module | Status | |---------------|------------------|------------------|--------| | AWS | Container Linux / Flatcar Linux | [aws/container-linux/kubernetes](aws/container-linux/kubernetes) | stable | -| Azure | Container Linux | [azure/container-linux/kubernetes](cl/azure.md) | alpha | +| Azure | Container Linux | [azure/container-linux/kubernetes](azure/container-linux/kubernetes) | alpha | | Bare-Metal | Container Linux / Flatcar Linux | [bare-metal/container-linux/kubernetes](bare-metal/container-linux/kubernetes) | stable | | Digital Ocean | Container Linux | [digital-ocean/container-linux/kubernetes](digital-ocean/container-linux/kubernetes) | beta | | Google Cloud | Container Linux | [google-cloud/container-linux/kubernetes](google-cloud/container-linux/kubernetes) | stable | From 9a395dbf88c295633e85a7978001cab7c9e3c7d7 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Sat, 29 Jun 2019 13:17:27 -0700 Subject: [PATCH 145/523] Update Grafana from v6.2.4 to v6.2.5 * https://github.com/grafana/grafana/releases/tag/v6.2.5 --- CHANGES.md | 4 ++++ addons/grafana/deployment.yaml | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index 19c0ba3db..ae2c994da 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -4,6 +4,10 @@ Notable changes between versions. ## Latest +#### Addons + +* Update Grafana from v6.2.4 to v6.2.5 + ## v1.15.0 * Kubernetes [v1.15.0](https://github.com/kubernetes/kubernetes/blob/master/CHANGELOG-1.15.md#v1150) diff --git a/addons/grafana/deployment.yaml b/addons/grafana/deployment.yaml index 62773f4e4..dc9f05847 100644 --- a/addons/grafana/deployment.yaml +++ b/addons/grafana/deployment.yaml @@ -23,7 +23,7 @@ spec: spec: containers: - name: grafana - image: grafana/grafana:6.2.4 + image: docker.io/grafana/grafana:6.2.5 env: - name: GF_PATHS_CONFIG value: "/etc/grafana/custom.ini" From 307aaf5e308270de322a5d4fdc273d5054681afc Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Sat, 29 Jun 2019 13:50:49 -0700 Subject: [PATCH 146/523] Use Terraform v0.12 syntax in ingress docs * Drop string interpolation in Google Cloud A records shown in Nginx ingress addon docs * Retain string interpolation syntax for CNAME records since Google Cloud DNS expects records to end in "." (some clouds add it automatically) --- docs/addons/ingress.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/addons/ingress.md b/docs/addons/ingress.md index 7e2bf84be..caf5244c9 100644 --- a/docs/addons/ingress.md +++ b/docs/addons/ingress.md @@ -64,7 +64,7 @@ resource "google_dns_record_set" "some-application" { name = "app.example.com." type = "A" ttl = 300 - rrdatas = ["${module.azure-ramius.ingress_static_ipv4}"] + rrdatas = [module.azure-ramius.ingress_static_ipv4] } ``` @@ -158,7 +158,7 @@ resource "google_dns_record_set" "app-record-a" { name = "app.example.com." type = "A" ttl = 300 - rrdatas = ["${module.google-cloud-yavin.ingress_static_ipv4}"] + rrdatas = [module.google-cloud-yavin.ingress_static_ipv4] } resource "google_dns_record_set" "app-record-aaaa" { @@ -169,6 +169,6 @@ resource "google_dns_record_set" "app-record-aaaa" { name = "app.example.com." type = "AAAA" ttl = 300 - rrdatas = ["${module.google-cloud-yavin.ingress_static_ipv6}"] + rrdatas = [module.google-cloud-yavin.ingress_static_ipv6] } ``` From 8d373b5850896bb65e6ab62cdb1128f3ceb89660 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Tue, 2 Jul 2019 20:18:02 -0700 Subject: [PATCH 147/523] Update Calico from v3.7.3 to v3.7.4 * https://docs.projectcalico.org/v3.7/release-notes/ --- CHANGES.md | 2 ++ aws/container-linux/kubernetes/bootkube.tf | 2 +- azure/container-linux/kubernetes/bootkube.tf | 2 +- bare-metal/container-linux/kubernetes/bootkube.tf | 2 +- digital-ocean/container-linux/kubernetes/bootkube.tf | 2 +- google-cloud/container-linux/kubernetes/bootkube.tf | 2 +- 6 files changed, 7 insertions(+), 5 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index ae2c994da..fe1d344e7 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -4,6 +4,8 @@ Notable changes between versions. ## Latest +* Update Calico from v3.7.3 to [v3.7.4](https://docs.projectcalico.org/v3.7/release-notes/) + #### Addons * Update Grafana from v6.2.4 to v6.2.5 diff --git a/aws/container-linux/kubernetes/bootkube.tf b/aws/container-linux/kubernetes/bootkube.tf index 68c4458a0..3f2e39d52 100644 --- a/aws/container-linux/kubernetes/bootkube.tf +++ b/aws/container-linux/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=62df9ad69cc0da35f47d40fa981370c4503ad581" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=95f6fc7fa5f6cedb1a887da3a823b0eeffa40f68" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/azure/container-linux/kubernetes/bootkube.tf b/azure/container-linux/kubernetes/bootkube.tf index 75b7ef131..5740d65d1 100644 --- a/azure/container-linux/kubernetes/bootkube.tf +++ b/azure/container-linux/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=62df9ad69cc0da35f47d40fa981370c4503ad581" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=95f6fc7fa5f6cedb1a887da3a823b0eeffa40f68" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/bare-metal/container-linux/kubernetes/bootkube.tf b/bare-metal/container-linux/kubernetes/bootkube.tf index 1cc9c16f9..287d779da 100644 --- a/bare-metal/container-linux/kubernetes/bootkube.tf +++ b/bare-metal/container-linux/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=62df9ad69cc0da35f47d40fa981370c4503ad581" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=95f6fc7fa5f6cedb1a887da3a823b0eeffa40f68" cluster_name = var.cluster_name api_servers = [var.k8s_domain_name] diff --git a/digital-ocean/container-linux/kubernetes/bootkube.tf b/digital-ocean/container-linux/kubernetes/bootkube.tf index 7ed6253c0..27e455c88 100644 --- a/digital-ocean/container-linux/kubernetes/bootkube.tf +++ b/digital-ocean/container-linux/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=62df9ad69cc0da35f47d40fa981370c4503ad581" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=95f6fc7fa5f6cedb1a887da3a823b0eeffa40f68" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/google-cloud/container-linux/kubernetes/bootkube.tf b/google-cloud/container-linux/kubernetes/bootkube.tf index 0bab5fc25..96397dd45 100644 --- a/google-cloud/container-linux/kubernetes/bootkube.tf +++ b/google-cloud/container-linux/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=62df9ad69cc0da35f47d40fa981370c4503ad581" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=95f6fc7fa5f6cedb1a887da3a823b0eeffa40f68" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] From 3fcb04f68c94bcbe2b4cb9825031adc1beba851a Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Fri, 5 Jul 2019 19:21:05 -0700 Subject: [PATCH 148/523] Improve apiserver backend service zone spanning * google_compute_backend_services use nested blocks to define backends (instance groups heterogeneous controllers) * Use Terraform v0.12.x dynamic blocks so the apiserver backend service can refer to (up to zone-many) controller instance groups * Previously, with Terraform v0.11.x, the apiserver backend service had to list a fixed set of backends to span controller nodes across zones in multi-controller setups. 3 backends were used because each GCP region offered at least 3 zones. Single-controller clusters had the cosmetic ugliness of unused instance groups * Allow controllers to span more than 3 zones if avilable in a region (e.g. currently only us-central1, with 4 zones) Related: * https://www.terraform.io/docs/providers/google/r/compute_backend_service.html * https://www.terraform.io/docs/configuration/expressions.html#dynamic-blocks --- CHANGES.md | 5 +++++ .../container-linux/kubernetes/apiserver.tf | 21 ++++++------------- .../container-linux/kubernetes/controllers.tf | 4 +--- 3 files changed, 12 insertions(+), 18 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index fe1d344e7..5d1e7f925 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -6,6 +6,11 @@ Notable changes between versions. * Update Calico from v3.7.3 to [v3.7.4](https://docs.projectcalico.org/v3.7/release-notes/) +#### Google Cloud + +* Allow controller nodes to span more than 3 zones if available in a region ([#504](https://github.com/poseidon/typhoon/pull/504)) +* Eliminate extraneous controller instance groups in single-controller clusters ([#504](https://github.com/poseidon/typhoon/pull/504)) + #### Addons * Update Grafana from v6.2.4 to v6.2.5 diff --git a/google-cloud/container-linux/kubernetes/apiserver.tf b/google-cloud/container-linux/kubernetes/apiserver.tf index c6080f9ce..62809a9e3 100644 --- a/google-cloud/container-linux/kubernetes/apiserver.tf +++ b/google-cloud/container-linux/kubernetes/apiserver.tf @@ -45,16 +45,11 @@ resource "google_compute_backend_service" "apiserver" { timeout_sec = "300" # controller(s) spread across zonal instance groups - backend { - group = google_compute_instance_group.controllers[0].self_link - } - - backend { - group = google_compute_instance_group.controllers[1].self_link - } - - backend { - group = google_compute_instance_group.controllers[2].self_link + dynamic "backend" { + for_each = google_compute_instance_group.controllers + content { + group = backend.value.self_link + } } health_checks = [google_compute_health_check.apiserver.self_link] @@ -64,11 +59,7 @@ resource "google_compute_backend_service" "apiserver" { resource "google_compute_instance_group" "controllers" { count = length(local.zones) - name = format( - "%s-controllers-%s", - var.cluster_name, - element(local.zones, count.index), - ) + name = format("%s-controllers-%s", var.cluster_name, element(local.zones, count.index)) zone = element(local.zones, count.index) named_port { diff --git a/google-cloud/container-linux/kubernetes/controllers.tf b/google-cloud/container-linux/kubernetes/controllers.tf index 2ba068c7b..06a921a66 100644 --- a/google-cloud/container-linux/kubernetes/controllers.tf +++ b/google-cloud/container-linux/kubernetes/controllers.tf @@ -20,9 +20,7 @@ data "google_compute_zones" "all" { } locals { - # TCP proxy load balancers require a fixed number of zonal backends. Spread - # controllers over up to 3 zones, since all GCP regions have at least 3. - zones = slice(data.google_compute_zones.all.names, 0, 3) + zones = data.google_compute_zones.all.names controllers_ipv4_public = google_compute_instance.controllers.*.network_interface.0.access_config.0.nat_ip } From 7a69bae75e4c530d1222f330f3f6ec9039e123ce Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Fri, 5 Jul 2019 19:48:49 -0700 Subject: [PATCH 149/523] Raise GCP network deletion timeout from 4m to 6m * Fix a GCP errata item https://github.com/poseidon/typhoon/wiki/Errata * Removal of a Google Cloud cluster often required 2 runs of `terraform apply` because network resource deletes timeout after 4m. Raise the network deletion timeout to 6m to ensure apply only needs to be run once to remove a cluster --- CHANGES.md | 1 + google-cloud/container-linux/kubernetes/network.tf | 4 ++++ 2 files changed, 5 insertions(+) diff --git a/CHANGES.md b/CHANGES.md index 5d1e7f925..8c58a0d7e 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -10,6 +10,7 @@ Notable changes between versions. * Allow controller nodes to span more than 3 zones if available in a region ([#504](https://github.com/poseidon/typhoon/pull/504)) * Eliminate extraneous controller instance groups in single-controller clusters ([#504](https://github.com/poseidon/typhoon/pull/504)) +* Raise network deletion timeout from 4m to 6m ([#505](https://github.com/poseidon/typhoon/pull/505)) #### Addons diff --git a/google-cloud/container-linux/kubernetes/network.tf b/google-cloud/container-linux/kubernetes/network.tf index 4f9d78a84..a0f3fad80 100644 --- a/google-cloud/container-linux/kubernetes/network.tf +++ b/google-cloud/container-linux/kubernetes/network.tf @@ -2,6 +2,10 @@ resource "google_compute_network" "network" { name = var.cluster_name description = "Network for the ${var.cluster_name} cluster" auto_create_subnetworks = true + + timeouts { + delete = "6m" + } } resource "google_compute_firewall" "allow-ssh" { From 69d064bfdf496e8912c9928a165c0de1d44cbbac Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Sat, 6 Jul 2019 13:11:37 -0700 Subject: [PATCH 150/523] Run kube-apiserver with lower privilege user (nobody) * Run kube-apiserver as a non-root user (nobody). User no longer needs to bind low number ports. * On most platforms, the kube-apiserver load balancer listens on 6443 and fronts controllers with kube-apiserver pods using port 6443. Google Cloud TCP proxy load balancers cannot listen on 6443. However, GCP's load balancer can be made to listen on 443, while kube-apiserver uses 6443 across all platforms. --- CHANGES.md | 1 + aws/container-linux/kubernetes/bootkube.tf | 2 +- azure/container-linux/kubernetes/bootkube.tf | 2 +- bare-metal/container-linux/kubernetes/bootkube.tf | 2 +- digital-ocean/container-linux/kubernetes/bootkube.tf | 2 +- google-cloud/container-linux/kubernetes/apiserver.tf | 4 ++-- google-cloud/container-linux/kubernetes/bootkube.tf | 4 ++-- google-cloud/container-linux/kubernetes/network.tf | 2 +- 8 files changed, 10 insertions(+), 9 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 8c58a0d7e..37c064193 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -5,6 +5,7 @@ Notable changes between versions. ## Latest * Update Calico from v3.7.3 to [v3.7.4](https://docs.projectcalico.org/v3.7/release-notes/) +* Run `kube-apiserver` with lower privilege user (nobody) ([#506](https://github.com/poseidon/typhoon/pull/506)) #### Google Cloud diff --git a/aws/container-linux/kubernetes/bootkube.tf b/aws/container-linux/kubernetes/bootkube.tf index 3f2e39d52..65647ded7 100644 --- a/aws/container-linux/kubernetes/bootkube.tf +++ b/aws/container-linux/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=95f6fc7fa5f6cedb1a887da3a823b0eeffa40f68" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=4caca47776dcff440c9f50445fbc8ec5b11728f4" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/azure/container-linux/kubernetes/bootkube.tf b/azure/container-linux/kubernetes/bootkube.tf index 5740d65d1..8ca8fcc67 100644 --- a/azure/container-linux/kubernetes/bootkube.tf +++ b/azure/container-linux/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=95f6fc7fa5f6cedb1a887da3a823b0eeffa40f68" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=4caca47776dcff440c9f50445fbc8ec5b11728f4" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/bare-metal/container-linux/kubernetes/bootkube.tf b/bare-metal/container-linux/kubernetes/bootkube.tf index 287d779da..b416e0980 100644 --- a/bare-metal/container-linux/kubernetes/bootkube.tf +++ b/bare-metal/container-linux/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=95f6fc7fa5f6cedb1a887da3a823b0eeffa40f68" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=4caca47776dcff440c9f50445fbc8ec5b11728f4" cluster_name = var.cluster_name api_servers = [var.k8s_domain_name] diff --git a/digital-ocean/container-linux/kubernetes/bootkube.tf b/digital-ocean/container-linux/kubernetes/bootkube.tf index 27e455c88..e3f1deeb8 100644 --- a/digital-ocean/container-linux/kubernetes/bootkube.tf +++ b/digital-ocean/container-linux/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=95f6fc7fa5f6cedb1a887da3a823b0eeffa40f68" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=4caca47776dcff440c9f50445fbc8ec5b11728f4" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/google-cloud/container-linux/kubernetes/apiserver.tf b/google-cloud/container-linux/kubernetes/apiserver.tf index 62809a9e3..b06ecf704 100644 --- a/google-cloud/container-linux/kubernetes/apiserver.tf +++ b/google-cloud/container-linux/kubernetes/apiserver.tf @@ -64,7 +64,7 @@ resource "google_compute_instance_group" "controllers" { named_port { name = "apiserver" - port = "443" + port = "6443" } # add instances in the zone into the instance group @@ -87,7 +87,7 @@ resource "google_compute_health_check" "apiserver" { unhealthy_threshold = 3 tcp_health_check { - port = "443" + port = "6443" } } diff --git a/google-cloud/container-linux/kubernetes/bootkube.tf b/google-cloud/container-linux/kubernetes/bootkube.tf index 96397dd45..87c989ccd 100644 --- a/google-cloud/container-linux/kubernetes/bootkube.tf +++ b/google-cloud/container-linux/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=95f6fc7fa5f6cedb1a887da3a823b0eeffa40f68" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=4caca47776dcff440c9f50445fbc8ec5b11728f4" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] @@ -15,6 +15,6 @@ module "bootkube" { enable_aggregation = var.enable_aggregation // temporary - apiserver_port = 443 + external_apiserver_port = 443 } diff --git a/google-cloud/container-linux/kubernetes/network.tf b/google-cloud/container-linux/kubernetes/network.tf index a0f3fad80..735fa3a84 100644 --- a/google-cloud/container-linux/kubernetes/network.tf +++ b/google-cloud/container-linux/kubernetes/network.tf @@ -54,7 +54,7 @@ resource "google_compute_firewall" "allow-apiserver" { allow { protocol = "tcp" - ports = [443] + ports = [6443] } source_ranges = ["0.0.0.0/0"] From 19596255a6822ccccf24e66d9b4b2fafc8867e42 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Mon, 8 Jul 2019 20:54:46 -0700 Subject: [PATCH 151/523] Fix malformed markdown table in OS docs --- docs/architecture/operating-systems.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/architecture/operating-systems.md b/docs/architecture/operating-systems.md index 804f70c1b..707ffe85e 100644 --- a/docs/architecture/operating-systems.md +++ b/docs/architecture/operating-systems.md @@ -15,7 +15,7 @@ Together, they diversify Typhoon to support a range of container technologies. ## Host Properties | Property | Container Linux / Flatcar Linux | Fedora Atomic | -|-------------------|-----------------|---------------|---------------| +|-------------------|---------------------------------|---------------| | host spec (bare-metal) | Container Linux Config | kickstart, cloud-init | | host spec (cloud) | Container Linux Config | cloud-init | | container runtime | docker | docker (CRIO planned) | From 28ab746068cfa413c4c83ba48b2db2ac12777eda Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Mon, 8 Jul 2019 21:32:50 -0700 Subject: [PATCH 152/523] Update Prometheus from v2.10.0 to v2.11.0-rc.0 * https://github.com/prometheus/prometheus/releases/tag/v2.11.0-rc.0 --- CHANGES.md | 1 + addons/prometheus/deployment.yaml | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index 37c064193..03ee80214 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -15,6 +15,7 @@ Notable changes between versions. #### Addons +* Update Prometheus from v2.10.0 to v2.11.0-rc.0 * Update Grafana from v6.2.4 to v6.2.5 ## v1.15.0 diff --git a/addons/prometheus/deployment.yaml b/addons/prometheus/deployment.yaml index 06abfa9e5..5dee34eb2 100644 --- a/addons/prometheus/deployment.yaml +++ b/addons/prometheus/deployment.yaml @@ -20,7 +20,7 @@ spec: serviceAccountName: prometheus containers: - name: prometheus - image: quay.io/prometheus/prometheus:v2.10.0 + image: quay.io/prometheus/prometheus:v2.11.0-rc.0 args: - --web.listen-address=0.0.0.0:9090 - --config.file=/etc/prometheus/prometheus.yaml From 40640f369793b1cc1ec86dec85439b696ecdb63f Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Mon, 8 Jul 2019 22:04:50 -0700 Subject: [PATCH 153/523] Upgrade nginx-ingress from v0.24.1 to v0.25.0 * Support networking.k8s.io/v1beta1 apiVersion * Update RBAC cluster-role for networking.k8s.io/v1beta1 * https://github.com/kubernetes/ingress-nginx/releases/tag/nginx-0.25.0 --- CHANGES.md | 2 ++ addons/nginx-ingress/aws/deployment.yaml | 2 +- .../nginx-ingress/aws/rbac/cluster-role.yaml | 24 +++++++++++++++---- addons/nginx-ingress/azure/deployment.yaml | 2 +- .../azure/rbac/cluster-role.yaml | 24 +++++++++++++++---- .../nginx-ingress/bare-metal/deployment.yaml | 2 +- .../bare-metal/rbac/cluster-role.yaml | 24 +++++++++++++++---- .../digital-ocean/daemonset.yaml | 2 +- .../digital-ocean/rbac/cluster-role.yaml | 24 +++++++++++++++---- .../google-cloud/deployment.yaml | 2 +- .../google-cloud/rbac/cluster-role.yaml | 24 +++++++++++++++---- 11 files changed, 102 insertions(+), 30 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 03ee80214..2b6eb0c1b 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -17,6 +17,8 @@ Notable changes between versions. * Update Prometheus from v2.10.0 to v2.11.0-rc.0 * Update Grafana from v6.2.4 to v6.2.5 +* Update nginx-ingress from v0.24.1 to v0.25.0 + * Support `networking.k8s.io/v1beta1` apiVersion ## v1.15.0 diff --git a/addons/nginx-ingress/aws/deployment.yaml b/addons/nginx-ingress/aws/deployment.yaml index e283c76d5..fdb9cc317 100644 --- a/addons/nginx-ingress/aws/deployment.yaml +++ b/addons/nginx-ingress/aws/deployment.yaml @@ -24,7 +24,7 @@ spec: node-role.kubernetes.io/node: "" containers: - name: nginx-ingress-controller - image: quay.io/kubernetes-ingress-controller/nginx-ingress-controller:0.24.1 + image: quay.io/kubernetes-ingress-controller/nginx-ingress-controller:0.25.0 args: - /nginx-ingress-controller - --ingress-class=public diff --git a/addons/nginx-ingress/aws/rbac/cluster-role.yaml b/addons/nginx-ingress/aws/rbac/cluster-role.yaml index 9fee9fde4..bf680e19e 100644 --- a/addons/nginx-ingress/aws/rbac/cluster-role.yaml +++ b/addons/nginx-ingress/aws/rbac/cluster-role.yaml @@ -28,6 +28,13 @@ rules: - get - list - watch + - apiGroups: + - "" + resources: + - events + verbs: + - create + - patch - apiGroups: - "extensions" resources: @@ -37,14 +44,21 @@ rules: - list - watch - apiGroups: - - "" + - "extensions" resources: - - events + - ingresses/status verbs: - - create - - patch + - update - apiGroups: - - "extensions" + - "networking.k8s.io" + resources: + - ingresses + verbs: + - get + - list + - watch + - apiGroups: + - "networking.k8s.io" resources: - ingresses/status verbs: diff --git a/addons/nginx-ingress/azure/deployment.yaml b/addons/nginx-ingress/azure/deployment.yaml index e283c76d5..fdb9cc317 100644 --- a/addons/nginx-ingress/azure/deployment.yaml +++ b/addons/nginx-ingress/azure/deployment.yaml @@ -24,7 +24,7 @@ spec: node-role.kubernetes.io/node: "" containers: - name: nginx-ingress-controller - image: quay.io/kubernetes-ingress-controller/nginx-ingress-controller:0.24.1 + image: quay.io/kubernetes-ingress-controller/nginx-ingress-controller:0.25.0 args: - /nginx-ingress-controller - --ingress-class=public diff --git a/addons/nginx-ingress/azure/rbac/cluster-role.yaml b/addons/nginx-ingress/azure/rbac/cluster-role.yaml index 9fee9fde4..bf680e19e 100644 --- a/addons/nginx-ingress/azure/rbac/cluster-role.yaml +++ b/addons/nginx-ingress/azure/rbac/cluster-role.yaml @@ -28,6 +28,13 @@ rules: - get - list - watch + - apiGroups: + - "" + resources: + - events + verbs: + - create + - patch - apiGroups: - "extensions" resources: @@ -37,14 +44,21 @@ rules: - list - watch - apiGroups: - - "" + - "extensions" resources: - - events + - ingresses/status verbs: - - create - - patch + - update - apiGroups: - - "extensions" + - "networking.k8s.io" + resources: + - ingresses + verbs: + - get + - list + - watch + - apiGroups: + - "networking.k8s.io" resources: - ingresses/status verbs: diff --git a/addons/nginx-ingress/bare-metal/deployment.yaml b/addons/nginx-ingress/bare-metal/deployment.yaml index 2048180cc..d45bbc263 100644 --- a/addons/nginx-ingress/bare-metal/deployment.yaml +++ b/addons/nginx-ingress/bare-metal/deployment.yaml @@ -22,7 +22,7 @@ spec: spec: containers: - name: nginx-ingress-controller - image: quay.io/kubernetes-ingress-controller/nginx-ingress-controller:0.24.1 + image: quay.io/kubernetes-ingress-controller/nginx-ingress-controller:0.25.0 args: - /nginx-ingress-controller - --ingress-class=public diff --git a/addons/nginx-ingress/bare-metal/rbac/cluster-role.yaml b/addons/nginx-ingress/bare-metal/rbac/cluster-role.yaml index 9fee9fde4..bf680e19e 100644 --- a/addons/nginx-ingress/bare-metal/rbac/cluster-role.yaml +++ b/addons/nginx-ingress/bare-metal/rbac/cluster-role.yaml @@ -28,6 +28,13 @@ rules: - get - list - watch + - apiGroups: + - "" + resources: + - events + verbs: + - create + - patch - apiGroups: - "extensions" resources: @@ -37,14 +44,21 @@ rules: - list - watch - apiGroups: - - "" + - "extensions" resources: - - events + - ingresses/status verbs: - - create - - patch + - update - apiGroups: - - "extensions" + - "networking.k8s.io" + resources: + - ingresses + verbs: + - get + - list + - watch + - apiGroups: + - "networking.k8s.io" resources: - ingresses/status verbs: diff --git a/addons/nginx-ingress/digital-ocean/daemonset.yaml b/addons/nginx-ingress/digital-ocean/daemonset.yaml index 11990ad82..ab43187e2 100644 --- a/addons/nginx-ingress/digital-ocean/daemonset.yaml +++ b/addons/nginx-ingress/digital-ocean/daemonset.yaml @@ -24,7 +24,7 @@ spec: node-role.kubernetes.io/node: "" containers: - name: nginx-ingress-controller - image: quay.io/kubernetes-ingress-controller/nginx-ingress-controller:0.24.1 + image: quay.io/kubernetes-ingress-controller/nginx-ingress-controller:0.25.0 args: - /nginx-ingress-controller - --ingress-class=public diff --git a/addons/nginx-ingress/digital-ocean/rbac/cluster-role.yaml b/addons/nginx-ingress/digital-ocean/rbac/cluster-role.yaml index 9fee9fde4..bf680e19e 100644 --- a/addons/nginx-ingress/digital-ocean/rbac/cluster-role.yaml +++ b/addons/nginx-ingress/digital-ocean/rbac/cluster-role.yaml @@ -28,6 +28,13 @@ rules: - get - list - watch + - apiGroups: + - "" + resources: + - events + verbs: + - create + - patch - apiGroups: - "extensions" resources: @@ -37,14 +44,21 @@ rules: - list - watch - apiGroups: - - "" + - "extensions" resources: - - events + - ingresses/status verbs: - - create - - patch + - update - apiGroups: - - "extensions" + - "networking.k8s.io" + resources: + - ingresses + verbs: + - get + - list + - watch + - apiGroups: + - "networking.k8s.io" resources: - ingresses/status verbs: diff --git a/addons/nginx-ingress/google-cloud/deployment.yaml b/addons/nginx-ingress/google-cloud/deployment.yaml index e283c76d5..fdb9cc317 100644 --- a/addons/nginx-ingress/google-cloud/deployment.yaml +++ b/addons/nginx-ingress/google-cloud/deployment.yaml @@ -24,7 +24,7 @@ spec: node-role.kubernetes.io/node: "" containers: - name: nginx-ingress-controller - image: quay.io/kubernetes-ingress-controller/nginx-ingress-controller:0.24.1 + image: quay.io/kubernetes-ingress-controller/nginx-ingress-controller:0.25.0 args: - /nginx-ingress-controller - --ingress-class=public diff --git a/addons/nginx-ingress/google-cloud/rbac/cluster-role.yaml b/addons/nginx-ingress/google-cloud/rbac/cluster-role.yaml index 9fee9fde4..bf680e19e 100644 --- a/addons/nginx-ingress/google-cloud/rbac/cluster-role.yaml +++ b/addons/nginx-ingress/google-cloud/rbac/cluster-role.yaml @@ -28,6 +28,13 @@ rules: - get - list - watch + - apiGroups: + - "" + resources: + - events + verbs: + - create + - patch - apiGroups: - "extensions" resources: @@ -37,14 +44,21 @@ rules: - list - watch - apiGroups: - - "" + - "extensions" resources: - - events + - ingresses/status verbs: - - create - - patch + - update - apiGroups: - - "extensions" + - "networking.k8s.io" + resources: + - ingresses + verbs: + - get + - list + - watch + - apiGroups: + - "networking.k8s.io" resources: - ingresses/status verbs: From eaf59bd33f08a54c390ad116098ea22093638f0e Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Tue, 9 Jul 2019 20:54:57 -0700 Subject: [PATCH 154/523] Update Prometheus from v2.11.0-rc.0 to v2.11.0 * https://github.com/prometheus/prometheus/releases/tag/v2.11.0 --- CHANGES.md | 2 +- addons/prometheus/deployment.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 2b6eb0c1b..cd068e390 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -15,7 +15,7 @@ Notable changes between versions. #### Addons -* Update Prometheus from v2.10.0 to v2.11.0-rc.0 +* Update Prometheus from v2.10.0 to v2.11.0 * Update Grafana from v6.2.4 to v6.2.5 * Update nginx-ingress from v0.24.1 to v0.25.0 * Support `networking.k8s.io/v1beta1` apiVersion diff --git a/addons/prometheus/deployment.yaml b/addons/prometheus/deployment.yaml index 5dee34eb2..94b096004 100644 --- a/addons/prometheus/deployment.yaml +++ b/addons/prometheus/deployment.yaml @@ -20,7 +20,7 @@ spec: serviceAccountName: prometheus containers: - name: prometheus - image: quay.io/prometheus/prometheus:v2.11.0-rc.0 + image: quay.io/prometheus/prometheus:v2.11.0 args: - --web.listen-address=0.0.0.0:9090 - --config.file=/etc/prometheus/prometheus.yaml From 9e91d7f01174fbca4de49e5ea5c0586728ce44e8 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Thu, 11 Jul 2019 21:00:03 -0700 Subject: [PATCH 155/523] Upgrade Calico from v3.7.4 to v3.8.0 * Enable CNI bandwidth plugin for traffic shaping * https://kubernetes.io/docs/concepts/extend-kubernetes/compute-storage-net/network-plugins/#support-traffic-shaping --- CHANGES.md | 3 ++- aws/container-linux/kubernetes/bootkube.tf | 2 +- azure/container-linux/kubernetes/bootkube.tf | 2 +- bare-metal/container-linux/kubernetes/bootkube.tf | 2 +- digital-ocean/container-linux/kubernetes/bootkube.tf | 2 +- google-cloud/container-linux/kubernetes/bootkube.tf | 2 +- 6 files changed, 7 insertions(+), 6 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index cd068e390..517d8e44e 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -4,7 +4,8 @@ Notable changes between versions. ## Latest -* Update Calico from v3.7.3 to [v3.7.4](https://docs.projectcalico.org/v3.7/release-notes/) +* Upgrade Calico from v3.7.3 to [v3.8.0](https://docs.projectcalico.org/v3.8/release-notes/) + * Enable CNI `bandwidth` plugin for [traffic shaping](https://kubernetes.io/docs/concepts/extend-kubernetes/compute-storage-net/network-plugins/#support-traffic-shaping) * Run `kube-apiserver` with lower privilege user (nobody) ([#506](https://github.com/poseidon/typhoon/pull/506)) #### Google Cloud diff --git a/aws/container-linux/kubernetes/bootkube.tf b/aws/container-linux/kubernetes/bootkube.tf index 65647ded7..9c17535d7 100644 --- a/aws/container-linux/kubernetes/bootkube.tf +++ b/aws/container-linux/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=4caca47776dcff440c9f50445fbc8ec5b11728f4" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=119cb00fa7b12e0ebd5a70c9c0a4e7eda2e8c3d6" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/azure/container-linux/kubernetes/bootkube.tf b/azure/container-linux/kubernetes/bootkube.tf index 8ca8fcc67..6fada064e 100644 --- a/azure/container-linux/kubernetes/bootkube.tf +++ b/azure/container-linux/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=4caca47776dcff440c9f50445fbc8ec5b11728f4" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=119cb00fa7b12e0ebd5a70c9c0a4e7eda2e8c3d6" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/bare-metal/container-linux/kubernetes/bootkube.tf b/bare-metal/container-linux/kubernetes/bootkube.tf index b416e0980..9965e7a85 100644 --- a/bare-metal/container-linux/kubernetes/bootkube.tf +++ b/bare-metal/container-linux/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=4caca47776dcff440c9f50445fbc8ec5b11728f4" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=119cb00fa7b12e0ebd5a70c9c0a4e7eda2e8c3d6" cluster_name = var.cluster_name api_servers = [var.k8s_domain_name] diff --git a/digital-ocean/container-linux/kubernetes/bootkube.tf b/digital-ocean/container-linux/kubernetes/bootkube.tf index e3f1deeb8..fab9828bf 100644 --- a/digital-ocean/container-linux/kubernetes/bootkube.tf +++ b/digital-ocean/container-linux/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=4caca47776dcff440c9f50445fbc8ec5b11728f4" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=119cb00fa7b12e0ebd5a70c9c0a4e7eda2e8c3d6" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/google-cloud/container-linux/kubernetes/bootkube.tf b/google-cloud/container-linux/kubernetes/bootkube.tf index 87c989ccd..3d3347a89 100644 --- a/google-cloud/container-linux/kubernetes/bootkube.tf +++ b/google-cloud/container-linux/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=4caca47776dcff440c9f50445fbc8ec5b11728f4" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=119cb00fa7b12e0ebd5a70c9c0a4e7eda2e8c3d6" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] From 70f5cfd33e818e08489b4f192d037b878f0a905d Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Sat, 13 Jul 2019 13:13:57 -0700 Subject: [PATCH 156/523] Update kube-state-metrics from v1.6.0 to v1.7.0-rc.1 * https://github.com/kubernetes/kube-state-metrics/releases/tag/v1.7.0-rc.1 * https://github.com/kubernetes/kube-state-metrics/releases/tag/v1.7.0-rc.0 --- CHANGES.md | 1 + addons/prometheus/exporters/kube-state-metrics/deployment.yaml | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index 517d8e44e..031ac860e 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -17,6 +17,7 @@ Notable changes between versions. #### Addons * Update Prometheus from v2.10.0 to v2.11.0 + * Update kube-state-metrics from v1.6.0 to v1.7.0-rc.1 * Update Grafana from v6.2.4 to v6.2.5 * Update nginx-ingress from v0.24.1 to v0.25.0 * Support `networking.k8s.io/v1beta1` apiVersion diff --git a/addons/prometheus/exporters/kube-state-metrics/deployment.yaml b/addons/prometheus/exporters/kube-state-metrics/deployment.yaml index d53afb964..dc17195f2 100644 --- a/addons/prometheus/exporters/kube-state-metrics/deployment.yaml +++ b/addons/prometheus/exporters/kube-state-metrics/deployment.yaml @@ -24,7 +24,7 @@ spec: serviceAccountName: kube-state-metrics containers: - name: kube-state-metrics - image: quay.io/coreos/kube-state-metrics:v1.6.0 + image: quay.io/coreos/kube-state-metrics:v1.7.0-rc.1 ports: - name: metrics containerPort: 8080 From dfa6bcfecf39602096408f72efc26bc2d32730c8 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Tue, 16 Jul 2019 22:04:00 -0700 Subject: [PATCH 157/523] Relax terraform-provider-ct version constraint * Allow updating terraform-provider-ct to any release beyond v0.3.2, but below v1.0. This relaxes the prior constraint that allowed only v0.3.y provider versions --- CHANGES.md | 2 ++ aws/container-linux/kubernetes/versions.tf | 2 +- azure/container-linux/kubernetes/versions.tf | 2 +- bare-metal/container-linux/kubernetes/versions.tf | 2 +- digital-ocean/container-linux/kubernetes/versions.tf | 2 +- google-cloud/container-linux/kubernetes/versions.tf | 2 +- 6 files changed, 7 insertions(+), 5 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 031ac860e..a2276bfc7 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -7,6 +7,8 @@ Notable changes between versions. * Upgrade Calico from v3.7.3 to [v3.8.0](https://docs.projectcalico.org/v3.8/release-notes/) * Enable CNI `bandwidth` plugin for [traffic shaping](https://kubernetes.io/docs/concepts/extend-kubernetes/compute-storage-net/network-plugins/#support-traffic-shaping) * Run `kube-apiserver` with lower privilege user (nobody) ([#506](https://github.com/poseidon/typhoon/pull/506)) +* Relax `terraform-provider-ct` version constraint (v0.3.2+) + * Allow provider versions below v1.0.0 (e.g. upgrading to v0.4) #### Google Cloud diff --git a/aws/container-linux/kubernetes/versions.tf b/aws/container-linux/kubernetes/versions.tf index 19d087c68..62ed9ebfe 100644 --- a/aws/container-linux/kubernetes/versions.tf +++ b/aws/container-linux/kubernetes/versions.tf @@ -4,7 +4,7 @@ terraform { required_version = "~> 0.12.0" required_providers { aws = "~> 2.7" - ct = "~> 0.3.2" + ct = "~> 0.3" template = "~> 2.1" null = "~> 2.1" } diff --git a/azure/container-linux/kubernetes/versions.tf b/azure/container-linux/kubernetes/versions.tf index e4e0a55ba..f42de22a4 100644 --- a/azure/container-linux/kubernetes/versions.tf +++ b/azure/container-linux/kubernetes/versions.tf @@ -4,7 +4,7 @@ terraform { required_version = "~> 0.12.0" required_providers { azurerm = "~> 1.27" - ct = "~> 0.3.2" + ct = "~> 0.3" template = "~> 2.1" null = "~> 2.1" } diff --git a/bare-metal/container-linux/kubernetes/versions.tf b/bare-metal/container-linux/kubernetes/versions.tf index 166ee67de..6caf55c73 100644 --- a/bare-metal/container-linux/kubernetes/versions.tf +++ b/bare-metal/container-linux/kubernetes/versions.tf @@ -4,7 +4,7 @@ terraform { required_version = "~> 0.12.0" required_providers { matchbox = "~> 0.3.0" - ct = "~> 0.3.2" + ct = "~> 0.3" template = "~> 2.1" null = "~> 2.1" } diff --git a/digital-ocean/container-linux/kubernetes/versions.tf b/digital-ocean/container-linux/kubernetes/versions.tf index e10ec96d3..1690ab5d2 100644 --- a/digital-ocean/container-linux/kubernetes/versions.tf +++ b/digital-ocean/container-linux/kubernetes/versions.tf @@ -4,7 +4,7 @@ terraform { required_version = "~> 0.12.0" required_providers { digitalocean = "~> 1.3" - ct = "~> 0.3.2" + ct = "~> 0.3" template = "~> 2.1" null = "~> 2.1" } diff --git a/google-cloud/container-linux/kubernetes/versions.tf b/google-cloud/container-linux/kubernetes/versions.tf index 9689ce3b2..2dc04ad21 100644 --- a/google-cloud/container-linux/kubernetes/versions.tf +++ b/google-cloud/container-linux/kubernetes/versions.tf @@ -4,7 +4,7 @@ terraform { required_version = "~> 0.12.0" required_providers { google = "~> 2.5" - ct = "~> 0.3.2" + ct = "~> 0.3" template = "~> 2.1" null = "~> 2.1" } From eb92f67125da5410e0141fac24d0c8237edd01f0 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Sun, 7 Jul 2019 15:41:04 -0700 Subject: [PATCH 158/523] Start prototype of Fedora CoreOS on bare-metal * Use terraform-provider-ct v0.4.0 with Fedora CoreOS Config support (not yet released) --- .../fedora-coreos/kubernetes/bootkube.tf | 19 +++ .../kubernetes/fcc/controller.yaml | 28 ++++ .../fedora-coreos/kubernetes/fcc/worker.yaml | 29 ++++ bare-metal/fedora-coreos/kubernetes/groups.tf | 22 +++ .../fedora-coreos/kubernetes/profiles.tf | 95 +++++++++++ .../fedora-coreos/kubernetes/variables.tf | 158 ++++++++++++++++++ .../fedora-coreos/kubernetes/versions.tf | 11 ++ 7 files changed, 362 insertions(+) create mode 100644 bare-metal/fedora-coreos/kubernetes/bootkube.tf create mode 100644 bare-metal/fedora-coreos/kubernetes/fcc/controller.yaml create mode 100644 bare-metal/fedora-coreos/kubernetes/fcc/worker.yaml create mode 100644 bare-metal/fedora-coreos/kubernetes/groups.tf create mode 100644 bare-metal/fedora-coreos/kubernetes/profiles.tf create mode 100644 bare-metal/fedora-coreos/kubernetes/variables.tf create mode 100644 bare-metal/fedora-coreos/kubernetes/versions.tf diff --git a/bare-metal/fedora-coreos/kubernetes/bootkube.tf b/bare-metal/fedora-coreos/kubernetes/bootkube.tf new file mode 100644 index 000000000..33df7bd05 --- /dev/null +++ b/bare-metal/fedora-coreos/kubernetes/bootkube.tf @@ -0,0 +1,19 @@ +# Self-hosted Kubernetes assets (kubeconfig, manifests) +module "bootkube" { + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=119cb00fa7b12e0ebd5a70c9c0a4e7eda2e8c3d6" + + cluster_name = var.cluster_name + api_servers = [var.k8s_domain_name] + etcd_servers = var.controller_domains + asset_dir = var.asset_dir + networking = var.networking + network_mtu = var.network_mtu + network_ip_autodetection_method = var.network_ip_autodetection_method + pod_cidr = var.pod_cidr + service_cidr = var.service_cidr + cluster_domain_suffix = var.cluster_domain_suffix + enable_reporting = var.enable_reporting + enable_aggregation = var.enable_aggregation +} + + diff --git a/bare-metal/fedora-coreos/kubernetes/fcc/controller.yaml b/bare-metal/fedora-coreos/kubernetes/fcc/controller.yaml new file mode 100644 index 000000000..e4d8934cd --- /dev/null +++ b/bare-metal/fedora-coreos/kubernetes/fcc/controller.yaml @@ -0,0 +1,28 @@ +--- +variant: fcos +version: 1.0.0 +storage: + files: + - path: /etc/kubernetes/kubelet.env + filesystem: root + mode: 0644 + contents: + inline: | + KUBELET_IMAGE_URL=docker://k8s.gcr.io/hyperkube + KUBELET_IMAGE_TAG=v1.15.0 + - path: /etc/hostname + filesystem: root + mode: 0644 + contents: + inline: + ${domain_name} + - path: /etc/sysctl.d/max-user-watches.conf + filesystem: root + contents: + inline: | + fs.inotify.max_user_watches=16184 +passwd: + users: + - name: core + ssh_authorized_keys: + - ${ssh_authorized_key} diff --git a/bare-metal/fedora-coreos/kubernetes/fcc/worker.yaml b/bare-metal/fedora-coreos/kubernetes/fcc/worker.yaml new file mode 100644 index 000000000..8e18b2f10 --- /dev/null +++ b/bare-metal/fedora-coreos/kubernetes/fcc/worker.yaml @@ -0,0 +1,29 @@ +--- +variant: fcos +version: 1.0.0 +storage: + files: + - path: /etc/kubernetes/kubelet.env + filesystem: root + mode: 0644 + contents: + inline: | + KUBELET_IMAGE_URL=docker://k8s.gcr.io/hyperkube + KUBELET_IMAGE_TAG=v1.15.0 + - path: /etc/hostname + filesystem: root + mode: 0644 + contents: + inline: + ${domain_name} + - path: /etc/sysctl.d/max-user-watches.conf + filesystem: root + contents: + inline: | + fs.inotify.max_user_watches=16184 +passwd: + users: + - name: core + ssh_authorized_keys: + - ${ssh_authorized_key} + diff --git a/bare-metal/fedora-coreos/kubernetes/groups.tf b/bare-metal/fedora-coreos/kubernetes/groups.tf new file mode 100644 index 000000000..fd9f089f3 --- /dev/null +++ b/bare-metal/fedora-coreos/kubernetes/groups.tf @@ -0,0 +1,22 @@ +# Match each controller or worker to a profile + +resource "matchbox_group" "controller" { + count = length(var.controller_names) + name = format("%s-%s", var.cluster_name, var.controller_names[count.index]) + profile = matchbox_profile.controllers.*.name[count.index] + + selector = { + mac = var.controller_macs[count.index] + } +} + +resource "matchbox_group" "worker" { + count = length(var.worker_names) + name = format("%s-%s", var.cluster_name, var.worker_names[count.index]) + profile = matchbox_profile.workers.*.name[count.index] + + selector = { + mac = var.worker_macs[count.index] + } +} + diff --git a/bare-metal/fedora-coreos/kubernetes/profiles.tf b/bare-metal/fedora-coreos/kubernetes/profiles.tf new file mode 100644 index 000000000..071b8ec56 --- /dev/null +++ b/bare-metal/fedora-coreos/kubernetes/profiles.tf @@ -0,0 +1,95 @@ +locals { + remote_kernel = "https://builds.coreos.fedoraproject.org/prod/streams/${var.os_stream}/builds/${var.os_version}/fedora-coreos-${var.os_version}-installer-kernel" + remote_initrd = "https://builds.coreos.fedoraproject.org/prod/streams/${var.os_stream}/builds/${var.os_version}/fedora-coreos-${var.os_version}-installer-initramfs.img" + remote_args = [ + "ip=dhcp", + "rd.neednet=1", + "coreos.inst=yes", + "coreos.inst.image_url=https://builds.coreos.fedoraproject.org/prod/streams/${var.os_stream}/builds/${var.os_version}/fedora-coreos-${var.os_version}-metal.raw.gz", + "coreos.inst.ignition_url=${var.matchbox_http_endpoint}/ignition?uuid=$${uuid}&mac=$${mac:hexhyp}", + "coreos.inst.install_dev=${var.install_disk}" + ] + + cached_kernel = "/assets/fedora-coreos/fedora-coreos-${var.os_version}-installer-kernel" + cached_initrd = "/assets/fedora-coreos/fedora-coreos-${var.os_version}-installer-initramfs.img" + cached_args = [ + "ip=dhcp", + "rd.neednet=1", + "coreos.inst=yes", + "coreos.inst.image_url=${var.matchbox_http_endpoint}/assets/fedora-coreos/fedora-coreos-${var.os_version}-metal-bios.raw.gz", + "coreos.inst.ignition_url=${var.matchbox_http_endpoint}/ignition?uuid=$${uuid}&mac=$${mac:hexhyp}", + "coreos.inst.install_dev=${var.install_disk}" + ] + + kernel = var.cached_install == "true" ? local.cached_kernel : local.remote_kernel + initrd = var.cached_install == "true" ? local.cached_initrd : local.remote_initrd + args = var.cached_install == "true" ? local.cached_args : local.remote_args +} + + +// Fedora CoreOS controller profile +resource "matchbox_profile" "controllers" { + count = length(var.controller_names) + name = format("%s-controller-%s", var.cluster_name, var.controller_names[count.index]) + + kernel = local.kernel + initrd = [ + local.initrd + ] + args = concat(local.args, var.kernel_args) + + raw_ignition = data.ct_config.controller-ignitions.*.rendered[count.index] +} + +data "ct_config" "controller-ignitions" { + count = length(var.controller_names) + + content = data.template_file.controller-configs.*.rendered[count.index] +} + +data "template_file" "controller-configs" { + count = length(var.controller_names) + + template = file("${path.module}/fcc/controller.yaml") + vars = { + domain_name = var.controller_domains[count.index] + etcd_name = var.controller_names[count.index] + etcd_initial_cluster = join(",", formatlist("%s=https://%s:2380", var.controller_names, var.controller_domains)) + cluster_dns_service_ip = module.bootkube.cluster_dns_service_ip + cluster_domain_suffix = var.cluster_domain_suffix + ssh_authorized_key = var.ssh_authorized_key + } +} + +// Fedora CoreOS worker profile +resource "matchbox_profile" "workers" { + count = length(var.worker_names) + name = format("%s-worker-%s", var.cluster_name, var.worker_names[count.index]) + + kernel = local.kernel + initrd = [ + local.initrd + ] + args = concat(local.args, var.kernel_args) + + raw_ignition = data.ct_config.worker-ignitions.*.rendered[count.index] +} + +data "ct_config" "worker-ignitions" { + count = length(var.worker_names) + + content = data.template_file.worker-configs.*.rendered[count.index] +} + +data "template_file" "worker-configs" { + count = length(var.worker_names) + + template = file("${path.module}/fcc/worker.yaml") + vars = { + domain_name = var.worker_domains[count.index] + cluster_dns_service_ip = module.bootkube.cluster_dns_service_ip + cluster_domain_suffix = var.cluster_domain_suffix + ssh_authorized_key = var.ssh_authorized_key + } +} + diff --git a/bare-metal/fedora-coreos/kubernetes/variables.tf b/bare-metal/fedora-coreos/kubernetes/variables.tf new file mode 100644 index 000000000..5cae2d30a --- /dev/null +++ b/bare-metal/fedora-coreos/kubernetes/variables.tf @@ -0,0 +1,158 @@ +variable "cluster_name" { + type = string + description = "Unique cluster name" +} + +# bare-metal + +variable "matchbox_http_endpoint" { + type = string + description = "Matchbox HTTP read-only endpoint (e.g. http://matchbox.example.com:8080)" +} + +variable "os_stream" { + type = string + description = "Fedora CoreOS release stream (e.g. testing, stable)" + default = "testing" +} + +variable "os_version" { + type = string + description = "Fedora CoreOS version to PXE and install (e.g. 30.20190712.0)" +} + +# machines +# Terraform's crude "type system" does not properly support lists of maps so we do this. + +variable "controller_names" { + type = list(string) + description = "Ordered list of controller names (e.g. [node1])" +} + +variable "controller_macs" { + type = list(string) + description = "Ordered list of controller identifying MAC addresses (e.g. [52:54:00:a1:9c:ae])" +} + +variable "controller_domains" { + type = list(string) + description = "Ordered list of controller FQDNs (e.g. [node1.example.com])" +} + +variable "worker_names" { + type = list(string) + description = "Ordered list of worker names (e.g. [node2, node3])" +} + +variable "worker_macs" { + type = list(string) + description = "Ordered list of worker identifying MAC addresses (e.g. [52:54:00:b2:2f:86, 52:54:00:c3:61:77])" +} + +variable "worker_domains" { + type = list(string) + description = "Ordered list of worker FQDNs (e.g. [node2.example.com, node3.example.com])" +} + +variable "snippets" { + type = map(list(string)) + description = "Map from machine names to lists of Fedora CoreOS Config snippets" + default = {} +} + +# configuration + +variable "k8s_domain_name" { + description = "Controller DNS name which resolves to a controller instance. Workers and kubeconfig's will communicate with this endpoint (e.g. cluster.example.com)" + type = string +} + +variable "ssh_authorized_key" { + type = string + description = "SSH public key for user 'core'" +} + +variable "asset_dir" { + description = "Path to a directory where generated assets should be placed (contains secrets)" + type = string +} + +variable "networking" { + description = "Choice of networking provider (flannel or calico)" + type = string + default = "calico" +} + +variable "network_mtu" { + description = "CNI interface MTU (applies to calico only)" + type = string + default = "1480" +} + +variable "network_ip_autodetection_method" { + description = "Method to autodetect the host IPv4 address (applies to calico only)" + type = string + default = "first-found" +} + +variable "pod_cidr" { + description = "CIDR IPv4 range to assign Kubernetes pods" + type = string + default = "10.2.0.0/16" +} + +variable "service_cidr" { + description = < Date: Sat, 13 Jul 2019 23:24:56 -0700 Subject: [PATCH 159/523] Run etcd-member.service across controllers * Running the etcd container with NOTIFY_SOCKET mounted (to use systemd Type=notify) causes podman to hang so for now just use exec * https://github.com/opencontainers/runc/pull/1807 --- .../kubernetes/fcc/controller.yaml | 65 ++++++++++++++++- bare-metal/fedora-coreos/kubernetes/ssh.tf | 73 +++++++++++++++++++ 2 files changed, 135 insertions(+), 3 deletions(-) create mode 100644 bare-metal/fedora-coreos/kubernetes/ssh.tf diff --git a/bare-metal/fedora-coreos/kubernetes/fcc/controller.yaml b/bare-metal/fedora-coreos/kubernetes/fcc/controller.yaml index e4d8934cd..b1d8bdbc0 100644 --- a/bare-metal/fedora-coreos/kubernetes/fcc/controller.yaml +++ b/bare-metal/fedora-coreos/kubernetes/fcc/controller.yaml @@ -1,23 +1,82 @@ --- variant: fcos version: 1.0.0 +systemd: + units: + - name: etcd-member.service + enabled: true + contents: | + [Unit] + Description=etcd (System Container) + Documentation=https://github.com/coreos/etcd + Wants=network-online.target network.target + After=network-online.target + [Service] + # https://github.com/opencontainers/runc/pull/1807 + # Type=notify + # NotifyAccess=exec + Type=exec + Restart=on-failure + RestartSec=10s + TimeoutStartSec=0 + LimitNOFILE=40000 + + ExecStartPre=/bin/chcon -t bin_t /opt/bin/etcd-wrapper + ExecStartPre=-/usr/bin/podman rm etcd + ExecStart=/opt/bin/etcd-wrapper + ExecStop=/usr/bin/podman stop etcd + [Install] + WantedBy=multi-user.target storage: files: + - path: /etc/etcd/etcd.env + mode: 0644 + contents: + inline: | + NOTIFY_SOCKET=/run/systemd/notify + ETCD_NAME=${etcd_name} + ETCD_DATA_DIR=/var/lib/etcd + ETCD_ADVERTISE_CLIENT_URLS=https://${domain_name}:2379 + ETCD_INITIAL_ADVERTISE_PEER_URLS=https://${domain_name}:2380 + ETCD_LISTEN_CLIENT_URLS=https://0.0.0.0:2379 + ETCD_LISTEN_PEER_URLS=https://0.0.0.0:2380 + ETCD_LISTEN_METRICS_URLS=http://0.0.0.0:2381 + ETCD_INITIAL_CLUSTER=${etcd_initial_cluster} + ETCD_STRICT_RECONFIG_CHECK=true + ETCD_TRUSTED_CA_FILE=/etc/ssl/certs/etcd/server-ca.crt + ETCD_CERT_FILE=/etc/ssl/certs/etcd/server.crt + ETCD_KEY_FILE=/etc/ssl/certs/etcd/server.key + ETCD_CLIENT_CERT_AUTH=true + ETCD_PEER_TRUSTED_CA_FILE=/etc/ssl/certs/etcd/peer-ca.crt + ETCD_PEER_CERT_FILE=/etc/ssl/certs/etcd/peer.crt + ETCD_PEER_KEY_FILE=/etc/ssl/certs/etcd/peer.key + ETCD_PEER_CLIENT_CERT_AUTH=true + - path: /opt/bin/etcd-wrapper + mode: 0544 + contents: + inline: | + #!/usr/bin/bash -e + #--volume $${NOTIFY_SOCKET}:/run/systemd/notify \ + set -x + mkdir -p /var/lib/etcd + exec podman run --name etcd \ + --env-file /etc/etcd/etcd.env \ + --network host \ + --volume /var/lib/etcd:/var/lib/etcd:rw,Z \ + --volume /etc/ssl/etcd:/etc/ssl/certs:ro,Z \ + quay.io/coreos/etcd:v3.3.13 - path: /etc/kubernetes/kubelet.env - filesystem: root mode: 0644 contents: inline: | KUBELET_IMAGE_URL=docker://k8s.gcr.io/hyperkube KUBELET_IMAGE_TAG=v1.15.0 - path: /etc/hostname - filesystem: root mode: 0644 contents: inline: ${domain_name} - path: /etc/sysctl.d/max-user-watches.conf - filesystem: root contents: inline: | fs.inotify.max_user_watches=16184 diff --git a/bare-metal/fedora-coreos/kubernetes/ssh.tf b/bare-metal/fedora-coreos/kubernetes/ssh.tf new file mode 100644 index 000000000..f11ad79f8 --- /dev/null +++ b/bare-metal/fedora-coreos/kubernetes/ssh.tf @@ -0,0 +1,73 @@ +# Secure copy etcd TLS assets and kubeconfig to controllers. Activates kubelet.service +resource "null_resource" "copy-controller-secrets" { + count = length(var.controller_names) + + # Without depends_on, remote-exec could start and wait for machines before + # matchbox groups are written, causing a deadlock. + depends_on = [ + matchbox_group.controller, + matchbox_group.worker, + ] + + connection { + type = "ssh" + host = var.controller_domains[count.index] + user = "core" + timeout = "60m" + } + + provisioner "file" { + content = module.bootkube.kubeconfig-kubelet + destination = "$HOME/kubeconfig" + } + + provisioner "file" { + content = module.bootkube.etcd_ca_cert + destination = "$HOME/etcd-client-ca.crt" + } + + provisioner "file" { + content = module.bootkube.etcd_client_cert + destination = "$HOME/etcd-client.crt" + } + + provisioner "file" { + content = module.bootkube.etcd_client_key + destination = "$HOME/etcd-client.key" + } + + provisioner "file" { + content = module.bootkube.etcd_server_cert + destination = "$HOME/etcd-server.crt" + } + + provisioner "file" { + content = module.bootkube.etcd_server_key + destination = "$HOME/etcd-server.key" + } + + provisioner "file" { + content = module.bootkube.etcd_peer_cert + destination = "$HOME/etcd-peer.crt" + } + + provisioner "file" { + content = module.bootkube.etcd_peer_key + destination = "$HOME/etcd-peer.key" + } + + provisioner "remote-exec" { + inline = [ + "sudo mkdir -p /etc/ssl/etcd/etcd", + "sudo mv etcd-client* /etc/ssl/etcd/", + "sudo cp /etc/ssl/etcd/etcd-client-ca.crt /etc/ssl/etcd/etcd/server-ca.crt", + "sudo mv etcd-server.crt /etc/ssl/etcd/etcd/server.crt", + "sudo mv etcd-server.key /etc/ssl/etcd/etcd/server.key", + "sudo cp /etc/ssl/etcd/etcd-client-ca.crt /etc/ssl/etcd/etcd/peer-ca.crt", + "sudo mv etcd-peer.crt /etc/ssl/etcd/etcd/peer.crt", + "sudo mv etcd-peer.key /etc/ssl/etcd/etcd/peer.key", + "sudo mv $HOME/kubeconfig /etc/kubernetes/kubeconfig", + ] + } +} + From 72c94f1c6afc17ef3fe52c73156aa3e9a04cce86 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Sun, 14 Jul 2019 01:39:08 -0700 Subject: [PATCH 160/523] Add Kubelet System Container and bootkube bootstrap * First semi-working cluster using 30.307-metal-bios * Enable CPU, Memory, and BlockIO accounting * Mount /var/lib/kubelet with `rshare` so mounted tmpfs Secrets (e.g. serviceaccount's) are visible within appropriate containers * SELinux relabel /etc/kubernetes so install-cni init containers can write the CNI config to the host /etc/kubernetes/net.d * SELinux relabel /var/lib/kubelet so ConfigMaps can be read by containers * SELinux relabel /opt/cni/bin so install-cni containers can write CNI binaries to the host * Set net.ipv4_conf.all.rp_filter to 1 (not 2, loose mode) to satisfy Calico requirement * Enable the QoS cgroup hierarchy for pod workloads (kubepods, burstable, besteffort). Mount /sys/fs/cgroup and /sys/fs/cgroup/systemd into the Kubelet. Its still rather racy whether Kubelet will fail on ContainerManager Delegation --- .../fedora-coreos/kubernetes/bootkube.tf | 2 + .../kubernetes/fcc/controller.yaml | 167 ++++++++++++++---- .../fedora-coreos/kubernetes/fcc/worker.yaml | 108 ++++++++++- .../fedora-coreos/kubernetes/outputs.tf | 4 + bare-metal/fedora-coreos/kubernetes/ssh.tf | 62 +++++++ 5 files changed, 302 insertions(+), 41 deletions(-) create mode 100644 bare-metal/fedora-coreos/kubernetes/outputs.tf diff --git a/bare-metal/fedora-coreos/kubernetes/bootkube.tf b/bare-metal/fedora-coreos/kubernetes/bootkube.tf index 33df7bd05..06c50989f 100644 --- a/bare-metal/fedora-coreos/kubernetes/bootkube.tf +++ b/bare-metal/fedora-coreos/kubernetes/bootkube.tf @@ -14,6 +14,8 @@ module "bootkube" { cluster_domain_suffix = var.cluster_domain_suffix enable_reporting = var.enable_reporting enable_aggregation = var.enable_aggregation + + trusted_certs_dir = "/etc/pki/tls/certs" } diff --git a/bare-metal/fedora-coreos/kubernetes/fcc/controller.yaml b/bare-metal/fedora-coreos/kubernetes/fcc/controller.yaml index b1d8bdbc0..bd7b4b7a9 100644 --- a/bare-metal/fedora-coreos/kubernetes/fcc/controller.yaml +++ b/bare-metal/fedora-coreos/kubernetes/fcc/controller.yaml @@ -20,19 +20,151 @@ systemd: RestartSec=10s TimeoutStartSec=0 LimitNOFILE=40000 - - ExecStartPre=/bin/chcon -t bin_t /opt/bin/etcd-wrapper + ExecStartPre=/bin/mkdir -p /var/lib/etcd ExecStartPre=-/usr/bin/podman rm etcd - ExecStart=/opt/bin/etcd-wrapper + #--volume $${NOTIFY_SOCKET}:/run/systemd/notify \ + ExecStart=/usr/bin/podman run --name etcd \ + --env-file /etc/etcd/etcd.env \ + --network host \ + --volume /var/lib/etcd:/var/lib/etcd:rw,Z \ + --volume /etc/ssl/etcd:/etc/ssl/certs:ro,Z \ + quay.io/coreos/etcd:v3.3.13 ExecStop=/usr/bin/podman stop etcd [Install] WantedBy=multi-user.target + - name: docker.service + enabled: true + - name: wait-for-dns.service + enabled: true + contents: | + [Unit] + Description=Wait for DNS entries + Before=kubelet.service + [Service] + Type=oneshot + RemainAfterExit=true + ExecStart=/bin/sh -c 'while ! /usr/bin/grep '^[^#[:space:]]' /etc/resolv.conf > /dev/null; do sleep 1; done' + [Install] + RequiredBy=kubelet.service + RequiredBy=etcd-member.service + - name: kubelet.service + contents: | + [Unit] + Description=Kubelet via Hyperkube (System Container) + Wants=rpc-statd.service + [Service] + ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d + ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests + ExecStartPre=/bin/mkdir -p /var/lib/calico + ExecStartPre=/bin/mkdir -p /var/lib/kubelet/volumeplugins + ExecStartPre=/bin/mkdir -p /opt/cni/bin + ExecStartPre=/usr/bin/bash -c "grep 'certificate-authority-data' /etc/kubernetes/kubeconfig | awk '{print $2}' | base64 -d > /etc/kubernetes/ca.crt" + ExecStartPre=-/usr/bin/podman rm kubelet + ExecStart=/usr/bin/podman run --name kubelet \ + --privileged \ + --pid host \ + --network host \ + --volume /etc/kubernetes:/etc/kubernetes:ro,z \ + --volume /usr/lib/os-release:/etc/os-release:ro \ + --volume /etc/ssl/certs:/etc/ssl/certs:ro \ + --volume /lib/modules:/lib/modules:ro \ + --volume /run:/run \ + --volume /sys/fs/cgroup:/sys/fs/cgroup:ro \ + --volume /sys/fs/cgroup/systemd:/sys/fs/cgroup/systemd \ + --volume /etc/pki/tls/certs:/usr/share/ca-certificates:ro \ + --volume /var/lib/calico:/var/lib/calico \ + --volume /var/lib/docker:/var/lib/docker \ + --volume /var/lib/kubelet:/var/lib/kubelet:rshared,z \ + --volume /var/log:/var/log \ + --volume /var/run:/var/run \ + --volume /var/run/lock:/var/run/lock:z \ + --volume /opt/cni/bin:/opt/cni/bin:z \ + --volume /etc/iscsi:/etc/iscsi \ + --volume /sbin/iscsiadm:/sbin/iscsiadm \ + k8s.gcr.io/hyperkube:v1.15.0 /hyperkube kubelet \ + --anonymous-auth=false \ + --authentication-token-webhook \ + --authorization-mode=Webhook \ + --cgroup-driver=systemd \ + --cgroups-per-qos=true \ + --enforce-node-allocatable=pods \ + --client-ca-file=/etc/kubernetes/ca.crt \ + --cluster_dns=${cluster_dns_service_ip} \ + --cluster_domain=${cluster_domain_suffix} \ + --cni-conf-dir=/etc/kubernetes/cni/net.d \ + --exit-on-lock-contention \ + --hostname-override=${domain_name} \ + --kubeconfig=/etc/kubernetes/kubeconfig \ + --lock-file=/var/run/lock/kubelet.lock \ + --network-plugin=cni \ + --node-labels=node-role.kubernetes.io/master \ + --node-labels=node-role.kubernetes.io/controller="true" \ + --pod-manifest-path=/etc/kubernetes/manifests \ + --read-only-port=0 \ + --register-with-taints=node-role.kubernetes.io/master=:NoSchedule \ + --volume-plugin-dir=/var/lib/kubelet/volumeplugins + ExecStop=-/usr/bin/podman stop kubelet + Delegate=yes + Restart=always + RestartSec=10 + [Install] + WantedBy=multi-user.target + - name: kubelet.path + enabled: true + contents: | + [Unit] + Description=Watch for kubeconfig + [Path] + PathExists=/etc/kubernetes/kubeconfig + [Install] + WantedBy=multi-user.target + - name: bootkube.service + contents: | + [Unit] + Description=Bootstrap a Kubernetes control plane + ConditionPathExists=!/opt/bootkube/init_bootkube.done + [Service] + Type=oneshot + RemainAfterExit=true + WorkingDirectory=/opt/bootkube + ExecStart=/usr/bin/bash -c 'set -x && \ + [ -n "$(ls /opt/bootkube/assets/manifests-*/* 2>/dev/null)" ] && mv /opt/bootkube/assets/manifests-*/* /opt/bootkube/assets/manifests && rm -rf /opt/bootkube/assets/manifests-* && exec podman run --name bootkube --privileged \ + --network host \ + --volume /opt/bootkube/assets:/assets \ + --volume /etc/kubernetes:/etc/kubernetes \ + quay.io/coreos/bootkube:v0.14.0 \ + /bootkube start --asset-dir=/assets' + ExecStartPost=/bin/touch /opt/bootkube/init_bootkube.done storage: + directories: + - path: /etc/kubernetes + - path: /opt/bootkube files: + - path: /etc/hostname + mode: 0644 + contents: + inline: + ${domain_name} + - path: /etc/sysctl.d/reverse-path-filter.conf + contents: + inline: | + net.ipv4.conf.all.rp_filter=1 + - path: /etc/sysctl.d/max-user-watches.conf + contents: + inline: | + fs.inotify.max_user_watches=16184 + - path: /etc/systemd/system.conf.d/accounting.conf + contents: + inline: | + [Manager] + DefaultCPUAccounting=yes + DefaultMemoryAccounting=yes + DefaultBlockIOAccounting=yes - path: /etc/etcd/etcd.env mode: 0644 contents: inline: | + # TODO: Use a systemd dropin once podman v1.4.5 is avail. NOTIFY_SOCKET=/run/systemd/notify ETCD_NAME=${etcd_name} ETCD_DATA_DIR=/var/lib/etcd @@ -51,35 +183,6 @@ storage: ETCD_PEER_CERT_FILE=/etc/ssl/certs/etcd/peer.crt ETCD_PEER_KEY_FILE=/etc/ssl/certs/etcd/peer.key ETCD_PEER_CLIENT_CERT_AUTH=true - - path: /opt/bin/etcd-wrapper - mode: 0544 - contents: - inline: | - #!/usr/bin/bash -e - #--volume $${NOTIFY_SOCKET}:/run/systemd/notify \ - set -x - mkdir -p /var/lib/etcd - exec podman run --name etcd \ - --env-file /etc/etcd/etcd.env \ - --network host \ - --volume /var/lib/etcd:/var/lib/etcd:rw,Z \ - --volume /etc/ssl/etcd:/etc/ssl/certs:ro,Z \ - quay.io/coreos/etcd:v3.3.13 - - path: /etc/kubernetes/kubelet.env - mode: 0644 - contents: - inline: | - KUBELET_IMAGE_URL=docker://k8s.gcr.io/hyperkube - KUBELET_IMAGE_TAG=v1.15.0 - - path: /etc/hostname - mode: 0644 - contents: - inline: - ${domain_name} - - path: /etc/sysctl.d/max-user-watches.conf - contents: - inline: | - fs.inotify.max_user_watches=16184 passwd: users: - name: core diff --git a/bare-metal/fedora-coreos/kubernetes/fcc/worker.yaml b/bare-metal/fedora-coreos/kubernetes/fcc/worker.yaml index 8e18b2f10..482bf2c7b 100644 --- a/bare-metal/fedora-coreos/kubernetes/fcc/worker.yaml +++ b/bare-metal/fedora-coreos/kubernetes/fcc/worker.yaml @@ -1,26 +1,116 @@ --- variant: fcos version: 1.0.0 +systemd: + units: + - name: docker.service + enabled: true + - name: wait-for-dns.service + enabled: true + contents: | + [Unit] + Description=Wait for DNS entries + Before=kubelet.service + [Service] + Type=oneshot + RemainAfterExit=true + ExecStart=/bin/sh -c 'while ! /usr/bin/grep '^[^#[:space:]]' /etc/resolv.conf > /dev/null; do sleep 1; done' + [Install] + RequiredBy=kubelet.service + - name: kubelet.service + contents: | + [Unit] + Description=Kubelet via Hyperkube (System Container) + Wants=rpc-statd.service + [Service] + ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d + ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests + ExecStartPre=/bin/mkdir -p /var/lib/calico + ExecStartPre=/bin/mkdir -p /var/lib/kubelet/volumeplugins + ExecStartPre=/bin/mkdir -p /opt/cni/bin + ExecStartPre=/usr/bin/bash -c "grep 'certificate-authority-data' /etc/kubernetes/kubeconfig | awk '{print $2}' | base64 -d > /etc/kubernetes/ca.crt" + ExecStartPre=-/usr/bin/podman rm kubelet + ExecStart=/usr/bin/podman run --name kubelet \ + --privileged \ + --pid host \ + --network host \ + --volume /etc/kubernetes:/etc/kubernetes:ro,z \ + --volume /usr/lib/os-release:/etc/os-release:ro \ + --volume /etc/ssl/certs:/etc/ssl/certs:ro \ + --volume /lib/modules:/lib/modules:ro \ + --volume /run:/run \ + --volume /sys/fs/cgroup:/sys/fs/cgroup:ro \ + --volume /sys/fs/cgroup/systemd:/sys/fs/cgroup/systemd \ + --volume /etc/pki/tls/certs:/usr/share/ca-certificates:ro \ + --volume /var/lib/calico:/var/lib/calico \ + --volume /var/lib/docker:/var/lib/docker \ + --volume /var/lib/kubelet:/var/lib/kubelet:rshared,z \ + --volume /var/log:/var/log \ + --volume /var/run:/var/run \ + --volume /var/run/lock:/var/run/lock:z \ + --volume /opt/cni/bin:/opt/cni/bin:z \ + --volume /etc/iscsi:/etc/iscsi \ + --volume /sbin/iscsiadm:/sbin/iscsiadm \ + k8s.gcr.io/hyperkube:v1.15.0 /hyperkube kubelet \ + --anonymous-auth=false \ + --authentication-token-webhook \ + --authorization-mode=Webhook \ + --cgroup-driver=systemd \ + --cgroups-per-qos=true \ + --enforce-node-allocatable=pods \ + --client-ca-file=/etc/kubernetes/ca.crt \ + --cluster_dns=${cluster_dns_service_ip} \ + --cluster_domain=${cluster_domain_suffix} \ + --cni-conf-dir=/etc/kubernetes/cni/net.d \ + --exit-on-lock-contention \ + --hostname-override=${domain_name} \ + --kubeconfig=/etc/kubernetes/kubeconfig \ + --lock-file=/var/run/lock/kubelet.lock \ + --network-plugin=cni \ + --node-labels=node-role.kubernetes.io/node \ + --pod-manifest-path=/etc/kubernetes/manifests \ + --read-only-port=0 \ + --volume-plugin-dir=/var/lib/kubelet/volumeplugins + ExecStop=-/usr/bin/podman stop kubelet + Delegate=yes + Restart=always + RestartSec=10 + [Install] + WantedBy=multi-user.target + - name: kubelet.path + enabled: true + contents: | + [Unit] + Description=Watch for kubeconfig + [Path] + PathExists=/etc/kubernetes/kubeconfig + [Install] + WantedBy=multi-user.target storage: + directories: + - path: /etc/kubernetes + - path: /opt/bootkube files: - - path: /etc/kubernetes/kubelet.env - filesystem: root - mode: 0644 - contents: - inline: | - KUBELET_IMAGE_URL=docker://k8s.gcr.io/hyperkube - KUBELET_IMAGE_TAG=v1.15.0 - path: /etc/hostname - filesystem: root mode: 0644 contents: inline: ${domain_name} + - path: /etc/sysctl.d/reverse-path-filter.conf + contents: + inline: | + net.ipv4.conf.all.rp_filter=1 - path: /etc/sysctl.d/max-user-watches.conf - filesystem: root contents: inline: | fs.inotify.max_user_watches=16184 + - path: /etc/systemd/system.conf.d/accounting.conf + contents: + inline: | + [Manager] + DefaultCPUAccounting=yes + DefaultMemoryAccounting=yes + DefaultBlockIOAccounting=yes passwd: users: - name: core diff --git a/bare-metal/fedora-coreos/kubernetes/outputs.tf b/bare-metal/fedora-coreos/kubernetes/outputs.tf new file mode 100644 index 000000000..1fd43af63 --- /dev/null +++ b/bare-metal/fedora-coreos/kubernetes/outputs.tf @@ -0,0 +1,4 @@ +output "kubeconfig-admin" { + value = module.bootkube.kubeconfig-admin +} + diff --git a/bare-metal/fedora-coreos/kubernetes/ssh.tf b/bare-metal/fedora-coreos/kubernetes/ssh.tf index f11ad79f8..da8329abc 100644 --- a/bare-metal/fedora-coreos/kubernetes/ssh.tf +++ b/bare-metal/fedora-coreos/kubernetes/ssh.tf @@ -71,3 +71,65 @@ resource "null_resource" "copy-controller-secrets" { } } +# Secure copy kubeconfig to all workers. Activates kubelet.service +resource "null_resource" "copy-worker-secrets" { + count = length(var.worker_names) + + # Without depends_on, remote-exec could start and wait for machines before + # matchbox groups are written, causing a deadlock. + depends_on = [ + matchbox_group.controller, + matchbox_group.worker, + ] + + connection { + type = "ssh" + host = var.worker_domains[count.index] + user = "core" + timeout = "60m" + } + + provisioner "file" { + content = module.bootkube.kubeconfig-kubelet + destination = "$HOME/kubeconfig" + } + + provisioner "remote-exec" { + inline = [ + "sudo mv $HOME/kubeconfig /etc/kubernetes/kubeconfig", + ] + } +} + +# Secure copy bootkube assets to ONE controller and start bootkube to perform +# one-time self-hosted cluster bootstrapping. +resource "null_resource" "bootkube-start" { + # Without depends_on, this remote-exec may start before the kubeconfig copy. + # Terraform only does one task at a time, so it would try to bootstrap + # while no Kubelets are running. + depends_on = [ + null_resource.copy-controller-secrets, + null_resource.copy-worker-secrets, + ] + + connection { + type = "ssh" + host = var.controller_domains[0] + user = "core" + timeout = "15m" + } + + provisioner "file" { + source = var.asset_dir + destination = "$HOME/assets" + } + + provisioner "remote-exec" { + inline = [ + "sudo mv $HOME/assets /opt/bootkube", + "sudo systemctl start bootkube", + ] + } +} + + From ce45e123fe047441efb503c683a652103cc2f573 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Tue, 16 Jul 2019 23:32:49 -0700 Subject: [PATCH 161/523] Port Typhoon Fedora CoreOS support to AWS * Use the newly minted "Fedora CoreOS Preview" AMI * Remove iscsi, kubelet.path activation, and kubeconfig distribution * As usual, bare-metal efforts make cloud provider ports much easier --- aws/fedora-coreos/kubernetes/LICENSE | 23 ++ aws/fedora-coreos/kubernetes/README.md | 23 ++ aws/fedora-coreos/kubernetes/ami.tf | 21 + aws/fedora-coreos/kubernetes/bootkube.tf | 19 + aws/fedora-coreos/kubernetes/controllers.tf | 86 +++++ .../kubernetes/fcc/controller.yaml | 179 +++++++++ aws/fedora-coreos/kubernetes/network.tf | 67 ++++ aws/fedora-coreos/kubernetes/nlb.tf | 94 +++++ aws/fedora-coreos/kubernetes/outputs.tf | 54 +++ aws/fedora-coreos/kubernetes/security.tf | 364 ++++++++++++++++++ aws/fedora-coreos/kubernetes/ssh.tf | 92 +++++ aws/fedora-coreos/kubernetes/variables.tf | 156 ++++++++ aws/fedora-coreos/kubernetes/versions.tf | 11 + aws/fedora-coreos/kubernetes/workers.tf | 23 ++ aws/fedora-coreos/kubernetes/workers/ami.tf | 21 + .../kubernetes/workers/fcc/worker.yaml | 108 ++++++ .../kubernetes/workers/ingress.tf | 48 +++ .../kubernetes/workers/outputs.tf | 10 + .../kubernetes/workers/variables.tf | 107 +++++ .../kubernetes/workers/versions.tf | 4 + .../kubernetes/workers/workers.tf | 89 +++++ 21 files changed, 1599 insertions(+) create mode 100644 aws/fedora-coreos/kubernetes/LICENSE create mode 100644 aws/fedora-coreos/kubernetes/README.md create mode 100644 aws/fedora-coreos/kubernetes/ami.tf create mode 100644 aws/fedora-coreos/kubernetes/bootkube.tf create mode 100644 aws/fedora-coreos/kubernetes/controllers.tf create mode 100644 aws/fedora-coreos/kubernetes/fcc/controller.yaml create mode 100644 aws/fedora-coreos/kubernetes/network.tf create mode 100644 aws/fedora-coreos/kubernetes/nlb.tf create mode 100644 aws/fedora-coreos/kubernetes/outputs.tf create mode 100644 aws/fedora-coreos/kubernetes/security.tf create mode 100644 aws/fedora-coreos/kubernetes/ssh.tf create mode 100644 aws/fedora-coreos/kubernetes/variables.tf create mode 100644 aws/fedora-coreos/kubernetes/versions.tf create mode 100644 aws/fedora-coreos/kubernetes/workers.tf create mode 100644 aws/fedora-coreos/kubernetes/workers/ami.tf create mode 100644 aws/fedora-coreos/kubernetes/workers/fcc/worker.yaml create mode 100644 aws/fedora-coreos/kubernetes/workers/ingress.tf create mode 100644 aws/fedora-coreos/kubernetes/workers/outputs.tf create mode 100644 aws/fedora-coreos/kubernetes/workers/variables.tf create mode 100644 aws/fedora-coreos/kubernetes/workers/versions.tf create mode 100644 aws/fedora-coreos/kubernetes/workers/workers.tf diff --git a/aws/fedora-coreos/kubernetes/LICENSE b/aws/fedora-coreos/kubernetes/LICENSE new file mode 100644 index 000000000..bd9a5eea2 --- /dev/null +++ b/aws/fedora-coreos/kubernetes/LICENSE @@ -0,0 +1,23 @@ +The MIT License (MIT) + +Copyright (c) 2017 Typhoon Authors +Copyright (c) 2017 Dalton Hubble + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. + diff --git a/aws/fedora-coreos/kubernetes/README.md b/aws/fedora-coreos/kubernetes/README.md new file mode 100644 index 000000000..ce8ee9a37 --- /dev/null +++ b/aws/fedora-coreos/kubernetes/README.md @@ -0,0 +1,23 @@ +# Typhoon + +Typhoon is a minimal and free Kubernetes distribution. + +* Minimal, stable base Kubernetes distribution +* Declarative infrastructure and configuration +* Free (freedom and cost) and privacy-respecting +* Practical for labs, datacenters, and clouds + +Typhoon distributes upstream Kubernetes, architectural conventions, and cluster addons, much like a GNU/Linux distribution provides the Linux kernel and userspace components. + +## Features + +* Kubernetes v1.15.0 (upstream, via [kubernetes-incubator/bootkube](https://github.com/kubernetes-incubator/bootkube)) +* Single or multi-master, [Calico](https://www.projectcalico.org/) or [flannel](https://github.com/coreos/flannel) networking +* On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/) +* Advanced features like [worker pools](https://typhoon.psdn.io/advanced/worker-pools/), [spot](https://typhoon.psdn.io/cl/aws/#spot) workers, and [snippets](https://typhoon.psdn.io/advanced/customization/#container-linux) customization +* Ready for Ingress, Prometheus, Grafana, and other optional [addons](https://typhoon.psdn.io/addons/overview/) + +## Docs + +Please see the [official docs](https://typhoon.psdn.io) and the AWS [tutorial](https://typhoon.psdn.io/cl/aws/). + diff --git a/aws/fedora-coreos/kubernetes/ami.tf b/aws/fedora-coreos/kubernetes/ami.tf new file mode 100644 index 000000000..f4ecec661 --- /dev/null +++ b/aws/fedora-coreos/kubernetes/ami.tf @@ -0,0 +1,21 @@ + +data "aws_ami" "fedora-coreos" { + most_recent = true + owners = ["125523088429"] + + filter { + name = "architecture" + values = ["x86_64"] + } + + filter { + name = "virtualization-type" + values = ["hvm"] + } + + // pin on known ok versions as preview matures + filter { + name = "name" + values = ["fedora-coreos-30.20190716.1-hvm"] + } +} diff --git a/aws/fedora-coreos/kubernetes/bootkube.tf b/aws/fedora-coreos/kubernetes/bootkube.tf new file mode 100644 index 000000000..9c58da8e1 --- /dev/null +++ b/aws/fedora-coreos/kubernetes/bootkube.tf @@ -0,0 +1,19 @@ +# Self-hosted Kubernetes assets (kubeconfig, manifests) +module "bootkube" { + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=119cb00fa7b12e0ebd5a70c9c0a4e7eda2e8c3d6" + + cluster_name = var.cluster_name + api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] + etcd_servers = aws_route53_record.etcds.*.fqdn + asset_dir = var.asset_dir + networking = var.networking + network_mtu = var.network_mtu + pod_cidr = var.pod_cidr + service_cidr = var.service_cidr + cluster_domain_suffix = var.cluster_domain_suffix + enable_reporting = var.enable_reporting + enable_aggregation = var.enable_aggregation + + trusted_certs_dir = "/etc/pki/tls/certs" +} + diff --git a/aws/fedora-coreos/kubernetes/controllers.tf b/aws/fedora-coreos/kubernetes/controllers.tf new file mode 100644 index 000000000..821ad6484 --- /dev/null +++ b/aws/fedora-coreos/kubernetes/controllers.tf @@ -0,0 +1,86 @@ +# Discrete DNS records for each controller's private IPv4 for etcd usage +resource "aws_route53_record" "etcds" { + count = var.controller_count + + # DNS Zone where record should be created + zone_id = var.dns_zone_id + + name = format("%s-etcd%d.%s.", var.cluster_name, count.index, var.dns_zone) + type = "A" + ttl = 300 + + # private IPv4 address for etcd + records = [aws_instance.controllers.*.private_ip[count.index]] +} + +# Controller instances +resource "aws_instance" "controllers" { + count = var.controller_count + + tags = { + Name = "${var.cluster_name}-controller-${count.index}" + } + + instance_type = var.controller_type + + ami = data.aws_ami.fedora-coreos.image_id + user_data = data.ct_config.controller-ignitions.*.rendered[count.index] + + # storage + root_block_device { + volume_type = var.disk_type + volume_size = var.disk_size + iops = var.disk_iops + } + + # network + associate_public_ip_address = true + subnet_id = aws_subnet.public.*.id[count.index] + vpc_security_group_ids = [aws_security_group.controller.id] + + lifecycle { + ignore_changes = [ + ami, + user_data, + ] + } +} + +# Controller Ignition configs +data "ct_config" "controller-ignitions" { + count = var.controller_count + content = data.template_file.controller-configs.*.rendered[count.index] + strict = true + snippets = var.controller_snippets +} + +# Controller Fedora CoreOS configs +data "template_file" "controller-configs" { + count = var.controller_count + + template = file("${path.module}/fcc/controller.yaml") + + vars = { + # Cannot use cyclic dependencies on controllers or their DNS records + etcd_name = "etcd${count.index}" + etcd_domain = "${var.cluster_name}-etcd${count.index}.${var.dns_zone}" + # etcd0=https://cluster-etcd0.example.com,etcd1=https://cluster-etcd1.example.com,... + etcd_initial_cluster = join(",", data.template_file.etcds.*.rendered) + kubeconfig = indent(10, module.bootkube.kubeconfig-kubelet) + ssh_authorized_key = var.ssh_authorized_key + cluster_dns_service_ip = cidrhost(var.service_cidr, 10) + cluster_domain_suffix = var.cluster_domain_suffix + } +} + +data "template_file" "etcds" { + count = var.controller_count + template = "etcd$${index}=https://$${cluster_name}-etcd$${index}.$${dns_zone}:2380" + + vars = { + index = count.index + cluster_name = var.cluster_name + dns_zone = var.dns_zone + } +} + diff --git a/aws/fedora-coreos/kubernetes/fcc/controller.yaml b/aws/fedora-coreos/kubernetes/fcc/controller.yaml new file mode 100644 index 000000000..7130c3047 --- /dev/null +++ b/aws/fedora-coreos/kubernetes/fcc/controller.yaml @@ -0,0 +1,179 @@ +--- +variant: fcos +version: 1.0.0 +systemd: + units: + - name: etcd-member.service + enabled: true + contents: | + [Unit] + Description=etcd (System Container) + Documentation=https://github.com/coreos/etcd + Wants=network-online.target network.target + After=network-online.target + [Service] + # https://github.com/opencontainers/runc/pull/1807 + # Type=notify + # NotifyAccess=exec + Type=exec + Restart=on-failure + RestartSec=10s + TimeoutStartSec=0 + LimitNOFILE=40000 + ExecStartPre=/bin/mkdir -p /var/lib/etcd + ExecStartPre=-/usr/bin/podman rm etcd + #--volume $${NOTIFY_SOCKET}:/run/systemd/notify \ + ExecStart=/usr/bin/podman run --name etcd \ + --env-file /etc/etcd/etcd.env \ + --network host \ + --volume /var/lib/etcd:/var/lib/etcd:rw,Z \ + --volume /etc/ssl/etcd:/etc/ssl/certs:ro,Z \ + quay.io/coreos/etcd:v3.3.13 + ExecStop=/usr/bin/podman stop etcd + [Install] + WantedBy=multi-user.target + - name: docker.service + enabled: true + - name: wait-for-dns.service + enabled: true + contents: | + [Unit] + Description=Wait for DNS entries + Before=kubelet.service + [Service] + Type=oneshot + RemainAfterExit=true + ExecStart=/bin/sh -c 'while ! /usr/bin/grep '^[^#[:space:]]' /etc/resolv.conf > /dev/null; do sleep 1; done' + [Install] + RequiredBy=kubelet.service + RequiredBy=etcd-member.service + - name: kubelet.service + enabled: true + contents: | + [Unit] + Description=Kubelet via Hyperkube (System Container) + Wants=rpc-statd.service + [Service] + ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d + ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests + ExecStartPre=/bin/mkdir -p /var/lib/calico + ExecStartPre=/bin/mkdir -p /var/lib/kubelet/volumeplugins + ExecStartPre=/bin/mkdir -p /opt/cni/bin + ExecStartPre=/usr/bin/bash -c "grep 'certificate-authority-data' /etc/kubernetes/kubeconfig | awk '{print $2}' | base64 -d > /etc/kubernetes/ca.crt" + ExecStartPre=-/usr/bin/podman rm kubelet + ExecStart=/usr/bin/podman run --name kubelet \ + --privileged \ + --pid host \ + --network host \ + --volume /etc/kubernetes:/etc/kubernetes:ro,z \ + --volume /usr/lib/os-release:/etc/os-release:ro \ + --volume /etc/ssl/certs:/etc/ssl/certs:ro \ + --volume /lib/modules:/lib/modules:ro \ + --volume /run:/run \ + --volume /sys/fs/cgroup:/sys/fs/cgroup:ro \ + --volume /sys/fs/cgroup/systemd:/sys/fs/cgroup/systemd \ + --volume /etc/pki/tls/certs:/usr/share/ca-certificates:ro \ + --volume /var/lib/calico:/var/lib/calico \ + --volume /var/lib/docker:/var/lib/docker \ + --volume /var/lib/kubelet:/var/lib/kubelet:rshared,z \ + --volume /var/log:/var/log \ + --volume /var/run:/var/run \ + --volume /var/run/lock:/var/run/lock:z \ + --volume /opt/cni/bin:/opt/cni/bin:z \ + k8s.gcr.io/hyperkube:v1.15.0 /hyperkube kubelet \ + --anonymous-auth=false \ + --authentication-token-webhook \ + --authorization-mode=Webhook \ + --cgroup-driver=systemd \ + --cgroups-per-qos=true \ + --enforce-node-allocatable=pods \ + --client-ca-file=/etc/kubernetes/ca.crt \ + --cluster_dns=${cluster_dns_service_ip} \ + --cluster_domain=${cluster_domain_suffix} \ + --cni-conf-dir=/etc/kubernetes/cni/net.d \ + --exit-on-lock-contention \ + --kubeconfig=/etc/kubernetes/kubeconfig \ + --lock-file=/var/run/lock/kubelet.lock \ + --network-plugin=cni \ + --node-labels=node-role.kubernetes.io/master \ + --node-labels=node-role.kubernetes.io/controller="true" \ + --pod-manifest-path=/etc/kubernetes/manifests \ + --read-only-port=0 \ + --register-with-taints=node-role.kubernetes.io/master=:NoSchedule \ + --volume-plugin-dir=/var/lib/kubelet/volumeplugins + ExecStop=-/usr/bin/podman stop kubelet + Delegate=yes + Restart=always + RestartSec=10 + [Install] + WantedBy=multi-user.target + - name: bootkube.service + contents: | + [Unit] + Description=Bootstrap a Kubernetes control plane + ConditionPathExists=!/opt/bootkube/init_bootkube.done + [Service] + Type=oneshot + RemainAfterExit=true + WorkingDirectory=/opt/bootkube + ExecStart=/usr/bin/bash -c 'set -x && \ + [ -n "$(ls /opt/bootkube/assets/manifests-*/* 2>/dev/null)" ] && mv /opt/bootkube/assets/manifests-*/* /opt/bootkube/assets/manifests && rm -rf /opt/bootkube/assets/manifests-* && exec podman run --name bootkube --privileged \ + --network host \ + --volume /opt/bootkube/assets:/assets \ + --volume /etc/kubernetes:/etc/kubernetes \ + quay.io/coreos/bootkube:v0.14.0 \ + /bootkube start --asset-dir=/assets' + ExecStartPost=/bin/touch /opt/bootkube/init_bootkube.done +storage: + directories: + - path: /etc/kubernetes + - path: /opt/bootkube + files: + - path: /etc/kubernetes/kubeconfig + mode: 0644 + contents: + inline: | + ${kubeconfig} + - path: /etc/sysctl.d/reverse-path-filter.conf + contents: + inline: | + net.ipv4.conf.all.rp_filter=1 + - path: /etc/sysctl.d/max-user-watches.conf + contents: + inline: | + fs.inotify.max_user_watches=16184 + - path: /etc/systemd/system.conf.d/accounting.conf + contents: + inline: | + [Manager] + DefaultCPUAccounting=yes + DefaultMemoryAccounting=yes + DefaultBlockIOAccounting=yes + - path: /etc/etcd/etcd.env + mode: 0644 + contents: + inline: | + # TODO: Use a systemd dropin once podman v1.4.5 is avail. + NOTIFY_SOCKET=/run/systemd/notify + ETCD_NAME=${etcd_name} + ETCD_DATA_DIR=/var/lib/etcd + ETCD_ADVERTISE_CLIENT_URLS=https://${etcd_domain}:2379 + ETCD_INITIAL_ADVERTISE_PEER_URLS=https://${etcd_domain}:2380 + ETCD_LISTEN_CLIENT_URLS=https://0.0.0.0:2379 + ETCD_LISTEN_PEER_URLS=https://0.0.0.0:2380 + ETCD_LISTEN_METRICS_URLS=http://0.0.0.0:2381 + ETCD_INITIAL_CLUSTER=${etcd_initial_cluster} + ETCD_STRICT_RECONFIG_CHECK=true + ETCD_TRUSTED_CA_FILE=/etc/ssl/certs/etcd/server-ca.crt + ETCD_CERT_FILE=/etc/ssl/certs/etcd/server.crt + ETCD_KEY_FILE=/etc/ssl/certs/etcd/server.key + ETCD_CLIENT_CERT_AUTH=true + ETCD_PEER_TRUSTED_CA_FILE=/etc/ssl/certs/etcd/peer-ca.crt + ETCD_PEER_CERT_FILE=/etc/ssl/certs/etcd/peer.crt + ETCD_PEER_KEY_FILE=/etc/ssl/certs/etcd/peer.key + ETCD_PEER_CLIENT_CERT_AUTH=true +passwd: + users: + - name: core + ssh_authorized_keys: + - ${ssh_authorized_key} diff --git a/aws/fedora-coreos/kubernetes/network.tf b/aws/fedora-coreos/kubernetes/network.tf new file mode 100644 index 000000000..a93b3f0cb --- /dev/null +++ b/aws/fedora-coreos/kubernetes/network.tf @@ -0,0 +1,67 @@ +data "aws_availability_zones" "all" { +} + +# Network VPC, gateway, and routes + +resource "aws_vpc" "network" { + cidr_block = var.host_cidr + assign_generated_ipv6_cidr_block = true + enable_dns_support = true + enable_dns_hostnames = true + + tags = { + "Name" = var.cluster_name + } +} + +resource "aws_internet_gateway" "gateway" { + vpc_id = aws_vpc.network.id + + tags = { + "Name" = var.cluster_name + } +} + +resource "aws_route_table" "default" { + vpc_id = aws_vpc.network.id + + route { + cidr_block = "0.0.0.0/0" + gateway_id = aws_internet_gateway.gateway.id + } + + route { + ipv6_cidr_block = "::/0" + gateway_id = aws_internet_gateway.gateway.id + } + + tags = { + "Name" = var.cluster_name + } +} + +# Subnets (one per availability zone) + +resource "aws_subnet" "public" { + count = length(data.aws_availability_zones.all.names) + + vpc_id = aws_vpc.network.id + availability_zone = data.aws_availability_zones.all.names[count.index] + + cidr_block = cidrsubnet(var.host_cidr, 4, count.index) + ipv6_cidr_block = cidrsubnet(aws_vpc.network.ipv6_cidr_block, 8, count.index) + map_public_ip_on_launch = true + assign_ipv6_address_on_creation = true + + tags = { + "Name" = "${var.cluster_name}-public-${count.index}" + } +} + +resource "aws_route_table_association" "public" { + count = length(data.aws_availability_zones.all.names) + + route_table_id = aws_route_table.default.id + subnet_id = aws_subnet.public.*.id[count.index] +} + diff --git a/aws/fedora-coreos/kubernetes/nlb.tf b/aws/fedora-coreos/kubernetes/nlb.tf new file mode 100644 index 000000000..2b87366a7 --- /dev/null +++ b/aws/fedora-coreos/kubernetes/nlb.tf @@ -0,0 +1,94 @@ +# Network Load Balancer DNS Record +resource "aws_route53_record" "apiserver" { + zone_id = var.dns_zone_id + + name = format("%s.%s.", var.cluster_name, var.dns_zone) + type = "A" + + # AWS recommends their special "alias" records for NLBs + alias { + name = aws_lb.nlb.dns_name + zone_id = aws_lb.nlb.zone_id + evaluate_target_health = true + } +} + +# Network Load Balancer for apiservers and ingress +resource "aws_lb" "nlb" { + name = "${var.cluster_name}-nlb" + load_balancer_type = "network" + internal = false + + subnets = aws_subnet.public.*.id + + enable_cross_zone_load_balancing = true +} + +# Forward TCP apiserver traffic to controllers +resource "aws_lb_listener" "apiserver-https" { + load_balancer_arn = aws_lb.nlb.arn + protocol = "TCP" + port = "6443" + + default_action { + type = "forward" + target_group_arn = aws_lb_target_group.controllers.arn + } +} + +# Forward HTTP ingress traffic to workers +resource "aws_lb_listener" "ingress-http" { + load_balancer_arn = aws_lb.nlb.arn + protocol = "TCP" + port = 80 + + default_action { + type = "forward" + target_group_arn = module.workers.target_group_http + } +} + +# Forward HTTPS ingress traffic to workers +resource "aws_lb_listener" "ingress-https" { + load_balancer_arn = aws_lb.nlb.arn + protocol = "TCP" + port = 443 + + default_action { + type = "forward" + target_group_arn = module.workers.target_group_https + } +} + +# Target group of controllers +resource "aws_lb_target_group" "controllers" { + name = "${var.cluster_name}-controllers" + vpc_id = aws_vpc.network.id + target_type = "instance" + + protocol = "TCP" + port = 6443 + + # TCP health check for apiserver + health_check { + protocol = "TCP" + port = 6443 + + # NLBs required to use same healthy and unhealthy thresholds + healthy_threshold = 3 + unhealthy_threshold = 3 + + # Interval between health checks required to be 10 or 30 + interval = 10 + } +} + +# Attach controller instances to apiserver NLB +resource "aws_lb_target_group_attachment" "controllers" { + count = var.controller_count + + target_group_arn = aws_lb_target_group.controllers.arn + target_id = aws_instance.controllers.*.id[count.index] + port = 6443 +} + diff --git a/aws/fedora-coreos/kubernetes/outputs.tf b/aws/fedora-coreos/kubernetes/outputs.tf new file mode 100644 index 000000000..471c3300e --- /dev/null +++ b/aws/fedora-coreos/kubernetes/outputs.tf @@ -0,0 +1,54 @@ +output "kubeconfig-admin" { + value = module.bootkube.kubeconfig-admin +} + +# Outputs for Kubernetes Ingress + +output "ingress_dns_name" { + value = aws_lb.nlb.dns_name + description = "DNS name of the network load balancer for distributing traffic to Ingress controllers" +} + +output "ingress_zone_id" { + value = aws_lb.nlb.zone_id + description = "Route53 zone id of the network load balancer DNS name that can be used in Route53 alias records" +} + +# Outputs for worker pools + +output "vpc_id" { + value = aws_vpc.network.id + description = "ID of the VPC for creating worker instances" +} + +output "subnet_ids" { + value = aws_subnet.public.*.id + description = "List of subnet IDs for creating worker instances" +} + +output "worker_security_groups" { + value = [aws_security_group.worker.id] + description = "List of worker security group IDs" +} + +output "kubeconfig" { + value = module.bootkube.kubeconfig-kubelet +} + +# Outputs for custom load balancing + +output "nlb_id" { + description = "ARN of the Network Load Balancer" + value = aws_lb.nlb.id +} + +output "worker_target_group_http" { + description = "ARN of a target group of workers for HTTP traffic" + value = module.workers.target_group_http +} + +output "worker_target_group_https" { + description = "ARN of a target group of workers for HTTPS traffic" + value = module.workers.target_group_https +} + diff --git a/aws/fedora-coreos/kubernetes/security.tf b/aws/fedora-coreos/kubernetes/security.tf new file mode 100644 index 000000000..aa2f84cb8 --- /dev/null +++ b/aws/fedora-coreos/kubernetes/security.tf @@ -0,0 +1,364 @@ +# Security Groups (instance firewalls) + +# Controller security group + +resource "aws_security_group" "controller" { + name = "${var.cluster_name}-controller" + description = "${var.cluster_name} controller security group" + + vpc_id = aws_vpc.network.id + + tags = { + "Name" = "${var.cluster_name}-controller" + } +} + +resource "aws_security_group_rule" "controller-ssh" { + security_group_id = aws_security_group.controller.id + + type = "ingress" + protocol = "tcp" + from_port = 22 + to_port = 22 + cidr_blocks = ["0.0.0.0/0"] +} + +resource "aws_security_group_rule" "controller-etcd" { + security_group_id = aws_security_group.controller.id + + type = "ingress" + protocol = "tcp" + from_port = 2379 + to_port = 2380 + self = true +} + +# Allow Prometheus to scrape etcd metrics +resource "aws_security_group_rule" "controller-etcd-metrics" { + security_group_id = aws_security_group.controller.id + + type = "ingress" + protocol = "tcp" + from_port = 2381 + to_port = 2381 + source_security_group_id = aws_security_group.worker.id +} + +resource "aws_security_group_rule" "controller-vxlan" { + count = var.networking == "flannel" ? 1 : 0 + + security_group_id = aws_security_group.controller.id + + type = "ingress" + protocol = "udp" + from_port = 4789 + to_port = 4789 + source_security_group_id = aws_security_group.worker.id +} + +resource "aws_security_group_rule" "controller-vxlan-self" { + count = var.networking == "flannel" ? 1 : 0 + + security_group_id = aws_security_group.controller.id + + type = "ingress" + protocol = "udp" + from_port = 4789 + to_port = 4789 + self = true +} + +resource "aws_security_group_rule" "controller-apiserver" { + security_group_id = aws_security_group.controller.id + + type = "ingress" + protocol = "tcp" + from_port = 6443 + to_port = 6443 + cidr_blocks = ["0.0.0.0/0"] +} + +# Allow Prometheus to scrape node-exporter daemonset +resource "aws_security_group_rule" "controller-node-exporter" { + security_group_id = aws_security_group.controller.id + + type = "ingress" + protocol = "tcp" + from_port = 9100 + to_port = 9100 + source_security_group_id = aws_security_group.worker.id +} + +# Allow apiserver to access kubelets for exec, log, port-forward +resource "aws_security_group_rule" "controller-kubelet" { + security_group_id = aws_security_group.controller.id + + type = "ingress" + protocol = "tcp" + from_port = 10250 + to_port = 10250 + source_security_group_id = aws_security_group.worker.id +} + +resource "aws_security_group_rule" "controller-kubelet-self" { + security_group_id = aws_security_group.controller.id + + type = "ingress" + protocol = "tcp" + from_port = 10250 + to_port = 10250 + self = true +} + +resource "aws_security_group_rule" "controller-bgp" { + security_group_id = aws_security_group.controller.id + + type = "ingress" + protocol = "tcp" + from_port = 179 + to_port = 179 + source_security_group_id = aws_security_group.worker.id +} + +resource "aws_security_group_rule" "controller-bgp-self" { + security_group_id = aws_security_group.controller.id + + type = "ingress" + protocol = "tcp" + from_port = 179 + to_port = 179 + self = true +} + +resource "aws_security_group_rule" "controller-ipip" { + security_group_id = aws_security_group.controller.id + + type = "ingress" + protocol = 4 + from_port = 0 + to_port = 0 + source_security_group_id = aws_security_group.worker.id +} + +resource "aws_security_group_rule" "controller-ipip-self" { + security_group_id = aws_security_group.controller.id + + type = "ingress" + protocol = 4 + from_port = 0 + to_port = 0 + self = true +} + +resource "aws_security_group_rule" "controller-ipip-legacy" { + security_group_id = aws_security_group.controller.id + + type = "ingress" + protocol = 94 + from_port = 0 + to_port = 0 + source_security_group_id = aws_security_group.worker.id +} + +resource "aws_security_group_rule" "controller-ipip-legacy-self" { + security_group_id = aws_security_group.controller.id + + type = "ingress" + protocol = 94 + from_port = 0 + to_port = 0 + self = true +} + +resource "aws_security_group_rule" "controller-egress" { + security_group_id = aws_security_group.controller.id + + type = "egress" + protocol = "-1" + from_port = 0 + to_port = 0 + cidr_blocks = ["0.0.0.0/0"] + ipv6_cidr_blocks = ["::/0"] +} + +# Worker security group + +resource "aws_security_group" "worker" { + name = "${var.cluster_name}-worker" + description = "${var.cluster_name} worker security group" + + vpc_id = aws_vpc.network.id + + tags = { + "Name" = "${var.cluster_name}-worker" + } +} + +resource "aws_security_group_rule" "worker-ssh" { + security_group_id = aws_security_group.worker.id + + type = "ingress" + protocol = "tcp" + from_port = 22 + to_port = 22 + cidr_blocks = ["0.0.0.0/0"] +} + +resource "aws_security_group_rule" "worker-http" { + security_group_id = aws_security_group.worker.id + + type = "ingress" + protocol = "tcp" + from_port = 80 + to_port = 80 + cidr_blocks = ["0.0.0.0/0"] +} + +resource "aws_security_group_rule" "worker-https" { + security_group_id = aws_security_group.worker.id + + type = "ingress" + protocol = "tcp" + from_port = 443 + to_port = 443 + cidr_blocks = ["0.0.0.0/0"] +} + +resource "aws_security_group_rule" "worker-vxlan" { + count = var.networking == "flannel" ? 1 : 0 + + security_group_id = aws_security_group.worker.id + + type = "ingress" + protocol = "udp" + from_port = 4789 + to_port = 4789 + source_security_group_id = aws_security_group.controller.id +} + +resource "aws_security_group_rule" "worker-vxlan-self" { + count = var.networking == "flannel" ? 1 : 0 + + security_group_id = aws_security_group.worker.id + + type = "ingress" + protocol = "udp" + from_port = 4789 + to_port = 4789 + self = true +} + +# Allow Prometheus to scrape node-exporter daemonset +resource "aws_security_group_rule" "worker-node-exporter" { + security_group_id = aws_security_group.worker.id + + type = "ingress" + protocol = "tcp" + from_port = 9100 + to_port = 9100 + self = true +} + +resource "aws_security_group_rule" "ingress-health" { + security_group_id = aws_security_group.worker.id + + type = "ingress" + protocol = "tcp" + from_port = 10254 + to_port = 10254 + cidr_blocks = ["0.0.0.0/0"] +} + +# Allow apiserver to access kubelets for exec, log, port-forward +resource "aws_security_group_rule" "worker-kubelet" { + security_group_id = aws_security_group.worker.id + + type = "ingress" + protocol = "tcp" + from_port = 10250 + to_port = 10250 + source_security_group_id = aws_security_group.controller.id +} + +# Allow Prometheus to scrape kubelet metrics +resource "aws_security_group_rule" "worker-kubelet-self" { + security_group_id = aws_security_group.worker.id + + type = "ingress" + protocol = "tcp" + from_port = 10250 + to_port = 10250 + self = true +} + +resource "aws_security_group_rule" "worker-bgp" { + security_group_id = aws_security_group.worker.id + + type = "ingress" + protocol = "tcp" + from_port = 179 + to_port = 179 + source_security_group_id = aws_security_group.controller.id +} + +resource "aws_security_group_rule" "worker-bgp-self" { + security_group_id = aws_security_group.worker.id + + type = "ingress" + protocol = "tcp" + from_port = 179 + to_port = 179 + self = true +} + +resource "aws_security_group_rule" "worker-ipip" { + security_group_id = aws_security_group.worker.id + + type = "ingress" + protocol = 4 + from_port = 0 + to_port = 0 + source_security_group_id = aws_security_group.controller.id +} + +resource "aws_security_group_rule" "worker-ipip-self" { + security_group_id = aws_security_group.worker.id + + type = "ingress" + protocol = 4 + from_port = 0 + to_port = 0 + self = true +} + +resource "aws_security_group_rule" "worker-ipip-legacy" { + security_group_id = aws_security_group.worker.id + + type = "ingress" + protocol = 94 + from_port = 0 + to_port = 0 + source_security_group_id = aws_security_group.controller.id +} + +resource "aws_security_group_rule" "worker-ipip-legacy-self" { + security_group_id = aws_security_group.worker.id + + type = "ingress" + protocol = 94 + from_port = 0 + to_port = 0 + self = true +} + +resource "aws_security_group_rule" "worker-egress" { + security_group_id = aws_security_group.worker.id + + type = "egress" + protocol = "-1" + from_port = 0 + to_port = 0 + cidr_blocks = ["0.0.0.0/0"] + ipv6_cidr_blocks = ["::/0"] +} + diff --git a/aws/fedora-coreos/kubernetes/ssh.tf b/aws/fedora-coreos/kubernetes/ssh.tf new file mode 100644 index 000000000..09e31c192 --- /dev/null +++ b/aws/fedora-coreos/kubernetes/ssh.tf @@ -0,0 +1,92 @@ +# Secure copy etcd TLS assets to controllers. +resource "null_resource" "copy-controller-secrets" { + count = var.controller_count + + connection { + type = "ssh" + host = aws_instance.controllers.*.public_ip[count.index] + user = "core" + timeout = "15m" + } + + provisioner "file" { + content = module.bootkube.etcd_ca_cert + destination = "$HOME/etcd-client-ca.crt" + } + + provisioner "file" { + content = module.bootkube.etcd_client_cert + destination = "$HOME/etcd-client.crt" + } + + provisioner "file" { + content = module.bootkube.etcd_client_key + destination = "$HOME/etcd-client.key" + } + + provisioner "file" { + content = module.bootkube.etcd_server_cert + destination = "$HOME/etcd-server.crt" + } + + provisioner "file" { + content = module.bootkube.etcd_server_key + destination = "$HOME/etcd-server.key" + } + + provisioner "file" { + content = module.bootkube.etcd_peer_cert + destination = "$HOME/etcd-peer.crt" + } + + provisioner "file" { + content = module.bootkube.etcd_peer_key + destination = "$HOME/etcd-peer.key" + } + + provisioner "remote-exec" { + inline = [ + "sudo mkdir -p /etc/ssl/etcd/etcd", + "sudo mv etcd-client* /etc/ssl/etcd/", + "sudo cp /etc/ssl/etcd/etcd-client-ca.crt /etc/ssl/etcd/etcd/server-ca.crt", + "sudo mv etcd-server.crt /etc/ssl/etcd/etcd/server.crt", + "sudo mv etcd-server.key /etc/ssl/etcd/etcd/server.key", + "sudo cp /etc/ssl/etcd/etcd-client-ca.crt /etc/ssl/etcd/etcd/peer-ca.crt", + "sudo mv etcd-peer.crt /etc/ssl/etcd/etcd/peer.crt", + "sudo mv etcd-peer.key /etc/ssl/etcd/etcd/peer.key", + "sudo chown -R etcd:etcd /etc/ssl/etcd", + "sudo chmod -R 500 /etc/ssl/etcd", + ] + } +} + +# Secure copy bootkube assets to ONE controller and start bootkube to perform +# one-time self-hosted cluster bootstrapping. +resource "null_resource" "bootkube-start" { + depends_on = [ + module.bootkube, + module.workers, + aws_route53_record.apiserver, + null_resource.copy-controller-secrets, + ] + + connection { + type = "ssh" + host = aws_instance.controllers[0].public_ip + user = "core" + timeout = "15m" + } + + provisioner "file" { + source = var.asset_dir + destination = "$HOME/assets" + } + + provisioner "remote-exec" { + inline = [ + "sudo mv $HOME/assets /opt/bootkube", + "sudo systemctl start bootkube", + ] + } +} + diff --git a/aws/fedora-coreos/kubernetes/variables.tf b/aws/fedora-coreos/kubernetes/variables.tf new file mode 100644 index 000000000..8df68d856 --- /dev/null +++ b/aws/fedora-coreos/kubernetes/variables.tf @@ -0,0 +1,156 @@ +variable "cluster_name" { + type = string + description = "Unique cluster name (prepended to dns_zone)" +} + +# AWS + +variable "dns_zone" { + type = string + description = "AWS Route53 DNS Zone (e.g. aws.example.com)" +} + +variable "dns_zone_id" { + type = string + description = "AWS Route53 DNS Zone ID (e.g. Z3PAABBCFAKEC0)" +} + +# instances + +variable "controller_count" { + type = string + default = "1" + description = "Number of controllers (i.e. masters)" +} + +variable "worker_count" { + type = string + default = "1" + description = "Number of workers" +} + +variable "controller_type" { + type = string + default = "t3.small" + description = "EC2 instance type for controllers" +} + +variable "worker_type" { + type = string + default = "t3.small" + description = "EC2 instance type for workers" +} + +variable "os_image" { + type = string + default = "coreos-stable" + description = "AMI channel for Fedora CoreOS (not yet used)" +} + +variable "disk_size" { + type = string + default = "40" + description = "Size of the EBS volume in GB" +} + +variable "disk_type" { + type = string + default = "gp2" + description = "Type of the EBS volume (e.g. standard, gp2, io1)" +} + +variable "disk_iops" { + type = string + default = "0" + description = "IOPS of the EBS volume (e.g. 100)" +} + +variable "worker_price" { + type = string + default = "" + description = "Spot price in USD for autoscaling group spot instances. Leave as default empty string for autoscaling group to use on-demand instances. Note, switching in-place from spot to on-demand is not possible: https://github.com/terraform-providers/terraform-provider-aws/issues/4320" +} + +variable "worker_target_groups" { + type = list(string) + description = "Additional target group ARNs to which worker instances should be added" + default = [] +} + +variable "controller_snippets" { + type = list(string) + description = "Controller Fedora CoreOS Config snippets" + default = [] +} + +variable "worker_snippets" { + type = list(string) + description = "Worker Fedora CoreOS Config snippets" + default = [] +} + +# configuration + +variable "ssh_authorized_key" { + type = string + description = "SSH public key for user 'core'" +} + +variable "asset_dir" { + description = "Path to a directory where generated assets should be placed (contains secrets)" + type = string +} + +variable "networking" { + description = "Choice of networking provider (calico or flannel)" + type = string + default = "calico" +} + +variable "network_mtu" { + description = "CNI interface MTU (applies to calico only). Use 8981 if using instances types with Jumbo frames." + type = string + default = "1480" +} + +variable "host_cidr" { + description = "CIDR IPv4 range to assign to EC2 nodes" + type = string + default = "10.0.0.0/16" +} + +variable "pod_cidr" { + description = "CIDR IPv4 range to assign Kubernetes pods" + type = string + default = "10.2.0.0/16" +} + +variable "service_cidr" { + description = < /dev/null; do sleep 1; done' + [Install] + RequiredBy=kubelet.service + - name: kubelet.service + enabled: true + contents: | + [Unit] + Description=Kubelet via Hyperkube (System Container) + Wants=rpc-statd.service + [Service] + ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d + ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests + ExecStartPre=/bin/mkdir -p /var/lib/calico + ExecStartPre=/bin/mkdir -p /var/lib/kubelet/volumeplugins + ExecStartPre=/bin/mkdir -p /opt/cni/bin + ExecStartPre=/usr/bin/bash -c "grep 'certificate-authority-data' /etc/kubernetes/kubeconfig | awk '{print $2}' | base64 -d > /etc/kubernetes/ca.crt" + ExecStartPre=-/usr/bin/podman rm kubelet + ExecStart=/usr/bin/podman run --name kubelet \ + --privileged \ + --pid host \ + --network host \ + --volume /etc/kubernetes:/etc/kubernetes:ro,z \ + --volume /usr/lib/os-release:/etc/os-release:ro \ + --volume /etc/ssl/certs:/etc/ssl/certs:ro \ + --volume /lib/modules:/lib/modules:ro \ + --volume /run:/run \ + --volume /sys/fs/cgroup:/sys/fs/cgroup:ro \ + --volume /sys/fs/cgroup/systemd:/sys/fs/cgroup/systemd \ + --volume /etc/pki/tls/certs:/usr/share/ca-certificates:ro \ + --volume /var/lib/calico:/var/lib/calico \ + --volume /var/lib/docker:/var/lib/docker \ + --volume /var/lib/kubelet:/var/lib/kubelet:rshared,z \ + --volume /var/log:/var/log \ + --volume /var/run:/var/run \ + --volume /var/run/lock:/var/run/lock:z \ + --volume /opt/cni/bin:/opt/cni/bin:z \ + k8s.gcr.io/hyperkube:v1.15.0 /hyperkube kubelet \ + --anonymous-auth=false \ + --authentication-token-webhook \ + --authorization-mode=Webhook \ + --cgroup-driver=systemd \ + --cgroups-per-qos=true \ + --enforce-node-allocatable=pods \ + --client-ca-file=/etc/kubernetes/ca.crt \ + --cluster_dns=${cluster_dns_service_ip} \ + --cluster_domain=${cluster_domain_suffix} \ + --cni-conf-dir=/etc/kubernetes/cni/net.d \ + --exit-on-lock-contention \ + --kubeconfig=/etc/kubernetes/kubeconfig \ + --lock-file=/var/run/lock/kubelet.lock \ + --network-plugin=cni \ + --node-labels=node-role.kubernetes.io/node \ + --pod-manifest-path=/etc/kubernetes/manifests \ + --read-only-port=0 \ + --volume-plugin-dir=/var/lib/kubelet/volumeplugins + ExecStop=-/usr/bin/podman stop kubelet + Delegate=yes + Restart=always + RestartSec=10 + [Install] + WantedBy=multi-user.target +storage: + directories: + - path: /etc/kubernetes + - path: /opt/bootkube + files: + - path: /etc/kubernetes/kubeconfig + mode: 0644 + contents: + inline: | + ${kubeconfig} + - path: /etc/sysctl.d/reverse-path-filter.conf + contents: + inline: | + net.ipv4.conf.all.rp_filter=1 + - path: /etc/sysctl.d/max-user-watches.conf + contents: + inline: | + fs.inotify.max_user_watches=16184 + - path: /etc/systemd/system.conf.d/accounting.conf + contents: + inline: | + [Manager] + DefaultCPUAccounting=yes + DefaultMemoryAccounting=yes + DefaultBlockIOAccounting=yes +passwd: + users: + - name: core + ssh_authorized_keys: + - ${ssh_authorized_key} + diff --git a/aws/fedora-coreos/kubernetes/workers/ingress.tf b/aws/fedora-coreos/kubernetes/workers/ingress.tf new file mode 100644 index 000000000..6b25152f0 --- /dev/null +++ b/aws/fedora-coreos/kubernetes/workers/ingress.tf @@ -0,0 +1,48 @@ +# Target groups of instances for use with load balancers + +resource "aws_lb_target_group" "workers-http" { + name = "${var.name}-workers-http" + vpc_id = var.vpc_id + target_type = "instance" + + protocol = "TCP" + port = 80 + + # HTTP health check for ingress + health_check { + protocol = "HTTP" + port = 10254 + path = "/healthz" + + # NLBs required to use same healthy and unhealthy thresholds + healthy_threshold = 3 + unhealthy_threshold = 3 + + # Interval between health checks required to be 10 or 30 + interval = 10 + } +} + +resource "aws_lb_target_group" "workers-https" { + name = "${var.name}-workers-https" + vpc_id = var.vpc_id + target_type = "instance" + + protocol = "TCP" + port = 443 + + # HTTP health check for ingress + health_check { + protocol = "HTTP" + port = 10254 + path = "/healthz" + + # NLBs required to use same healthy and unhealthy thresholds + healthy_threshold = 3 + unhealthy_threshold = 3 + + # Interval between health checks required to be 10 or 30 + interval = 10 + } +} + diff --git a/aws/fedora-coreos/kubernetes/workers/outputs.tf b/aws/fedora-coreos/kubernetes/workers/outputs.tf new file mode 100644 index 000000000..22f378855 --- /dev/null +++ b/aws/fedora-coreos/kubernetes/workers/outputs.tf @@ -0,0 +1,10 @@ +output "target_group_http" { + description = "ARN of a target group of workers for HTTP traffic" + value = aws_lb_target_group.workers-http.arn +} + +output "target_group_https" { + description = "ARN of a target group of workers for HTTPS traffic" + value = aws_lb_target_group.workers-https.arn +} + diff --git a/aws/fedora-coreos/kubernetes/workers/variables.tf b/aws/fedora-coreos/kubernetes/workers/variables.tf new file mode 100644 index 000000000..a1de562fb --- /dev/null +++ b/aws/fedora-coreos/kubernetes/workers/variables.tf @@ -0,0 +1,107 @@ +variable "name" { + type = string + description = "Unique name for the worker pool" +} + +# AWS + +variable "vpc_id" { + type = string + description = "Must be set to `vpc_id` output by cluster" +} + +variable "subnet_ids" { + type = list(string) + description = "Must be set to `subnet_ids` output by cluster" +} + +variable "security_groups" { + type = list(string) + description = "Must be set to `worker_security_groups` output by cluster" +} + +# instances + +variable "worker_count" { + type = string + default = "1" + description = "Number of instances" +} + +variable "instance_type" { + type = string + default = "t3.small" + description = "EC2 instance type" +} + +variable "os_image" { + type = string + default = "coreos-stable" + description = "AMI channel for Fedora CoreOS (not yet used)" +} + +variable "disk_size" { + type = string + default = "40" + description = "Size of the EBS volume in GB" +} + +variable "disk_type" { + type = string + default = "gp2" + description = "Type of the EBS volume (e.g. standard, gp2, io1)" +} + +variable "disk_iops" { + type = string + default = "0" + description = "IOPS of the EBS volume (required for io1)" +} + +variable "spot_price" { + type = string + default = "" + description = "Spot price in USD for autoscaling group spot instances. Leave as default empty string for autoscaling group to use on-demand instances. Note, switching in-place from spot to on-demand is not possible: https://github.com/terraform-providers/terraform-provider-aws/issues/4320" +} + +variable "target_groups" { + type = list(string) + description = "Additional target group ARNs to which instances should be added" + default = [] +} + +variable "snippets" { + type = list(string) + description = "Fedora CoreOS Config snippets" + default = [] +} + +# configuration + +variable "kubeconfig" { + type = string + description = "Must be set to `kubeconfig` output by cluster" +} + +variable "ssh_authorized_key" { + type = string + description = "SSH public key for user 'core'" +} + +variable "service_cidr" { + description = < Date: Wed, 17 Jul 2019 23:52:34 -0700 Subject: [PATCH 162/523] Add docs for Fedora CoreOS AWS and bare-metal --- README.md | 7 + aws/fedora-coreos/kubernetes/README.md | 2 +- bare-metal/fedora-coreos/kubernetes/LICENSE | 23 ++ bare-metal/fedora-coreos/kubernetes/README.md | 23 ++ docs/architecture/operating-systems.md | 63 +--- docs/fedora-coreos/aws.md | 237 ++++++++++++ docs/fedora-coreos/bare-metal.md | 354 ++++++++++++++++++ docs/index.md | 7 + mkdocs.yml | 3 + 9 files changed, 667 insertions(+), 52 deletions(-) create mode 100644 bare-metal/fedora-coreos/kubernetes/LICENSE create mode 100644 bare-metal/fedora-coreos/kubernetes/README.md create mode 100644 docs/fedora-coreos/aws.md create mode 100644 docs/fedora-coreos/bare-metal.md diff --git a/README.md b/README.md index 794fba238..eb3a0c4fe 100644 --- a/README.md +++ b/README.md @@ -29,6 +29,13 @@ Typhoon provides a Terraform Module for each supported operating system and plat | Digital Ocean | Container Linux | [digital-ocean/container-linux/kubernetes](digital-ocean/container-linux/kubernetes) | beta | | Google Cloud | Container Linux | [google-cloud/container-linux/kubernetes](google-cloud/container-linux/kubernetes) | stable | +A preview of Typhoon for [Fedora CoreOS](https://getfedora.org/coreos/) is available for testing. + +| Platform | Operating System | Terraform Module | Status | +|---------------|------------------|------------------|--------| +| AWS | Fedora CoreOS | [aws/fedora-coreos/kubernetes](aws/fedora-coreos/kubernetes) | preview | +| Bare-Metal | Fedora CoreOS | [bare-metal/fedora-coreos/kubernetes](bare-metal/fedora-coreos/kubernetes) | preview | + ## Documentation * [Docs](https://typhoon.psdn.io) diff --git a/aws/fedora-coreos/kubernetes/README.md b/aws/fedora-coreos/kubernetes/README.md index ce8ee9a37..c48573298 100644 --- a/aws/fedora-coreos/kubernetes/README.md +++ b/aws/fedora-coreos/kubernetes/README.md @@ -19,5 +19,5 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Docs -Please see the [official docs](https://typhoon.psdn.io) and the AWS [tutorial](https://typhoon.psdn.io/cl/aws/). +Please see the [official docs](https://typhoon.psdn.io) and the AWS [tutorial](https://typhoon.psdn.io/fedora-coreos/aws/). diff --git a/bare-metal/fedora-coreos/kubernetes/LICENSE b/bare-metal/fedora-coreos/kubernetes/LICENSE new file mode 100644 index 000000000..bd9a5eea2 --- /dev/null +++ b/bare-metal/fedora-coreos/kubernetes/LICENSE @@ -0,0 +1,23 @@ +The MIT License (MIT) + +Copyright (c) 2017 Typhoon Authors +Copyright (c) 2017 Dalton Hubble + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. + diff --git a/bare-metal/fedora-coreos/kubernetes/README.md b/bare-metal/fedora-coreos/kubernetes/README.md new file mode 100644 index 000000000..3896705ac --- /dev/null +++ b/bare-metal/fedora-coreos/kubernetes/README.md @@ -0,0 +1,23 @@ +# Typhoon + +Typhoon is a minimal and free Kubernetes distribution. + +* Minimal, stable base Kubernetes distribution +* Declarative infrastructure and configuration +* Free (freedom and cost) and privacy-respecting +* Practical for labs, datacenters, and clouds + +Typhoon distributes upstream Kubernetes, architectural conventions, and cluster addons, much like a GNU/Linux distribution provides the Linux kernel and userspace components. + +## Features + +* Kubernetes v1.15.0 (upstream, via [kubernetes-incubator/bootkube](https://github.com/kubernetes-incubator/bootkube)) +* Single or multi-master, [Calico](https://www.projectcalico.org/) or [flannel](https://github.com/coreos/flannel) networking +* On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/) +* Advanced features like [snippets](https://typhoon.psdn.io/advanced/customization/#container-linux) customization +* Ready for Ingress, Prometheus, Grafana, and other optional [addons](https://typhoon.psdn.io/addons/overview/) + +## Docs + +Please see the [official docs](https://typhoon.psdn.io) and the bare-metal [tutorial](https://typhoon.psdn.io/fedora-coreos/bare-metal/). + diff --git a/docs/architecture/operating-systems.md b/docs/architecture/operating-systems.md index 707ffe85e..7e40d27f0 100644 --- a/docs/architecture/operating-systems.md +++ b/docs/architecture/operating-systems.md @@ -1,6 +1,6 @@ # Operating Systems -Typhoon supports [Container Linux](https://coreos.com/why/) and Fedora [Atomic](https://www.projectatomic.io/) 28 (deprecated). These two operating systems were chosen because they offer: +Typhoon supports [Container Linux](https://coreos.com/why/), [Flatcar Linux](https://www.flatcar-linux.org/) and [Fedora CoreOS](https://getfedora.org/coreos/) (preview). These operating systems were chosen because they offer: * Minimalism and focus on clustered operation * Automated and atomic operating system upgrades @@ -10,18 +10,19 @@ Typhoon supports [Container Linux](https://coreos.com/why/) and Fedora [Atomic]( Together, they diversify Typhoon to support a range of container technologies. * Container Linux: Gentoo core, rkt-fly, docker -* Fedora Atomic: RHEL core, rpm-ostree, system containers (i.e. runc), CRI-O (future) +* Fedora CoreOS: rpm-ostree, podman, moby ## Host Properties -| Property | Container Linux / Flatcar Linux | Fedora Atomic | +| Property | Container Linux / Flatcar Linux | Fedora CoreOS | |-------------------|---------------------------------|---------------| -| host spec (bare-metal) | Container Linux Config | kickstart, cloud-init | -| host spec (cloud) | Container Linux Config | cloud-init | -| container runtime | docker | docker (CRIO planned) | -| cgroup driver | cgroupfs (except Flatcar edge) | systemd | -| logging driver | json-file | journald | +| Ignition system | Ignition v2.x spec | Ignition v3.x spec | +| Container Engine | docker | docker | | storage driver | overlay2 | overlay2 | +| logging driver | json-file | journald | +| cgroup driver | cgroupfs (except Flatcar edge) | systemd | +| Networking | systemd-networkd | NetworkManager | +| Username | core | core | ## Kubernetes Properties @@ -32,10 +33,10 @@ Together, they diversify Typhoon to support a range of container technologies. | control plane | self-hosted | self-hosted | | kubelet image | upstream hyperkube | upstream hyperkube via [system container](https://github.com/poseidon/system-containers) | | control plane images | upstream hyperkube | upstream hyperkube | -| on-host etcd | rkt-fly | system container (runc) | -| on-host kubelet | rkt-fly | system container (runc) | +| on-host etcd | rkt-fly | podman | +| on-host kubelet | rkt-fly | podman | | CNI plugins | calico or flannel | calico or flannel | -| coordinated drain & OS update | [CLUO](https://github.com/coreos/container-linux-update-operator) addon | manual (planned) | +| coordinated drain & OS update | [CLUO](https://github.com/coreos/container-linux-update-operator) addon | (planned) | ## Directory Locations @@ -47,43 +48,3 @@ Typhoon conventional directories. | pod-manifest-path | /etc/kubernetes/manifests | | volume-plugin-dir | /var/lib/kubelet/volumeplugins | -## Kubelet Mounts - -### Container Linux - -| Mount location | Host location | Options | -|-------------------|-------------------|---------| -| /etc/kubernetes | /etc/kubernetes | ro | -| /etc/ssl/certs | /etc/ssl/certs | ro | -| /usr/share/ca-certificates | /usr/share/ca-certificates | ro | -| /var/lib/kubelet | /var/lib/kubelet | recursive | -| /var/lib/docker | /var/lib/docker | | -| /var/lib/cni | /var/lib/cni | | -| /var/lib/calico | /var/lib/calico | | -| /var/log | /var/log | | -| /etc/os-release | /usr/lib/os-release | ro | -| /run | /run | | -| /lib/modules | /lib/modules | ro | -| /etc/resolv.conf | /etc/resolv.conf | | -| /opt/cni/bin | /opt/cni/bin | | - - -### Fedora Atomic - -| Mount location | Host location | Options | -|--------------------|------------------|---------| -| /rootfs | / | ro | -| /etc/kubernetes | /etc/kubernetes | ro | -| /etc/ssl/certs | /etc/ssl/certs | ro | -| /etc/pki/tls/certs | /usr/share/ca-certificates | ro | -| /var/lib | /var/lib | | -| /var/lib/kubelet | /var/lib/kubelet | recursive | -| /var/log | /var/log | ro | -| /etc/os-release | /etc/os-release | ro | -| /var/run/secrets | /var/run/secrets | | -| /run | /run | | -| /lib/modules | /lib/modules | ro | -| /etc/hosts | /etc/hosts | ro | -| /etc/resolv.conf | /etc/resolv.conf | ro | -| /opt/cni/bin | /opt/cni/bin (changing in future) | | - diff --git a/docs/fedora-coreos/aws.md b/docs/fedora-coreos/aws.md new file mode 100644 index 000000000..8382745da --- /dev/null +++ b/docs/fedora-coreos/aws.md @@ -0,0 +1,237 @@ +# AWS + +!!! danger + Typhoon for Fedora CoreOS is an early preview! Fedora CoreOS itself is a preview! Expect bugs and design shifts. Please help both projects solve problems. Report Fedora CoreOS bugs to [Fedora](https://github.com/coreos/fedora-coreos-tracker/issues). Report Typhoon issues to Typhoon. + +In this tutorial, we'll create a Kubernetes v1.15.0 cluster on AWS with Fedora CoreOS. + +We'll declare a Kubernetes cluster using the Typhoon Terraform module. Then apply the changes to create a VPC, gateway, subnets, security groups, controller instances, worker auto-scaling group, network load balancer, and TLS assets. + +Controllers are provisioned to run an `etcd-member` peer and a `kubelet` service. Workers run just a `kubelet` service. A one-time [bootkube](https://github.com/kubernetes-incubator/bootkube) bootstrap schedules the `apiserver`, `scheduler`, `controller-manager`, and `coredns` on controllers and schedules `kube-proxy` and `calico` (or `flannel`) on every node. A generated `kubeconfig` provides `kubectl` access to the cluster. + +## Requirements + +* AWS Account and IAM credentials +* AWS Route53 DNS Zone (registered Domain Name or delegated subdomain) +* Terraform v0.12.x and [terraform-provider-ct](https://github.com/poseidon/terraform-provider-ct) installed locally + +## Terraform Setup + +Install [Terraform](https://www.terraform.io/downloads.html) v0.12.x on your system. + +```sh +$ terraform version +Terraform v0.12.2 +``` + +Add the [terraform-provider-ct](https://github.com/poseidon/terraform-provider-ct) plugin binary for your system to `~/.terraform.d/plugins/`, noting the final name. + +```sh +wget https://github.com/poseidon/terraform-provider-ct/releases/download/v0.4.0/terraform-provider-ct-v0.4.0-linux-amd64.tar.gz +tar xzf terraform-provider-ct-v0.4.0-linux-amd64.tar.gz +mv terraform-provider-ct-v0.4.0-linux-amd64/terraform-provider-ct ~/.terraform.d/plugins/terraform-provider-ct_v0.4.0 +``` + +Read [concepts](/architecture/concepts/) to learn about Terraform, modules, and organizing resources. Change to your infrastructure repository (e.g. `infra`). + +``` +cd infra/clusters +``` + +## Provider + +Login to your AWS IAM dashboard and find your IAM user. Select "Security Credentials" and create an access key. Save the id and secret to a file that can be referenced in configs. + +``` +[default] +aws_access_key_id = xxx +aws_secret_access_key = yyy +``` + +Configure the AWS provider to use your access key credentials in a `providers.tf` file. + +```tf +provider "aws" { + version = "2.19.0" + region = "us-east-1" # MUST be us-east-1 right now! + shared_credentials_file = "/home/user/.config/aws/credentials" +} + +provider "ct" { + version = "0.4.0" +} +``` + +Additional configuration options are described in the `aws` provider [docs](https://www.terraform.io/docs/providers/aws/). + +!!! tip + Regions are listed in [docs](http://docs.aws.amazon.com/general/latest/gr/rande.html#ec2_region) or with `aws ec2 describe-regions`. + +## Cluster + +Define a Kubernetes cluster using the module `aws/fedora-coreos/kubernetes`. + +```tf +module "aws-tempest" { + source = "git::https://github.com/poseidon/typhoon//aws/fedora-coreos/kubernetes?ref=DEVELOPMENT_SHA" + + # AWS + cluster_name = "tempest" + dns_zone = "aws.example.com" + dns_zone_id = "Z3PAABBCFAKEC0" + + # configuration + ssh_authorized_key = "ssh-rsa AAAAB3Nz..." + asset_dir = "/home/user/.secrets/clusters/tempest" + + # optional + worker_count = 2 + worker_type = "t3.small" +} +``` + +Reference the [variables docs](#variables) or the [variables.tf](https://github.com/poseidon/typhoon/blob/master/aws/fedora-coreos/kubernetes/variables.tf) source. + +## ssh-agent + +Initial bootstrapping requires `bootkube.service` be started on one controller node. Terraform uses `ssh-agent` to automate this step. Add your SSH private key to `ssh-agent`. + +```sh +ssh-add ~/.ssh/id_rsa +ssh-add -L +``` + +## Apply + +Initialize the config directory if this is the first use with Terraform. + +```sh +terraform init +``` + +Plan the resources to be created. + +```sh +$ terraform plan +Plan: 98 to add, 0 to change, 0 to destroy. +``` + +Apply the changes to create the cluster. + +```sh +$ terraform apply +... +module.aws-tempest.null_resource.bootkube-start: Still creating... (4m50s elapsed) +module.aws-tempest.null_resource.bootkube-start: Still creating... (5m0s elapsed) +module.aws-tempest.null_resource.bootkube-start: Creation complete after 11m8s (ID: 3961816482286168143) + +Apply complete! Resources: 98 added, 0 changed, 0 destroyed. +``` + +In 4-8 minutes, the Kubernetes cluster will be ready. + +## Verify + +[Install kubectl](https://kubernetes.io/docs/tasks/tools/install-kubectl/) on your system. Use the generated `kubeconfig` credentials to access the Kubernetes cluster and list nodes. + +``` +$ export KUBECONFIG=/home/user/.secrets/clusters/tempest/auth/kubeconfig +$ kubectl get nodes +NAME STATUS ROLES AGE VERSION +ip-10-0-3-155 Ready controller,master 10m v1.15.0 +ip-10-0-26-65 Ready node 10m v1.15.0 +ip-10-0-41-21 Ready node 10m v1.15.0 +``` + +List the pods. + +``` +$ kubectl get pods --all-namespaces +NAMESPACE NAME READY STATUS RESTARTS AGE +kube-system calico-node-1m5bf 2/2 Running 0 34m +kube-system calico-node-7jmr1 2/2 Running 0 34m +kube-system calico-node-bknc8 2/2 Running 0 34m +kube-system coredns-1187388186-wx1lg 1/1 Running 0 34m +kube-system coredns-1187388186-qjnvp 1/1 Running 0 34m +kube-system kube-apiserver-4mjbk 1/1 Running 0 34m +kube-system kube-controller-manager-3597210155-j2jbt 1/1 Running 1 34m +kube-system kube-controller-manager-3597210155-j7g7x 1/1 Running 0 34m +kube-system kube-proxy-14wxv 1/1 Running 0 34m +kube-system kube-proxy-9vxh2 1/1 Running 0 34m +kube-system kube-proxy-sbbsh 1/1 Running 0 34m +kube-system kube-scheduler-3359497473-5plhf 1/1 Running 0 34m +kube-system kube-scheduler-3359497473-r7zg7 1/1 Running 1 34m +kube-system pod-checkpointer-4kxtl 1/1 Running 0 34m +kube-system pod-checkpointer-4kxtl-ip-10-0-3-155 1/1 Running 0 33m +``` + +## Going Further + +Learn about [maintenance](/topics/maintenance/) and [addons](/addons/overview/). + +## Variables + +Check the [variables.tf](https://github.com/poseidon/typhoon/blob/master/aws/fedora-coreos/kubernetes/variables.tf) source. + +### Required + +| Name | Description | Example | +|:-----|:------------|:--------| +| cluster_name | Unique cluster name (prepended to dns_zone) | "tempest" | +| dns_zone | AWS Route53 DNS zone | "aws.example.com" | +| dns_zone_id | AWS Route53 DNS zone id | "Z3PAABBCFAKEC0" | +| ssh_authorized_key | SSH public key for user 'core' | "ssh-rsa AAAAB3NZ..." | +| asset_dir | Absolute path to a directory where generated assets should be placed (contains secrets) | "/home/user/.secrets/clusters/tempest" | + +#### DNS Zone + +Clusters create a DNS A record `${cluster_name}.${dns_zone}` to resolve a network load balancer backed by controller instances. This FQDN is used by workers and `kubectl` to access the apiserver(s). In this example, the cluster's apiserver would be accessible at `tempest.aws.example.com`. + +You'll need a registered domain name or delegated subdomain on AWS Route53. You can set this up once and create many clusters with unique names. + +```tf +resource "aws_route53_zone" "zone-for-clusters" { + name = "aws.example.com." +} +``` + +Reference the DNS zone id with `"${aws_route53_zone.zone-for-clusters.zone_id}"`. + +!!! tip "" + If you have an existing domain name with a zone file elsewhere, just delegate a subdomain that can be managed on Route53 (e.g. aws.mydomain.com) and [update nameservers](http://docs.aws.amazon.com/Route53/latest/DeveloperGuide/SOA-NSrecords.html). + +### Optional + +| Name | Description | Default | Example | +|:-----|:------------|:--------|:--------| +| controller_count | Number of controllers (i.e. masters) | 1 | 1 | +| worker_count | Number of workers | 1 | 3 | +| controller_type | EC2 instance type for controllers | "t3.small" | See below | +| worker_type | EC2 instance type for workers | "t3.small" | See below | +| os_image | AMI channel for Fedora CoreOS | not yet used | ? | +| disk_size | Size of the EBS volume in GB | "40" | "100" | +| disk_type | Type of the EBS volume | "gp2" | standard, gp2, io1 | +| disk_iops | IOPS of the EBS volume | "0" (i.e. auto) | "400" | +| worker_target_groups | Target group ARNs to which worker instances should be added | [] | ["${aws_lb_target_group.app.id}"] | +| worker_price | Spot price in USD for workers. Leave as default empty string for regular on-demand instances | "" | "0.10" | +| controller_snippets | Controller Fedora CoreOS Config snippets | [] | UNSUPPORTED | +| worker_clc_snippets | Worker Fedora CoreOS Config snippets | [] | UNSUPPORTED | +| networking | Choice of networking provider | "calico" | "calico" or "flannel" | +| network_mtu | CNI interface MTU (calico only) | 1480 | 8981 | +| host_cidr | CIDR IPv4 range to assign to EC2 instances | "10.0.0.0/16" | "10.1.0.0/16" | +| pod_cidr | CIDR IPv4 range to assign to Kubernetes pods | "10.2.0.0/16" | "10.22.0.0/16" | +| service_cidr | CIDR IPv4 range to assign to Kubernetes services | "10.3.0.0/16" | "10.3.0.0/24" | +| cluster_domain_suffix | FQDN suffix for Kubernetes services answered by coredns. | "cluster.local" | "k8s.example.com" | + +Check the list of valid [instance types](https://aws.amazon.com/ec2/instance-types/). + +!!! warning + Do not choose a `controller_type` smaller than `t2.small`. Smaller instances are not sufficient for running a controller. + +!!! tip "MTU" + If your EC2 instance type supports [Jumbo frames](http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/network_mtu.html#jumbo_frame_instances) (most do), we recommend you change the `network_mtu` to 8981! You will get better pod-to-pod bandwidth. + +#### Spot + +Add `worker_price = "0.10"` to use spot instance workers (instead of "on-demand") and set a maximum spot price in USD. Clusters can tolerate spot market interuptions fairly well (reschedules pods, but cannot drain) to save money, with the tradeoff that requests for workers may go unfulfilled. + diff --git a/docs/fedora-coreos/bare-metal.md b/docs/fedora-coreos/bare-metal.md new file mode 100644 index 000000000..f08f0688d --- /dev/null +++ b/docs/fedora-coreos/bare-metal.md @@ -0,0 +1,354 @@ +# Bare-Metal + +!!! danger + Typhoon for Fedora CoreOS is an early preview! Fedora CoreOS itself is a preview! Expect bugs and design shifts. Please help both projects solve problems. Report Fedora CoreOS bugs to [Fedora](https://github.com/coreos/fedora-coreos-tracker/issues). Report Typhoon issues to Typhoon. + +In this tutorial, we'll network boot and provision a Kubernetes v1.15.0 cluster on bare-metal with Fedora CoreOS. + +First, we'll deploy a [Matchbox](https://github.com/poseidon/matchbox) service and setup a network boot environment. Then, we'll declare a Kubernetes cluster using the Typhoon Terraform module and power on machines. On PXE boot, machines will install Fedora CoreOS to disk, reboot into the disk install, and provision themselves as Kubernetes controllers or workers via Ignition. + +Controllers are provisioned to run an `etcd-member` peer and a `kubelet` service. Workers run just a `kubelet` service. A one-time [bootkube](https://github.com/kubernetes-incubator/bootkube) bootstrap schedules the `apiserver`, `scheduler`, `controller-manager`, and `coredns` on controllers and schedules `kube-proxy` and `calico` (or `flannel`) on every node. A generated `kubeconfig` provides `kubectl` access to the cluster. + +## Requirements + +* Machines with 2GB RAM, 30GB disk, PXE-enabled NIC, IPMI +* PXE-enabled [network boot](https://coreos.com/matchbox/docs/latest/network-setup.html) environment (with HTTPS support) +* Matchbox v0.6+ deployment with API enabled +* Matchbox credentials `client.crt`, `client.key`, `ca.crt` +* Terraform v0.12.x, [terraform-provider-matchbox](https://github.com/poseidon/terraform-provider-matchbox), and [terraform-provider-ct](https://github.com/poseidon/terraform-provider-ct) installed locally + +## Machines + +Collect a MAC address from each machine. For machines with multiple PXE-enabled NICs, pick one of the MAC addresses. MAC addresses will be used to match machines to profiles during network boot. + +* 52:54:00:a1:9c:ae (node1) +* 52:54:00:b2:2f:86 (node2) +* 52:54:00:c3:61:77 (node3) + +Configure each machine to boot from the disk through IPMI or the BIOS menu. + +``` +ipmitool -H node1 -U USER -P PASS chassis bootdev disk options=persistent +``` + +During provisioning, you'll explicitly set the boot device to `pxe` for the next boot only. Machines will install (overwrite) the operating system to disk on PXE boot and reboot into the disk install. + +!!! tip "" + Ask your hardware vendor to provide MACs and preconfigure IPMI, if possible. With it, you can rack new servers, `terraform apply` with new info, and power on machines that network boot and provision into clusters. + +## DNS + +Create a DNS A (or AAAA) record for each node's default interface. Create a record that resolves to each controller node (or re-use the node record if there's one controller). + +* node1.example.com (node1) +* node2.example.com (node2) +* node3.example.com (node3) +* myk8s.example.com (node1) + +Cluster nodes will be configured to refer to the control plane and themselves by these fully qualified names and they'll be used in generated TLS certificates. + +## Matchbox + +Matchbox is an open-source app that matches network-booted bare-metal machines (based on labels like MAC, UUID, etc.) to profiles to automate cluster provisioning. + +Install Matchbox on a Kubernetes cluster or dedicated server. + +* Installing on [Kubernetes](https://coreos.com/matchbox/docs/latest/deployment.html#kubernetes) (recommended) +* Installing on a [server](https://coreos.com/matchbox/docs/latest/deployment.html#download) + +!!! tip + Deploy Matchbox as service that can be accessed by all of your bare-metal machines globally. This provides a single endpoint to use Terraform to manage bare-metal clusters at different sites. Typhoon will never include secrets in provisioning user-data so you may even deploy matchbox publicly. + +Matchbox provides a TLS client-authenticated API that clients, like Terraform, can use to manage machine matching and profiles. Think of it like a cloud provider API, but for creating bare-metal instances. + +[Generate TLS](https://coreos.com/matchbox/docs/latest/deployment.html#generate-tls-certificates) client credentials. Save the `ca.crt`, `client.crt`, and `client.key` where they can be referenced in Terraform configs. + +```sh +mv ca.crt client.crt client.key ~/.config/matchbox/ +``` + +Verify the matchbox read-only HTTP endpoints are accessible (port is configurable). + +```sh +$ curl http://matchbox.example.com:8080 +matchbox +``` + +Verify your TLS client certificate and key can be used to access the Matchbox API (port is configurable). + +```sh +$ openssl s_client -connect matchbox.example.com:8081 \ + -CAfile ~/.config/matchbox/ca.crt \ + -cert ~/.config/matchbox/client.crt \ + -key ~/.config/matchbox/client.key +``` + +## PXE Environment + +Create an iPXE-enabled network boot environment. Configure PXE clients to chainload [iPXE](http://ipxe.org/cmd) firmware compiled to support [HTTPS downloads](https://ipxe.org/crypto). Instruct iPXE clients to chainload from your Matchbox service's `/boot.ipxe` endpoint. + +For networks already supporting iPXE clients, you can add a `default.ipxe` config. + +```ini +# /var/www/html/ipxe/default.ipxe +chain http://matchbox.foo:8080/boot.ipxe +``` + +For networks with Ubiquiti Routers, you can [configure the router](/topics/hardware/#ubiquiti) itself to chainload machines to iPXE and Matchbox. + +Read about the [many ways](https://coreos.com/matchbox/docs/latest/network-setup.html) to setup a compliant iPXE-enabled network. There is quite a bit of flexibility: + +* Continue using existing DHCP, TFTP, or DNS services +* Configure specific machines, subnets, or architectures to chainload from Matchbox +* Place Matchbox behind a menu entry (timeout and default to Matchbox) + +!!! note "" + TFTP chainloading to modern boot firmware, like iPXE, avoids issues with old NICs and allows faster transfer protocols like HTTP to be used. + +!!! warning + Compile iPXE from [source](https://github.com/ipxe/ipxe) with support for [HTTPS downloads](https://ipxe.org/crypto). iPXE's pre-built firmware binaries do not enable this. If you cannot enable HTTPS downloads, set `download_protocol = "http"` (discouraged). + +## Terraform Setup + +Install [Terraform](https://www.terraform.io/downloads.html) v0.12.x on your system. + +```sh +$ terraform version +Terraform v0.12.2 +``` + +Add the [terraform-provider-matchbox](https://github.com/poseidon/terraform-provider-matchbox) plugin binary for your system to `~/.terraform.d/plugins/`, noting the final name. + +```sh +wget https://github.com/poseidon/terraform-provider-matchbox/releases/download/v0.3.0/terraform-provider-matchbox-v0.3.0-linux-amd64.tar.gz +tar xzf terraform-provider-matchbox-v0.3.0-linux-amd64.tar.gz +mv terraform-provider-matchbox-v0.3.0-linux-amd64/terraform-provider-matchbox ~/.terraform.d/plugins/terraform-provider-matchbox_v0.3.0 +``` + +Add the [terraform-provider-ct](https://github.com/poseidon/terraform-provider-ct) plugin binary for your system to `~/.terraform.d/plugins/`, noting the final name. + +```sh +wget https://github.com/poseidon/terraform-provider-ct/releases/download/v0.4.0/terraform-provider-ct-v0.4.0-linux-amd64.tar.gz +tar xzf terraform-provider-ct-v0.4.0-linux-amd64.tar.gz +mv terraform-provider-ct-v0.4.0-linux-amd64/terraform-provider-ct ~/.terraform.d/plugins/terraform-provider-ct_v0.4.0 +``` + +Read [concepts](/architecture/concepts/) to learn about Terraform, modules, and organizing resources. Change to your infrastructure repository (e.g. `infra`). + +``` +cd infra/clusters +``` + +## Provider + +Configure the Matchbox provider to use your Matchbox API endpoint and client certificate in a `providers.tf` file. + +```tf +provider "matchbox" { + version = "0.3.0" + endpoint = "matchbox.example.com:8081" + client_cert = "${file("~/.config/matchbox/client.crt")}" + client_key = "${file("~/.config/matchbox/client.key")}" + ca = "${file("~/.config/matchbox/ca.crt")}" +} + +provider "ct" { + version = "0.4.0" +} +``` + +## Cluster + +Define a Kubernetes cluster using the module `bare-metal/fedora-coreos/kubernetes`. + +```tf +module "bare-metal-mercury" { + source = "git::https://github.com/poseidon/typhoon//bare-metal/fedora-coreos/kubernetes?ref=DEVELOPMENT_SHA" + + # bare-metal + cluster_name = "mercury" + matchbox_http_endpoint = "http://matchbox.example.com" + os_stream = "testing" + os_version = "30.20190716.1" + cached_install = false + + # configuration + k8s_domain_name = "node1.example.com" + ssh_authorized_key = "ssh-rsa AAAAB3Nz..." + asset_dir = "/home/user/.secrets/clusters/mercury" + + # machines + controller_names = ["node1"] + controller_macs = ["52:54:00:a1:9c:ae"] + controller_domains = ["node1.example.com"] + worker_names = [ + "node2", + "node3", + ] + worker_macs = [ + "52:54:00:b2:2f:86", + "52:54:00:c3:61:77", + ] + worker_domains = [ + "node2.example.com", + "node3.example.com", + ] +} +``` + +Reference the [variables docs](#variables) or the [variables.tf](https://github.com/poseidon/typhoon/blob/master/bare-metal/fedora-coreos/kubernetes/variables.tf) source. + +## ssh-agent + +Initial bootstrapping requires `bootkube.service` be started on one controller node. Terraform uses `ssh-agent` to automate this step. Add your SSH private key to `ssh-agent`. + +```sh +ssh-add ~/.ssh/id_rsa +ssh-add -L +``` + +## Apply + +Initialize the config directory if this is the first use with Terraform. + +```sh +terraform init +``` + +Plan the resources to be created. + +```sh +$ terraform plan +Plan: 55 to add, 0 to change, 0 to destroy. +``` + +Apply the changes. Terraform will generate bootkube assets to `asset_dir` and create Matchbox profiles (e.g. controller, worker) and matching rules via the Matchbox API. + +```sh +$ terraform apply +module.bare-metal-mercury.null_resource.copy-kubeconfig.0: Provisioning with 'file'... +module.bare-metal-mercury.null_resource.copy-etcd-secrets.0: Provisioning with 'file'... +module.bare-metal-mercury.null_resource.copy-kubeconfig.0: Still creating... (10s elapsed) +module.bare-metal-mercury.null_resource.copy-etcd-secrets.0: Still creating... (10s elapsed) +... +``` + +Apply will then loop until it can successfully copy credentials to each machine and start the one-time Kubernetes bootstrap service. Proceed to the next step while this loops. + +### Power + +Power on each machine with the boot device set to `pxe` for the next boot only. + +```sh +ipmitool -H node1.example.com -U USER -P PASS chassis bootdev pxe +ipmitool -H node1.example.com -U USER -P PASS power on +``` + +Machines will network boot, install Fedora CoreOS to disk, reboot into the disk install, and provision themselves as controllers or workers. + +!!! tip "" + If this is the first test of your PXE-enabled network boot environment, watch the SOL console of a machine to spot any misconfigurations. + +### Bootstrap + +Wait for the `bootkube-start` step to finish bootstrapping the Kubernetes control plane. This may take 5-15 minutes depending on your network. + +``` +module.bare-metal-mercury.null_resource.bootkube-start: Still creating... (6m10s elapsed) +module.bare-metal-mercury.null_resource.bootkube-start: Still creating... (6m20s elapsed) +module.bare-metal-mercury.null_resource.bootkube-start: Still creating... (6m30s elapsed) +module.bare-metal-mercury.null_resource.bootkube-start: Still creating... (6m40s elapsed) +module.bare-metal-mercury.null_resource.bootkube-start: Creation complete (ID: 5441741360626669024) + +Apply complete! Resources: 55 added, 0 changed, 0 destroyed. +``` + +To watch the bootstrap process in detail, SSH to the first controller and journal the logs. + +``` +$ ssh core@node1.example.com +$ journalctl -f -u bootkube +bootkube[5]: Pod Status: pod-checkpointer Running +bootkube[5]: Pod Status: kube-apiserver Running +bootkube[5]: Pod Status: kube-scheduler Running +bootkube[5]: Pod Status: kube-controller-manager Running +bootkube[5]: All self-hosted control plane components successfully started +bootkube[5]: Tearing down temporary bootstrap control plane... +``` + +## Verify + +[Install kubectl](https://kubernetes.io/docs/tasks/tools/install-kubectl/) on your system. Use the generated `kubeconfig` credentials to access the Kubernetes cluster and list nodes. + +``` +$ export KUBECONFIG=/home/user/.secrets/clusters/mercury/auth/kubeconfig +$ kubectl get nodes +NAME STATUS ROLES AGE VERSION +node1.example.com Ready controller,master 10m v1.15.0 +node2.example.com Ready node 10m v1.15.0 +node3.example.com Ready node 10m v1.15.0 +``` + +List the pods. + +``` +$ kubectl get pods --all-namespaces +NAMESPACE NAME READY STATUS RESTARTS AGE +kube-system calico-node-6qp7f 2/2 Running 1 11m +kube-system calico-node-gnjrm 2/2 Running 0 11m +kube-system calico-node-llbgt 2/2 Running 0 11m +kube-system coredns-1187388186-dj3pd 1/1 Running 0 11m +kube-system coredns-1187388186-mx9rt 1/1 Running 0 11m +kube-system kube-apiserver-7336w 1/1 Running 0 11m +kube-system kube-controller-manager-3271970485-b9chx 1/1 Running 0 11m +kube-system kube-controller-manager-3271970485-v30js 1/1 Running 1 11m +kube-system kube-proxy-50sd4 1/1 Running 0 11m +kube-system kube-proxy-bczhp 1/1 Running 0 11m +kube-system kube-proxy-mp2fw 1/1 Running 0 11m +kube-system kube-scheduler-3895335239-fd3l7 1/1 Running 1 11m +kube-system kube-scheduler-3895335239-hfjv0 1/1 Running 0 11m +kube-system pod-checkpointer-wf65d 1/1 Running 0 11m +kube-system pod-checkpointer-wf65d-node1.example.com 1/1 Running 0 11m +``` + +## Going Further + +Learn about [maintenance](/topics/maintenance/) and [addons](/addons/overview/). + +## Variables + +Check the [variables.tf](https://github.com/poseidon/typhoon/blob/master/bare-metal/fedora-coreos/kubernetes/variables.tf) source. + +### Required + +| Name | Description | Example | +|:-----|:------------|:--------| +| cluster_name | Unique cluster name | mercury | +| matchbox_http_endpoint | Matchbox HTTP read-only endpoint | http://matchbox.example.com:port | +| os_stream | Fedora CoreOS release stream | testing | +| os_version | Fedora CoreOS version to PXE and install | 30.20190716.1 | +| k8s_domain_name | FQDN resolving to the controller(s) nodes. Workers and kubectl will communicate with this endpoint | "myk8s.example.com" | +| ssh_authorized_key | SSH public key for user 'core' | "ssh-rsa AAAAB3Nz..." | +| asset_dir | Absolute path to a directory where generated assets should be placed (contains secrets) | "/home/user/.secrets/clusters/mercury" | +| controller_names | Ordered list of controller short names | ["node1"] | +| controller_macs | Ordered list of controller identifying MAC addresses | ["52:54:00:a1:9c:ae"] | +| controller_domains | Ordered list of controller FQDNs | ["node1.example.com"] | +| worker_names | Ordered list of worker short names | ["node2", "node3"] | +| worker_macs | Ordered list of worker identifying MAC addresses | ["52:54:00:b2:2f:86", "52:54:00:c3:61:77"] | +| worker_domains | Ordered list of worker FQDNs | ["node2.example.com", "node3.example.com"] | + +### Optional + +| Name | Description | Default | Example | +|:-----|:------------|:--------|:--------| +| cached_install | PXE boot and install from the Matchbox `/assets` cache. Admin MUST have downloaded Fedora CoreOS images into the cache | false | true | +| install_disk | Disk device where Fedora CoreOS should be installed | "sda" (not "/dev/sda" like Container Linux) | "sdb" | +| networking | Choice of networking provider | "calico" | "calico" or "flannel" | +| network_mtu | CNI interface MTU (calico-only) | 1480 | - | +| snippets | Map from machine names to lists of Fedora CoreOS Config snippets | {} | UNSUPPORTED | +| network_ip_autodetection_method | Method to detect host IPv4 address (calico-only) | first-found | can-reach=10.0.0.1 | +| pod_cidr | CIDR IPv4 range to assign to Kubernetes pods | "10.2.0.0/16" | "10.22.0.0/16" | +| service_cidr | CIDR IPv4 range to assign to Kubernetes services | "10.3.0.0/16" | "10.3.0.0/24" | +| cluster_domain_suffix | FQDN suffix for Kubernetes services answered by coredns. | "cluster.local" | "k8s.example.com" | +| kernel_args | Additional kernel args to provide at PXE boot | [] | "kvm-intel.nested=1" | + diff --git a/docs/index.md b/docs/index.md index a9153d6b6..af432c385 100644 --- a/docs/index.md +++ b/docs/index.md @@ -29,6 +29,13 @@ Typhoon provides a Terraform Module for each supported operating system and plat | Digital Ocean | Container Linux | [digital-ocean/container-linux/kubernetes](cl/digital-ocean.md) | beta | | Google Cloud | Container Linux | [google-cloud/container-linux/kubernetes](cl/google-cloud.md) | stable | +A preview of Typhoon for [Fedora CoreOS](https://getfedora.org/coreos/) is available for testing. + +| Platform | Operating System | Terraform Module | Status | +|---------------|------------------|------------------|--------| +| AWS | Fedora CoreOS | [aws/fedora-coreos/kubernetes](aws/fedora-coreos/kubernetes) | preview | +| Bare-Metal | Fedora CoreOS | [bare-metal/fedora-coreos/kubernetes](bare-metal/fedora-coreos/kubernetes) | preview | + ## Documentation * Architecture [concepts](architecture/concepts.md) and [operating-systems](architecture/operating-systems.md) diff --git a/mkdocs.yml b/mkdocs.yml index ba0fa18e3..0c0a4549a 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -59,6 +59,9 @@ nav: - 'Bare-Metal': 'cl/bare-metal.md' - 'Digital Ocean': 'cl/digital-ocean.md' - 'Google Cloud': 'cl/google-cloud.md' + - 'Fedora CoreOS': + - 'AWS': 'fedora-coreos/aws.md' + - 'Bare-Metal': 'fedora-coreos/bare-metal.md' - 'Topics': - 'Maintenance': 'topics/maintenance.md' - 'Hardware': 'topics/hardware.md' From 5fdeb9bc78394b42d1812f9104fc295f376a86cb Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Thu, 18 Jul 2019 00:58:11 -0700 Subject: [PATCH 163/523] Adjust Fedora CoreOS image locations * Use the xz compressed images published by Fedora testing, instead of gzippped tarballs. This is possible because the initramfs now supports xz and coreos-installer 0.8 was added * Separate bios and uefi raw images are no longer needed --- bare-metal/fedora-coreos/kubernetes/profiles.tf | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/bare-metal/fedora-coreos/kubernetes/profiles.tf b/bare-metal/fedora-coreos/kubernetes/profiles.tf index 071b8ec56..b17a655a5 100644 --- a/bare-metal/fedora-coreos/kubernetes/profiles.tf +++ b/bare-metal/fedora-coreos/kubernetes/profiles.tf @@ -1,11 +1,11 @@ locals { - remote_kernel = "https://builds.coreos.fedoraproject.org/prod/streams/${var.os_stream}/builds/${var.os_version}/fedora-coreos-${var.os_version}-installer-kernel" - remote_initrd = "https://builds.coreos.fedoraproject.org/prod/streams/${var.os_stream}/builds/${var.os_version}/fedora-coreos-${var.os_version}-installer-initramfs.img" + remote_kernel = "https://builds.coreos.fedoraproject.org/prod/streams/${var.os_stream}/builds/${var.os_version}/x86_64/fedora-coreos-${var.os_version}-installer-kernel" + remote_initrd = "https://builds.coreos.fedoraproject.org/prod/streams/${var.os_stream}/builds/${var.os_version}/x86_64/fedora-coreos-${var.os_version}-installer-initramfs.img" remote_args = [ "ip=dhcp", "rd.neednet=1", "coreos.inst=yes", - "coreos.inst.image_url=https://builds.coreos.fedoraproject.org/prod/streams/${var.os_stream}/builds/${var.os_version}/fedora-coreos-${var.os_version}-metal.raw.gz", + "coreos.inst.image_url=https://builds.coreos.fedoraproject.org/prod/streams/${var.os_stream}/builds/${var.os_version}/x86_64/fedora-coreos-${var.os_version}-metal.raw.xz", "coreos.inst.ignition_url=${var.matchbox_http_endpoint}/ignition?uuid=$${uuid}&mac=$${mac:hexhyp}", "coreos.inst.install_dev=${var.install_disk}" ] @@ -16,7 +16,7 @@ locals { "ip=dhcp", "rd.neednet=1", "coreos.inst=yes", - "coreos.inst.image_url=${var.matchbox_http_endpoint}/assets/fedora-coreos/fedora-coreos-${var.os_version}-metal-bios.raw.gz", + "coreos.inst.image_url=${var.matchbox_http_endpoint}/assets/fedora-coreos/fedora-coreos-${var.os_version}-metal.raw.xz", "coreos.inst.ignition_url=${var.matchbox_http_endpoint}/ignition?uuid=$${uuid}&mac=$${mac:hexhyp}", "coreos.inst.install_dev=${var.install_disk}" ] @@ -44,7 +44,8 @@ resource "matchbox_profile" "controllers" { data "ct_config" "controller-ignitions" { count = length(var.controller_names) - content = data.template_file.controller-configs.*.rendered[count.index] + content = data.template_file.controller-configs.*.rendered[count.index] + strict = true } data "template_file" "controller-configs" { @@ -78,7 +79,8 @@ resource "matchbox_profile" "workers" { data "ct_config" "worker-ignitions" { count = length(var.worker_names) - content = data.template_file.worker-configs.*.rendered[count.index] + content = data.template_file.worker-configs.*.rendered[count.index] + strict = true } data "template_file" "worker-configs" { From c7ff1a2e01f1406908feffc28c2a1317978809e0 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Thu, 18 Jul 2019 03:02:10 -0700 Subject: [PATCH 164/523] Announce a preview with Fedora CoreOS preview --- docs/announce.md | 10 ++++++++++ docs/architecture/operating-systems.md | 4 ++-- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/docs/announce.md b/docs/announce.md index 1c46d42ff..864bade71 100644 --- a/docs/announce.md +++ b/docs/announce.md @@ -1,5 +1,15 @@ # Announce +## Jul 18, 2019 + +Introducing a preview of Typhoon Kubernetes clusters with Fedora CoreOS! + +Fedora recently [announced](https://lists.fedoraproject.org/archives/list/coreos@lists.fedoraproject.org/thread/3HTW5SLUY6X2Y5SFXJSE4BWEDNJ2J5SL/) the first preview release of Fedora CoreOS, aiming to blend the best of CoreOS and Fedora for containerized workloads. To spur testing, Typhoon is sharing preview modules for Kubernetes v1.15 on [AWS](https://typhoon.psdn.io/fedora-coreos/aws/) and [bare-metal](https://typhoon.psdn.io/fedora-coreos/bare-metal/) using the new Fedora CoreOS preview. What better way to test drive than by running Kubernetes? + +While Typhoon uses Container Linux (or Flatcar Linux) for stable modules, the project hasn't been a stranger to Fedora ideas, once developing a [Fedora Atomic](https://typhoon.psdn.io/announce/#april-26-2018) variant in 2018. That makes the Fedora CoreOS fushion both exciting and familiar. Typhoon with Fedora CoreOS uses Ignition v3 for provisioning, uses rpm-ostree for layering and updates, tries swapping system containers for podman, and brings SELinux enforcement ([table](https://typhoon.psdn.io/architecture/operating-systems/)). This is an early preview (don't go to prod), but do try it out and help identify and solve issues (getting started links above). + +About: For newcomers, Typhoon is a minimal and free (cost and freedom) Kubernetes distribution providing upstream Kubernetes, declarative configuration via Terraform, and support for AWS, Azure, Google Cloud, DigitalOcean, and bare-metal. It is run by former CoreOS engineer [@dghubble](https://twitter.com/dghubble) to power his clusters with freedom [motivations](https://typhoon.psdn.io/#motivation). + ## March 27, 2019 Last April, Typhoon [introduced](#april-26-2018) alpha support for creating Kubernetes clusters with Fedora Atomic on AWS, Google Cloud, DigitalOcean, and bare-metal. Fedora Atomic shared many of Container Linux's aims for a container-optimized operating system, introduced novel ideas, and provided technical diversification for an uncertain future. However, Project Atomic efforts were merged into Fedora CoreOS and future Fedora Atomic releases are [not expected](http://www.projectatomic.io/blog/2018/06/welcome-to-fedora-coreos/). *Typhoon modules for Fedora Atomic will not be updated much beyond Kubernetes v1.13*. They may later be removed. diff --git a/docs/architecture/operating-systems.md b/docs/architecture/operating-systems.md index 7e40d27f0..3a3b8e4c5 100644 --- a/docs/architecture/operating-systems.md +++ b/docs/architecture/operating-systems.md @@ -26,12 +26,12 @@ Together, they diversify Typhoon to support a range of container technologies. ## Kubernetes Properties -| Property | Container Linux | Fedora Atomic | +| Property | Container Linux | Fedora CoreOS | |-------------------|-----------------|---------------| | single-master | all platforms | all platforms | | multi-master | all platforms | all platforms | | control plane | self-hosted | self-hosted | -| kubelet image | upstream hyperkube | upstream hyperkube via [system container](https://github.com/poseidon/system-containers) | +| kubelet image | upstream hyperkube | upstream hyperkube | | control plane images | upstream hyperkube | upstream hyperkube | | on-host etcd | rkt-fly | podman | | on-host kubelet | rkt-fly | podman | From bb557b4ba092716ff7fadf2fd73de57d7543a65c Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Thu, 18 Jul 2019 23:44:08 -0700 Subject: [PATCH 165/523] Fix Fedora CoreOS preview links on docs site --- docs/index.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/index.md b/docs/index.md index af432c385..a2f2063b1 100644 --- a/docs/index.md +++ b/docs/index.md @@ -33,8 +33,8 @@ A preview of Typhoon for [Fedora CoreOS](https://getfedora.org/coreos/) is avail | Platform | Operating System | Terraform Module | Status | |---------------|------------------|------------------|--------| -| AWS | Fedora CoreOS | [aws/fedora-coreos/kubernetes](aws/fedora-coreos/kubernetes) | preview | -| Bare-Metal | Fedora CoreOS | [bare-metal/fedora-coreos/kubernetes](bare-metal/fedora-coreos/kubernetes) | preview | +| AWS | Fedora CoreOS | [aws/fedora-coreos/kubernetes](fedora-coreos/aws.md) | preview | +| Bare-Metal | Fedora CoreOS | [bare-metal/fedora-coreos/kubernetes](fedora-coreos/bare-metal.md) | preview | ## Documentation From 6cd3e65267fd129dc05bc7aa6b0cc69366591282 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Thu, 18 Jul 2019 23:50:49 -0700 Subject: [PATCH 166/523] Update kube-state-metrics from v1.7.0-rc.1 to v1.7.0 * https://github.com/kubernetes/kube-state-metrics/releases/tag/v1.7.0 * Add storageclasses and verticalpodautoscalers to ClusterRole --- CHANGES.md | 2 +- .../kube-state-metrics/cluster-role.yaml | 15 +++++++++++++++ .../exporters/kube-state-metrics/deployment.yaml | 2 +- 3 files changed, 17 insertions(+), 2 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index a2276bfc7..1123efce4 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -19,7 +19,7 @@ Notable changes between versions. #### Addons * Update Prometheus from v2.10.0 to v2.11.0 - * Update kube-state-metrics from v1.6.0 to v1.7.0-rc.1 + * Update kube-state-metrics from v1.6.0 to v1.7.0 * Update Grafana from v6.2.4 to v6.2.5 * Update nginx-ingress from v0.24.1 to v0.25.0 * Support `networking.k8s.io/v1beta1` apiVersion diff --git a/addons/prometheus/exporters/kube-state-metrics/cluster-role.yaml b/addons/prometheus/exporters/kube-state-metrics/cluster-role.yaml index 6cd144b78..c19e6f45f 100644 --- a/addons/prometheus/exporters/kube-state-metrics/cluster-role.yaml +++ b/addons/prometheus/exporters/kube-state-metrics/cluster-role.yaml @@ -70,3 +70,18 @@ rules: verbs: - list - watch +- apiGroups: + - storage.k8s.io + resources: + - storageclasses + verbs: + - list + - watch +- apiGroups: + - autoscaling.k8s.io + resources: + - verticalpodautoscalers + verbs: + - list + - watch + diff --git a/addons/prometheus/exporters/kube-state-metrics/deployment.yaml b/addons/prometheus/exporters/kube-state-metrics/deployment.yaml index dc17195f2..829fd2ffb 100644 --- a/addons/prometheus/exporters/kube-state-metrics/deployment.yaml +++ b/addons/prometheus/exporters/kube-state-metrics/deployment.yaml @@ -24,7 +24,7 @@ spec: serviceAccountName: kube-state-metrics containers: - name: kube-state-metrics - image: quay.io/coreos/kube-state-metrics:v1.7.0-rc.1 + image: quay.io/coreos/kube-state-metrics:v1.7.0 ports: - name: metrics containerPort: 8080 From 339e323491d16deb0ad06c15855a7b1986589767 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Thu, 18 Jul 2019 23:57:53 -0700 Subject: [PATCH 167/523] Temporarily turn off QoS cgroups on Fedora CoreOS controllers * Kubelets can hit the ContainerManager Delegation issue and fail to start (noted in 72c94f1c6). Its unclear why this occurs only to some Kubelets (possibly an ordering concern) * QoS cgroups remain a goal * When a controller node is affected, bootstrapping fails, which makes other development harder. Temporarily disable QoS on controllers only. This should safeguard bring-up and hopefully still allow the issue to occur on some workers for debugging --- aws/fedora-coreos/kubernetes/fcc/controller.yaml | 4 ++-- bare-metal/fedora-coreos/kubernetes/fcc/controller.yaml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/aws/fedora-coreos/kubernetes/fcc/controller.yaml b/aws/fedora-coreos/kubernetes/fcc/controller.yaml index 7130c3047..a688390a3 100644 --- a/aws/fedora-coreos/kubernetes/fcc/controller.yaml +++ b/aws/fedora-coreos/kubernetes/fcc/controller.yaml @@ -85,8 +85,8 @@ systemd: --authentication-token-webhook \ --authorization-mode=Webhook \ --cgroup-driver=systemd \ - --cgroups-per-qos=true \ - --enforce-node-allocatable=pods \ + --cgroups-per-qos=false \ + --enforce-node-allocatable="" \ --client-ca-file=/etc/kubernetes/ca.crt \ --cluster_dns=${cluster_dns_service_ip} \ --cluster_domain=${cluster_domain_suffix} \ diff --git a/bare-metal/fedora-coreos/kubernetes/fcc/controller.yaml b/bare-metal/fedora-coreos/kubernetes/fcc/controller.yaml index bd7b4b7a9..9c946c4eb 100644 --- a/bare-metal/fedora-coreos/kubernetes/fcc/controller.yaml +++ b/bare-metal/fedora-coreos/kubernetes/fcc/controller.yaml @@ -86,8 +86,8 @@ systemd: --authentication-token-webhook \ --authorization-mode=Webhook \ --cgroup-driver=systemd \ - --cgroups-per-qos=true \ - --enforce-node-allocatable=pods \ + --cgroups-per-qos=false \ + --enforce-node-allocatable="" \ --client-ca-file=/etc/kubernetes/ca.crt \ --cluster_dns=${cluster_dns_service_ip} \ --cluster_domain=${cluster_domain_suffix} \ From e0c7676a1571220d5556bdca2e14802cbd9c461a Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Fri, 19 Jul 2019 01:15:15 -0700 Subject: [PATCH 168/523] Update Kubernetes from v1.15.0 to v1.15.1 * https://github.com/kubernetes/kubernetes/blob/master/CHANGELOG-1.15.md#downloads-for-v1151 --- CHANGES.md | 1 + README.md | 10 +++++----- aws/container-linux/kubernetes/README.md | 2 +- aws/container-linux/kubernetes/bootkube.tf | 2 +- .../kubernetes/cl/controller.yaml.tmpl | 2 +- .../kubernetes/workers/cl/worker.yaml.tmpl | 4 ++-- aws/fedora-coreos/kubernetes/README.md | 2 +- aws/fedora-coreos/kubernetes/bootkube.tf | 2 +- aws/fedora-coreos/kubernetes/fcc/controller.yaml | 2 +- .../kubernetes/workers/fcc/worker.yaml | 2 +- azure/container-linux/kubernetes/README.md | 2 +- azure/container-linux/kubernetes/bootkube.tf | 2 +- .../kubernetes/cl/controller.yaml.tmpl | 2 +- .../kubernetes/workers/cl/worker.yaml.tmpl | 4 ++-- bare-metal/container-linux/kubernetes/README.md | 2 +- bare-metal/container-linux/kubernetes/bootkube.tf | 2 +- .../kubernetes/cl/controller.yaml.tmpl | 2 +- .../container-linux/kubernetes/cl/worker.yaml.tmpl | 2 +- bare-metal/fedora-coreos/kubernetes/README.md | 2 +- bare-metal/fedora-coreos/kubernetes/bootkube.tf | 2 +- .../fedora-coreos/kubernetes/fcc/controller.yaml | 2 +- .../fedora-coreos/kubernetes/fcc/worker.yaml | 2 +- digital-ocean/container-linux/kubernetes/README.md | 2 +- .../container-linux/kubernetes/bootkube.tf | 2 +- .../kubernetes/cl/controller.yaml.tmpl | 2 +- .../container-linux/kubernetes/cl/worker.yaml.tmpl | 4 ++-- docs/advanced/worker-pools.md | 14 +++++++------- docs/cl/aws.md | 10 +++++----- docs/cl/azure.md | 10 +++++----- docs/cl/bare-metal.md | 14 +++++++------- docs/cl/digital-ocean.md | 10 +++++----- docs/cl/google-cloud.md | 10 +++++----- docs/fedora-coreos/aws.md | 8 ++++---- docs/fedora-coreos/bare-metal.md | 8 ++++---- docs/index.md | 10 +++++----- docs/topics/maintenance.md | 10 +++++----- google-cloud/container-linux/kubernetes/README.md | 2 +- .../container-linux/kubernetes/bootkube.tf | 2 +- .../kubernetes/cl/controller.yaml.tmpl | 2 +- .../kubernetes/workers/cl/worker.yaml.tmpl | 4 ++-- 40 files changed, 90 insertions(+), 89 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 1123efce4..b4362f2c8 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -4,6 +4,7 @@ Notable changes between versions. ## Latest +* Kubernetes [v1.15.1](https://github.com/kubernetes/kubernetes/blob/master/CHANGELOG-1.15.md#v1151) * Upgrade Calico from v3.7.3 to [v3.8.0](https://docs.projectcalico.org/v3.8/release-notes/) * Enable CNI `bandwidth` plugin for [traffic shaping](https://kubernetes.io/docs/concepts/extend-kubernetes/compute-storage-net/network-plugins/#support-traffic-shaping) * Run `kube-apiserver` with lower privilege user (nobody) ([#506](https://github.com/poseidon/typhoon/pull/506)) diff --git a/README.md b/README.md index eb3a0c4fe..4b5a41286 100644 --- a/README.md +++ b/README.md @@ -11,7 +11,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features -* Kubernetes v1.15.0 (upstream, via [kubernetes-incubator/bootkube](https://github.com/kubernetes-incubator/bootkube)) +* Kubernetes v1.15.1 (upstream, via [kubernetes-incubator/bootkube](https://github.com/kubernetes-incubator/bootkube)) * Single or multi-master, [Calico](https://www.projectcalico.org/) or [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/) * Advanced features like [worker pools](https://typhoon.psdn.io/advanced/worker-pools/), [preemptible](https://typhoon.psdn.io/cl/google-cloud/#preemption) workers, and [snippets](https://typhoon.psdn.io/advanced/customization/#container-linux) customization @@ -48,7 +48,7 @@ Define a Kubernetes cluster by using the Terraform module for your chosen platfo ```tf module "google-cloud-yavin" { - source = "git::https://github.com/poseidon/typhoon//google-cloud/container-linux/kubernetes?ref=v1.15.0" + source = "git::https://github.com/poseidon/typhoon//google-cloud/container-linux/kubernetes?ref=v1.15.1" # Google Cloud cluster_name = "yavin" @@ -82,9 +82,9 @@ In 4-8 minutes (varies by platform), the cluster will be ready. This Google Clou $ export KUBECONFIG=/home/user/.secrets/clusters/yavin/auth/kubeconfig $ kubectl get nodes NAME ROLES STATUS AGE VERSION -yavin-controller-0.c.example-com.internal controller,master Ready 6m v1.15.0 -yavin-worker-jrbf.c.example-com.internal node Ready 5m v1.15.0 -yavin-worker-mzdm.c.example-com.internal node Ready 5m v1.15.0 +yavin-controller-0.c.example-com.internal controller,master Ready 6m v1.15.1 +yavin-worker-jrbf.c.example-com.internal node Ready 5m v1.15.1 +yavin-worker-mzdm.c.example-com.internal node Ready 5m v1.15.1 ``` List the pods. diff --git a/aws/container-linux/kubernetes/README.md b/aws/container-linux/kubernetes/README.md index ce8ee9a37..bb7645fac 100644 --- a/aws/container-linux/kubernetes/README.md +++ b/aws/container-linux/kubernetes/README.md @@ -11,7 +11,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features -* Kubernetes v1.15.0 (upstream, via [kubernetes-incubator/bootkube](https://github.com/kubernetes-incubator/bootkube)) +* Kubernetes v1.15.1 (upstream, via [kubernetes-incubator/bootkube](https://github.com/kubernetes-incubator/bootkube)) * Single or multi-master, [Calico](https://www.projectcalico.org/) or [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/) * Advanced features like [worker pools](https://typhoon.psdn.io/advanced/worker-pools/), [spot](https://typhoon.psdn.io/cl/aws/#spot) workers, and [snippets](https://typhoon.psdn.io/advanced/customization/#container-linux) customization diff --git a/aws/container-linux/kubernetes/bootkube.tf b/aws/container-linux/kubernetes/bootkube.tf index 9c17535d7..dae12fec8 100644 --- a/aws/container-linux/kubernetes/bootkube.tf +++ b/aws/container-linux/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=119cb00fa7b12e0ebd5a70c9c0a4e7eda2e8c3d6" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=5b9faa903182505ac534f42a2cf250d5d215941b" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/aws/container-linux/kubernetes/cl/controller.yaml.tmpl b/aws/container-linux/kubernetes/cl/controller.yaml.tmpl index 5005bf15e..09c3d9a06 100644 --- a/aws/container-linux/kubernetes/cl/controller.yaml.tmpl +++ b/aws/container-linux/kubernetes/cl/controller.yaml.tmpl @@ -125,7 +125,7 @@ storage: contents: inline: | KUBELET_IMAGE_URL=docker://k8s.gcr.io/hyperkube - KUBELET_IMAGE_TAG=v1.15.0 + KUBELET_IMAGE_TAG=v1.15.1 - path: /etc/sysctl.d/max-user-watches.conf filesystem: root contents: diff --git a/aws/container-linux/kubernetes/workers/cl/worker.yaml.tmpl b/aws/container-linux/kubernetes/workers/cl/worker.yaml.tmpl index 3f4495200..a2bd65db4 100644 --- a/aws/container-linux/kubernetes/workers/cl/worker.yaml.tmpl +++ b/aws/container-linux/kubernetes/workers/cl/worker.yaml.tmpl @@ -95,7 +95,7 @@ storage: contents: inline: | KUBELET_IMAGE_URL=docker://k8s.gcr.io/hyperkube - KUBELET_IMAGE_TAG=v1.15.0 + KUBELET_IMAGE_TAG=v1.15.1 - path: /etc/sysctl.d/max-user-watches.conf filesystem: root contents: @@ -113,7 +113,7 @@ storage: --volume config,kind=host,source=/etc/kubernetes \ --mount volume=config,target=/etc/kubernetes \ --insecure-options=image \ - docker://k8s.gcr.io/hyperkube:v1.15.0 \ + docker://k8s.gcr.io/hyperkube:v1.15.1 \ --net=host \ --dns=host \ --exec=/kubectl -- --kubeconfig=/etc/kubernetes/kubeconfig delete node $(hostname) diff --git a/aws/fedora-coreos/kubernetes/README.md b/aws/fedora-coreos/kubernetes/README.md index c48573298..375287bc1 100644 --- a/aws/fedora-coreos/kubernetes/README.md +++ b/aws/fedora-coreos/kubernetes/README.md @@ -11,7 +11,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features -* Kubernetes v1.15.0 (upstream, via [kubernetes-incubator/bootkube](https://github.com/kubernetes-incubator/bootkube)) +* Kubernetes v1.15.1 (upstream, via [kubernetes-incubator/bootkube](https://github.com/kubernetes-incubator/bootkube)) * Single or multi-master, [Calico](https://www.projectcalico.org/) or [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/) * Advanced features like [worker pools](https://typhoon.psdn.io/advanced/worker-pools/), [spot](https://typhoon.psdn.io/cl/aws/#spot) workers, and [snippets](https://typhoon.psdn.io/advanced/customization/#container-linux) customization diff --git a/aws/fedora-coreos/kubernetes/bootkube.tf b/aws/fedora-coreos/kubernetes/bootkube.tf index 9c58da8e1..d01a9a586 100644 --- a/aws/fedora-coreos/kubernetes/bootkube.tf +++ b/aws/fedora-coreos/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=119cb00fa7b12e0ebd5a70c9c0a4e7eda2e8c3d6" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=5b9faa903182505ac534f42a2cf250d5d215941b" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/aws/fedora-coreos/kubernetes/fcc/controller.yaml b/aws/fedora-coreos/kubernetes/fcc/controller.yaml index a688390a3..b98424180 100644 --- a/aws/fedora-coreos/kubernetes/fcc/controller.yaml +++ b/aws/fedora-coreos/kubernetes/fcc/controller.yaml @@ -80,7 +80,7 @@ systemd: --volume /var/run:/var/run \ --volume /var/run/lock:/var/run/lock:z \ --volume /opt/cni/bin:/opt/cni/bin:z \ - k8s.gcr.io/hyperkube:v1.15.0 /hyperkube kubelet \ + k8s.gcr.io/hyperkube:v1.15.1 /hyperkube kubelet \ --anonymous-auth=false \ --authentication-token-webhook \ --authorization-mode=Webhook \ diff --git a/aws/fedora-coreos/kubernetes/workers/fcc/worker.yaml b/aws/fedora-coreos/kubernetes/workers/fcc/worker.yaml index 12e517afb..3940ef6e8 100644 --- a/aws/fedora-coreos/kubernetes/workers/fcc/worker.yaml +++ b/aws/fedora-coreos/kubernetes/workers/fcc/worker.yaml @@ -50,7 +50,7 @@ systemd: --volume /var/run:/var/run \ --volume /var/run/lock:/var/run/lock:z \ --volume /opt/cni/bin:/opt/cni/bin:z \ - k8s.gcr.io/hyperkube:v1.15.0 /hyperkube kubelet \ + k8s.gcr.io/hyperkube:v1.15.1 /hyperkube kubelet \ --anonymous-auth=false \ --authentication-token-webhook \ --authorization-mode=Webhook \ diff --git a/azure/container-linux/kubernetes/README.md b/azure/container-linux/kubernetes/README.md index 2cc47ccd9..24a90b2b0 100644 --- a/azure/container-linux/kubernetes/README.md +++ b/azure/container-linux/kubernetes/README.md @@ -11,7 +11,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features -* Kubernetes v1.15.0 (upstream, via [kubernetes-incubator/bootkube](https://github.com/kubernetes-incubator/bootkube)) +* Kubernetes v1.15.1 (upstream, via [kubernetes-incubator/bootkube](https://github.com/kubernetes-incubator/bootkube)) * Single or multi-master, [Calico](https://www.projectcalico.org/) or [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/) * Advanced features like [worker pools](https://typhoon.psdn.io/advanced/worker-pools/), [low-priority](https://typhoon.psdn.io/cl/azure/#low-priority) workers, and [snippets](https://typhoon.psdn.io/advanced/customization/#container-linux) customization diff --git a/azure/container-linux/kubernetes/bootkube.tf b/azure/container-linux/kubernetes/bootkube.tf index 6fada064e..86aa94811 100644 --- a/azure/container-linux/kubernetes/bootkube.tf +++ b/azure/container-linux/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=119cb00fa7b12e0ebd5a70c9c0a4e7eda2e8c3d6" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=5b9faa903182505ac534f42a2cf250d5d215941b" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/azure/container-linux/kubernetes/cl/controller.yaml.tmpl b/azure/container-linux/kubernetes/cl/controller.yaml.tmpl index 34755c18b..a6eea4a4a 100644 --- a/azure/container-linux/kubernetes/cl/controller.yaml.tmpl +++ b/azure/container-linux/kubernetes/cl/controller.yaml.tmpl @@ -123,7 +123,7 @@ storage: contents: inline: | KUBELET_IMAGE_URL=docker://k8s.gcr.io/hyperkube - KUBELET_IMAGE_TAG=v1.15.0 + KUBELET_IMAGE_TAG=v1.15.1 - path: /etc/sysctl.d/max-user-watches.conf filesystem: root contents: diff --git a/azure/container-linux/kubernetes/workers/cl/worker.yaml.tmpl b/azure/container-linux/kubernetes/workers/cl/worker.yaml.tmpl index ec3000399..b94e23d18 100644 --- a/azure/container-linux/kubernetes/workers/cl/worker.yaml.tmpl +++ b/azure/container-linux/kubernetes/workers/cl/worker.yaml.tmpl @@ -93,7 +93,7 @@ storage: contents: inline: | KUBELET_IMAGE_URL=docker://k8s.gcr.io/hyperkube - KUBELET_IMAGE_TAG=v1.15.0 + KUBELET_IMAGE_TAG=v1.15.1 - path: /etc/sysctl.d/max-user-watches.conf filesystem: root contents: @@ -111,7 +111,7 @@ storage: --volume config,kind=host,source=/etc/kubernetes \ --mount volume=config,target=/etc/kubernetes \ --insecure-options=image \ - docker://k8s.gcr.io/hyperkube:v1.15.0 \ + docker://k8s.gcr.io/hyperkube:v1.15.1 \ --net=host \ --dns=host \ --exec=/kubectl -- --kubeconfig=/etc/kubernetes/kubeconfig delete node $(hostname | tr '[:upper:]' '[:lower:]') diff --git a/bare-metal/container-linux/kubernetes/README.md b/bare-metal/container-linux/kubernetes/README.md index 4342252a7..001a140b2 100644 --- a/bare-metal/container-linux/kubernetes/README.md +++ b/bare-metal/container-linux/kubernetes/README.md @@ -11,7 +11,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features -* Kubernetes v1.15.0 (upstream, via [kubernetes-incubator/bootkube](https://github.com/kubernetes-incubator/bootkube)) +* Kubernetes v1.15.1 (upstream, via [kubernetes-incubator/bootkube](https://github.com/kubernetes-incubator/bootkube)) * Single or multi-master, [Calico](https://www.projectcalico.org/) or [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/) * Advanced features like [snippets](https://typhoon.psdn.io/advanced/customization/#container-linux) customization diff --git a/bare-metal/container-linux/kubernetes/bootkube.tf b/bare-metal/container-linux/kubernetes/bootkube.tf index 9965e7a85..55e57608a 100644 --- a/bare-metal/container-linux/kubernetes/bootkube.tf +++ b/bare-metal/container-linux/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=119cb00fa7b12e0ebd5a70c9c0a4e7eda2e8c3d6" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=5b9faa903182505ac534f42a2cf250d5d215941b" cluster_name = var.cluster_name api_servers = [var.k8s_domain_name] diff --git a/bare-metal/container-linux/kubernetes/cl/controller.yaml.tmpl b/bare-metal/container-linux/kubernetes/cl/controller.yaml.tmpl index 15a1316d2..7528c3cac 100644 --- a/bare-metal/container-linux/kubernetes/cl/controller.yaml.tmpl +++ b/bare-metal/container-linux/kubernetes/cl/controller.yaml.tmpl @@ -130,7 +130,7 @@ storage: contents: inline: | KUBELET_IMAGE_URL=docker://k8s.gcr.io/hyperkube - KUBELET_IMAGE_TAG=v1.15.0 + KUBELET_IMAGE_TAG=v1.15.1 - path: /etc/hostname filesystem: root mode: 0644 diff --git a/bare-metal/container-linux/kubernetes/cl/worker.yaml.tmpl b/bare-metal/container-linux/kubernetes/cl/worker.yaml.tmpl index 24270cfdc..26d826efc 100644 --- a/bare-metal/container-linux/kubernetes/cl/worker.yaml.tmpl +++ b/bare-metal/container-linux/kubernetes/cl/worker.yaml.tmpl @@ -91,7 +91,7 @@ storage: contents: inline: | KUBELET_IMAGE_URL=docker://k8s.gcr.io/hyperkube - KUBELET_IMAGE_TAG=v1.15.0 + KUBELET_IMAGE_TAG=v1.15.1 - path: /etc/hostname filesystem: root mode: 0644 diff --git a/bare-metal/fedora-coreos/kubernetes/README.md b/bare-metal/fedora-coreos/kubernetes/README.md index 3896705ac..7d6ac2382 100644 --- a/bare-metal/fedora-coreos/kubernetes/README.md +++ b/bare-metal/fedora-coreos/kubernetes/README.md @@ -11,7 +11,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features -* Kubernetes v1.15.0 (upstream, via [kubernetes-incubator/bootkube](https://github.com/kubernetes-incubator/bootkube)) +* Kubernetes v1.15.1 (upstream, via [kubernetes-incubator/bootkube](https://github.com/kubernetes-incubator/bootkube)) * Single or multi-master, [Calico](https://www.projectcalico.org/) or [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/) * Advanced features like [snippets](https://typhoon.psdn.io/advanced/customization/#container-linux) customization diff --git a/bare-metal/fedora-coreos/kubernetes/bootkube.tf b/bare-metal/fedora-coreos/kubernetes/bootkube.tf index 06c50989f..2e2751c18 100644 --- a/bare-metal/fedora-coreos/kubernetes/bootkube.tf +++ b/bare-metal/fedora-coreos/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=119cb00fa7b12e0ebd5a70c9c0a4e7eda2e8c3d6" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=5b9faa903182505ac534f42a2cf250d5d215941b" cluster_name = var.cluster_name api_servers = [var.k8s_domain_name] diff --git a/bare-metal/fedora-coreos/kubernetes/fcc/controller.yaml b/bare-metal/fedora-coreos/kubernetes/fcc/controller.yaml index 9c946c4eb..22d20b81e 100644 --- a/bare-metal/fedora-coreos/kubernetes/fcc/controller.yaml +++ b/bare-metal/fedora-coreos/kubernetes/fcc/controller.yaml @@ -81,7 +81,7 @@ systemd: --volume /opt/cni/bin:/opt/cni/bin:z \ --volume /etc/iscsi:/etc/iscsi \ --volume /sbin/iscsiadm:/sbin/iscsiadm \ - k8s.gcr.io/hyperkube:v1.15.0 /hyperkube kubelet \ + k8s.gcr.io/hyperkube:v1.15.1 /hyperkube kubelet \ --anonymous-auth=false \ --authentication-token-webhook \ --authorization-mode=Webhook \ diff --git a/bare-metal/fedora-coreos/kubernetes/fcc/worker.yaml b/bare-metal/fedora-coreos/kubernetes/fcc/worker.yaml index 482bf2c7b..5fa9c25ca 100644 --- a/bare-metal/fedora-coreos/kubernetes/fcc/worker.yaml +++ b/bare-metal/fedora-coreos/kubernetes/fcc/worker.yaml @@ -51,7 +51,7 @@ systemd: --volume /opt/cni/bin:/opt/cni/bin:z \ --volume /etc/iscsi:/etc/iscsi \ --volume /sbin/iscsiadm:/sbin/iscsiadm \ - k8s.gcr.io/hyperkube:v1.15.0 /hyperkube kubelet \ + k8s.gcr.io/hyperkube:v1.15.1 /hyperkube kubelet \ --anonymous-auth=false \ --authentication-token-webhook \ --authorization-mode=Webhook \ diff --git a/digital-ocean/container-linux/kubernetes/README.md b/digital-ocean/container-linux/kubernetes/README.md index 01ad9f01b..ea079a815 100644 --- a/digital-ocean/container-linux/kubernetes/README.md +++ b/digital-ocean/container-linux/kubernetes/README.md @@ -11,7 +11,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features -* Kubernetes v1.15.0 (upstream, via [kubernetes-incubator/bootkube](https://github.com/kubernetes-incubator/bootkube)) +* Kubernetes v1.15.1 (upstream, via [kubernetes-incubator/bootkube](https://github.com/kubernetes-incubator/bootkube)) * Single or multi-master, [Calico](https://www.projectcalico.org/) or [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/) * Advanced features like [snippets](https://typhoon.psdn.io/advanced/customization/#container-linux) customization diff --git a/digital-ocean/container-linux/kubernetes/bootkube.tf b/digital-ocean/container-linux/kubernetes/bootkube.tf index fab9828bf..f42bfa507 100644 --- a/digital-ocean/container-linux/kubernetes/bootkube.tf +++ b/digital-ocean/container-linux/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=119cb00fa7b12e0ebd5a70c9c0a4e7eda2e8c3d6" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=5b9faa903182505ac534f42a2cf250d5d215941b" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/digital-ocean/container-linux/kubernetes/cl/controller.yaml.tmpl b/digital-ocean/container-linux/kubernetes/cl/controller.yaml.tmpl index 5891e4029..97c7ae26a 100644 --- a/digital-ocean/container-linux/kubernetes/cl/controller.yaml.tmpl +++ b/digital-ocean/container-linux/kubernetes/cl/controller.yaml.tmpl @@ -129,7 +129,7 @@ storage: contents: inline: | KUBELET_IMAGE_URL=docker://k8s.gcr.io/hyperkube - KUBELET_IMAGE_TAG=v1.15.0 + KUBELET_IMAGE_TAG=v1.15.1 - path: /etc/sysctl.d/max-user-watches.conf filesystem: root contents: diff --git a/digital-ocean/container-linux/kubernetes/cl/worker.yaml.tmpl b/digital-ocean/container-linux/kubernetes/cl/worker.yaml.tmpl index cca36471d..57276d074 100644 --- a/digital-ocean/container-linux/kubernetes/cl/worker.yaml.tmpl +++ b/digital-ocean/container-linux/kubernetes/cl/worker.yaml.tmpl @@ -99,7 +99,7 @@ storage: contents: inline: | KUBELET_IMAGE_URL=docker://k8s.gcr.io/hyperkube - KUBELET_IMAGE_TAG=v1.15.0 + KUBELET_IMAGE_TAG=v1.15.1 - path: /etc/sysctl.d/max-user-watches.conf filesystem: root contents: @@ -117,7 +117,7 @@ storage: --volume config,kind=host,source=/etc/kubernetes \ --mount volume=config,target=/etc/kubernetes \ --insecure-options=image \ - docker://k8s.gcr.io/hyperkube:v1.15.0 \ + docker://k8s.gcr.io/hyperkube:v1.15.1 \ --net=host \ --dns=host \ --exec=/kubectl -- --kubeconfig=/etc/kubernetes/kubeconfig delete node $(hostname) diff --git a/docs/advanced/worker-pools.md b/docs/advanced/worker-pools.md index 0e5d758a9..650abcaa2 100644 --- a/docs/advanced/worker-pools.md +++ b/docs/advanced/worker-pools.md @@ -76,7 +76,7 @@ Create a cluster following the Azure [tutorial](../cl/azure.md#cluster). Define ```tf module "ramius-worker-pool" { - source = "git::https://github.com/poseidon/typhoon//azure/container-linux/kubernetes/workers?ref=v1.15.0" + source = "git::https://github.com/poseidon/typhoon//azure/container-linux/kubernetes/workers?ref=v1.15.1" # Azure region = module.azure-ramius.region @@ -142,7 +142,7 @@ Create a cluster following the Google Cloud [tutorial](../cl/google-cloud.md#clu ```tf module "yavin-worker-pool" { - source = "git::https://github.com/poseidon/typhoon//google-cloud/container-linux/kubernetes/workers?ref=v1.15.0" + source = "git::https://github.com/poseidon/typhoon//google-cloud/container-linux/kubernetes/workers?ref=v1.15.1" # Google Cloud region = "europe-west2" @@ -173,11 +173,11 @@ Verify a managed instance group of workers joins the cluster within a few minute ``` $ kubectl get nodes NAME STATUS AGE VERSION -yavin-controller-0.c.example-com.internal Ready 6m v1.15.0 -yavin-worker-jrbf.c.example-com.internal Ready 5m v1.15.0 -yavin-worker-mzdm.c.example-com.internal Ready 5m v1.15.0 -yavin-16x-worker-jrbf.c.example-com.internal Ready 3m v1.15.0 -yavin-16x-worker-mzdm.c.example-com.internal Ready 3m v1.15.0 +yavin-controller-0.c.example-com.internal Ready 6m v1.15.1 +yavin-worker-jrbf.c.example-com.internal Ready 5m v1.15.1 +yavin-worker-mzdm.c.example-com.internal Ready 5m v1.15.1 +yavin-16x-worker-jrbf.c.example-com.internal Ready 3m v1.15.1 +yavin-16x-worker-mzdm.c.example-com.internal Ready 3m v1.15.1 ``` ### Variables diff --git a/docs/cl/aws.md b/docs/cl/aws.md index 3d09ff6d9..edf0b0de0 100644 --- a/docs/cl/aws.md +++ b/docs/cl/aws.md @@ -1,6 +1,6 @@ # AWS -In this tutorial, we'll create a Kubernetes v1.15.0 cluster on AWS with Container Linux. +In this tutorial, we'll create a Kubernetes v1.15.1 cluster on AWS with Container Linux. We'll declare a Kubernetes cluster using the Typhoon Terraform module. Then apply the changes to create a VPC, gateway, subnets, security groups, controller instances, worker auto-scaling group, network load balancer, and TLS assets. @@ -70,7 +70,7 @@ Define a Kubernetes cluster using the module `aws/container-linux/kubernetes`. ```tf module "aws-tempest" { - source = "git::https://github.com/poseidon/typhoon//aws/container-linux/kubernetes?ref=v1.15.0" + source = "git::https://github.com/poseidon/typhoon//aws/container-linux/kubernetes?ref=v1.15.1" # AWS cluster_name = "tempest" @@ -135,9 +135,9 @@ In 4-8 minutes, the Kubernetes cluster will be ready. $ export KUBECONFIG=/home/user/.secrets/clusters/tempest/auth/kubeconfig $ kubectl get nodes NAME STATUS ROLES AGE VERSION -ip-10-0-3-155 Ready controller,master 10m v1.15.0 -ip-10-0-26-65 Ready node 10m v1.15.0 -ip-10-0-41-21 Ready node 10m v1.15.0 +ip-10-0-3-155 Ready controller,master 10m v1.15.1 +ip-10-0-26-65 Ready node 10m v1.15.1 +ip-10-0-41-21 Ready node 10m v1.15.1 ``` List the pods. diff --git a/docs/cl/azure.md b/docs/cl/azure.md index ec9931afe..84777f205 100644 --- a/docs/cl/azure.md +++ b/docs/cl/azure.md @@ -3,7 +3,7 @@ !!! danger Typhoon for Azure is alpha. For production, use AWS, Google Cloud, or bare-metal. As Azure matures, check [errata](https://github.com/poseidon/typhoon/wiki/Errata) for known shortcomings. -In this tutorial, we'll create a Kubernetes v1.15.0 cluster on Azure with Container Linux. +In this tutorial, we'll create a Kubernetes v1.15.1 cluster on Azure with Container Linux. We'll declare a Kubernetes cluster using the Typhoon Terraform module. Then apply the changes to create a resource group, virtual network, subnets, security groups, controller availability set, worker scale set, load balancer, and TLS assets. @@ -66,7 +66,7 @@ Define a Kubernetes cluster using the module `azure/container-linux/kubernetes`. ```tf module "azure-ramius" { - source = "git::https://github.com/poseidon/typhoon//azure/container-linux/kubernetes?ref=v1.15.0" + source = "git::https://github.com/poseidon/typhoon//azure/container-linux/kubernetes?ref=v1.15.1" # Azure cluster_name = "ramius" @@ -132,9 +132,9 @@ In 4-8 minutes, the Kubernetes cluster will be ready. $ export KUBECONFIG=/home/user/.secrets/clusters/ramius/auth/kubeconfig $ kubectl get nodes NAME STATUS ROLES AGE VERSION -ramius-controller-0 Ready controller,master 24m v1.15.0 -ramius-worker-000001 Ready node 25m v1.15.0 -ramius-worker-000002 Ready node 24m v1.15.0 +ramius-controller-0 Ready controller,master 24m v1.15.1 +ramius-worker-000001 Ready node 25m v1.15.1 +ramius-worker-000002 Ready node 24m v1.15.1 ``` List the pods. diff --git a/docs/cl/bare-metal.md b/docs/cl/bare-metal.md index 2941e15fe..76c85a3f1 100644 --- a/docs/cl/bare-metal.md +++ b/docs/cl/bare-metal.md @@ -1,6 +1,6 @@ # Bare-Metal -In this tutorial, we'll network boot and provision a Kubernetes v1.15.0 cluster on bare-metal with Container Linux. +In this tutorial, we'll network boot and provision a Kubernetes v1.15.1 cluster on bare-metal with Container Linux. First, we'll deploy a [Matchbox](https://github.com/poseidon/matchbox) service and setup a network boot environment. Then, we'll declare a Kubernetes cluster using the Typhoon Terraform module and power on machines. On PXE boot, machines will install Container Linux to disk, reboot into the disk install, and provision themselves as Kubernetes controllers or workers via Ignition. @@ -160,7 +160,7 @@ Define a Kubernetes cluster using the module `bare-metal/container-linux/kuberne ```tf module "bare-metal-mercury" { - source = "git::https://github.com/poseidon/typhoon//bare-metal/container-linux/kubernetes?ref=v1.15.0" + source = "git::https://github.com/poseidon/typhoon//bare-metal/container-linux/kubernetes?ref=v1.15.1" # bare-metal cluster_name = "mercury" @@ -265,9 +265,9 @@ Apply complete! Resources: 55 added, 0 changed, 0 destroyed. To watch the install to disk (until machines reboot from disk), SSH to port 2222. ``` -# before v1.15.0 +# before v1.15.1 $ ssh debug@node1.example.com -# after v1.15.0 +# after v1.15.1 $ ssh -p 2222 core@node1.example.com ``` @@ -292,9 +292,9 @@ bootkube[5]: Tearing down temporary bootstrap control plane... $ export KUBECONFIG=/home/user/.secrets/clusters/mercury/auth/kubeconfig $ kubectl get nodes NAME STATUS ROLES AGE VERSION -node1.example.com Ready controller,master 10m v1.15.0 -node2.example.com Ready node 10m v1.15.0 -node3.example.com Ready node 10m v1.15.0 +node1.example.com Ready controller,master 10m v1.15.1 +node2.example.com Ready node 10m v1.15.1 +node3.example.com Ready node 10m v1.15.1 ``` List the pods. diff --git a/docs/cl/digital-ocean.md b/docs/cl/digital-ocean.md index fa66e077b..71cf4b2ac 100644 --- a/docs/cl/digital-ocean.md +++ b/docs/cl/digital-ocean.md @@ -1,6 +1,6 @@ # Digital Ocean -In this tutorial, we'll create a Kubernetes v1.15.0 cluster on DigitalOcean with Container Linux. +In this tutorial, we'll create a Kubernetes v1.15.1 cluster on DigitalOcean with Container Linux. We'll declare a Kubernetes cluster using the Typhoon Terraform module. Then apply the changes to create controller droplets, worker droplets, DNS records, tags, and TLS assets. @@ -65,7 +65,7 @@ Define a Kubernetes cluster using the module `digital-ocean/container-linux/kube ```tf module "digital-ocean-nemo" { - source = "git::https://github.com/poseidon/typhoon//digital-ocean/container-linux/kubernetes?ref=v1.15.0" + source = "git::https://github.com/poseidon/typhoon//digital-ocean/container-linux/kubernetes?ref=v1.15.1" # Digital Ocean cluster_name = "nemo" @@ -130,9 +130,9 @@ In 3-6 minutes, the Kubernetes cluster will be ready. $ export KUBECONFIG=/home/user/.secrets/clusters/nemo/auth/kubeconfig $ kubectl get nodes NAME STATUS ROLES AGE VERSION -10.132.110.130 Ready controller,master 10m v1.15.0 -10.132.115.81 Ready node 10m v1.15.0 -10.132.124.107 Ready node 10m v1.15.0 +10.132.110.130 Ready controller,master 10m v1.15.1 +10.132.115.81 Ready node 10m v1.15.1 +10.132.124.107 Ready node 10m v1.15.1 ``` List the pods. diff --git a/docs/cl/google-cloud.md b/docs/cl/google-cloud.md index cb562a673..d6aef86d3 100644 --- a/docs/cl/google-cloud.md +++ b/docs/cl/google-cloud.md @@ -1,6 +1,6 @@ # Google Cloud -In this tutorial, we'll create a Kubernetes v1.15.0 cluster on Google Compute Engine with Container Linux. +In this tutorial, we'll create a Kubernetes v1.15.1 cluster on Google Compute Engine with Container Linux. We'll declare a Kubernetes cluster using the Typhoon Terraform module. Then apply the changes to create a network, firewall rules, health checks, controller instances, worker managed instance group, load balancers, and TLS assets. @@ -71,7 +71,7 @@ Define a Kubernetes cluster using the module `google-cloud/container-linux/kuber ```tf module "google-cloud-yavin" { - source = "git::https://github.com/poseidon/typhoon//google-cloud/container-linux/kubernetes?ref=v1.15.0" + source = "git::https://github.com/poseidon/typhoon//google-cloud/container-linux/kubernetes?ref=v1.15.1" # Google Cloud cluster_name = "yavin" @@ -138,9 +138,9 @@ In 4-8 minutes, the Kubernetes cluster will be ready. $ export KUBECONFIG=/home/user/.secrets/clusters/yavin/auth/kubeconfig $ kubectl get nodes NAME ROLES STATUS AGE VERSION -yavin-controller-0.c.example-com.internal controller,master Ready 6m v1.15.0 -yavin-worker-jrbf.c.example-com.internal node Ready 5m v1.15.0 -yavin-worker-mzdm.c.example-com.internal node Ready 5m v1.15.0 +yavin-controller-0.c.example-com.internal controller,master Ready 6m v1.15.1 +yavin-worker-jrbf.c.example-com.internal node Ready 5m v1.15.1 +yavin-worker-mzdm.c.example-com.internal node Ready 5m v1.15.1 ``` List the pods. diff --git a/docs/fedora-coreos/aws.md b/docs/fedora-coreos/aws.md index 8382745da..0060c6525 100644 --- a/docs/fedora-coreos/aws.md +++ b/docs/fedora-coreos/aws.md @@ -3,7 +3,7 @@ !!! danger Typhoon for Fedora CoreOS is an early preview! Fedora CoreOS itself is a preview! Expect bugs and design shifts. Please help both projects solve problems. Report Fedora CoreOS bugs to [Fedora](https://github.com/coreos/fedora-coreos-tracker/issues). Report Typhoon issues to Typhoon. -In this tutorial, we'll create a Kubernetes v1.15.0 cluster on AWS with Fedora CoreOS. +In this tutorial, we'll create a Kubernetes v1.15.1 cluster on AWS with Fedora CoreOS. We'll declare a Kubernetes cluster using the Typhoon Terraform module. Then apply the changes to create a VPC, gateway, subnets, security groups, controller instances, worker auto-scaling group, network load balancer, and TLS assets. @@ -138,9 +138,9 @@ In 4-8 minutes, the Kubernetes cluster will be ready. $ export KUBECONFIG=/home/user/.secrets/clusters/tempest/auth/kubeconfig $ kubectl get nodes NAME STATUS ROLES AGE VERSION -ip-10-0-3-155 Ready controller,master 10m v1.15.0 -ip-10-0-26-65 Ready node 10m v1.15.0 -ip-10-0-41-21 Ready node 10m v1.15.0 +ip-10-0-3-155 Ready controller,master 10m v1.15.1 +ip-10-0-26-65 Ready node 10m v1.15.1 +ip-10-0-41-21 Ready node 10m v1.15.1 ``` List the pods. diff --git a/docs/fedora-coreos/bare-metal.md b/docs/fedora-coreos/bare-metal.md index f08f0688d..816fec3b0 100644 --- a/docs/fedora-coreos/bare-metal.md +++ b/docs/fedora-coreos/bare-metal.md @@ -3,7 +3,7 @@ !!! danger Typhoon for Fedora CoreOS is an early preview! Fedora CoreOS itself is a preview! Expect bugs and design shifts. Please help both projects solve problems. Report Fedora CoreOS bugs to [Fedora](https://github.com/coreos/fedora-coreos-tracker/issues). Report Typhoon issues to Typhoon. -In this tutorial, we'll network boot and provision a Kubernetes v1.15.0 cluster on bare-metal with Fedora CoreOS. +In this tutorial, we'll network boot and provision a Kubernetes v1.15.1 cluster on bare-metal with Fedora CoreOS. First, we'll deploy a [Matchbox](https://github.com/poseidon/matchbox) service and setup a network boot environment. Then, we'll declare a Kubernetes cluster using the Typhoon Terraform module and power on machines. On PXE boot, machines will install Fedora CoreOS to disk, reboot into the disk install, and provision themselves as Kubernetes controllers or workers via Ignition. @@ -284,9 +284,9 @@ bootkube[5]: Tearing down temporary bootstrap control plane... $ export KUBECONFIG=/home/user/.secrets/clusters/mercury/auth/kubeconfig $ kubectl get nodes NAME STATUS ROLES AGE VERSION -node1.example.com Ready controller,master 10m v1.15.0 -node2.example.com Ready node 10m v1.15.0 -node3.example.com Ready node 10m v1.15.0 +node1.example.com Ready controller,master 10m v1.15.1 +node2.example.com Ready node 10m v1.15.1 +node3.example.com Ready node 10m v1.15.1 ``` List the pods. diff --git a/docs/index.md b/docs/index.md index a2f2063b1..cdd5bb00d 100644 --- a/docs/index.md +++ b/docs/index.md @@ -11,7 +11,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features -* Kubernetes v1.15.0 (upstream, via [kubernetes-incubator/bootkube](https://github.com/kubernetes-incubator/bootkube)) +* Kubernetes v1.15.1 (upstream, via [kubernetes-incubator/bootkube](https://github.com/kubernetes-incubator/bootkube)) * Single or multi-master, [Calico](https://www.projectcalico.org/) or [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/) * Advanced features like [worker pools](advanced/worker-pools/), [preemptible](cl/google-cloud/#preemption) workers, and [snippets](advanced/customization/#container-linux) customization @@ -47,7 +47,7 @@ Define a Kubernetes cluster by using the Terraform module for your chosen platfo ```tf module "google-cloud-yavin" { - source = "git::https://github.com/poseidon/typhoon//google-cloud/container-linux/kubernetes?ref=v1.15.0" + source = "git::https://github.com/poseidon/typhoon//google-cloud/container-linux/kubernetes?ref=v1.15.1" # Google Cloud cluster_name = "yavin" @@ -80,9 +80,9 @@ In 4-8 minutes (varies by platform), the cluster will be ready. This Google Clou $ export KUBECONFIG=/home/user/.secrets/clusters/yavin/auth/kubeconfig $ kubectl get nodes NAME ROLES STATUS AGE VERSION -yavin-controller-0.c.example-com.internal controller,master Ready 6m v1.15.0 -yavin-worker-jrbf.c.example-com.internal node Ready 5m v1.15.0 -yavin-worker-mzdm.c.example-com.internal node Ready 5m v1.15.0 +yavin-controller-0.c.example-com.internal controller,master Ready 6m v1.15.1 +yavin-worker-jrbf.c.example-com.internal node Ready 5m v1.15.1 +yavin-worker-mzdm.c.example-com.internal node Ready 5m v1.15.1 ``` List the pods. diff --git a/docs/topics/maintenance.md b/docs/topics/maintenance.md index 0d1237b25..49253a4c8 100644 --- a/docs/topics/maintenance.md +++ b/docs/topics/maintenance.md @@ -18,7 +18,7 @@ module "google-cloud-yavin" { } module "bare-metal-mercury" { - source = "git::https://github.com/poseidon/typhoon//bare-metal/container-linux/kubernetes?ref=v1.15.0" + source = "git::https://github.com/poseidon/typhoon//bare-metal/container-linux/kubernetes?ref=v1.15.1" ... } ``` @@ -279,15 +279,15 @@ Typhoon modules have been adapted for Terraform v0.12. Provider plugins requirem | Typhoon Release | Terraform version | |-------------------|---------------------| -| v1.15.0 - ? | v0.12.x | -| v1.10.3 - v1.15.0 | v0.11.x | +| v1.15.1 - ? | v0.12.x | +| v1.10.3 - v1.15.1 | v0.11.x | | v1.9.2 - v1.10.2 | v0.10.4+ or v0.11.x | | v1.7.3 - v1.9.1 | v0.10.x | | v1.6.4 - v1.7.2 | v0.9.x | ### New users -New users can start with Terraform v0.12.x and follow the docs for Typhoon v1.15.0+ without issue. +New users can start with Terraform v0.12.x and follow the docs for Typhoon v1.15.1+ without issue. ### Existing users @@ -404,7 +404,7 @@ tree . └── infraB <- new Terraform v0.12.x configs ``` -Define Typhoon clusters in the new config directory using Terraform v0.12 syntax. Follow the Typhoon v1.15.0+ docs (e.g. use `terraform12` in the `infraB` dir). See [AWS](/cl/aws), [Azure](/cl/azure), [Bare-Metal](/cl/bare-metal), [Digital Ocean](/cl/digital-ocean), or [Google-Cloud](/cl/google-cloud)) to create new clusters. Follow the usual [upgrade](/topics/maintenance/#upgrades) process to apply workloads and shift traffic. Later, switch back to the old config directory and deprovision clusters with Terraform v0.11. +Define Typhoon clusters in the new config directory using Terraform v0.12 syntax. Follow the Typhoon v1.15.1+ docs (e.g. use `terraform12` in the `infraB` dir). See [AWS](/cl/aws), [Azure](/cl/azure), [Bare-Metal](/cl/bare-metal), [Digital Ocean](/cl/digital-ocean), or [Google-Cloud](/cl/google-cloud)) to create new clusters. Follow the usual [upgrade](/topics/maintenance/#upgrades) process to apply workloads and shift traffic. Later, switch back to the old config directory and deprovision clusters with Terraform v0.11. ```shell terraform12 init diff --git a/google-cloud/container-linux/kubernetes/README.md b/google-cloud/container-linux/kubernetes/README.md index 8aa176fa4..2c996e92c 100644 --- a/google-cloud/container-linux/kubernetes/README.md +++ b/google-cloud/container-linux/kubernetes/README.md @@ -11,7 +11,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features -* Kubernetes v1.15.0 (upstream, via [kubernetes-incubator/bootkube](https://github.com/kubernetes-incubator/bootkube)) +* Kubernetes v1.15.1 (upstream, via [kubernetes-incubator/bootkube](https://github.com/kubernetes-incubator/bootkube)) * Single or multi-master, [Calico](https://www.projectcalico.org/) or [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/) * Advanced features like [worker pools](https://typhoon.psdn.io/advanced/worker-pools/), [preemptible](https://typhoon.psdn.io/cl/google-cloud/#preemption) workers, and [snippets](https://typhoon.psdn.io/advanced/customization/#container-linux) customization diff --git a/google-cloud/container-linux/kubernetes/bootkube.tf b/google-cloud/container-linux/kubernetes/bootkube.tf index 3d3347a89..eb519c2ce 100644 --- a/google-cloud/container-linux/kubernetes/bootkube.tf +++ b/google-cloud/container-linux/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=119cb00fa7b12e0ebd5a70c9c0a4e7eda2e8c3d6" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=5b9faa903182505ac534f42a2cf250d5d215941b" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/google-cloud/container-linux/kubernetes/cl/controller.yaml.tmpl b/google-cloud/container-linux/kubernetes/cl/controller.yaml.tmpl index 64ec62929..9bf32eb27 100644 --- a/google-cloud/container-linux/kubernetes/cl/controller.yaml.tmpl +++ b/google-cloud/container-linux/kubernetes/cl/controller.yaml.tmpl @@ -124,7 +124,7 @@ storage: contents: inline: | KUBELET_IMAGE_URL=docker://k8s.gcr.io/hyperkube - KUBELET_IMAGE_TAG=v1.15.0 + KUBELET_IMAGE_TAG=v1.15.1 - path: /etc/sysctl.d/max-user-watches.conf filesystem: root contents: diff --git a/google-cloud/container-linux/kubernetes/workers/cl/worker.yaml.tmpl b/google-cloud/container-linux/kubernetes/workers/cl/worker.yaml.tmpl index a34cca82f..4ce7668e8 100644 --- a/google-cloud/container-linux/kubernetes/workers/cl/worker.yaml.tmpl +++ b/google-cloud/container-linux/kubernetes/workers/cl/worker.yaml.tmpl @@ -94,7 +94,7 @@ storage: contents: inline: | KUBELET_IMAGE_URL=docker://k8s.gcr.io/hyperkube - KUBELET_IMAGE_TAG=v1.15.0 + KUBELET_IMAGE_TAG=v1.15.1 - path: /etc/sysctl.d/max-user-watches.conf filesystem: root contents: @@ -112,7 +112,7 @@ storage: --volume config,kind=host,source=/etc/kubernetes \ --mount volume=config,target=/etc/kubernetes \ --insecure-options=image \ - docker://k8s.gcr.io/hyperkube:v1.15.0 \ + docker://k8s.gcr.io/hyperkube:v1.15.1 \ --net=host \ --dns=host \ --exec=/kubectl -- --kubeconfig=/etc/kubernetes/kubeconfig delete node $(hostname) From 56d0b9eae4ac0ee15a2aead14e5d39bb89e12dad Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Sat, 20 Jul 2019 16:51:29 -0700 Subject: [PATCH 169/523] Avoid creating extraneous GCE controller instance groups * Intended as part of #504 improvement * Single controller clusters only require one controller instance group (previously created zone-many) * Multi-controller clusters must "wrap" controllers over zonal heterogeneous instance groups. For example, 5 controllers over 3 zones (no change) --- google-cloud/container-linux/kubernetes/apiserver.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/google-cloud/container-linux/kubernetes/apiserver.tf b/google-cloud/container-linux/kubernetes/apiserver.tf index b06ecf704..318d9c1fb 100644 --- a/google-cloud/container-linux/kubernetes/apiserver.tf +++ b/google-cloud/container-linux/kubernetes/apiserver.tf @@ -57,7 +57,7 @@ resource "google_compute_backend_service" "apiserver" { # Instance group of heterogeneous (unmanged) controller instances resource "google_compute_instance_group" "controllers" { - count = length(local.zones) + count = min(var.controller_count, length(local.zones)) name = format("%s-controllers-%s", var.cluster_name, element(local.zones, count.index)) zone = element(local.zones, count.index) From e0be091accf89fafa82348288f07daff2d4c4b09 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Sat, 20 Jul 2019 20:17:08 -0700 Subject: [PATCH 170/523] Update kube-state-metrics from v1.7.0 to v1.7.1 * https://github.com/kubernetes/kube-state-metrics/releases/tag/v1.7.1 --- CHANGES.md | 2 +- addons/prometheus/exporters/kube-state-metrics/deployment.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index b4362f2c8..b37f153f1 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -20,7 +20,7 @@ Notable changes between versions. #### Addons * Update Prometheus from v2.10.0 to v2.11.0 - * Update kube-state-metrics from v1.6.0 to v1.7.0 + * Update kube-state-metrics from v1.6.0 to v1.7.1 * Update Grafana from v6.2.4 to v6.2.5 * Update nginx-ingress from v0.24.1 to v0.25.0 * Support `networking.k8s.io/v1beta1` apiVersion diff --git a/addons/prometheus/exporters/kube-state-metrics/deployment.yaml b/addons/prometheus/exporters/kube-state-metrics/deployment.yaml index 829fd2ffb..52c654454 100644 --- a/addons/prometheus/exporters/kube-state-metrics/deployment.yaml +++ b/addons/prometheus/exporters/kube-state-metrics/deployment.yaml @@ -24,7 +24,7 @@ spec: serviceAccountName: kube-state-metrics containers: - name: kube-state-metrics - image: quay.io/coreos/kube-state-metrics:v1.7.0 + image: quay.io/coreos/kube-state-metrics:v1.7.1 ports: - name: metrics containerPort: 8080 From f543f08867aa9b2a1e5acd6162d9dd43dbded56c Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Sat, 20 Jul 2019 20:31:06 -0700 Subject: [PATCH 171/523] Compact nginx-ingress ClusterRole rules * https://github.com/kubernetes/ingress-nginx/pull/4302 --- addons/nginx-ingress/aws/rbac/cluster-role.yaml | 14 +------------- addons/nginx-ingress/azure/rbac/cluster-role.yaml | 14 +------------- .../bare-metal/rbac/cluster-role.yaml | 14 +------------- .../digital-ocean/rbac/cluster-role.yaml | 14 +------------- .../google-cloud/rbac/cluster-role.yaml | 14 +------------- 5 files changed, 5 insertions(+), 65 deletions(-) diff --git a/addons/nginx-ingress/aws/rbac/cluster-role.yaml b/addons/nginx-ingress/aws/rbac/cluster-role.yaml index bf680e19e..5682d3974 100644 --- a/addons/nginx-ingress/aws/rbac/cluster-role.yaml +++ b/addons/nginx-ingress/aws/rbac/cluster-role.yaml @@ -37,19 +37,6 @@ rules: - patch - apiGroups: - "extensions" - resources: - - ingresses - verbs: - - get - - list - - watch - - apiGroups: - - "extensions" - resources: - - ingresses/status - verbs: - - update - - apiGroups: - "networking.k8s.io" resources: - ingresses @@ -58,6 +45,7 @@ rules: - list - watch - apiGroups: + - "extensions" - "networking.k8s.io" resources: - ingresses/status diff --git a/addons/nginx-ingress/azure/rbac/cluster-role.yaml b/addons/nginx-ingress/azure/rbac/cluster-role.yaml index bf680e19e..5682d3974 100644 --- a/addons/nginx-ingress/azure/rbac/cluster-role.yaml +++ b/addons/nginx-ingress/azure/rbac/cluster-role.yaml @@ -37,19 +37,6 @@ rules: - patch - apiGroups: - "extensions" - resources: - - ingresses - verbs: - - get - - list - - watch - - apiGroups: - - "extensions" - resources: - - ingresses/status - verbs: - - update - - apiGroups: - "networking.k8s.io" resources: - ingresses @@ -58,6 +45,7 @@ rules: - list - watch - apiGroups: + - "extensions" - "networking.k8s.io" resources: - ingresses/status diff --git a/addons/nginx-ingress/bare-metal/rbac/cluster-role.yaml b/addons/nginx-ingress/bare-metal/rbac/cluster-role.yaml index bf680e19e..5682d3974 100644 --- a/addons/nginx-ingress/bare-metal/rbac/cluster-role.yaml +++ b/addons/nginx-ingress/bare-metal/rbac/cluster-role.yaml @@ -37,19 +37,6 @@ rules: - patch - apiGroups: - "extensions" - resources: - - ingresses - verbs: - - get - - list - - watch - - apiGroups: - - "extensions" - resources: - - ingresses/status - verbs: - - update - - apiGroups: - "networking.k8s.io" resources: - ingresses @@ -58,6 +45,7 @@ rules: - list - watch - apiGroups: + - "extensions" - "networking.k8s.io" resources: - ingresses/status diff --git a/addons/nginx-ingress/digital-ocean/rbac/cluster-role.yaml b/addons/nginx-ingress/digital-ocean/rbac/cluster-role.yaml index bf680e19e..5682d3974 100644 --- a/addons/nginx-ingress/digital-ocean/rbac/cluster-role.yaml +++ b/addons/nginx-ingress/digital-ocean/rbac/cluster-role.yaml @@ -37,19 +37,6 @@ rules: - patch - apiGroups: - "extensions" - resources: - - ingresses - verbs: - - get - - list - - watch - - apiGroups: - - "extensions" - resources: - - ingresses/status - verbs: - - update - - apiGroups: - "networking.k8s.io" resources: - ingresses @@ -58,6 +45,7 @@ rules: - list - watch - apiGroups: + - "extensions" - "networking.k8s.io" resources: - ingresses/status diff --git a/addons/nginx-ingress/google-cloud/rbac/cluster-role.yaml b/addons/nginx-ingress/google-cloud/rbac/cluster-role.yaml index bf680e19e..5682d3974 100644 --- a/addons/nginx-ingress/google-cloud/rbac/cluster-role.yaml +++ b/addons/nginx-ingress/google-cloud/rbac/cluster-role.yaml @@ -37,19 +37,6 @@ rules: - patch - apiGroups: - "extensions" - resources: - - ingresses - verbs: - - get - - list - - watch - - apiGroups: - - "extensions" - resources: - - ingresses/status - verbs: - - update - - apiGroups: - "networking.k8s.io" resources: - ingresses @@ -58,6 +45,7 @@ rules: - list - watch - apiGroups: + - "extensions" - "networking.k8s.io" resources: - ingresses/status From c8df349e5554e794fbb108359e7087d89d28a3ce Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Sun, 21 Jul 2019 10:32:58 -0700 Subject: [PATCH 172/523] Fix to add all Azure controller nodes to address pool * Add all Azure controllers to the apiserver load balancer backend address pool * Previously, kube-apiserver availability relied on the 0th controller being up. Multi-controller was just providing etcd data redundancy --- CHANGES.md | 5 +++++ azure/container-linux/kubernetes/controllers.tf | 4 +++- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index b37f153f1..f0bef47fa 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -11,6 +11,11 @@ Notable changes between versions. * Relax `terraform-provider-ct` version constraint (v0.3.2+) * Allow provider versions below v1.0.0 (e.g. upgrading to v0.4) +#### Azure + +* Fix to add all controller nodes to the apiserver load balancer backend address pool ([#518](https://github.com/poseidon/typhoon/pull/518)) + * kube-apiserver availability relied on the 0th controller + #### Google Cloud * Allow controller nodes to span more than 3 zones if available in a region ([#504](https://github.com/poseidon/typhoon/pull/504)) diff --git a/azure/container-linux/kubernetes/controllers.tf b/azure/container-linux/kubernetes/controllers.tf index 2739eb6ea..c57f42da3 100644 --- a/azure/container-linux/kubernetes/controllers.tf +++ b/azure/container-linux/kubernetes/controllers.tf @@ -114,7 +114,9 @@ resource "azurerm_network_interface" "controllers" { # Add controller NICs to the controller backend address pool resource "azurerm_network_interface_backend_address_pool_association" "controllers" { - network_interface_id = azurerm_network_interface.controllers[0].id + count = var.controller_count + + network_interface_id = azurerm_network_interface.controllers[count.index].id ip_configuration_name = "ip0" backend_address_pool_id = azurerm_lb_backend_address_pool.controller.id } From 68d8717924a16d8ea6306b1ef9a401fdb0020cd6 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Sun, 21 Jul 2019 11:02:18 -0700 Subject: [PATCH 173/523] Refresh Prometheus rules/alerts and Grafana dashboards * Refresh rules, alerts, and dashboards from upstreams --- CHANGES.md | 1 + addons/grafana/dashboards-etcd.yaml | 2 +- addons/grafana/dashboards-k8s-nodes.yaml | 5005 ++++++++++++++++++ addons/grafana/dashboards-k8s-resources.yaml | 2035 ++++++- addons/grafana/dashboards-k8s.yaml | 3309 ++++++------ addons/grafana/dashboards-prom.yaml | 2160 ++++++++ addons/grafana/deployment.yaml | 10 + addons/prometheus/rules.yaml | 263 +- 8 files changed, 11050 insertions(+), 1735 deletions(-) create mode 100644 addons/grafana/dashboards-k8s-nodes.yaml create mode 100644 addons/grafana/dashboards-prom.yaml diff --git a/CHANGES.md b/CHANGES.md index f0bef47fa..99d35df18 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -25,6 +25,7 @@ Notable changes between versions. #### Addons * Update Prometheus from v2.10.0 to v2.11.0 + * Refresh rules, alerts, and dashboards from upstreams * Update kube-state-metrics from v1.6.0 to v1.7.1 * Update Grafana from v6.2.4 to v6.2.5 * Update nginx-ingress from v0.24.1 to v0.25.0 diff --git a/addons/grafana/dashboards-etcd.yaml b/addons/grafana/dashboards-etcd.yaml index 21f9880ff..94bcaf06d 100644 --- a/addons/grafana/dashboards-etcd.yaml +++ b/addons/grafana/dashboards-etcd.yaml @@ -19,7 +19,7 @@ data: "links": [ ], - "refresh": false, + "refresh": "10s", "rows": [ { "collapse": false, diff --git a/addons/grafana/dashboards-k8s-nodes.yaml b/addons/grafana/dashboards-k8s-nodes.yaml new file mode 100644 index 000000000..a0d7963cd --- /dev/null +++ b/addons/grafana/dashboards-k8s-nodes.yaml @@ -0,0 +1,5005 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: grafana-dashboards-k8s-nodes + namespace: monitoring +data: + kubelet.json: |- + { + "__inputs": [ + + ], + "__requires": [ + + ], + "annotations": { + "list": [ + + ] + }, + "editable": false, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "id": null, + "links": [ + + ], + "refresh": "", + "rows": [ + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "$datasource", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + + }, + "id": 2, + "interval": null, + "links": [ + + ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "span": 2, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum(up{job=\"kubelet\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": "", + "title": "Up", + "tooltip": { + "shared": false + }, + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "min" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "$datasource", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + + }, + "id": 3, + "interval": null, + "links": [ + + ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "span": 2, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum(kubelet_running_pod_count{job=\"kubelet\", instance=~\"$instance\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{instance}}", + "refId": "A" + } + ], + "thresholds": "", + "title": "Running Pods", + "tooltip": { + "shared": false + }, + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "min" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "$datasource", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + + }, + "id": 4, + "interval": null, + "links": [ + + ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "span": 2, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum(kubelet_running_container_count{job=\"kubelet\", instance=~\"$instance\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{instance}}", + "refId": "A" + } + ], + "thresholds": "", + "title": "Running Container", + "tooltip": { + "shared": false + }, + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "min" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "$datasource", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + + }, + "id": 5, + "interval": null, + "links": [ + + ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "span": 2, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum(volume_manager_total_volumes{job=\"kubelet\", instance=~\"$instance\", state=\"actual_state_of_world\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{instance}}", + "refId": "A" + } + ], + "thresholds": "", + "title": "Actual Volume Count", + "tooltip": { + "shared": false + }, + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "min" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "$datasource", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + + }, + "id": 6, + "interval": null, + "links": [ + + ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "span": 2, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum(volume_manager_total_volumes{job=\"kubelet\", instance=~\"$instance\",state=\"desired_state_of_world\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{instance}}", + "refId": "A" + } + ], + "thresholds": "", + "title": "Desired Volume Count", + "tooltip": { + "shared": false + }, + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "min" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "$datasource", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + + }, + "id": 7, + "interval": null, + "links": [ + + ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "span": 2, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum(rate(kubelet_node_config_error{job=\"kubelet\", instance=~\"$instance\"}[5m]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{instance}}", + "refId": "A" + } + ], + "thresholds": "", + "title": "Config Error Count", + "tooltip": { + "shared": false + }, + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "min" + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "gridPos": { + + }, + "id": 8, + "legend": { + "alignAsTable": "true", + "avg": false, + "current": "true", + "max": false, + "min": false, + "rightSide": "true", + "show": "true", + "total": false, + "values": "true" + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(kubelet_runtime_operations_total{job=\"kubelet\",instance=~\"$instance\"}[5m])) by (operation_type, instance)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{instance}} {{operation_type}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Operation Rate", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "gridPos": { + + }, + "id": 9, + "legend": { + "alignAsTable": "true", + "avg": false, + "current": "true", + "max": false, + "min": false, + "rightSide": "true", + "show": "true", + "total": false, + "values": "true" + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(kubelet_runtime_operations_errors_total{job=\"kubelet\",instance=~\"$instance\"}[5m])) by (instance, operation_type)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{instance}} {{operation_type}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Operation Error Rate", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "gridPos": { + + }, + "id": 10, + "legend": { + "alignAsTable": "true", + "avg": false, + "current": "true", + "max": false, + "min": false, + "rightSide": "true", + "show": "true", + "total": false, + "values": "true" + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(kubelet_runtime_operations_duration_seconds_bucket{job=\"kubelet\",instance=~\"$instance\"}[5m])) by (instance, operation_type, le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{instance}} {{operation_type}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Operation duration 99th quantile", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "gridPos": { + + }, + "id": 11, + "legend": { + "alignAsTable": "true", + "avg": false, + "current": "true", + "max": false, + "min": false, + "rightSide": "true", + "show": "true", + "total": false, + "values": "true" + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(kubelet_pod_start_duration_seconds_count{job=\"kubelet\",instance=~\"$instance\"}[5m])) by (instance)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{instance}} pod", + "refId": "A" + }, + { + "expr": "sum(rate(kubelet_pod_worker_duration_seconds_count{job=\"kubelet\",instance=~\"$instance\"}[5m])) by (instance)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{instance}} worker", + "refId": "B" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Pod Start Rate", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "gridPos": { + + }, + "id": 12, + "legend": { + "alignAsTable": "true", + "avg": false, + "current": "true", + "max": false, + "min": false, + "rightSide": "true", + "show": "true", + "total": false, + "values": "true" + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(kubelet_pod_start_duration_seconds_count{job=\"kubelet\",instance=~\"$instance\"}[5m])) by (instance, le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{instance}} pod", + "refId": "A" + }, + { + "expr": "histogram_quantile(0.99, sum(rate(kubelet_pod_worker_duration_seconds_bucket{job=\"kubelet\",instance=~\"$instance\"}[5m])) by (instance, le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{instance}} worker", + "refId": "B" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Pod Start Duration", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "gridPos": { + + }, + "id": 13, + "legend": { + "alignAsTable": "true", + "avg": false, + "current": "true", + "hideEmpty": "true", + "hideZero": "true", + "max": false, + "min": false, + "rightSide": "true", + "show": "true", + "total": false, + "values": "true" + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(storage_operation_duration_seconds_count{job=\"kubelet\",instance=~\"$instance\"}[5m])) by (instance, operation_name, volume_plugin)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{instance}} {{operation_name}} {{volume_plugin}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Storage Operation Rate", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "gridPos": { + + }, + "id": 14, + "legend": { + "alignAsTable": "true", + "avg": false, + "current": "true", + "hideEmpty": "true", + "hideZero": "true", + "max": false, + "min": false, + "rightSide": "true", + "show": "true", + "total": false, + "values": "true" + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(storage_operation_errors_total{job=\"kubelet\",instance=~\"$instance\"}[5m])) by (instance, operation_name, volume_plugin)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{instance}} {{operation_name}} {{volume_plugin}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Storage Operation Error Rate", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "gridPos": { + + }, + "id": 15, + "legend": { + "alignAsTable": "true", + "avg": false, + "current": "true", + "hideEmpty": "true", + "hideZero": "true", + "max": false, + "min": false, + "rightSide": "true", + "show": true, + "total": false, + "values": "true" + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(storage_operation_duration_seconds_bucket{job=\"kubelet\", instance=~\"$instance\"}[5m])) by (instance, operation_name, volume_plugin, le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{instance}} {{operation_name}} {{volume_plugin}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Storage Operation Duration 99th quantile", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "gridPos": { + + }, + "id": 16, + "legend": { + "alignAsTable": "true", + "avg": false, + "current": "true", + "max": false, + "min": false, + "rightSide": "true", + "show": "true", + "total": false, + "values": "true" + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(kubelet_cgroup_manager_duration_seconds_count{job=\"kubelet\", instance=~\"$instance\"}[5m])) by (instance, operation_type)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{operation_type}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Cgroup manager operation rate", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "gridPos": { + + }, + "id": 17, + "legend": { + "alignAsTable": "true", + "avg": false, + "current": "true", + "max": false, + "min": false, + "rightSide": "true", + "show": "true", + "total": false, + "values": "true" + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(kubelet_cgroup_manager_duration_seconds_bucket{job=\"kubelet\", instance=~\"$instance\"}[5m])) by (instance, operation_type, le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{instance}} {{operation_type}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Cgroup manager 99th quantile", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "Pod lifecycle event generator", + "fill": 1, + "gridPos": { + + }, + "id": 18, + "legend": { + "alignAsTable": "true", + "avg": false, + "current": "true", + "max": false, + "min": false, + "rightSide": "true", + "show": "true", + "total": false, + "values": "true" + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(kubelet_pleg_relist_duration_seconds_count{job=\"kubelet\", instance=~\"$instance\"}[5m])) by (instance)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{instance}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "PLEG relist rate", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "gridPos": { + + }, + "id": 19, + "legend": { + "alignAsTable": "true", + "avg": false, + "current": "true", + "max": false, + "min": false, + "rightSide": "true", + "show": "true", + "total": false, + "values": "true" + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(kubelet_pleg_relist_interval_seconds_bucket{job=\"kubelet\",instance=~\"$instance\"}[5m])) by (instance, le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{instance}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "PLEG relist interval", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "gridPos": { + + }, + "id": 20, + "legend": { + "alignAsTable": "true", + "avg": false, + "current": "true", + "max": false, + "min": false, + "rightSide": "true", + "show": "true", + "total": false, + "values": "true" + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(kubelet_pleg_relist_duration_seconds_bucket{job=\"kubelet\",instance=~\"$instance\"}[5m])) by (instance, le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{instance}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "PLEG relist duration", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "gridPos": { + + }, + "id": 21, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(rest_client_requests_total{job=\"kubelet\", instance=~\"$instance\",code=~\"2..\"}[5m]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "2xx", + "refId": "A" + }, + { + "expr": "sum(rate(rest_client_requests_total{job=\"kubelet\", instance=~\"$instance\",code=~\"3..\"}[5m]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "3xx", + "refId": "B" + }, + { + "expr": "sum(rate(rest_client_requests_total{job=\"kubelet\", instance=~\"$instance\",code=~\"4..\"}[5m]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "4xx", + "refId": "C" + }, + { + "expr": "sum(rate(rest_client_requests_total{job=\"kubelet\", instance=~\"$instance\",code=~\"5..\"}[5m]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "5xx", + "refId": "D" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "RPC Rate", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "gridPos": { + + }, + "id": 22, + "legend": { + "alignAsTable": "true", + "avg": false, + "current": "true", + "max": false, + "min": false, + "rightSide": "true", + "show": "true", + "total": false, + "values": "true" + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(rest_client_request_latency_seconds_bucket{job=\"kubelet\", instance=~\"$instance\"}[5m])) by (instance, verb, url, le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{instance}} {{verb}} {{url}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Request duration 99th quantile", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "gridPos": { + + }, + "id": 23, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "process_resident_memory_bytes{job=\"kubelet\",instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{instance}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Memory", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "gridPos": { + + }, + "id": 24, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(process_cpu_seconds_total{job=\"kubelet\",instance=~\"$instance\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{instance}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU usage", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "gridPos": { + + }, + "id": 25, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "go_goroutines{job=\"kubelet\",instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{instance}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Goroutines", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + } + ], + "schemaVersion": 14, + "style": "dark", + "tags": [ + "kubernetes-mixin" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": null, + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": true, + "label": null, + "multi": false, + "name": "instance", + "options": [ + + ], + "query": "label_values(kubelet_runtime_operations{job=\"kubelet\"}, instance)", + "refresh": 2, + "regex": "", + "sort": 0, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "", + "title": "Kubernetes / Kubelet", + "uid": "3138fa155d5915769fbded898ac09fd9", + "version": 0 + } + nodes.json: |- + { + "__inputs": [ + + ], + "__requires": [ + + ], + "annotations": { + "list": [ + + ] + }, + "editable": false, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "id": null, + "links": [ + + ], + "refresh": "", + "rows": [ + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "gridPos": { + + }, + "id": 2, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "max(node_load1{cluster=\"$cluster\", job=\"node-exporter\", instance=\"$instance\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "load 1m", + "refId": "A" + }, + { + "expr": "max(node_load5{cluster=\"$cluster\", job=\"node-exporter\", instance=\"$instance\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "load 5m", + "refId": "B" + }, + { + "expr": "max(node_load15{cluster=\"$cluster\", job=\"node-exporter\", instance=\"$instance\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "load 15m", + "refId": "C" + }, + { + "expr": "count(node_cpu_seconds_total{cluster=\"$cluster\", job=\"node-exporter\", instance=\"$instance\", mode=\"user\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "logical cores", + "refId": "D" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "System load", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "gridPos": { + + }, + "id": 3, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum by (cpu) (irate(node_cpu_seconds_total{cluster=\"$cluster\", job=\"node-exporter\", mode!=\"idle\", instance=\"$instance\"}[5m]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{cpu}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Usage Per Core", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "gridPos": { + + }, + "id": 4, + "legend": { + "alignAsTable": "true", + "avg": "true", + "current": "true", + "max": "false", + "min": "false", + "rightSide": "true", + "show": "true", + "total": "false", + "values": "true" + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 9, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "max (sum by (cpu) (irate(node_cpu_seconds_total{cluster=\"$cluster\", job=\"node-exporter\", mode!=\"idle\", instance=\"$instance\"}[2m])) ) * 100\n", + "format": "time_series", + "intervalFactor": 10, + "legendFormat": "{{ cpu }}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU Utilization", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "percent", + "label": null, + "logBase": 1, + "max": 100, + "min": 0, + "show": true + }, + { + "format": "percent", + "label": null, + "logBase": 1, + "max": 100, + "min": 0, + "show": true + } + ] + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "datasource": "$datasource", + "format": "percent", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": true, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + + }, + "id": 5, + "interval": null, + "links": [ + + ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "span": 3, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "avg(sum by (cpu) (irate(node_cpu_seconds_total{cluster=\"$cluster\", job=\"node-exporter\", mode!=\"idle\", instance=\"$instance\"}[2m]))) * 100\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": "80, 90", + "title": "CPU Usage", + "tooltip": { + "shared": false + }, + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "gridPos": { + + }, + "id": 6, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 9, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "max(\n node_memory_MemTotal_bytes{cluster=\"$cluster\", job=\"node-exporter\", instance=\"$instance\"}\n - node_memory_MemFree_bytes{cluster=\"$cluster\", job=\"node-exporter\", instance=\"$instance\"}\n - node_memory_Buffers_bytes{cluster=\"$cluster\", job=\"node-exporter\", instance=\"$instance\"}\n - node_memory_Cached_bytes{cluster=\"$cluster\", job=\"node-exporter\", instance=\"$instance\"}\n)\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "memory used", + "refId": "A" + }, + { + "expr": "max(node_memory_Buffers_bytes{cluster=\"$cluster\", job=\"node-exporter\", instance=\"$instance\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "memory buffers", + "refId": "B" + }, + { + "expr": "max(node_memory_Cached_bytes{cluster=\"$cluster\", job=\"node-exporter\", instance=\"$instance\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "memory cached", + "refId": "C" + }, + { + "expr": "max(node_memory_MemFree_bytes{cluster=\"$cluster\", job=\"node-exporter\", instance=\"$instance\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "memory free", + "refId": "D" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Memory Usage", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "datasource": "$datasource", + "format": "percent", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": true, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + + }, + "id": 7, + "interval": null, + "links": [ + + ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "span": 3, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "max(\n (\n (\n node_memory_MemTotal_bytes{cluster=\"$cluster\", job=\"node-exporter\", instance=\"$instance\"}\n - node_memory_MemFree_bytes{cluster=\"$cluster\", job=\"node-exporter\", instance=\"$instance\"}\n - node_memory_Buffers_bytes{cluster=\"$cluster\", job=\"node-exporter\", instance=\"$instance\"}\n - node_memory_Cached_bytes{cluster=\"$cluster\", job=\"node-exporter\", instance=\"$instance\"}\n )\n / node_memory_MemTotal_bytes{cluster=\"$cluster\", job=\"node-exporter\", instance=\"$instance\"}\n ) * 100)\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": "80, 90", + "title": "Memory Usage", + "tooltip": { + "shared": false + }, + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "gridPos": { + + }, + "id": 8, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + { + "alias": "read", + "yaxis": 1 + }, + { + "alias": "io time", + "yaxis": 2 + } + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "max(rate(node_disk_read_bytes_total{cluster=\"$cluster\", job=\"node-exporter\", instance=\"$instance\"}[2m]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "read", + "refId": "A" + }, + { + "expr": "max(rate(node_disk_written_bytes_total{cluster=\"$cluster\", job=\"node-exporter\", instance=\"$instance\"}[2m]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "written", + "refId": "B" + }, + { + "expr": "max(rate(node_disk_io_time_seconds_total{cluster=\"$cluster\", job=\"node-exporter\", instance=\"$instance\"}[2m]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "io time", + "refId": "C" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Disk I/O", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "ms", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "gridPos": { + + }, + "id": 9, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "node:node_filesystem_usage:{cluster=\"$cluster\", instance=\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{device}} disk used", + "refId": "A" + }, + { + "expr": "node:node_filesystem_usage:{cluster=\"$cluster\", instance=\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{device}} disk free", + "refId": "B" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Disk Space Usage", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "gridPos": { + + }, + "id": 10, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "rate(node_network_receive_bytes_total{cluster=\"$cluster\", job=\"node-exporter\", instance=\"$instance\", device!~\"lo\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{device}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Network Received", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "gridPos": { + + }, + "id": 11, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "rate(node_network_transmit_bytes_total{cluster=\"$cluster\", job=\"node-exporter\", instance=\"$instance\", device!~\"lo\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{device}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Network Transmitted", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "gridPos": { + + }, + "id": 12, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 9, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "max(\n node_filesystem_files{cluster=\"$cluster\", job=\"node-exporter\", instance=\"$instance\"}\n - node_filesystem_files_free{cluster=\"$cluster\", job=\"node-exporter\", instance=\"$instance\"}\n)\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "inodes used", + "refId": "A" + }, + { + "expr": "max(node_filesystem_files_free{cluster=\"$cluster\", job=\"node-exporter\", instance=\"$instance\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "inodes free", + "refId": "B" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Inodes Usage", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "datasource": "$datasource", + "format": "percent", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": true, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + + }, + "id": 13, + "interval": null, + "links": [ + + ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "span": 3, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "max(\n (\n (\n node_filesystem_files{cluster=\"$cluster\", job=\"node-exporter\", instance=\"$instance\"}\n - node_filesystem_files_free{cluster=\"$cluster\", job=\"node-exporter\", instance=\"$instance\"}\n )\n / node_filesystem_files{cluster=\"$cluster\", job=\"node-exporter\", instance=\"$instance\"}\n ) * 100)\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": "80, 90", + "title": "Inodes Usage", + "tooltip": { + "shared": false + }, + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + } + ], + "schemaVersion": 14, + "style": "dark", + "tags": [ + "kubernetes-mixin" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": null, + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + + }, + "datasource": "$datasource", + "hide": 2, + "includeAll": false, + "label": "cluster", + "multi": false, + "name": "cluster", + "options": [ + + ], + "query": "label_values(kube_pod_info, cluster)", + "refresh": 2, + "regex": "", + "sort": 0, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "instance", + "options": [ + + ], + "query": "label_values(node_boot_time_seconds{cluster=\"$cluster\", job=\"node-exporter\"}, instance)", + "refresh": 2, + "regex": "", + "sort": 0, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "", + "title": "Kubernetes / Nodes", + "uid": "fa49a4706d07a042595b664c87fb33ea", + "version": 0 + } + proxy.json: |- + { + "__inputs": [ + + ], + "__requires": [ + + ], + "annotations": { + "list": [ + + ] + }, + "editable": false, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "id": null, + "links": [ + + ], + "refresh": "", + "rows": [ + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "$datasource", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + + }, + "id": 2, + "interval": null, + "links": [ + + ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "span": 2, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum(up{job=\"kube-proxy\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": "", + "title": "Up", + "tooltip": { + "shared": false + }, + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "min" + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "gridPos": { + + }, + "id": 3, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 5, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(kubeproxy_sync_proxy_rules_duration_seconds_count{job=\"kube-proxy\", instance=~\"$instance\"}[5m]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "rate", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rules Sync Rate", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "gridPos": { + + }, + "id": 4, + "legend": { + "alignAsTable": "true", + "avg": false, + "current": "true", + "max": false, + "min": false, + "rightSide": "true", + "show": "true", + "total": false, + "values": "true" + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 5, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99,rate(kubeproxy_sync_proxy_rules_duration_seconds_bucket{job=\"kube-proxy\", instance=~\"$instance\"}[5m]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{instance}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rule Sync Latency 99th Quantile", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "gridPos": { + + }, + "id": 5, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(kubeproxy_network_programming_duration_seconds_count{job=\"kube-proxy\", instance=~\"$instance\"}[5m]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "rate", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Network Programming Rate", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "gridPos": { + + }, + "id": 6, + "legend": { + "alignAsTable": "true", + "avg": false, + "current": "true", + "max": false, + "min": false, + "rightSide": "true", + "show": "true", + "total": false, + "values": "true" + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(kubeproxy_network_programming_duration_seconds_bucket{job=\"kube-proxy\", instance=~\"$instance\"}[5m])) by (instance, le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{instance}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Network Programming Latency 99th Quantile", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "gridPos": { + + }, + "id": 7, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(rest_client_requests_total{job=\"kube-proxy\", instance=~\"$instance\",code=~\"2..\"}[5m]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "2xx", + "refId": "A" + }, + { + "expr": "sum(rate(rest_client_requests_total{job=\"kube-proxy\", instance=~\"$instance\",code=~\"3..\"}[5m]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "3xx", + "refId": "B" + }, + { + "expr": "sum(rate(rest_client_requests_total{job=\"kube-proxy\", instance=~\"$instance\",code=~\"4..\"}[5m]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "4xx", + "refId": "C" + }, + { + "expr": "sum(rate(rest_client_requests_total{job=\"kube-proxy\", instance=~\"$instance\",code=~\"5..\"}[5m]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "5xx", + "refId": "D" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Kube API Request Rate", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "gridPos": { + + }, + "id": 8, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 8, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(rest_client_request_latency_seconds_bucket{job=\"kube-proxy\",instance=~\"$instance\",verb=\"POST\"}[5m])) by (verb, url, le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{verb}} {{url}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Post Request Latency 99th Quantile", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "gridPos": { + + }, + "id": 9, + "legend": { + "alignAsTable": "true", + "avg": false, + "current": "true", + "max": false, + "min": false, + "rightSide": "true", + "show": "true", + "total": false, + "values": "true" + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(rest_client_request_latency_seconds_bucket{job=\"kube-proxy\", instance=~\"$instance\", verb=\"GET\"}[5m])) by (verb, url, le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{verb}} {{url}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Get Request Latency 99th Quantile", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "gridPos": { + + }, + "id": 10, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "process_resident_memory_bytes{job=\"kube-proxy\",instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{instance}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Memory", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "gridPos": { + + }, + "id": 11, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(process_cpu_seconds_total{job=\"kube-proxy\",instance=~\"$instance\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{instance}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU usage", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "gridPos": { + + }, + "id": 12, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "go_goroutines{job=\"kube-proxy\",instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{instance}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Goroutines", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + } + ], + "schemaVersion": 14, + "style": "dark", + "tags": [ + "kubernetes-mixin" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": null, + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": true, + "label": null, + "multi": false, + "name": "instance", + "options": [ + + ], + "query": "label_values(kubeproxy_network_programming_duration_seconds_bucket{job=\"kube-proxy\"}, instance)", + "refresh": 2, + "regex": "", + "sort": 0, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "", + "title": "Kubernetes / Proxy", + "uid": "632e265de029684c40b21cb76bca4f94", + "version": 0 + } diff --git a/addons/grafana/dashboards-k8s-resources.yaml b/addons/grafana/dashboards-k8s-resources.yaml index f00af428b..98a17712c 100644 --- a/addons/grafana/dashboards-k8s-resources.yaml +++ b/addons/grafana/dashboards-k8s-resources.yaml @@ -4,6 +4,1907 @@ metadata: name: grafana-dashboards-k8s-resources namespace: monitoring data: + k8s-cluster-rsrc-use.json: |- + { + "annotations": { + "list": [ + + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "links": [ + + ], + "refresh": "10s", + "rows": [ + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 1, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "node:cluster_cpu_utilisation:ratio{cluster=\"$cluster\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{node}}", + "legendLink": "/d/4ac4f123aae0ff6dbaf4f4f66120033b/k8s-node-rsrc-use", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU Utilisation", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": 1, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 2, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "node:node_cpu_saturation_load1:{cluster=\"$cluster\"} / scalar(sum(min(kube_pod_info{cluster=\"$cluster\"}) by (node)))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{node}}", + "legendLink": "/d/4ac4f123aae0ff6dbaf4f4f66120033b/k8s-node-rsrc-use", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU Saturation (Load1)", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": 1, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "CPU", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 3, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "node:cluster_memory_utilisation:ratio{cluster=\"$cluster\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{node}}", + "legendLink": "/d/4ac4f123aae0ff6dbaf4f4f66120033b/k8s-node-rsrc-use", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Memory Utilisation", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": 1, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 4, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "node:node_memory_swap_io_bytes:sum_rate{cluster=\"$cluster\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{node}}", + "legendLink": "/d/4ac4f123aae0ff6dbaf4f4f66120033b/k8s-node-rsrc-use", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Memory Saturation (Swap I/O)", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Memory", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 5, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "node:node_disk_utilisation:avg_irate{cluster=\"$cluster\"} / scalar(:kube_pod_info_node_count:{cluster=\"$cluster\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{node}}", + "legendLink": "/d/4ac4f123aae0ff6dbaf4f4f66120033b/k8s-node-rsrc-use", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Disk IO Utilisation", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": 1, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 6, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "node:node_disk_saturation:avg_irate{cluster=\"$cluster\"} / scalar(:kube_pod_info_node_count:{cluster=\"$cluster\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{node}}", + "legendLink": "/d/4ac4f123aae0ff6dbaf4f4f66120033b/k8s-node-rsrc-use", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Disk IO Saturation", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": 1, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Disk", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 7, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "node:node_net_utilisation:sum_irate{cluster=\"$cluster\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{node}}", + "legendLink": "/d/4ac4f123aae0ff6dbaf4f4f66120033b/k8s-node-rsrc-use", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Net Utilisation (Transmitted)", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 8, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "node:node_net_saturation:sum_irate{cluster=\"$cluster\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{node}}", + "legendLink": "/d/4ac4f123aae0ff6dbaf4f4f66120033b/k8s-node-rsrc-use", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Net Saturation (Dropped)", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Network", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 9, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(max(node_filesystem_size_bytes{fstype=~\"ext[234]|btrfs|xfs|zfs\", cluster=\"$cluster\"} - node_filesystem_avail_bytes{fstype=~\"ext[234]|btrfs|xfs|zfs\", cluster=\"$cluster\"}) by (device,pod,namespace)) by (pod,namespace)\n/ scalar(sum(max(node_filesystem_size_bytes{fstype=~\"ext[234]|btrfs|xfs|zfs\", cluster=\"$cluster\"}) by (device,pod,namespace)))\n* on (namespace, pod) group_left (node) node_namespace_pod:kube_pod_info:{cluster=\"$cluster\"}\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{node}}", + "legendLink": "/d/4ac4f123aae0ff6dbaf4f4f66120033b/k8s-node-rsrc-use", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Disk Capacity", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": 1, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Storage", + "titleSize": "h6" + } + ], + "schemaVersion": 14, + "style": "dark", + "tags": [ + "kubernetes-mixin" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": null, + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + "text": "prod", + "value": "prod" + }, + "datasource": "$datasource", + "hide": 2, + "includeAll": false, + "label": "cluster", + "multi": false, + "name": "cluster", + "options": [ + + ], + "query": "label_values(kube_node_info, cluster)", + "refresh": 1, + "regex": "", + "sort": 2, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "", + "title": "Kubernetes / USE Method / Cluster", + "uid": "a6e7d1362e1ddbb79db21d5bb40d7137", + "version": 0 + } + k8s-node-rsrc-use.json: |- + { + "annotations": { + "list": [ + + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "links": [ + + ], + "refresh": "10s", + "rows": [ + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 1, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "node:node_cpu_utilisation:avg1m{cluster=\"$cluster\", node=\"$node\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Utilisation", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU Utilisation", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 2, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "node:node_cpu_saturation_load1:{cluster=\"$cluster\", node=\"$node\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Saturation", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU Saturation (Load1)", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "CPU", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 3, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "node:node_memory_utilisation:{cluster=\"$cluster\", node=\"$node\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Memory", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Memory Utilisation", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 4, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "node:node_memory_swap_io_bytes:sum_rate{cluster=\"$cluster\", node=\"$node\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Swap IO", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Memory Saturation (Swap I/O)", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Memory", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 5, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "node:node_disk_utilisation:avg_irate{cluster=\"$cluster\", node=\"$node\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Utilisation", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Disk IO Utilisation", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 6, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "node:node_disk_saturation:avg_irate{cluster=\"$cluster\", node=\"$node\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Saturation", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Disk IO Saturation", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Disk", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 7, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "node:node_net_utilisation:sum_irate{cluster=\"$cluster\", node=\"$node\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Utilisation", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Net Utilisation (Transmitted)", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 8, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "node:node_net_saturation:sum_irate{cluster=\"$cluster\", node=\"$node\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Saturation", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Net Saturation (Dropped)", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Net", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 9, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "node:node_filesystem_usage:{cluster=\"$cluster\"}\n* on (namespace, pod) group_left (node) node_namespace_pod:kube_pod_info:{cluster=\"$cluster\", node=\"$node\"}\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{device}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Disk Utilisation", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Disk", + "titleSize": "h6" + } + ], + "schemaVersion": 14, + "style": "dark", + "tags": [ + "kubernetes-mixin" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": null, + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + "text": "prod", + "value": "prod" + }, + "datasource": "$datasource", + "hide": 2, + "includeAll": false, + "label": "cluster", + "multi": false, + "name": "cluster", + "options": [ + + ], + "query": "label_values(kube_node_info, cluster)", + "refresh": 1, + "regex": "", + "sort": 2, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + "text": "prod", + "value": "prod" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": "node", + "multi": false, + "name": "node", + "options": [ + + ], + "query": "label_values(kube_node_info{cluster=\"$cluster\"}, node)", + "refresh": 1, + "regex": "", + "sort": 2, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "", + "title": "Kubernetes / USE Method / Node", + "uid": "4ac4f123aae0ff6dbaf4f4f66120033b", + "version": 0 + } k8s-resources-cluster.json: |- { "annotations": { @@ -147,7 +2048,7 @@ data: "steppedLine": false, "targets": [ { - "expr": "sum(kube_pod_container_resource_requests_cpu_cores{cluster=\"$cluster\"}) / sum(node:node_num_cpu:sum{cluster=\"$cluster\"})", + "expr": "sum(kube_pod_container_resource_requests_cpu_cores{cluster=\"$cluster\"}) / sum(kube_node_status_allocatable_cpu_cores{cluster=\"$cluster\"})", "format": "time_series", "instant": true, "intervalFactor": 2, @@ -231,7 +2132,7 @@ data: "steppedLine": false, "targets": [ { - "expr": "sum(kube_pod_container_resource_limits_cpu_cores{cluster=\"$cluster\"}) / sum(node:node_num_cpu:sum{cluster=\"$cluster\"})", + "expr": "sum(kube_pod_container_resource_limits_cpu_cores{cluster=\"$cluster\"}) / sum(kube_node_status_allocatable_cpu_cores{cluster=\"$cluster\"})", "format": "time_series", "instant": true, "intervalFactor": 2, @@ -315,7 +2216,7 @@ data: "steppedLine": false, "targets": [ { - "expr": "1 - sum(:node_memory_MemFreeCachedBuffers_bytes:sum{cluster=\"$cluster\"}) / sum(:node_memory_MemTotal_bytes:sum{cluster=\"$cluster\"})", + "expr": "1 - sum(:node_memory_MemFreeCachedBuffers_bytes:sum{cluster=\"$cluster\"}) / sum(kube_node_status_allocatable_memory_bytes{cluster=\"$cluster\"})", "format": "time_series", "instant": true, "intervalFactor": 2, @@ -399,7 +2300,7 @@ data: "steppedLine": false, "targets": [ { - "expr": "sum(kube_pod_container_resource_requests_memory_bytes{cluster=\"$cluster\"}) / sum(:node_memory_MemTotal_bytes:sum{cluster=\"$cluster\"})", + "expr": "sum(kube_pod_container_resource_requests_memory_bytes{cluster=\"$cluster\"}) / sum(kube_node_status_allocatable_memory_bytes{cluster=\"$cluster\"})", "format": "time_series", "instant": true, "intervalFactor": 2, @@ -483,7 +2384,7 @@ data: "steppedLine": false, "targets": [ { - "expr": "sum(kube_pod_container_resource_limits_memory_bytes{cluster=\"$cluster\"}) / sum(:node_memory_MemTotal_bytes:sum{cluster=\"$cluster\"})", + "expr": "sum(kube_pod_container_resource_limits_memory_bytes{cluster=\"$cluster\"}) / sum(kube_node_status_allocatable_memory_bytes{cluster=\"$cluster\"})", "format": "time_series", "instant": true, "intervalFactor": 2, @@ -578,7 +2479,7 @@ data: "steppedLine": false, "targets": [ { - "expr": "sum(namespace_pod_name_container_name:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\"}) by (namespace)", + "expr": "sum(namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\"}) by (namespace)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{namespace}}", @@ -861,7 +2762,7 @@ data: "step": 10 }, { - "expr": "sum(namespace_pod_name_container_name:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\"}) by (namespace)", + "expr": "sum(namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\"}) by (namespace)", "format": "table", "instant": true, "intervalFactor": 2, @@ -879,7 +2780,7 @@ data: "step": 10 }, { - "expr": "sum(namespace_pod_name_container_name:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\"}) by (namespace) / sum(kube_pod_container_resource_requests_cpu_cores{cluster=\"$cluster\"}) by (namespace)", + "expr": "sum(namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\"}) by (namespace) / sum(kube_pod_container_resource_requests_cpu_cores{cluster=\"$cluster\"}) by (namespace)", "format": "table", "instant": true, "intervalFactor": 2, @@ -897,7 +2798,7 @@ data: "step": 10 }, { - "expr": "sum(namespace_pod_name_container_name:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\"}) by (namespace) / sum(kube_pod_container_resource_limits_cpu_cores{cluster=\"$cluster\"}) by (namespace)", + "expr": "sum(namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\"}) by (namespace) / sum(kube_pod_container_resource_limits_cpu_cores{cluster=\"$cluster\"}) by (namespace)", "format": "table", "instant": true, "intervalFactor": 2, @@ -997,7 +2898,7 @@ data: "steppedLine": false, "targets": [ { - "expr": "sum(container_memory_rss{cluster=\"$cluster\", container_name!=\"\"}) by (namespace)", + "expr": "sum(container_memory_rss{cluster=\"$cluster\", container!=\"\"}) by (namespace)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{namespace}}", @@ -1280,7 +3181,7 @@ data: "step": 10 }, { - "expr": "sum(container_memory_rss{cluster=\"$cluster\", container_name!=\"\"}) by (namespace)", + "expr": "sum(container_memory_rss{cluster=\"$cluster\", container!=\"\"}) by (namespace)", "format": "table", "instant": true, "intervalFactor": 2, @@ -1298,7 +3199,7 @@ data: "step": 10 }, { - "expr": "sum(container_memory_rss{cluster=\"$cluster\", container_name!=\"\"}) by (namespace) / sum(kube_pod_container_resource_requests_memory_bytes{cluster=\"$cluster\"}) by (namespace)", + "expr": "sum(container_memory_rss{cluster=\"$cluster\", container!=\"\"}) by (namespace) / sum(kube_pod_container_resource_requests_memory_bytes{cluster=\"$cluster\"}) by (namespace)", "format": "table", "instant": true, "intervalFactor": 2, @@ -1316,7 +3217,7 @@ data: "step": 10 }, { - "expr": "sum(container_memory_rss{cluster=\"$cluster\", container_name!=\"\"}) by (namespace) / sum(kube_pod_container_resource_limits_memory_bytes{cluster=\"$cluster\"}) by (namespace)", + "expr": "sum(container_memory_rss{cluster=\"$cluster\", container!=\"\"}) by (namespace) / sum(kube_pod_container_resource_limits_memory_bytes{cluster=\"$cluster\"}) by (namespace)", "format": "table", "instant": true, "intervalFactor": 2, @@ -1519,10 +3420,10 @@ data: "steppedLine": false, "targets": [ { - "expr": "sum(namespace_pod_name_container_name:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod_name)", + "expr": "sum(namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)", "format": "time_series", "intervalFactor": 2, - "legendFormat": "{{pod_name}}", + "legendFormat": "{{pod}}", "legendLink": null, "step": 10 } @@ -1748,7 +3649,7 @@ data: ], "targets": [ { - "expr": "sum(label_replace(namespace_pod_name_container_name:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\"}, \"pod\", \"$1\", \"pod_name\", \"(.*)\")) by (pod)", + "expr": "sum(namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)", "format": "table", "instant": true, "intervalFactor": 2, @@ -1766,7 +3667,7 @@ data: "step": 10 }, { - "expr": "sum(label_replace(namespace_pod_name_container_name:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\"}, \"pod\", \"$1\", \"pod_name\", \"(.*)\")) by (pod) / sum(kube_pod_container_resource_requests_cpu_cores{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)", + "expr": "sum(namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod) / sum(kube_pod_container_resource_requests_cpu_cores{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)", "format": "table", "instant": true, "intervalFactor": 2, @@ -1784,7 +3685,7 @@ data: "step": 10 }, { - "expr": "sum(label_replace(namespace_pod_name_container_name:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\"}, \"pod\", \"$1\", \"pod_name\", \"(.*)\")) by (pod) / sum(kube_pod_container_resource_limits_cpu_cores{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)", + "expr": "sum(namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod) / sum(kube_pod_container_resource_limits_cpu_cores{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)", "format": "table", "instant": true, "intervalFactor": 2, @@ -1884,10 +3785,10 @@ data: "steppedLine": false, "targets": [ { - "expr": "sum(container_memory_usage_bytes{cluster=\"$cluster\", namespace=\"$namespace\", container_name!=\"\"}) by (pod_name)", + "expr": "sum(container_memory_usage_bytes{cluster=\"$cluster\", namespace=\"$namespace\", container!=\"\"}) by (pod)", "format": "time_series", "intervalFactor": 2, - "legendFormat": "{{pod_name}}", + "legendFormat": "{{pod}}", "legendLink": null, "step": 10 } @@ -2167,7 +4068,7 @@ data: ], "targets": [ { - "expr": "sum(label_replace(container_memory_usage_bytes{cluster=\"$cluster\", namespace=\"$namespace\",container_name!=\"\"}, \"pod\", \"$1\", \"pod_name\", \"(.*)\")) by (pod)", + "expr": "sum(container_memory_usage_bytes{cluster=\"$cluster\", namespace=\"$namespace\",container!=\"\"}) by (pod)", "format": "table", "instant": true, "intervalFactor": 2, @@ -2185,7 +4086,7 @@ data: "step": 10 }, { - "expr": "sum(label_replace(container_memory_usage_bytes{cluster=\"$cluster\", namespace=\"$namespace\",container_name!=\"\"}, \"pod\", \"$1\", \"pod_name\", \"(.*)\")) by (pod) / sum(kube_pod_container_resource_requests_memory_bytes{namespace=\"$namespace\"}) by (pod)", + "expr": "sum(container_memory_usage_bytes{cluster=\"$cluster\", namespace=\"$namespace\",container!=\"\"}) by (pod) / sum(kube_pod_container_resource_requests_memory_bytes{namespace=\"$namespace\"}) by (pod)", "format": "table", "instant": true, "intervalFactor": 2, @@ -2203,7 +4104,7 @@ data: "step": 10 }, { - "expr": "sum(label_replace(container_memory_usage_bytes{cluster=\"$cluster\", namespace=\"$namespace\",container_name!=\"\"}, \"pod\", \"$1\", \"pod_name\", \"(.*)\")) by (pod) / sum(kube_pod_container_resource_limits_memory_bytes{namespace=\"$namespace\"}) by (pod)", + "expr": "sum(container_memory_usage_bytes{cluster=\"$cluster\", namespace=\"$namespace\",container!=\"\"}) by (pod) / sum(kube_pod_container_resource_limits_memory_bytes{namespace=\"$namespace\"}) by (pod)", "format": "table", "instant": true, "intervalFactor": 2, @@ -2212,7 +4113,7 @@ data: "step": 10 }, { - "expr": "sum(label_replace(container_memory_rss{cluster=\"$cluster\", namespace=\"$namespace\",container_name!=\"\"}, \"pod\", \"$1\", \"pod_name\", \"(.*)\")) by (pod)", + "expr": "sum(container_memory_rss{cluster=\"$cluster\", namespace=\"$namespace\",container!=\"\"}) by (pod)", "format": "table", "instant": true, "intervalFactor": 2, @@ -2221,7 +4122,7 @@ data: "step": 10 }, { - "expr": "sum(label_replace(container_memory_cache{cluster=\"$cluster\", namespace=\"$namespace\",container_name!=\"\"}, \"pod\", \"$1\", \"pod_name\", \"(.*)\")) by (pod)", + "expr": "sum(container_memory_cache{cluster=\"$cluster\", namespace=\"$namespace\",container!=\"\"}) by (pod)", "format": "table", "instant": true, "intervalFactor": 2, @@ -2230,7 +4131,7 @@ data: "step": 10 }, { - "expr": "sum(label_replace(container_memory_swap{cluster=\"$cluster\", namespace=\"$namespace\",container_name!=\"\"}, \"pod\", \"$1\", \"pod_name\", \"(.*)\")) by (pod)", + "expr": "sum(container_memory_swap{cluster=\"$cluster\", namespace=\"$namespace\",container!=\"\"}) by (pod)", "format": "table", "instant": true, "intervalFactor": 2, @@ -2460,10 +4361,10 @@ data: "steppedLine": false, "targets": [ { - "expr": "sum(namespace_pod_name_container_name:container_cpu_usage_seconds_total:sum_rate{namespace=\"$namespace\", pod_name=\"$pod\", container_name!=\"POD\", cluster=\"$cluster\"}) by (container_name)", + "expr": "sum(namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{namespace=\"$namespace\", pod=\"$pod\", container!=\"POD\", cluster=\"$cluster\"}) by (container)", "format": "time_series", "intervalFactor": 2, - "legendFormat": "{{container_name}}", + "legendFormat": "{{container}}", "legendLink": null, "step": 10 } @@ -2689,7 +4590,7 @@ data: ], "targets": [ { - "expr": "sum(label_replace(namespace_pod_name_container_name:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\", pod_name=\"$pod\", container_name!=\"POD\"}, \"container\", \"$1\", \"container_name\", \"(.*)\")) by (container)", + "expr": "sum(namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\", container!=\"POD\"}) by (container)", "format": "table", "instant": true, "intervalFactor": 2, @@ -2707,7 +4608,7 @@ data: "step": 10 }, { - "expr": "sum(label_replace(namespace_pod_name_container_name:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\", pod_name=\"$pod\"}, \"container\", \"$1\", \"container_name\", \"(.*)\")) by (container) / sum(kube_pod_container_resource_requests_cpu_cores{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container)", + "expr": "sum(namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container) / sum(kube_pod_container_resource_requests_cpu_cores{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container)", "format": "table", "instant": true, "intervalFactor": 2, @@ -2725,7 +4626,7 @@ data: "step": 10 }, { - "expr": "sum(label_replace(namespace_pod_name_container_name:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\", pod_name=\"$pod\"}, \"container\", \"$1\", \"container_name\", \"(.*)\")) by (container) / sum(kube_pod_container_resource_limits_cpu_cores{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container)", + "expr": "sum(namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container) / sum(kube_pod_container_resource_limits_cpu_cores{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container)", "format": "table", "instant": true, "intervalFactor": 2, @@ -2825,26 +4726,26 @@ data: "steppedLine": false, "targets": [ { - "expr": "sum(container_memory_rss{cluster=\"$cluster\", namespace=\"$namespace\", pod_name=\"$pod\", container_name!=\"POD\", container_name!=\"\"}) by (container_name)", + "expr": "sum(container_memory_rss{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\", container!=\"POD\", container!=\"\"}) by (container)", "format": "time_series", "intervalFactor": 2, - "legendFormat": "{{container_name}} (RSS)", + "legendFormat": "{{container}} (RSS)", "legendLink": null, "step": 10 }, { - "expr": "sum(container_memory_cache{cluster=\"$cluster\", namespace=\"$namespace\", pod_name=\"$pod\", container_name!=\"POD\", container_name!=\"\"}) by (container_name)", + "expr": "sum(container_memory_cache{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\", container!=\"POD\", container!=\"\"}) by (container)", "format": "time_series", "intervalFactor": 2, - "legendFormat": "{{container_name}} (Cache)", + "legendFormat": "{{container}} (Cache)", "legendLink": null, "step": 10 }, { - "expr": "sum(container_memory_swap{cluster=\"$cluster\", namespace=\"$namespace\", pod_name=\"$pod\", container_name!=\"POD\", container_name!=\"\"}) by (container_name)", + "expr": "sum(container_memory_swap{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\", container!=\"POD\", container!=\"\"}) by (container)", "format": "time_series", "intervalFactor": 2, - "legendFormat": "{{container_name}} (Swap)", + "legendFormat": "{{container}} (Swap)", "legendLink": null, "step": 10 } @@ -3124,7 +5025,7 @@ data: ], "targets": [ { - "expr": "sum(label_replace(container_memory_usage_bytes{cluster=\"$cluster\", namespace=\"$namespace\", pod_name=\"$pod\", container_name!=\"POD\", container_name!=\"\"}, \"container\", \"$1\", \"container_name\", \"(.*)\")) by (container)", + "expr": "sum(container_memory_usage_bytes{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\", container!=\"POD\", container!=\"\"}) by (container)", "format": "table", "instant": true, "intervalFactor": 2, @@ -3142,7 +5043,7 @@ data: "step": 10 }, { - "expr": "sum(label_replace(container_memory_usage_bytes{cluster=\"$cluster\", namespace=\"$namespace\", pod_name=\"$pod\"}, \"container\", \"$1\", \"container_name\", \"(.*)\")) by (container) / sum(kube_pod_container_resource_requests_memory_bytes{namespace=\"$namespace\", pod=\"$pod\"}) by (container)", + "expr": "sum(container_memory_usage_bytes{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container) / sum(kube_pod_container_resource_requests_memory_bytes{namespace=\"$namespace\", pod=\"$pod\"}) by (container)", "format": "table", "instant": true, "intervalFactor": 2, @@ -3160,7 +5061,7 @@ data: "step": 10 }, { - "expr": "sum(label_replace(container_memory_usage_bytes{cluster=\"$cluster\", namespace=\"$namespace\", pod_name=\"$pod\", container_name!=\"\"}, \"container\", \"$1\", \"container_name\", \"(.*)\")) by (container) / sum(kube_pod_container_resource_limits_memory_bytes{namespace=\"$namespace\", pod=\"$pod\"}) by (container)", + "expr": "sum(container_memory_usage_bytes{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\", container!=\"\"}) by (container) / sum(kube_pod_container_resource_limits_memory_bytes{namespace=\"$namespace\", pod=\"$pod\"}) by (container)", "format": "table", "instant": true, "intervalFactor": 2, @@ -3169,7 +5070,7 @@ data: "step": 10 }, { - "expr": "sum(label_replace(container_memory_rss{cluster=\"$cluster\", namespace=\"$namespace\", pod_name=\"$pod\", container_name != \"\", container_name != \"POD\"}, \"container\", \"$1\", \"container_name\", \"(.*)\")) by (container)", + "expr": "sum(container_memory_rss{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\", container != \"\", container != \"POD\"}) by (container)", "format": "table", "instant": true, "intervalFactor": 2, @@ -3178,7 +5079,7 @@ data: "step": 10 }, { - "expr": "sum(label_replace(container_memory_cache{cluster=\"$cluster\", namespace=\"$namespace\", pod_name=\"$pod\", container_name != \"\", container_name != \"POD\"}, \"container\", \"$1\", \"container_name\", \"(.*)\")) by (container)", + "expr": "sum(container_memory_cache{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\", container != \"\", container != \"POD\"}) by (container)", "format": "table", "instant": true, "intervalFactor": 2, @@ -3187,7 +5088,7 @@ data: "step": 10 }, { - "expr": "sum(label_replace(container_memory_swap{cluster=\"$cluster\", namespace=\"$namespace\", pod_name=\"$pod\", container_name != \"\", container_name != \"POD\"}, \"container\", \"$1\", \"container_name\", \"(.*)\")) by (container)", + "expr": "sum(container_memory_swap{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\", container != \"\", container != \"POD\"}) by (container)", "format": "table", "instant": true, "intervalFactor": 2, @@ -3444,7 +5345,7 @@ data: "steppedLine": false, "targets": [ { - "expr": "sum(\n label_replace(\n namespace_pod_name_container_name:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\"},\n \"pod\", \"$1\", \"pod_name\", \"(.*)\"\n ) * on(namespace,pod) group_left(workload, workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", + "expr": "sum(\n namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\"}\n * on(namespace,pod)\n group_left(workload, workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{pod}}", @@ -3673,7 +5574,7 @@ data: ], "targets": [ { - "expr": "sum(\n label_replace(\n namespace_pod_name_container_name:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\"},\n \"pod\", \"$1\", \"pod_name\", \"(.*)\"\n ) * on(namespace,pod) group_left(workload, workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", + "expr": "sum(\n namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\"}\n * on(namespace,pod)\n group_left(workload, workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", "format": "table", "instant": true, "intervalFactor": 2, @@ -3682,7 +5583,7 @@ data: "step": 10 }, { - "expr": "sum(\n kube_pod_container_resource_requests_cpu_cores{cluster=\"$cluster\", namespace=\"$namespace\"}\n * on(namespace,pod) group_left(workload, workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", + "expr": "sum(\n kube_pod_container_resource_requests_cpu_cores{cluster=\"$cluster\", namespace=\"$namespace\"}\n * on(namespace,pod)\n group_left(workload, workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", "format": "table", "instant": true, "intervalFactor": 2, @@ -3691,7 +5592,7 @@ data: "step": 10 }, { - "expr": "sum(\n label_replace(\n namespace_pod_name_container_name:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\"},\n \"pod\", \"$1\", \"pod_name\", \"(.*)\"\n ) * on(namespace,pod) group_left(workload, workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n/sum(\n kube_pod_container_resource_requests_cpu_cores{cluster=\"$cluster\", namespace=\"$namespace\"}\n * on(namespace,pod) group_left(workload, workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", + "expr": "sum(\n namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\"}\n * on(namespace,pod)\n group_left(workload, workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n/sum(\n kube_pod_container_resource_requests_cpu_cores{cluster=\"$cluster\", namespace=\"$namespace\"}\n * on(namespace,pod)\n group_left(workload, workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", "format": "table", "instant": true, "intervalFactor": 2, @@ -3700,7 +5601,7 @@ data: "step": 10 }, { - "expr": "sum(\n kube_pod_container_resource_limits_cpu_cores{cluster=\"$cluster\", namespace=\"$namespace\"}\n * on(namespace,pod) group_left(workload, workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", + "expr": "sum(\n kube_pod_container_resource_limits_cpu_cores{cluster=\"$cluster\", namespace=\"$namespace\"}\n * on(namespace,pod)\n group_left(workload, workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", "format": "table", "instant": true, "intervalFactor": 2, @@ -3709,7 +5610,7 @@ data: "step": 10 }, { - "expr": "sum(\n label_replace(\n namespace_pod_name_container_name:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\"},\n \"pod\", \"$1\", \"pod_name\", \"(.*)\"\n ) * on(namespace,pod) group_left(workload, workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n/sum(\n kube_pod_container_resource_limits_cpu_cores{cluster=\"$cluster\", namespace=\"$namespace\"}\n * on(namespace,pod) group_left(workload, workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", + "expr": "sum(\n namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\"}\n * on(namespace,pod)\n group_left(workload, workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n/sum(\n kube_pod_container_resource_limits_cpu_cores{cluster=\"$cluster\", namespace=\"$namespace\"}\n * on(namespace,pod)\n group_left(workload, workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", "format": "table", "instant": true, "intervalFactor": 2, @@ -3809,7 +5710,7 @@ data: "steppedLine": false, "targets": [ { - "expr": "sum(\n label_replace(\n container_memory_usage_bytes{cluster=\"$cluster\", namespace=\"$namespace\", container_name!=\"\"},\n \"pod\", \"$1\", \"pod_name\", \"(.*)\"\n ) * on(namespace,pod) group_left(workload, workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n ) by (pod)\n", + "expr": "sum(\n container_memory_usage_bytes{cluster=\"$cluster\", namespace=\"$namespace\", container!=\"\"}\n * on(namespace,pod)\n group_left(workload, workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{pod}}", @@ -4038,7 +5939,7 @@ data: ], "targets": [ { - "expr": "sum(\n label_replace(\n container_memory_usage_bytes{cluster=\"$cluster\", namespace=\"$namespace\", container_name!=\"\"},\n \"pod\", \"$1\", \"pod_name\", \"(.*)\"\n ) * on(namespace,pod) group_left(workload, workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n ) by (pod)\n", + "expr": "sum(\n container_memory_usage_bytes{cluster=\"$cluster\", namespace=\"$namespace\", container!=\"\"}\n * on(namespace,pod)\n group_left(workload, workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", "format": "table", "instant": true, "intervalFactor": 2, @@ -4047,7 +5948,7 @@ data: "step": 10 }, { - "expr": "sum(\n kube_pod_container_resource_requests_memory_bytes{cluster=\"$cluster\", namespace=\"$namespace\"}\n * on(namespace,pod) group_left(workload, workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", + "expr": "sum(\n kube_pod_container_resource_requests_memory_bytes{cluster=\"$cluster\", namespace=\"$namespace\"}\n * on(namespace,pod)\n group_left(workload, workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", "format": "table", "instant": true, "intervalFactor": 2, @@ -4056,7 +5957,7 @@ data: "step": 10 }, { - "expr": "sum(\n label_replace(\n container_memory_usage_bytes{cluster=\"$cluster\", namespace=\"$namespace\", container_name!=\"\"},\n \"pod\", \"$1\", \"pod_name\", \"(.*)\"\n ) * on(namespace,pod) group_left(workload, workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n ) by (pod)\n/sum(\n kube_pod_container_resource_requests_memory_bytes{cluster=\"$cluster\", namespace=\"$namespace\"}\n * on(namespace,pod) group_left(workload, workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", + "expr": "sum(\n container_memory_usage_bytes{cluster=\"$cluster\", namespace=\"$namespace\", container!=\"\"}\n * on(namespace,pod)\n group_left(workload, workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n/sum(\n kube_pod_container_resource_requests_memory_bytes{cluster=\"$cluster\", namespace=\"$namespace\"}\n * on(namespace,pod)\n group_left(workload, workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", "format": "table", "instant": true, "intervalFactor": 2, @@ -4065,7 +5966,7 @@ data: "step": 10 }, { - "expr": "sum(\n kube_pod_container_resource_limits_memory_bytes{cluster=\"$cluster\", namespace=\"$namespace\"}\n * on(namespace,pod) group_left(workload, workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", + "expr": "sum(\n kube_pod_container_resource_limits_memory_bytes{cluster=\"$cluster\", namespace=\"$namespace\"}\n * on(namespace,pod)\n group_left(workload, workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", "format": "table", "instant": true, "intervalFactor": 2, @@ -4074,7 +5975,7 @@ data: "step": 10 }, { - "expr": "sum(\n label_replace(\n container_memory_usage_bytes{cluster=\"$cluster\", namespace=\"$namespace\", container_name!=\"\"},\n \"pod\", \"$1\", \"pod_name\", \"(.*)\"\n ) * on(namespace,pod) group_left(workload, workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n ) by (pod)\n/sum(\n kube_pod_container_resource_limits_memory_bytes{cluster=\"$cluster\", namespace=\"$namespace\"}\n * on(namespace,pod) group_left(workload, workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", + "expr": "sum(\n container_memory_usage_bytes{cluster=\"$cluster\", namespace=\"$namespace\", container!=\"\"}\n * on(namespace,pod)\n group_left(workload, workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n/sum(\n kube_pod_container_resource_limits_memory_bytes{cluster=\"$cluster\", namespace=\"$namespace\"}\n * on(namespace,pod)\n group_left(workload, workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", "format": "table", "instant": true, "intervalFactor": 2, @@ -4358,7 +6259,7 @@ data: "steppedLine": false, "targets": [ { - "expr": "sum(\n label_replace(\n namespace_pod_name_container_name:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\"},\n \"pod\", \"$1\", \"pod_name\", \"(.*)\"\n ) * on(namespace,pod) group_left(workload, workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\"}\n) by (workload, workload_type)\n", + "expr": "sum(\n namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\"}\n* on(namespace,pod)\n group_left(workload, workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\"}\n) by (workload, workload_type)\n", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{workload}} - {{workload_type}}", @@ -4632,7 +6533,7 @@ data: "step": 10 }, { - "expr": "sum(\n label_replace(\n namespace_pod_name_container_name:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\"},\n \"pod\", \"$1\", \"pod_name\", \"(.*)\"\n ) * on(namespace,pod) group_left(workload, workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\"}\n) by (workload, workload_type)\n", + "expr": "sum(\n namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\"}\n* on(namespace,pod)\n group_left(workload, workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\"}\n) by (workload, workload_type)\n", "format": "table", "instant": true, "intervalFactor": 2, @@ -4641,7 +6542,7 @@ data: "step": 10 }, { - "expr": "sum(\n kube_pod_container_resource_requests_cpu_cores{cluster=\"$cluster\", namespace=\"$namespace\"}\n * on(namespace,pod) group_left(workload, workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\"}\n) by (workload, workload_type)\n", + "expr": "sum(\n kube_pod_container_resource_requests_cpu_cores{cluster=\"$cluster\", namespace=\"$namespace\"}\n* on(namespace,pod)\n group_left(workload, workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\"}\n) by (workload, workload_type)\n", "format": "table", "instant": true, "intervalFactor": 2, @@ -4650,7 +6551,7 @@ data: "step": 10 }, { - "expr": "sum(\n label_replace(\n namespace_pod_name_container_name:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\"},\n \"pod\", \"$1\", \"pod_name\", \"(.*)\"\n ) * on(namespace,pod) group_left(workload, workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\"}\n) by (workload, workload_type)\n/sum(\n kube_pod_container_resource_requests_cpu_cores{cluster=\"$cluster\", namespace=\"$namespace\"}\n * on(namespace,pod) group_left(workload, workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\"}\n) by (workload, workload_type)\n", + "expr": "sum(\n namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\"}\n* on(namespace,pod)\n group_left(workload, workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\"}\n) by (workload, workload_type)\n/sum(\n kube_pod_container_resource_requests_cpu_cores{cluster=\"$cluster\", namespace=\"$namespace\"}\n* on(namespace,pod)\n group_left(workload, workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\"}\n) by (workload, workload_type)\n", "format": "table", "instant": true, "intervalFactor": 2, @@ -4659,7 +6560,7 @@ data: "step": 10 }, { - "expr": "sum(\n kube_pod_container_resource_limits_cpu_cores{cluster=\"$cluster\", namespace=\"$namespace\"}\n * on(namespace,pod) group_left(workload, workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\"}\n) by (workload, workload_type)\n", + "expr": "sum(\n kube_pod_container_resource_limits_cpu_cores{cluster=\"$cluster\", namespace=\"$namespace\"}\n* on(namespace,pod)\n group_left(workload, workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\"}\n) by (workload, workload_type)\n", "format": "table", "instant": true, "intervalFactor": 2, @@ -4668,7 +6569,7 @@ data: "step": 10 }, { - "expr": "sum(\n label_replace(\n namespace_pod_name_container_name:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\"},\n \"pod\", \"$1\", \"pod_name\", \"(.*)\"\n ) * on(namespace,pod) group_left(workload, workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\"}\n) by (workload, workload_type)\n/sum(\n kube_pod_container_resource_limits_cpu_cores{cluster=\"$cluster\", namespace=\"$namespace\"}\n * on(namespace,pod) group_left(workload, workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\"}\n) by (workload, workload_type)\n", + "expr": "sum(\n namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\"}\n* on(namespace,pod)\n group_left(workload, workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\"}\n) by (workload, workload_type)\n/sum(\n kube_pod_container_resource_limits_cpu_cores{cluster=\"$cluster\", namespace=\"$namespace\"}\n* on(namespace,pod)\n group_left(workload, workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\"}\n) by (workload, workload_type)\n", "format": "table", "instant": true, "intervalFactor": 2, @@ -4768,7 +6669,7 @@ data: "steppedLine": false, "targets": [ { - "expr": "sum(\n label_replace(\n container_memory_usage_bytes{cluster=\"$cluster\", namespace=\"$namespace\", container_name!=\"\"},\n \"pod\", \"$1\", \"pod_name\", \"(.*)\"\n ) * on(namespace,pod) group_left(workload, workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\"}\n ) by (workload, workload_type)\n", + "expr": "sum(\n container_memory_usage_bytes{cluster=\"$cluster\", namespace=\"$namespace\", container!=\"\"}\n * on(namespace,pod)\n group_left(workload, workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\"}\n) by (workload, workload_type)\n", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{workload}} - {{workload_type}}", @@ -5042,7 +6943,7 @@ data: "step": 10 }, { - "expr": "sum(\n label_replace(\n container_memory_usage_bytes{cluster=\"$cluster\", namespace=\"$namespace\", container_name!=\"\"},\n \"pod\", \"$1\", \"pod_name\", \"(.*)\"\n ) * on(namespace,pod) group_left(workload, workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\"}\n ) by (workload, workload_type)\n", + "expr": "sum(\n container_memory_usage_bytes{cluster=\"$cluster\", namespace=\"$namespace\", container!=\"\"}\n * on(namespace,pod)\n group_left(workload, workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\"}\n) by (workload, workload_type)\n", "format": "table", "instant": true, "intervalFactor": 2, @@ -5051,7 +6952,7 @@ data: "step": 10 }, { - "expr": "sum(\n kube_pod_container_resource_requests_memory_bytes{cluster=\"$cluster\", namespace=\"$namespace\"}\n * on(namespace,pod) group_left(workload, workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\"}\n) by (workload, workload_type)\n", + "expr": "sum(\n kube_pod_container_resource_requests_memory_bytes{cluster=\"$cluster\", namespace=\"$namespace\"}\n* on(namespace,pod)\n group_left(workload, workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\"}\n) by (workload, workload_type)\n", "format": "table", "instant": true, "intervalFactor": 2, @@ -5060,7 +6961,7 @@ data: "step": 10 }, { - "expr": "sum(\n label_replace(\n container_memory_usage_bytes{cluster=\"$cluster\", namespace=\"$namespace\", container_name!=\"\"},\n \"pod\", \"$1\", \"pod_name\", \"(.*)\"\n ) * on(namespace,pod) group_left(workload, workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\"}\n ) by (workload, workload_type)\n/sum(\n kube_pod_container_resource_requests_memory_bytes{cluster=\"$cluster\", namespace=\"$namespace\"}\n * on(namespace,pod) group_left(workload, workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\"}\n) by (workload, workload_type)\n", + "expr": "sum(\n container_memory_usage_bytes{cluster=\"$cluster\", namespace=\"$namespace\", container!=\"\"}\n * on(namespace,pod)\n group_left(workload, workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\"}\n) by (workload, workload_type)\n/sum(\n kube_pod_container_resource_requests_memory_bytes{cluster=\"$cluster\", namespace=\"$namespace\"}\n* on(namespace,pod)\n group_left(workload, workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\"}\n) by (workload, workload_type)\n", "format": "table", "instant": true, "intervalFactor": 2, @@ -5069,7 +6970,7 @@ data: "step": 10 }, { - "expr": "sum(\n kube_pod_container_resource_limits_memory_bytes{cluster=\"$cluster\", namespace=\"$namespace\"}\n * on(namespace,pod) group_left(workload, workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\"}\n) by (workload, workload_type)\n", + "expr": "sum(\n kube_pod_container_resource_limits_memory_bytes{cluster=\"$cluster\", namespace=\"$namespace\"}\n* on(namespace,pod)\n group_left(workload, workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\"}\n) by (workload, workload_type)\n", "format": "table", "instant": true, "intervalFactor": 2, @@ -5078,7 +6979,7 @@ data: "step": 10 }, { - "expr": "sum(\n label_replace(\n container_memory_usage_bytes{cluster=\"$cluster\", namespace=\"$namespace\", container_name!=\"\"},\n \"pod\", \"$1\", \"pod_name\", \"(.*)\"\n ) * on(namespace,pod) group_left(workload, workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\"}\n ) by (workload, workload_type)\n/sum(\n kube_pod_container_resource_limits_memory_bytes{cluster=\"$cluster\", namespace=\"$namespace\"}\n * on(namespace,pod) group_left(workload, workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\"}\n) by (workload, workload_type)\n", + "expr": "sum(\n container_memory_usage_bytes{cluster=\"$cluster\", namespace=\"$namespace\", container!=\"\"}\n * on(namespace,pod)\n group_left(workload, workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\"}\n) by (workload, workload_type)\n/sum(\n kube_pod_container_resource_limits_memory_bytes{cluster=\"$cluster\", namespace=\"$namespace\"}\n* on(namespace,pod)\n group_left(workload, workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\"}\n) by (workload, workload_type)\n", "format": "table", "instant": true, "intervalFactor": 2, diff --git a/addons/grafana/dashboards-k8s.yaml b/addons/grafana/dashboards-k8s.yaml index d83a49cfb..0f5a8c2eb 100644 --- a/addons/grafana/dashboards-k8s.yaml +++ b/addons/grafana/dashboards-k8s.yaml @@ -4,26 +4,117 @@ metadata: name: grafana-dashboards-k8s namespace: monitoring data: - k8s-cluster-rsrc-use.json: |- + apiserver.json: |- { + "__inputs": [ + + ], + "__requires": [ + + ], "annotations": { "list": [ ] }, - "editable": true, + "editable": false, "gnetId": null, "graphTooltip": 0, "hideControls": false, + "id": null, "links": [ ], - "refresh": "10s", + "refresh": "", "rows": [ { "collapse": false, - "height": "250px", + "collapsed": false, "panels": [ + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "$datasource", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + + }, + "id": 2, + "interval": null, + "links": [ + + ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "span": 2, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum(up{job=\"apiserver\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": "", + "title": "Up", + "tooltip": { + "shared": false + }, + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "min" + }, { "aliasColors": { @@ -32,42 +123,68 @@ data: "dashLength": 10, "dashes": false, "datasource": "$datasource", - "fill": 10, - "id": 1, + "fill": 1, + "gridPos": { + + }, + "id": 3, "legend": { + "alignAsTable": false, "avg": false, "current": false, "max": false, "min": false, + "rightSide": false, "show": true, "total": false, "values": false }, "lines": true, - "linewidth": 0, + "linewidth": 1, "links": [ ], - "nullPointMode": "null as zero", + "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", + "repeat": null, "seriesOverrides": [ ], "spaceLength": 10, - "span": 6, - "stack": true, + "span": 5, + "stack": false, "steppedLine": false, "targets": [ { - "expr": "node:cluster_cpu_utilisation:ratio{cluster=\"$cluster\"}", + "expr": "sum(rate(apiserver_request_total{job=\"apiserver\", instance=~\"$instance\",code=~\"2..\"}[5m]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "2xx", + "refId": "A" + }, + { + "expr": "sum(rate(apiserver_request_total{job=\"apiserver\", instance=~\"$instance\",code=~\"3..\"}[5m]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "3xx", + "refId": "B" + }, + { + "expr": "sum(rate(apiserver_request_total{job=\"apiserver\", instance=~\"$instance\",code=~\"4..\"}[5m]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "4xx", + "refId": "C" + }, + { + "expr": "sum(rate(apiserver_request_total{job=\"apiserver\", instance=~\"$instance\",code=~\"5..\"}[5m]))", "format": "time_series", "intervalFactor": 2, - "legendFormat": "{{node}}", - "legendLink": "/d/4ac4f123aae0ff6dbaf4f4f66120033b/k8s-node-rsrc-use", - "step": 10 + "legendFormat": "5xx", + "refId": "D" } ], "thresholds": [ @@ -75,7 +192,7 @@ data: ], "timeFrom": null, "timeShift": null, - "title": "CPU Utilisation", + "title": "RPC Rate", "tooltip": { "shared": false, "sort": 0, @@ -93,20 +210,20 @@ data: }, "yaxes": [ { - "format": "percentunit", + "format": "ops", "label": null, "logBase": 1, - "max": 1, - "min": 0, + "max": null, + "min": null, "show": true }, { - "format": "short", + "format": "ops", "label": null, "logBase": 1, "max": null, "min": null, - "show": false + "show": true } ] }, @@ -118,42 +235,47 @@ data: "dashLength": 10, "dashes": false, "datasource": "$datasource", - "fill": 10, - "id": 2, + "fill": 1, + "gridPos": { + + }, + "id": 4, "legend": { + "alignAsTable": "true", "avg": false, - "current": false, + "current": "true", "max": false, "min": false, - "show": true, + "rightSide": "true", + "show": "true", "total": false, - "values": false + "values": "true" }, "lines": true, - "linewidth": 0, + "linewidth": 1, "links": [ ], - "nullPointMode": "null as zero", + "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", + "repeat": null, "seriesOverrides": [ ], "spaceLength": 10, - "span": 6, - "stack": true, + "span": 5, + "stack": false, "steppedLine": false, "targets": [ { - "expr": "node:node_cpu_saturation_load1:{cluster=\"$cluster\"} / scalar(sum(min(kube_pod_info{cluster=\"$cluster\"}) by (node)))", + "expr": "histogram_quantile(0.99, sum(rate(apiserver_request_duration_seconds_bucket{job=\"apiserver\", instance=~\"$instance\"}[5m])) by (verb, le))", "format": "time_series", "intervalFactor": 2, - "legendFormat": "{{node}}", - "legendLink": "/d/4ac4f123aae0ff6dbaf4f4f66120033b/k8s-node-rsrc-use", - "step": 10 + "legendFormat": "{{verb}}", + "refId": "A" } ], "thresholds": [ @@ -161,7 +283,7 @@ data: ], "timeFrom": null, "timeShift": null, - "title": "CPU Saturation (Load1)", + "title": "Request duration 99th quantile", "tooltip": { "shared": false, "sort": 0, @@ -179,20 +301,20 @@ data: }, "yaxes": [ { - "format": "percentunit", + "format": "s", "label": null, "logBase": 1, - "max": 1, - "min": 0, + "max": null, + "min": null, "show": true }, { - "format": "short", + "format": "s", "label": null, "logBase": 1, "max": null, "min": null, - "show": false + "show": true } ] } @@ -200,13 +322,14 @@ data: "repeat": null, "repeatIteration": null, "repeatRowId": null, - "showTitle": true, - "title": "CPU", - "titleSize": "h6" + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" }, { "collapse": false, - "height": "250px", + "collapsed": false, "panels": [ { "aliasColors": { @@ -216,42 +339,47 @@ data: "dashLength": 10, "dashes": false, "datasource": "$datasource", - "fill": 10, - "id": 3, + "fill": 1, + "gridPos": { + + }, + "id": 5, "legend": { + "alignAsTable": false, "avg": false, "current": false, "max": false, "min": false, - "show": true, + "rightSide": false, + "show": false, "total": false, "values": false }, "lines": true, - "linewidth": 0, + "linewidth": 1, "links": [ ], - "nullPointMode": "null as zero", + "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", + "repeat": null, "seriesOverrides": [ ], "spaceLength": 10, "span": 6, - "stack": true, + "stack": false, "steppedLine": false, "targets": [ { - "expr": "node:cluster_memory_utilisation:ratio{cluster=\"$cluster\"}", + "expr": "sum(rate(workqueue_adds_total{job=\"apiserver\", instance=~\"$instance\"}[5m])) by (instance, name)", "format": "time_series", "intervalFactor": 2, - "legendFormat": "{{node}}", - "legendLink": "/d/4ac4f123aae0ff6dbaf4f4f66120033b/k8s-node-rsrc-use", - "step": 10 + "legendFormat": "{{instance}} {{name}}", + "refId": "A" } ], "thresholds": [ @@ -259,7 +387,7 @@ data: ], "timeFrom": null, "timeShift": null, - "title": "Memory Utilisation", + "title": "Work Queue Add Rate", "tooltip": { "shared": false, "sort": 0, @@ -277,20 +405,20 @@ data: }, "yaxes": [ { - "format": "percentunit", + "format": "ops", "label": null, "logBase": 1, - "max": 1, + "max": null, "min": 0, "show": true }, { - "format": "short", + "format": "ops", "label": null, "logBase": 1, "max": null, - "min": null, - "show": false + "min": 0, + "show": true } ] }, @@ -302,42 +430,47 @@ data: "dashLength": 10, "dashes": false, "datasource": "$datasource", - "fill": 10, - "id": 4, + "fill": 1, + "gridPos": { + + }, + "id": 6, "legend": { + "alignAsTable": false, "avg": false, "current": false, "max": false, "min": false, - "show": true, + "rightSide": false, + "show": false, "total": false, "values": false }, "lines": true, - "linewidth": 0, + "linewidth": 1, "links": [ ], - "nullPointMode": "null as zero", + "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", + "repeat": null, "seriesOverrides": [ ], "spaceLength": 10, "span": 6, - "stack": true, + "stack": false, "steppedLine": false, "targets": [ { - "expr": "node:node_memory_swap_io_bytes:sum_rate{cluster=\"$cluster\"}", + "expr": "sum(rate(workqueue_depth{job=\"apiserver\", instance=~\"$instance\"}[5m])) by (instance, name)", "format": "time_series", "intervalFactor": 2, - "legendFormat": "{{node}}", - "legendLink": "/d/4ac4f123aae0ff6dbaf4f4f66120033b/k8s-node-rsrc-use", - "step": 10 + "legendFormat": "{{instance}} {{name}}", + "refId": "A" } ], "thresholds": [ @@ -345,7 +478,7 @@ data: ], "timeFrom": null, "timeShift": null, - "title": "Memory Saturation (Swap I/O)", + "title": "Work Queue Depth", "tooltip": { "shared": false, "sort": 0, @@ -363,7 +496,7 @@ data: }, "yaxes": [ { - "format": "Bps", + "format": "short", "label": null, "logBase": 1, "max": null, @@ -375,23 +508,11 @@ data: "label": null, "logBase": 1, "max": null, - "min": null, - "show": false + "min": 0, + "show": true } ] - } - ], - "repeat": null, - "repeatIteration": null, - "repeatRowId": null, - "showTitle": true, - "title": "Memory", - "titleSize": "h6" - }, - { - "collapse": false, - "height": "250px", - "panels": [ + }, { "aliasColors": { @@ -400,42 +521,47 @@ data: "dashLength": 10, "dashes": false, "datasource": "$datasource", - "fill": 10, - "id": 5, + "fill": 1, + "gridPos": { + + }, + "id": 7, "legend": { + "alignAsTable": "true", "avg": false, - "current": false, + "current": "true", "max": false, "min": false, - "show": true, + "rightSide": "true", + "show": "true", "total": false, - "values": false + "values": "true" }, "lines": true, - "linewidth": 0, + "linewidth": 1, "links": [ ], - "nullPointMode": "null as zero", + "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", + "repeat": null, "seriesOverrides": [ ], "spaceLength": 10, - "span": 6, - "stack": true, + "span": 12, + "stack": false, "steppedLine": false, "targets": [ { - "expr": "node:node_disk_utilisation:avg_irate{cluster=\"$cluster\"} / scalar(:kube_pod_info_node_count:{cluster=\"$cluster\"})", + "expr": "histogram_quantile(0.99, sum(rate(workqueue_queue_duration_seconds_bucket{job=\"apiserver\", instance=~\"$instance\"}[5m])) by (instance, name, le))", "format": "time_series", "intervalFactor": 2, - "legendFormat": "{{node}}", - "legendLink": "/d/4ac4f123aae0ff6dbaf4f4f66120033b/k8s-node-rsrc-use", - "step": 10 + "legendFormat": "{{instance}} {{name}}", + "refId": "A" } ], "thresholds": [ @@ -443,7 +569,7 @@ data: ], "timeFrom": null, "timeShift": null, - "title": "Disk IO Utilisation", + "title": "Work Queue Latency", "tooltip": { "shared": false, "sort": 0, @@ -461,23 +587,36 @@ data: }, "yaxes": [ { - "format": "percentunit", + "format": "s", "label": null, "logBase": 1, - "max": 1, - "min": 0, + "max": null, + "min": null, "show": true }, { - "format": "short", + "format": "s", "label": null, "logBase": 1, "max": null, "min": null, - "show": false + "show": true } ] - }, + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ { "aliasColors": { @@ -486,42 +625,47 @@ data: "dashLength": 10, "dashes": false, "datasource": "$datasource", - "fill": 10, - "id": 6, + "fill": 1, + "gridPos": { + + }, + "id": 8, "legend": { + "alignAsTable": false, "avg": false, "current": false, "max": false, "min": false, + "rightSide": false, "show": true, "total": false, "values": false }, "lines": true, - "linewidth": 0, + "linewidth": 1, "links": [ ], - "nullPointMode": "null as zero", + "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", + "repeat": null, "seriesOverrides": [ ], "spaceLength": 10, - "span": 6, - "stack": true, + "span": 4, + "stack": false, "steppedLine": false, "targets": [ { - "expr": "node:node_disk_saturation:avg_irate{cluster=\"$cluster\"} / scalar(:kube_pod_info_node_count:{cluster=\"$cluster\"})", + "expr": "etcd_helper_cache_entry_total{job=\"apiserver\", instance=~\"$instance\"}", "format": "time_series", "intervalFactor": 2, - "legendFormat": "{{node}}", - "legendLink": "/d/4ac4f123aae0ff6dbaf4f4f66120033b/k8s-node-rsrc-use", - "step": 10 + "legendFormat": "{{instance}}", + "refId": "A" } ], "thresholds": [ @@ -529,7 +673,7 @@ data: ], "timeFrom": null, "timeShift": null, - "title": "Disk IO Saturation", + "title": "ETCD Cache Entry Total", "tooltip": { "shared": false, "sort": 0, @@ -547,10 +691,10 @@ data: }, "yaxes": [ { - "format": "percentunit", + "format": "short", "label": null, "logBase": 1, - "max": 1, + "max": null, "min": 0, "show": true }, @@ -559,23 +703,11 @@ data: "label": null, "logBase": 1, "max": null, - "min": null, - "show": false + "min": 0, + "show": true } ] - } - ], - "repeat": null, - "repeatIteration": null, - "repeatRowId": null, - "showTitle": true, - "title": "Disk", - "titleSize": "h6" - }, - { - "collapse": false, - "height": "250px", - "panels": [ + }, { "aliasColors": { @@ -584,42 +716,54 @@ data: "dashLength": 10, "dashes": false, "datasource": "$datasource", - "fill": 10, - "id": 7, + "fill": 1, + "gridPos": { + + }, + "id": 9, "legend": { + "alignAsTable": false, "avg": false, "current": false, "max": false, "min": false, + "rightSide": false, "show": true, "total": false, "values": false }, "lines": true, - "linewidth": 0, + "linewidth": 1, "links": [ ], - "nullPointMode": "null as zero", + "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", + "repeat": null, "seriesOverrides": [ ], "spaceLength": 10, - "span": 6, - "stack": true, + "span": 4, + "stack": false, "steppedLine": false, "targets": [ { - "expr": "node:node_net_utilisation:sum_irate{cluster=\"$cluster\"}", + "expr": "sum(rate(etcd_helper_cache_hit_total{job=\"apiserver\",instance=~\"$instance\"}[5m])) by (intance)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{instance}} hit", + "refId": "A" + }, + { + "expr": "sum(rate(etcd_helper_cache_miss_total{job=\"apiserver\",instance=~\"$instance\"}[5m])) by (instance)", "format": "time_series", "intervalFactor": 2, - "legendFormat": "{{node}}", - "legendLink": "/d/4ac4f123aae0ff6dbaf4f4f66120033b/k8s-node-rsrc-use", - "step": 10 + "legendFormat": "{{instance}} miss", + "refId": "B" } ], "thresholds": [ @@ -627,7 +771,7 @@ data: ], "timeFrom": null, "timeShift": null, - "title": "Net Utilisation (Transmitted)", + "title": "ETCD Cache Hit/Miss Rate", "tooltip": { "shared": false, "sort": 0, @@ -645,7 +789,7 @@ data: }, "yaxes": [ { - "format": "Bps", + "format": "ops", "label": null, "logBase": 1, "max": null, @@ -653,12 +797,12 @@ data: "show": true }, { - "format": "short", + "format": "ops", "label": null, "logBase": 1, "max": null, - "min": null, - "show": false + "min": 0, + "show": true } ] }, @@ -670,42 +814,54 @@ data: "dashLength": 10, "dashes": false, "datasource": "$datasource", - "fill": 10, - "id": 8, + "fill": 1, + "gridPos": { + + }, + "id": 10, "legend": { + "alignAsTable": false, "avg": false, "current": false, "max": false, "min": false, + "rightSide": false, "show": true, "total": false, "values": false }, "lines": true, - "linewidth": 0, + "linewidth": 1, "links": [ ], - "nullPointMode": "null as zero", + "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", + "repeat": null, "seriesOverrides": [ ], "spaceLength": 10, - "span": 6, - "stack": true, + "span": 4, + "stack": false, "steppedLine": false, "targets": [ { - "expr": "node:node_net_saturation:sum_irate{cluster=\"$cluster\"}", + "expr": "histogram_quantile(0.99,sum(rate(etcd_request_cache_get_duration_seconds_bucket{job=\"apiserver\",instance=~\"$instance\"}[5m])) by (instance, le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{instance}} get", + "refId": "A" + }, + { + "expr": "histogram_quantile(0.99,sum(rate(etcd_request_cache_add_duration_seconds_bucket{job=\"apiserver\",instance=~\"$instance\"}[5m])) by (instance, le))", "format": "time_series", "intervalFactor": 2, - "legendFormat": "{{node}}", - "legendLink": "/d/4ac4f123aae0ff6dbaf4f4f66120033b/k8s-node-rsrc-use", - "step": 10 + "legendFormat": "{{instance}} miss", + "refId": "B" } ], "thresholds": [ @@ -713,7 +869,7 @@ data: ], "timeFrom": null, "timeShift": null, - "title": "Net Saturation (Dropped)", + "title": "ETCD Cache Duration 99th Quantile", "tooltip": { "shared": false, "sort": 0, @@ -731,7 +887,7 @@ data: }, "yaxes": [ { - "format": "Bps", + "format": "s", "label": null, "logBase": 1, "max": null, @@ -739,12 +895,12 @@ data: "show": true }, { - "format": "short", + "format": "s", "label": null, "logBase": 1, "max": null, - "min": null, - "show": false + "min": 0, + "show": true } ] } @@ -752,13 +908,14 @@ data: "repeat": null, "repeatIteration": null, "repeatRowId": null, - "showTitle": true, - "title": "Network", - "titleSize": "h6" + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" }, { "collapse": false, - "height": "250px", + "collapsed": false, "panels": [ { "aliasColors": { @@ -768,42 +925,47 @@ data: "dashLength": 10, "dashes": false, "datasource": "$datasource", - "fill": 10, - "id": 9, + "fill": 1, + "gridPos": { + + }, + "id": 11, "legend": { + "alignAsTable": false, "avg": false, "current": false, "max": false, "min": false, + "rightSide": false, "show": true, "total": false, "values": false }, "lines": true, - "linewidth": 0, + "linewidth": 1, "links": [ ], - "nullPointMode": "null as zero", + "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", + "repeat": null, "seriesOverrides": [ ], "spaceLength": 10, - "span": 12, - "stack": true, + "span": 4, + "stack": false, "steppedLine": false, "targets": [ { - "expr": "sum(max(node_filesystem_size_bytes{fstype=~\"ext[234]|btrfs|xfs|zfs\", cluster=\"$cluster\"} - node_filesystem_avail_bytes{fstype=~\"ext[234]|btrfs|xfs|zfs\", cluster=\"$cluster\"}) by (device,pod,namespace)) by (pod,namespace)\n/ scalar(sum(max(node_filesystem_size_bytes{fstype=~\"ext[234]|btrfs|xfs|zfs\", cluster=\"$cluster\"}) by (device,pod,namespace)))\n* on (namespace, pod) group_left (node) node_namespace_pod:kube_pod_info:{cluster=\"$cluster\"}\n", + "expr": "process_resident_memory_bytes{job=\"apiserver\",instance=~\"$instance\"}", "format": "time_series", "intervalFactor": 2, - "legendFormat": "{{node}}", - "legendLink": "/d/4ac4f123aae0ff6dbaf4f4f66120033b/k8s-node-rsrc-use", - "step": 10 + "legendFormat": "{{instance}}", + "refId": "A" } ], "thresholds": [ @@ -811,7 +973,7 @@ data: ], "timeFrom": null, "timeShift": null, - "title": "Disk Capacity", + "title": "Memory", "tooltip": { "shared": false, "sort": 0, @@ -829,138 +991,23 @@ data: }, "yaxes": [ { - "format": "percentunit", + "format": "bytes", "label": null, "logBase": 1, - "max": 1, - "min": 0, + "max": null, + "min": null, "show": true }, { - "format": "short", + "format": "bytes", "label": null, "logBase": 1, "max": null, "min": null, - "show": false + "show": true } ] - } - ], - "repeat": null, - "repeatIteration": null, - "repeatRowId": null, - "showTitle": true, - "title": "Storage", - "titleSize": "h6" - } - ], - "schemaVersion": 14, - "style": "dark", - "tags": [ - "kubernetes-mixin" - ], - "templating": { - "list": [ - { - "current": { - "text": "Prometheus", - "value": "Prometheus" - }, - "hide": 0, - "label": null, - "name": "datasource", - "options": [ - - ], - "query": "prometheus", - "refresh": 1, - "regex": "", - "type": "datasource" - }, - { - "allValue": null, - "current": { - "text": "prod", - "value": "prod" }, - "datasource": "$datasource", - "hide": 2, - "includeAll": false, - "label": "cluster", - "multi": false, - "name": "cluster", - "options": [ - - ], - "query": "label_values(kube_node_info, cluster)", - "refresh": 1, - "regex": "", - "sort": 2, - "tagValuesQuery": "", - "tags": [ - - ], - "tagsQuery": "", - "type": "query", - "useTags": false - } - ] - }, - "time": { - "from": "now-1h", - "to": "now" - }, - "timepicker": { - "refresh_intervals": [ - "5s", - "10s", - "30s", - "1m", - "5m", - "15m", - "30m", - "1h", - "2h", - "1d" - ], - "time_options": [ - "5m", - "15m", - "1h", - "6h", - "12h", - "24h", - "2d", - "7d", - "30d" - ] - }, - "timezone": "", - "title": "Kubernetes / USE Method / Cluster", - "uid": "a6e7d1362e1ddbb79db21d5bb40d7137", - "version": 0 - } - k8s-node-rsrc-use.json: |- - { - "annotations": { - "list": [ - - ] - }, - "editable": true, - "gnetId": null, - "graphTooltip": 0, - "hideControls": false, - "links": [ - - ], - "refresh": "10s", - "rows": [ - { - "collapse": false, - "height": "250px", - "panels": [ { "aliasColors": { @@ -970,12 +1017,17 @@ data: "dashes": false, "datasource": "$datasource", "fill": 1, - "id": 1, + "gridPos": { + + }, + "id": 12, "legend": { + "alignAsTable": false, "avg": false, "current": false, "max": false, "min": false, + "rightSide": false, "show": true, "total": false, "values": false @@ -985,26 +1037,26 @@ data: "links": [ ], - "nullPointMode": "null as zero", + "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", + "repeat": null, "seriesOverrides": [ ], "spaceLength": 10, - "span": 6, + "span": 4, "stack": false, "steppedLine": false, "targets": [ { - "expr": "node:node_cpu_utilisation:avg1m{cluster=\"$cluster\", node=\"$node\"}", + "expr": "rate(process_cpu_seconds_total{job=\"apiserver\",instance=~\"$instance\"}[5m])", "format": "time_series", "intervalFactor": 2, - "legendFormat": "Utilisation", - "legendLink": null, - "step": 10 + "legendFormat": "{{instance}}", + "refId": "A" } ], "thresholds": [ @@ -1012,7 +1064,7 @@ data: ], "timeFrom": null, "timeShift": null, - "title": "CPU Utilisation", + "title": "CPU usage", "tooltip": { "shared": false, "sort": 0, @@ -1030,7 +1082,7 @@ data: }, "yaxes": [ { - "format": "percentunit", + "format": "short", "label": null, "logBase": 1, "max": null, @@ -1042,8 +1094,8 @@ data: "label": null, "logBase": 1, "max": null, - "min": null, - "show": false + "min": 0, + "show": true } ] }, @@ -1056,12 +1108,17 @@ data: "dashes": false, "datasource": "$datasource", "fill": 1, - "id": 2, + "gridPos": { + + }, + "id": 13, "legend": { + "alignAsTable": false, "avg": false, "current": false, "max": false, "min": false, + "rightSide": false, "show": true, "total": false, "values": false @@ -1071,26 +1128,26 @@ data: "links": [ ], - "nullPointMode": "null as zero", + "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", + "repeat": null, "seriesOverrides": [ ], "spaceLength": 10, - "span": 6, + "span": 4, "stack": false, "steppedLine": false, "targets": [ { - "expr": "node:node_cpu_saturation_load1:{cluster=\"$cluster\", node=\"$node\"}", + "expr": "go_goroutines{job=\"apiserver\",instance=~\"$instance\"}", "format": "time_series", "intervalFactor": 2, - "legendFormat": "Saturation", - "legendLink": null, - "step": 10 + "legendFormat": "{{instance}}", + "refId": "A" } ], "thresholds": [ @@ -1098,7 +1155,7 @@ data: ], "timeFrom": null, "timeShift": null, - "title": "CPU Saturation (Load1)", + "title": "Goroutines", "tooltip": { "shared": false, "sort": 0, @@ -1116,11 +1173,11 @@ data: }, "yaxes": [ { - "format": "percentunit", + "format": "short", "label": null, "logBase": 1, "max": null, - "min": 0, + "min": null, "show": true }, { @@ -1129,7 +1186,7 @@ data: "logBase": 1, "max": null, "min": null, - "show": false + "show": true } ] } @@ -1137,99 +1194,207 @@ data: "repeat": null, "repeatIteration": null, "repeatRowId": null, - "showTitle": true, - "title": "CPU", - "titleSize": "h6" - }, - { - "collapse": false, - "height": "250px", - "panels": [ - { - "aliasColors": { - - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 1, - "id": 3, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + } + ], + "schemaVersion": 14, + "style": "dark", + "tags": [ + "kubernetes-mixin" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": null, + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": true, + "label": null, + "multi": false, + "name": "instance", + "options": [ + + ], + "query": "label_values(apiserver_request_total{job=\"apiserver\"}, instance)", + "refresh": 2, + "regex": "", + "sort": 0, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "", + "title": "Kubernetes / API server", + "uid": "09ec8aa1e996d6ffcd6817bbaff4db1b", + "version": 0 + } + controller-manager.json: |- + { + "__inputs": [ + + ], + "__requires": [ + + ], + "annotations": { + "list": [ + + ] + }, + "editable": false, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "id": null, + "links": [ + + ], + "refresh": "", + "rows": [ + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "$datasource", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true }, - "lines": true, - "linewidth": 1, + "gridPos": { + + }, + "id": 2, + "interval": null, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } ], - "spaceLength": 10, - "span": 6, - "stack": false, - "steppedLine": false, + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "span": 2, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", "targets": [ { - "expr": "node:node_memory_utilisation:{cluster=\"$cluster\", node=\"$node\"}", + "expr": "sum(up{job=\"kube-controller-manager\"})", "format": "time_series", "intervalFactor": 2, - "legendFormat": "Memory", - "legendLink": null, - "step": 10 + "legendFormat": "", + "refId": "A" } ], - "thresholds": [ - - ], - "timeFrom": null, - "timeShift": null, - "title": "Memory Utilisation", + "thresholds": "", + "title": "Up", "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ - - ] + "shared": false }, - "yaxes": [ - { - "format": "percentunit", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false + "op": "=", + "text": "N/A", + "value": "null" } - ] + ], + "valueName": "min" }, { "aliasColors": { @@ -1240,41 +1405,46 @@ data: "dashes": false, "datasource": "$datasource", "fill": 1, - "id": 4, + "gridPos": { + + }, + "id": 3, "legend": { + "alignAsTable": "true", "avg": false, - "current": false, + "current": "true", "max": false, "min": false, - "show": true, + "rightSide": "true", + "show": "true", "total": false, - "values": false + "values": "true" }, "lines": true, "linewidth": 1, "links": [ ], - "nullPointMode": "null as zero", + "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", + "repeat": null, "seriesOverrides": [ ], "spaceLength": 10, - "span": 6, + "span": 10, "stack": false, "steppedLine": false, "targets": [ { - "expr": "node:node_memory_swap_io_bytes:sum_rate{cluster=\"$cluster\", node=\"$node\"}", + "expr": "sum(rate(workqueue_adds_total{job=\"kube-controller-manager\", instance=~\"$instance\"}[5m])) by (instance, name)", "format": "time_series", "intervalFactor": 2, - "legendFormat": "Swap IO", - "legendLink": null, - "step": 10 + "legendFormat": "{{instance}} {{name}}", + "refId": "A" } ], "thresholds": [ @@ -1282,7 +1452,7 @@ data: ], "timeFrom": null, "timeShift": null, - "title": "Memory Saturation (Swap I/O)", + "title": "Work Queue Add Rate", "tooltip": { "shared": false, "sort": 0, @@ -1300,20 +1470,20 @@ data: }, "yaxes": [ { - "format": "Bps", + "format": "ops", "label": null, "logBase": 1, "max": null, - "min": 0, + "min": null, "show": true }, { - "format": "short", + "format": "ops", "label": null, "logBase": 1, "max": null, "min": null, - "show": false + "show": true } ] } @@ -1321,13 +1491,14 @@ data: "repeat": null, "repeatIteration": null, "repeatRowId": null, - "showTitle": true, - "title": "Memory", - "titleSize": "h6" + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" }, { "collapse": false, - "height": "250px", + "collapsed": false, "panels": [ { "aliasColors": { @@ -1338,41 +1509,46 @@ data: "dashes": false, "datasource": "$datasource", "fill": 1, - "id": 5, + "gridPos": { + + }, + "id": 4, "legend": { + "alignAsTable": "true", "avg": false, - "current": false, + "current": "true", "max": false, "min": false, - "show": true, + "rightSide": "true", + "show": "true", "total": false, - "values": false + "values": "true" }, "lines": true, "linewidth": 1, "links": [ ], - "nullPointMode": "null as zero", + "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", + "repeat": null, "seriesOverrides": [ ], "spaceLength": 10, - "span": 6, + "span": 12, "stack": false, "steppedLine": false, "targets": [ { - "expr": "node:node_disk_utilisation:avg_irate{cluster=\"$cluster\", node=\"$node\"}", + "expr": "sum(rate(workqueue_depth{job=\"kube-controller-manager\", instance=~\"$instance\"}[5m])) by (instance, name)", "format": "time_series", "intervalFactor": 2, - "legendFormat": "Utilisation", - "legendLink": null, - "step": 10 + "legendFormat": "{{instance}} {{name}}", + "refId": "A" } ], "thresholds": [ @@ -1380,7 +1556,7 @@ data: ], "timeFrom": null, "timeShift": null, - "title": "Disk IO Utilisation", + "title": "Work Queue Depth", "tooltip": { "shared": false, "sort": 0, @@ -1398,7 +1574,7 @@ data: }, "yaxes": [ { - "format": "percentunit", + "format": "short", "label": null, "logBase": 1, "max": null, @@ -1410,11 +1586,24 @@ data: "label": null, "logBase": 1, "max": null, - "min": null, - "show": false + "min": 0, + "show": true } ] - }, + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ { "aliasColors": { @@ -1424,41 +1613,46 @@ data: "dashes": false, "datasource": "$datasource", "fill": 1, - "id": 6, + "gridPos": { + + }, + "id": 5, "legend": { + "alignAsTable": "true", "avg": false, - "current": false, + "current": "true", "max": false, "min": false, - "show": true, + "rightSide": "true", + "show": "true", "total": false, - "values": false + "values": "true" }, "lines": true, "linewidth": 1, "links": [ ], - "nullPointMode": "null as zero", + "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", + "repeat": null, "seriesOverrides": [ ], "spaceLength": 10, - "span": 6, + "span": 12, "stack": false, "steppedLine": false, "targets": [ { - "expr": "node:node_disk_saturation:avg_irate{cluster=\"$cluster\", node=\"$node\"}", + "expr": "histogram_quantile(0.99, sum(rate(workqueue_queue_duration_seconds_bucket{job=\"kube-controller-manager\", instance=~\"$instance\"}[5m])) by (instance, name, le))", "format": "time_series", "intervalFactor": 2, - "legendFormat": "Saturation", - "legendLink": null, - "step": 10 + "legendFormat": "{{instance}} {{name}}", + "refId": "A" } ], "thresholds": [ @@ -1466,7 +1660,7 @@ data: ], "timeFrom": null, "timeShift": null, - "title": "Disk IO Saturation", + "title": "Work Queue Latency", "tooltip": { "shared": false, "sort": 0, @@ -1484,20 +1678,20 @@ data: }, "yaxes": [ { - "format": "percentunit", + "format": "s", "label": null, "logBase": 1, "max": null, - "min": 0, + "min": null, "show": true }, { - "format": "short", + "format": "s", "label": null, "logBase": 1, "max": null, "min": null, - "show": false + "show": true } ] } @@ -1505,13 +1699,14 @@ data: "repeat": null, "repeatIteration": null, "repeatRowId": null, - "showTitle": true, - "title": "Disk", - "titleSize": "h6" + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" }, { "collapse": false, - "height": "250px", + "collapsed": false, "panels": [ { "aliasColors": { @@ -1522,12 +1717,17 @@ data: "dashes": false, "datasource": "$datasource", "fill": 1, - "id": 7, + "gridPos": { + + }, + "id": 6, "legend": { + "alignAsTable": false, "avg": false, "current": false, "max": false, "min": false, + "rightSide": false, "show": true, "total": false, "values": false @@ -1537,26 +1737,47 @@ data: "links": [ ], - "nullPointMode": "null as zero", + "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", + "repeat": null, "seriesOverrides": [ ], "spaceLength": 10, - "span": 6, + "span": 4, "stack": false, "steppedLine": false, "targets": [ { - "expr": "node:node_net_utilisation:sum_irate{cluster=\"$cluster\", node=\"$node\"}", + "expr": "sum(rate(rest_client_requests_total{job=\"kube-controller-manager\", instance=~\"$instance\",code=~\"2..\"}[5m]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "2xx", + "refId": "A" + }, + { + "expr": "sum(rate(rest_client_requests_total{job=\"kube-controller-manager\", instance=~\"$instance\",code=~\"3..\"}[5m]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "3xx", + "refId": "B" + }, + { + "expr": "sum(rate(rest_client_requests_total{job=\"kube-controller-manager\", instance=~\"$instance\",code=~\"4..\"}[5m]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "4xx", + "refId": "C" + }, + { + "expr": "sum(rate(rest_client_requests_total{job=\"kube-controller-manager\", instance=~\"$instance\",code=~\"5..\"}[5m]))", "format": "time_series", "intervalFactor": 2, - "legendFormat": "Utilisation", - "legendLink": null, - "step": 10 + "legendFormat": "5xx", + "refId": "D" } ], "thresholds": [ @@ -1564,7 +1785,7 @@ data: ], "timeFrom": null, "timeShift": null, - "title": "Net Utilisation (Transmitted)", + "title": "Kube API Request Rate", "tooltip": { "shared": false, "sort": 0, @@ -1582,20 +1803,20 @@ data: }, "yaxes": [ { - "format": "Bps", + "format": "ops", "label": null, "logBase": 1, "max": null, - "min": 0, + "min": null, "show": true }, { - "format": "short", + "format": "ops", "label": null, "logBase": 1, "max": null, "min": null, - "show": false + "show": true } ] }, @@ -1608,12 +1829,17 @@ data: "dashes": false, "datasource": "$datasource", "fill": 1, - "id": 8, + "gridPos": { + + }, + "id": 7, "legend": { + "alignAsTable": false, "avg": false, "current": false, "max": false, "min": false, + "rightSide": false, "show": true, "total": false, "values": false @@ -1623,26 +1849,26 @@ data: "links": [ ], - "nullPointMode": "null as zero", + "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", + "repeat": null, "seriesOverrides": [ ], "spaceLength": 10, - "span": 6, + "span": 8, "stack": false, "steppedLine": false, "targets": [ { - "expr": "node:node_net_saturation:sum_irate{cluster=\"$cluster\", node=\"$node\"}", + "expr": "histogram_quantile(0.99, sum(rate(rest_client_request_latency_seconds_bucket{job=\"kube-controller-manager\", instance=~\"$instance\", verb=\"POST\"}[5m])) by (verb, url, le))", "format": "time_series", "intervalFactor": 2, - "legendFormat": "Saturation", - "legendLink": null, - "step": 10 + "legendFormat": "{{verb}} {{url}}", + "refId": "A" } ], "thresholds": [ @@ -1650,7 +1876,7 @@ data: ], "timeFrom": null, "timeShift": null, - "title": "Net Saturation (Dropped)", + "title": "Post Request Latency 99th Quantile", "tooltip": { "shared": false, "sort": 0, @@ -1668,7 +1894,7 @@ data: }, "yaxes": [ { - "format": "Bps", + "format": "s", "label": null, "logBase": 1, "max": null, @@ -1676,12 +1902,12 @@ data: "show": true }, { - "format": "short", + "format": "s", "label": null, "logBase": 1, "max": null, - "min": null, - "show": false + "min": 0, + "show": true } ] } @@ -1689,13 +1915,14 @@ data: "repeat": null, "repeatIteration": null, "repeatRowId": null, - "showTitle": true, - "title": "Net", - "titleSize": "h6" + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" }, { "collapse": false, - "height": "250px", + "collapsed": false, "panels": [ { "aliasColors": { @@ -1706,26 +1933,32 @@ data: "dashes": false, "datasource": "$datasource", "fill": 1, - "id": 9, + "gridPos": { + + }, + "id": 8, "legend": { + "alignAsTable": "true", "avg": false, - "current": false, + "current": "true", "max": false, "min": false, - "show": true, + "rightSide": "true", + "show": "true", "total": false, - "values": false + "values": "true" }, "lines": true, "linewidth": 1, "links": [ ], - "nullPointMode": "null as zero", + "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", + "repeat": null, "seriesOverrides": [ ], @@ -1735,12 +1968,11 @@ data: "steppedLine": false, "targets": [ { - "expr": "node:node_filesystem_usage:{cluster=\"$cluster\"}\n* on (namespace, pod) group_left (node) node_namespace_pod:kube_pod_info:{cluster=\"$cluster\", node=\"$node\"}\n", + "expr": "histogram_quantile(0.99, sum(rate(rest_client_request_latency_seconds_bucket{job=\"kube-controller-manager\", instance=~\"$instance\", verb=\"GET\"}[5m])) by (verb, url, le))", "format": "time_series", "intervalFactor": 2, - "legendFormat": "{{device}}", - "legendLink": null, - "step": 10 + "legendFormat": "{{verb}} {{url}}", + "refId": "A" } ], "thresholds": [ @@ -1748,7 +1980,7 @@ data: ], "timeFrom": null, "timeShift": null, - "title": "Disk Utilisation", + "title": "Get Request Latency 99th Quantile", "tooltip": { "shared": false, "sort": 0, @@ -1766,7 +1998,7 @@ data: }, "yaxes": [ { - "format": "percentunit", + "format": "s", "label": null, "logBase": 1, "max": null, @@ -1774,12 +2006,12 @@ data: "show": true }, { - "format": "short", + "format": "s", "label": null, "logBase": 1, "max": null, - "min": null, - "show": false + "min": 0, + "show": true } ] } @@ -1787,151 +2019,106 @@ data: "repeat": null, "repeatIteration": null, "repeatRowId": null, - "showTitle": true, - "title": "Disk", - "titleSize": "h6" - } - ], - "schemaVersion": 14, - "style": "dark", - "tags": [ - "kubernetes-mixin" - ], - "templating": { - "list": [ - { - "current": { - "text": "Prometheus", - "value": "Prometheus" - }, - "hide": 0, - "label": null, - "name": "datasource", - "options": [ + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { - ], - "query": "prometheus", - "refresh": 1, - "regex": "", - "type": "datasource" - }, - { - "allValue": null, - "current": { - "text": "prod", - "value": "prod" - }, - "datasource": "$datasource", - "hide": 2, - "includeAll": false, - "label": "cluster", - "multi": false, - "name": "cluster", - "options": [ + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "gridPos": { - ], - "query": "label_values(kube_node_info, cluster)", - "refresh": 1, - "regex": "", - "sort": 2, - "tagValuesQuery": "", - "tags": [ + }, + "id": 9, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ - ], - "tagsQuery": "", - "type": "query", - "useTags": false - }, - { - "allValue": null, - "current": { - "text": "prod", - "value": "prod" - }, - "datasource": "$datasource", - "hide": 0, - "includeAll": false, - "label": "node", - "multi": false, - "name": "node", - "options": [ + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ - ], - "query": "label_values(kube_node_info{cluster=\"$cluster\"}, node)", - "refresh": 1, - "regex": "", - "sort": 2, - "tagValuesQuery": "", - "tags": [ + ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "process_resident_memory_bytes{job=\"kube-controller-manager\",instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{instance}}", + "refId": "A" + } + ], + "thresholds": [ - ], - "tagsQuery": "", - "type": "query", - "useTags": false - } - ] - }, - "time": { - "from": "now-1h", - "to": "now" - }, - "timepicker": { - "refresh_intervals": [ - "5s", - "10s", - "30s", - "1m", - "5m", - "15m", - "30m", - "1h", - "2h", - "1d" - ], - "time_options": [ - "5m", - "15m", - "1h", - "6h", - "12h", - "24h", - "2d", - "7d", - "30d" - ] - }, - "timezone": "", - "title": "Kubernetes / USE Method / Node", - "uid": "4ac4f123aae0ff6dbaf4f4f66120033b", - "version": 0 - } - nodes.json: |- - { - "__inputs": [ - - ], - "__requires": [ - - ], - "annotations": { - "list": [ - - ] - }, - "editable": false, - "gnetId": null, - "graphTooltip": 0, - "hideControls": false, - "id": null, - "links": [ + ], + "timeFrom": null, + "timeShift": null, + "title": "Memory", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ - ], - "refresh": "", - "rows": [ - { - "collapse": false, - "collapsed": false, - "panels": [ + ] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, { "aliasColors": { @@ -1944,7 +2131,7 @@ data: "gridPos": { }, - "id": 2, + "id": 10, "legend": { "alignAsTable": false, "avg": false, @@ -1971,37 +2158,16 @@ data: ], "spaceLength": 10, - "span": 6, + "span": 4, "stack": false, "steppedLine": false, "targets": [ { - "expr": "max(node_load1{cluster=\"$cluster\", job=\"node-exporter\", instance=\"$instance\"})", + "expr": "rate(process_cpu_seconds_total{job=\"kube-controller-manager\",instance=~\"$instance\"}[5m])", "format": "time_series", "intervalFactor": 2, - "legendFormat": "load 1m", + "legendFormat": "{{instance}}", "refId": "A" - }, - { - "expr": "max(node_load5{cluster=\"$cluster\", job=\"node-exporter\", instance=\"$instance\"})", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "load 5m", - "refId": "B" - }, - { - "expr": "max(node_load15{cluster=\"$cluster\", job=\"node-exporter\", instance=\"$instance\"})", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "load 15m", - "refId": "C" - }, - { - "expr": "count(node_cpu_seconds_total{cluster=\"$cluster\", job=\"node-exporter\", instance=\"$instance\", mode=\"user\"})", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "logical cores", - "refId": "D" } ], "thresholds": [ @@ -2009,7 +2175,7 @@ data: ], "timeFrom": null, "timeShift": null, - "title": "System load", + "title": "CPU usage", "tooltip": { "shared": false, "sort": 0, @@ -2031,7 +2197,7 @@ data: "label": null, "logBase": 1, "max": null, - "min": null, + "min": 0, "show": true }, { @@ -2039,7 +2205,7 @@ data: "label": null, "logBase": 1, "max": null, - "min": null, + "min": 0, "show": true } ] @@ -2056,7 +2222,7 @@ data: "gridPos": { }, - "id": 3, + "id": 11, "legend": { "alignAsTable": false, "avg": false, @@ -2083,15 +2249,15 @@ data: ], "spaceLength": 10, - "span": 6, + "span": 4, "stack": false, "steppedLine": false, "targets": [ { - "expr": "sum by (cpu) (irate(node_cpu_seconds_total{cluster=\"$cluster\", job=\"node-exporter\", mode!=\"idle\", instance=\"$instance\"}[5m]))", + "expr": "go_goroutines{job=\"kube-controller-manager\",instance=~\"$instance\"}", "format": "time_series", "intervalFactor": 2, - "legendFormat": "{{cpu}}", + "legendFormat": "{{instance}}", "refId": "A" } ], @@ -2100,7 +2266,7 @@ data: ], "timeFrom": null, "timeShift": null, - "title": "Usage Per Core", + "title": "Goroutines", "tooltip": { "shared": false, "sort": 0, @@ -2118,7 +2284,7 @@ data: }, "yaxes": [ { - "format": "percentunit", + "format": "short", "label": null, "logBase": 1, "max": null, @@ -2126,7 +2292,7 @@ data: "show": true }, { - "format": "percentunit", + "format": "short", "label": null, "logBase": 1, "max": null, @@ -2143,7 +2309,116 @@ data: "title": "Dashboard Row", "titleSize": "h6", "type": "row" - }, + } + ], + "schemaVersion": 14, + "style": "dark", + "tags": [ + "kubernetes-mixin" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": null, + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": true, + "label": null, + "multi": false, + "name": "instance", + "options": [ + + ], + "query": "label_values(process_cpu_seconds_total{job=\"kube-controller-manager\"}, instance)", + "refresh": 2, + "regex": "", + "sort": 0, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "", + "title": "Kubernetes / Controller Manager", + "uid": "72e0e05bef5099e5f049b05fdc429ed4", + "version": 0 + } + persistentvolumesusage.json: |- + { + "__inputs": [ + + ], + "__requires": [ + + ], + "annotations": { + "list": [ + + ] + }, + "editable": false, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "id": null, + "links": [ + + ], + "refresh": "", + "rows": [ { "collapse": false, "collapsed": false, @@ -2160,17 +2435,17 @@ data: "gridPos": { }, - "id": 4, + "id": 2, "legend": { - "alignAsTable": "true", - "avg": "true", - "current": "true", - "max": "false", - "min": "false", - "rightSide": "true", - "show": "true", - "total": "false", - "values": "true" + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "total": false, + "values": true }, "lines": true, "linewidth": 1, @@ -2188,15 +2463,22 @@ data: ], "spaceLength": 10, "span": 9, - "stack": false, + "stack": true, "steppedLine": false, "targets": [ { - "expr": "max (sum by (cpu) (irate(node_cpu_seconds_total{cluster=\"$cluster\", job=\"node-exporter\", mode!=\"idle\", instance=\"$instance\"}[2m])) ) * 100\n", + "expr": "(\n sum without(instance, node) (kubelet_volume_stats_capacity_bytes{cluster=\"$cluster\", job=\"kubelet\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"})\n -\n sum without(instance, node) (kubelet_volume_stats_available_bytes{cluster=\"$cluster\", job=\"kubelet\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"})\n)\n", "format": "time_series", - "intervalFactor": 10, - "legendFormat": "{{ cpu }}", + "intervalFactor": 1, + "legendFormat": "Used Space", "refId": "A" + }, + { + "expr": "sum without(instance, node) (kubelet_volume_stats_available_bytes{cluster=\"$cluster\", job=\"kubelet\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"})\n", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Free Space", + "refId": "B" } ], "thresholds": [ @@ -2204,7 +2486,7 @@ data: ], "timeFrom": null, "timeShift": null, - "title": "CPU Utilization", + "title": "Volume Space Usage", "tooltip": { "shared": false, "sort": 0, @@ -2222,18 +2504,18 @@ data: }, "yaxes": [ { - "format": "percent", + "format": "bytes", "label": null, "logBase": 1, - "max": 100, + "max": null, "min": 0, "show": true }, { - "format": "percent", + "format": "bytes", "label": null, "logBase": 1, - "max": 100, + "max": null, "min": 0, "show": true } @@ -2260,7 +2542,7 @@ data: "gridPos": { }, - "id": 5, + "id": 3, "interval": null, "links": [ @@ -2300,7 +2582,7 @@ data: "tableColumn": "", "targets": [ { - "expr": "avg(sum by (cpu) (irate(node_cpu_seconds_total{cluster=\"$cluster\", job=\"node-exporter\", mode!=\"idle\", instance=\"$instance\"}[2m]))) * 100\n", + "expr": "(\n kubelet_volume_stats_capacity_bytes{cluster=\"$cluster\", job=\"kubelet\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"}\n -\n kubelet_volume_stats_available_bytes{cluster=\"$cluster\", job=\"kubelet\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"}\n)\n/\nkubelet_volume_stats_capacity_bytes{cluster=\"$cluster\", job=\"kubelet\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"}\n* 100\n", "format": "time_series", "intervalFactor": 2, "legendFormat": "", @@ -2308,7 +2590,7 @@ data: } ], "thresholds": "80, 90", - "title": "CPU Usage", + "title": "Volume Space Usage", "tooltip": { "shared": false }, @@ -2348,17 +2630,17 @@ data: "gridPos": { }, - "id": 6, + "id": 4, "legend": { - "alignAsTable": false, - "avg": false, - "current": false, - "max": false, - "min": false, + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, "rightSide": false, "show": true, "total": false, - "values": false + "values": true }, "lines": true, "linewidth": 1, @@ -2376,36 +2658,22 @@ data: ], "spaceLength": 10, "span": 9, - "stack": false, + "stack": true, "steppedLine": false, "targets": [ { - "expr": "max(\n node_memory_MemTotal_bytes{cluster=\"$cluster\", job=\"node-exporter\", instance=\"$instance\"}\n - node_memory_MemFree_bytes{cluster=\"$cluster\", job=\"node-exporter\", instance=\"$instance\"}\n - node_memory_Buffers_bytes{cluster=\"$cluster\", job=\"node-exporter\", instance=\"$instance\"}\n - node_memory_Cached_bytes{cluster=\"$cluster\", job=\"node-exporter\", instance=\"$instance\"}\n)\n", + "expr": "sum without(instance, node) (kubelet_volume_stats_inodes_used{cluster=\"$cluster\", job=\"kubelet\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"})\n", "format": "time_series", - "intervalFactor": 2, - "legendFormat": "memory used", + "intervalFactor": 1, + "legendFormat": "Used inodes", "refId": "A" }, { - "expr": "max(node_memory_Buffers_bytes{cluster=\"$cluster\", job=\"node-exporter\", instance=\"$instance\"})", + "expr": "(\n sum without(instance, node) (kubelet_volume_stats_inodes{cluster=\"$cluster\", job=\"kubelet\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"})\n -\n sum without(instance, node) (kubelet_volume_stats_inodes_used{cluster=\"$cluster\", job=\"kubelet\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"})\n)\n", "format": "time_series", - "intervalFactor": 2, - "legendFormat": "memory buffers", + "intervalFactor": 1, + "legendFormat": " Free inodes", "refId": "B" - }, - { - "expr": "max(node_memory_Cached_bytes{cluster=\"$cluster\", job=\"node-exporter\", instance=\"$instance\"})", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "memory cached", - "refId": "C" - }, - { - "expr": "max(node_memory_MemFree_bytes{cluster=\"$cluster\", job=\"node-exporter\", instance=\"$instance\"})", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "memory free", - "refId": "D" } ], "thresholds": [ @@ -2413,7 +2681,7 @@ data: ], "timeFrom": null, "timeShift": null, - "title": "Memory Usage", + "title": "Volume inodes Usage", "tooltip": { "shared": false, "sort": 0, @@ -2431,19 +2699,19 @@ data: }, "yaxes": [ { - "format": "bytes", + "format": "none", "label": null, "logBase": 1, "max": null, - "min": null, + "min": 0, "show": true }, { - "format": "bytes", + "format": "none", "label": null, "logBase": 1, "max": null, - "min": null, + "min": 0, "show": true } ] @@ -2469,7 +2737,7 @@ data: "gridPos": { }, - "id": 7, + "id": 5, "interval": null, "links": [ @@ -2509,7 +2777,7 @@ data: "tableColumn": "", "targets": [ { - "expr": "max(\n (\n (\n node_memory_MemTotal_bytes{cluster=\"$cluster\", job=\"node-exporter\", instance=\"$instance\"}\n - node_memory_MemFree_bytes{cluster=\"$cluster\", job=\"node-exporter\", instance=\"$instance\"}\n - node_memory_Buffers_bytes{cluster=\"$cluster\", job=\"node-exporter\", instance=\"$instance\"}\n - node_memory_Cached_bytes{cluster=\"$cluster\", job=\"node-exporter\", instance=\"$instance\"}\n )\n / node_memory_MemTotal_bytes{cluster=\"$cluster\", job=\"node-exporter\", instance=\"$instance\"}\n ) * 100)\n", + "expr": "kubelet_volume_stats_inodes_used{cluster=\"$cluster\", job=\"kubelet\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"}\n/\nkubelet_volume_stats_inodes{cluster=\"$cluster\", job=\"kubelet\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"}\n* 100\n", "format": "time_series", "intervalFactor": 2, "legendFormat": "", @@ -2517,7 +2785,7 @@ data: } ], "thresholds": "80, 90", - "title": "Memory Usage", + "title": "Volume inodes Usage", "tooltip": { "shared": false }, @@ -2540,7 +2808,181 @@ data: "title": "Dashboard Row", "titleSize": "h6", "type": "row" - }, + } + ], + "schemaVersion": 14, + "style": "dark", + "tags": [ + "kubernetes-mixin" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": null, + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + + }, + "datasource": "$datasource", + "hide": 2, + "includeAll": false, + "label": "cluster", + "multi": false, + "name": "cluster", + "options": [ + + ], + "query": "label_values(kubelet_volume_stats_capacity_bytes, cluster)", + "refresh": 2, + "regex": "", + "sort": 0, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": "Namespace", + "multi": false, + "name": "namespace", + "options": [ + + ], + "query": "label_values(kubelet_volume_stats_capacity_bytes{cluster=\"$cluster\", job=\"kubelet\"}, namespace)", + "refresh": 2, + "regex": "", + "sort": 0, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": "PersistentVolumeClaim", + "multi": false, + "name": "volume", + "options": [ + + ], + "query": "label_values(kubelet_volume_stats_capacity_bytes{cluster=\"$cluster\", job=\"kubelet\", namespace=\"$namespace\"}, persistentvolumeclaim)", + "refresh": 2, + "regex": "", + "sort": 0, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-7d", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "", + "title": "Kubernetes / Persistent Volumes", + "uid": "919b92a8e8041bd567af9edab12c840c", + "version": 0 + } + pods.json: |- + { + "__inputs": [ + + ], + "__requires": [ + + ], + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "$datasource", + "enable": true, + "expr": "time() == BOOL timestamp(rate(kube_pod_container_status_restarts_total{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}[2m]) > 0)", + "hide": false, + "iconColor": "rgba(215, 44, 44, 1)", + "name": "Restarts", + "showIn": 0, + "tags": [ + "restart" + ], + "type": "rows" + } + ] + }, + "editable": false, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "id": null, + "links": [ + + ], + "refresh": "", + "rows": [ { "collapse": false, "collapsed": false, @@ -2557,14 +2999,14 @@ data: "gridPos": { }, - "id": 8, + "id": 2, "legend": { - "alignAsTable": false, - "avg": false, - "current": false, + "alignAsTable": true, + "avg": true, + "current": true, "max": false, "min": false, - "rightSide": false, + "rightSide": true, "show": true, "total": false, "values": false @@ -2581,131 +3023,40 @@ data: "renderer": "flot", "repeat": null, "seriesOverrides": [ - { - "alias": "read", - "yaxis": 1 - }, - { - "alias": "io time", - "yaxis": 2 - } + ], "spaceLength": 10, - "span": 6, + "span": 12, "stack": false, "steppedLine": false, "targets": [ { - "expr": "max(rate(node_disk_read_bytes_total{cluster=\"$cluster\", job=\"node-exporter\", instance=\"$instance\"}[2m]))", + "expr": "sum by(container) (container_memory_usage_bytes{job=\"kubernetes-cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\", container=~\"$container\", container!=\"POD\"})", "format": "time_series", "intervalFactor": 2, - "legendFormat": "read", + "legendFormat": "Current: {{ container }}", "refId": "A" }, { - "expr": "max(rate(node_disk_written_bytes_total{cluster=\"$cluster\", job=\"node-exporter\", instance=\"$instance\"}[2m]))", + "expr": "sum by(container) (kube_pod_container_resource_requests{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"memory\", pod=\"$pod\", container=~\"$container\"})", "format": "time_series", "intervalFactor": 2, - "legendFormat": "written", + "legendFormat": "Requested: {{ container }}", "refId": "B" }, { - "expr": "max(rate(node_disk_io_time_seconds_total{cluster=\"$cluster\", job=\"node-exporter\", instance=\"$instance\"}[2m]))", + "expr": "sum by(container) (kube_pod_container_resource_limits{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"memory\", pod=\"$pod\", container=~\"$container\"})", "format": "time_series", "intervalFactor": 2, - "legendFormat": "io time", + "legendFormat": "Limit: {{ container }}", "refId": "C" - } - ], - "thresholds": [ - - ], - "timeFrom": null, - "timeShift": null, - "title": "Disk I/O", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ - - ] - }, - "yaxes": [ - { - "format": "bytes", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true }, { - "format": "ms", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] - }, - { - "aliasColors": { - - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 1, - "gridPos": { - - }, - "id": 9, - "legend": { - "alignAsTable": false, - "avg": false, - "current": false, - "max": false, - "min": false, - "rightSide": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [ - - ], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "repeat": null, - "seriesOverrides": [ - - ], - "spaceLength": 10, - "span": 6, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "node:node_filesystem_usage:{cluster=\"$cluster\"}", + "expr": "sum by(container) (container_memory_cache{job=\"kubernetes-cadvisor\", namespace=\"$namespace\", pod=~\"$pod\", container=~\"$container\", container!=\"POD\"})", "format": "time_series", "intervalFactor": 2, - "legendFormat": "{{device}}", - "refId": "A" + "legendFormat": "Cache: {{ container }}", + "refId": "D" } ], "thresholds": [ @@ -2713,7 +3064,7 @@ data: ], "timeFrom": null, "timeShift": null, - "title": "Disk Space Usage", + "title": "Memory Usage", "tooltip": { "shared": false, "sort": 0, @@ -2731,19 +3082,19 @@ data: }, "yaxes": [ { - "format": "percentunit", + "format": "bytes", "label": null, "logBase": 1, "max": null, - "min": null, + "min": 0, "show": true }, { - "format": "percentunit", + "format": "bytes", "label": null, "logBase": 1, "max": null, - "min": null, + "min": 0, "show": true } ] @@ -2773,14 +3124,14 @@ data: "gridPos": { }, - "id": 10, + "id": 3, "legend": { - "alignAsTable": false, - "avg": false, - "current": false, + "alignAsTable": true, + "avg": true, + "current": true, "max": false, "min": false, - "rightSide": false, + "rightSide": true, "show": true, "total": false, "values": false @@ -2800,16 +3151,30 @@ data: ], "spaceLength": 10, - "span": 6, + "span": 12, "stack": false, "steppedLine": false, "targets": [ { - "expr": "max(rate(node_network_receive_bytes_total{cluster=\"$cluster\", job=\"node-exporter\", instance=\"$instance\", device!~\"lo\"}[5m]))", + "expr": "sum by (container) (irate(container_cpu_usage_seconds_total{job=\"kubernetes-cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\", image!=\"\", pod=\"$pod\", container=~\"$container\", container!=\"POD\"}[4m]))", "format": "time_series", "intervalFactor": 2, - "legendFormat": "{{device}}", + "legendFormat": "Current: {{ container }}", "refId": "A" + }, + { + "expr": "sum by(container) (kube_pod_container_resource_requests{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"cpu\", pod=\"$pod\", container=~\"$container\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Requested: {{ container }}", + "refId": "B" + }, + { + "expr": "sum by(container) (kube_pod_container_resource_limits{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"cpu\", pod=\"$pod\", container=~\"$container\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Limit: {{ container }}", + "refId": "C" } ], "thresholds": [ @@ -2817,7 +3182,7 @@ data: ], "timeFrom": null, "timeShift": null, - "title": "Network Received", + "title": "CPU Usage", "tooltip": { "shared": false, "sort": 0, @@ -2835,23 +3200,36 @@ data: }, "yaxes": [ { - "format": "bytes", + "format": "short", "label": null, "logBase": 1, "max": null, - "min": null, + "min": 0, "show": true }, { - "format": "bytes", + "format": "short", "label": null, "logBase": 1, "max": null, - "min": null, + "min": 0, "show": true } ] - }, + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ { "aliasColors": { @@ -2864,14 +3242,14 @@ data: "gridPos": { }, - "id": 11, + "id": 4, "legend": { - "alignAsTable": false, - "avg": false, - "current": false, + "alignAsTable": true, + "avg": true, + "current": true, "max": false, "min": false, - "rightSide": false, + "rightSide": true, "show": true, "total": false, "values": false @@ -2891,16 +3269,23 @@ data: ], "spaceLength": 10, - "span": 6, + "span": 12, "stack": false, "steppedLine": false, "targets": [ { - "expr": "max(rate(node_network_transmit_bytes_total{cluster=\"$cluster\", job=\"node-exporter\", instance=\"$instance\", device!~\"lo\"}[5m]))", + "expr": "sort_desc(sum by (pod) (irate(container_network_receive_bytes_total{job=\"kubernetes-cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}[4m])))", "format": "time_series", "intervalFactor": 2, - "legendFormat": "{{device}}", + "legendFormat": "RX: {{ pod }}", "refId": "A" + }, + { + "expr": "sort_desc(sum by (pod) (irate(container_network_transmit_bytes_total{job=\"kubernetes-cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}[4m])))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "TX: {{ pod }}", + "refId": "B" } ], "thresholds": [ @@ -2908,7 +3293,7 @@ data: ], "timeFrom": null, "timeShift": null, - "title": "Network Transmitted", + "title": "Network I/O", "tooltip": { "shared": false, "sort": 0, @@ -2930,7 +3315,7 @@ data: "label": null, "logBase": 1, "max": null, - "min": null, + "min": 0, "show": true }, { @@ -2938,7 +3323,7 @@ data: "label": null, "logBase": 1, "max": null, - "min": null, + "min": 0, "show": true } ] @@ -2968,14 +3353,14 @@ data: "gridPos": { }, - "id": 12, + "id": 5, "legend": { - "alignAsTable": false, - "avg": false, - "current": false, + "alignAsTable": true, + "avg": true, + "current": true, "max": false, "min": false, - "rightSide": false, + "rightSide": true, "show": true, "total": false, "values": false @@ -2995,23 +3380,16 @@ data: ], "spaceLength": 10, - "span": 9, + "span": 12, "stack": false, "steppedLine": false, "targets": [ { - "expr": "max(\n node_filesystem_files{cluster=\"$cluster\", job=\"node-exporter\", instance=\"$instance\"}\n - node_filesystem_files_free{cluster=\"$cluster\", job=\"node-exporter\", instance=\"$instance\"}\n)\n", + "expr": "max by (container) (kube_pod_container_status_restarts_total{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\", container=~\"$container\"})", "format": "time_series", "intervalFactor": 2, - "legendFormat": "inodes used", + "legendFormat": "Restarts: {{ container }}", "refId": "A" - }, - { - "expr": "max(node_filesystem_files_free{cluster=\"$cluster\", job=\"node-exporter\", instance=\"$instance\"})", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "inodes free", - "refId": "B" } ], "thresholds": [ @@ -3019,7 +3397,7 @@ data: ], "timeFrom": null, "timeShift": null, - "title": "Inodes Usage", + "title": "Total Restarts Per Container", "tooltip": { "shared": false, "sort": 0, @@ -3041,7 +3419,7 @@ data: "label": null, "logBase": 1, "max": null, - "min": null, + "min": 0, "show": true }, { @@ -3049,127 +3427,95 @@ data: "label": null, "logBase": 1, "max": null, - "min": null, + "min": 0, "show": true } ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + } + ], + "schemaVersion": 14, + "style": "dark", + "tags": [ + "kubernetes-mixin" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" }, - { - "cacheTimeout": null, - "colorBackground": false, - "colorValue": false, - "colors": [ - "rgba(50, 172, 45, 0.97)", - "rgba(237, 129, 40, 0.89)", - "rgba(245, 54, 54, 0.9)" - ], - "datasource": "$datasource", - "format": "percent", - "gauge": { - "maxValue": 100, - "minValue": 0, - "show": true, - "thresholdLabels": false, - "thresholdMarkers": true - }, - "gridPos": { + "hide": 0, + "label": null, + "name": "datasource", + "options": [ - }, - "id": 13, - "interval": null, - "links": [ + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { - ], - "mappingType": 1, - "mappingTypes": [ - { - "name": "value to text", - "value": 1 - }, - { - "name": "range to text", - "value": 2 - } - ], - "maxDataPoints": 100, - "nullPointMode": "connected", - "nullText": null, - "postfix": "", - "postfixFontSize": "50%", - "prefix": "", - "prefixFontSize": "50%", - "rangeMaps": [ - { - "from": "null", - "text": "N/A", - "to": "null" - } - ], - "span": 3, - "sparkline": { - "fillColor": "rgba(31, 118, 189, 0.18)", - "full": false, - "lineColor": "rgb(31, 120, 193)", - "show": false - }, - "tableColumn": "", - "targets": [ - { - "expr": "max(\n (\n (\n node_filesystem_files{cluster=\"$cluster\", job=\"node-exporter\", instance=\"$instance\"}\n - node_filesystem_files_free{cluster=\"$cluster\", job=\"node-exporter\", instance=\"$instance\"}\n )\n / node_filesystem_files{cluster=\"$cluster\", job=\"node-exporter\", instance=\"$instance\"}\n ) * 100)\n", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "", - "refId": "A" - } - ], - "thresholds": "80, 90", - "title": "Inodes Usage", - "tooltip": { - "shared": false - }, - "type": "singlestat", - "valueFontSize": "80%", - "valueMaps": [ - { - "op": "=", - "text": "N/A", - "value": "null" - } - ], - "valueName": "current" - } - ], - "repeat": null, - "repeatIteration": null, - "repeatRowId": null, - "showTitle": false, - "title": "Dashboard Row", - "titleSize": "h6", - "type": "row" - } - ], - "schemaVersion": 14, - "style": "dark", - "tags": [ - "kubernetes-mixin" - ], - "templating": { - "list": [ + }, + "datasource": "$datasource", + "hide": 2, + "includeAll": false, + "label": "cluster", + "multi": false, + "name": "cluster", + "options": [ + + ], + "query": "label_values(kube_pod_info, cluster)", + "refresh": 2, + "regex": "", + "sort": 0, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, { + "allValue": null, "current": { - "text": "Prometheus", - "value": "Prometheus" + }, + "datasource": "$datasource", "hide": 0, - "label": null, - "name": "datasource", + "includeAll": false, + "label": "Namespace", + "multi": false, + "name": "namespace", "options": [ ], - "query": "prometheus", - "refresh": 1, + "query": "label_values(kube_pod_info{cluster=\"$cluster\"}, namespace)", + "refresh": 2, "regex": "", - "type": "datasource" + "sort": 0, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false }, { "allValue": null, @@ -3177,15 +3523,15 @@ data: }, "datasource": "$datasource", - "hide": 2, + "hide": 0, "includeAll": false, - "label": "cluster", + "label": "Pod", "multi": false, - "name": "cluster", + "name": "pod", "options": [ ], - "query": "label_values(kube_pod_info, cluster)", + "query": "label_values(kube_pod_info{cluster=\"$cluster\", namespace=~\"$namespace\"}, pod)", "refresh": 2, "regex": "", "sort": 0, @@ -3204,14 +3550,14 @@ data: }, "datasource": "$datasource", "hide": 0, - "includeAll": false, - "label": null, + "includeAll": true, + "label": "Container", "multi": false, - "name": "instance", + "name": "container", "options": [ ], - "query": "label_values(node_boot_time_seconds{cluster=\"$cluster\", job=\"node-exporter\"}, instance)", + "query": "label_values(kube_pod_container_info{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}, container)", "refresh": 2, "regex": "", "sort": 0, @@ -3255,11 +3601,11 @@ data: ] }, "timezone": "", - "title": "Kubernetes / Nodes", - "uid": "fa49a4706d07a042595b664c87fb33ea", + "title": "Kubernetes / Pods", + "uid": "ab4f13a9892a76a4d21ce8c2445bf4ea", "version": 0 } - persistentvolumesusage.json: |- + scheduler.json: |- { "__inputs": [ @@ -3286,126 +3632,28 @@ data: "collapse": false, "collapsed": false, "panels": [ - { - "aliasColors": { - - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 1, - "gridPos": { - - }, - "id": 2, - "legend": { - "alignAsTable": true, - "avg": true, - "current": true, - "max": true, - "min": true, - "rightSide": false, - "show": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [ - - ], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "repeat": null, - "seriesOverrides": [ - - ], - "spaceLength": 10, - "span": 9, - "stack": true, - "steppedLine": false, - "targets": [ - { - "expr": "(\n sum without(instance, node) (kubelet_volume_stats_capacity_bytes{cluster=\"$cluster\", job=\"kubelet\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"})\n -\n sum without(instance, node) (kubelet_volume_stats_available_bytes{cluster=\"$cluster\", job=\"kubelet\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"})\n)\n", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "Used Space", - "refId": "A" - }, - { - "expr": "sum without(instance, node) (kubelet_volume_stats_available_bytes{cluster=\"$cluster\", job=\"kubelet\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"})\n", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "Free Space", - "refId": "B" - } - ], - "thresholds": [ - - ], - "timeFrom": null, - "timeShift": null, - "title": "Volume Space Usage", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ - - ] - }, - "yaxes": [ - { - "format": "bytes", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "bytes", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - } - ] - }, { "cacheTimeout": null, "colorBackground": false, "colorValue": false, "colors": [ - "rgba(50, 172, 45, 0.97)", + "#299c46", "rgba(237, 129, 40, 0.89)", - "rgba(245, 54, 54, 0.9)" + "#d44a3a" ], "datasource": "$datasource", - "format": "percent", + "format": "none", "gauge": { "maxValue": 100, "minValue": 0, - "show": true, + "show": false, "thresholdLabels": false, "thresholdMarkers": true }, "gridPos": { }, - "id": 3, + "id": 2, "interval": null, "links": [ @@ -3435,7 +3683,7 @@ data: "to": "null" } ], - "span": 3, + "span": 2, "sparkline": { "fillColor": "rgba(31, 118, 189, 0.18)", "full": false, @@ -3445,15 +3693,15 @@ data: "tableColumn": "", "targets": [ { - "expr": "(\n kubelet_volume_stats_capacity_bytes{cluster=\"$cluster\", job=\"kubelet\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"}\n -\n kubelet_volume_stats_available_bytes{cluster=\"$cluster\", job=\"kubelet\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"}\n)\n/\nkubelet_volume_stats_capacity_bytes{cluster=\"$cluster\", job=\"kubelet\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"}\n* 100\n", + "expr": "sum(up{job=\"kube-scheduler\"})", "format": "time_series", "intervalFactor": 2, "legendFormat": "", "refId": "A" } ], - "thresholds": "80, 90", - "title": "Volume Space Usage", + "thresholds": "", + "title": "Up", "tooltip": { "shared": false }, @@ -3466,21 +3714,8 @@ data: "value": "null" } ], - "valueName": "current" - } - ], - "repeat": null, - "repeatIteration": null, - "repeatRowId": null, - "showTitle": false, - "title": "Dashboard Row", - "titleSize": "h6", - "type": "row" - }, - { - "collapse": false, - "collapsed": false, - "panels": [ + "valueName": "min" + }, { "aliasColors": { @@ -3493,17 +3728,17 @@ data: "gridPos": { }, - "id": 4, + "id": 3, "legend": { - "alignAsTable": true, - "avg": true, - "current": true, - "max": true, - "min": true, - "rightSide": false, - "show": true, + "alignAsTable": "true", + "avg": false, + "current": "true", + "max": false, + "min": false, + "rightSide": "true", + "show": "true", "total": false, - "values": true + "values": "true" }, "lines": true, "linewidth": 1, @@ -3520,23 +3755,37 @@ data: ], "spaceLength": 10, - "span": 9, - "stack": true, + "span": 5, + "stack": false, "steppedLine": false, "targets": [ { - "expr": "sum without(instance, node) (kubelet_volume_stats_inodes_used{cluster=\"$cluster\", job=\"kubelet\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"})\n", + "expr": "sum(rate(scheduler_e2e_scheduling_duration_seconds_count{job=\"kube-scheduler\", instance=~\"$instance\"}[5m])) by (instance)", "format": "time_series", - "intervalFactor": 1, - "legendFormat": "Used inodes", + "intervalFactor": 2, + "legendFormat": "{{instance}} e2e", "refId": "A" }, { - "expr": "(\n sum without(instance, node) (kubelet_volume_stats_inodes{cluster=\"$cluster\", job=\"kubelet\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"})\n -\n sum without(instance, node) (kubelet_volume_stats_inodes_used{cluster=\"$cluster\", job=\"kubelet\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"})\n)\n", + "expr": "sum(rate(scheduler_binding_duration_seconds_count{job=\"kube-scheduler\", instance=~\"$instance\"}[5m])) by (instance)", "format": "time_series", - "intervalFactor": 1, - "legendFormat": " Free inodes", + "intervalFactor": 2, + "legendFormat": "{{instance}} binding", "refId": "B" + }, + { + "expr": "sum(rate(scheduler_scheduling_algorithm_duration_seconds_count{job=\"kube-scheduler\", instance=~\"$instance\"}[5m])) by (instance)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{instance}} scheduling algorithm", + "refId": "C" + }, + { + "expr": "sum(rate(scheduler_volume_scheduling_duration_seconds_count{job=\"kube-scheduler\", instance=~\"$instance\"}[5m])) by (instance)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{instance}} volume", + "refId": "D" } ], "thresholds": [ @@ -3544,7 +3793,7 @@ data: ], "timeFrom": null, "timeShift": null, - "title": "Volume inodes Usage", + "title": "Scheduling Rate", "tooltip": { "shared": false, "sort": 0, @@ -3562,7 +3811,7 @@ data: }, "yaxes": [ { - "format": "none", + "format": "ops", "label": null, "logBase": 1, "max": null, @@ -3570,7 +3819,7 @@ data: "show": true }, { - "format": "none", + "format": "ops", "label": null, "logBase": 1, "max": null, @@ -3580,88 +3829,116 @@ data: ] }, { - "cacheTimeout": null, - "colorBackground": false, - "colorValue": false, - "colors": [ - "rgba(50, 172, 45, 0.97)", - "rgba(237, 129, 40, 0.89)", - "rgba(245, 54, 54, 0.9)" - ], - "datasource": "$datasource", - "format": "percent", - "gauge": { - "maxValue": 100, - "minValue": 0, - "show": true, - "thresholdLabels": false, - "thresholdMarkers": true + "aliasColors": { + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, "gridPos": { }, - "id": 5, - "interval": null, + "id": 4, + "legend": { + "alignAsTable": "true", + "avg": false, + "current": "true", + "max": false, + "min": false, + "rightSide": "true", + "show": "true", + "total": false, + "values": "true" + }, + "lines": true, + "linewidth": 1, "links": [ ], - "mappingType": 1, - "mappingTypes": [ + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 5, + "stack": false, + "steppedLine": false, + "targets": [ { - "name": "value to text", - "value": 1 + "expr": "histogram_quantile(0.99, sum(rate(scheduler_e2e_scheduling_duration_seconds_bucket{job=\"kube-scheduler\",instance=~\"$instance\"}[5m])) by (instance, le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{instance}} e2e", + "refId": "A" }, { - "name": "range to text", - "value": 2 - } - ], - "maxDataPoints": 100, - "nullPointMode": "connected", - "nullText": null, - "postfix": "", - "postfixFontSize": "50%", - "prefix": "", - "prefixFontSize": "50%", - "rangeMaps": [ + "expr": "histogram_quantile(0.99, sum(rate(scheduler_binding_duration_seconds_bucket{job=\"kube-scheduler\",instance=~\"$instance\"}[5m])) by (instance, le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{instance}} binding", + "refId": "B" + }, { - "from": "null", - "text": "N/A", - "to": "null" - } - ], - "span": 3, - "sparkline": { - "fillColor": "rgba(31, 118, 189, 0.18)", - "full": false, - "lineColor": "rgb(31, 120, 193)", - "show": false - }, - "tableColumn": "", - "targets": [ + "expr": "histogram_quantile(0.99, sum(rate(scheduler_scheduling_algorithm_duration_seconds_bucket{job=\"kube-scheduler\",instance=~\"$instance\"}[5m])) by (instance, le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{instance}} scheduling algorithm", + "refId": "C" + }, { - "expr": "kubelet_volume_stats_inodes_used{cluster=\"$cluster\", job=\"kubelet\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"}\n/\nkubelet_volume_stats_inodes{cluster=\"$cluster\", job=\"kubelet\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"}\n* 100\n", + "expr": "histogram_quantile(0.99, sum(rate(scheduler_volume_scheduling_duration_seconds_bucket{job=\"kube-scheduler\",instance=~\"$instance\"}[5m])) by (instance, le))", "format": "time_series", "intervalFactor": 2, - "legendFormat": "", - "refId": "A" + "legendFormat": "{{instance}} volume", + "refId": "D" } ], - "thresholds": "80, 90", - "title": "Volume inodes Usage", + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Scheduling latency 99th Quantile", "tooltip": { - "shared": false + "shared": false, + "sort": 0, + "value_type": "individual" }, - "type": "singlestat", - "valueFontSize": "80%", - "valueMaps": [ + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ { - "op": "=", - "text": "N/A", - "value": "null" + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true } - ], - "valueName": "current" + ] } ], "repeat": null, @@ -3671,181 +3948,223 @@ data: "title": "Dashboard Row", "titleSize": "h6", "type": "row" - } - ], - "schemaVersion": 14, - "style": "dark", - "tags": [ - "kubernetes-mixin" - ], - "templating": { - "list": [ - { - "current": { - "text": "Prometheus", - "value": "Prometheus" - }, - "hide": 0, - "label": null, - "name": "datasource", - "options": [ - - ], - "query": "prometheus", - "refresh": 1, - "regex": "", - "type": "datasource" - }, - { - "allValue": null, - "current": { - - }, - "datasource": "$datasource", - "hide": 2, - "includeAll": false, - "label": "cluster", - "multi": false, - "name": "cluster", - "options": [ + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { - ], - "query": "label_values(kubelet_volume_stats_capacity_bytes, cluster)", - "refresh": 2, - "regex": "", - "sort": 0, - "tagValuesQuery": "", - "tags": [ + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "gridPos": { - ], - "tagsQuery": "", - "type": "query", - "useTags": false - }, - { - "allValue": null, - "current": { + }, + "id": 5, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ - }, - "datasource": "$datasource", - "hide": 0, - "includeAll": false, - "label": "Namespace", - "multi": false, - "name": "namespace", - "options": [ + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ - ], - "query": "label_values(kubelet_volume_stats_capacity_bytes{cluster=\"$cluster\", job=\"kubelet\"}, namespace)", - "refresh": 2, - "regex": "", - "sort": 0, - "tagValuesQuery": "", - "tags": [ + ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(rest_client_requests_total{job=\"kube-scheduler\", instance=~\"$instance\",code=~\"2..\"}[5m]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "2xx", + "refId": "A" + }, + { + "expr": "sum(rate(rest_client_requests_total{job=\"kube-scheduler\", instance=~\"$instance\",code=~\"3..\"}[5m]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "3xx", + "refId": "B" + }, + { + "expr": "sum(rate(rest_client_requests_total{job=\"kube-scheduler\", instance=~\"$instance\",code=~\"4..\"}[5m]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "4xx", + "refId": "C" + }, + { + "expr": "sum(rate(rest_client_requests_total{job=\"kube-scheduler\", instance=~\"$instance\",code=~\"5..\"}[5m]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "5xx", + "refId": "D" + } + ], + "thresholds": [ - ], - "tagsQuery": "", - "type": "query", - "useTags": false - }, - { - "allValue": null, - "current": { + ], + "timeFrom": null, + "timeShift": null, + "title": "Kube API Request Rate", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + ] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] }, - "datasource": "$datasource", - "hide": 0, - "includeAll": false, - "label": "PersistentVolumeClaim", - "multi": false, - "name": "volume", - "options": [ + { + "aliasColors": { - ], - "query": "label_values(kubelet_volume_stats_capacity_bytes{cluster=\"$cluster\", job=\"kubelet\", namespace=\"$namespace\"}, persistentvolumeclaim)", - "refresh": 2, - "regex": "", - "sort": 0, - "tagValuesQuery": "", - "tags": [ + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "gridPos": { - ], - "tagsQuery": "", - "type": "query", - "useTags": false - } - ] - }, - "time": { - "from": "now-7d", - "to": "now" - }, - "timepicker": { - "refresh_intervals": [ - "5s", - "10s", - "30s", - "1m", - "5m", - "15m", - "30m", - "1h", - "2h", - "1d" - ], - "time_options": [ - "5m", - "15m", - "1h", - "6h", - "12h", - "24h", - "2d", - "7d", - "30d" - ] - }, - "timezone": "", - "title": "Kubernetes / Persistent Volumes", - "uid": "919b92a8e8041bd567af9edab12c840c", - "version": 0 - } - pods.json: |- - { - "__inputs": [ + }, + "id": 6, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ - ], - "__requires": [ + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ - ], - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": "$datasource", - "enable": true, - "expr": "time() == BOOL timestamp(rate(kube_pod_container_status_restarts_total{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}[2m]) > 0)", - "hide": false, - "iconColor": "rgba(215, 44, 44, 1)", - "name": "Restarts", - "showIn": 0, - "tags": [ - "restart" - ], - "type": "rows" - } - ] - }, - "editable": false, - "gnetId": null, - "graphTooltip": 0, - "hideControls": false, - "id": null, - "links": [ + ], + "spaceLength": 10, + "span": 8, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(rest_client_request_latency_seconds_bucket{job=\"kube-scheduler\", instance=~\"$instance\", verb=\"POST\"}[5m])) by (verb, url, le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{verb}} {{url}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Post Request Latency 99th Quantile", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ - ], - "refresh": "", - "rows": [ + ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + }, { "collapse": false, "collapsed": false, @@ -3862,17 +4181,17 @@ data: "gridPos": { }, - "id": 2, + "id": 7, "legend": { - "alignAsTable": true, - "avg": true, - "current": true, + "alignAsTable": "true", + "avg": false, + "current": "true", "max": false, "min": false, - "rightSide": true, - "show": true, + "rightSide": "true", + "show": "true", "total": false, - "values": false + "values": "true" }, "lines": true, "linewidth": 1, @@ -3894,32 +4213,11 @@ data: "steppedLine": false, "targets": [ { - "expr": "sum by(container_name) (container_memory_usage_bytes{job=\"kubernetes-cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\", pod_name=\"$pod\", container_name=~\"$container\", container_name!=\"POD\"})", + "expr": "histogram_quantile(0.99, sum(rate(rest_client_request_latency_seconds_bucket{job=\"kube-scheduler\", instance=~\"$instance\", verb=\"GET\"}[5m])) by (verb, url, le))", "format": "time_series", "intervalFactor": 2, - "legendFormat": "Current: {{ container_name }}", + "legendFormat": "{{verb}} {{url}}", "refId": "A" - }, - { - "expr": "sum by(container) (kube_pod_container_resource_requests{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"memory\", pod=\"$pod\", container=~\"$container\"})", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "Requested: {{ container }}", - "refId": "B" - }, - { - "expr": "sum by(container) (kube_pod_container_resource_limits{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"memory\", pod=\"$pod\", container=~\"$container\"})", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "Limit: {{ container }}", - "refId": "C" - }, - { - "expr": "sum by(container_name) (container_memory_cache{job=\"kubernetes-cadvisor\", namespace=\"$namespace\", pod_name=~\"$pod\", container_name=~\"$container\", container_name!=\"POD\"})", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "Cache: {{ container_name }}", - "refId": "D" } ], "thresholds": [ @@ -3927,7 +4225,7 @@ data: ], "timeFrom": null, "timeShift": null, - "title": "Memory Usage", + "title": "Get Request Latency 99th Quantile", "tooltip": { "shared": false, "sort": 0, @@ -3945,7 +4243,7 @@ data: }, "yaxes": [ { - "format": "bytes", + "format": "s", "label": null, "logBase": 1, "max": null, @@ -3953,7 +4251,7 @@ data: "show": true }, { - "format": "bytes", + "format": "s", "label": null, "logBase": 1, "max": null, @@ -3987,14 +4285,14 @@ data: "gridPos": { }, - "id": 3, + "id": 8, "legend": { - "alignAsTable": true, - "avg": true, - "current": true, + "alignAsTable": false, + "avg": false, + "current": false, "max": false, "min": false, - "rightSide": true, + "rightSide": false, "show": true, "total": false, "values": false @@ -4014,30 +4312,16 @@ data: ], "spaceLength": 10, - "span": 12, + "span": 4, "stack": false, "steppedLine": false, "targets": [ { - "expr": "sum by (container_name) (rate(container_cpu_usage_seconds_total{job=\"kubernetes-cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\", image!=\"\", pod_name=\"$pod\", container_name=~\"$container\", container_name!=\"POD\"}[1m]))", + "expr": "process_resident_memory_bytes{job=\"kube-scheduler\", instance=~\"$instance\"}", "format": "time_series", "intervalFactor": 2, - "legendFormat": "Current: {{ container_name }}", + "legendFormat": "{{instance}}", "refId": "A" - }, - { - "expr": "sum by(container) (kube_pod_container_resource_requests{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"cpu\", pod=\"$pod\", container=~\"$container\"})", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "Requested: {{ container }}", - "refId": "B" - }, - { - "expr": "sum by(container) (kube_pod_container_resource_limits{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"cpu\", pod=\"$pod\", container=~\"$container\"})", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "Limit: {{ container }}", - "refId": "C" } ], "thresholds": [ @@ -4045,7 +4329,7 @@ data: ], "timeFrom": null, "timeShift": null, - "title": "CPU Usage", + "title": "Memory", "tooltip": { "shared": false, "sort": 0, @@ -4063,36 +4347,23 @@ data: }, "yaxes": [ { - "format": "short", + "format": "bytes", "label": null, "logBase": 1, "max": null, - "min": 0, + "min": null, "show": true }, { - "format": "short", + "format": "bytes", "label": null, "logBase": 1, "max": null, - "min": 0, + "min": null, "show": true } ] - } - ], - "repeat": null, - "repeatIteration": null, - "repeatRowId": null, - "showTitle": false, - "title": "Dashboard Row", - "titleSize": "h6", - "type": "row" - }, - { - "collapse": false, - "collapsed": false, - "panels": [ + }, { "aliasColors": { @@ -4105,14 +4376,14 @@ data: "gridPos": { }, - "id": 4, + "id": 9, "legend": { - "alignAsTable": true, - "avg": true, - "current": true, + "alignAsTable": false, + "avg": false, + "current": false, "max": false, "min": false, - "rightSide": true, + "rightSide": false, "show": true, "total": false, "values": false @@ -4132,23 +4403,16 @@ data: ], "spaceLength": 10, - "span": 12, + "span": 4, "stack": false, "steppedLine": false, "targets": [ { - "expr": "sort_desc(sum by (pod_name) (rate(container_network_receive_bytes_total{job=\"kubernetes-cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\", pod_name=\"$pod\"}[1m])))", + "expr": "rate(process_cpu_seconds_total{job=\"kube-scheduler\", instance=~\"$instance\"}[5m])", "format": "time_series", "intervalFactor": 2, - "legendFormat": "RX: {{ pod_name }}", + "legendFormat": "{{instance}}", "refId": "A" - }, - { - "expr": "sort_desc(sum by (pod_name) (rate(container_network_transmit_bytes_total{job=\"kubernetes-cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\", pod_name=\"$pod\"}[1m])))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "TX: {{ pod_name }}", - "refId": "B" } ], "thresholds": [ @@ -4156,7 +4420,7 @@ data: ], "timeFrom": null, "timeShift": null, - "title": "Network I/O", + "title": "CPU usage", "tooltip": { "shared": false, "sort": 0, @@ -4190,20 +4454,7 @@ data: "show": true } ] - } - ], - "repeat": null, - "repeatIteration": null, - "repeatRowId": null, - "showTitle": false, - "title": "Dashboard Row", - "titleSize": "h6", - "type": "row" - }, - { - "collapse": false, - "collapsed": false, - "panels": [ + }, { "aliasColors": { @@ -4216,14 +4467,14 @@ data: "gridPos": { }, - "id": 5, + "id": 10, "legend": { - "alignAsTable": true, - "avg": true, - "current": true, + "alignAsTable": false, + "avg": false, + "current": false, "max": false, "min": false, - "rightSide": true, + "rightSide": false, "show": true, "total": false, "values": false @@ -4243,15 +4494,15 @@ data: ], "spaceLength": 10, - "span": 12, + "span": 4, "stack": false, "steppedLine": false, "targets": [ { - "expr": "max by (container) (kube_pod_container_status_restarts_total{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\", container=~\"$container\"})", + "expr": "go_goroutines{job=\"kube-scheduler\",instance=~\"$instance\"}", "format": "time_series", "intervalFactor": 2, - "legendFormat": "Restarts: {{ container }}", + "legendFormat": "{{instance}}", "refId": "A" } ], @@ -4260,7 +4511,7 @@ data: ], "timeFrom": null, "timeShift": null, - "title": "Total Restarts Per Container", + "title": "Goroutines", "tooltip": { "shared": false, "sort": 0, @@ -4282,7 +4533,7 @@ data: "label": null, "logBase": 1, "max": null, - "min": 0, + "min": null, "show": true }, { @@ -4290,7 +4541,7 @@ data: "label": null, "logBase": 1, "max": null, - "min": 0, + "min": null, "show": true } ] @@ -4332,95 +4583,17 @@ data: "allValue": null, "current": { - }, - "datasource": "$datasource", - "hide": 2, - "includeAll": false, - "label": "cluster", - "multi": false, - "name": "cluster", - "options": [ - - ], - "query": "label_values(kube_pod_info, cluster)", - "refresh": 2, - "regex": "", - "sort": 0, - "tagValuesQuery": "", - "tags": [ - - ], - "tagsQuery": "", - "type": "query", - "useTags": false - }, - { - "allValue": null, - "current": { - - }, - "datasource": "$datasource", - "hide": 0, - "includeAll": false, - "label": "Namespace", - "multi": false, - "name": "namespace", - "options": [ - - ], - "query": "label_values(kube_pod_info{cluster=\"$cluster\"}, namespace)", - "refresh": 2, - "regex": "", - "sort": 0, - "tagValuesQuery": "", - "tags": [ - - ], - "tagsQuery": "", - "type": "query", - "useTags": false - }, - { - "allValue": null, - "current": { - - }, - "datasource": "$datasource", - "hide": 0, - "includeAll": false, - "label": "Pod", - "multi": false, - "name": "pod", - "options": [ - - ], - "query": "label_values(kube_pod_info{cluster=\"$cluster\", namespace=~\"$namespace\"}, pod)", - "refresh": 2, - "regex": "", - "sort": 0, - "tagValuesQuery": "", - "tags": [ - - ], - "tagsQuery": "", - "type": "query", - "useTags": false - }, - { - "allValue": null, - "current": { - }, "datasource": "$datasource", "hide": 0, "includeAll": true, - "label": "Container", + "label": null, "multi": false, - "name": "container", + "name": "instance", "options": [ ], - "query": "label_values(kube_pod_container_info{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}, container)", + "query": "label_values(process_cpu_seconds_total{job=\"kube-scheduler\"}, instance)", "refresh": 2, "regex": "", "sort": 0, @@ -4464,8 +4637,8 @@ data: ] }, "timezone": "", - "title": "Kubernetes / Pods", - "uid": "ab4f13a9892a76a4d21ce8c2445bf4ea", + "title": "Kubernetes / Scheduler", + "uid": "2e6b6a3b4bddf1427b3a55aa1311c656", "version": 0 } statefulset.json: |- @@ -4555,7 +4728,7 @@ data: "tableColumn": "", "targets": [ { - "expr": "sum(rate(container_cpu_usage_seconds_total{job=\"kubernetes-cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\", pod_name=~\"$statefulset.*\"}[3m]))", + "expr": "sum(rate(container_cpu_usage_seconds_total{job=\"kubernetes-cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"$statefulset.*\"}[3m]))", "format": "time_series", "intervalFactor": 2, "legendFormat": "", @@ -4638,7 +4811,7 @@ data: "tableColumn": "", "targets": [ { - "expr": "sum(container_memory_usage_bytes{job=\"kubernetes-cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\", pod_name=~\"$statefulset.*\"}) / 1024^3", + "expr": "sum(container_memory_usage_bytes{job=\"kubernetes-cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"$statefulset.*\"}) / 1024^3", "format": "time_series", "intervalFactor": 2, "legendFormat": "", @@ -4721,7 +4894,7 @@ data: "tableColumn": "", "targets": [ { - "expr": "sum(rate(container_network_transmit_bytes_total{job=\"kubernetes-cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\", pod_name=~\"$statefulset.*\"}[3m])) + sum(rate(container_network_receive_bytes_total{cluster=\"$cluster\", namespace=\"$namespace\",pod_name=~\"$statefulset.*\"}[3m]))", + "expr": "sum(rate(container_network_transmit_bytes_total{job=\"kubernetes-cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"$statefulset.*\"}[3m])) + sum(rate(container_network_receive_bytes_total{cluster=\"$cluster\", namespace=\"$namespace\",pod=~\"$statefulset.*\"}[3m]))", "format": "time_series", "intervalFactor": 2, "legendFormat": "", diff --git a/addons/grafana/dashboards-prom.yaml b/addons/grafana/dashboards-prom.yaml new file mode 100644 index 000000000..b1f4c06c5 --- /dev/null +++ b/addons/grafana/dashboards-prom.yaml @@ -0,0 +1,2160 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: grafana-dashboards-prom + namespace: monitoring +data: + prometheus-remote-write.json: |- + { + "annotations": { + "list": [ + + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "links": [ + + ], + "refresh": "10s", + "rows": [ + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 1, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "prometheus_remote_storage_highest_timestamp_in_seconds{cluster=~\"$cluster\", instance=~\"$instance\"} - ignoring(queue) group_right(instance) prometheus_remote_storage_queue_highest_sent_timestamp_seconds{cluster=~\"$cluster\", instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{cluster}}:{{instance}}-{{queue}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Highest Timestamp In vs. Highest Timestamp Sent", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 2, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(prometheus_remote_storage_highest_timestamp_in_seconds{cluster=~\"$cluster\", instance=~\"$instance\"}[5m]) - ignoring (queue) group_right(instance) rate(prometheus_remote_storage_queue_highest_sent_timestamp_seconds{cluster=~\"$cluster\", instance=~\"$instance\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{cluster}}:{{instance}}-{{queue}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate[5m]", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Timestamps", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 3, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(prometheus_remote_storage_samples_in_total{cluster=~\"$cluster\", instance=~\"$instance\"}[5m])- ignoring(queue) group_right(instance) rate(prometheus_remote_storage_succeeded_samples_total{cluster=~\"$cluster\", instance=~\"$instance\"}[5m]) - rate(prometheus_remote_storage_dropped_samples_total{cluster=~\"$cluster\", instance=~\"$instance\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{cluster}}:{{instance}}-{{queue}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate, in vs. succeeded or dropped [5m]", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Samples", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 4, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "prometheus_remote_storage_shards{cluster=~\"$cluster\", instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{cluster}}:{{instance}}-{{queue}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Num. Shards", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 5, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "prometheus_remote_storage_shard_capacity{cluster=~\"$cluster\", instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{cluster}}:{{instance}}-{{queue}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Capacity", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Shards", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 6, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 3, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(prometheus_remote_storage_dropped_samples_total{cluster=~\"$cluster\", instance=~\"$instance\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{cluster}}:{{instance}}-{{queue}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Dropped Samples", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 7, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 3, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(prometheus_remote_storage_failed_samples_total{cluster=~\"$cluster\", instance=~\"$instance\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{cluster}}:{{instance}}-{{queue}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Failed Samples", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 8, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 3, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(prometheus_remote_storage_retried_samples_total{cluster=~\"$cluster\", instance=~\"$instance\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{cluster}}:{{instance}}-{{queue}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Retried Samples", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 9, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 3, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(prometheus_remote_storage_enqueue_retries_total{cluster=~\"$cluster\", instance=~\"$instance\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{cluster}}:{{instance}}-{{queue}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Enqueue Retries", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Misc Rates.", + "titleSize": "h6" + } + ], + "schemaVersion": 14, + "style": "dark", + "tags": [ + + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": null, + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + "selected": true, + "text": "All", + "value": "$__all" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": true, + "label": "instance", + "multi": true, + "name": "instance", + "options": [ + + ], + "query": "label_values(prometheus_build_info, instance)", + "refresh": 1, + "regex": "", + "sort": 2, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + "selected": true, + "text": "All", + "value": "$__all" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": true, + "label": "cluster", + "multi": true, + "name": "cluster", + "options": [ + + ], + "query": "label_values(kube_pod_container_info{image=~\".*prometheus.*\"}, cluster)", + "refresh": 1, + "regex": "", + "sort": 2, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "utc", + "title": "Prometheus Remote Write", + "uid": "", + "version": 0 + } + prometheus.json: |- + { + "annotations": { + "list": [ + + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "links": [ + + ], + "refresh": "10s", + "rows": [ + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 1, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "Count", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #A", + "thresholds": [ + + ], + "type": "hidden", + "unit": "short" + }, + { + "alias": "Uptime", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #B", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "Instance", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "instance", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "Job", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "job", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "Version", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "version", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "/.*/", + "thresholds": [ + + ], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "expr": "count by (job, instance, version) (prometheus_build_info{job=~\"$job\", instance=~\"$instance\"})", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10 + }, + { + "expr": "max by (job, instance) (time() - process_start_time_seconds{job=~\"$job\", instance=~\"$instance\"})", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Prometheus Stats", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "transform": "table", + "type": "table", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Prometheus Stats", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 2, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(prometheus_target_sync_length_seconds_sum{job=~\"$job\",instance=~\"$instance\"}[5m])) by (scrape_job) * 1e3", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{scrape_job}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Target Sync", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "ms", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 3, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(prometheus_sd_discovered_targets{job=~\"$job\",instance=~\"$instance\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Targets", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Targets", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Discovery", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 4, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(prometheus_target_interval_length_seconds_sum{job=~\"$job\",instance=~\"$instance\"}[5m]) / rate(prometheus_target_interval_length_seconds_count{job=~\"$job\",instance=~\"$instance\"}[5m]) * 1e3", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{interval}} configured", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Average Scrape Interval Duration", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "ms", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 5, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 4, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum by (job) (rate(prometheus_target_scrapes_exceeded_sample_limit_total[1m]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "exceeded sample limit: {{job}}", + "legendLink": null, + "step": 10 + }, + { + "expr": "sum by (job) (rate(prometheus_target_scrapes_sample_duplicate_timestamp_total[1m]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "duplicate timestamp: {{job}}", + "legendLink": null, + "step": 10 + }, + { + "expr": "sum by (job) (rate(prometheus_target_scrapes_sample_out_of_bounds_total[1m]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "out of bounds: {{job}}", + "legendLink": null, + "step": 10 + }, + { + "expr": "sum by (job) (rate(prometheus_target_scrapes_sample_out_of_order_total[1m]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "out of order: {{job}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Scrape failures", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 6, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 4, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "rate(prometheus_tsdb_head_samples_appended_total{job=~\"$job\",instance=~\"$instance\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{job}} {{instance}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Appended Samples", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Retrieval", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 7, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "prometheus_tsdb_head_series{job=~\"$job\",instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{job}} {{instance}} head series", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Head Series", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 8, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "prometheus_tsdb_head_chunks{job=~\"$job\",instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{job}} {{instance}} head chunks", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Head Chunks", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Storage", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 9, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "rate(prometheus_engine_query_duration_seconds_count{job=~\"$job\",instance=~\"$instance\",slice=\"inner_eval\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{job}} {{instance}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Query Rate", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 10, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "max by (slice) (prometheus_engine_query_duration_seconds{quantile=\"0.9\",job=~\"$job\",instance=~\"$instance\"}) * 1e3", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{slice}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Stage Duration", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "ms", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Query", + "titleSize": "h6" + } + ], + "schemaVersion": 14, + "style": "dark", + "tags": [ + + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": null, + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + "selected": true, + "text": "All", + "value": "$__all" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": true, + "label": "job", + "multi": true, + "name": "job", + "options": [ + + ], + "query": "label_values(prometheus_build_info, job)", + "refresh": 1, + "regex": "", + "sort": 2, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + "selected": true, + "text": "All", + "value": "$__all" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": true, + "label": "instance", + "multi": true, + "name": "instance", + "options": [ + + ], + "query": "label_values(prometheus_build_info, instance)", + "refresh": 1, + "regex": "", + "sort": 2, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "utc", + "title": "Prometheus", + "uid": "", + "version": 0 + } diff --git a/addons/grafana/deployment.yaml b/addons/grafana/deployment.yaml index dc9f05847..bdc2d891f 100644 --- a/addons/grafana/deployment.yaml +++ b/addons/grafana/deployment.yaml @@ -56,8 +56,12 @@ spec: mountPath: /etc/grafana/provisioning/dashboards - name: dashboards-etcd mountPath: /etc/grafana/dashboards/etcd + - name: dashboards-prom + mountPath: /etc/grafana/dashboards/prom - name: dashboards-k8s mountPath: /etc/grafana/dashboards/k8s + - name: dashboards-k8s-nodes + mountPath: /etc/grafana/dashboards/k8s-nodes - name: dashboards-k8s-resources mountPath: /etc/grafana/dashboards/k8s-resources volumes: @@ -73,9 +77,15 @@ spec: - name: dashboards-etcd configMap: name: grafana-dashboards-etcd + - name: dashboards-prom + configMap: + name: grafana-dashboards-prom - name: dashboards-k8s configMap: name: grafana-dashboards-k8s + - name: dashboards-k8s-nodes + configMap: + name: grafana-dashboards-k8s-nodes - name: dashboards-k8s-resources configMap: name: grafana-dashboards-k8s-resources diff --git a/addons/prometheus/rules.yaml b/addons/prometheus/rules.yaml index 2962028a1..22e54a912 100644 --- a/addons/prometheus/rules.yaml +++ b/addons/prometheus/rules.yaml @@ -163,32 +163,32 @@ data: "name": "k8s.rules", "rules": [ { - "expr": "sum(rate(container_cpu_usage_seconds_total{job=\"kubernetes-cadvisor\", image!=\"\", container_name!=\"\"}[5m])) by (namespace)\n", + "expr": "sum(rate(container_cpu_usage_seconds_total{job=\"kubernetes-cadvisor\", image!=\"\", container!=\"POD\"}[5m])) by (namespace)\n", "record": "namespace:container_cpu_usage_seconds_total:sum_rate" }, { - "expr": "sum by (namespace, pod_name, container_name) (\n rate(container_cpu_usage_seconds_total{job=\"kubernetes-cadvisor\", image!=\"\", container_name!=\"\"}[5m])\n)\n", - "record": "namespace_pod_name_container_name:container_cpu_usage_seconds_total:sum_rate" + "expr": "sum by (namespace, pod, container) (\n rate(container_cpu_usage_seconds_total{job=\"kubernetes-cadvisor\", image!=\"\", container!=\"POD\"}[5m])\n)\n", + "record": "namespace_pod_container:container_cpu_usage_seconds_total:sum_rate" }, { - "expr": "sum(container_memory_usage_bytes{job=\"kubernetes-cadvisor\", image!=\"\", container_name!=\"\"}) by (namespace)\n", + "expr": "sum(container_memory_usage_bytes{job=\"kubernetes-cadvisor\", image!=\"\", container!=\"POD\"}) by (namespace)\n", "record": "namespace:container_memory_usage_bytes:sum" }, { - "expr": "sum by (namespace, label_name) (\n sum(rate(container_cpu_usage_seconds_total{job=\"kubernetes-cadvisor\", image!=\"\", container_name!=\"\"}[5m])) by (namespace, pod_name)\n * on (namespace, pod_name) group_left(label_name)\n label_replace(kube_pod_labels{job=\"kube-state-metrics\"}, \"pod_name\", \"$1\", \"pod\", \"(.*)\")\n)\n", - "record": "namespace_name:container_cpu_usage_seconds_total:sum_rate" + "expr": "sum by (namespace, label_name) (\n sum(rate(container_cpu_usage_seconds_total{job=\"kubernetes-cadvisor\", image!=\"\", container!=\"POD\"}[5m])) by (namespace, pod)\n * on (namespace, pod)\n group_left(label_name) kube_pod_labels{job=\"kube-state-metrics\"}\n)\n", + "record": "namespace:container_cpu_usage_seconds_total:sum_rate" }, { - "expr": "sum by (namespace, label_name) (\n sum(container_memory_usage_bytes{job=\"kubernetes-cadvisor\",image!=\"\", container_name!=\"\"}) by (pod_name, namespace)\n* on (namespace, pod_name) group_left(label_name)\n label_replace(kube_pod_labels{job=\"kube-state-metrics\"}, \"pod_name\", \"$1\", \"pod\", \"(.*)\")\n)\n", - "record": "namespace_name:container_memory_usage_bytes:sum" + "expr": "sum by (namespace, label_name) (\n sum(container_memory_usage_bytes{job=\"kubernetes-cadvisor\",image!=\"\", container!=\"POD\"}) by (pod, namespace)\n * on (namespace, pod)\n group_left(label_name) kube_pod_labels{job=\"kube-state-metrics\"}\n)\n", + "record": "namespace:container_memory_usage_bytes:sum" }, { - "expr": "sum by (namespace, label_name) (\n sum(kube_pod_container_resource_requests_memory_bytes{job=\"kube-state-metrics\"} * on (endpoint, instance, job, namespace, pod, service) group_left(phase) (kube_pod_status_phase{phase=~\"^(Pending|Running)$\"} == 1)) by (namespace, pod)\n* on (namespace, pod) group_left(label_name)\n label_replace(kube_pod_labels{job=\"kube-state-metrics\"}, \"pod_name\", \"$1\", \"pod\", \"(.*)\")\n)\n", - "record": "namespace_name:kube_pod_container_resource_requests_memory_bytes:sum" + "expr": "sum by (namespace, label_name) (\n sum(kube_pod_container_resource_requests_memory_bytes{job=\"kube-state-metrics\"} * on (endpoint, instance, job, namespace, pod, service) group_left(phase) (kube_pod_status_phase{phase=~\"^(Pending|Running)$\"} == 1)) by (namespace, pod)\n * on (namespace, pod)\n group_left(label_name) kube_pod_labels{job=\"kube-state-metrics\"}\n)\n", + "record": "namespace:kube_pod_container_resource_requests_memory_bytes:sum" }, { - "expr": "sum by (namespace, label_name) (\n sum(kube_pod_container_resource_requests_cpu_cores{job=\"kube-state-metrics\"} * on (endpoint, instance, job, namespace, pod, service) group_left(phase) (kube_pod_status_phase{phase=~\"^(Pending|Running)$\"} == 1)) by (namespace, pod)\n* on (namespace, pod) group_left(label_name)\n label_replace(kube_pod_labels{job=\"kube-state-metrics\"}, \"pod_name\", \"$1\", \"pod\", \"(.*)\")\n)\n", - "record": "namespace_name:kube_pod_container_resource_requests_cpu_cores:sum" + "expr": "sum by (namespace, label_name) (\n sum(kube_pod_container_resource_requests_cpu_cores{job=\"kube-state-metrics\"} * on (endpoint, instance, job, namespace, pod, service) group_left(phase) (kube_pod_status_phase{phase=~\"^(Pending|Running)$\"} == 1)) by (namespace, pod)\n * on (namespace, pod)\n group_left(label_name) kube_pod_labels{job=\"kube-state-metrics\"}\n)\n", + "record": "namespace:kube_pod_container_resource_requests_cpu_cores:sum" }, { "expr": "sum(\n label_replace(\n label_replace(\n kube_pod_owner{job=\"kube-state-metrics\", owner_kind=\"ReplicaSet\"},\n \"replicaset\", \"$1\", \"owner_name\", \"(.*)\"\n ) * on(replicaset, namespace) group_left(owner_name) kube_replicaset_owner{job=\"kube-state-metrics\"},\n \"workload\", \"$1\", \"owner_name\", \"(.*)\"\n )\n) by (namespace, workload, pod)\n", @@ -217,67 +217,67 @@ data: "name": "kube-scheduler.rules", "rules": [ { - "expr": "histogram_quantile(0.99, sum(rate(scheduler_e2e_scheduling_latency_microseconds_bucket{job=\"kube-scheduler\"}[5m])) without(instance, pod)) / 1e+06\n", + "expr": "histogram_quantile(0.99, sum(rate(scheduler_e2e_scheduling_duration_seconds_bucket{job=\"kube-scheduler\"}[5m])) without(instance, pod))\n", "labels": { "quantile": "0.99" }, - "record": "cluster_quantile:scheduler_e2e_scheduling_latency:histogram_quantile" + "record": "cluster_quantile:scheduler_e2e_scheduling_duration_seconds:histogram_quantile" }, { - "expr": "histogram_quantile(0.99, sum(rate(scheduler_scheduling_algorithm_latency_microseconds_bucket{job=\"kube-scheduler\"}[5m])) without(instance, pod)) / 1e+06\n", + "expr": "histogram_quantile(0.99, sum(rate(scheduler_scheduling_algorithm_duration_seconds_bucket{job=\"kube-scheduler\"}[5m])) without(instance, pod))\n", "labels": { "quantile": "0.99" }, - "record": "cluster_quantile:scheduler_scheduling_algorithm_latency:histogram_quantile" + "record": "cluster_quantile:scheduler_scheduling_algorithm_duration_seconds:histogram_quantile" }, { - "expr": "histogram_quantile(0.99, sum(rate(scheduler_binding_latency_microseconds_bucket{job=\"kube-scheduler\"}[5m])) without(instance, pod)) / 1e+06\n", + "expr": "histogram_quantile(0.99, sum(rate(scheduler_binding_duration_seconds_bucket{job=\"kube-scheduler\"}[5m])) without(instance, pod))\n", "labels": { "quantile": "0.99" }, - "record": "cluster_quantile:scheduler_binding_latency:histogram_quantile" + "record": "cluster_quantile:scheduler_binding_duration_seconds:histogram_quantile" }, { - "expr": "histogram_quantile(0.9, sum(rate(scheduler_e2e_scheduling_latency_microseconds_bucket{job=\"kube-scheduler\"}[5m])) without(instance, pod)) / 1e+06\n", + "expr": "histogram_quantile(0.9, sum(rate(scheduler_e2e_scheduling_duration_seconds_bucket{job=\"kube-scheduler\"}[5m])) without(instance, pod))\n", "labels": { "quantile": "0.9" }, - "record": "cluster_quantile:scheduler_e2e_scheduling_latency:histogram_quantile" + "record": "cluster_quantile:scheduler_e2e_scheduling_duration_seconds:histogram_quantile" }, { - "expr": "histogram_quantile(0.9, sum(rate(scheduler_scheduling_algorithm_latency_microseconds_bucket{job=\"kube-scheduler\"}[5m])) without(instance, pod)) / 1e+06\n", + "expr": "histogram_quantile(0.9, sum(rate(scheduler_scheduling_algorithm_duration_seconds_bucket{job=\"kube-scheduler\"}[5m])) without(instance, pod))\n", "labels": { "quantile": "0.9" }, - "record": "cluster_quantile:scheduler_scheduling_algorithm_latency:histogram_quantile" + "record": "cluster_quantile:scheduler_scheduling_algorithm_duration_seconds:histogram_quantile" }, { - "expr": "histogram_quantile(0.9, sum(rate(scheduler_binding_latency_microseconds_bucket{job=\"kube-scheduler\"}[5m])) without(instance, pod)) / 1e+06\n", + "expr": "histogram_quantile(0.9, sum(rate(scheduler_binding_duration_seconds_bucket{job=\"kube-scheduler\"}[5m])) without(instance, pod))\n", "labels": { "quantile": "0.9" }, - "record": "cluster_quantile:scheduler_binding_latency:histogram_quantile" + "record": "cluster_quantile:scheduler_binding_duration_seconds:histogram_quantile" }, { - "expr": "histogram_quantile(0.5, sum(rate(scheduler_e2e_scheduling_latency_microseconds_bucket{job=\"kube-scheduler\"}[5m])) without(instance, pod)) / 1e+06\n", + "expr": "histogram_quantile(0.5, sum(rate(scheduler_e2e_scheduling_duration_seconds_bucket{job=\"kube-scheduler\"}[5m])) without(instance, pod))\n", "labels": { "quantile": "0.5" }, - "record": "cluster_quantile:scheduler_e2e_scheduling_latency:histogram_quantile" + "record": "cluster_quantile:scheduler_e2e_scheduling_duration_seconds:histogram_quantile" }, { - "expr": "histogram_quantile(0.5, sum(rate(scheduler_scheduling_algorithm_latency_microseconds_bucket{job=\"kube-scheduler\"}[5m])) without(instance, pod)) / 1e+06\n", + "expr": "histogram_quantile(0.5, sum(rate(scheduler_scheduling_algorithm_duration_seconds_bucket{job=\"kube-scheduler\"}[5m])) without(instance, pod))\n", "labels": { "quantile": "0.5" }, - "record": "cluster_quantile:scheduler_scheduling_algorithm_latency:histogram_quantile" + "record": "cluster_quantile:scheduler_scheduling_algorithm_duration_seconds:histogram_quantile" }, { - "expr": "histogram_quantile(0.5, sum(rate(scheduler_binding_latency_microseconds_bucket{job=\"kube-scheduler\"}[5m])) without(instance, pod)) / 1e+06\n", + "expr": "histogram_quantile(0.5, sum(rate(scheduler_binding_duration_seconds_bucket{job=\"kube-scheduler\"}[5m])) without(instance, pod))\n", "labels": { "quantile": "0.5" }, - "record": "cluster_quantile:scheduler_binding_latency:histogram_quantile" + "record": "cluster_quantile:scheduler_binding_duration_seconds:histogram_quantile" } ] }, @@ -285,25 +285,25 @@ data: "name": "kube-apiserver.rules", "rules": [ { - "expr": "histogram_quantile(0.99, sum(rate(apiserver_request_latencies_bucket{job=\"apiserver\"}[5m])) without(instance, pod)) / 1e+06\n", + "expr": "histogram_quantile(0.99, sum(rate(apiserver_request_duration_seconds_bucket{job=\"apiserver\"}[5m])) without(instance, pod))\n", "labels": { "quantile": "0.99" }, - "record": "cluster_quantile:apiserver_request_latencies:histogram_quantile" + "record": "cluster_quantile:apiserver_request_duration_seconds:histogram_quantile" }, { - "expr": "histogram_quantile(0.9, sum(rate(apiserver_request_latencies_bucket{job=\"apiserver\"}[5m])) without(instance, pod)) / 1e+06\n", + "expr": "histogram_quantile(0.9, sum(rate(apiserver_request_duration_seconds_bucket{job=\"apiserver\"}[5m])) without(instance, pod))\n", "labels": { "quantile": "0.9" }, - "record": "cluster_quantile:apiserver_request_latencies:histogram_quantile" + "record": "cluster_quantile:apiserver_request_duration_seconds:histogram_quantile" }, { - "expr": "histogram_quantile(0.5, sum(rate(apiserver_request_latencies_bucket{job=\"apiserver\"}[5m])) without(instance, pod)) / 1e+06\n", + "expr": "histogram_quantile(0.5, sum(rate(apiserver_request_duration_seconds_bucket{job=\"apiserver\"}[5m])) without(instance, pod))\n", "labels": { "quantile": "0.5" }, - "record": "cluster_quantile:apiserver_request_latencies:histogram_quantile" + "record": "cluster_quantile:apiserver_request_duration_seconds:histogram_quantile" } ] }, @@ -403,11 +403,11 @@ data: "record": "node:node_disk_saturation:avg_irate" }, { - "expr": "max by (namespace, pod, device) ((node_filesystem_size_bytes{fstype=~\"ext[234]|btrfs|xfs|zfs\"}\n- node_filesystem_avail_bytes{fstype=~\"ext[234]|btrfs|xfs|zfs\"})\n/ node_filesystem_size_bytes{fstype=~\"ext[234]|btrfs|xfs|zfs\"})\n", + "expr": "max by (instance, namespace, pod, device) ((node_filesystem_size_bytes{fstype=~\"ext[234]|btrfs|xfs|zfs\"}\n- node_filesystem_avail_bytes{fstype=~\"ext[234]|btrfs|xfs|zfs\"})\n/ node_filesystem_size_bytes{fstype=~\"ext[234]|btrfs|xfs|zfs\"})\n", "record": "node:node_filesystem_usage:" }, { - "expr": "max by (namespace, pod, device) (node_filesystem_avail_bytes{fstype=~\"ext[234]|btrfs|xfs|zfs\"} / node_filesystem_size_bytes{fstype=~\"ext[234]|btrfs|xfs|zfs\"})\n", + "expr": "max by (instance, namespace, pod, device) (node_filesystem_avail_bytes{fstype=~\"ext[234]|btrfs|xfs|zfs\"} / node_filesystem_size_bytes{fstype=~\"ext[234]|btrfs|xfs|zfs\"})\n", "record": "node:node_filesystem_avail:" }, { @@ -510,7 +510,7 @@ data: "message": "Pod {{ $labels.namespace }}/{{ $labels.pod }} has been in a non-ready state for longer than an hour.", "runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubepodnotready" }, - "expr": "sum by (namespace, pod) (kube_pod_status_phase{job=\"kube-state-metrics\", phase=~\"Pending|Unknown\"}) > 0\n", + "expr": "sum by (namespace, pod) (kube_pod_status_phase{job=\"kube-state-metrics\", phase=~\"Failed|Pending|Unknown\"}) > 0\n", "for": "1h", "labels": { "severity": "critical" @@ -659,7 +659,7 @@ data: "message": "Cluster has overcommitted CPU resource requests for Pods and cannot tolerate node failure.", "runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubecpuovercommit" }, - "expr": "sum(namespace_name:kube_pod_container_resource_requests_cpu_cores:sum)\n /\nsum(node:node_num_cpu:sum)\n >\n(count(node:node_num_cpu:sum)-1) / count(node:node_num_cpu:sum)\n", + "expr": "sum(namespace:kube_pod_container_resource_requests_cpu_cores:sum)\n /\nsum(kube_node_status_allocatable_cpu_cores)\n >\n(count(kube_node_status_allocatable_cpu_cores)-1) / count(kube_node_status_allocatable_cpu_cores)\n", "for": "5m", "labels": { "severity": "warning" @@ -671,7 +671,7 @@ data: "message": "Cluster has overcommitted memory resource requests for Pods and cannot tolerate node failure.", "runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubememovercommit" }, - "expr": "sum(namespace_name:kube_pod_container_resource_requests_memory_bytes:sum)\n /\nsum(node_memory_MemTotal_bytes)\n >\n(count(node:node_num_cpu:sum)-1)\n /\ncount(node:node_num_cpu:sum)\n", + "expr": "sum(namespace:kube_pod_container_resource_requests_memory_bytes:sum)\n /\nsum(kube_node_status_allocatable_memory_bytes)\n >\n(count(kube_node_status_allocatable_memory_bytes)-1)\n /\ncount(kube_node_status_allocatable_memory_bytes)\n", "for": "5m", "labels": { "severity": "warning" @@ -683,7 +683,7 @@ data: "message": "Cluster has overcommitted CPU resource requests for Namespaces.", "runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubecpuovercommit" }, - "expr": "sum(kube_resourcequota{job=\"kube-state-metrics\", type=\"hard\", resource=\"cpu\"})\n /\nsum(node:node_num_cpu:sum)\n > 1.5\n", + "expr": "sum(kube_resourcequota{job=\"kube-state-metrics\", type=\"hard\", resource=\"cpu\"})\n /\nsum(kube_node_status_allocatable_cpu_cores)\n > 1.5\n", "for": "5m", "labels": { "severity": "warning" @@ -695,7 +695,7 @@ data: "message": "Cluster has overcommitted memory resource requests for Namespaces.", "runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubememovercommit" }, - "expr": "sum(kube_resourcequota{job=\"kube-state-metrics\", type=\"hard\", resource=\"memory\"})\n /\nsum(node_memory_MemTotal_bytes{job=\"node-exporter\"})\n > 1.5\n", + "expr": "sum(kube_resourcequota{job=\"kube-state-metrics\", type=\"hard\", resource=\"memory\"})\n /\nsum(kube_node_status_allocatable_memory_bytes{job=\"node-exporter\"})\n > 1.5\n", "for": "5m", "labels": { "severity": "warning" @@ -716,10 +716,10 @@ data: { "alert": "CPUThrottlingHigh", "annotations": { - "message": "{{ printf \"%0.0f\" $value }}% throttling of CPU in namespace {{ $labels.namespace }} for container {{ $labels.container_name }} in pod {{ $labels.pod_name }}.", + "message": "{{ printf \"%0.0f\" $value }}% throttling of CPU in namespace {{ $labels.namespace }} for container {{ $labels.container }} in pod {{ $labels.pod }}.", "runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-cputhrottlinghigh" }, - "expr": "100 * sum(increase(container_cpu_cfs_throttled_periods_total{container_name!=\"\", }[5m])) by (container_name, pod_name, namespace)\n /\nsum(increase(container_cpu_cfs_periods_total{}[5m])) by (container_name, pod_name, namespace)\n > 100 \n", + "expr": "100 * sum(increase(container_cpu_cfs_throttled_periods_total{container!=\"\", }[5m])) by (container, pod, namespace)\n /\nsum(increase(container_cpu_cfs_periods_total{}[5m])) by (container, pod, namespace)\n > 100 \n", "for": "15m", "labels": { "severity": "warning" @@ -837,7 +837,7 @@ data: "message": "The API server has a 99th percentile latency of {{ $value }} seconds for {{ $labels.verb }} {{ $labels.resource }}.", "runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubeapilatencyhigh" }, - "expr": "cluster_quantile:apiserver_request_latencies:histogram_quantile{job=\"apiserver\",quantile=\"0.99\",subresource!=\"log\",verb!~\"^(?:LIST|WATCH|WATCHLIST|PROXY|CONNECT)$\"} > 1\n", + "expr": "cluster_quantile:apiserver_request_duration_seconds:histogram_quantile{job=\"apiserver\",quantile=\"0.99\",subresource!=\"log\",verb!~\"^(?:LIST|WATCH|WATCHLIST|PROXY|CONNECT)$\"} > 1\n", "for": "10m", "labels": { "severity": "warning" @@ -849,7 +849,7 @@ data: "message": "The API server has a 99th percentile latency of {{ $value }} seconds for {{ $labels.verb }} {{ $labels.resource }}.", "runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubeapilatencyhigh" }, - "expr": "cluster_quantile:apiserver_request_latencies:histogram_quantile{job=\"apiserver\",quantile=\"0.99\",subresource!=\"log\",verb!~\"^(?:LIST|WATCH|WATCHLIST|PROXY|CONNECT)$\"} > 4\n", + "expr": "cluster_quantile:apiserver_request_duration_seconds:histogram_quantile{job=\"apiserver\",quantile=\"0.99\",subresource!=\"log\",verb!~\"^(?:LIST|WATCH|WATCHLIST|PROXY|CONNECT)$\"} > 4\n", "for": "10m", "labels": { "severity": "critical" @@ -861,7 +861,7 @@ data: "message": "API server is returning errors for {{ $value }}% of requests.", "runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubeapierrorshigh" }, - "expr": "sum(rate(apiserver_request_count{job=\"apiserver\",code=~\"^(?:5..)$\"}[5m]))\n /\nsum(rate(apiserver_request_count{job=\"apiserver\"}[5m])) * 100 > 3\n", + "expr": "sum(rate(apiserver_request_total{job=\"apiserver\",code=~\"^(?:5..)$\"}[5m]))\n /\nsum(rate(apiserver_request_total{job=\"apiserver\"}[5m])) * 100 > 3\n", "for": "10m", "labels": { "severity": "critical" @@ -873,7 +873,7 @@ data: "message": "API server is returning errors for {{ $value }}% of requests.", "runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubeapierrorshigh" }, - "expr": "sum(rate(apiserver_request_count{job=\"apiserver\",code=~\"^(?:5..)$\"}[5m]))\n /\nsum(rate(apiserver_request_count{job=\"apiserver\"}[5m])) * 100 > 1\n", + "expr": "sum(rate(apiserver_request_total{job=\"apiserver\",code=~\"^(?:5..)$\"}[5m]))\n /\nsum(rate(apiserver_request_total{job=\"apiserver\"}[5m])) * 100 > 1\n", "for": "10m", "labels": { "severity": "warning" @@ -885,7 +885,7 @@ data: "message": "API server is returning errors for {{ $value }}% of requests for {{ $labels.verb }} {{ $labels.resource }} {{ $labels.subresource }}.", "runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubeapierrorshigh" }, - "expr": "sum(rate(apiserver_request_count{job=\"apiserver\",code=~\"^(?:5..)$\"}[5m])) by (resource,subresource,verb)\n /\nsum(rate(apiserver_request_count{job=\"apiserver\"}[5m])) by (resource,subresource,verb) * 100 > 10\n", + "expr": "sum(rate(apiserver_request_total{job=\"apiserver\",code=~\"^(?:5..)$\"}[5m])) by (resource,subresource,verb)\n /\nsum(rate(apiserver_request_total{job=\"apiserver\"}[5m])) by (resource,subresource,verb) * 100 > 10\n", "for": "10m", "labels": { "severity": "critical" @@ -897,7 +897,7 @@ data: "message": "API server is returning errors for {{ $value }}% of requests for {{ $labels.verb }} {{ $labels.resource }} {{ $labels.subresource }}.", "runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubeapierrorshigh" }, - "expr": "sum(rate(apiserver_request_count{job=\"apiserver\",code=~\"^(?:5..)$\"}[5m])) by (resource,subresource,verb)\n /\nsum(rate(apiserver_request_count{job=\"apiserver\"}[5m])) by (resource,subresource,verb) * 100 > 5\n", + "expr": "sum(rate(apiserver_request_total{job=\"apiserver\",code=~\"^(?:5..)$\"}[5m])) by (resource,subresource,verb)\n /\nsum(rate(apiserver_request_total{job=\"apiserver\"}[5m])) by (resource,subresource,verb) * 100 > 5\n", "for": "10m", "labels": { "severity": "warning" @@ -1047,52 +1047,73 @@ data: ] }, { - "name": "prometheus.rules", + "name": "general.rules", "rules": [ { - "alert": "PrometheusConfigReloadFailed", + "alert": "TargetDown", "annotations": { - "description": "Reloading Prometheus' configuration has failed for {{$labels.namespace}}/{{$labels.pod}}", - "summary": "Reloading Prometheus' configuration failed" + "message": "{{ $value }}% of the {{ $labels.job }} targets are down." }, - "expr": "prometheus_config_last_reload_successful{job=\"prometheus\"} == 0\n", + "expr": "100 * (count(up == 0) BY (job) / count(up) BY (job)) > 10", "for": "10m", "labels": { "severity": "warning" } + } + ] + } + ] + } + prom.yaml: |- + { + "groups": [ + { + "name": "prometheus", + "rules": [ + { + "alert": "PrometheusBadConfig", + "annotations": { + "description": "Prometheus {{$labels.instance}} has failed to reload its configuration.", + "summary": "Failed Prometheus configuration reload." + }, + "expr": "# Without max_over_time, failed scrapes could create false negatives, see\n# https://www.robustperception.io/alerting-on-gauges-in-prometheus-2-0 for details.\nmax_over_time(prometheus_config_last_reload_successful{job=\"prometheus\"}[5m]) == 0\n", + "for": "10m", + "labels": { + "severity": "critical" + } }, { "alert": "PrometheusNotificationQueueRunningFull", "annotations": { - "description": "Prometheus' alert notification queue is running full for {{$labels.namespace}}/{{ $labels.pod}}", - "summary": "Prometheus' alert notification queue is running full" + "description": "Alert notification queue of Prometheus {{$labels.instance}} is running full.", + "summary": "Prometheus alert notification queue predicted to run full in less than 30m." }, - "expr": "predict_linear(prometheus_notifications_queue_length{job=\"prometheus\"}[5m], 60 * 30) > prometheus_notifications_queue_capacity{job=\"prometheus\"}\n", - "for": "10m", + "expr": "# Without min_over_time, failed scrapes could create false negatives, see\n# https://www.robustperception.io/alerting-on-gauges-in-prometheus-2-0 for details.\n(\n predict_linear(prometheus_notifications_queue_length{job=\"prometheus\"}[5m], 60 * 30)\n>\n min_over_time(prometheus_notifications_queue_capacity{job=\"prometheus\"}[5m])\n)\n", + "for": "15m", "labels": { "severity": "warning" } }, { - "alert": "PrometheusErrorSendingAlerts", + "alert": "PrometheusErrorSendingAlertsToSomeAlertmanagers", "annotations": { - "description": "Errors while sending alerts from Prometheus {{$labels.namespace}}/{{ $labels.pod}} to Alertmanager {{$labels.Alertmanager}}", - "summary": "Errors while sending alert from Prometheus" + "description": "{{ printf \"%.1f\" $value }}% errors while sending alerts from Prometheus {{$labels.instance}} to Alertmanager {{$labels.alertmanager}}.", + "summary": "Prometheus has encountered more than 1% errors sending alerts to a specific Alertmanager." }, - "expr": "rate(prometheus_notifications_errors_total{job=\"prometheus\"}[5m]) / rate(prometheus_notifications_sent_total{job=\"prometheus\"}[5m]) > 0.01\n", - "for": "10m", + "expr": "(\n rate(prometheus_notifications_errors_total{job=\"prometheus\"}[5m])\n/\n rate(prometheus_notifications_sent_total{job=\"prometheus\"}[5m])\n)\n* 100\n> 1\n", + "for": "15m", "labels": { "severity": "warning" } }, { - "alert": "PrometheusErrorSendingAlerts", + "alert": "PrometheusErrorSendingAlertsToAnyAlertmanager", "annotations": { - "description": "Errors while sending alerts from Prometheus {{$labels.namespace}}/{{ $labels.pod}} to Alertmanager {{$labels.Alertmanager}}", - "summary": "Errors while sending alerts from Prometheus" + "description": "{{ printf \"%.1f\" $value }}% minimum errors while sending alerts from Prometheus {{$labels.instance}} to any Alertmanager.", + "summary": "Prometheus encounters more than 3% errors sending alerts to any Alertmanager." }, - "expr": "rate(prometheus_notifications_errors_total{job=\"prometheus\"}[5m]) / rate(prometheus_notifications_sent_total{job=\"prometheus\"}[5m]) > 0.03\n", - "for": "10m", + "expr": "min without(alertmanager) (\n rate(prometheus_notifications_errors_total{job=\"prometheus\"}[5m])\n/\n rate(prometheus_notifications_sent_total{job=\"prometheus\"}[5m])\n)\n* 100\n> 3\n", + "for": "15m", "labels": { "severity": "critical" } @@ -1100,10 +1121,10 @@ data: { "alert": "PrometheusNotConnectedToAlertmanagers", "annotations": { - "description": "Prometheus {{ $labels.namespace }}/{{ $labels.pod}} is not connected to any Alertmanagers", - "summary": "Prometheus is not connected to any Alertmanagers" + "description": "Prometheus {{$labels.instance}} is not connected to any Alertmanagers.", + "summary": "Prometheus is not connected to any Alertmanagers." }, - "expr": "prometheus_notifications_alertmanagers_discovered{job=\"prometheus\"} < 1\n", + "expr": "# Without max_over_time, failed scrapes could create false negatives, see\n# https://www.robustperception.io/alerting-on-gauges-in-prometheus-2-0 for details.\nmax_over_time(prometheus_notifications_alertmanagers_discovered{job=\"prometheus\"}[5m]) < 1\n", "for": "10m", "labels": { "severity": "warning" @@ -1112,11 +1133,11 @@ data: { "alert": "PrometheusTSDBReloadsFailing", "annotations": { - "description": "{{$labels.job}} at {{$labels.instance}} had {{$value | humanize}} reload failures over the last four hours.", - "summary": "Prometheus has issues reloading data blocks from disk" + "description": "Prometheus {{$labels.instance}} has detected {{$value | humanize}} reload failures over the last 3h.", + "summary": "Prometheus has issues reloading blocks from disk." }, - "expr": "increase(prometheus_tsdb_reloads_failures_total{job=\"prometheus\"}[2h]) > 0\n", - "for": "12h", + "expr": "increase(prometheus_tsdb_reloads_failures_total{job=\"prometheus\"}[3h]) > 0\n", + "for": "4h", "labels": { "severity": "warning" } @@ -1124,11 +1145,11 @@ data: { "alert": "PrometheusTSDBCompactionsFailing", "annotations": { - "description": "{{$labels.job}} at {{$labels.instance}} had {{$value | humanize}} compaction failures over the last four hours.", - "summary": "Prometheus has issues compacting sample blocks" + "description": "Prometheus {{$labels.instance}} has detected {{$value | humanize}} compaction failures over the last 3h.", + "summary": "Prometheus has issues compacting blocks." }, - "expr": "increase(prometheus_tsdb_compactions_failed_total{job=\"prometheus\"}[2h]) > 0\n", - "for": "12h", + "expr": "increase(prometheus_tsdb_compactions_failed_total{job=\"prometheus\"}[3h]) > 0\n", + "for": "4h", "labels": { "severity": "warning" } @@ -1136,10 +1157,10 @@ data: { "alert": "PrometheusTSDBWALCorruptions", "annotations": { - "description": "{{$labels.job}} at {{$labels.instance}} has a corrupted write-ahead log (WAL).", - "summary": "Prometheus write-ahead log is corrupted" + "description": "Prometheus {{$labels.instance}} has detected {{$value | humanize}} corruptions of the write-ahead log (WAL) over the last 3h.", + "summary": "Prometheus is detecting WAL corruptions." }, - "expr": "prometheus_tsdb_wal_corruptions_total{job=\"prometheus\"} > 0\n", + "expr": "increase(tsdb_wal_corruptions_total{job=\"prometheus\"}[3h]) > 0\n", "for": "4h", "labels": { "severity": "warning" @@ -1148,8 +1169,8 @@ data: { "alert": "PrometheusNotIngestingSamples", "annotations": { - "description": "Prometheus {{ $labels.namespace }}/{{ $labels.pod}} isn't ingesting samples.", - "summary": "Prometheus isn't ingesting samples" + "description": "Prometheus {{$labels.instance}} is not ingesting samples.", + "summary": "Prometheus is not ingesting samples." }, "expr": "rate(prometheus_tsdb_head_samples_appended_total{job=\"prometheus\"}[5m]) <= 0\n", "for": "10m", @@ -1158,32 +1179,76 @@ data: } }, { - "alert": "PrometheusTargetScrapesDuplicate", + "alert": "PrometheusDuplicateTimestamps", "annotations": { - "description": "{{$labels.namespace}}/{{$labels.pod}} has many samples rejected due to duplicate timestamps but different values", - "summary": "Prometheus has many samples rejected" + "description": "Prometheus {{$labels.instance}} is dropping {{$value | humanize}} samples/s with different values but duplicated timestamp.", + "summary": "Prometheus is dropping samples with duplicate timestamps." }, - "expr": "increase(prometheus_target_scrapes_sample_duplicate_timestamp_total{job=\"prometheus\"}[5m]) > 0\n", + "expr": "rate(prometheus_target_scrapes_sample_duplicate_timestamp_total{job=\"prometheus\"}[5m]) > 0\n", "for": "10m", "labels": { "severity": "warning" } - } - ] - }, - { - "name": "general.rules", - "rules": [ + }, { - "alert": "TargetDown", + "alert": "PrometheusOutOfOrderTimestamps", "annotations": { - "message": "{{ $value }}% of the {{ $labels.job }} targets are down." + "description": "Prometheus {{$labels.instance}} is dropping {{$value | humanize}} samples/s with timestamps arriving out of order.", + "summary": "Prometheus drops samples with out-of-order timestamps." }, - "expr": "100 * (count(up == 0) BY (job) / count(up) BY (job)) > 10", + "expr": "rate(prometheus_target_scrapes_sample_out_of_order_total{job=\"prometheus\"}[5m]) > 0\n", "for": "10m", "labels": { "severity": "warning" } + }, + { + "alert": "PrometheusRemoteStorageFailures", + "annotations": { + "description": "Prometheus {{$labels.instance}} failed to send {{ printf \"%.1f\" $value }}% of the samples to queue {{$labels.queue}}.", + "summary": "Prometheus fails to send samples to remote storage." + }, + "expr": "(\n rate(prometheus_remote_storage_failed_samples_total{job=\"prometheus\"}[5m])\n/\n (\n rate(prometheus_remote_storage_failed_samples_total{job=\"prometheus\"}[5m])\n +\n rate(prometheus_remote_storage_succeeded_samples_total{job=\"prometheus\"}[5m])\n )\n)\n* 100\n> 1\n", + "for": "15m", + "labels": { + "severity": "critical" + } + }, + { + "alert": "PrometheusRemoteWriteBehind", + "annotations": { + "description": "Prometheus {{$labels.instance}} remote write is {{ printf \"%.1f\" $value }}s behind for queue {{$labels.queue}}.", + "summary": "Prometheus remote write is behind." + }, + "expr": "# Without max_over_time, failed scrapes could create false negatives, see\n# https://www.robustperception.io/alerting-on-gauges-in-prometheus-2-0 for details.\n(\n max_over_time(prometheus_remote_storage_highest_timestamp_in_seconds{job=\"prometheus\"}[5m])\n- on(job, instance) group_right\n max_over_time(prometheus_remote_storage_queue_highest_sent_timestamp_seconds{job=\"prometheus\"}[5m])\n)\n> 120\n", + "for": "15m", + "labels": { + "severity": "critical" + } + }, + { + "alert": "PrometheusRuleFailures", + "annotations": { + "description": "Prometheus {{$labels.instance}} has failed to evaluate {{ printf \"%.0f\" $value }} rules in the last 5m.", + "summary": "Prometheus is failing rule evaluations." + }, + "expr": "increase(prometheus_rule_evaluation_failures_total{job=\"prometheus\"}[5m]) > 0\n", + "for": "15m", + "labels": { + "severity": "critical" + } + }, + { + "alert": "PrometheusMissingRuleEvaluations", + "annotations": { + "description": "Prometheus {{$labels.instance}} has missed {{ printf \"%.0f\" $value }} rule group evaluations in the last 5m.", + "summary": "Prometheus is missing rule evaluations due to slow rule group evaluation." + }, + "expr": "increase(prometheus_rule_group_iterations_missed_total{job=\"prometheus\"}[5m]) > 0\n", + "for": "15m", + "labels": { + "severity": "warning" + } } ] } From b9ccfedfe585986da085240b020f18854e2edf3e Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Sun, 21 Jul 2019 11:58:56 -0700 Subject: [PATCH 174/523] Update CHANGES for v1.15.1 release --- CHANGES.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGES.md b/CHANGES.md index 99d35df18..116bc3b3e 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -4,6 +4,8 @@ Notable changes between versions. ## Latest +## v1.15.1 + * Kubernetes [v1.15.1](https://github.com/kubernetes/kubernetes/blob/master/CHANGELOG-1.15.md#v1151) * Upgrade Calico from v3.7.3 to [v3.8.0](https://docs.projectcalico.org/v3.8/release-notes/) * Enable CNI `bandwidth` plugin for [traffic shaping](https://kubernetes.io/docs/concepts/extend-kubernetes/compute-storage-net/network-plugins/#support-traffic-shaping) From 8cb7fe48a11630540966417ed810ed693905d919 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Sat, 27 Jul 2019 15:18:29 -0700 Subject: [PATCH 175/523] Update Fedora CoreOS to testing 30.20190725.0 * Fedora CoreOS Preview AMI are pinned until maturity --- aws/fedora-coreos/kubernetes/ami.tf | 2 +- aws/fedora-coreos/kubernetes/workers/ami.tf | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/aws/fedora-coreos/kubernetes/ami.tf b/aws/fedora-coreos/kubernetes/ami.tf index f4ecec661..7d27a2908 100644 --- a/aws/fedora-coreos/kubernetes/ami.tf +++ b/aws/fedora-coreos/kubernetes/ami.tf @@ -16,6 +16,6 @@ data "aws_ami" "fedora-coreos" { // pin on known ok versions as preview matures filter { name = "name" - values = ["fedora-coreos-30.20190716.1-hvm"] + values = ["fedora-coreos-30.20190725.0-hvm"] } } diff --git a/aws/fedora-coreos/kubernetes/workers/ami.tf b/aws/fedora-coreos/kubernetes/workers/ami.tf index f4ecec661..7d27a2908 100644 --- a/aws/fedora-coreos/kubernetes/workers/ami.tf +++ b/aws/fedora-coreos/kubernetes/workers/ami.tf @@ -16,6 +16,6 @@ data "aws_ami" "fedora-coreos" { // pin on known ok versions as preview matures filter { name = "name" - values = ["fedora-coreos-30.20190716.1-hvm"] + values = ["fedora-coreos-30.20190725.0-hvm"] } } From 1409bc62d8e12a1b87b1361692f7bea796bd5347 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Sat, 27 Jul 2019 15:23:34 -0700 Subject: [PATCH 176/523] Remove download_protocol variable from Fedora CoreOS * For Fedora CoreOS, only HTTPS downloads are available. Any iPXE firmware must be compiled to support TLS fetching. * For Container Linux, using public kernel/initramfs images defaults to using HTTPS, but can be set to HTTP for iPXE firmware that hasn't been custom compiled to support TLS --- bare-metal/fedora-coreos/kubernetes/variables.tf | 6 ------ docs/fedora-coreos/bare-metal.md | 2 +- 2 files changed, 1 insertion(+), 7 deletions(-) diff --git a/bare-metal/fedora-coreos/kubernetes/variables.tf b/bare-metal/fedora-coreos/kubernetes/variables.tf index 5cae2d30a..30411c9ed 100644 --- a/bare-metal/fedora-coreos/kubernetes/variables.tf +++ b/bare-metal/fedora-coreos/kubernetes/variables.tf @@ -120,12 +120,6 @@ variable "cluster_domain_suffix" { default = "cluster.local" } -variable "download_protocol" { - type = string - default = "https" - description = "Protocol iPXE should use to download the kernel and initrd. Defaults to https, which requires iPXE compiled with crypto support. Unused if cached_install is true." -} - variable "cached_install" { type = string default = "false" diff --git a/docs/fedora-coreos/bare-metal.md b/docs/fedora-coreos/bare-metal.md index 816fec3b0..5401130e2 100644 --- a/docs/fedora-coreos/bare-metal.md +++ b/docs/fedora-coreos/bare-metal.md @@ -106,7 +106,7 @@ Read about the [many ways](https://coreos.com/matchbox/docs/latest/network-setup TFTP chainloading to modern boot firmware, like iPXE, avoids issues with old NICs and allows faster transfer protocols like HTTP to be used. !!! warning - Compile iPXE from [source](https://github.com/ipxe/ipxe) with support for [HTTPS downloads](https://ipxe.org/crypto). iPXE's pre-built firmware binaries do not enable this. If you cannot enable HTTPS downloads, set `download_protocol = "http"` (discouraged). + Compile iPXE from [source](https://github.com/ipxe/ipxe) with support for [HTTPS downloads](https://ipxe.org/crypto). iPXE's pre-built firmware binaries do not enable this. Fedora does not provide images over HTTP. ## Terraform Setup From dcd6733649eeb6a912d73fb7a27dc85c2166f4ca Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Sat, 27 Jul 2019 15:31:13 -0700 Subject: [PATCH 177/523] Update Calico from v3.8.0 to v3.8.1 * https://docs.projectcalico.org/v3.8/release-notes/ --- CHANGES.md | 2 ++ aws/container-linux/kubernetes/bootkube.tf | 2 +- aws/fedora-coreos/kubernetes/bootkube.tf | 2 +- azure/container-linux/kubernetes/bootkube.tf | 2 +- bare-metal/container-linux/kubernetes/bootkube.tf | 2 +- bare-metal/fedora-coreos/kubernetes/bootkube.tf | 2 +- digital-ocean/container-linux/kubernetes/bootkube.tf | 2 +- google-cloud/container-linux/kubernetes/bootkube.tf | 2 +- 8 files changed, 9 insertions(+), 7 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 116bc3b3e..a45a9ffc9 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -4,6 +4,8 @@ Notable changes between versions. ## Latest +* Update Calico from v3.8.0 to [v3.8.1](https://docs.projectcalico.org/v3.8/release-notes/) + ## v1.15.1 * Kubernetes [v1.15.1](https://github.com/kubernetes/kubernetes/blob/master/CHANGELOG-1.15.md#v1151) diff --git a/aws/container-linux/kubernetes/bootkube.tf b/aws/container-linux/kubernetes/bootkube.tf index dae12fec8..ea1721113 100644 --- a/aws/container-linux/kubernetes/bootkube.tf +++ b/aws/container-linux/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=5b9faa903182505ac534f42a2cf250d5d215941b" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=83dd5a7cfc5daa9afc259fd4c3ceb22dec9437da" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/aws/fedora-coreos/kubernetes/bootkube.tf b/aws/fedora-coreos/kubernetes/bootkube.tf index d01a9a586..1a2d53dc4 100644 --- a/aws/fedora-coreos/kubernetes/bootkube.tf +++ b/aws/fedora-coreos/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=5b9faa903182505ac534f42a2cf250d5d215941b" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=83dd5a7cfc5daa9afc259fd4c3ceb22dec9437da" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/azure/container-linux/kubernetes/bootkube.tf b/azure/container-linux/kubernetes/bootkube.tf index 86aa94811..1899ac55d 100644 --- a/azure/container-linux/kubernetes/bootkube.tf +++ b/azure/container-linux/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=5b9faa903182505ac534f42a2cf250d5d215941b" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=83dd5a7cfc5daa9afc259fd4c3ceb22dec9437da" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/bare-metal/container-linux/kubernetes/bootkube.tf b/bare-metal/container-linux/kubernetes/bootkube.tf index 55e57608a..c8a68e184 100644 --- a/bare-metal/container-linux/kubernetes/bootkube.tf +++ b/bare-metal/container-linux/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=5b9faa903182505ac534f42a2cf250d5d215941b" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=83dd5a7cfc5daa9afc259fd4c3ceb22dec9437da" cluster_name = var.cluster_name api_servers = [var.k8s_domain_name] diff --git a/bare-metal/fedora-coreos/kubernetes/bootkube.tf b/bare-metal/fedora-coreos/kubernetes/bootkube.tf index 2e2751c18..9188500a1 100644 --- a/bare-metal/fedora-coreos/kubernetes/bootkube.tf +++ b/bare-metal/fedora-coreos/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=5b9faa903182505ac534f42a2cf250d5d215941b" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=83dd5a7cfc5daa9afc259fd4c3ceb22dec9437da" cluster_name = var.cluster_name api_servers = [var.k8s_domain_name] diff --git a/digital-ocean/container-linux/kubernetes/bootkube.tf b/digital-ocean/container-linux/kubernetes/bootkube.tf index f42bfa507..e9e09078a 100644 --- a/digital-ocean/container-linux/kubernetes/bootkube.tf +++ b/digital-ocean/container-linux/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=5b9faa903182505ac534f42a2cf250d5d215941b" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=83dd5a7cfc5daa9afc259fd4c3ceb22dec9437da" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/google-cloud/container-linux/kubernetes/bootkube.tf b/google-cloud/container-linux/kubernetes/bootkube.tf index eb519c2ce..56845e09e 100644 --- a/google-cloud/container-linux/kubernetes/bootkube.tf +++ b/google-cloud/container-linux/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=5b9faa903182505ac534f42a2cf250d5d215941b" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=83dd5a7cfc5daa9afc259fd4c3ceb22dec9437da" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] From a12833531ebbc9e850c4ee30068cd3d1a9003d57 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Mon, 29 Jul 2019 22:41:10 -0700 Subject: [PATCH 178/523] Add new load balancing, TCP/UDP, and firewall docs/diagrams * Describe kube-apiserver load balancing on each platform * Describe HTTP/S Ingress load balancing on each platform * Describe TCP/UDP load balancing apps on each platform (some clouds don't support UDP) * Describe firewall customization (e.g. for TCP/UDP apps) * Update IPv6 status for each platform --- docs/addons/ingress.md | 4 +- docs/architecture/aws.md | 82 ++++++++++++++++- docs/architecture/azure.md | 70 ++++++++++++++- docs/architecture/bare-metal.md | 16 ++++ docs/architecture/digitalocean.md | 83 ++++++++++++++++- docs/architecture/google-cloud.md | 85 +++++++++++++++++- docs/img/typhoon-aws-load-balancing.png | Bin 0 -> 38362 bytes docs/img/typhoon-azure-load-balancing.png | Bin 0 -> 39794 bytes .../typhoon-digitalocean-load-balancing.png | Bin 0 -> 50263 bytes docs/img/typhoon-gcp-load-balancing.png | Bin 0 -> 70481 bytes 10 files changed, 331 insertions(+), 9 deletions(-) create mode 100644 docs/img/typhoon-aws-load-balancing.png create mode 100644 docs/img/typhoon-azure-load-balancing.png create mode 100644 docs/img/typhoon-digitalocean-load-balancing.png create mode 100644 docs/img/typhoon-gcp-load-balancing.png diff --git a/docs/addons/ingress.md b/docs/addons/ingress.md index caf5244c9..4bbbfbe1a 100644 --- a/docs/addons/ingress.md +++ b/docs/addons/ingress.md @@ -101,7 +101,7 @@ resource "google_dns_record_set" "some-application" { ## Digital Ocean -On Digital Ocean, DNS A and AAAA records (e.g. FQDN `nemo-workers.example.com`) resolve to each worker[^1] running an Ingress controller DaemonSet on host ports 80 and 443. Firewall rules allow IPv4 and IPv6 traffic to ports 80 and 443. +On DigitalOcean, DNS A and AAAA records (e.g. FQDN `nemo-workers.example.com`) resolve to each worker[^1] running an Ingress controller DaemonSet on host ports 80 and 443. Firewall rules allow IPv4 and IPv6 traffic to ports 80 and 443. Create the Ingress controller daemonset, service, RBAC roles, RBAC bindings, and namespace. @@ -127,7 +127,7 @@ resource "google_dns_record_set" "some-application" { !!! note Hosting IPv6 apps is possible, but requires editing the nginx-ingress addon to use `hostNetwork: true`. -[^1]: Digital Ocean does offer load balancers. We've opted not to use them to keep the Digital Ocean setup simple and cheap for developers. +[^1]: DigitalOcean does offer load balancers. We've opted not to use them to keep the DigitalOcean cluster cheap for developers. ## Google Cloud diff --git a/docs/architecture/aws.md b/docs/architecture/aws.md index 11c71a3e9..bdcc6aa14 100644 --- a/docs/architecture/aws.md +++ b/docs/architecture/aws.md @@ -1,8 +1,87 @@ # AWS +## Load Balancing + +![Load Balancing](/img/typhoon-aws-load-balancing.png) + +### kube-apiserver + +A network load balancer (NLB) distributes IPv4 TCP/6443 traffic across a target group of controller nodes with a healthy `kube-apiserver`. Clusters with multiple controllers span zones in a region to tolerate zone outages. + +### HTTP/HTTPS Ingress + +A network load balancer (NLB) distributes IPv4 TCP/80 and TCP/443 traffic across two target groups of worker nodes with a healthy Ingress controller. Workers span the zones in a region to tolerate zone outages. + +The AWS NLB has a DNS alias record (regional) resolving to 3 zonal IPv4 addresses. The alias record is output as `ingress_dns_name` for use in application DNS CNAME records. See [Ingress on AWS](/addons/ingress/#aws). + +### TCP Services + +Load balance TCP applications by adding a listener and target group. A listener and target group may map different ports (e.g 3333 external, 30333 internal). + +```tf +# Forward TCP traffic to a target group +resource "aws_lb_listener" "some-app" { + load_balancer_arn = module.tempest.nlb_id + protocol = "TCP" + port = "3333" + + default_action { + type = "forward" + target_group_arn = aws_lb_target_group.some-app.arn + } +} + +# Target group of workers for some-app +resource "aws_lb_target_group" "some-app" { + name = "some-app" + vpc_id = module.tempest.vpc_id + target_type = "instance" + + protocol = "TCP" + port = 3333 + + health_check { + protocol = "TCP" + port = 30333 + } +} +``` + +Pass `worker_target_groups` to the cluster to register worker instances into custom target groups. + +```tf +module "tempest" { +... + worker_target_groups = [ + aws_lb_target_group.some-app.id, + ] +} +``` + +Notes: + +* AWS NLBs and target groups do not support UDP +* Global Accelerator does support UDP, but its expensive + +## Firewalls + +Add firewall rules to the worker security group. + +```tf +resource "aws_security_group_rule" "some-app" { + security_group_id = module.tempest.worker_security_groups[0] + + type = "ingress" + protocol = "tcp" + from_port = 3333 + to_port = 30333 + cidr_blocks = ["0.0.0.0/0"] +} +``` + ## IPv6 -Status of IPv6 on Typhoon AWS clusters. +AWS Network Load Balancers do not support `dualstack`. | IPv6 Feature | Supported | |-------------------------|-----------| @@ -10,4 +89,3 @@ Status of IPv6 on Typhoon AWS clusters. | Node Outbound IPv6 | Yes | | Kubernetes Ingress IPv6 | No | -* AWS Network Load Balancers do not support `dualstack`. diff --git a/docs/architecture/azure.md b/docs/architecture/azure.md index e699e889c..a19384b36 100644 --- a/docs/architecture/azure.md +++ b/docs/architecture/azure.md @@ -1,13 +1,77 @@ # Azure +## Load Balancing + +![Load Balancing](/img/typhoon-azure-load-balancing.png) + +### kube-apiserver + +A load balancer distributes IPv4 TCP/6443 traffic across a backend address pool of controllers with a healthy `kube-apiserver`. Clusters with multiple controllers use an availability set with 2 fault domains to tolerate hardware failures within Azure. + +### HTTP/HTTPS Ingress + +A load balancer distributes IPv4 TCP/80 and TCP/443 traffic across a backend address pool of workers with a healthy Ingress controller. + +The Azure LB IPv4 address is output as `ingress_static_ipv4` for use in DNS A records. See [Ingress on Azure](/addons/ingress/#azure). + +### TCP/UDP Services + +Load balance TCP/UDP applications by adding rules to the Azure LB (output). A rule may map different ports (e.g. 3333 external, 30333 internal). + +```tf +# Forward traffic to the worker backend address pool +resource "azurerm_lb_rule" "some-app-tcp" { + resource_group_name = module.ramius.resource_group_name + + name = "some-app-tcp" + loadbalancer_id = module.ramius.loadbalancer_id + frontend_ip_configuration_name = "ingress" + + protocol = "Tcp" + frontend_port = 3333 + backend_port = 30333 + backend_address_pool_id = module.ramius.backend_address_pool_id + probe_id = azurerm_lb_probe.some-app.id +} + +# Health check some-app +resource "azurerm_lb_probe" "some-app" { + resource_group_name = module.ramius.resource_group_name + + name = "some-app" + loadbalancer_id = module.ramius.loadbalancer_id + protocol = "Tcp" + port = 30333 +} +``` + +## Firewalls + +Add firewall rules to the worker security group. + +```tf +resource "azurerm_network_security_rule" "some-app" { + resource_group_name = "${module.ramius.resource_group_name}" + + name = "some-app" + network_security_group_name = module.ramius.worker_security_group_name + priority = "3001" + access = "Allow" + direction = "Inbound" + protocol = "Tcp" + source_port_range = "*" + destination_port_range = "30333" + source_address_prefix = "*" + destination_address_prefix = module.ramius.worker_address_prefix +} +``` + ## IPv6 -Status of IPv6 on Typhoon Azure clusters. +Azure does not provide public IPv6 addresses at the standard SKU. | IPv6 Feature | Supported | |-------------------------|-----------| | Node IPv6 address | No | | Node Outbound IPv6 | No | | Kubernetes Ingress IPv6 | No | - -* Azure does not allow reserving a static IPv6 address diff --git a/docs/architecture/bare-metal.md b/docs/architecture/bare-metal.md index c799e7518..4d728fa6c 100644 --- a/docs/architecture/bare-metal.md +++ b/docs/architecture/bare-metal.md @@ -1,5 +1,21 @@ # Bare-Metal +## Load Balancing + +### kube-apiserver + +Load balancing across controller nodes with a healthy `kube-apiserver` is determined by your unique bare-metal environment and its capabilities. + +### HTTP/HTTPS Ingress + +Load balancing across worker nodes with a healthy Ingress Controller is determined by your unique bare-metal environment and its capabilities. + +See the `nginx-ingress` addon to run [Nginx as the Ingress Controller](/addons/ingress/#bare-metal) for bare-metal. + +### TCP/UDP Services + +Load balancing across worker nodes with TCP/UDP services is determined by your unique bare-metal environment and its capabilities. + ## IPv6 Status of IPv6 on Typhoon bare-metal clusters. diff --git a/docs/architecture/digitalocean.md b/docs/architecture/digitalocean.md index f6f328fc2..ba8a21dec 100644 --- a/docs/architecture/digitalocean.md +++ b/docs/architecture/digitalocean.md @@ -1,11 +1,92 @@ # DigitalOcean +## Load Balancing + +![Load Balancing](/img/typhoon-digitalocean-load-balancing.png) + +### kube-apiserver + +DNS A records round-robin[^1] resolve IPv4 TCP/6443 traffic to controller droplets (regardless of whether their `kube-apiserver` is healthy). Clusters with multiple controllers are supported, but round-robin means 1/3 down causes ~1/3 of apiserver requests will fail). + +[^1]: DigitalOcean does offer load balancers. We've opted not to use them to keep the DigitalOcean cluster cheap for developers. + +### HTTP/HTTPS Ingress + +DNS records (A and AAAA) round-robin[^1] resolve the `workers_dns` name (e.g. `nemo-workers.example.com`) to a worker droplet's IPv4 and IPv6 address. This allows running an Ingress controller Daemonset across workers (resolved regardless of whether its the controller is healthy). + +The DNS record name is output as `workers_dns` for use in application DNS CNAME records. See [Ingess on DigitalOcean](/addons/ingress/#digital-ocean). + +### TCP/UDP Services + +DNS records (A and AAAA) round-robin[^1] resolve the `workers_dns` name (e.g. `nemo-workers.example.com`) to a worker droplet's IPv4 and IPv6 address. The DNS record name is output as `workers_dns` for use in application DNS CNAME records. + +With round-robin as "load balancing", TCP/UDP services can be served via the same CNAME. Don't forget to add a firewall rule for the application. + +### Custom Load Balancer + +Add a DigitalOcean load balancer to distribute IPv4 TCP traffic (HTTP/HTTPS Ingress or TCP service) across worker droplets (tagged with `worker_tag`) with a healthy Ingress controller. A load balancer adds cost, but adds redundancy against worker failures (closer to Typhoon clusters on other platforms). + +```tf +resource "digitalocean_loadbalancer" "ingress" { + name = "ingress" + region = "fra1" + droplet_tag = module.nemo.worker_tag + + healthcheck { + protocol = "http" + port = "10254" + path = "/healthz" + healthy_threshold = 2 + } + + forwarding_rule { + entry_protocol = "tcp" + entry_port = 80 + target_protocol = "tcp" + target_port = 80 + } + + forwarding_rule { + entry_protocol = "tcp" + entry_port = 443 + target_protocol = "tcp" + target_port = 443 + } + + forwarding_rule { + entry_protocol = "tcp" + entry_port = 3333 + target_protocol = "tcp" + target_port = 30300 + } +} +``` + +Define DNS A records to `digitalocean_loadbalancer.ingress.ip` instead of CNAMEs. + +## Firewalls + +Add firewall rules matching worker droplets with `worker_tag`. + +```tf +resource "digitalocean_firewall" "some-app" { + name = "some-app" + tags = [module.nemo.worker_tag] + inbound_rule { + protocol = "tcp" + port_range = "30300" + source_addresses = ["0.0.0.0/0"] + } +} +``` + ## IPv6 -Status of IPv6 on Typhoon DigitalOcean clusters. +DigitalOcean load balancers do not have an IPv6 address. Resolving individual droplets' IPv6 addresses and using an Ingress controller with `hostNetwork: true` is a possible way to serve IPv6 traffic, if one must. | IPv6 Feature | Supported | |-------------------------|-----------| | Node IPv6 address | Yes | | Node Outbound IPv6 | Yes | | Kubernetes Ingress IPv6 | Possible | + diff --git a/docs/architecture/google-cloud.md b/docs/architecture/google-cloud.md index c70129fcd..186082a8f 100644 --- a/docs/architecture/google-cloud.md +++ b/docs/architecture/google-cloud.md @@ -1,11 +1,94 @@ # Google Cloud +## Load Balancing + +![Load Balancing](/img/typhoon-gcp-load-balancing.png) + +### kube-apiserver + +A global forwarding rule (IPv4 anycast) and TCP Proxy distribute IPv4 TCP/443 traffic across a backend service with zonal instance groups of controller(s) with a healthy `kube-apiserver` (TCP/6443). Clusters with multiple controllers span zones in a region to tolerate zone outages. + +Notes: + +* GCP TCP Proxy limits external port options (e.g. must use 443, not 6443) +* A regional NLB cannot be used for multi-controller (see [#190](https://github.com/poseidon/typhoon/pull/190)) + +### HTTP/HTTP Ingress + +Global forwarding rules and a TCP Proxy distribute IPv4/IPv6 TCP/80 and TCP/443 traffic across a managed instance group of workers with a healthy Ingress Controller. Workers span zones in a region to tolerate zone outages. + +The IPv4 and IPv6 anycast addresses are output as `ingress_static_ipv4` and `ingress_static_ipv6` for use in DNS A and AAAA records. See [Ingress on Google Cloud](/addons/ingress/#google-cloud). + +### TCP/UDP Services + +Load balance TCP/UDP applications by adding a forwarding rule to the worker target pool (output). + +```tf +# Static IPv4 address for some-app Load Balancing +resource "google_compute_address" "some-app-ipv4" { + name = "some-app-ipv4" +} + +# Forward IPv4 TCP traffic to the target pool +resource "google_compute_forwarding_rule" "some-app-tcp" { + name = "some-app-tcp" + ip_address = google_compute_address.some-app-ipv4.address + ip_protocol = "TCP" + port_range = "3333" + target = module.yavin.worker_target_pool +} + + +# Forward IPv4 UDP traffic to the target pool +resource "google_compute_forwarding_rule" "some-app-udp" { + name = "some-app-udp" + ip_address = google_compute_address.some-app-ipv4.address + ip_protocol = "UDP" + port_range = "3333" + target = module.yavin.worker_target_pool +} +``` + +Notes: + +* GCP Global Load Balancers aren't appropriate for custom TCP/UDP. + * Backend Services require a named port corresponding to an instance group (output by Typhoon) port. Typhoon shouldn't accept a list of every TCP/UDP service that may later be hosted on the cluster. + * Backend Services don't support UDP (i.e. rules out global load balancers) +* IPv4 Only: Regional Load Balancers use a regional IPv4 address (e.g. `google_compute_address`), no IPv6. +* Forward rules don't support differing external and internal ports. Some Ingress controllers (e.g. nginx) can proxy TCP/UDP traffic to achieve this. +* Worker target pool health checks workers `HTTP:10254/healthz` (i.e. `nginx-ingress`) + +## Firewalls + +Add firewall rules to the cluster's network. + +```tf +resource "google_compute_firewall" "some-app" { + name = "some-app" + network = module.yavin.network_self_link + + allow { + protocol = "tcp" + ports = [3333] + } + + allow { + protocol = "udp" + ports = [3333] + } + + source_ranges = ["0.0.0.0/0"] + target_tags = ["yavin-worker"] +} +``` + ## IPv6 -Status of IPv6 on Typhoon Google Cloud clusters. +Applications exposed via HTTP/HTTPS Ingress can be served over IPv6. | IPv6 Feature | Supported | |-------------------------|-----------| | Node IPv6 address | No | | Node Outbound IPv6 | No | | Kubernetes Ingress IPv6 | Yes | + diff --git a/docs/img/typhoon-aws-load-balancing.png b/docs/img/typhoon-aws-load-balancing.png new file mode 100644 index 0000000000000000000000000000000000000000..36cbe52eafe88ff517a2d13ee9b6028cc182b80e GIT binary patch literal 38362 zcmdqJcT`hr-zXTW2q>b0(k%x;s(?suqH+|EV50?sNK-%PcJO__@4GX1X5BUGuDN%Of8fr}-p}uOe!b?l_X&>; zikgZb5NN~EKM$P(f#eK9An9BMdEm}&#&s|Vlm$9^$n8wZ=ebFzJZ)dB*|&Y)h9cY} zT4TI3f-!rutv2CZDhW0;sWDPA5w-&kXQOoGrXtzGF&ezFCU z(`|borTo)WN*}mFKRv>e;1l)< zgcXj>3bVee4#ytNTRUvlBwx0m+(+24vPjF1RC$rHL@y!{k;;a&bV?mD$T8n>c8ojh zQ||$P-Vl+#V9?|)^roUq9WkS^+TEX5cl-6n=xp)M%d>MQD42WTw`O+R z@@@YanyT!Q=R4M`sBB_v#;cOF`k>8Zv@>5zcn6llTV`4cG$_`XM!t=@w}izf8h|a{ za3Vg{`srh$L@^2oL&RcKjej~#A!}l*| zWRDPc9wHQ896D$w^Hdp==BLgyF?A$*8)44kU*@o3<*lU(fB(QN_$_Sb+7`Jh6ye8Z z2op&v?@w$nq&+J9Slgx-jM1%OSp2v@S%1_~ol#hObYY@yyr+FN#!8`%VQbG`)qD@r zb*hI@(jhCzg}7~$-Q7LPdJ83?h{jS|w*NOBMj=Tnh>=!%z8yOG@i%#G=0>xcZd}li z=EWPi*HY!M>&{f5jJyB-{L&n}j(pdwGe7GkW`!AQFq<$#Lwhbo&-H5ViTz8x*#CwdH1CjUwsq-o#?ou;@p0SK(hs3Y`1w)7!&bw1L z6e5!6I|f@JF>V)?*JM%2;Q*?}&08hi((z@eM(@IpCgrz-Ll~L=58qPT84+s(wu*ofqW3<^r*I4@m~s~qu(^-ijw8(v%Qd}c zIk7syNujJsB4WGk4_12gHuji>;BNDc0c#W_;O=;^4BDFZEfp_ckUChgsV(>g8va{T zCihoKg>R@&Q|g+_n~c1g=aZ@$z&{$YjM<%Yr(th9gU>Mx&E#>E)e!rN@I&C}L1{uL z-f%+8o8P!kn6Y3_hu1U*QY>~aaMB?`_0D2#Q$yIellFY=ZlMKFJ)f64b_K3+Jn)s8 z_v}wKY5ket-i<(X@IU@Y#gAlyWUj^+>oe{<-X*aGb(D z%a$59A06|=n16me9~g9c@-QvV)xVqK45624G%`PeH_q}Z@kNYMiecT+!2CtKPSUhR zkrXOg)t`@wE{ zYY!&0bTngChnd6uZ4ex0?4`n0HV9?+YDAcZN8`~Py@h1!oG0i#+JD)ecJHEpNKE{- z;sjtX7~$E2vpR47+<4*JqUG^gN{xJO>xm7C3)>Nfk0+;!p#5KU#&3y6%sw`*etVR5 zZq&VN@w1qpkPaq+nKyK%BB%Yty*PB3d=alBs`}V(v!&ByfXdnNV%-&ypfRU6ItRZ? zJ3#3wcgbnHR!lSIxOSgtAn!Tbp!+=(DrcYO^bT3F zy)pfLw~u7zr^G_4SB0m`Z4l{f>wfr=%yx8)(wgFr9V^S)tZV$gr?JesIsg3yedGoG zM_vrG6CyL+|1TS^i>dbVJQEF>Y>b(tKtUysvo=c@`gL~_TEVbfJ5N+qBs1Y8eQcA2 z#Z<;e5@CSJ$|NN@vd2OunRChY#fm%Y5gQrKUMc(jV@U;EnyW_0IKG=kQt8sV{z%li zw7~Qtws#9M^*Y&K0%TFx&!muGy+`?V*4Q{d* zG&@}8m;=+=kl2#$>O>uwSWNoDB)&<-UF;%qESSzzFFxnl9F5>veDmOM|6@8t>pG0> zIek_}Vz|syF*s8f$_En}EcWI;7TiD1%NVZ`K1CvCPz&MQ78l*Z#nu~7NP8HZ^Glm$ zJ8eytIy8+kOoZu;>Fc+-m<0!%?u?@Rfmz5bm``pUNMD_b(TnrtbH4msY(Ja~*WL8n z{mHsndjnPx-z0wyFI!-}12IHKw54n9xxpY7orC=)o6VNTFLchg%NIcL!=&BruNic^ zv6tM#h!Za4jM^yIr7*f5bE#{xMOS0Cb7FLzqr!in`|SEpO%JEDb3erVnpj?F$54m) zi$prJj?(H_?wCnzrZdeZeCJ!H{>l^1&p2AXUP-b2=j1nE_qS;q&BvXCO=t5W|NElX#t&YC7|r$yKao z5-zaURCUUPmeu67%8g#KUrfgZntLN`?8^L=2+3NEbTlo+GH1TlAvji>8IZF0EP8i6 zz4=XQXG^a$Xz(w|CSUspnNAIUEuF;GZ_KA}zt0|Da5h^wnpj8RTi#6{CJP(VAOEqZ zv(qUWX8a2=jG?rrGidb(@)$ic0DEeK=YLS$$@as^PB)WpGGYg^%A9pEoP?zVqqU)b zIh*3|30pq5s${`7OXMfs$Crrx;^XWS*_9EjOLVQpUaKJy_p!Sz%Z>{Fs+#ExBI94r zm6-=8dy*QppcFQRNHH~ZDXe$$o1>sk++Ub&m`M$MNK2s%KF#xe%xF;4B{TOaMogJb zCB1P;0xH~8hK!(cM8qnJO! z4No8N--nn)f=V8&`C>^Qn>J0ateL?E;Ld;Na(IM0^=0Ovr&RobHJh*&LI9Kmz3%xP zK1iYliMt8fwtLNVuf-FR&w)8>Al4!c$>(^B|IZibpHe68s+0tl(+965s{hNX?0WRo zqKhR7eOmTO>E|!ORe;Oa*Vjk=Y+EOJ^t`Nf%A^c%6>o7xvqSPA%lmN8d5b-|S2QJ$ z@Xlv1>!WsHQe>puHULj%r8OQFZPxX7Q!bj`Ap5Ud7C(xzu56Fnp%C?&sSEm)eX>h- z>%0Ga1Na)XL;nBf>(?7PR04EgKg(FJ(vT%DrN0I?%Y*BD0YG(f}xI^-<_5r9yu*;$)YM&*70^Wb1-l&98EFJH@NQuFU1{FS>wdDIt}GOc9r8-W{{b z3>Gi12sK@Mlg^7qlcs{_n4>?<4mLTG{S`F3@yF6moz3*Ux}-jCgSo*vqWQ*fpwbF2lt$>b@K zh*6%&GJ16$m!=E^Ou`7))Z0GKh&x?tDF+G_!kF-F-d9la0D-1wnOzEommLCJN#tyA zF7Vc>L0I*dFNq3Ui`=JM4BioV9+x9-6sY>KR81{lBKikum@z7cmXANO^TLhiP^pXB z0w3(`&?;>rVjS!fkk#9GNQqdxBYwVP8FQre%)-)AJ5|l8v!0~py{l5H9sz2p=LO3a zyw)*AVaddw4M`r5jc>ph*||68RT_6q1SlEitBv2$Kjf-8J})&{W=7SBH>*V;HcXvw zM<4!vs_l8O0jdTMsXzKkzSh*F$bI{S%$t-GY_lnF=xoY8c&S3Ehbedgx;ptaPCcJK zW!f2`^x?4U?cayhC1*<5;8hki!2Dp{^IV%&?JHPiU4&o`ZANWmReJI*9=tNSiK<1wT8{%-vb15z&Gtxksm&{*G1YaYHCLq_uAfYXtlx&xu_d@<8rglGfoGZjA|9P z#3B!{M}2M}t(6BF;N)G_dZZm_T|~MbWsle_bppAoY!%yAF60rJ6Ib8=2Lj6e3j)Br z5fJpEsf<-W;dOMaR{XUz)rN4vY}t0o|MO;x zJcRKhrm);}Y`HN{@aXCJ5)YOAMVljLc@U)}k4e1Md+nei=iS0NxP_7#?(&Y`Bxn8? zlDk&R-xo3Lzj)>awah6ro}ZjpE#oo&b{Y%VSE#H8ec!6)-3!DX``iCAzsmZKM!;%@ zWpMtkW09P}O@PNtef!EhbwIEZ6UqEl$7r5T;;%Sn{9De2K4QQGPYg+?L2+{@0%dB# z_hHV*+{OZ-B`K)bJL1#f^jBfkZvpys4~|B}gxe|_LR+>mikQDx!-!r~4P!`fZ|8+U z>SU4QU}%8%;2xD9QsHc$GW;{Se`%7Nr~VBg2B*EH(d$a{@hjNR_Y*GCv?GdU@6EmK zQ1|BdB}D9u)Co=WUvCKAJ_^2K79-Eq<%U<2oMUO!qlrHjpxf9(Ip0&2;kXWn(;qp^ zGE=L_oe5nVn|a*0(e_*Bz5h~4@m~LyP62P1LUxvk%**hPqU5a|yo7C5?+E@cirhVM znu|U5OYP_}T=U^+lkLpV@L0Km)QrYSTZW&D%-i&pOfg$5^1`w;KZU=xRr1726WYX| zM;zQoOZ*JlARnt%qO0lS!gf)4A%}1Idfx=f{#!gk{w*H%mp#o? zj~xM>ZC4X8`g?=>k0ze~E(M7x)gVUWPL6J4)}kBcm^}xq&ee2PzX=#L)cq2y?OPUC zk{M(}&c<_F5XK+;L9I!pmVcDiKA=?T!Me&)19^*)l=Mp?ZriWA37%jK?udUov0$?x z|1XVLHzH9IiWhXhXl0LY`fZbz&83fdyI%blLb4et$dr*^Q`~I@F19-LYBktyYbypba#JU&xR0=J;2Oaq+~&o z-OZ^C;e5GI?o_5&6hGH1`Hx+mvHX3(ITV*ND6vT-wisBtURO*n3U;p7BHhOSu;--Q zDB!d&Pxy&%>SZh%1@_~aonGq=7aU>Y#(}hrOTtmeN*yG7F-Y4N@?tb8a0{T6s-?oW z2Atg2Gu^Iow1njDXfE}98ov%W2jGFo7Dy9fT*NC2zL!H{ZKQj*z6q?`lJ5~;P(J4IXf+qWsbSGWo!nM*M*0GG;b>fP+se8oEg#Uq}-S&mpYfQqNz%TjwA-S%EJoCAg)E)a3N9~l$q(>>V0-?SP&3&c)p6!eL)6ws z3Wf}G&t$|AS~}3Upr)tL8flK)mMDx_pG7nulF9;qwp)a?TzyjM}n7m zxyW8x!tsP?(W5-O)lS?HPtDuNe3SRHwGD@EG^CKL@cWcB>bH2$c18jz439|Ogz`41 z2yKST7Su?2eM+qu4C+1>>Mb22^KHtz^eElcY|IIS(q@O9h%M#w(uoz?l2T>Q1 z>hbWO(viz0&(p&jo-)3!$-arw}_DA^NcD5jr=zXD|k2gu6kT}rOasvEy^ z*F1kOcy)wExPspJRqMLIM7t*R;*7m@!93$^z8$|L}u=;&ko zlgx^VV3#@^zWhxBu9MS_W+_P%5D4AEb+%T|RiXjE{%2>rD9aMO>xfe&*`e9F&??T{ zc+1Ey3qyf?0lbDV28$F9E`O^zG0CJa!8U9OMc56eki|!MD>Vj#ZFS|JUjss~`0Zi} zMQps~aLQ%kC4`^=4Nv{*D=Tdhx z>xae|xd`)p#rdD9w5hhv7bQhmr(7?p{)@awz}}IXDsPUkh!xc6#uJ}vqu3qMvQ0Lq zn$Uy2n6Cn8ZctPlu9IC88aF~=ZohAr5jKCZX#DZWLbJ05{{G}!7~;kLSfJJ+y*!g( z10<56)9S&Y!AQqd@l73|Lp1D=25q*owh!Gjp626Xv^o!w($dz>w9NQM+jzXMCFDI?o4uD`-^kj$rYX~@68Gm%snU?JyvTc|8zXDAVVGuxMa!D( zm|w5Wv8UIG=fC(}3$gPISaHr6#B(3*LJ@s3J-_EUE6psU-_uC2J3N?>xZc#zgYht5 z$4WiiiBwWhY0gVya0JXowb8oVryAqgp>;AY^>D#90CaD~M#H}X!y2Q_CrAQhw(yke z_XRWQW78kyF3FyAdX@b;CZ^=Q!lxg{&o!Qi`@U_w*6s5t`A@aAa#pU)odL30yhrC< ztU*9Vc-HPy=5RCyV&e zJ4egQC&z^Rh=KwDH}B8`AsbHX0wqHBRYOXAu%h8)m-I1q zwZ#7KMQ!!oh5FSd?RM+Gv+8Rn{r~nQSE&us$LLbh1P^}O9d*{w$5LjWt_*b@CnL41 ze^1z@;AL&*DnMyN6i4dAItTh?6{s`;x=aD;eoNY4nLZ6kbFHg9N^rR`kHd3@JQ(7N z1XG!-A|SX+ar4@eC9!HIp))W;QnC{_+>UAeH)2s) zz3W`{<_gw1*Q_^;vp_la07bYA1NS>P%8VpnTtS*~7LtK_9*H?$Fn!jw51HAObFMCb zXBp{x{uFwXIcv%(JZNHE^$$%~`ol zqyeAtigWuh?bsS(yd{e_|60ii6mgbem)uA(z3|Ul zG9;WdQ7s^iNgJZvT&sX>??owV*dLO!GwSi*o39W4751%=d2P|z_?zm1clVixv~+6JPLrQwIQy91zuqolbE>gmE>i{8z` zzQT%B6aT^y?);}L%Q7wDZQ4MSMPV(uK>C>AUvBb8fJXstUZ?6E;oDcKO*xi<`PS#- zwp0@EWzb?GP2c`%&g#S{WP@Yc^VQM$kq?P+u8()Ro1h>1OfrD_(hUus{cLJ4jk>P2 zUMhS-FQMyKlfL-!T|vZ#EllLy%jb2D?PTW$M1*J02KanpFw!ati*{=~QlwP5fze!(9 zO$|IA@Jcp&M`SfI7I7+xQ$P`n0;SN#pzYX8$2G!YANnJ6h@IM6-hIA0>)o_8|!o8`@B0cG`^OltQZnqrP z)95cP0IOLTGQBq=2)F(HKexZq1Gdt3MIf5tr+J*6^EpwbEJ zw7m0-zK)4n*A>_$(aWG11C%4450t!_c6_pD1?|xQe^Q1lbSD3;SeO<1s|1}822^I9 z9x+XgJ*4PQfYOI{X!&(SSX_JyPbELP$_}d0iuFvFuLe^9xmkst%m)arEZXxCF*iks=tn(!}4%w=bPg|A1~^Q1Q%Dd z1sACg5bdr4a(QEhY4>@()L2gqN|I0EtY33usfpDO+h+g48JGQ0KOZqL$0Im{v^(?s z`Brne0(IV>pr3D8=@4W9H8TK`5m2_?P!5Q~ij@qDayjD>Z_dd^bC5oHPOx*Bk$!|6JH7{f@2(0Cqpe8XRltVDL-TJJDB$pgh4v zmkKW6d=n==pKRE6Ic$S8t(60F_AzLzVvbm%NSA`d7aOd{;pSRQ++$%?_`A<8h##Dh zMugBMHBryMuqR;_h5!Qm`EZDy%`kGwPzI>N>&?X)qH6U1aY+w=lyCP@T-){K%D?j? z{u$6W{=h-Y$EfNa`mjmuYh}F0hfU)@Xf-{dc74a=&Tb{;#`Lm}(CTsn><4sR;1bDG z-!wa;hdzAfMy>{GhSmt#V7bti7Z5?Fb+NzvoyAR%D`=*`uifIoO9h7ubU3gHsNGV( z$-W%MJJGmITSk85aGzAQx1}2|tMu2jWN9KBv8uT_3XJ>uz4QyUGtmkGvg6 z0x^b!PDwI^w)B-(9ZRnqyNuQ9qW9i2J#?b>&uF;Tmfx?sn^2w|K9`i5ch{2Y0ZX}# zU&#O>MxY3wV91VjZ8fOrc&O;BQ`u<_2Z@E7y&*P%YCvDn$MMQ6?wO(Mtz9pUF#Apl zE&%G{6i~(aY3n>XiPwKiz?K(o!=C*@cr!eZ+OJcenNN=IJ>Ds*e1r@a>No+k=?`n@Rc! zAWyQX0NS#)uZfW??%?tJTTGQ;!5mQlvnLc_)julqL!)hEdZ`~R%e5|t)D$yE)bDB- zMKJM;Poyryn!6Ly(#Kv}4w@d`UX0&^k_8eL$0TYF1k&|%UHUT2T^{Dn)tzktGAA$M z@@#aoS)V8|ZvH%w1tr@x@fA*4&B}WYzr}@uO4#a6wm49I8vk~)_mv&8Qu-{S!&Yx2 zY=OfaOWk{xhI@EHyZbNfv#s7#9o1u$_2t0fTMvY}$GA<_6Tevi2@gWCn)nh^+#umf z_G?B!_}Ars!=nK3P;cQCZw8{99%+|bzP*Dc{Dlddh;n=U@HzDgV47RiFmo66x0wk)uJmdvHLXL8>8C@L33F{YwG{rb zv$`I^<^ir`vzn!jv~-Rid-h4tfN&JRZ2Q<@i&fUii+XTUKE>6s%q?)Gmc|>o+S&<` zPf&e}r*S$;c*s8>!p6yz7+&FZ_;_RK;;M^y!KG^~*yY7)I~q9y<64(b0Z}Gw@kSeI z(V1P_(JAHyi+KZpE;~CJ8HgF{MBT@Zf}dNd%*T=U9F6M{7|a`DOveeQ3peam5J ztMEi2DbMyrxc;Yc`xisJ7MAsA7;7n|D^G76K7q(RGM?H>gWEVzs{7C>uz$EZpcG}P z@H5Anw34swI@x*)|Ln@W+FhZ2ANGKK4$B#lbMaRFeg`5rc?2((v9Sx_=V*IamX)hj zq@}mw+8{pno6uoC)bp*iK7U9P7G6pBH#l^$r7m?2E_;JGFrNaC`l@uNUZ$NpKc&W} zZg;_Z-dAWwu)FB9rL92ZB4&tJjB@lup(yG)p5G7LaQ$JOm*;BXVWk(0@{F-A%j?{N zR%tG)v|_wv$$O@6enOHdH?EM`OYv{njY@w7K`Z`(za=ymK{)|+f{e~Qri;Z79GKlW zp-lkxgS=?f5yh$ARo8Xlrf8{-`QuyAL`Ii#Ckn88Ps=Rwj zH}%ABDC3f~EjgUl7>dOvv3T~7+BH_*`(=jNZSPChjKQ&vFqWT|x;s%Q)E^*M*H-Yg zKb+#<>+7;jTchOkWxAvN$Bf;q57jUQ$4EeCGe0*%i(PAnI2UlWGQ-412S_)1`F78K zCKy5<_*b#jgv#Bpj=typvQxpQmp=DTQQ%GVIC*Ykm$H7^Hz)fWKAEe{QfRlRHC;X3=o&7{H8uA~~g<~PdJl|=aIiqof%n*cZ<}#?y6Ax&= zrG~7J$|j!7n(7__d`~?+b9~hR3ag?NqI~Rxf+2IW8q+B#krYq|ga`QpPuM%ro_w;) zXawDT3LoG)CVqNAi~k2Pcg!VX*@9v5i~FMJbTgNMVF^kFbRWZ|?J@)dz~$O|nOf|I zI-oRV`H0Y{7Hl3IFpYEI+FDMhwI^nfj*aqOn(BR?x}wW^UyEjlW_h*v107#nTy}a~MlTe3Y&kTD%FJEV7D5U{$^AbUfq( z>wN~30IMF#Q(UPTE-v>#D;dwj|1 zYs&oCVUq#}?#KOIoI*?V?@8~~VUsg?(Ih4uI|t*kVlDWE{*7yRL5 z1%e_8)u3*6(4+e`zwR&=X*kZ3fHZyO73cA!q^JEM8<+VJwG9zL7@_mm7TBLxKkKjl zBTevsZ>e}|L9k@g{{3hb;D98hwoiC(W1|r+?Ax6uFePPZs;~uW}bday#R4# z6o@Adums^qa+nnTBG3!xX)gaXi0Wm&=kllQaL0%+XSQg)+jk+IX|dPpRz|V{kNmc| znodkXBS=*avn2q{@zHG2Ib0eGzv3hatO(c0->`Jv4tc*MpO~(E%xN>oZ0`G)4fL3$ zedy!{!JPEIv>Nlc!r{pFIjS~h>FbU9fdHGV)8s8c9xWYJUtMEMITkyAVifPkq_qzi ziI-sVH=I88KW>9Q??|rC0qrVtDc+g>NzStE6n}kGlRZcB8C+nCQ&uZ_!7 zHViHbZMD}1mySM{4{aSho4TRHgR!P)#KsyLuIfjgbsk0P6)~(X{A?3HD?osqb!6dr0}tliXvpu-&bViX~S6!$n_AB58^AN92B;bSHx z0$v4Fx46vs8uR)+UnZwa@4h51@E0xcoUl#zd9;*7TWaSZp3aolLJVG<8JK}{BQ%na zPNhY&`TP7YAMC@@-(qjnA-YVGkA#F?hw-CWv#bY=SS{%tj5sX_|6`E z#ZgfvYP9uu?+Kz}vQBW!#ZU{Npg#8U;^RwnAX~NE_=(n41>`=y*5nKDQJ#HskN~l~ zb^JUfO5;_>R_~_WD8~qO>YCdD2uy*7MlXqQQ(^H3GUK@Md(CCM05Ff@$lfw!K11h~apVsRPqqS}8Kd--DP0 zw`GtZN`Qxqppxwd>hM*R3#3-PH3mx0h*`fB&A@S@Y1(Dxppst^od;wlvgynS4uU!j zF`+eX5nvP?_n>OVkwFJ>6k{e&6bbFeHNU49e;nT&H5f6{FaC#DCto4%kF71ji=SyP zoNZ_EWWJBTnKBYPEUh-=M&C}Dvu8|{1ErK4UT4DMkBw7AFNGKc_Y~ z$C}Kfl$VDjpxDhySQV?L5SjuJm!xn5|5m&Uu{Chy(-D@Yx)kfc&__9|& z=SR5DpMS+~;h)AP52_bU}qaHa85 zGkyHcJz*%|A_GcM*2xJ}D>cb{W1W8LV4JT;oiEtmQD;RIky#zxxR~@43-r-WzDyuV zP^>7a-~G-!=m{emA_1tJ&i)2&)wl|%MMa9Fp(dd76v~HVMYAI-s+YgPpbW7U(nEp*HO3+{M4I zBh78fnKu45_?EVSG3td<7~AW*CvYHHHwA+A zL=gx`ycSP0=l?jKo)+fnpW9}sx5c~D;e0S}u^cUOHoZJn-IUY&RXf!`&jMf95io0N zSUxXt4uG`v)T@QQ=h!PqO#0+}#-}t`Olk4x8omze%I=g@V5C9PgfH6*oTAe@cblZQ z=d{CI#x?er0l@)eaPQS2#(X-q(Ek=ccJnJf16LN!n6%Wrnh3>JhdV1ReTzfmt92-t zdf2e=^q0-vFW+C?A-jf->3i}8Z26G$*K)^9m;1Q$Ew`GpEX%<-sz!9_Zj`d!7XHm} z$7(?tDrtU&s(b~zytwejx+9>oKTGuEJud<;h~i=ijrR2$UAn-XF-yy@0>_=0C9q7R zKp+*PuW-=2?n|dEXc2#NvvzpaBU5_f+-v22=Yn%lGkx6W4s3jPl5;_MGV-;vyX(?F z(NNOg&Hz_!BT{Il8HU12A>wnLKZGQRaY0aub*3I9A(>M@kV{~kNj?tJsq3-u;}(3d z)kV!rLf(I)X=O_W4k{JyT;>*>KIxzZi#aLaY&wpcP&iPuIj2+0k-z*Nmga-u{kp{q zp0mXk5sUm^PZct`|L8CuxbG{XO#jtNHvfD8po)FxGM3=okPlr-TuS3FV2RiV%Jcq= zk~JJE8pgc=x+zzOaLVDKqw3YYQ+(2BkP>tS&Js zB0BTnv{cF6DHqYNVTMS6Jj-dH@YQDTWzti7mw+j<u0tW&KC-xIr$K+p7=-(}elSp6vS{U)`al{4ZcM!9&! ztdsQmsn5GeiZ%bt;8EeBxaiyAY(0IwU!9?tgAD-w2hnX=nYAMQGLhdd+9;Gt|&KjGEjH zMM35;?Uu2nxuAM1HQ*KMo$z@gEw4Ze5O5$U;Qll^D@}Na3h^?GKmOtFgX8bm%VC-o zp?Ao>a7XNgH=n>^hm~rc=UCV%ZQ|Dz1$om1LjllFY9mhB5(HKKhJWP`z?n2cIbw)m zVsK`v6+d&ZTMs7Dq*YTPj6%%l=cV)OC}0=U@GBddQ)nV!fo+5h0wC3@ za383A5CDfWf>gRvEB1f(OmBQ_OF$U2f*N3SF#CB9vaZkz>>}>ZPG#XQte`i0G_C3(e9pn5%*iZK4q-1oV z=Yq`N-fvN_NLWbJZo?ng_I7RQDVLd4mml1Y^ex zjc5!-%|!L-%3qO6HH%M#0nmr&w;^U{$71$-P|?-}X%G#F18s0su)pKkT~n zO`5Rjfzzk1kTV^y4lMn{B&Q_E8VaqfT=AH`J3jG5W6>Vi-;X;1+KVoxwDp^-j-aUJyfuhxWQ;u>YL?l>wKo?KH+AlKgtR)TF%<$vf!5a`T| z+omgt0}@e@JpM0D_WSWQW&eMDT#`5bAA39t54nf+GpSZ0SZ7P58?;vc2f%%mAS-KJ z!3Vep0>zg8FYlcHU(~y=`_Q1gFuAi?iydWK~vL2Yx%4+u61<}dK93f8y ztg?5LgU0!IdY8=mZ4YRO3_teavEKHF3#mZt48O{Np%b0sRPd9BN0)(AdC1R_B8SyN zUxz_GkeO%<7*&|AJa$OMa0*SQarL)W6UBBZbL_r05}`5Oc5Y!{I+eW+FIfKB{vuop zFiMdg+O$RnY-ErzKa)CA%r60~)MdIbw~Iak^i)ivT7ZM-gp`& z0Yb`45jhrXvqze!hedsPF)j%RX~*{@bpU|J*S4DmmXg-@x>((y_>0)FGz zNR<@5DgENo;~;FgX>fp7-j!A-+Q~e=zc{0@K0n-rw2nk#F0B59iG^K8l#b+~u7J*0 z(zUe^W}^$ZGxqQkypa1I4r=KtHwD1ty*xDWFw0V=QS(T|g{5@Hfgc=5>C;p2(k#oX z9ow#kGM`c*70B5sb38X4QWqYRy4+MF_|cFa*m=pkD{7obONZH#tGO$To8?m*`3wK7 z=+tjycE$-un>T_4f@V2j%%Wj4C${NV*30(XEN)bywnmoI#qq_KL}$Gs;<^I!!npR7 zn~_9Htvsz{cMYO*A?1j$-<1;swvp+Y8!UZ#Zjv%gANRRmUl`|(|JfE(FV5vI>8)e- z2+WCS36x~<8kCa{Z{e(Bk{KHf5iNd^uQDddBseM68Tc$*N;e@PtWzM@YE{p52IZCa zISA|&4TZ2^Ec0hZqM6f&(wKTod<~JAREItsS;8=kpZL#p{f$(ylRbz}%bbyO;<3*^ zN6u*_S7j%=lw)s(QXN$P0#ewIy--%48SOfmU8{?gXJ@{p4PdZp3XG}mecYHu%`Xu# zt*wzdAQn4P36S^-Bh}<^Wx`m3lS87GLi1r3ux2AHiKe5t{3YMNgZ zv8OtB9(R#Mf-R@`m6-*%(YuPZiPfb{P(L9wyT>xP#bXuUF6LqBU(6xjff|ntfk8}p zRw&f=Cf}{k3-UKsueT}7EY0Hjj|Jbj1CY&N_;C#Tr~_5-fR=$vcQ_EK7YrlW18xa0 zceWSx3g|tYzMO(peTK7L$si}Pd5MA8x!RvUd3p0Q7EVqfTJ<)AjLFA?j4@TlaJe9D zDeZiz4Y9Lvn}@JOB#OunMvKCY36T!LDqchmp2B$0HN|wlrNP9NXit&tLBT+eE*7wC z4yc(B$yir^_rOVo#SpuxLoUD;HUYLvl7x@~{R3cjKF=rLmzb2C0yIo10Ht<7+(EX8 zKmCoZ0=%SuKYR$NU&cevEB94*pYQi@mCP{${oj(2(oNp0*fm!W@DESUuRi&z>0HQ^ zR7sux5U&clWR;fwhq|6>Kb!#Cl7PAQGAj5e4*FE$dnSCs34tFy zSUn943rdD*R&V|l?Fl%1AI}2~dqDE=|FP3)-Otlke5LdSPNWSVvrK& z>+0lbat}_&EliAm=LLAIUK*Ray5hv?K*v#s6jdMvIAY$a1MQFHlre6NxWWw`TAAUUKgkpTL@F>^ErtVBXU_@h#hMY>sNmxOn zeRN;kQ3Q@UM?Z8Uji);(G=(gT*;}x2(Y3AG0T#XdrtDPH=3UA)B$DM`ft4mEBq&A%6PC;c!A7PKpuCed)vyogI(p8*K(!~$NOG=+v0Uz ziz8H?`60T;4EH~ceOz1LTyrbb4?3VBd^|})5I)-W`ki+fOWdEYPizY4EY2Un_em4Qk{xFA-P&*TuT^6lYK&nz>X?NH9KC^oNjg&h-^18zJNB$&a zgDKEOs$u!=6!_u@vP{!E;?c?a(6UI!x@T`COu2eX_`Jw;N#$CH4fK&8`>OI*rnh2h z+$T$8@P10uGiMiy7P>>bZxcE$F0S>S`0uLlPZtnNq}wNn@12186iC_lxMteH|FW%p zriKguZ+e*yAO4@x%S6(eM(TlFuYY+XTYy#)oa#l^JY+&F>VtG5Jyo_+@@U%Q;PyR8 zb#}C!-IE$d0bn)@obdGkGL#H=2i6DWN7m0%KQ>~oP>q_Gw-+Jp4xmcpvYdcFwMb#J z;I?C#V<(Vxr|xQw^ZGBR{wWnsXq>dHshQM@l%iMmgqq#h$%pG4JMU#@ zVeE-0;sY@KeSdbAPe9^s8=-!s%5<=0K$ZI(7qb%2sFGMDCsco3>+I+=e(Yu5{lbqRkLSs+{l6Mq|$zD_J?PB5sOB}!1X zAacY1gbt>YC%@DIg4IHYIStQe6%9L#=ByQR7`>z=ztVUA)cF##oQL6}7jMYX z6kK$TAcFOxP!;;r!>Q9gDNbzQ#oA45vK>s1RRMG{uAfPZPz1DRrKz~c>Uf8`V1SHk zRzE)2`+VSOK?+;+9fYO>Ck}X(rLt?_@>^sRQQzJXo{fDmG zI!Ajv-;Ei;`)A`5?>}DM=~SjQo->u(8>EH4^u#IS$68C>j{ZKEsO-ub+3>9U`S>*J z_x()!m#!mgO-*+QG)2S6dS+!bg#xE;8%r}{p-a*u)Yad?zOK$o1$E(@j}yGjF_i8{ zqrQb6h_`nGUD$%fC#Gc)^xn^@?_i;&wu2|V&yaRvhlv$wt79+s=RxBh??j~smfJMR z#od?!yMvfa84+@n>U~gN~=X(wuVKS1ANM-1Z z!?zAHTrAUGmv_4!YOw!5x_k3@DBJgc7)3};k&pHJ< zJC5^xypQ*>)%svp5qI&$#k0x$uJF-S-1hhHm8)m*PW5R2?<&6AzKLCaP)*ioE#OQj_F*7k)iH2Y?7djDwHpDosg?m86{`futO z^$R-LZSk#;dX6b&~^18YxF4Zlr4^863)KC*G(nBk0G9o%C#eQF7k%cJ=PD zo-${1+~@*MC)dn-XL_T}*I$cYqx=siZf^=8&CJXM&Tb5lz>kxgyzNS@6@wSu5e zyLe-hc-JAk#AO16`)NR1hU)gBul8udp_Uen;-ZWTMLyc`Iv+4%Cp`NRu8OIYPa{B4 zpfFjbazWM~Kk~o;J!xx{M!Ylq$8n7Fpo~6L25_PcGH}rk- z;iGg-Ps5}xKjld9W5k^bu9!4NF94v;b_UFTTHo5;OA6%2Q7Tn7#1}&n2xf0wI360t z>|!KE`F{<^|G|F8v{tI ztl(d$1e{q;;`_;gKu_V9*If)qL%n5SQgPRUXFYpc{XoW~=-}*jOn|A4H@_Reqy;6g-#y6w= zFmulABV^1>{0YMG?14A~lW)i^NtoS9%S)h(sHtT4&h~qaBcF6xAR3!yJaHsvTH=s5 zHfKohQv82Dik$6KR@IyS;3}pCsW~xt^jznWZ><{JFsIyMkB{`7BY zzjxPE*eVQF9Rq>~A*|`i%OQK>>EA>&HYa6}7n*Jg%|N4j`q7wAuARb}m@BOBDvg9- zefP4?C{>VUtJ)SW+brGnsSFuns;b?eyD4_CJQSjvHs5?qftTnT*qi0b}eGAY0fR*sQ$nMU1_#md-aktWI_I6pId%O4~G zXft%}R><vq+=W97DZ#zC1Vzm0 zlA3v0e^`j2pFXZ*X*lbgu7+DkZ zUV5v+NDvm@)Oy%zn_RiLW7X$Pj}2&Ef=K~lZ}Q@vVJk{)@XEFZXiC=sOkjeYZIBkf zeBZA^dpfI6L^(l35m_D=Y94o}pt=0I&J%^ylK82sex`d9mM-o^ZSUitgDj9>?gE-xs0&_w(tmx954b|+-Df{L*wmng* zc9%O(#2Bha2<@92QesIEMlqdPkbWJzn+EX%p*Q|a>zXL5!B1Ny4}XA81t%*h?b)ix ztv9`=jNX4d#9qIIDGIC zmc~oj1MwO#klhzk9L^+{v_@=GovJ_jUA7}iKeX6P=yF@H;m($Af=RCt^S=Cf;3W+H zmQ6;245jshV%07n$@S{=duNCstv&hm?ljA=Z?HJdaZwnmsC7i^j%TI;tel3VU3mLe zD2dY;S*ay-zv}fEu`(GL?MMwE7C-Lmp z?-R4?9t5*A4hVWJoKBt?pdh_b%d5f%Z^#6+s6NLBT+xT_H?rtrmwA^Yk_UU_?M&lO zJiD|H`g!cc)Y|8~V-N=2C3fRof!U z$PLZLPZ3=A)EQZrYHFC-?7^xY)}^+RU^DM)HYQ!R5nnaj&%D3xyBq>;{F1WtZAXB7 zj{c$l*3Qxr?X41SDk^NIX4k`axm%KOU;9rT6w4nbroQ*8_N0BCeck31)^e$jmJXP} zKaVG!iIGidQ-N)N|8)Tf#;p=hn3-W@ZH32KpKoZ`sklW|?QZWwx$f%iY4Ev2XQGP0 z9bI&=#|{RYxm+PWeKLQT`|L9@b0*gv{)Q%$Wbmul`A(J;B1`(bt!hOS0K=os$RWnB zx_iq{O5T&Xmt+iODRDV?ZK&3frRfCHpf&tbO&(KqE4@bS!eN-*Id+Ner*u-d z)6D_oGTR83@1a-tDaC>m4d?HlMvtvEWoNNEY1cLctN1B+zzobCoS6Ik;*pk={oqKx za`3i2n0Wwcy?R&}Pp@?6O*oPqBEBhS+Ih4f|MorS+TcjZ^AWZVVYXQxW8%Gy(^!@< zNeo5?88KeI`@;iSZOz(yzJf`fiDP5rwa(11gX=<@2- za;$3%#O5xk_e#IGX5;785YJ2`5C-hLRxNZA_l)s_b~s${PRg?vjBt8Mb)d!cxV4(B zVN-1Ji(zX2JdUdx;ee_$uTSJINrAJq9WbviQNE_1O2|bCePy>j(`;b0yY4>JU3!~Z zm2CS@dk$6!oX^(YoV0;4RquHvXp!k%8@LeG?rfl{RxyQX^`W*ywrN@aNA8?!GYfHcFMPKZk`euma&zd^1;c3_p zot$2hdRfHm!O)l#Q``UAzelDDrqt`@?G@}$E z_c$WO@c@-#`Y$EdkcQ3(y~dekJ}ol4Vaa+%lo@=uP(+?)T8;30LB#dy+^3p``F-b{ zpi{Aa{CSUBO%4Jub%e!o$=047MSW8I)z6eYSTKVMie0M>a6Z!(xtK{mYPXjBOcBkEfuWsssBHVuIq(3R*I`3mWUaaJn3! zmu<)cEOTDnTPM6MhLBTw{0fnm@xrX~$9JMp{Vz3Y@3|ZGquf8%Xg$ri7}H%Fkr0#H zVwH@^+maLx^Harf^6oGqny=^a+)7Cy#3f^1^xi0h?y~ekqS@tY+I8p6$$M{%Z$3%| zY4B(J`&#A{UO#YMEmZ@INw3qtGIHL=BsT%1+UNnlg~g6_Mjd#g{X2zDPe`eV(L%h|@3_vO86qoCT!1})Yk5{>;S=~0LKqK$G zd^$>MzpHOR|NVNytT7&&+r6#ho4Hxd6_fT|gvUhk)s>U_0kAR$q0-mAuOI^y=3zjWa&~!^ZCYJ!&jPRL6w8w9KGO73CYA@ANU)4wTM*O zr-({hOJPMG=9Ex0G{Jt9jTvf2{QQDp@m6N_W6gcj#*XnOII%+?TU0^jX;vb<@2Q`! z(cW@~TTCq0)phn@WIS3YWvp1xb=YBuHOD@(m6@WoF{zFtyPm?PFaaMGZ{|xrv)5LZ zZOh5~cv+~B&bA4xKs2|W5fZzU*39W4KT>~Xb$+U0`L())q$EG3QB~XHajBS6s`1uB zkgZ8a6)Ev%e_7;6bBkYSaH`7o)J7MbYeU@Sgujl+Ouz$G_|YS9!bp;_fi9*zT_(kgohfHncpYtg5^NTei^t}HNi(_7)F{bf z=go%jo+5BQURb*P3y+&|4<^Zx7A?NU6;tgMdPRyVC%x%+-_qes(~Rosu><(&a&T+h zD)l7%%@OnYIk7YDB!UIaljm=3K%&M*I>OQ(drF;crKUxsr*c*>IXN+V`a6m%bEe_+ z$=(ONwp~w^!KVjS<_ry(Cuyf10b7@$Z`8)D=BMnt`AtGpR^rdG>9n@f%6S^76I+!L}r<)k{Dy0uAA9g8-XNxgT@I?trRm z6DfZ>;ZU#CwnF$o(6yooN>i_k{(sy-KqWW6 zb&Pv=6)P99HVOzQd0%O`_A-!zsaUREf(x>aJn8k=Kr3FKPV`yX_lmrItxWee+J z!S=IKcbE4<7E-guMro2vSLY+Rk;2KEKDa{J1iO7Uy}tvgA;f!&%ec?2arMJus+e?^ z7d%z^TAHvrl{xhHA?mJoz6 zG{JrGP<0eL)ZCyMzdxZUJ_Oa?PX3P`*?liYdh@urB-`sjg;MlrCLTEO0)%^G zkwrZc<-I4!$jeUu)J=^3KFy?JnB5N?K+d4T_1GRYNNE6V& zH_E(+S09F@rf-GA9@Rn-YcsyWKv103dYCVWWw6h-=^Ee~G*d|C+u_8%JrwF}|Mk7z ztmPlUqZQt7iW)1kOJ(~;$oZs@%CgSKEpuL(-B3@eMNdHQy$^WWj1JnlGIcNvH>Vv| zL#Bg{&d(6|Zs1fHYq7k*Q=9xsEFYD2Ii=1hT~$pD^41H$>8{RFz6Zft=?OaV`YAwg zHazoNXL_ZX*TV8nYK0p~=fR#>B9iNuf!;!_3j(gbsOOkYcKCbxJB__%%Ak?=b;riK?uY2aI+fOdwmbLv*IO!}EC* z@@WrpMQhoa-{>E9Lx21dA}Bg3?2zW^G4Y1Ce40Wo4{%aB{^*&ri_ z`wgZ$8YcLJ#AJ=%SO-Bok@`0JO@$?u!DKT-R`Xp32rMh(DurGK6wKZNR>d|E%;ufwVIFNQ;?6+w7B6&r+V z>*O}to55o$&39{m1+#el*0Q%$a>8(B4p4UZa_DTMDARwL(Lbgm=@GYZYJ7eINPY*% z`Uceq@+HT_fu_ ziSqlND$3|aTR}==asV%-*y%P{l2OAz9jNHlZLxg+zkJ8z^wYhRbFXC+e*KuAx%P?$ zC1P9+VK*N57KBY3JEW#FkDSTqZZD(WIi}B@DUVQKn9ybn(D=dhG!CrP+t>Wmp?_|R z2AT&}mEpB~OntZ(p9YxOh0SMV*aqDe{Co}Hd0Pf5)7ChoGEA<4 zp(BNoyaAc=e@inoch2IvT%~`*sl&wvW$%|`10sBApl?5qAVZJWL2X!7l)v z(UwA@m+$z@y!VS`vI_yWyC?CV#vja7;leF+?Uo~Seg#{8>7jgrXR9&j8qKJRRk`C? zSfbMR!W}9-kK2|PFe4hEPC;kFYhOTYj?RM|BGv|&9Yu8>hN?z7-irua<4k8TORK1L z@xvp%+#mk+3Dc~<#sQaZeL#8p;QEXKKi7wY_XqXkU}^Dw`G5YB^?ml=2Z{Z&760>~ zjeZ4B{vVgz))q3^{n$19$L?h`0aU6;sU-}}v0Q-JK%H800j7n%>j$8~Y_C%Qs@~biXxs6k;$K$LbGZD3jf99V6C&M6A7Hg#^OiZd0pA zx95zry^n1^YmQ8OJusWD?YlO`F4qSFIc-~xVO9RUZ5%)4@CYF?@*q&qLATKQ(1cYz z({S$69GP(M(K@Z9${?rMNj15_Ws{*ZhdlAsW)2)@f1lHLR(Kg$lqWQ@eGjgIpH>dW*qm|{_AiAnnPO&<*)c&+m zPF*c%!g~Oih1;VCnzE>GJW#uz^aiq+0U`=TC8PnDl_*>Ki-qR7Yi|a|HH_c=?hkj> zxzefTLO>-!Sf0X~8I1JNO)cJOuhj{yhO1|1_c;gXP`)TtG`(Na7^DsAYUNyTy_s$5 zzZx+!=QkSgBw!V9t4utcGgi4J$r=6(fv);dQMHn39&ls8p-?MzL8)oLK~;kuZuQ_U zXL&NQJiEVQo99ez7RJls+}mL$8wTw!Q{e&pkzh5cj+V!?%`0UNG{)RB^E#hCTCfK5(XvSsiSDrY|-e#C6 zPaFrDZ~te`1t(jgJ)3s7)6zarEoZDopml1?e62mlcd=UT^S@2^@c{;i})n%p$o{^Qv8R}wByNF_6j24~r?uv$FS zc6OVhgyz>TMHMTC=HOSi#Xb$HO~O`1TnWD`Ho-&9dGtAC$CqQM7vBbr1wI)jpBOkz z$6WqA#D284TC81EmvQc)u8*%T;ThysQ`>vu45jfe8`rU{DF3=T{lUSPfII?rj(s_G zMxEh*ceEEi119)Op&(ncG@VZTuJNPE)tlbsSuR_h$dVEOoA8?xM;3+{^3)I+ z-w*qNKi4@PCZ`-tY$&m|WIFkZO@9H&dkmwps@%;FXIP%=7y|I!Gr-4;7!y13CIz`p6A+j#Jf@PwxKAC09S0kqf=p zR2%%(-#-wO1+dc1il}%;qZUwzK-r$23Pp2#5|ZPcB^Rx-wL1NDR6(rIlmELEqExVi z%pMK7df-QAX14 zmz&F%z0JN#GbNf10L-~hRS?EXoQX@$=LHM+_$iqXMfXi}_G|j!;#z+aPv)i$yc{Yq zBZ8b=66H4IGwR$BI!JxVBhTpbRT`)5DciFh;bn?du+II23zT9(FShWZHdW5?zvMY&B=ZFuDMtQtJ% zjA&Ub$6;qOvZF45ftg0C3y* z6a=b$nq+ z0|)@ehmRchee-*ID1Y4-9_T)BWUo>N;qGlI<(JjS?10BHigA^;~-S@T4N!Gancu{JT zx_VGoJ-Sk>B!@Lu;A?Y(!sYR3Or%RC!oWTSm8 ze<5E+ZRiY6c{bc263je_M!}5n+x3l{H`WQp5UKjNtl6##E0-vJ=?OqD7Xrd!u_N2hEzT z2TS=QI9|JV_3H8pLD8t#j`}z}O;%J9S;rXC9SzGy<|8lcxI|h85yS*vXb~q4?I?Gp zL|ZSV&WZ@dL_L3>2v3Er*+U&%jcf>So#9VMS!KhW1EHK+0h|ZK5@%58P1U*82GX$% z2oVmnIPXH{CGX&&NOumGRJSaMnW#|M1($?NpUJAcFN(WvEcg5%+RoC!XeJ2)g z!tt%duwJNv$vldKaqXeJ6aaAS-gB(eIIW>ns9#GxVs&ZG5$Nx~LG61HRhnXp7Mj#T zWO<__QUZ=XEZw(%kYr@Qo;5z{s6gv+7U5$SV?buGAIo_<*Q9r{jj%=|L@Znth|h@Z zFzt&RAt|HThlxjWTz-|~M|{=9t)QZc3wpX{e(x@fJGIjzII7m{Y482sSSanT?SP|i zEI3Eb;8QkdrWL-bDwr;c&0V&MSMaV|sTrAFpd0jjHf29OKuo5(k2*VQFts>W=Li0- zDbMbnW}F{QXEb#&#W%Hz%15psm%HL=7k=BV@+i#&%Las{{@yP4hD@rwOFpBC8b4U1 zesI(Dz_`@DMUD%8^FI8&{6b9V_bT^Hz!`-TU}rvj=`gKQ82h?eN&@T%GFe z9*Lj%+kjD-2r*ru^z~a|=@ysYs7>wnoKXmGu(Cu#q)?PQ23F|);E3VrvU>btNyrQ; zZ~QPZ?1YidR(be(F`dmxed36C$xGx8mw8U`U(__o;XrqZUxSN{yhVr`lq$3-QK#oP`(o7=0VB;*MXrX8!aT zO}Pl2|ISa0)+E}XPuOzXnUGl6+;5n2PTo0kZMyOKm54qz64Nn4mjfDXsh)3j7Y-2X zj)Dk?{CZU{fmWIb*o8@6czbaj;GWcZd#jPqe7QoCHch5#sD@{4P-6J3jRF$uI=O)F zIPpy5sll-vS+#8)H6kZivS8kuiYW#?NVd~%ne;xIivp7Zk#a1nq5K|Zb%f+|AhG@c zfh9s~I=nqpFAdwXIS=<0r&dteZW#JaM=!;eFfegE)+Jl*m}%jzg}Rb-HjuT8FZW^A z2DF247%}?!Z_CQq3{1Wquv8MgN-D$|`QuWX9vm^~D&~77L?(y5;Al3L1XPZv$~L|) z5>-SY64U8Vh+)L>oQk1o3Twe>swFE*6Lw^5v|vYZuw`YQO>LwQkkNfYt=IpJ_J^@lXuu-m`A#y&;N6AwR*# zR`59fre0>#d!_>?M#kB4JU8HSN3#6gQzxF!U0!$$92LoYPog~>rk1b=VOtuqyi8b4 z;0CDYTgyDQq-zeNhCyj<@0x@3h=!X?A|*fZO?P;iO@%ytxL@vvJ$0#YeH`I+6I zjG7>wU9j&!t`P?%24S!IpnCg{ZlMIr?g8G*cuqU6heg(aNmNvSOdtPdT1OC-=h|m+ zaQgY%ye$z-KL1-topDWCe@AxjJ>t-DCD?92+3*A--7UZ9q&ia^DXdAW^5o)jd!`#x z{Oa3GQt^&%MmiC}-Rzi^{^=JOe3`FJK~8bHLeIsd5r^+A>hv4D0&V+x9~yv|t$c#y z`LFxxM?VL7q6{pWhw&0Vi~NBEztaY9KSa0tX*)=V-pmBODB?6FOS_86o-2#6{-s;0 zo3M8E!wbV~hf1wl8C=wJWno3o^jfg?w>)mUT{U57X$4djrM@x5Iw&2adHS5kXi+|ZsI7IXLRf}D3bm)6Hm6&^xToMkrX141 zilnoc+kRI^(R>>S8LoRb^gW32vS?9}Nu~uf9;qIGe+1gacvMioh?53+)}KL6^}e>T z%Pjp~wY{*$R^WumJU$|`0G1gpI;wnJYSlBV#RPatB3$CWBi9F)^fHvFP_JGBJI)P6 zL{RJtOD{ZH>JwJ$Sp1%xz|I1-9z#e6HQ39jAD+}g#t=gzpDmob+D~$wdC^y>pe@jP zrEwiu1s?X#%PL7ZPOA~#_&MoB!O2AzvM| z;)bBzhS>W_Bjj1+k5YqULRpN^lJKtQYwmOnx|m{ZVR?ziVD|5>l6foCK_0I3d}l8S z56Tn$waYU|kl-6FI4Yl0tmn@Se%5u<+^D6eDzj*cr`%cf;h#5Qw^BL+_PPot<6M?i z+jDtIA|nSR#SLnnY~v-dpNUC^{4*M*_gbVcP{+mWwDWjSG!Dme8wS_1iY3940LuNq zWf+V3zXd7e)eKH&QHE=S91ZAXVuIsr@HKc*-`w&7)p1sMZH>bwWgCVMU=#Y#j@fsr zrr~Q#WVClije}gIoUdHa@cX*>yiPy;K%P8X?PuSwOiY|^{FUCexwoUiz!kwVf#&i0 ziogpidUj=gcT%~TA6fN&EHwb!cBPLcxv%|0X@A80QW>$1%Jucei>k8AtecHWmRG|L^A6Bxh#$aR&21GJtpJf-b{pXPS?L2c3TDE%f;6-+&F zvNjHsZ*ul;Vqu+fRPPr5%CSBG4K0zdf6{ZsGu5ka?aD=LWk(4^$tnNc^`xg`U6gEI z0HCU9^en~Mn=r68;0FXKBAf#cuasX({;zf(zL^gaoCh_Kr69q1s=O6XOu@T_9*b`G;6P8e-y`B_jPu31B#TR`s&l(}BX@!D%y|mg+R6JUj&nLZ z=b4NT?xWqmot$gEU9M0{y6C0qye;)h+felZQ$j_#z*9U;?s)*3?6ybX{45~q8mz=c zzjBUzN-lQN2>+19s5CUP3AFc71zoltv{BoCQxR9^@2&1&c@}GiK3Ui7nR)(=iK$W{ zv)Gzw?dTrFEina1jVWF*>4N|m`dRIi1&AuMI90}{WBju$k7fHC+OU@W^Y5SeAEHH< zS6rVl=oS);r*tI;8a{}cRgy~HOVW@`8qZyym2QMG*6a|5AzmI2a~k^w>4^QWvDUab zksM>=5tVh@SBG$9zFf7Z?om|vhv(7T0)}kd1E+ZtCg|;G^;x4dCDMrR7~7eS`1frf zV4kF&fX#!Yr*?nd0@BkLI_vd4yO!?o$uFzML6u?l?y@7=dAkn3e2_*?oeJkPcFur^ zt8@r=By#a9+cp4+_2|{I&&HW`JKu z(&hEI+4}nU?_UBHg8q-bbQ}2(VtOvh706M^MVf;uUDejn=DQz9&718U5do5pC{6HS zDPC*{{QMUQj(UHXZv7|Y8491_PFT_G@}J>k|| zY4&{w_UDKS*bFs-(6l1T|2XOkxC1YGS^Zi7+jM8giyUtE z$oiBOn%Jkl6EVf@`mg^QbKm0EPlaM##!mOv=t3)0IJe;6KN+>KX1PlFp2l78DQ_Nx z!r&3Bs{rSocHnlTYiB#w1<&>97ywXvL5;9F?`k)Y{u?cc^DuJQ#9-$LJQ1}jw|Cod z3H>8HL{9Kf5QbaCd^$=#A`;U?^tFA+0zfy3r*APH%ClVJC&%luz)wmD57#b_+nkKe zVgl_}-jUw{v8~3_s7mXmxv<|8={oC%ph2Ozkc)1n4SqB7M`k9 z%O_a8{f!ZJvFIlDLh1rU0sz*r+dVm#4cv?3`p7*5M%yRM24Zc?IR{0f1L`o;1frJV z2mx){h3N>*WG-TV+w>ApNu$efdj!PtZ10NJKD2k=zR+X0ONk!p@S?T7tkkv=NPjKf z0$A=tp0veO11oR+ zT+hrlK-o2@Vpx@ZSadbz$EN!m$n_AL|D#6uEyjJ0{+GzO!J9S-Re;B%ODm9?Dj4@TTq8qaVRRv60 z0!()llmRm~g2)1B9%-ht7HY6i(-W~dX{a~KUyQN9&a8J98wy3uydU|ty*s5lhIoGF z$fV4dH6|zk5FtmCs*cy@pVAhX0?e&S?W{BRdSppu{FLhKj8QAX{*CE7@>i6mifZsLQnu=erBE8or!AYe9_IQzX&Kj2CpIxCZGjlZ^5t*;fe?vv#`AwUYc5;TaR8-gk7Hgjd8Q% z+&y3%H#@Ry5;^4W%QcmWYx&ma-e$LCBpC#^<5ZM`<~sd)0*OG(aUFIGG6XRt_1c0{ zO2JJaPx*6OYnT_Ak^mag_H9IipE4~Z7c1}*YSh_l>U3H1vK+j5sNv|^aR@R+H0$WHWMRo@SgVma0Y#?!*q`s|T2vQwt)Ir+2phmb$r zU<_=fBQcE{4_2@?%@swiOZ&aeWP;dHo%e9bvAeDifpRm0suR`F6N~AerbQIhoOxRJ z1u?C|t;E~Mr~}G>SNc3p?29bcOtvj)lo(ljT%|GY`U(4A$b^~YPRY+|Fme6+K=-%L3-*vziX^6GURnUt{*3NDC zYhqo#nbKoSnfFgj(LX0GQ!H^;-Wx03EBsJU4=cJiUsrbGXwOMW)y`P~!&zV29FUTm z(6Xs**R?by>lQ2)%A-w6x_xamP%10PE6Wsd-&qvb+WYw3aE3~cD$@BS-rHv5IqF7R zHYcqR5-qz%0e~J;AhS7X!$$u8KpAovY0YM9&b&XxgXDQb)4(UUrSz^mvbBy~A5@>U3lq zVzqhOgHu%edBx3#R^Wz_D~pp&Ow0A@vFVO#PN9NlgQFw5yT}^t<079F^?bY~yA2Yl z3h}+nwKg6GA?^B#K^73{x>XDO0{fCX#g$ARmfk&5=%MRPsJsFc8X~!@p0ym7q$lUJ z+!+H0Q#ta$+*^U8)OT;Q3#VE{Tj%Q>7@6d>8Etsm3S(iQ*VQV{@dm2sKhWx-?(>TL z@*QovJL~XguZGxg93OZjz~F2C7tFH_Iptpov@+;rE8$eel;_&D>#mSl@g<5rm*Tia zNoTCYiZU?KY0fgGU1fL&#}q0lJ2JptGKonhg-S}z7A(%1e)x>#9EzGI+9t3Vl}yyd znUWJN*=zQO70!AR2!V#l&Q9*rK}@^8F1L|RS(ahk&twPtuPv2DsvCN-Q8@cA3S$cO zK8!cqbjST5j7a#(S0mJ1uv&1JTP&_5c}4OMi?ixh!i@9z7F#%E7`GZBZ*k(Gtp_SV z3`0g|qviSzCnLfMl5UF}=K1{4y0vR;{OpuDNQw|sgggFGTvxn?9{46DcIamDuaZc< zE|WHoqaxJH=hCS1`9*XZPGma@#Md(}Zu^|DUox}b(OZppd!o8>z>lf1^%Oe-xf){N zUyL1H3Cc7rNoT0O<{r-lm~0CP40H7W{HL2KYq}wQ!Pf+V!zN@0u9Ynf-u14l8}FO> zygau$VJ!@NDwx*OPL#P|0RGXf;^VE~YkJQyE(wh~AAjn-I!NFQYSU&)qR$JLS17$* z8Q3x}vX?d&VUKAontR?2wf~Al*5l6?THD0I(S3hgCs4F99)Q?q>5}i@XsTT5n1dBJ z_1NwFOdu<1Joe3e47Y5UOa4Y(q$&s(XLks+hK;KAb;b(YC~#VX&jrN?6}|KWvi4U| zqkqn76RpBO9+@6qc;70IttvtA&R(fwKbF&PrNB}Ma92Z_6sQ?feZGG=Fblh5j@kly z1m8A@3aroe23f_R-vNZDBMcKGC|%c!}3IrlRoWG z15NT#zgywUrkq^sKY9W^IA!8WM4JSKUBze-o2v1obir*RTc?Jmf>EwLV8p2-%2%{(;m z$G0JVWW2k*8{wuKgAF_`6xk!Rkb4^41ZBsCN%^0R4+^}1{o8!1XABcj3iGYRy+vQ! zsz_WL69*`zNlSV?Ujxrj9fKAIhgFA#w=e-uzOW6mX>j$Hj!!2MvjEp*!!imQrim%U zd15oyi+@_NCl6j)XO?g%@@tsXi5~ZvtA-Vf(e2mW9H}$@N8xFLK)A*X9bJgV6$L<) z6H206`3R()IzlndpFZ2lf6aaLcRjbQG+Osg8Z*G+govVw^}E&8$hE=9n&)fi53LnM zixcDQx%+Vjuw73iJFszKd|mJfsllJqn_{pL&P88vorf35OuZ!7iW3<@am$vz5x7Ps zzpboVfjewBKM3l3MaclAplz%DI|ucFlFd!u9LTW~B#4S^c%^Js)%?uu42W&>a`z6z zcvyCZ?@^k8P5=1o7TRKtU_7alz{PoUf9%ljjHkJ7)gY4Fm$|dnyLKQV93=&Ma|k3k zo9h%y%MvLUeT3>i*ioc+sTc68b3S;k_pSNO}nqW{7`q?#b^Orb)I(n1}Sye+(U97Kcs z&2ReRc2WnouC8U=ir~8)&QHlw_nU!y>X%Kzm;0GIa=Yq`A&2-K;cJk@g-KFmx%dty zS=RAw7<}YUg&^6;;QEAqUC01#F=~G18Tu%2PW3D-n6-8+ok7(iagSsB9BV(K%;mrK z+9&S8e`KS0gMl+LTDDXSk>oh8M7L|%y&si2%WNozw#om{=`=}h$AqgSPL0`UEocty z3={i9BDAx&zdPH-c1C9Af6!^9=j5q*V_AP*Fo;3#C}x<5p15#hl|-CwlhPjfznbT~EDxbHW4nGHm57 zMy=3ir;MkhgnN;+7m#bv`rV41O7D7=a+~xLN#h+nEDXIrXZ2l0$RYG=E={{+%Je}X ze%NvpF;hXhqEkr&=artnHlk3ZdV*j4utt)sNw*3I*U$1w?-_L^aJG}3fw9!?9q1p4 z7vTjWZUe5Fuim|CEPPj_-CBJ}JA}%vc_R9LGzF~;b%Y8FVD!C!6KWr6Utiy@7n!7g zH6$})U@rEY>T(3?d-V#;L@gDqd>THz>?{rTi2(7!9wS|3Nu1;vytq;_9A9`i_q|3k z!ugxdR_St5sKY1~Y==m9W*e?b>G}MseW#(-i4vgp`KeYQZNL|DsPf(=e)-fLh}rG( z=yGn+D8m3IY*AYnsJZp$l>4nnPtwKFKL@!3NLpEmX(ds{S5Bg9{6$kZdPbXbnwi`; zt5;Pey+z-yKC9^**!f2f=dAkm*Z;x(yez}&{}ubQZL4hGYuE73mRPXaWfAOq)#jij znY7oHab8^NCUiGOjv72g>!N-Nqiw7ogX8vAO7Z?8?&JQZMt#chsS)H{uG2uR@p3pYzh-*6x009l7kstx*a zj@LQBRU)n94F~7A0zi7hK?oEmT#Q`VHnWXy))Bl$tS3iE%VxyvUUM5Le300JU3M_c z@U6AB?i|0Jk=bX3ouIl#P-psSi?@YkCGxYj(N>R#%rv!qj}@bmhTd|f&J3lt5FGfu zx;)e3hT(jXh*Sc3y1T^-)JZ&BdYP999i1u49n^!WK z5IlDSS-_~_g=H3ipKC}6WlSxAI1U`n!@2-V6HY*}W*6R?X9v!_&a$J?%L47OB;6D( zEN8%6ZZuo1FBKijL``Vr<2=~)9x`w10taOH_`sX(%lXPJiH_xYuHs`oeb}cnyztP8 z7F#viA-)U3-$2!(2q>V4QDLHcO-leVhly?E6>28>%5M_C9MncaCO!(s6)nb1Sm75c$LJO8XcP3N^I8tof zS2Bv(&9c0H`oWR#LC`L6MzZCmR=(>k4Lxo%lV8>E!R?7BRz45Z142ayd>#aQ&Y&yP z-eFkQ{XU#xmg^*aKEp3Pb|A#^$)TH>1{acv-zq&t zwc&m}pFhC$`Za3u*&)xyk;abx3s`D=BzMh8(O=T|{`$kl@831)>GaGU&jqUOD8LnK zZCW(YiN&5Lo*n&63IM{`TMgIIEuo3$uO*aodn2;iv^;-BUIY7D7lMDx-+@vQxEyRy zFNSOc5hro8jn^22B4)Qq0M&sjoa3GQLqPF5#(R?%*KxZUZK^Ge-ftcsQ1Z{o%rqX9 z0FapSz6J`!%71lrC=5wXMigvL7X%TnyFl)uuP4xN+P-*FTPb?_%B4T97FSkVKEqWk z|KX`ezOxH8UN?j^Y-`f52oUoel-%eNa7$rnr6haRGVwykpW%Cx5&aQLxti8vFD)Eq z3!)2=q(C-M8ovA28r_2(=jz{GFr{XO!}Wd{e<;U}#dlvEvqxoPFN~L7BvkY|{ubVg zh&`W=O&@Fbu{yd{+D#jMZlcR1s;D@+FrY8q(&#V%lA)z_os_Sh-Kyhe3*);z3|xxZU>44kHS|dj-InXT&L)$}F^whz`0X3%`RNdM0A z+o;P<2XgGW!;!4nVQr+bTUJvA5bt<@dIeqfv$o#6`30R=NOvV(tLal^?EYtDR1qSU z_nD${D@Le5okf0#<<436Tq3+}w@LW|M#f2gcjYz>?pmZp@8k z3h_{c?ZOupBX9C{0;!z)4VyPE;ca;8pEte%Yh4Im1N2sWB@6;OIhsG&$x zLQz^Ep(6wcA@l&DyjSpkp8J2+yUvGqowLqbhYzf)?Y(Euo;@>r<~Mn1WT4A-oc}l- z9Ua@9+c%Br=#FyJ(a~dBn1Cm{>Yc%KbYXONZeBO>$1RSk#tNFRU!0S9#H!^e!S0l! z%2XXTn^acC@LM3`*J4;$g4U0u(_hcX(EVoWO8Bj#`bDmhjxBky;a83tpP=yNIDh{D zu_+xhGgFOv4i*zrmpU=At5fC_S$$xv|Ns9rifnYIFzb%ur zxc;_lW{f^-#0b3Oyp41yQokMA7yTQ`%izV38Ms2;hW&}p}KBg z5jx%fe(v{PYvng$RD`P3IM4OBeRmW6cIdlV{|ClJMKQAHrN@CGrc1Dso@UNsS~3J%VHJ$vV*e#IAJddbI`DO%s-^W>N(?uj5eR=xz zhjYPp340Y11pSMTs8u3UWd0i($-}V(@v5MD5m*jcyD`o=xhJ>SexiLI;?|njrSWom z*VB2wFxA!I;G1Pz%5&$s@dkw4xB9N5|K*DOJ>v71@+G*Fpg}ESfZW)$!4^-tfnCH8 zal_3l%UTWcM`x^-w_6B~^yAl(Tk-`xNe#~&H!O10T;3?SqNK=|I$p3C$7*L+Ai*F7 ztR!s-8{NLyTP1uCsz3QoU+%iien?*or7B>rbscK@M=-tnUc>Cqb!hyoZTXm@9C6RZ zMfIxJZk1gw__nz=x`s{qqjP=g&gA^7dFshH%kL|0

      ~jDMfRG%`*u@%CQ{ny6U3 z>py6go%XTO{hHC@QEO$g+YXBVv8R872svkMtirCr{6ru5;|4;$T!ZY2`dk@nxfft% zQHgS2>#YyE7Gg)*b{y92TZh)ZjQXFH56d4hExtK!^HAR0cdP4O80FisX0y_?uQ<&hc4o$V}4Soy{O z2(X|28{fWTx!`+y?-0egh|D2Bg;l2=g|ETB7`jClmyp@Jps@mvHQ2+Dt7=d^!+Hx# zY<3AbAX#?_b1{t+jCkeTIILQ!wGyYfzqO9{YzWyQ7B?6;gdle2bEUCuZ+(lj>&`UW zN(&tB7X(g5@-EqVgD|>}^k7)Al^bn$tACJSn{3DQqBvz`%+O9%{AGscW^v*3$Hghz zNYhL^#q0Qy9kq;MX7VQSemk;W#4Xn8ffkO(H^|kM*X&8(%mm&*W+MU&X_-dP+ z8!Z8PWWQG_UGx-a8+FZ0B8~_kwkD$tJ;&=$gO1ZS{yviU?EbbDNodA|u3Ky}FmDUJfqoEZf{Yr|a6d^cBo1oHBF{yhFOkpZhmLHhE)kd-%35fEvD4@?xl!E@Ycx-v%Z8kCll#!9kL*~ zti`(6m5_4M*g5DanPNZsxw~S6%+ptyy%8unIlZ?#p6IMDY(q|RO?VVz1gi{H$EyVo zFhb)x%F;O&BMKsS>m{bQI~7{WG!mfx{RM=sYj)58#So~7kJN26>FPv*ZD1zv_0i&n z{hj$lJH+-nAtThU{UtW^V2>g_-NI=aPsuEP?f>EF^y7*-ak~@Rc2SuZQ*MS zpqadVGn=SB(+A(&g)|{|ezNk*d2~AWi=uL2+Cmp8`km#Gaogaf;v^lKlJ*K=jP^Pr zC2!RE=8E1a9i@amol7;`WJB@Wn3F*dP8B&DomqR~?T{{;vFGark0r+AvJWjK%77K!nFcS=ey{`=@7+?o}KV|8SWtY z4I8bM&*XnUx+u+jS3O=!qAht-d@*oZzjE$_`0Q{bM2y&ff;&iEm+)8~)ai^hevAC$ zV4YVI?N*k!z-r}~vAGGqmyyk#YjEr~ZQ&a?hV>7gy;QBG@5YKVMWdGsCHKs8_=vJv>fOy@3wMxE#h;52#fsmkQy8(f&aGD9hSwxfMjlspL-Enb9Y5(8i>6n4 zHsZFiqI!0-h;-|mLn}4WY2%YQbfVd3$Ah`V%sTe0gsNP!O1Oh3qAyabw6fdsV_5>D zpk;3M~#^E zUH4MMw`X3;VCx&+-`qD6e7iUK#DWEEa~0?c`Uc!v^+*U;0wZN`90MOc zG_$S~y{gWZ$H`YYA^2mR#v$?`4EQ~MlQCB;-l#Lc(=r$Q^1EM+1j!;dg^9iu9Q0m( zsIwcEV_-VF*q99?pFcFURjdSl5K8?5;f+wL)6Y7Vp>vF{L>bDh7Nqbyufim9dNvOh zi1JCVww>G0KbW_fd85GnS;bo_*Gy8_mNr1J=AbvM#!bd(k<`rU1lM%^l|0?z15bU+ z`QXsI@AL0edc-02iZ+Qt42n|UMM(%&mtRu+hSCNBO7+RtL(;-Z!7tAb6o zl9P%x2Bfl}zFT)X{r>4U$#rSkioY~gj?~;zqyPTzdR$Gl&g5+Z7dtsv-zbF=i%GYt zp>Wx!<*2(pb;8914<%+rj#&9Ayfe+^uq22B!es4dkVwR)z+l^Y_;=YHi7gefV>Gn@8}F&J<+0pdtHYxIm*HHK#cZUtwKoC36n5H_e5Fau$q)gA(#LD6)ery}su z>jQIV?J8wF)H8jSX3nm5>Qk|Hp_26(eFuQ*V=jIo<%m)9sfG|k4G@4bHPqZ&?Ye%>fPTM~! zKJU6;Ks;4Ev)QK7GJgM)&q>htTef5p?N}7M8n(?D?(oAn40(CEf}`#nsPkQv{4eXg z)sYj^D8C!V*o#=tB6Rz2E!n|P_q(83i1I-}P8vze#SJsUQ4Ov(ki8j})bywS*$c~F zpUdHoMh&@)W)a<_SS;N3DqR{NBKQ9l5e<{fr0W)JQ#r;vQVb$oZ2`KG^B-dw- zX!l=`w9-G<_;y}D?!HROcdtaNx2yaKCi*$Oz(&SHUnJwL_1s-15~-4U7f@>*ZgNhA z;Zs)gKi^PIGFcMg|CQib?W57dNA5Dh&|0OD!ASR|Yo~l#A@`$;3x(Bs`R+=O6xfei zeN~WLUrjZA0l9;bxX!ByxhSnqBl@Ypl=$<8QC9sLfZe4*A3lLQM3aji2Sbv zahwLZTtXMr;7@SNabt|1einLu(?4-7!zD0g$+zt!=4WrHtkV!`P#M#IDJu!eIY@%& z9dxnrfoA=q7745#a4?ds_3b<|izV^N|AlJHonj`N2N#z+9$wK~(HFQ;li0)bW zA0Ji1&7oqldM8n)nFJMsd1SK^R=IOTiSMqIZ}-)%#~tsoz_n?7gUGcQ?*;#A81yV3 zk)L>ay8Cq>3~Q4)wA6>XT%pA!MfkUC>T&LJexHP|DK2^AsN&}=m=1=O22pb}V%nxq zVt-zjY}f7fb9|{?@$@kP23G*mPoUKZn}etmazVM44DEWeA zyKSiNN&8*CFSlDYUThD5KOK$sQvO$J#(XoCx*nVKCVaI%&JFoi|Ls_H*?Dl&%+q zN;6L31o!0bzgn*D3X(Dcivo{Yc9+2Yi_ERFB{6&9a-Vi8}zNbo|4zB z1*FF?#?RGD_2JWh0tvf8)SQH+m<+LLkI_hf^_Pw9_T@?Q5kUFWOg4O;SdBZjG81JQDQzq7X}Y3sX9S(Npk^BJKbj& zy61ae)WV}j9G0lXWzR(WhAvV>V(G>+m@Ii5K_z{Z4{7w;;5ZYtv(}^8zzC5A zG_BGf7h;{s!YFJyBs>=#xNo=wlE!t8is8-s__V8Xz6lkbC0x2?-(rg>W zHR(NjSMBNZ!+CHog=iIbpS3h2kQ(Z?r_J~Ry(dt7;=g44j%dvqNEQo=9{+sanr$C; z3giN^!0Rr?`*{#fXW0PSt{|UyI{OH5)!{|2FWep+%VTdD`#`(juXaJD-A7KQ(+yth z?jKO#KFRg94N*MlR0I+JYy++ZetuvaAme3#sT1yst z>FE!Y%CZT4`ELVL+{wU85lw~ocySP(R8wfv;|GQlGpI#u7TCdtB|wY&;Ixc0UpW5l z^C_)QFSl>nZ?zd(aVxq|LWvoH1o*uMK);b7?@Do_PNQ-xViH=Jk8<_DEf5W@8onZw zgqcIKDYG#akcde2j_kdEnQ#{78zfbri2qnHBc*G*|IbW*e(P%wlg}71bmFLy{ zmoN!4Eq=}q18@6I7CcomzgmKIaV-2)CvQ}zZ~c<~mLP){J3Sp8&*nq#+ZWHu+FDidMaL*%O zoZwf8Bq;F=wmpWE?#=GQuvhdfisFY-i4wU&(kQoY@RTd;+?sr>%&cB_urvn06Omwl zJ8A#H=9SSUz4fm6b1@STT+~y(qEtUUT=nR=}$Dp`%lqI|lf_^tYZJuXk;%=Afe! zUO#k$fq!&i%<7z=^rmW4)uo>Xk+3U)Te0uTA?hLRs|)_0`-DD`s^%>4mra({n<-`M zYTbK~x=;mP*1+N?qq0hxoIv1lWvZvJTDn362#oZ3HthKsc`nG>c8Vd2o+ta({hmj8 znAaAJ9eS}q)I|M}^>cuT*h@X(TXnH9SgM5XDPEe&T`rrQJw=Y|bp5m2Tup8odni*? zRK3KTD#8N9RsHS59U_^PZ=U4}cVK!&ISm?XC499V>_xQ&-2moJ;0Emj+V=pF@cHIL zHb&hA<6c99$s%y0;_9MH6 z^pbr-%)`Lu zA05}19WpKtP~8DCs?$Jk}7tDTYTAU zuth674plKt+u;J|fR(wIS#_;{*$a|66#0bfUOyH~-&#-;aJx3LFA|(Cx2u-j3#h8_ zsRS)qJ*s~8<&{UrTHUg82?|MS=Z#`EeynLQU<9u++Ty|E5miOu#E-;Rz~K(|LMzkX zfvWnwLSlwn+vIvoNfz6A6c9E+u|s~gBInDV^+PpuNh+Ht`KK)pE+>PGxIw~JGs<`A zTsEeVo5u)&0ZZ}s>hv=$RHKsx7^8(HXzXXhCPI`sQbnpBhHI@j(6=HNnjzs1DC8hh z1T^!lc>E`m-rJc;=uYACXH#>in(a+>cx<>shmqH}i!;>$f3`Qi=!<7SE_XhoPCYO9IKmXcC)oWw5Ez3=fa|simr0w7 z#B)yiTZ(O1eo$M0Z107+UbV8zK6qBG;(jltNO-tIdeHsn{DgXDgqLme#;we`U9HSV zGA!K15=I5$tkV=0DRvEE4>uu>CzMD>S_w`hv+uzTCYGmsp0jW-2 zoDV3xxSs9o9F)s-cF^dm9-AvZdZRA=jEeJ51z~uNl{_`I`wl^)y{4$Oy6;M1(CwGI zg{!IRN23ZbS>h?WiN1HKgkL?sL*&IuXXMB=v-QWq&ozoO&(XKCDgJrj7AK?KO5=ib z?#q=m(T871t+*WPv>vOJDBAQ0f*`<$KL7ghS0B84AG$k!L6dW8Hx^!&{#lP;7a2WV zzV>DBCdZh0u&uR9lx~oFLz`AIY#?D&+$wsK)a~Cd3GJLN4)g2e6s!Kf{B-7 zu=UDzK^hZM8-c*Dl!;}JZI`18s!!hcPH$Dl#-3`>z4XIhI%3ZP!6pnx2a3OP!1+rL zZfM+fJjSg&0|der2~V7Za|cWIik@b|*}2O;lmuZHU;ZkEt2isQKgJpQsmr z|Sq=)MU#QaHA7z7W7--B4L1VNm&Sw-d_he4#bH-=2 zoW6ThWOV^=f_rem>x)K@$De6&HSz#QovfzHBN z1({a?uBXJ2A^l^)4gvC+odE+bY(8I!Jp z*{2_In_klQ^3xKKY3H8@OQxLvF~8t1Z?yRQrEdUN5|TKg;3*?d+L-Z|U&J`K>^h_l zErFv9^sWkG-UTW5p#F@^mFcwS<1O?eQPi)nprz0Ll5ws@PYOJ3DQ~@L+wZ39p2Q5Q zMWvl8uT<0?_*m_q6{ycV8qIQggf{}75yZ}&o7{a3w@}2Tp!2d}C(uv9ETVRC{cFeN zT1$r#K#$1C0`@${sFtc!rE&fEEaQvD*iQ7B^QH{EI_z!@K}1Qn;XzV${i^~tsn z&J)Tjx_kw6+y7aVu>VQ~UARMWo;#BYa^1SHCKdg)61Z=fGFCDtV6FFie_bI}odKbm zW*(i5pR^s_6vS;_&`m@-rv>7b%HmdsTukS{L9gWnZ3SHQWP~s<)#;zUu|39DH0?@7 z{`bM)=fWMnn!YMOKh|$T?s9L028DbU=Z3s&Rv8CpRx1=|W92wKu=z~S^{%!=Z@4kE zju`TNij{(3Q~{Fz~Kq3)f>hG06e9{0WhzO?K-(57dD-v3WiGjyx$wn-I@h__5%1x%{{ONB| z^nrwN@b!#GnMsN$h$(m}2uzu`!CHTvXxpmqQXiTgAv|ViotqO=?R_D-4^!>?dc>~o zjJ*LAUL2bu6o2#VhXF8NVGjqnh3FXsN5c1IVMW#kJZqR09fnrcb{|CPUiFe5RYK2R z%5)kQ)F@2bh;aN6Ubnhf?$7JICS`+2FRrs~vfX>?SG!B;TSv^%(^h)tV`Kk^4p?z& z)%Qg)pEvRM>e1vV*5EK4;mt_*ryH@_V<%{E2!J*tS>C$4^2<+r)7~6Yi&vnQN!Ihn zAjkL6VG@M**DZ?m8@~0OT6qNIWh$VGE~jU+LlTbI;;-umWyr*zO#O`E7xIJ&If}G; zyHR|1ZS}l2V{L=(>9i^9WbzURir|PboPnzjpcjsFbAjl$o}K}nGvQq+E9j$IN4|o^ zG(}JbG?raGYtZ&ZH?8iS66X}N!FZQhz8QPWrL>vXtMf(y%XiOHqqMF%bg`=o;>PC- zhEF4j*L>poil}pdMzpqWAHjNi;9`vyN-=v$1F0N*u1oN1y<(3&(-QL@#KkBE2HRk1 zVT&V0Ko)o>z<(vcKlpOE!$?INB;Y;1_P3|r=0HH}`u%~6`$UZ=?|Bt-Wx+9k7AAjo zz$JN_Fh|c%YvQ2uPyNvFv_aJ8eS%t$TucB3HiborVns!I9YPN9p;XUu4g+*Z8nM!u z56&UzdIp4W|K5tF!dqXjFs)erfQv_tHHRX)*W}XB0h`|=Hm%-fG*}guQ6`cJCc%+w zYq%n9jfu88A4~W_<{Ub4;2L4Dp^`QH9A%=Ze!9t*Gy&hVK#CvEqpA-G2!hD4?Fnz9 zmMnhp=jOYz8a?UTUyHF|f!vUR_=1j;1JB&U-V9_ryvUTuR~ztqpYTVe?J-j2KI0i* zzP@_fFCG^`?XTQ?fK>Tcwk%iaxCgoXX0`0f2Ug-0W-A>5`-Fqt3|BlP4-Jd-2>oWO zlZJv3O!BgAEix^l;vy!e2F7`@-AoJ0zfj5Y@p+ybb+0qz8)5k~Gs<%=WwPbF=QK^;u-DYPrGEv@HrQCasl$hH3wlsM(%2Zh znAiu?^;E}sm#Js-hqd*v%9m9**+vcRFpXFS>xsCiT5YV^MpW$#zwU8U@4=;8!$0mQ!?YW1!21#B`VE8viAH){s3Xo~ zLFmCTks@LUL_w-;M^bmw$8{l@-3=xiabqveG6ut zaJDl5N&UiOHLE7)Z)>e%{j_4V`xM)0jnS25mz`Y4L|s@6mE@-yICM{6Jlch*8XoVBkkw^^nCVF4;@5S4gF zk!`r>VIlwU32uB|EA&pIpQ^k3?=8N*N>aD;rS@v*j}Vzk@>R7$VgmjR=G$7Hc5Edo z<_a+VZ?`zcGV1;O-l$)z%=o>d{K&Jd{IPw4l}ElB!Bx8Fz18tbHX(eEx0!19nmjfO zz5k~=VGYW#p_Ukk`LR}I%OoseAY)Ybp6u^six)h1oV&*eK|TQ-iRlsrfwSZF?IHyqEiDL6sVs3 zRJC_*IKKhf#b9=}Qr<|AbWFWX`Ahrb@N;2WD;klCeUf)UZQ4&FEs$S&M-m?;fUT}% z4JEL2YCY5|?nb})QuuBpL8oH_hRu}}Qo{E6+8(ePmzkL0&8AYb1>OWp&;EvBiweJo z{J1KmX4Gk{W++Y4B(=YR3Ze*BOpgKb_X?1cwgMm&2IJ+4O9T~Q6Xp3zD6`i2*D6UW zQt>gu=$~gm{M}Ce({ilZSfl{ zlXaBk57ET?`0K^R?Pb$sex+E(XypvtykloMh4nNAjGWMhs@CsMBh5kDnJkSTwe_5l z&^95P8?p^&%T??Pchs;b`-Hm-z8%I$(s7r$UuL(unc}cGt)BwX{O|gm2}Godi)RRK zqezhrI%Js-8x+Xie5IM6III8Dcqnl#Ccv4a4xLi!hpW2u%~Y?apkoc%^Zd78TakuQ zK{f+q8O*S~3IByy!qb&PwM!9$_|R)Db?#p+v7*iG{12s`$8ThCq6K!wO1quc1K+hp zv?7Ds;VK6cFNwBb=`|54?1*f?x()VI`RBRyxlzgPhEe%5VO1nov_jVdRrXHmKCZ&Z zuf?nxsV41?Uz9}*-tKU7tlR17w1B0XZwP~$Ygkv^*)lEs3c2Dbmx03{y5T3C9659$ z#2l)p9y!DuaPWb8KBo|=K6v~JV5+J}%{tUD@8vG7;>Tcw=pdbJ!<%XJ?ke}{@RZo#x)fQ%#-T(4-f16#XZIAa6V^$T@g zi%l3ZXC5TllQM3hGqy$OdYrBrz}{9BKuSt^lE44?vxY}SxTDnP1jf0ex&9?p31oi* zqKFz{NErXz2FuRIb2M52NfIPLkaEbnI8Sm=Inyl-=6*sn*!AtU;V-cMa}_As^@nri zQF`|~-`~O&D$DnDJ@#_lt*f$N+Qny#>q2AFzX01?SV0FDrP|TeXyZGMTDK3D9n3qj za;v64(jnGbWHm7l$gk<nA!gx)G=Chmau%9eaPdLa$6;Cw{}xT(+r zs!09XpRT_CdCqAIj8dQ0%IN;=!HV`6Z$>SD9u{6%pMKpD!9;Y`gl!UE`wQ$0^pLOh zu@h}nU#%!FjoS3gAAJUOYY9|OR_Qvobq|;_itNw+IU?#+eG&8+FkR-cClaUGlOQ{f zitc>cv`Ni(0}vtjpvdYAuY7h{m9gE!BauVh62Y9>ObO8Tm+(oqk;GPVyF{Uc5#w@6 z>Bf=iiIi{@zX3H+z^8S5J8aAVzb`Z$yPH^8naX9EZBNR{VS0|9=_EziR94MfK(qp2 zJJ|tPiZ11Wr$*&B72l_dJccm7bAiiC!ELLNm6YN9BY|j3a!C4Y_q+$7wLcn|5r?mm zH}{Y|+r3wHdgM9AY&xE(Mg6?|j=uF(L(#2E6Ck!-olO5!BIY)ATyOebc0gA34G#EG z{=jetRp=hGyXP=mNC8FgpmKDAMV}$z=)vxeWq4sif)-a;cGPPRktGF18DWK3BcGnG zN`Yy}mDi4T%9ItM(7dvrae}k^rEjw%r>>(zWO_~xtpu7$rf!Nyt!9WKKlnvP%OcVg z+Ip-wH!@^=hvC^7K=cd8c<*`6JaE+9`^IP79m8|NS_cJRmB*7~o2rBy@uF5<=O zFj}%b#eTz3=64W+$6(DPJ7gp0Z#6CY}2f=s-mfa5u6 zC$@nF`8fi2Ub|4rGR5RaY*Z&v*}s2wbfB%steIB#b!P9`=g*}Aiz3y zD>I33*LqY1>joPpDW?!7tX_VX>^TQdGXv0PoS)Rfz;PusdZvK*5lO8-^K3*DuG&gr z{>^lM4*Tl$dOv=7-@9}(ySQ%NSrz}U+50}j!%e0MWgWdbq%y+ym#94y>Y8bhHS_=l z$cK9>(j=S6PYDP~z8i-LFzA1yeo!>sQk{}9y;)~*g1em5n$4*>@a_c?yMc7BBzTZ( zPW3stz6w~P(6`p?AsJUke1Vv9SQOhxuqsbubsQ1L1obSEm%uDbFf>*F$|5;sk-msV zgum@IkT01#iXd$b+mBc&y~Y1ZxX@|*z|!GTE<{gehA4mD+jcsuNO~op^JE#4+pAc+ z$uD)MC*n8Y!7dE5PmWZOG{toiM^7*T4u?tx_}i49df6}S#oQ#rdl}S46rlt_&Tt4+ zZTm@u!87vfi@K0c%nE&%fBBWY5goTtD&+>D`nn}+6SVk`U38Bp&4;lII|T6Q{84DH z0b5ti)%dAk6FqXTY2a5ZQ;FQAv3&?I;qLGzxW_mW?PnO|uq%n_R=A7b9@uW#xcu}x zxHcEP@k7DV5oZ(U_*f8tL^d`v!RpgaOpNGeR-FV%y?#K!hHUcU%~k|SvI_Li{R@R~ zmwVv))h+&|n(kHp`dOcVaODZWlNw>2n8jDqb%zw~)+k`ZZ!r3E^$sX-6FmpiAS3Tz^L{$^3p+O!+kRoY7`FqI$8G%M1aElxM@eYCPd(SDpT*gdTbjlLC zh`$5D8wmoglAOB=WbKNx?4yvcE<2tqigM{W)H2#wd56T13G$@00+KI$jz}_J&qTCm|rkfg(^;L!#JM76tq+8^|%5`ylho3de;wvZSb)i zs~KmYhQ+?Az%Xcy7NTCb!~T00Oc~*6lS;((agHWI?NwQj<`#+9y&AzTzdnRp7uy!^oK2gSj_)iq-CnKx zxeSrIjd71qK2X-1-Uz(W)!Puo=q8(C#kTy**s!$7QbkS}CpWkFJoSDrsI7pwhbjnA=+tu<5Wt?of;F30t&Dd~`&)!H=FrW@X)zM(d%zy3!WQ++de-5am+pq!}p zRuBTe{;XbQ^{D4As@L9O*qsC(&beGs<=6fVD!l&PKYyeyJO%VlGIDYq43a^!iw_|Xi0k)!B-Y82C6*3w%{q=$TRhvf7Db>Rvlx|1kxt;K(ef{ zt5N7^G#|IiizItJe7a?Ei;<+giJo~iu%89576CKdM*)gRi?Dnbzo!(M!n-`ci-M*TTrx6>Ah3Q6q zz6+WP7qP+G*a1-e373f@46VV^&56NPp3Sc$N=-%n;1>XJD&c!65L>CL{BRopV2Irm z_@{E+?k3UfZGE@HussKYP>Oq2)!?(!e+kpw)9mX-1CrcAP zE^G+T0~*1rK&up+GXaE+vdqn-VG$L|$dA4;Jfb#r3E@_36=fZvlGS)s4xM&L&#;;% zHZXDQ^3#Fci%q@tU(p&~51X1^=L!Oc{=jKuO)DhGfZR`+v9=4igDR#zWLrV=IDGk>iTNa_GS4L{2D^oW09 zx;7fcrP~H!XpqQGnwE_3BtqA0SOnH$_dX6pCC;W~IpUS??rVz{9}9RrzkwR9JyZA^ z3$u>Nz-GH;sp}IogtwY}yEcvj5D^$ttRN^hVKq8xga4XS=39;xd8Z!+=N-PHkwwoH zXnR5o06z6}Y{=o6OrqAKZLr@+VP7Lj_oqq{ENoN&7FZuwlG{x=(R*&IAUn?SQS&db zF9Eq$f$|-y)p2^+G-BsV^{HJG;W=0lIgq_JW^9o}4b^L09H9tt{40U#jM1X_E0~20 zB=-Y0@!3fHGi6x)(P*d_k~_!QcWO6ASj?y(;~+1R_?gf&=g)V1^X`K4iw2}nEB^71 zlYwcn10ll2J@G8isDo9zeatU^*@mGp@W!m8pf#bM?9ch--=P3uM>-Al&u>)g#=9O0 z+Z*wrGAC=cK%pP=m!3!(?%mw_4HOIpWuDcYo_1#CR|5S>? z&$8*z|12o7r34AT&kloEP-e&c`7%)hrG&t&Cri_RrCe!QElGe5NQeQMLi}5z=^i&o zg=guCQB046b%*qR_gU|cZtuxJHc&#YdELUmcHfHm5|FtQ)nhcsK9#*2q-Th|gms%a34G~!;Ccz8{nV5;DIhKR&k~vm@8eOwSCOcl*^iHMJ&KOc_HNOZ&e{n% z@w>;5NpB@+Q_aSCU=oii=gBhr@Z!~sv+3kp_@i}zKxw>6r

      ?Y0WRVeTM?hm&r*dF(#}9LeM%g3UUFe6>;!rxinlETRvycf%1l?q*Qg%_ zKaxuQn+ZJ8%|J7)In6ZFdYC`mI7&DI$eLH*BrThsi?EgBWmQS@3Xui+V9WEL~8(rMSt;8bPtnc7*ZpEc680p_$CDZ78V@>sRZ$BJz+`_-D#u2 z-t7nTLL)q*1L(n`;WnPM8QM|P;*g3)C@;z;vL|We)=zHE0#2{4t-;M zFcu;_MvejvYK?V%Qn| za;XQdsXWeUz+p-jI+PAOo^>3KD^^${ zsO2b9x79?duz}ag_oX+B8W986hl;yjOPOta#FU2=(A~YKmXH+*=ju2OXLYA{T7K360N`=Jq z#ya%urbDwr7vE^k+wHiR2bF}J#eBoQ$|FNR>fPcFdQoZ%g7`$|WM4Fx6lIvUx0|Od zOvdacEFs69jD;$~V>Fi9J;{^c?t&oLTGu{-Sk~|i$ot9BU}fC!@G0UdZh9X7>%swI zyz`3QD|NS7Fq`G2ojC=x|Q@TPT0>!JJAR zS;F(l1dWiC?z-}>CY}=+DJrD3ZdnX8u;PH~SZ-*?J_12fx9=xEVN=VvhTn3Vl^d(= z2^t>GEw;Tt$9V{T+H#jN^2jG5JlgWJIk2;6Ys)CsFc`Z<9GH&s)pEA|+qec;ApNyR z-`6pP*PWrz)QfU>8_Mu!Hmb{)M8fFzm1L$(_@3bUtZ8Ctg_I@zRqX~=Xog}&d}7x) z5Fl(%!)n&tqdVJ42MF%+fK6tV0=RI@3f~dbpX_1|enkN)&CL1=*M`c3Yv&=zIb}l? zj9b)bBy{1B>BXXTR?-De)~v?F-J`GsVgpHUDOt z(l!r;dxjR1Qv5;(Lk1w@Ue}Cl@VNl0tB5ME=TmPHfPTUOoGSiyK5+)fU1lsD!0?I% z;(r#dDTAd;qQmBG2^#=PS=DAZ9($DzgW3#COKR~MTf}a!L_*zP!}@;g>S9Il2oF6o z6p1c&F$H{jmC}d3Jm_n^Kve81b6>W^Ld-YUOk&&H=kc4z?bya3Gy4dvM1IKL%6D}e z{Gd~;*ndcP*eD9%No4ts-+FZVIu-+zv3U<$wsozCR3-Mc^*re!#^pBSe1dIfx4n1x zc?Z15?%744tTy}}rv&Pt=0k;&05M%hDIXw^-s2TiIqbG=GP!nkqaCPSIWYDC@ZtGxcu6Fs{L z_Xp>MHXH+byR-#CCvoy#~t+l$dY=1L&}oVzF}|$T^RnaJ_V4f z|F?=5fV9f|pXl_5gP(hvy9kpr&!O1&J=JvWYh{VL=O%G9+G~A)FbT*!4@0|;V#(e_%^@r{ zOcg!v$!*}Cq5N#j8_v$Fgkjp345>q=Lw zM^8%fd>*#zT2HPgL)()pK!-rB>R;P55$?Hz*DCcbpUkp{TEguuK8yrahBinJQxKX3 zU7u~oN4bQbCbW%87U(_8;A_#4Dt2~j(M==utYk9Ld zQC-rULd)1nvLy+(UyXTG}0Qm2rkQ;#3EEIPyVc(D^y8bLG*Njqgip8&#UtW7Z z4?yz3Zxnjd9XyjawpxMuwiz`N5E%f91<9LR4HDG4!m|35`h>EA?7PGo&E$H?>|`R< zVOj}LM~7J8Pl|iFCqXV@rp?W)d>QQBpNq&#EF9M>k8#WS2aQ=)Dm)Vyuyzq{@OZWA zHk^obBD}gfhS^*8-mw}D#CA$!*ESHgPn>shsSV?g6V_@3+eZ;2Pr9z^v>o{?17Xbd z^9{*-IIVBKBH>do7xk+pf&HD+)899PQ-w8*f;B?yW|Y+cy5f28f|Izurj2&)NZJ3Lo@Cw5 z&W)ET+4SGd3|aXi5<2+%hH z71Qr|$phEM?e#p96&c!#L(t|6C6x_8-r~%}tasqG5Rn8OUQ0cl*cgU?JeiAueW913 zpzcEK``2qq@e16P9UF*h83WJc&*4+;$xt`El+6k5Cvw`>9WwhcJqO{d>?50MOoybr z1q1@93St>zN>iLk0Ml*_Gq| z0?_}Y>_1@T7SZ2})zO)ywcoT55}kja98h@F5A7Ta0E*WV+NoIytRsb;WM8=u7*Hp= z?7w5B57hM9g4tt%8O+93H$6-OV3reO!gYqSM!_l@s!V*z?U&305b7cLV;KFlv&-(z zJ7ZUM8Cowo2k_C?r`D6}oMDK#j~gM1K$&e08K2;Rj}qi#hQSLbed2)rXqDq$95AyR z#RMvaaJLQp5oP)Se1ZyQ>%54aUpcbok1SJH)WxM=6hP&sEdjoku2gYuDT*Nold6h< z7n48EIA3bL7?Bax?z}fm=u4jFA*1ne6bF~R!M1Ru*UKxavyBv+QviOx_N2Y7bj2$_ zA1ZPBP3?%8K1%b`M9y^?_DZpz0XiHgzxa%#M-Y z)D$u-4WKOBfBBRCRy^(R5OaaLQ;5GMeC;Jgceb4Sg($6zFHDs!aWRspIy)j|v^$e% zSHCN){edO*?pdXp0+l%4DKZU+Iv%jMM;ZTHCQFy5pM_p zy^md%q3@>QLR0bv?(?5KY$Pc7mqE&~Ox`25OL*ia2R7k!xKo`7^y-*`QA|L~I2WjG z6{rSS&bIPX4Lv+5lwA7Dad@qkEPc)5ePz9aTSk>;qt%rKpj*5R?VMz1)#&@Two%~@ z1=a=T)?La8T!*Verv&<^W$OQihfHN>;dWHd2^pAYm4N@?d<+dbtfSOY{HMjp<(~WW zGWk5kNJAwo243tm1-L7?`<)|a0q`S$18=3C=Qa(XX2{R3h4E%f4Gw*>&| z*9m&rckd9l2&5l?C5iYhUOTxHtK81PEtZPW^8SWf`wK>V@VL9OqtWXl0JW-|`+DQ` zVLf9k|Nmf5 z;h!M1wPAh!S8PzeA5N0v{0?z}+X%K@_n;T|qh;yaaZ6J+nY@@EMzTSE=@l73WDOt@ z{~uX99d66$57$muhP1!xMGSxIc%+rMHs@RaE}P}^O$SwjUE%Dr(`X4aFQ&N3vB><& zpk&T#ii0g7LDJ~4KCP!Lub;50dPji6z(H+d;{U5V#7kNod_KG0Mf=i2HZEQHyO?&L zfQpw5y{{l6n@s?z99)?ZSbU5|TiEbl+Pa-1?P--UM$0$^)Bk*@6kw>yA+5m?7w6k4 z{hPKi<|~2v7z$9o0zNFLN&F=dN!x>dJ3#UG3 zJ}EQyYJQr+%g*O2#fmYGEL|K6Y#HxKSEpWu!3@brzk^l<Tm+eV*A(|-9Yk&UA@65__u&_Ij)7J(D04nIYF?14l z_kc1uTv3{o-dbywtbT2A&dIONcUt-B+j>4=etJo@b-Cr+7}RNA?Y_->2Gi*cSO4TR zYfO+wV;6QEGMp~;)Y#IrB3+e+aC%R(4bV9yBg#A@>Y`Zt?neM^9bVNR@U#CIP=W1s z0XRAB>9{q5@z2pEC6Y*s*D-P@BM02X7(FUI0eEwQPN#9|-_8tZo!!n<;+F#30_YWI zi}qK`(1j0GK$abfP!3`Akd0GuLQb>(DLe02CoF#c^ZLVD7-YI7@YbCU6(_frR9GQ} z08|2wAT-Xe|DF8y+H$Z}`=nGP3y+=Q!Q;hMa#_(OBO89{@Acolovbr?SDI#Lb}9B` z7&J3Uo5_=lRcF442KgS}P~<_a&(E-O24zw4Zg65pnY7 zZ=WoRQtUeo)EFvv^!fu4O17!8Ka;rKs}p!!&O^p1J{^NJ;}7s@CxJNMA8xrJL2m)) zgK*b>b8!19uxk5+b;J1yU8(ytB^&al)lDR-&@>&-xw5@3&t3gdtCZ*62AGUIW)V@B zy}gmAgO!odyV%yVxSnm^s(e>Gmc~aRRii~@)(}cLAUu33RI(v7b$@=B_S-8-jL`%i zK>*tgeX%uK#uz;wmxxx>4ND}6#CA$EG^A=oN^~WOGnJor+*C>vXOrvfY``8u&Y*Gz zRQG#v*Q@7UNT zXnF7CT(f6%;X9`j6g{lE`eV23nnUCZ<(pe(+Km=E$YlWriGrVf&Kt_2Uk(t zA#}u%=_Q{O*uj^OY+(6Gs8z9RrD|Jk(FO$!!Rh4P(VoQvf*s>(upShTzTAuqgi$Ft zBG)_TxSxEWK@=Au2ZA?=ISF7Ts=eLsG-|ic>-PyXPy%6ko}pd7t0(ck->y$-N%z#w zXPmv?E_q1zh43k5!6;Vl&mGPt%_F0JmcctY$Gu3et5I40FV*=xk6}@M3h<_IhroJG z4q}xG4A3B2Z(4r}MU4_Bf66Ad0>3IzmiC>6r(C^BkK}(d(?4HJ_STCn?_Tc{zyMDr ziYAm;xLt3wvh=v;iXmG*0rhi3S<`2LcZ`-ct=^oV@80ySU;gu8{qO-pYemZA5y0St zoqcsejU6axNzsy>uR01ex_*YoR}iS41wf|%89{35e=+yw@ldw^`><@Ov?yh#xQoiJ z>{|(yY}vDgkY%jdw@4czC6qlO`##n|7$ajzmccL+GJ`QRw!v7Qb9CR`pYQW}zQ5=9 zdp+wv_bukS=DN=HeID=QINsnJCPQiAnSb9Ub;nZ{4QNufyc}b^A5_4;ItNtNJ2h;0 zc06?@Ti%+LwB4evU=J<#Hm2i#>!=<3(nC0=$i8NEjoeGm!Lswc;gNeIg(}w$8x<>u z4K&pdfY#c#rXe<8wEgSkaGS<6E2Yyjf$e?Vd6sz;)&o<9v?$INW1kG1tVBD37OviK>+jBy zt(^AIH%doA1)r2M5%|v*o}c9(N2-cxF*sz7Fg`Tvl0aBrdYP1EarA0-q_qN;TP(C= z+xErX(7Gmbcs-)g`C*-UWOAJ{jQDl4BNVKW4=3D-O{_wq|1@1MPS!#@GxlmN$y9{zqTA}m~ z+m_aSA>SC2>Xv%z5~k=ZPWk-ZZEsbMnE;vJ%Lc25HHKa8_n25K=_diYTqxpL`+mjp zUa!S{8IYtaGB?hMeJp{K`Mz#fV$je$b~33h1R!1GO{au<|9MKCt^KF`G7h-^*;%HM zUL1oW?|5VM#L%Z>GB`TnRUEA|Ui_I0KE4$`F#K(9DDh6o(*%xI2TEKWta{MK4Lwd8 zIr-x-Yx6$?FZHKq?FUuQlD{SZDG(y6@p_m*|LaD}q0lj6yT8k6XtjJ!Iz;h0a zzAOu`R+kmO$9I*>KbZ4r`tD}%?2~s|N7Waw#(&t#?IQ-ymk--K-j zZWpCx39s?P41Fk28>wMn{7>A++8^|Iy$X znm(xiV(Z}QU+A|=qc|Y0gN&B=njtX8;UB32X*JQN=`%L*KrG0*Z(;Gc@Gxuk!4W3X z$Acpa#nKVb;H4&7c!U30_oR1dPS^+zimUBcD+0?b6D?f9r<|we;V`|h=M_01!#O@Um_11WX^W5kH6*{ zQuiGPgj%W$V!RndxPS=9tbbG%6Za+>mgynN-c7Qt7oJN-zVt-p7OuqIm%4pSCnstd zUE?0dDfEmQh$1PZzi)<(j-Ibor`m0hwQIuwjhl&8CHhCnGglx0VWz2+_i0MZn!1zFx0Fe;)M&@W4{H%%|x^@ju~bEfP;J!e-CK8}Rat(_)4`d3RB(D2 z2VgjbG}zhCGaGd0#r`HT{6yY;>;2^i045ugz5sI?T6GV#-TS@u(`Tq&B@;4}yWn9w zjKC59QK}l_O9O+Ge=E&~;e`HkT{kn?r?qo#OrNuUNR*FtD^%@&B~g1WOx$tQ6MQp$ z)MARq9%!1{c_7kdS{pk{jpL5YdIbmoZDL5x=5=!rW>zgg3ROv%M0;^|+;bOSU!EGO ze32^I1gDVN0#(oAtzY-RLL{;-Z=e5h%e#>`ttf^*Y81Kpu&)lEKcHo4NZ&g~=r~je^6BR>y6cJda2iG|OC6`UeE8_SiUphfonKb|ZxHf|x zTzliz7v{rK-AJ9gx$mvgr*~4JM|m~DIx%DKcrp7+rmFtuNV`6D`O#Q&XfOgnHV+o7 zZVP$DoS3tx`{f+r4aIpbBdW%1_#!bwO+}MD6oX8Bt(EED6mA>ZJM7Fn$(Hw5^xHFt2xnFBQ005#(xk%8ws+n!k9X zbL(5yC(5nImw{A2+C^)r!_$sa=CFiiu;&b$-FBMHiOtaM7|$sU{%6qn zNpj2TDQP8DNcrmYRGi%UcFQ9udYd7aif%X0WqRG+0)!+dqXVqoG54>%UTzGiXIYkb4(2p>1+zQq_S z*>tI%Cm=-O=T-)BkgtIin8K)LcD_F;G&R!p`t5A(%xl&1kjJsl*Xqs+@v;jDcU|(* zI*#QKhWkTkz8&edO-*%g=q|c9a+!FhbjZneoE^ve47iJMb(*&}Fbhv*DbPi(^Syg6 z>;h>V?=THr91x)hpSz4PH+4MTdnaBJ%w08? z#uAZHKE=p_Eay8ZsLSDiuFOJR@DRKm3%LguYG>HPZ52BmGEMn}2n3%ODBNw*V3zXD znsBr3hLG8~=lxWr+1fQ}{Askl7H2h(n(?i?40(;@i8m6khI>S{AY~7f4szM1QF#OB0@URB#J519_FJ`9edTV6 zH$>+zyb+p!Bsq^HAzy6T3o=7gYcBAuSFY;yR*JloqH2lerH9{S5tR$j=@g;DbHjIy zs(ETRLLQ{5(ZcV2EHoLC#FT#VR2t5nKEzros8_UC^pFl#=i9aN;T`_=jxZjtTyo0g zi@*78$-P?@hWurT-+IEX53Y}+zefZt+UI}eeBA59X}0&}?%uU>V3optd*UA~rJQJ|F+~kN(Z)Yu#Owt4s?F$2o2S ztuG`Md zy`+`%%D16W9=(+%2C2F%Zo~J>)^98MrMb|-8`rCl-H>}OrLbKAM~DJ0amZ>90BzJw zgkLXmdTh)S`BoB>t4DgRD#3fUa;JD9Soleam8OH7m0}r=&PjQ^PidjXd9E^l;KDtT zY6DbpiM8?u9t*ix&v~!+oz>XMo7KqT^Qr#_LOMo9nDm%N$0+9=$FlepvoT!#tNs2F z(_47QyqNYJk~u;VRNqd^S_2B^Bo6$|)XUYEJa)Pre$`6%jWIeiOIRj8=yn1L6yu>1 zd?*CrabuxvBsST4BG$@ja$SDFsu{p|f4@nC#F(cRnsItjln#z*b5mv?{_b8q@1=OZ z;~UT^<`)S;Eg@*MpM9EnNi>{>hu246>%W|X7 z9tU7V8CD=yXV>c`*@Q~cC;+dwIP}fxVKY>Cf2!|K4>w}&W~{ky+Ck`^#EyY82}279 zLn)EwgF^u`l(_aP5h#+65rTulabt~%R(-!RElW9BMZcc7Je+rw)mz2>g*;WWSOtQ< zqsJ-|oJ6zPd}Ct3YBdi*-~HN_+y``ZZ~`-HrFVibq;_!F5eEIacJX?9Q3;yS*Ck`R zk$3&4;>x|(=Qk2@@29JJ?i7LMI5}WmN++=Q*Q`-?L2Y78|EaYmT>FGMoV3gJg!wtC#vMm6xbJLO@S>+@h=BWPKkCbgeusJMoop+ zKsT31A)K0LuNYJX6*6aKCsOwL+I0#| z6BMjWIB0oIoR_s@XG)ev%-eGo^ang+yu;#Ryo*#N$VN-G!zW+vk9sN4sR2pKVUU)0 z8EevKxi+~`VWRDLDTVwlZyCI~*0Ofa1HVShv)E0)?i#BwZ@E$ZyIexUE)Kr(1}QlY z(amXv4m<;c$KK5=j$u+NVZ(LyL$SGIVcQ@qrH>+uHWeL%&;c(YT6I~0JtTfo#TdX# z8121B%U)Hg#-k>Pmi9lKH4$%e9KBTapmjTB2+GRaPFo56}=dz zF(_B%;g!2L-yE3uP8Uc#YmWq?;S_Vkf&_E|Me++tUgBw3PyQ|CYafE=tbCC3yfP7DFoOcBH;6zs6|2WQq*%t>(v0c6BkOJMVU8ESJvNiPre*~ zW}uz!L1On;Jr}onel&vYc*4Pm(?wK3!DC=DZLCHpD#ZR@JB^!($E^G$9 zXIL|!`7UC+6~MO2XRBoj&2laWi7teA;?z`&C4D<(Y1Ok%#wwW*NYBJQB@9<})oPQ1 zKI8!ib>P0^E$XTU#xURKOKLk2FQztGy6x3)2Kl~2g|V-v;a6araDTJ3i-jgA=@(w9 z`360g(8^s-c{(h6@t*Wi*UxhYm5dSSzyNRq!{42JMVE}9M|d4u+-GC~`-)qcvhSW@ z)TDQ^>*FAAG7a@o-6v|`}LfI1nkxJBcob&?x7GPe6oo`NavdElG&;aB@}0+m4WF=b#%{hwK;?=1!q0#Y3pgA0+ylxPFnWP_%L4E|McRZ(Y#*;o z4JMctO=8BjD@>}Hj@xbebY2=Zlb6(z>>z-%tjK5y`k!Weq=G`x8fL zI!GVYLgpo^#UxQ^ep<@;r5|Cnk=0YYa-(qz3`iQF?Evd)qz#}>CMsH_STnvMuDVFd zrmp9(1m&F21TR>B2y`5mddo&wDDSR&I6w2mz&H~lHADtymfm&D_~n3F>1L~6rfn_4 zY3wSM14);Hd}4h$G&|%f_i#r*^Gr?-et+qn>ke?6^Xp2uJJeDd%jU+MqU z65Si{scWT+e5Hg1Am-sya7JJ}5*fuZrmpa8baxWqtrKkvJ95ubm9<1Q5aI@Np7KF& zV@VaOlYiQ(iYQoOgbWUK!;+FjBO>NvKFHR!s+KEZk5RQZwHo^?N)oG(h8~-BF9$6E za!F%=Xq&@QX@ckGXD>Q;zq`)4PsFKrr;&#Gl`uOULtc{f0S#-h`W zq-$$Op*DD$jkXQM7Y~gs?fHv5+dAF`(l>?2H~NEbK|{Dzrk60=(`yeU&!@=EJ!K&E z^;o#UG+&auIrJ6yaRx3o1|gr3pO@e#R$k0h6FT)5odPF6s`g)P{rw(|e~}nciL4!L zj-dw-NtgF`7hC?%rY-`WSMRV2t=k86^wZx8k?jjHQ@9z8ovz*xe4tq5I3Vb$qK~#K z;c2!Pdc6eiTSzE=#vsvLd)z7MJ+kvPKO&zjELZ9}bv`Mcz_s#%7<~?v{fvcsC;dVj z96_s&VnQ+PPYqf%_{QBHv5Ad0e%>PB6wM?!4~Mkf2%lP}b6A>G(AxU6kRWpVoPE!; zjl2hm^idg`x?RFMZ&yBixOd{JaPpDEQ%!#@>5qLl<1=0#6Dam}dI+_=i?$?~yNj9%}Ke<;K~IRjZQCV$m`7<6~wgg$k7%Our4gz3_CA6-Sj{H}5HbcJgYK zsXPv1`b|^-dLFOx&iPdwVGj*?&<=-vieDwV?V$94AczU@jcUE~+#GX7}h&lV@ zN${_@Z)fSAE)@MRwYheb87Q40RYGa3OVw)TX`UrABMS>zj0ZmiP|+?+ff5quWavq1 z@DBnZi-Lz@<-V=Uo_wOu1k=Px(>JNlY^Ip3Npm8{+ga!h2O=^~YGD0VEQvF_C+OJv&jO&6uL?)Vlj|4a6(&$A6s-Yh*XH7*b;^%PY z0OB~#)6mzZed@2K_q$Fi<#>|v&22vORxq9_f=%7|>MLJWvx5@phN}Pb^6aEthgR#1PLmBr(0?>Rn81Kk&OdI7@zaN6K2$O_x3gj;*1N zSJTm(($=}8^BFS7k^-Nc*K03-{$%KwiC69D!59}S*Q2Kxq6pa00PcCeu*Bi_>pWuJ zJ8hDDUDHdvLC{29+Q*l{qtfxlr5P~;%^O#9GB^EE*_L|d{iYR)$+8>I%(u-j0+B-73i{TR6 zw=(#M%`|s*}d*?oj+Ci z#V-6K_@4Aa>9PNM1xmFXT577rJ&ZwL-=65r4$%ghxz=*LGcgy!Q?e0zkb3st-&nkv zShFA5Y^U>74)Tuh-z++R#$^asw8%Z3cW($#iud>5H1wm|DFtAK^!L82tEA`poQWno z!3J$$?pyKZoSm^>Z-9dlUTDY^eUEj2x2@wmK0Fyk{gVBQ{6}hmabw9*_%JnPtsV5z ztXl2~!qrj2DlP#3`aCZc9M+J*QbP#Qjwimg`{?KXBl1cE(Ks{9F^!AuMT@Br;o&zg zwA5mzaTM3Yz6S0oOEyvW~s`!XOWY=QT?I4dw9@b5G5C z>;s|urpbz~sQ?KZN+Gn_K`!fO`o9n7|M9HR^9@hQZ?KS_ECz(DvBI}cYLuC1vEp3% zrKisH8{6J7Y5RJ<|2;Ct$mfhu1lRKA#&arj$4sap4y#ZEg$+DG?dJYp20WMlwDov1 zjsHkX4{&@Qf34D3Ip+cQQ>0GGAG-|u8a43Kv+c=+Tq(joe*prVUuEGbU@!L*8oLS` zglbQ`#&(VR)>h0Y0AIH5FTyh$G)aVhOA1taiT76 z%B0=S;6z#p`lz72upqx80rdgQkw#itEv{1aS}z=p+kM@eH|CiZTTYH?wM@|pZ=U|* z$MCvQg5}Mrw`uKXS)3>BMhS+cRQnUppMkr$2W;HB2hRzr1K zdLWnnF8iLxQ=_ewQgrh?Bgk-qa0eLd!JqDUeVV#^qSnwvk*^d*zMi@@K88nU2q?xp zUitb~5S82B{_tk3sGj0w&06YLtE2!c>3=TH-{H?hHd)s1P2RO+x!FPR)a^)NjMCTG z<_@iUMYR1EhfZfR zO?$eM?qM_~a)q$qtOHeV7<1~?*K`Fp*%motVdJw~6w0@E>pyM+1Rb&Kmxm0e`dOmv z9ddxT+W2|B7~h1RK`NC9dE(KWKTs=lHH?OB3rJy^p(_CR|sfVPWWWD}tA z4kUfwIh1D{u=wgu0nF;=PSL#!rjLZHpm)@#yMn5iii2zAv~^>k}yX22WYO}`|t zlx8Z{;q_KkVlm0|f`$;z0geaQsUsYGS5w$LebiniHE*`Amg!Fq@=4wtLcQ8S>6Zoa z_-ih!JJwY*Gb3+gCKswBSe=czs9$w;vve8o{`07r?2+T)Q(Yi;aKbwW9~~W?Mu?j0 zCEcTc;UmOh-?0+l4)+();7xq0Znn%Dx7ifv?Cbv5CsopYTJ<9(Wzw*@-twKelD}^g zP9J4p-l$xk*`=MQjVJj0Fu@*|(2&shVT$$hyrdT+FKO8QE-jiO$QzjX@XP7p*$ju? zS7C)g;Mv_WR7vUT_t_3$1``Fg$98P1tVIhXIltL+z~ClctTr(>UCA#I5{D(Nkp0p6 zVLXZs4fpz?{_c%uAZZ2Wnf|5WA3@JoVjkhgclF8OhU3cN{M$|8Q)XZw6V_$yL)ACK zP*jdejcz7Wvo-bmW8o@^o2sO}c|e_dj26_3#0u(F{Wy%kVUfX^?C8DkRAxtw7Wla4 z{>OX1Q|~@#^!INOJk+MNtG+TJSF`v1&u=+|(oY{L@fg?0Rlu*y)Pr2#k9B~9)>0en zS3^kHE7!fx4d51Tdl3u0^R(+H!Ce>@*A>8UZ-JvMhpK%uRKl9%V`bbi+lgOHxXDYw z+Xp%KKeVgxs%>p}DZFp5v+tI`Ob20UN1lcT;X(3Y4WB}lT8jaXTVF7d4~9=gn3P)W zeAk8HN93sQG>-M-d`^I0FdTb6pgo#bzd5VCH=wkQSsqcCt<6TivZ%jf5#6WE`x39K^532;<) z4_}{PjS*@LEcC=nC!+KFu9}sLD8oQ+|rn2YQi_DE3=pl&7ptF|= z4@rXM4poy_Z|75x;}0 z46zD@?ftr`qVjo*@Zd-4BT0SKXsc(MA;L;}FLa>_c8qmp>3JSpn33j3tZ37`Ixhw* zZlTx}B;?R?=A0?L3vkoiW?5GZ-QKwOI4S&|Yq>SO3rpr&S2W5C%S^Cp4&BM#7E-W| z^CDavT~n^JR%C;)PdHu#YYPg}EsdfKqMNsoz~|oL*~@@!-E_se^}rB|(WQo}O|ZEh zDFJ=vNmN>0#>+Z}UPLX)QL4t4)a!OtT;|PhB`gyXI4_1h$QZcvt)XZ>2utrG5Hb+DE2yn9gt&V_8C_%4 zps}^tc#&Kow|LR|gYO{3Ns}$KXuTK-OMMum9E#Ztk+SEb0Y*xsv=d9 z*UpsADJ9LoJ?;14p&aZ`gi%~)%$DX@iT^XE{O-30e3T(CkVD7GV(RIa{sm(K~k z-eZxON(_}-*<9VqLiM?)l!-m*IhfK%-f5AWi_>XAQBr<dTcd8{}Mwl@oPMIU(fL5I(ipy`+_D?#+stQyulm^uHIHmL52K zC{1H5UZo~`nD-rFz)Z#Xi%n@ro;kNcU#G;Y1eJ=uuiaq{ODRq4b1Ca9Dk*Y_Drc_O z#pN{LU8WRdR2Gp>`E;kf?M=bHth|(56xKJk13kp5s}Qo&??g)@8>#Af#X=FJYiXY!W_!}1+?5b0iLfA)R2MeKv_knB`wZ+r+pdpG5DN8&XbrI`Rx)d^`m zS2pLbAyT#6PAl=hccHU-4**yH$(EhUX(E?gZe5wnT(U-&^%&k2muzi}koqoTFnNy`i45<{yON<^n$c`XoPYW@XPY}M zK<<|rj;S-y^IhO2G7{(Kctvk}O7RfQ0%gjD4~=vM{moQCPclH`*lAG|r+MZZ#&&H>$+Crwg_SCe~44 zM^I(0lLgt!>y3U2YAxX!cPb7)#<-2f=6e)8qOG<1wg%l?WS0Z-C+b{r%BZsowPFDC zM2fuWPT7oE^;Wu0+Kj~m>I?qG7qU?b#yGAyfgo@eaNBEo-_o>j(!qTrhx8ev0{EK2 zsL)^VzYtZDFC)w)>xU&1S7y*t6{(xxF!uM?{lXZ66=Is4x)7P zcu!!#d|r195ndzD4~WphbGC9dbkdKdzx6VlayoTKIm6!yEaOG3Py;ICu<^=(RICWd z%RPYM(LuCOf#XSCE_oWLwHr(v*{inE2*^m%z*Fxl*zd{$WPta14ybz0^F{@@lDyHl zz^oL3!>k!>T@;4=k}yax>bJba>(z`))bTZ0!ZB zxL{OjJOBlOMi$T`kFtbq(&7AoQ!&26%=E8_3Z^CC`?tj#QMcv&EGCCjB3Xb#;Bhp$ zy|dLqJSQzwsisex4lX1`*Y(h4ctF{tl#_=THT^jQ86PjlS?=Zyaz+9cGk}m^v8PI^ zI-Km~GgYa5b2VqH;ezH{*yo50F{4Z$<4}y<67tTst0+uhHEyM6z5LJ#YiY{hkpLLB z%5OC}5Ns`Iy;-zc_WB;y@SpYA4;1L_vgl4 zJRW3#fOze5*Z22ZX?;TmpV1!W=@_1KwxvD9P7UhW%IAq{`GWM!X7H4ITAp_2z%!LT zg)pwPt9=>oh`Cm%Yi5a?jZTPw&AQr~A$#)&&3Okpv0(;?D|dH6!V0nNX@LEj7$Q|O zqO%w_CYOkDOvz?Zk+yGi-`f0iiodByIpciT9s0*hDc`*3E&RS85(nwHmM;uZ=*SnE zY(g4860-?`D!N1zrc!*SvxT`2hcQXw5m>`5=CLouWlBKvD>T;UV0ow}%u0YTY}Z*+ z2v^?jO&r9fK+ULU$k5AmdL4s(T78Zl>kqTGD|OBeCz3M{7eXPLqg9jrEkNA0Ih2wV zoA3BD8G13S>Zs!rV(NU#z)R_jBv4zf2JxNZO7Im~Te)S>H|~gTS}HPFpKD)%NkbtS z3b9gohiL9&CG}$Vq$pF*!X9(0XVw-neH|N07O;rc4@XFLxv?1(YV5hupYyo9?MDv+ zdz#*^(mffn2I!f})I}~bUCe!nykER+N;FhEaao#O1;E{u?KlF%TGGFi;FoA^XHToG zVv3?3iCxJ`*<2t5j9s{X9FRYin;jp4qBZ zLmExrEMReJlYy102hsMAPSI7GaQ=mrAPpJtNAA(5m~)$3YlNXFq=5N?Ic0TjU?{4G z8Y{?`j0hSF$Ko{!FUVmaFw-Dne41UuimvdW3r$KxohTtu>o*zzL9}cjw7(SS2^$(~ zYiqZHr?49VNxJ~vuW$PEbUcBX$wW}bL7=@r)S}cXY~$BtTn52|u!8S1>4Otsfws|S zy#~ok*s#cC?;W`sYv@JjNFHYC1fag|YHT^UH7`svX%YGyFGijgYTf$xwTtcN=f+>u zRvaqeZ;$#>a%^6*w(GuVTSLBnL|vF=Zz{Wysq}#v3lLBv%S6VEYjPIYYt`i>Xv;+1 zO(Y`3$GCUQ9cbZTEBp2eKUG;U(?on|5YEqy`qPpyz=>hGF+(sga1`i{VhNMG?J25P zMAyX*kvTd3PFpWA>j#vg?!hr>F7_F{CKVO>^@rs32HC{^$5p38*zE0vWKRqA650CT zeGm`!zIP&;k%w7rLjLWGVQwJ@|I;?Z(%Iy$JDJG>hmxpX-~2_;C@u=C=oLejM1q&4J%SG0UksPDwi)ugb?avt1HMll88|t&co?(s= z^g^|O`Zt8lKLe-+r&{i}^aaogmg;o*UPQdvT*09)D@hDQQ_kTo zA+-?62FczxrVaIE>#YSpH#k2rN}t^Lx6gRmnBeRTwIFlvR=4E=<6j*&oET|t60=#t zybz#Nhg-+q?cTv@{922(zhv>MBD>k^1IeO}!*$&lQSzji6DnMpWJv?q(?-^UtU7S`h zk+P_0;y!33E$|n?7f))ZOarIQzCG?ScTwSPJZ9EkCRA{Wutp0>7W&6KnFw52OfTEjs(9K7QY#llWOK_H_d%+6( zhf;zl_TpnYmA!p#P9N%2S%!E1eRm=_zS|QYwrt2fX_xejYgRkNs#Yb2<{YRYOpiJ+ z5Z0|?YUTjnGgzapz&lvfGgvOrD`~%%xBk4L->lcxuQyCH^O=`;5mngr^awfl26PF{ zHuI{=+qt><-3Qb`1YiXr% zLxomr1?1Ryuxlk*w;!BWnh(?+EX151B*f9L9cEoykr*PlbBCGdqvX_J^G2nCRLR@; zrLge>`;GDVi2Lux@R&UMhVS6iB%Q07I!$fL8Cct25Xv!E{d8i=f zL6AnI{)@wdh5|qvGpbnV@zs5n$}uYTPXh{F0N^*0ZeJ8>N=J=;CRITqlZIv(Y^Y#! z2E_q>UxIlB3U643?rv`)e7Wp{ced7s$H<_nTp=i4JT5?0F1BzovAW}WQe;fnTC~pD zETYg>u)wRVlD+Rpzb4xXy?l>yR8qAV!1x;UpvzgkMxeU5-t5nfCZI}786vWMhU_26 z#qp`^>JNjBks2GA{<-lII7*Qz;68EUV!YT-B}|;tYko_Z9e+1x=SP&__lHFLO>;##m1*l~_$HwFH*8;yw_x2OM$R-K(% zU4;g;z|;<76o6~n4BjQpcU>G@V}e)MWhQ6yWGDuB8Cl2Z#UMx)gP?@-I%pqi(*iOz zs5P2^yf0eAOw=GmwUk*AlWCBIy&1&#$H!4W0pWpY~kMvfzYuN zDWcZm-&l2h1*j>_G|;gXYMJcB7YK>KiesR2@Fs0MgDSkYwmYyH|rSOm3UKT;-DA`2INqpL4jRHE!mbw13HjHAa~4uWfIg5 z0D&3rzLfEQKcoRy>wk3?{I{1(*=(&7(KGnIj8ZB=ycray_&pUx?4{R5U`=!CDbMn&?wwdenMZHK0s5<(XM~QE|SAHqv z7mC=ATgwZZ&^bu(ufMqDngy}mt%pInlD&qV97{2PggYHuq&aL7gs>$M)g#JanryBSP;N8;N^C#^8I$F!1#n80lj{<%eHo0@ zKrm8kCzEEs8$S;?UhujAIF;xl5WTzhr>2J1L_%+rm2$ZnU1Xl@q=ew#>*SV=!v8Uf#`WindH8`{qyTvmuu$AN%q?_ zVqu&1&4Klvjfs<%P8vrv#FJ;JxFMq1$0I50zM^%_<*xKo1F4m1n;1oms}`^O;y@E` z$jLrObj9(YtUOhgnQn56WCQ|^nCds>Ql4h5p3xzMG(OGc#c8wTJ|m zZym7SvRm0gUJ@=$^SfSICpMpF6i`i0x$m}$3>DQAmLe9ZA#=!$vx5>wi^yJ=NuvRFvfNbX7j{suZVRK1Inej0*};k;zd)uYGhj5xb-0(j}Mj@Knp`@F^dT z*r6;7_bb-<5pB}`F*K#eW+i`>1+LH2&~S;|x^d0Wh45JoDJ(_X1K_M{jy#8P2?XW^ z$!&u6TT!}f)*z4$@&`4Ioq9GcpjYD>lrE4TEHFk%r$7SfXx6!f*00wN&$S12-C!SL ze0^NHZ>JmqVojzDx6$Fcxwg8#BuU@C_P3Wl^@Pr9^Msi*0P~;e6&;TNGlmMH5-XqK zQgNO`Gy6kS63%a0y-qoeQn506fQgmu%!RK`Fp_&opvCiOjtceJDHP9g%}kaZqMa>qika=_&=Gf%kB18FHnu#A`z(VkyLH zD|Nx6jrd0&cYNN9U7&>rCd}QcQ@*$2G>{>d;?;a*(jDDfzpc6J4T=*^M_05zY~I2= zXA#A{Xk=alrnjPzlBd6|MbtC16DXIU=PY&JrXN}N%EZC{c51B@bJCU-C7;&9WNZ@jRniZ$dUvgi?=WdDz4ITq*k=H&5>9$MN9Iyb z@<&h?X%&Yc1RXWv#qB*kgic=lMsOUiW)cjJ`NNiOhn@xUO zi^Zp$W8dcCCOBHSuvF_;Ox2XMUN`0Ju>H*H!pQuO%A>!gVEwqdx$g!C?hl)jZ&|tr zVZUN%Wf~ZPnwq*o*YF9Y8(Y_-h3oYXI@Kv%+KCV4@P{P}*%e@t6zHR1kE>9_m6EsNw)-~ys!Nqdyw1Lv0ro6M`v!`tX!T{a0pA-qt-Xr}XyHWTfY?ki zTIh)6XQX&(Q=e_j;g}X-O_AXPV?YKT_h{kiTNve}LmH{6zIR+OPY16oF$IvR)NA;0 z!==00OR*VJ8>xjgRx@(BmYT3pNBha9I9hdTG~vf3l=wa4<9MKcNvU#U58K-2jZH1` z4CT8M`7X>I=^I@UBYSZrn|<%$Ja@7CqIZ6tApT}2hIO{SRJnKX-MJZrNZih~O^y$- zca9zmHK2uWW{TDL4&j-;^4~;K&LnDQ8ta+^S%dfKpWiv)s3WY|y-bP(bvyoA7qrom zfpp;=G3`D#x}op%@AUf&VWY#`S!M?_03(Lq6)*{DY93tF>%Lgxj@%A8Lr~X%ZhlO! z&$|z!xkb4L``igC=$)`K&KjK!()gM1W94N_@loaGJFDb?I^OiyarCm&obIg*5KAa8 zR^nizC2w^s!GC&|+UDAqI=%Pg4k&@gz1YIC+9MMAhO!n>>5mV7diCTq)v9uQr;t>&1JV1XEpQvQGiPo9bBtqWB*&p7n={HO^RmWpUC)2Gax> zcNl9bqv`bp1kk+iG6e8CXXcA*-iAx3PIy_z4u@3CAqRMrlN$Y|$Aj`>$^9*!sbW22<{6NlFi#`PttTO%zq<>_XUd3cO_0~%kxuueaUP_eM$W|Box$Q@o@ir{_sEn0Kt=^8-B({&q7rh zgbrgXAf(wPr86-T9*bhUJv~X&bsIIuP6P9=%!@Uj1;~2Ya%Ux|@Fn_4JHIS*x;z-^TrQ}Awx1Ctb5ONIfS5-mOJ+% zO@Kh~vPO&jFQYgR5*4_%7CbcIiB$K0zvT3!Q&Q@qc4LncWQ|CSMrLxx+;_w2?Nl%N>AOlC3pxz%z=0GL zKcXH;Wmi?pyYFmHoYyN_Dy=0X;F&kQK23f%MEed}T9Pf4)~?LgH8l`}E@;#dr)0bZ z#58jiQ5iPR615D9q+WD;;pbmgEPKB{dFi%xz*u*4l(T>cM;b-1mVVwwUJ!e)Q*$vIDSsX?J43O(pN1`e!L>sfFT zF0zlD>KsH%%U;qJXFF%oC@ncw{;0LU!m=JDQ|iBI>AYLi4v3ZXLdFMZWeWk#SI~~+ z=^|Xz_d4qkETT^TMb5o`c@sT1LT9F@6KBwHh7Ss1PBvkLdqV)`bm{jA{k{lUFZD$9wJP&ISB+V0^@__`40bfnd_+ZftUNnB6B^ z#yb-AqIm~VS!0$5qlk~D9vWEsg^0CEZ>JOrs#WCY57wL(8jkK-`0fZYziU-^j8G@j zDXHM%Jav$wd0uaT04T*1;9UsL=ugy+vr5T!>y;alZxr#Nvga0og)Uu7y(Nd&JMCX6 z9lrEJN2j0D5QeD!W^~cV5H3J`G7}qngV5hoyjmSlkj_0cva1Xd1Xus=cPua=)MEKd zMC2RG0sR=AnEpT(c6IjC2du;%B7+#*Dqm~4bFPVAvy8OrFY$L5y>n7vFE`dbKV7!T zA0~5Kf+9NKpqrgxqZ^#tui_I1C#@A8a)B&Rs44TH;yqG3xd_Z7wi8X$^+08%|4XpF-dIY`i=VmnC~B`>z}{< zpF&;#A6)bQHyG|uBmV>UJq}vI?rr-eNv9CqWE^>Ce^0P*4}=3)_QQVz0oNG0TXqkz z+JNE}|0`JBe;EaS(?B<$1c*?(UB`Rd%+nLyCL8Fy#Uj~Qvz0SJwpt? z?%qtMtU&?e>FCoKZ+k_strnzOs&xn-{7)1QfNe3~#CQ?wAWh~H{^Oyg4~JRrG$Jl6 zVQq)v2l#d@!+&lJ&@c23)r?G_I~pOAe>tMVe`lh>PbEF&1LuUYwZlF|$`T?m!wI0T z_3NOL;db(y8C|xfH$Qte;LQpr|Vu($Eb=p&rZ4Imv|3X@BfT}kmnG6uNhW>hf z@gkF;*fKHNmfGWXjaItaq6&;r5|R>7>7T++$U#6A@Go?>2rgXqLY@%k6)XOxSkvX5 z*bmKKI3c8=Fy$GOUi1VxBFXHd0%EL%@INo4FjE&QLfaTGIcAhDm9W6@#{M~Av#2J% z|Kau19|*FJ{XsqVR3-A*lb*(h8Q4FdVb4CONefTzumBBZ5g@OrxGsI960A+f|MTN? zO%-M0NDB^@UARlRE(rGLF)twZN6=y>(Xp>9?F)b**7~0gp@5{LVhn?kR}Mb8tI@?W z0Na+lYwQcJs(&uv5p#dkxw(iSdb5p8~J{=Sx#w{wTIOn=>J~oq~3M#OawR zQ5_cZ*TOrFI4~eXs%H}BM+6sPrqAPvkLI>u5`mJfw+sNW`Cp{=zYOXBIYjz@?h=1l zz^3-y++4vIhM|UW09T`cw6ll7KZq(Xl|5)-c?Z$#;^G|%7`T(q{%LweZk8@})tX!n z9l+>~GzUoBK0>@7U2_9z$!~x!40GhyP@##*0W;SO<&gSqFlWRla2PEX7UZ`H0CFUt z^cDUO@HJF}wau`?P)Uw{JTZhTFP$3iRxbnjZm4yY6fa_zRuh0de{jR)OS0WZMwx!v zA~OhjBoHf`^eKVK=bW)26B@ew^K124(F~;I?s8J|c7=6RnTQqgTiF$m`1~P?4A|Zv z#f5>;_IgKflbcBbjKI{kzRAQ?xhtf?>lm^Yy4f^aIG(qN#)go9*+uu?3zjO$@eh>6 z{T@KH_nyU&EEppsyh@L;A}_Fahe(+gYHr)VC*5<`PZX+jjp!5-T<^=2#J(lQ;H8S9 z!=z@TGnb`7v1qwinYeRk6k2TSBIU2KWa93G<|U9E{40Fhy4&OfQ!B|m(u3vk(4?)-;XTXnQ1PFlMbwhd1dWj_AGsZ*g19T9I)I6*ka^>Y`T>G?a95AB^ie8p zB}`lVs?j1(>Y2OdDZ(9}8<{%1SOFqfQKwqYyJ;dn`Cw`HiJ4<#X^9m1D^S4b#`qWI z$~kihT_eD=QN0|LDHp?+)qFlg~3TELm^da+y~hHTwAOU<>>&;Z)v z_SIQrvE`avYZ+6wr0eCBA4TcVMphx5Ga^UoUMV2Sq_O-Mitf0fih>qm@?4GITYX5k zM5O@po|(_c?{{k4+PeJv29;PVW8_clOaQ4Dn5%$bP9N2bp_7Pcs(Y&E)oi7DW+^4G zIMjWq=>;i0$4hgp*~P;oDFZUMrWO;}9Un&|L9ab%Y`StAB;CQ~WnkXBQ9={!ZO(B} zw?ObKQ;FY4ZSZ4@_<#9ve?bMV`};?I;CXjvU)%x4jLmO^3HVQ*0-6u%P416?h*+SI z20@^D9nL5LN++PXZY>Yw=_X;D!!nBULUaNBen|=DJBSAUJ7EbVAT0Pwh5 z2?eY^6M^W!`T8j(AM3SP7v~18{toCa*ci8EpMk8M50-6*r4$iLEZ0(+L7K-LF5oke`m&w(yZ~TOBU9JRht>X0nY1w~3O7;uyM+oC|(^)cS*SYKB4VeTxFYClmRZ zzXzCy8sel7Gz;CCfcNh!F-Wg$;&!UNG6d!EF+nMr=qkMm6z%+dqFviLJ593Ki@Osb zYVM`lX}>F<=e56z z9Jzc#Gv$BV@}}*zz%vX!l)SsKEq2P%Gk3&`^cdzuJ>7Y4S=ePU(6|J{ohD#^eL2&n zIh%F?hs_-mt(`WS9E_PFy}tL3{03l$mg7Xym6&S#!xvlQidL26pDLa*F}m+ew3I9G ztil_wr?0F#E4s9J|F3=h#ft^Jw=ODOx8TmTiH^zd_f6Qlz~c3~{_XG9|M~O$7Wi~k zh8?WHC6I0pg432Q`Np(;DRBG!t9+FfU=hDPa>^2&o?WM_HLe9OUj9mQrq~|vApYK~ z-#vP^U1yAKY5ExX)ZQK1e*E!fu%|S8-D72 z$!lJ(d+&Oqt-knciier$)L+|x699oTV}IoUEBJZe?ntvrCMLD5|GMSH&Z<`d<*y{b zZ4Ub};CZ9=hDv(k?=#%Lp1u2BU;OEeS#fiKgGk46 zQ*3v)WDDvXtNwiBk;QT`g~(OveTKi6i=TRWNd3F!?TuSboVrqLy14$;zplqGYs-|s zrbO<&-}`6pjiqxT#YoY9;69Q(un!KrBC6<{0p21949vseE%rc$#5BhCvuV3`pPdQ1 o+9X03+>ZqX_s|Co_s{%i{u9~asea+nEs%FSUHx3vIVCg!0E8$29smFU literal 0 HcmV?d00001 diff --git a/docs/img/typhoon-digitalocean-load-balancing.png b/docs/img/typhoon-digitalocean-load-balancing.png new file mode 100644 index 0000000000000000000000000000000000000000..0fbd9f7e0d39d96c0094b559bd6139b777f18fe8 GIT binary patch literal 50263 zcmd43WmHye6fOGFEvO(!D@rOLCEW-p-6|<9p>#I_f(QyoODOe%bV#EJ2uMhGm(mSV zXYKEt^XH6l#<=&-9rqhU2IA)3&wie@=2~mcxxd$HD)JpNd{$QFd-90mG_@#$|Kp~v??Qb;5#P{B8P(BxtC-QWWHY+&`|7~p&A*c)w|fmne2DoDCH$`3ouy%~sK~q3E4q)Qa(UV3TV=^~>aLU1 zEM3|*+G8P=CGD>VI*Hv$}Pu!rY}B9y+1~N5t;loYe%}{v=YQ zjoisEJ0Pf-NW(Mz{_`$JutjCXok-z>?fS{TLQIEG)zTLq-BRoOu%Z@Y{#eCnnq@B4 z!|QJU1A`5Lu@cAkS~SeTsvD0p{t_hlpA|BRC%4z-|0%5W4^WDph%UA1bQu+2FL3!7 zL?0<_u`XoCMTx)CQ~l(ln3{6H&HChJLj!rCSFG9A?;7Uzy*YI(goF2}oXRv_kpDE0 zJsGLu`rChS*R9(kY@uY|IdH60@e#90@+%v|FXPHDB<)g7x4&?I8?0R17;6(#7Uddn z6$my-m=f<4e1aCZD{O7}>a$hj`;P1TC34pIasncrxWyDd@Q(k?;6EQxTW7JXP%ppn z)$QO*->(VhFxwHMzjaubIt`ergio~JZ(2#N)^?S>)~FivK#$$foy`BsTe+{l%4!<0 zIBJnJ?t)20jZYXzel$5XWEVg2cHTU%gU&atQ%P<@l@j}J@u;-RSmoowaxFT#iB+x5 zH)~S5)YrY0I&Tfx-(8=w9e%XwF>O6DCC=-qX6u~S_ZHQn#hSBsO}13+>+&k*mCf!d zU0GXwu`X{yl<4E{=F!vA|n6q3Psp5KRb$BU|y8X z{PGQK_ah%&^~4b}I^OmbFMFT2c8{w>Ei4ybp%z43q6jW~3tUGv1cf|Zr;W44F&L_F zjT@&JxOJ81Wa>wmn0%Z>p3*@>J9HeLRDl=`5nLyu7dO{kQ)J1m)t8 zGs!-ExG!@}B$=S=8&_)dh++$giQE(ZF!a3Ld(HvdwWZGfk%gCLdM39%MEXmA%VmPyjPTt(nWU?{sut}s3 zKYH%?XmdYqStuyP>1*!nC(@nTFShBAG{`T1J`;HuAc#$63vt^our^I z&;1Wg7nW8dGeBIw6vc-@>dA6S!evBO`RGc)hk1o1<|_&(WOzJgAqPbfJ7!jFQrMXj zIZwXa59Q?QR$oT3rd23&v3$me*1y91x;vh<-qwcWMEI0pfyU={uMk0)!pDuLY;OCG zweq{4ldU*+9?lszh+hroxas~&t{IauAuSTyz1Fp7Xs%vK&*>dc#xVWA^H;68myJJb z_LYpK8)ZGRG|GDN3IEJ>;qpDsE15|$a>dqG-`RceWIZ)INFupAW_kX(OtimkxYm+b zJgRlM?1b?Ebw`P%93?SReUs;r_i)vYdAt6gTgHcw9gVAULRUYZbm^Ve4e$<0IgBWX zjJVsR|9kvXORAMPp`SkkE2-wH*+w{7Pqe3~4E~EYJ(Ca=gP67df+y#b2P65LR@hz| zkqz(2l7rh@A4$8D8OsaLF)GC~zuOj#`fkVai>=(tRNncs&yPzR$~bZFtP}$2O*EfB zd4?62MnNZ^o5t-)Bdz9gMQSvWNX>CiR?$L`<_+#5yLv{&Zi2I-tyv~c@OBpIt96Rh ze`{(!!na)iewff}`E*LF`*D(^w(TCqQVmHZ7PW_WL2&iL*FZHBg3FP`xB){BluLuRolBF?Yov4IV;N}N!obP zX?g0a_}fNu?{Rt1L1o%FVMpLB=4xflkn#c5Jb6OumBh>iJuG=F>eqD(CDzoBRKzBw zg)MY#w(Cus2UcoUu)p++s!)23IM_0Xztg-!ALPUr;;UQp$_tY=On4HD2$!QBx87Qn zd417EGX?eCz2q{jhD1}2yNHN4Xa7uWvu(fJ15T6c)`que6d64R=IB{upQyyGC0XPD zPQf{pF-mWV5x6s-shE+fpxQ_gpbU1ZR1X;YGlSY`;pZjlzt?X~Rd*cz~ zKkX35*{MiP=k%m!DGXF>UhB=}9Kcs&40+hMqG5KQL!PQI#rY-bA)a#!$;(QUIrXco zuODJo9&IumX0{3WJKV8lA)TJpaBe(Krd7CoMsu!1f3~vQ6bMZvwv)7?7C!vvjc*o) zLNTHgWu>&-Q`RQkyvbeuHEs9&vQ>O_YmXV9Fem*KTa?^dKsJe1LFiHmBQ;U@I`$$q zHa1SDVoWw`xE2mM**-QlMOIiQ9|pnQXtFCB_iM{|8zn3(%3k}YD-iaa?2qY|mT5NB zws(u}deh+F2qu&M|K}I|F!B?J>{UW!g<-o&Y&&Hun=D=%cTVb2g)71xY#;J1i>Ec| zR-IQrt~Vf0;ees;=NH$Qq8UOHzqQ}f^9wM49vS8>b_Y+=p3XNm(7P-47mX|Z4A`{35_nr&Xy@JWd5@vm= zyoFt}>D+pRv>1~bizE6^Fok5Izl1GaYM_7HsJuD`W5*P;RS`{jeCQ?@sh%AdX-M1+ z;|u!oC`$7SzUG5XQMXE`A-5?_GLEhOh&5e#WZNl4hc}BHrxuKH?&5CUi6)@ML}SGG zqnmu1WEro2W%!CGi2DY!&;LO0@zI^Oi31-1HoQw4pcIxE%rId0+xfMi}BYp zPcLJ$!@LK(3<2koSTpJM23OOu z)bQMij`}#=-2GXQJtmS>{FyE!DoNN&w1R&J>--X5nNbu@VOz&bZ4CBjI5vJ*8#r?T z_nK_Ijz2XG>Pe}Paaf@-1~3g#zBq=hgu!Faw39h8ZU)@Y`=*bV&dGgcg!tEwj*04$ zz@G}ncXT9=iVeD1iZLr+tWEQ$U+JacSnO{}Cb$k#q_a!>I=6-dBXzfSG`$>Oo11chYLVe=$){fY& zZ-gmh7;zTSCP+98V7kbp+OP&-Cz|%Q_FT&z{H>*p9dy?EE`u&^5uaGd4d? zdWIA074W5LwMkc!rhu`ar2f>Y!2cd8holo)=@k-G0c?ub`R?l&S5U4{R{pYP(jj1# zm#RqXzxJzqIA2+6={eI&k>V??s!}o+_Ils?O$vyUFwI8CMXqx+p7pNV)ip8B2+a@{ zp!;$40w#x>Tu&;b4jf8Ztxhj+bgpf%7cg=$aO@tMkGMuX!-;|I5&LtC5l%yY=AnlF z2h^{Y=V>lnI`aPfO?G;A-!|F^zkBNij2Y-9mYn!A_~YmezFS-$-fSau&Bk&w)m9z) zK^S}q6>n_*P1qPmd$NSbCdi*eFR@myKxv#tFKy7Ome^VkV`w@+zxJ2zGaO@;Wt^)V zox>#g9j^CuTx~zq8wf;O#XrViA4v-`>;5k)QVve?PZv}E*6rv$&aY_VEI>Cumc;JG zOu(c4h7Qpx2}G}!1eRSqt1vpDKMJj1r#x`7#3#ZT38wh$()K+6;I_`7SzW!osE(RI zpJC(8V$3j%bMtspLYQ>tiCta9;&cZYox49-mdZOxXdpJA;BVNJ0}96DQ0pvgv6bf<2+PFkw6{kqW|?=`--X&p*-lkr!0k1PBq z_3Bwu7gdB(cveitubZsmgvYKcS>6*IUq|kLgOimfuo3y@;hXmfQdbe!{F)y3*)Rq9 z4*B{2w_juxGs)dBx~ojG^L#}~NCYU{)H`C85BUex$;-ZW!eaaa39VWT!gy}tQ?(fU zuhn%Y5-|r^BxPr^hL2jb|9-?%h83Tc%|Mip``jTuUrSTDe=#!qVfD}Sq<0MtXZ{b` zl$GS9c&w$vG~_UpX0FWeF#5+SU$yK(i8kRfCI@?MpL1NX8mwvwBUgV%)Y?L7UDGOHuNJ>|@1Ck| zzK-&+i-Be=5p$@6N<#K)G}_bAv6SWh{X-RFj4SD9CX`D4A9i-1;~4wZMA^qzfBIQ6o(?U|qpM1(n{kPcJA6~RcoX?&g%}1s7Xk|`F6xtPOxA2PT2@{@ znLa8fAwrKmF`G?Wb6I-yQtt!S*1$l^irel9GT5uwl zjun^{Bi^Nu`?lz~=pjmvqqdedN1T}b+Jwp~DVyM`AFnEvK&3&u1`R^&-dXaD%g%J6Z*k#WIPRJ}R`loG*#rrnN8Y(LG|U zyiw7r4m1BHVrehFafZ73(pit49YRl?hCCtdq;Pf@N6SZYH#!+U_ml?d=raeCWj!s1 zC4am>F2B zBGI;5T57(-4_l5yABCsfAG@+*)w0MwAK&cslbcve`!)}c24-2K+UR){AJLNlYTR5$ z)4et}kqUC)fg8Yt9$wG?&+(O z^C=UZZ8%m-2Skmpi<_ge(d_h5%b$&MW9*y++~nnkMFw4c8`BmR7JAzyH=qfdn-CoI zI5q2Ojt`v{Ok3N}JP~wVqw?_ZND*`hoNtQ?9TFPkpLagFa(ftuE?r2fG|FM6H$%E}wn{TXFyf=rLO8f*~v!-<1eg-3+Woq|8Znn z^Z1>6)B7HoXlQ6~E5{hry56(wro{|E71iw!+6)!UzLTfVcv@wzlp^>}{nG;sa`Zp! z1OYqh{%mE_-D?Cno~jpoOKS_sPi-nXI2be=c@f?(P-trd;jKkXcw>rX-ws z|Ni~Y?(PTL+JWB~g@lB#F)`jUNW9Ht#SW6<;o{dYB9CIA5kIOTXp^PDUM7Nr-%*@PY4o_eq zfEKTo>LwcfB&@Bizv^=;^)GDAHk+B7ze`Gb=;+9Uvad=u@;i%zb@o<^a&eLN^C>=h zB%@C5=;S2yat3vLe7rpW>9U_jVp_zm5+rGJ2+5zRdXabU-Z}s2do})|F%>pq`*gR* zsbBd%d}zmRk3`Rp*H^ba>u=q<72F_r`SN9b$M)NN_7e#Vey8ssT2MNJxh7P(=#GQ+ z@zEL=4qnS{GdQljKmDQM6ddo<(xQe}7bEXa8E;Hf4}5tVKieEyX5M-2+8sTAH2TMn zAKVUevaoYeVh2>UuItkMe1<;z+J6W0+df5!KeVu5uCA^|0*%w4hTFIab+9(tY*#h; zU~{ToQAz2uVZA4O1Q{KjvaBq2kJuqC-X&T_F0LrJGmr17`^}p-2`5VkZpgCDPaM}2 z2l6r5j#i7QaffAF+m4pLx*-!nhL8QScJkLpvE(2;(j9p4`E7T6+#q5);nsjD2+vkZlN6QL4y=e|GjQf7tv_CxyP{E?lVou9vc5l@ylZ%ib{#*!m{@9wtPTlsA} zR{r`$!!d5rFij1X$5Nstq*jJzj>?3`a_XI@m2UvrLo|MMpD%SKxA70_%fPhBg;u$_C z2W!)fX?dDCJN>E>B`;^C>3#OpLoU&Un^3KdmWAdQH6&Sn^ayd7Z@o-O*%r&D85R-2 z1VQLuHRTxr56J`BSL3o8(zDF*k<&qLE68ug}zf1tiekI%sT3fTNGvxYaOlC0rIx^B6(%pNz zg%bG}#Te#$UI(@UFQ$~9KW8r~DS309H)OFhzRc&uHH3tw6@r>gw~*E!+wyjTS+8w) zrZLq6R;SpQ7#S6nOG(+YyT@DZqQumEVQ{=WW}?HZ3x zfnGq*@_hZ_l*yRQA*S?t7cpM6E?J|B5suhRCq+qBPp2M33Yp1Cz~ zQSLL3B!Z5tY2S2=z651_2XQvN06n1@3PyT89B+aGyn~cgtfcJM+17wH(SSaom*`lb zOu%N!47-F7U;iF`Lx!6;2tXkwApruP&~B_82~1eTrw5S`&9eRFY9jaIJLRSue2`<9 zZ;eEvJ1h(zl3!k5pLTWV3lk?N=fKjh)%{Y7E@}W%cL;ENY)mb**6WlM+M7&F2zVuB z-veN9-I!2}q8I!67oG2Ze!QGz@PvQ`gIcHAvuYf{1~mG8YHC-(<1(x11~GMYb=Gic zLqmEouLHCFzk_e+#gfq4`B(6%k%wmE@-SF~41G zJIdqe=m6EI)ecTx4lwq3 zHk9EA{-BQjBzVIvmv_aghSanWo(_&ZjQPUmihi!C}19j%_``7|hDa;&0kC z9q=f?%O9)hf$^&|!DmG$Irg_wE9>E40l|yy#s#SPteOB8asdY7^&JT$PPed>llW!< zyN=F$q18xznyO;O8BdhfLy5L?ct8xL6G?5C)gg=_m%l&XF+p!l`S(>3nms&|?CRBK ze+(=@%v|)t$r_i$d!`RED!XG?RGEc@5_3L!hI?(+#~)7n83~8R8DB}qFf58k3e?5< zVWX^^T#4~FOo;q8-X2jIPft%PT0S{-DjqZZuH^e|tNT|N7F_=C8qLFu_9^iGvBy@A-G%1*33ttjUm%>Xr@x<=TB(o z;2|2a5&!PxE3)O77^w(s>XAmg5@8~JTM7F%^ua0T&705O_blg=SFVpT^cK2oyNe=` zuWRHJ2M5jTwxMt6cjf{}*_v?A6!6>xoQWSAb-fb~&ICuw}e^ zH}a`{HIAJ65meQqf2&1`3JU2SyGs**KN+Uby>1;Zr#W@;539i>q?Y4PC*m3%8~a1~ z{$wzeYe2=lJ!kv6?FZy?-7$jYX_uY_S2 zIUA0ic^<3*MtSf3me+R~)E+=<6Zu1K1Kb<+LS1hn9CKY8`3AGk_Q@F%90Pfp%jHHx z1(MO@@w798tN9KdI9wr@`TAWVYjtV4C9aRXh&k{7ORg0^&gWcx`>J$`s){9wbjwdq zJ4IMW#OAJTeOkb+s5ait%|kZ@C8au>j##z_jAUx5G-G9pDZGvU{&I-_TVXoij`HiR z^Ke296&dwFvtl(~d3V^zPi%B_)OM;a5nAdibaY)%a>Ambmft_(G=YBfM_Zd_wd0~% z8?Qg~Dm*WyM4`kf3>O-(0q(927v%tMe9p;6PIkql}$2k{qWWQ0@}Myab>(CaB19t=30sX?6VF zr$a;uFZ|B^6j?~r3HeiNev_zK{_SorvL#K(w*HIJEXomLk`S^z9vRL_gd5aY+dV2g zTGC_jSe=Dqp5fh};VxGqUw%LFC8Bk~7E9b(PSvi$uM>6XuNNEA_!tF0&<)P?5~!;I z_L3FF`^>kQnUmmCeH$-SQ)wbTnvyghb(Qv~hW0EaIWP}r@s4iv=;t^a+*j`B%TZ0U zue6^W<5TV@#>TW@B;!`@-*a0V`Hk&4mgeN{f2TW8%R$4vhIv`#yLCB1@ZY`E61x;( zA4@ueVHrB&@0QnJ2EW+kCjQiGT%J9-#8_97TV|x(FMZ;SmQI})Ty%JQx#6Rek|oeu z=6u9VT5q>1FZ<10)xw-EQnn)8_%yutl1{%-@*+s&igcKS^f$uR4pSWLlYpucVS-ax zsajtD+~eKl<-@hn-4qui;>JsY%NolQYkUR{V;_W8C1(R#TnxCJFS+{OCuSP>0YhK! zv8Z**(N0xYdYAB28p{ zBxPQ3Jjqp6-P%oO2N}Mu>l}VeOjOj;8h8BCXpC`ibYuruH8ya^Hi-S>fefUkWOSi}073b01%nC-i zs*`%957vhRqN81Vd4^7b+--@suE@4KNu9TR&%c&RzBL~u(G6w53Z|nB)a|-|_fAz> zuU{}jufV6_>~7Je$d-0zgn*%S4l3@_PAAtg6x~*ML}pFW#(csw*+Z>>{=5$thSe#i zLVvcl*oXSpb%9P~jSKW&Yol&gWo~Y+*M%;=vqhf4N7mO@ylUE4BtX)iRN^#xFke@*(bs3AW|b4J zGP@RVWh3ykQ!pTrm+Wp6GyzIFI@i<$X($Xs)84(4VkE<_n5GtXW|cTIn=G(SY6X@} z(EI3RhM#+md^Ya|wO(S?RTh=tCF}88H!c(~E{)SYa1CJvgA;C-m_hO9Rbjv$GYg9l z3!#@9v&i@6=4Z@Vg+ONLb=LrQur^+$T52U7*7vuLWOqJ=sWYCd2YO2tLqkKWr4JEY z4GF=)*g9XH$W_@-j#SuC(g`~oL)o72T>SzcyYmf?q1gM_v9SK&30mh%M<-X~uaOc{ z8M|l-x!I%bd89foEH1u_i)(@Is=cd=grseIuU~Z+{)849amGg=<0fF&_LSEq#?X-^ zSN-0V`;*RB(|rF~Tm8(*Yn@#-BqHn3p45d3yT5pm8IgvjGgti+ zE41%XqC5DHKHM<@Y7A)xknXRpPE;jD@LR>O;UnlC5+wY@SQgY&%QZXMji;`7OFr+M$e8A1@^x_e+NupwyWD8 zdg={rBeYh@0(S0eC2a^hA^(>41`p5sDDi(`EFawiq5CX#Tzq(qQ(plh0%nzEsBO16 zIaLDrZv+bxEL+scMO-Vl`WQy#e>9 zy?ObrO2P)L^kosDkcQpw;+&XPF#71oom|djd#>e2Yb%fQiki1L0|mPlzILQa5+6ZY zk32GstE%Qah@^(bgxBI1mFM2d<&GcxR=xg@%Pf9Fr*es2^d}67GQ5*NkLwp|Uh-JE zq6MLK2&MN7gN#A?ETo)2dGaI|{Q`C^)@`@aAd@)>jlkBV+w}HNrYN;C3uP31bSH$>bv`RH2+NtK04!Nq z*)5o2x5sRXerL$wZ8To^w!XP8qoHwyt8NPuDT`G1?P(!=Zu9b%^^QBLkint+M;O5j zgK!71vQpEwOCP;=6wl8#{dgT06p*eBImrMlJS2i(44XB&_(#bk{}uoi*_9_%_FWUz zPNVf+_Yn$8trw-k{1QsXv?KM5rKM#|TAHe>t3bB3BQ#Smp#lSdC21uPfPsapmLZJ; zq#KM%r$Cc?1Ms||6?%_LLQTfNz(D=j82$n2<`CcjE|1r=l~|us*Xq&|L@)-<`tQ~# zh8_qDrHk`38GbHkGIK1tl8}RiZZQOxlQeBgEmqWABFF-dBE-ZWjWBq<5^GuUh8@rn zhgv6P&NIrqyMNoeCkGg*+9#^9+m2Vr8m^eYkRAIoh=ulNJ_lz8k~K)()5 zWHvmZzC#NIfJzCBT77+edKJ$p8c%nKfTIIK?+L?4O>>TE(;LA@BspR)jyy&^j=USx zinXdaCV#PdxM1ppW0XVkLs%hc{dL@{Q zeAYGD)_d#Y@jx2@fPoDjR|Dosm^NY5$oTVFssG3F?yIqC z7=0sb3adj@CC*|I?hnwTzu2T%T1-6EP+C=V8yjXmk_xU|gIWRL8#AK5Cz7hDlLLX8 zZH@4uor}QM0fSNRb(jskr}OGiBsKqYRC;?nl+$j7BdR3GpO)4Ao$WRT;00*lV1R&p zLv#Nv#ONnhLSQFMd()*5UYErENDY2>ARXvOtwQ}oU^Wm6aOlfZSTRiPAzTedfdKWd zDUH0}J>skm2ATl!wHauw(FPxp&BhCH0s;bk-{VCjB;lEu2iOW9#!kF2AO(sFOaF1nK=ZQ%5-b$9`TbqP{S>O3no$nX4Y6q=bbg4;>(%?r~ zkN6`16C`~Rnh*$^H*_K&I10aJ0`iyoKo;D-ckdk%g1|a))~wuwb^Zv%A%cg%h21F7 zuYM29_+4t=*&AO9OU-a>0K%bew_5~3YNT=jQlrv4DE07fsI&{E4WYFoLK%Do2e2Ou zyC5SiRPUc6n{~}Vb$(s^@RW+%7zLbAxy|s~iHQldYaHGMJj(sR8^K++|K=AVL@GjW zB6K9oejz4M1yr@P$gyyVXI}W7(AZar@ZsCRaUcY&uy7J26+%W06&iHHW_S4{77q1x}FEguR=UPrp4)y7d#Qza%+6kB_`Ra>nQPrlfW!2&jygZg~uUm*01* z*3Y>2QlE49#ggtLWo2{7AYhzA`}zSqfS9;d{!8(lh*P7`B4Juum&Zok#)R^#9y%5# zRiYa|5@A#kPH`%9nsO;5RdKr3&*yijziPXwFB5)=lg9VVQE>QDPGm_xA8vHs$A_

      aXa?#XTkjA1fHVw6xDtg{UBUg{-Tb@^A*jos-|9%((Du<-%%bk;Jo7I z>OiT6f8&S8ykoV3va)u=@e<8lF0P(GKho*MJjn??JHsUuF!>x>wNToFp}s!0p-bCl z)KUh6qK0KDop$=~M62Z=y?bpQ6OT#%M43m3yGsq z$|-F9TG}lXbYA+(BHS}eVN6e>SSfAJ2k0I=sCM5C5lLSB(nxe|Pb;v+Kn?LiRHs`f zamuMTB;9{uxVb&wmd_dA>0CRx1a}w&VRCt7^?J@^gkTmCvt$g}c@M9$no_LjWZZ>g z<5S0^sjA=g*3X}&&Nj#OSlwYLAbN`*_E&sfaZ^fFeZ^cbyib@F$um z_6KN+hGC`(hw-3yHF*?o*IOQ|a}#9FxOs4BNk%GQS`ocro%+u_p@j@8W016ziD_vU z$$w&g_}+^*IzVXTcBuUw&~Yq*%u9pl(d9 z?R=b_YUUFZ6ZOybS>&@d_EP&J(Rb+jkIybdNAIPoXGLrzYu@3ii7fjy`k;bg*8I+2 zv$O!@cCv{srHq^2bedvaHvio{N1OIKA4|j_nF5B!`l4ZaNPU4283rw$sHBH++8OU9 zg=RwcEB-P~Iy3Ai72Y$9TaQLck}!Wa*%v4csh=d!rx(^owMy3ccvsbL1|2w{N89-P z!(9{ou&GhnD28a_HDVD9%wFO!Y~RW#y7P1G^ah_dw1k1egW@W^hWAB@@Ng1EEX9Jp zkIGnnRAOP&`aUEw%7bi8EcdcV{ya7<^OgV^%C|Uv7}bh7jdH_?$5J)MG1OjPPa2B~ z^&gVFU2vF=wTZR3b~b^fPh3zDwU4PYGenY0Y+D}nxSz94R}!zFX>0nR;nUigK{$og zq~!~vZr^omZb}aKeB4`O=I7+OSW2ygO=uM9$hd9#4DTW4Zosxya_9>z0rO#NpSp943htF`#ZHh6u{6_BaHg{|X)_e08<4}dgxj1zk6dE*mt($iR zh!!;6rQr}fLAjGyg~l1!*lwOYIB|(TYe<=IdxrDYWqlvRz9g#Rw{OoOEc~1Jd5&H? z9kssKfyQk96nsalKZ$XgKZNQYf@w^6r)26W83)#fDDATqjHHn^TT<+v>qV6vd5*yV@&jiz@uNe_N0-o_!QFg3 zbJa^iC4H$*l~jl9kW)v}g+xbZ9k+mem|ySPt%Di~%qY}JfJkv*;S;@YhL$HzKU+H{ z?&1ny+%4v@TrWYJ+z+Y0pSMAdghTWkac^ByXOq{#?nAvqCN7 z4&)v5<+54F)6*_uf>#&la$Yw3u}F@;;Bp$@Y+>ss@bdUj|I4QF&%$j-?{sPns zmg4LA_4+q;;+6?#gwjQH;{ITJjqZ|1Gb+QLvfcgp$or#3py02B9R;42T4wj+Z4W(@L6hOU}q zUjry;;gESdr_$^Im;ULz&WG4BMU;31EI5o$84oC*Hgvm4sxz|M$^6%`@Mc z{hrSzwY*xb$K-~bn~!NP%+jR13O>dU!UMXEcdC9kE_nIge2^{08rz2IeOOEZxSpVO z=|knOlsZk;ZYjMzwu~3*9K6}nW8q~v-iWnZER0e2TKo1>=ZEMmQ=y_)T$}Id?}9sC2S9 z>1FK8+T&nDFa`qY$XlFG`B^!yc@lHB^_**M*FDo#)@kou<&?Ac0|RGL=1h__7HICe zua9noH*d`{hu@4;e??}=;cfVAkWX1#JysG1jz}oN|1jL{efYvhgphsojC!!T^$uzO zY;Ng$+{)Dtjf@v6mGHM&f{l2H<o@Gdf^?MWSU~{? zdUic20#+`Q_HOGi(k6*pJ*deH;p6bQ*+JMCN~bpAWgUJpE*VUxF{ z3|izOx-VXeM&Q@;7Xq|{KR&yTO?7;s(XrkCe&OLzO+kJ&wvWa&oYy#MMq?(L!SHD5 z=*Hbr{XZuX*QCiQmkEET!=!Kb(S44fw5$wl7BJs~=t0n}q-lSw+R9A~W2IiH8G+4k zAx?D9UoN$OoVH94sJJnmzFM<0k@I?fKS)_Q{bd{xzUW+_V4zR+2JI0-y>;xEIr;lN zKWKno)?hRL0$)m?@p4^lJyZ}2NUSy3)UOfh7I8JGFNa8hF_HWfvcLjwwwcwBhPJ1*u1AnJTQ0w;6=lP6WRHs3io<~^dtvQG~#Z7utIAs1&)6VDS zv}F7ms6k;wl6geEjJEcdz_wZ1MbNwO;x3YKH}0PPm+}w6UHN}}lb|t*`$Z6(xj;^C zJH-}zWToVBs#M+aR%`5!ya7(j@J#Zt;p1!P;($6SMl7}2VML+WR+f2Iy$jg-h#~ZQ z$_!xGR{EMVhMFQ3N==1O8h?9Oc+zFYd3k2KHsAxzT3^x8Cw~oQHn!JbNGLOD#y!}a zHey{7_CDeUkHD+Id{70z4g&H!ICwxF(5rD~1Cl1^bO#hxV1X4>RY}d)zW=FtJH6S) zO$_Ne>;lGt07=I3LK7-b4KM>FS;L%%9EP}RK9<2nEL6m$X6;ul(F*#*bD7r^c7FZ} zs}c$PX1p@-bT4I9)Y*Wvej{Zy)ydi3p3}7LtvTWvbSf_MK}@i*y+YGMQ6eV}zQW74 z>cWctuGgFma=_szdbZb(C>d?^UT+XXMsp}B7GioBE4OZb$0Uye=L$mSdV}K&w3@y~ zKfl(ube%xy9)mOc8!(7OIL6?Y>gPk;Dxf7l11A-0_)fC_&qc@CZFlK_^#8cP|K@vb zZOAa7ykIWm>jJ0bKe=Hnn}3(0+JdsJ5g!KR#Z@_V=}vvl_=z|$hi~Gi5w;60V`e0K zYHQ0ZYsPn?e`%24c78kLl6w2mhiIT&gz-eiMNU9AdDT5Xd*U?Yba3|}!*OToD`W1T zzdy-)%mmxqcmN+r%^iZ$OR9WF>5iVgOG`DL8n*Nc9b7 zPYgs62i+Vt{n|fFw(#$W0uzKwGE!Tm})~4MZ$=a2v#;1kMUz>I!D#jeKW8hajfn z_6OTFVjcwB6!-*XA3S&u%GYPb7%*Q96g-Z3`l3EAA%RLUgXqRXb{q~2lb0WtQO?~F zKteDpCktc&udnn`G!ZD~KJYsrsvC$u!v=12Al}>r=MC^(y;U!ae9w<9!E*Khm_o$W zB>jreU6dX?MTi;E9W07ruV0&hVu8?sK)3ROZ$!X$qzNHatDP)YL>oY+1e2BZXz6Vr z^ZR75?KW38^z#{g9|;tSAj$bICf#H0jI#32b7k^mnZxn z2z_f~@KXY_q{mJ>i;SDJO2{e*fQT>d4THo-goXzF=_;$$>NT)oflvebVh&KRh`a*| z*6&)k7Y*nCY{2>?55_H0u|2ZH#6((A_h`80U2LpuqwkrZ;{raS(tPyVFaf0l9Lhm> z4Blj*bV86+uNKzks%2n8HkbqNjcjtfq33JxWB!G{N{3HiVK_@y8_i zPcpt6Nob~MW_@*Ce=_BJVh7we?C>*Co>9M&_#;4q+6Ml!41^$XdbNOi0CAi02!%+FPEPIwR6y8q^Ue&;Sw?V^(WZ8xf(*WKP+^V_y)jZ(tom)=xv-lZ(qug%%8Y8*>HI}QILM5w| z_h+6Ir{@rHaNHsF_&NXG^J__9vAZ!R_w}#eeCf*>Lh9do!TtS4D|+&VAXY^{O7T9N zbkld6bV0-d#3%$7Ob{PNKv@GCDF?Jp=glbuK7fe_F=z>dYD5qMSOy7H1-NwJmKnjO z2Yw~Qs04b%>$h($ATGcc2qM9CP}0E@mjS=>D@`Kp*Dod9OSB5k&U}ocWql9eI2NEc zdh-UlFNk^}FwsIR@;x8GHy8%snEmk7J_87CL|X>E^vdPSQeY+p;o=@B@Ma({AkrNm zK_WyC#oukj8~7}V8J2CiK2`y?0>l%{rA3kiYGB3l!CbApD>a2>FQ&!8!!0vf2z}nBbd7W8jE&6Ca>YjJB$^uA#GvRh%VfSBHFmzMEKn?fUpUx>B67OU)TSs zKl?#VHQ1bwI?>{(G{YdxeUM~qnrn$bocFm;t2j6759y&l&IVH#kf&;hFx^q&wrL2P zln0_oZh#;7Pa!tvKr2IDMF7xi;vS&3CINzJsa&-bFdqGsBcO-bO#VNT^Tg z@Mj#^R%)E@QOte+WNiL3GeDtW<`Ld`kFO}Mx241o#MpU4TpY!=E=4P!@|7!B!oW=n z2sHteA)?2Q+t)^djZy=I@Mor`*ZH55BjPDiNyI$&npXE6Pmi|MNR`2&4mp*`XMF`+ zvBVXs;vho8p_#+Vz_w5656uQhlkfsTng_rZB2kb~^QFVvKeR%Q!5}!rg(W|CSh^)C zLIk@7s2>tC0uB`Jde8m2-|`GCe9DN28B~j~Ojdp8-*QM2fa8r$x&JFnK8Wi9!E|`N zV5O!T%p?lXWk?$(0X)APx2tj(HIIK4Sm}4RYJ@n=KgmaFWId!nN)w32pp>BMz<{fi z#P=FRtqe3f@*)7}pC*7b|I6=PPDsq=0XH`wS3>vkTZ5J`tA zO7y#Fl-L7UGqR}H;o&zqIp0F9y$A0ofPUEAObY+w3DNL(Xja1Wy-)mQm*|hF@Y$tg z?)5kY8Q+k7oo?)NsLeQJn2qUs;zqjVcCsz$>wAt*_&mDXqdu*_Y^Rmx#Vsh$fVRl% z1y1{ex=vuehV(~TxuJr`N(tNnpdUsjB>aTvt_B2AC&wuo?$p*bc-6CP3MYlQ6cO-K zN)t~-Km=?`?;*>e1vu!T7kv*%0I#U1YsVoO4_emESTeFK`MkVndW)pL zVIRff9;6zcdhaPyR;_EKB6iO1U6HMmGccTkoCj@4MvV^&ah2kLA#U_`<8)NEbp{vG zNkCCY^u_DfALp`shQ@Z$k>W`<`Z=rJdL5K*X_@DO@TvrOJ0Y|}Dtn;3Q4xcamx1sI zT4iW7K}5fyWx+~oODEw=8`UPGrKOd-D-F6Ly!!)aE~KUG*^r=jksDg-vT?q^ST;7C z?)hf?=ys>pMaA4!!$M-v3-O~rwL-FTQuZ4jcazC7WnW`tV|-!5Z*peD(TmbvtS*K1 zwSy8#mzI!_AZXqZYs^U2o01XIon_3ez)eN0ZT|1N0P*3?NB!q%^&(MGnF}U@asnAwRgHkD!L}sPEXfGNXG!)WKsZ^3mg`!16 z>p3p>{d@j;p6Ac!e!cGd{(kwa_xpWa*Lj}Dc^t>-?J-~e`U;j!nF`YSS5-=lepmE8 z=EM&(_lPn~?4_`_T~Eh)A=~wBwnt7z41&8;o-`4k1X;H$h2Gj(MknI? zT9KP6Vxe`cOk0v0jG;RvC32Td7tADUB**>iI8(yP%`bW1S96o2-V)KIDVWx@^Zu7( z!HlPv*aCTCm#LQMB&aH=JWa)|&z`N4Pj_Nny}-ifHi0fBG+~P?Y$}tbGQwu_Wm8jA zvt?-QKNZ}`Sb66}L-trD(+&5P%vs84nfeR!hR1B%rl`Uvmrch4?;bEP^>CBqR{Fja z#M-`2h1qESf@A;i-0}CcUA!M;eZ1z`pL8DBdYU_^V){?sX^M$N^_kxbGv^S-<4>ufy@V6gildsI`yhEe7`3A zIxw3el5h5f$K8Ur2d+s;daEzUN7WcfPh2?AP+w^)Drge8D^H{;lshA&RBrG@_)Z^#H2m1M{M;|dIT&QtSVM;J)ims@MjyNJ}luF+g zy`@RB>+j_CD{lF11(}sOhcYR_pde{64qb92C$!k6o&PEIQs-(>+j;j*oQ^7T zY3+50$-s1^z!Z2sW5XS!d?2{vVsd_A{iEVPIZX>sLT&1KmAfaPTydS|BLmY~Cs(!{HBM zTfyZy=NBi40HERb8JZb^`^c#f0tlrC8CLPN0t6D*`AI7Uj~{2}4nTz}h-%0RFdAZ4 z@|Xgl+dw0S+^2W$`GW`R$@hW=;i3>178dr@zFl}dJG(s?{g4eu@7r>$h_N+0;LHwW zf#eWGu=w~(*FJ}quicMzgrYFv%8EP!OF6$}7%Q(Bqfqsu8)+L)9pJBc>0|K}zuu-X zQXM&bxE-rn)7}$fEwS?Kpm~6+BA$IJ?^~p}e#OhX#lG_s19>%z zX4eH{*ff?H7~D=4O2ycj6bztb0_TDmJVVRT<~YJH2RSXvIOknclQjxt{0maagp>wueDvs< z*TL&Rk$M15Qvj*7o}H+x`_}pn@}T8;KYr*Uqo_XcGDT~*d;4~7O#waOpm;4)sPm>%Cpu_j}v>6nhd%UL4=9Z4| z66q4&k4@rRvNYYW0Ln}qNl)?6NM#5`(0JdAM|hJ#-YI1tm4X;^a7YOK2#*5Q<-mo~ zkA?Q^eJ?KBPJgW<;x#};d$6vMDs8p6x8-n=0wW4NqN4#M@yz|ZKDiS_o9vx$LLge&UIX+;R;5Xb=+vpU zqq|%IM?HM>$hM{ByNetVjgXp_f~1KkmS`>UVc8ay@hB2tP9x>lPWAaFDyCpgTixOIUt7Dhpd7rU$sjb)!Zd z_mOZ_L1SKf=jRh%cJ_V9n}{|BneZ?Y{~;7WOgww8W90(tt5Tl(bgg!XATi_*!F?me z1W3_P{LaR@6CyR(z_TY4_M|7X39UCZm&@L{v(tBkunHIFCTx)v0(l^eZb*IV@Q*hE z1`N1-aaY*QbPYirS-&XA@egOQV?`w-&SJHYXMzM1D@=fJF|h_idw<$g`BM^wC(`R4 zt#}H;6+7QwL3mk6lqkq)5-aaE-`l@JDz>$2*M9PuX(9j+wge!Z1ef9P;}4>v#Vl^) zh)!285LzL`Y4A9S)EAcrfuE;ZQkal}LeDSiVb2#oSwFtIMvHqieD}%{;XDuEQI4#x zOstj+hKQ3Q3I!~PA?y&75_u9zML=bwO#B3;BB=wwq;~~O@xJ`>MG&q|khla80wwrn zl=fGzQUMp4@2XsRGnQAW4Tc|)#E|HK(hkLBIAo@;oSzc_N>6i&kk#;02{eeT*}fx_ z11XE;tE+1XF&%A^gyW~Y)=I{>U4($0>_pUoSSE>2o?K5)7r_OpA#o(w8z{|0#|28^ zCN|P__;jIR2av=VT-s-WN+-k#rN7@Ee{yfvn%&aZxmP@E!UVVo5hWU=9!mn{nE;3{ zv0Dk#8dV2qz<@MuaPPKm+x9-?C>JQ64XEaj2Fj=||7J(e!Wuvq5o~F-lO_gM_U+&( z1~j*XjQdY#vF^k}rzMci?%nJ1x^C4BR87pW^E+vqq1Hv9JA)fWP&Dk9UW;!zzqX@F z8p2y7PL|xmy^11=EkL)qW3!R=*siE$NpRNFTL-G{O>@S*0Fh}}1&*Ow8>f6P=rgGf zl3){v)r2cqmP@Mi{?(O>DvRS%MBaS*^y#nB8?7=AFh_UXzKDbX*=cCaz$O4-pi6)3 zXfyu8&dz@3Vtz|gQwr{$gvnF?U6o4@NC^vaDz#1PaTAkw2za`hyCLu*fFSAWAZFoE zKM55th0pSzF2Vv|8nakU22~rkaT);21!(yQ&WEMH4+L3AN94M-0*0KqKu$o<`?_k zkn+{l^%;K{=Y|bXu&q%8u49jH^sDUFs`)$1!NoN^Rf@Q_xV))lmY_-XP^8U z7|yllC0thWU#thXkN{mLp~=!X@5D^wn>TOX7dalU;!}EtSnR2K8*K?G=#}wn4&jjZ z7fZ-cquVUDK_&LSj9p2pz4ZCdv4Hv?#>YFi1dd-QoyRIaYGt(<4-z0zI{;zEPk{4|i(Hn2)@QeVRAKo;P)51c~M+E53&53Fz- z^4e0j(VnIx{D^JbOsHR})v(4{Zd_Z-$f>KVTSXMWh>AzFwMU>Bz(IE^>FA@Ko~wJt zG^T+16d?E!_J1l84iG!vAsL!S2qX;&LgEKn1m}ilz>q<~JPH_5C4u#BEp?>Mf{EZ? zfN?YU%-wDy#|TRmY4doRR8HCNy3`39Ey+={;rIN|C(mtS50Z`ZWaPZ^P6 zlcHj8Zu6XXIiIq37}9GqE{|O z$7BI9nM6wxQ;9ef;T*hJVD;*N(@Vzl8^(F4t7~^2#fb{_CK6C$gaACzu053*JxGGO z5OOw2SV)*4JaXs0GAnKx9EF1@7!HaCc8zU=9|JHANvrBLcAiX%k*FTG|6ghadr zz?Pp~^j~AQ?n#fKo=6Jx9wS7!h5J`KG}T&QtK-m_eFO+9bl#B6kfsS94YCrVcLZ2O zlte^6rLIniz?F*hlBAHtLqOCZ^{L00F`(ObL`DoK5m;a{e&B4_A!Ckh$4fwE}054NIT<*ecMT3P^~+??~<2@+?$| zgcJ&CC>dT>lPVn!7X%_B@d4YEf@Eeu$w}&r(;WLT4R3!rSP2?$v_ZLo{No0( zfk5l40KH5?Zzlf*R$+i-FR=al*XO;?1?|r&!sdQ-G*rZoK=lQbmGo~eO>QDxGOX^j z#6*7}-9)47rKYx|a3;KsNBZ4+zmZ?NxCC`IML8xnW-`@Cav^y$`PpCDe^Qi97l4mZ zfSQ_U$_JdxhT7V9rWbs~_U=6{nqp{WmDB@%b9rcy#EugVv!QOiVnxARv7&ayHB=4=LJl}e>qloYP^aQsa)6lKSt!P-c)1cgrDz5xc`gp(@qYNy zgx3x|yW1^xB}=Kjtu3r&)L_^&O}@M;FE=V7m4$O2zia+zMbgfwi!R3}F7EB1;8rxQ z3qL^TD%2UN(oXbFDPatT?tSt+xMyDFpW7V6_23=b)MSLZdUj8ZB>7o+O|?UFiycVQ zjMUZDquiAR?)rwZ&|DMSzCEo`7W|C`9~C2g{bVHlxKGv*TwGj_x&iX1);BaP8r9~6 z=|lFUi((_Nd{wun0`45@l*X?`;(Gj291O0}j1;p1Ycku$!FyL+UYY#paQP|rykMQf6!N$6?Ph@fM;dCOOVya#r%qJQ{ONCdusWZXfpI32brh-kdv`x zn7rZeWjV<34ddoa?S(0UoK(%KiqEEV_l80Zc$zbB7K|K!o|^z_KFY>(=d#NsmQ>vL z_3^7@t&HsyjVN#Y4M)?z(Z4lHEZ z#ZAs-FB?lgvbFx6{xofexZ<7qJKL5#=Fc&9opfVH1bDi5%%m52*t>Y1s-8m&`MDyy z09Poc3Y=%)(EX-%_2^!TEp<7Lk&gozH}4M@xw0$$o4^IJ`u;7|39-cy$K4YYPsP{o zV!g6UNqL=Vrn$dO#y4t%6_1kXD~ZEhiti`3zgT^EWa3h9eB(q~{_@l;*D2v0W`&99 z$=058%=q?x{GsyO`e~#1I|p@N9XAa+C4R77%141EZOe$AYm&PmyN2OSSY)L2UYR}H zrrhD1&{%$W{~d>oi8rMhB$>jj)Jj|Oa}}GrX+Nc0>#ACPv^w6GlN4^p`~JvH`WNh< zL<8I&P=cz(@jz{&QE?Zwbiv5SO_tv}&L-Y_mu9aZcW1}=rr4C5Mw+5(UAm|EqPM7g zcQO3@E2~1#V=8HSEOcYq5w|qJTU(-2y9>lj433&oe-3HC=(pBxCSYUux)dG;_boko z6)J}e&t#^aXxCHcaH&18+cW!fjQF39mQy_Y+rQf5`{io0MYH%=ZdfTlrQt@dc|l8s zx;cU3r)lEFRL)`dw@#vMG9H`0OpEiIYZNkG4N6`^Th`7$^nr@yZK#w(i(}dm7f{HI zYJPeW-w%TN^ku5M$?-v!@so<@)KFpEUdkT1_(1VI>14ihk;U@um$vV1nQ&2XPkCZA zb~RDAELYLMu~UvuaN`7}oub{nH6R@7$rv6xcKbO>fX7$nYXtoYQ(4wr)y%lGsm3hN zA^+lv;(01*d_6_y?q36$%^|@|L(<>f$j{gs%(E|hcEP8Z&sMoqdsTB($KQr0>$m#P z)l4Km$-D1oot zFOaoR#HA}(Ae$ravMq&`CD`E@75PCz`F3HlR-S7=`LiCTD!UQUuzZ4069@V7ujYAI zsWgsgS$rF{Rg&a1Rng%1y>H~L!N>OP zQ{@k1$2 zlW$G#aiMCGy!I@jO55_Ro#~k;{~`V-*E_Av&G>OF*zM5YM0&jL{*l`3BLy^Rq1D;) zZbsIruS+*m67J}_sBFQ?T171ZwmR$_6u8P(z)7HGUm$tSuJL9R3*~Hu$XWD}|oCH@Y z{99rI9+NFX@jlE&A?mE>|G&ptn1^VQnW*p$-x zui3Qo+~OuS_IklEPnyuTOiC$k>9~QelyVl1t=U!ES*5;Emhcq>jrX-q6ra()@m@fL zen&4&$-Y%jecqSljOS>)DH%m`%FEWz3rcSPy;!TJWaBTxEoE|Rtx7Eu%>?bvx7qg{ z8D3IlQ%6(R`2R_oD?UvVSR;9@@NPhHK$b>>L*TRmjU9y&J>Ob~Gc-3hiaKTfvpx(K z>AWP#YjaO3y3MA_wCYyUpKRB&5?p6wjb>lPYM$A8Z>>A?3Bg3gNdu9bXJOj9z3c@U zg&SMfNo^Xt$#a&UK4(sN(%5eKh9q_6SO9aR%(;yLiu+bos?MdFGv2N~SGLR|WwMc@ ze%}TcuE`g=(W94t;t&oVY8NT*JWuURR~~VsZ5PF;BePk_B(g9+V*g z3RM;XzDXW}G_ra*$uVs2xi8bD!)t9`MJ}FJ_)Rp#WC6)51Bz0;NGznJU(mFFq)CKat4>p>W;>Cp#k?IMI1)e_{8qJCy?dvtX@x=@) zc=!p;Pb%ZM=(-2H7q0~@)Gks!*#_wD!98=F@>{5d1{UGNFvaT&IpwOZ8rR&Erj9uu^%!&XNm)I7=BP4b$0AP<`# zYxdi0uPoYQ3{J(I1+qW&8&&Ka_l*^&6q>qI*|6u_zKPh<5z~=oX5|-U5wH;0pShoH z@TFmQnCAA4VaCsWFNM!UxhG3!890%}!2VFUnbE}KNA1_d@qJVwZ55WCa%Xr)s3|%H zm#0#l#444odbz`v{Y}0rh@9AH8yafk#9m<$m#=O)zsGU1XY!?SSjE4~O!~1Q(P+at zo-mq&1L{q6%^5ru%3*P{8OcAzqqV9m>ionR@~f#89Tmkl70~39<-cF%cpO#XkbK|s zzixe<7VlS{R1!^*Yrs7}h~98UtvmSd)Yb^5+w$+k+r0n!{eK~gO{n@XA%kNj{f~KQ zu;luGr=`aJ|6!>L-CjdN$p8CeVw>vPidrkDH*n}InRox?$dCv%pqC5Pj7V_iwH=fb z+7|jlKYnx%tF2I|lq+AyaBTcd=_g7p=Xmwww`&g_8{Vw6%S^<4EWya{f}p%BuWf;Z z*ooD`xF(99hEl%5Ecy8lFYj!ZW3OE&YH#>x`u26D{nf7fHSum+S!U4|shERX!wy^$ zC~S44o(%MhKPE6dk!<)R?2V}WM#|_vwPAFxsb5`~;G$lz8Z+(vK#?q0N+m)OQoqBE zqAn?6d$Rw@M%7uDorxny&asZT4!ev`y|ENHdXN3kc!e%6XV$UxB0FBNr|9WcV@^`|24wi@h5sAD!4aY%05}BSnc_W3_p1+MP<&(AEUqoQtPob2Pp(JSF4h7l&aq7EKs3v zwv=t%IJ4%s=bmf7y&^6ChYJuXSfxrWwaTr9g>xSpzq82_AFTIx|K{89in0lYcvgeZ;c2}JKR5T^j2qL# zwJ-0!3^rxyoGsdYO=OI3t%jEg^QW6UAFn60bM02$vdz#>dpWWz@OzboNcr)x$Wf1- zBfh2i(uJ*lLji{Z=0q-Q@1o+g`mD25TX(}qTrTp=M1yU<5_N6ezIz7ZE%mv(Y#Y1j zTC;CiY)^FkR>WcYLydQ`^68)K7+2QD^E#dRAtwFYE41TDzl9!4Zu#16~Gj zN#>Egk~_xBH#Dl!XbZ6(s5&0Gy}`_aQp^8N*s`TjSkRq?pH9A|?m3CcQ%6$uDwJL= zUHfr!p=qfr!j{j*>+R%NLxJpw*YMV9T~jaFwF;AC_3Vwee9zypVCh`(sp1Nwkv0}s zVV#LJQU5(PeUCajX}sSDHn&45&u+AB)R`Y0WoSJAxb|pG{Saqk;@+hbaqJ~Q_0)aS z3nrmn?2M{u`td2&qt#67Gc%QPhUD3YyXC{QjUv)E&hJP&9$%N_6WiQxd#2HGJ|iNn z=q2aKpRS(g`93CH<$3#5b5yhsnh0bsi!>%qnOnM*DxI~W38ArWu|L$f`2J?|4eF1~ z=a|K*9hZ;FPx*Fho?#s6KR7QR`gRH6*rU61#cN0YEYjH)E9v~6^UWW|`cWsR8P~n0 z$Q+?TDpy&-^&+cJ=)(>ca2$wY(XL?b3OOgPIds-2?Cn&vthm0O%uCMt+M9Q36gorN z($mMqj_#{3a4!$3pE+Bs8sp{fc3E3SGlp9XZ!wGr3LAZ9AIl`#l5cM~I;~{TrJNP@ zUd$~^C+}LD@be$;+5_XC4`~aB8tiBp3Xl6wT$`*qz0u~%Ub&*gL$PVv+r<8*G;E{5 z5dUw-u6&G+E5qhH`$SV*dz}R2c^DTz_;K~a{LrrRW}}6fGutxbCS6;fQ|g@7 z*5g!BO-oOGLZcPzwj(WurJwfNf$m3&islz%3O;)zE+y8@rOaH_Pb~fNWpqu~m0xQ= z7dG1)*=o`0`63AI=hzL)-6|v(teKj zeA--@ACuTy>DIXaE|_!$*JS=&M*3i$!d(3xs&Cq9)JEBkENP5OMsZsZ8Ml&4KakXT z-#NBEcKB!G6+Y&gSRw7MmSbGSmaXUBxXEdXFEQ0TedHJ&lsi`2mte&$l2ZS~b;UJ$ zRNH7w*N{#x5bE?~UPXg|sFdWy%w+##qJhRo63V3Fwje# zHI0Z5eIOTg21kzk}&)CpPwx^Lznk_`5#V{tY;y} z{ypCP5G|?RrZ{vdLF>Vf2(p?4L2wd1-fnecv@Fon@w~^)MDw$2+qP|<_pEo!EG~QM zrj02?r^4sTdQ@KN-pqo}XztTMwn3%`C|omlr?s1=a@J*bn1SH|lb6@u-S@y5iIC^f zQn&;&31YJZms;Yn0Zf;)D+}%0OTfP+4HHq(j7Aj(28K%@NR+OwEK+{^kiibr`Z3zL z{|qzGA5MDu^j=nOSqM;K(rE?@32hzXPX#~+5;AVtbKJxS3Z3D%LsP^w3tS@ND@<%i z;dMdmGeFyQE*YRf->(tKU=$1kNn0F!lwlCthoGVug`vily?Y;_^VkNDzCG$7&cw2Q z4cYd3tq>a>8jr+88I%;#1|`1JK9+?Oe<(S0j=k9F8eNA#gt51)6V68jM1)m zL#%qy1OS+84FZX@Q_qK7e#Z~#>oBU|*k9wEljv*^c?ZyMn2x*$`t3LOqJIi23XC0= zF#U7A&tc{dS^#1Az+V`hv>+>PTn`9}Ah4%eRQmn?)fYq9H)zD6FA>oEFCpQ) z;Bjmxc+e3;C!**9Lwj^hG z3`!N^>#VXef98&Way_76Vh{`N#CB-ANP8DubHX$GJ2!C`G+WSJNXs9GlO%33#bU;h z8cCwli->GwRUMxJ@yeBGZ5XOL`cN`Bsv#771E?8JO+^3Bln&c0L2>Ce)sO*cdefK zh?VxBuH$I1!#jp}!cYR?2_5d?roH_{RRG20O*r~Zg8(!QwhFPb*gi3!06VI#mDj!M$;83jC{(YM}>&$M1yTJSW((jfE>xVT2b;%N`eVbHGE zI(=G**mMy~Vu0jCy8}MZB$gEvOb5UyC`H?lf;92KpTQy{_U>ps5qT78ec{6iX%9Uu z;bW0%XoZ6Jkp#T}EZ+NBCP`tnS^IE#chg#zv)PW(f4y&`<6_vF*Gz+k1bQ4eNbAS` zh$Xb7Q3d$xYi3wV+U0Sr{N11Zvu18!ZqW$M_`tEn<&`s?tU5Jge*_9D!Ubz24JEk^ zf#gq|SfsZYnYvWZcXUB1UT>t7Aw91)eajj1_Ue`lHrvR9_6O=^zXY|8&3N&$xITDt z(;(QPdBEjkk-2fG{wB7kwB9mI)M@QUHnE9F$w}zi+?JM%krI#5s`6H7=dC@~<)PSw z-^1!*6>3g1^6?y@P4@p7viYeRD{~;M_vdrk@}gWAAX3Y-<(MeQDgc@{axtd zqWO=R=22ee31eY!4E=q6NAXpulrVl>Q+3mMr8wh6+mwtk?9!%=wQOBE&D#|SqB{F5CNXrB=KF0ch50M=EI_nwbUh)&frG$j4#Z=E?Y=9h&7 z!r3$;zlv&;4~gF0m2;vN(T=GK1oUd0_YL3yw!l@E(A^dg2OFeSgLI0{HVr_c;X;Uo z*p(Dw;}`LRxx_LswSY4gv}Xg2g(}%b@_l4*QaP zYv*a`vKr$QA7PX9=-kCXGm)b>F5r)a7AN7|L%9OeUFIqfeu$wV4`>;n2@HV_MC=pM zCw2wxex`LFE8!bqd!xUbKDY%)#ybKM42GH3ocHgB?@s96X}l_fQEAEsd~K1SFm{sF17H@eZP`6Jc&-;P zO@pen!uK+sL%mXty|P+srjdAV<%%~6c_2fKRf7_bN{Ihngh$D3jE>p<=Q ztea>V^eP;>BF(QX|559<(SiNjX&jWZ6P?-AosAR_Fce)=p~tYQ$3Zsl)u!fzHr~04(w0b=+tn z$-5HM)tR#)I`wK<>Q(Rge1|z5GgNW5H%WLuAg$XmHJkhSwqA5lJ+s!U40{nr(~c1s z05uE#@7s?|ozzqRCxA-{)6wK^h3&&h?IG7cVSL_m=*G1->sC$nE9x^lefpthG`f$@ z)#@QxfW{(BTM|Xu4*E{qaM*J=hcx2|V%E%{WuGfo{CMB&*SC+QrMX7zzO56tQw!TC zbvpTX-}k5$8LhWJcj%2YYVT1BW*dzC_gOalxA#)>n)iR{v^O|yx2n_by5BwQaVxC4 zOzq36*}|KwP9_PO~W&aA54Bvunt7-P5-OZAy1dMLvtfncYy-kX7Wg4shBs+WO!aH z5tEeG{o$dkwu$>U+i&|J$qnhZhQDiX-p_$0k9{v&aWl))_*xImR7WO_jjFou3yT@f zg{z`0=I!QpA{BVpd&}k)m+mObWi~x~TN=hWtXpyYDzrU2qS@|ikv&hJ@$6fV+-;UO zT0L?*FEn<9FfP~Y-pOq4;(1IN%}GaNm|SeqaX#Y74vj5|r(30slpR{74Rzm?#O^=f z)L))AOMNfODy}QDIfiY}ggkL=vyqkM-OOdDdV)5d3i`HRm!YuWToIa_3QpWok;)P^D$A-7 zY#>CA-X}cU0xUOuV^DDPkaRg1_b08Y;i6}|+}PcCM}==R+<)o+#`7dyF;3@@R_od_ zm+YXX+jW&?iSoQ;vuKfx#{QDS7_1qQt?0(SW3x4ronQ(`?u z6McXNZ4ZyCl}2!Z8hHqLY^NT1*i6i6ib`v3S5-|wA#h>x3#|0^f^9-_ThL@6Rl1R@ zs1Xp(!xBgPQI#9x{g^@9`c9_YfslhjTta9AIW+S?<9i z#!tf%5?Wv#xo~B5V)gxVrxOf}j7Kyzg=A$pA%@k_*$#hABFY=HQ5$uU!!Q(NWo_wt zTTqsxJpNWjw78%(B7CFTcm*?KBH|{zHd)ySQ~^j49Wq4Q&BMnf+R#e5iv3Nl>$TPu2|lVy4N0Z z75jT4*V+DEilp(q1OkLx1~F>zojbzYwguKb++Kj0G+>2kf=xV+ zl$Yq}iLP@V4!}C#yv-B41k{yC4KNBS3LwK>lxSd)R}pR)SS((615w2gb5%m!C6sR% zmZJP|LB%Qfumwap7nJ$bWR$^Ee3{?U)FHwzd$8^3B~)-wi+>HE75L}!2$na>K;oE9 zxN5keAnttfTl~2Mfd6kHyG_=2fml6Sq|`!v@?H>YAHJz;3jJ*cOb}-5BPFBJBW_ z_Cy9kJULNx5r+p-65(FWplpSdmC>dp-l;3r6h#1RSI=L<3ImOsi1YD$|1B>R7A*V< zYyA5sh`}+I0>Bu;zJ@A-m>R>&VGoM8gLehA@=lf8eMrB*#$JPtY>xfc+dTMdB;VE7 zW4%RWXG=gz6I45tgj@s)^9U>(!qdf-5!48xjYLpgPkMG+|8wTV{U-BNK7KU83cO7r_#-iwWR4-&{QWoPn>tfrNMi2LRVj31DXe1MvWvP5d*$K4T@n|O*nku4Y}i?DiUPuAfgFI zZYzk&6>wJ^c$0}?gQyKC2rm~ZB`^}{gW?(fTv8&LFb!zkClH&2`EKA8?D{gX|PkMh(QNlnXg2<3LMzC-Ic@5R*e*gfiU*gq`Jx{P548b79 zGuVe9n3I)Awi$jsIOUWvd(S@?R3oHq0D;5*MRADs?{a%O_96zHN`0&-(g;6=nM8VxQAEOXCjL+Dzf75kE}fn_+RlNOhZZu(+5}3 z0OI)LVB$J}DojT6IH05_JVLw;H0%$U!Sw-9;kb|K_gL41x@*H3(jT7)zoHp8ak0F$ zLVMneW1YBKj}CcYDul_S)0o_bGn* zY@_U1dyKF{gdXUJ;6c|6NFG{Ym02~_kuzI<6RVxJtQMfpaHHnsMV=sacq|QH;D2Z= z$z8a3?R%MXr7RQmqXz5ISGRWN6* z9BauBGLZ?%clK``%cz~~&tdsJ+)sUn-X;4>R`MTymP1jN4Q9K1qJSA+Et;;M=r`jO z5HWjYa^U+|*`3~>xzfI9lk!dnww+fs%7N~q!H;TRd_{wrfr|Rc?fHVkJcTIJ= zJYKxjad?Akk2tA}9oZzwM1ivRErdFZx~amTI`JPeE^h4ieSVuoCG2%@7}$8 zy69`_TL-+EN)6x^HYxTZR0GN%0x?bv`wmrlkIcy+D;JdNpstTE^9H>ldYQVqXEW;r z=zjGU-^BrkD}bwqT9*fQ7P0*%iZUWD5>$zWIH1?fKO!s)LxaqRWgMS5o4=h#1N(>| zPR{)W3Lo>}TMUbg$s4Lm0mQC@IGF%7A_m7~^FfP6IN>mWB>*9)_O~GwBR=;CHCSdt zh<`SNuWu6l6x1V9p5rFq_r6J3U9!bF;3CIIAT-tG?>giNz%U7z2fzeE7AyaWGP}PO zHyNM-1qk870lrlx--vUPkyD~-_yVEI6Yw1g3gNkfYM%nlE7{3Jf`(ZIdzRaq!xJ_7 z3-jJFm&X-1S>#ab$8nWrtFlnf+|;JehUrsSX1$f{e^4So6Bf+el4=m{T-5CH__MM& zkp=&q4tQ%9p{*bTYkm)G!Ymwhh&v*1E#FZDMQeC$Y&IFv79JCmweq*CG7}!@=Aait zPLzxJLfM!qkcB+r1JKCxMkXfdaB)tD(*p>x4}iSguWxBN4%LDIKtj!ow#|Efl?<_T zpyu|#Rm(u4=Ycp+_`uLV9M1qjJOfWWQ`V_RB(E#$l;46i2Rr-Yiq)0NMSyZUKu`Yz zSxxwniR?Ws4=q3+yq)1usb@;PUfaF-!0YbYOa-p$g#7l}OTEhj99gij$p|i@_C!zu z)&nO36|6lnoeuag@jiG8)dCd7c);WjfvOXBKQ3UQMsTGHH=0;_0Bsot-aQIr{|rDu zVkAYv`+q$Vh;KES-31(vsFVPl6YULo0mL7G`uNGj101%5v5y4d-+VD}AfW_Bs6j9$ zY8K3rAeu@7vpDztjSWwrjAl8@#f!#E{sjuT>ApLOBaE6k$JVrKtX@?+{VlCTjch8p zT?no%qeO^@Rd@f$HX3cadCnM^QM?9`*i?U!pUW`s;Xeh21HfDxhE;~ z*LUkc_kU%1kD%3L@gr`&2bZ5HmB>^rh!RL@jKs$p%^rf163=r8G$6&i4G35C0FY9M z3sW|1+opJOp80>c0NWrq`sbmSfaZ$3`F3a@;5l&Q2t63!_^bH#X{o9t$ET}yPwPH! z`FzB34Ru?ISB0(`f7OW&o^_*l{vq%hlH5-atcV_e($w+lZzM{j7e>Y{0&9H@xrGHT z66|G`2x+Y$*v*g$VcAgNU~^q3-V$hw+#wLW;;)ix@F>IcK7=!d%m#!M7r9#?G8EXP zYhhsC_!1ny2pqULK#3Cw^#MZ38JJy=v>Fyvr@V$a{v#k9A&&5fIUll)ldQI9=2ASP z!Wum}g%57d=BtKdV=>DfCl0zWmC0)VO*|kuc6Xmu^Hvw<$KHA=Pxw#JxoWhWx+&`{ z^W&~^dV+3cUAcGfD~98@X&9Q-{ZU7ev{cWtN!W`dbs>N{K59Cqe<_G0DOh9JLIkfy`vxB>>HJ9! z3d8X;ii;aG+rx|>WMt(`@c(`&X*w%t=u^g~GdCaTgQxdIsF}DmxA00?Y?wZzmf7x{MK-_Qf=%X|43Nv^46^)sB_I$m* zDCc-~XY)*W;~Ou^hr+;Mz0?Dn((Iw_KV2!~at_cOaV#-KkCIGr+!w?|`E<0KDPfC2 zpw0THBUObFl-47og%RNiH`+D3H1~-fK&NW>Lbsd0LYy|z?Z2z%Cn_F4&YW4jt2$9G z^_@dQV}gy^M1(;kDD|j8NU({C?lXz2Rk`^u#Ddc;)hmn*=l`1bbr`MVShH>e+ey79$^kH8+)} z3P0~+)A+D+J+j~;jW^!W95oM?KT0x@;gJ|ahm04dBVA)2xkVQjiyf_v%ZvGck53*q z&(V58QMT+oD@6Ii4ktJ&Kw2g8+VGU$cn@E0PN@V;R2+Kld<_3wplt3b|Zxe)P?`#H0TQ_@=99zyZNAC z%!D9QOkBJ{eio5G6M9Y1z_Cvw9s0_O6uf_@mm%5iz!ZbWVfsHHRM*8c0$smKi>TCv z-rNsL8%^Jso~RRRW{hM46XOTd>r;e?KMmjMj_%gGA0ujg0T9J)H zP4)Gsu`O||6Ok?2hPb=U<%b~j#y;j&@sUH8Pc*j8gV%5-Q4vgALqh`&4BK|@Z2s=c z4j@{SDw-|1IbH69sfQzzQAN?g1DQUX)h=Cg5a9TEJe6<#m6ZiQreT;i zda-hmA8wjZiIQQu$J*UJ7j6j7>Rp~`+6$pES|ij%DScu5<8G`ojDJr3kB8D4%uzyG zl`;pvSBv`yap`Wy+PA8z_J6O_*PMceaSt*qQuC3awkUB3`hjB*w@>fn$zzb1xK&u2 zhClBh?Nm%6vB8k1C#k8h76Qz12F-R7y5WC@Q)XdEa5Xr0tXVP>eiqxeBAfrg^rBV_ z>PoamU{^Navjdp}&Ji-WGXdpy8>)B`7)kKKh^z)^UB{WO{V4mjM(T2ZAp#6h&){sO zXJ8mavanr3f_N^j=j40~Nlo28R^`_avj+%H)cnKsr?N7gvgEQ1%=xubn#+Z4JVw_x ziZvx{$&zyvHzmj0>L7j*pV6t}g44SqthginR3r4cgYMXyY3oFpuVj>oQOaty!>Xux z*~7d>(l5OxV@x5!8g>O|e?_qkRti(1pCH(p>BAxsooZz$_1doFB)7l*7td9;x6MT^ zrLd~hTX@ka&0O3Ysvj%!r6ap)#g~4w-!ZI{!AE)w5`rdvx}0_a@|vE{MUp9(ZYWwU zeehk4xg3A7o@z___?+-&tza{C^6J>qV__+!JGMx7-u=b!;aBo%kn(6+bBBG~tJnNV z0mh>hdpG|OB1e*&O$Vdvi%_4qoLw{nETUstS>iJ}hTVtkGmf|fJAa6pV66AkDPd|f zuv7m$~St&xd2~PMi_mN9ImMr`%4oOz7mQ zjJ}!Dl&MIW(|BB_j{@2Wy2d|K|5#dFlDILX;HEr?TZw<#?@v}e{%M^hS(EsuHQfI{ zUh?`(dWcSrrMb~Xcu4z+=SDmGvn9{!GKAK8Z_h{W}igv!mHaJdxdvjE3(iib7TE#;zBiH^|71EsC zFz?A(tn$#1^YqqtbekQn;|zJ-qMvA=*sRngqJ5E;xnt(;!j6VEhcfd@PIhGrz}A#}1DKlU~#c)l%2U7)T#8m1x6*ryS!wuP+w z8JpzYj44Yty_46!tv%K>oe%wqy->m9qwTVqyZzl60M?z6I(r~&AXmSG|20S z1~ynH+Iuf}S$Fl^^(Y@N(dR64#HyEk7Hb^gT z;Nk*DhLVn+MOaCQilQN3R}*4pO;Kb>K~_&aW`ucP*S6y@T=J`Q+f^gkS~%04mr7Yi z#UI9G&!D%FWD{#PDB?u?O5Al}*hok@AY1`}CR6PZ-d##hhUdQb;5T1&HIZf-I&`;z z>Qi^#Uaf~;zF1PkqdN^|1bHc{A}B5aD=w1q87VtxOc?n-0`DDV*!LYuSw#KQ`}3ga zn{jzJZaJ4tIIb_IbQx&wdB~Q$ha5Tnr)dkJZH0vN(&Q^zV$}(=3d{(@OmZ;3$QWsW zVaT;KFddtgRbVSB6%?b>Gc&}16lt&(@T)bnwCy0Ipql~P8Gwc?a4ZPJs|fo9(NhjH zh2eP%`ydh_FmSvM{JUD~N9G4jOd{En$6FBHFqZQUX~hFgM}Fi5W*Si;?}ZlxWsREIBqg^`V9GUp{Y(GX-1?F3R4!FG{C5~wm6 z;0;ann~mpRs(X7Y0LI+~{F`{RoSFRcirjmW>0{~;DRlt`&x6$h`Dw-nZOrr}^}^k| zgpWr^Nhp?Z_lS==xk^ev^=FZfp=_jsr<99}B#P1zpt4)V#cu^Na!q3jynKK0?^OED zYTh{KDaAxYf)TebjK*Drcb^*4O^|N>>t#F*|I#*0P|JPsO95!Lj-8zdajihrM-b~c z#fzZX2_o;osBT!}7?_wGCKDNI`U>GtN5Tu_VfZeEi3H=AIyL4CEQRL4d0OJZxrD4c+sbAQoywZV=6HxxdCIsLQ?IC#{#N# zwII1bHfwTxQ-vtgHjVw!MZbMvz0M#_XNeKO}A(&{K#e)8gS$m*Z!k+XXYUsF*M}$CyM4!w$1QJ55FVL(w zo)L#jB;6HE*$iUg*F@P4BM>Y26yYvCgA1&?`6yvn!p#+bfI}fRkO2-D1Y#$Ssz9|y zFNYzFQbd77K_}^{hO-8VC45&??e)#fLi#7jIOzeP9Rx=o!E^t&e8I6q%8RCAbm}@j zzGL9M5La8$hl4c+FoRYAHRuBHVAm13e-y^60*bc=oJATrI91T~K}bIm*GAg-c)WACiNoC@~ia+UUGVS~?8H!b4b{0LHh&6ocUlfN^TkJtK@x ztn|~c?8P$83z2(Df~G)1Zu_iy)lUR=y`b2c`P=*hRfl`nA=FUsQO5M^33JAQw2%&V%ZvB;y=__PP1y@)!ot^EO8&49v{bW5O{}wf8{aQ+J zY}sW$k*7gr=CqCSCzY@pIYP1qn^!ox!>Pw>%W@xwU6|#j7E{ifH))q1^?H-DV|-}# zPjsq$o|S5GOU9+UKbrG;R2uSQTcT5q91J}A6WBEV#joswwU06_3aAY|T3Tjh7QNoI z*j`@#@ym3#oe{G$$Am#qwZ8-%3Wp8znN1)iJ)RFE+jClU)=55h`IZ$>>6v60Yq2eR1!7zB(jb)>K zsRt}tV(N?jVCJmJo53FxsA{F1KSg3&5N{nEbnr8SJBXxp{RZOT0v9d9(3bc8D~yhpI1z^Pl$d$}Ujk(P{K) z$-h&O-lENaeZ9bwiz1FJ#IcRQzc3qTLG)$UQF;Yz@HOh@5_AXQyJd~;6yevA?la)v zN#gbm;GH<|5Uw)Jf-XnwvS9hLrU(Z;arGeva_AV48-^fBJhT9u5hw&O7;()9*L!5w zl?6s(;YKLlFzrSOn(x#r3v|C8_4)iv!fGRq^R-Koh2ASyp4)x*QlUP`Gv^mVIjwh8 zMOv-w#I8+yP?fBED$yW>K-t!&kH!W`d~hIC=Sz@LyTrGMbVXsgqKOFXHF$G7;enGU zj(5Pv=Rs5y4qcyWScH}0T;zez4BQflgAGy_Ldqs%Qqip^1Y7jZ$oyDzfTl9AZ}uxBA6 zf6~wel6CH&>jwRU)-6h_ahtulU9ze=0WN9CL z@bF=}vRwA{>#OlBNR)>~y(s!gE}C1-(NLuzEZOe9RZI@uf^!+GhIk>RCHU2o(bM4F zk}fY=Qd{6c2Izv!CznFA*w*d>>VE=ft}Z034uGsV1paa|A{yb=6TBELazz;h*kA60 z!y%$4AK$nkC!Aanb@$-nky(%w;C$t>Ec%Uvb3tgoXjM$YU9BT(!r0ZhFRzI;i1Fj) zwgmT|!;Y%^+ciDHoJGHf)4T9mQswwR;B~$=&wPaB$I=b?sNHd5v}6>0|KlBJ7Kdkm zc#Nii#g%0lGSeG5L-U#ybhoOjt1sd4BHI+Gak!TSwXW-pi3D$wm1OhMt z4goy9BLq$?40~89qT#WDgF{bOR}B}2Z3!F+Fxu4OG6nt^e%|2TV!&y>>(D{sS*)d% zg3U+bsOr)H>k?*(TBE^rEKVUDp7%?3=bz7LT-(HT>QPdhhUneYx~HZ)5-B-R=Se^X zPtM<_+*g1i@HN{0b>Jj`TMlf*xpbJ5&@7Ss!&Mo_SmXB}7&J;24I~I_)~1t+#hWw~T(>SW%PrQ^2pRVZV*`k_{s!k78-FdUy z)~49w#6Ahngp$DTNG)R(JVNjr-OyIR_O-+O2x2vdjdK#@UUGlYY)U-0nh$GD7vNy% z;2c0w1Rg(eMISp?;J$ec)L z0M0~Y{cPU6nb28CmO`QusO3|jMUeRb_zCv#*~8-*|5W;rB+cmm{mZ#{UwI`-NNDZ- z^T|ADLa^)HiTk{}&%Jhhm3F!K#s^JE^U_DHv%*N~-taY*x)oGBwv~`fMz5)vStfDhEP|=-Xk`C{YTDxR zk0+eAPTPv)c3Do>ie9I)%k*hrHcIl`7UpymP!^faL2MB1;6#YE?B@BaxFyHTwCZz5 zzhAar@}SzAA8R;_APmhkWK92PY~DrmKK7qVc57y6-l7<1b`(E_*}oihe28;uix~=0 z9CN}iFj_Wth@*znaA(OWbEq@hbd4Emg^YlEU>)!YMyx;HF>zwmC2YqDXR`?PfFqxlV zqxk5D+s2#hZ!!Zxz<0%qItSg;EQCy+H6`sX&5redQdO8(5bV|okUst|Bp4`$ z#Yj6iBMHx24H_mwc=39?f`akb2o$Z1O4lAvf%aeriLJ$K$pWOnV-}+e@HyGelF2U- z*Ldmwl1TLA-}n6(gV84Hd0EUls7#JKG7_yn6KM*E(SJ}?NUl_h53zXvB9Jl&VE3k^ z4E@rO_BJ8l(z~*z?g6TeN@MC_ z|`sb>8vAkl$(V=9(vMqKQI^uMBMwzv>!4bjK$zImuJ~E zUWVF;@z&wxq>s<+qRUj4r5(zMaLU^&;(NAjz~$zcI|W}LQm8U>nHS9D&?N)6_%ARS z9R{x_6pF~=c$y`ROZ%4l@Ju*l_rg%m3fl%i%(a_0E0O$6j-D7K1cY${g%+D#UcL-h zdL*6m_@OTtnwetw=nGs-Dm8zbuJBF@M!pRMdXKTIC`W;HxSAok(DzgI?WtsbgjVM< zz9H58MlqG#l?-aaaXE(Ja-;$ZA9E}0Kq+Vdw-&{vfCBqZYl%Z>Dfm}}Mi4@@plhDm zxCPxE&c0?3=E9m&7Qz7eV{f)W>Glvf%u7&1Xc8%Q2o8d{$B+vZdcQmAVRRX20OCJT zE0fF+nuWC(Uqiv6g_aTQI(k*=A3+R8sGL#6^A(4tj9imZAWi9gP?j(|_E-?H3}ohr zWtg9y~<)H|QQWCY@BmuCt0<^eSzel0XTyhB#vsizvf zT4kmmb3pIl@k={T1Qn+D9b^6awOYJGx455LHkse_ zJG07B`L_YnH`hP8u_xr64%D_>SpTmYy_oU+qnk})wASn_HkdE6T5tJ_3n{a6&k=09 zVI)$r$SNW|bNX`UBjOs13|m$>6{nt83cvo)vRLe-#B2s*fw$I{&4xMS%x&RkRK0oa z{{`}wE7l6n)3?a<4BTulvU`4gsMZ(J{mgN5`)PKo3$D7GG9;%(R`~o-V6qfSqRp9u z6Ka1bj4l;c8-M&<@71{&-z&px$J(ZQ?y*fdsfw~K$uk8{noEDW%iOU z)F(#C0_oSE!Ixw=nZBk_tLfODBgsBmV#n`V9iBO-W4poqzmkPC7hL`ov;7YiL$@x3 zx$sq*Wzi-Hju`cg`!wTLUm53#ITZG~wxoN8?(N>F8pf6OOQ9|X?W5*KEr*Xqk9v3~ zRdDK=(imMbHl942@$-y{Rm$J}Z+@5VU|JhaW{0nBx0Q%axw_@VDHQa)w;haYYd%3@6ub*o-|;nw*ny6O#1vRM3UxO|@O z(2H4pEnsI{`4sj;Z7CKXkKnvG%?9^2k4r-JKEf}2lBT^E(&ycoRw`@ieaipFL+x%; zQ@QrAkyXu#alQHHCZg31(rZ@@_PY&RxY<29I&v;e*7dHKkzdhd9fS39TV40CfpV*w zyWE?_JFoBheSqhYR}o_+W2%f@rdike#p+RN7uX?cO{`Q+!E-n36sjAU9q*dt3bs#L zSFWoD4x7T4^GCql^;!b#$wvt;p%xz_(>b53Z^iSdRPeErdznN36!5!s?vky^HK~86 z$J{co7%RTBMb}EXzc6;ZaL&{Om&NG|Bh0HgKQCVx_TW2tw?sJh+t%~qvDcIBYdzH` z_#C6`nV* z$;p(KoQ^jTh{xJJ%NPcwEa$JC3cC)ArrUO3eR+D3lVOZqXy?NAJ)=L;JtDqa9xe+w z7uIR)e@rCCs8hQqxkp8=d^r4Dj8wqc+Y4X3NSMu|PAAPC_D+>ln~)twzmHChgkl*< zU8r`;Pm2i)YaoFf7Zo}YhCnH138X8S#~-LkeT(bPPd9cw61ki$7 zeL6Qg<(@~oqMEvTUrpSMFe?m=fzL+Y>W9fzkPbjgV-Z-!2sM9R7?9pS-kpH3mj(+$liPT4Ro62p7PnCs8`CWD5`*I))R9L(uE$|kpiq#}18zkYj`-js zL@-&0X>ltY^CTW5hhku{bV;LrC1e#GKaDY?ed+$@N+CF>arMw(0GSg5YNHO6B$BWt z$L&)I!l4YKh_9TR%={`oERmK@)mMU;`!zuGN?`jw`u?a`2s)?QjMR-JsK;u{ig3g2;~15HjJgV{`l0ZmaUZ~viCb}`aDHYyiM9<^JF`G>iLQ+xb)E#_R;zM{TX30a*wUI3$j zoCaD$v7UHrHb9a>li`>UUW{rBv-iWzmH{L9O;1S^FcwI`RLzRGn5qwyt{umy4`e>LfRsi{Q5PzHQbOOmy7S0h9 zn-%hn*VoT4&PJ3ig`XV-P(4#fhzomrd(&V9%12FreVNkQt~D5Qz+G^bl=QHTpu!yW z4zy#pQ57M{ry64^iHUd7Q&7{tV1X7Uw-am}fR1Ft^q+(curj!;&sEQT`C=alLwsPg zR}yTHQS~R%1N8J}v!wtOW=dbew#5JD=zI@A#mT}@$5+<^r%%m+l*)!icKPRk4}jn#?|qQA$hw2P*p1!jnvgb8 zjrkn;;~hFE|7m<3uotKXY5_V26S0V3WdPD+D0rcHM$`4~<+xUO>2}-Vb#F!aqxNGs zb5ORfkdSz?;Fle>{>X2GYNV8g5Ep~sH0V_ zlmh}%&eHAe2R>*3ItOBW`27+QfWtl1eCf)S+8^G(mn`bs*Ju8{|3h!DvEnWO1c*F| zw*1}=+m9r`p4?ZI3)N@Bs~9($VgB>uG(%@z$5Dz`wlfTa+TBjRa>V6euwmPU@t5=y9LDe zglXW5-^t!^qefFhW8$M6u&KeQeM9D(;+wJq^4gDy${+c7e` z_C}!K-#C`5Q#anyv2n~Di+y)>QoFXxeK#mgjOe_`1@(s}6K|m;kh#Xy{KQGmnA&^E z&*gM6qo!|Y`2O$gK^q~P)(lK~80fWoD0^Y|L-ePvrFF4R{?V_q6K__RnSTSFa6*3F zQz`Ex;ZMZ}P38mmCP!{+lshLrpY-R6S>3~-Oqe%T zs1y*!THRoX`A85qf_RVf_)*Z0f!NXG8W{X7tM0#%B@3vn-P_`{>;K=nol3T{tA z@@X^)LIGI8>mm||Xn&f$!{HMPixVtsChS5ey9I9*oKc`^0lk1!h(g;^SuHn@4-I^>Z%3EpNGa-h1PwF9P2 zl7ZL|3*gt34Fa(O@V9%?moVT+PD%tIqk@4)PA^J`vH=%IpdzBI!*4%j;BClI)GfxD z2zyPip?ir^M{wi*ESelsr2#5@BT}nY0d)!jpN*AeY}3)97+W0RGz1Df7MCT8$?Z`J zJ4;nxme2Fj_kNhOq+!X&XHIU(TpQlc%!OxA+0Fh|q~5%plb z0SCr|=s>{oI^Zpc7%BleLj?@s<0vIpK>q^<0RXx>ewv(*sI0~IqhS;5cSz@-W5-fG z1jzwzRhkW4Y=6I5rudJqO2N4S&pkigFt+{u>7UphEw4fbwVw5S-1r8G%N&}&O0rsZI&yNpAQPPya1z?ndXXvQGJPVYW@BpG|NYp|U zL4XgV3D$9ngJWySUg4SL5iO76-5QOzPRO;3?OZjtPZ^R~euJ3$uGFh$8S8N2ok5+% z!G;dKI@tcm7~K!io?7La6k+}3Kn_xv>j|wr`F;t)KMq6NT zp{@HpC5?7@j(~HhRQ~a)rMv#^I2@AFk>n7oQ)kE}&>OuL=PLN{?jH7&^iGodbW|QS z=*PAPYrd#2!mpvlVniL4`Sk0$JgkT=U2?qX=i{?MPHt6&gRNz2(~SfzC_wORgt8Tk zpx4;~#5SrQm6BJMSP_nt#9fE6I838OKnb$B(povgQTBE>V-L5byNhQUCcD?|G148- zk>t1d&e0l+SoC=OcUGYC*e`!)mSCwplfQXkd+U!Rkq-FWXRjT=2kFDki7%-0V1Aq_ zv4x97ElG4)XZd@bjBtEev;SEl2d>rJzK-+s6mw8!Pm&+r_YnNbt6eK1D1_OJ0 zzqx|MebzgIlD1FaO!BbL488defy@q_+6a^#1y-~p1CkEZM0n5kDU_u2+}K(;4Z zSKVY#v@|KwGBagB4&;H+zK*F@#(!iZ&Fo-1h3F%48~xX5P$LW|YaGwL!wvYbC(;g0 zK+hdgjCS=-8)jBQXMY#Ov>XuU29QWJ%9ZOm=|Q;*h_z1v<&%h(D-mV!9m>B!kk=ZN zbHM%$F_!I3p%1Ol3z6;}wSJ_2@)6+#eRcuR*kp8-OgTBDfsO)GWtd-afNL8j*d#-T z&(1y8zNY}ZItPPHwitU(#!1V;huz7Ra)#VH*>k*GoP%SNgQtgs8!--r)-9%_q#pVd z+9t`M_iUkgA_)p6Or_$d?AJ(4?V7h_|4Z>$j0zCH#lZiM!%+vChR2IM@o=6Lj(t|b z+^|4#@D3Gngwne|V-JpcN)Ts|@*Dr|ZWJ0jBkH@EU{w%c46Hx#)JQtcM4?RW>9I!x z@(hG0YoK?#G=G%^#vyQG*5V|$-kpmhRC-@w6uL%m-ne-Qw8!8LL~3szCkPzplko88 zJ|_?(LEVV*01p*>nxNCVhRXru25wZx9v_o1(#xIA8nwh6yBMxve1IX0^pBMAuQ7~# zDE{}~uOK(CQvE$71i+u0F^rm>J5>C!AtgLpwC;n$T1WyxlU@ZJG2k)+RUcwc!ZTqV zhp9Ch*TUA5jK}0#0&Z9I{MHSL0!R4(>4ynpUkSPiCl)x=c6?s4Ze0+D#Zi+xzLF8Z zU*(Ci46~{Mh1ucSziPUm=zI4x4z%UHfm_8Wlnr-#M3-(Pm~57j_na6eLQ|0gsQFLmk|;B#**?MhqUtN@nGa zWBC`+l&wV+9bMA!aeF1W{>@u~xYjloxIX1^?^owf$`zlygt5frLx2JykCU82p<^U} zK(SLP2Z*4K6)k9)GHY$?jTf!GA0h59hDeg^yC`{xeCPrrBiZOQPvfG+*J%4)ej%^}rvX>B|f_T1q+`!SHLF~|m zj~j8b$GaI8^rG9V_-yha%0FQJ@s3n{#p#6rV%YmSv#kTlU%xzY(=;p|d=005^y`?r zEjVomyA0cWMKLt+->sp*Z}cu(vF&2f#jIJ98hiIn?dlTU|3JG*PP$?lYZ*9@Um>^B zWL{^r{~i#Hj)}rAwRU%RAMMErt*T+^C8w^Ve5dGsog$%v$HuSHi){9#gv8<=t-N>3 zPW|;vZ+RGM6TrMvIbNDSy!MHjQ0fCd{!D56K2?DQocyyhcYiP_a8fUoDQ_Kku&M6J z+eAMR-AzV$iMn+xmilw!1ESa8IVhbRF^%bu2^WX$j>)@-_?Cfbm9@T|QJ<%0O2MY z`z|Kvd6YW&&7U@>?WvI8ZOg}B@64RTc)!S)cjrasEtk-dBV*d*d43jW{_#*i@vlpMy$r@QE&=oZpZ{5{GbwI((!8Iu@*9Kx(bCwyC0%X*x&Hx4R#@}^ literal 0 HcmV?d00001 diff --git a/docs/img/typhoon-gcp-load-balancing.png b/docs/img/typhoon-gcp-load-balancing.png new file mode 100644 index 0000000000000000000000000000000000000000..412fa98f56edcf0d6033d7980e00f52d66fa6aea GIT binary patch literal 70481 zcmeFYX;_lo7dL8UgISqcSy@_|`lzLqnKNctR!)^Ej#!y?X# z6+5f#Tla63kdWAZ>7wN|35kt05)vC6wrm34MCxTxBqZ)hT(Y#d9z~rM5tEO6U7C!> zlU=)N^D?Hlc1!<=xew0ohqYY0AkJ?|yAX5_c6>WzgK)5&@$W=Pt+Wf|wbABT&&GrQ{xqgxIM<%6 zlfT@3pGzOpDN_IZs3IB9^0L6U4k>d|P!VF<<;7g)Bv**}4C9N% z;bJk(5ZB7Una3-;HYwM6t&dumj=mmEJjmpSf>${lb5u3ruRElf< z85gN=osk^n8LXU0Y6B1Jo?X_w{A4ZoYA(l$?m-4Q=0+csO(F&`TLXkFL*Mj>K}Ure zqT$s+q==DQl=4LEiTV1kH9>a6hIQB9DSggJHU#FvNQ&}I9`l4MH?F_59Kc)+YZ?ND zYla$hifDEXL|Q}g7P>hd?XM;WyV~ISnW9R94ZAjpV}w6OP4>_|lT7JsK=vBG6whjo<=AJVfZmZHRq{4SNQUyxE&Sahuo7)FUjCWJDnGis4c{FWi~oX^?xiU&(d_=J zYPc;k%u9R}YX24sH|R9m$kSA3u-qAF{uC@k$od!<6;{ZuOAnrqpIOvthVgY}y8pWn z+hN1;X9FHmb&`>UvST^Kp}{!b$n0DBRBVHw%i15o7c?nG({dU=maQFgcpDQ0LPaDIBY^O0I%YSA67@4`L zHjcs}X5}V(P5+tIPsg2!jH$s&KqxT%px*PLBr* z+sv19pNoI|JD3D`N0;|M1Gm_oA^*8?{yZa+l7AcGJ?xl~xvkF@bt@fmXSJWqOO z&3|;Mg}L5h8^--i-?m!Me_lx>pV_&=!cDkw3spkmXUyiNxI@kh+ax4U?5wHv1jLh& z*g}o_Gr73EXLj}>E&goP@t+%Q-mhy}mJoRQ5&tvT21ttp{oa9l^B$;Ue0FS zifpBBMXhl$QdItRk9UybTrFfpg+QzT%6axI&27m3u={^00A zF>{)T`u4D%O~BSGiL&O)Z2CxEL%v#@ zm}$pRz#1y$!?uzy=`n5R&7v`i55g^)+ONw#1nXufyYXK652BQ-yKAm>AhHz_+0IlZ zE|d7HRGYLxq_%~6Z}T3TGv#>%sZTuPqkVQfWm-Uh_f2_B2CeG%t2~d?+}BSAB>f91 z8gG_R6D}e44W|=!UunxpB?7mo_4}_bX=Q6YGfy5lOeiZH6E6|hC@5XG+*Gnlb2dim zMgb;#+O>r3M0vd^k7GJAuua+)cnneC&KzhB*QsWECm$ymix2A_IQf-5c-t}Av5~u+ zo0rpQrnYzQb4AXlD7h4c8k~GdV~-hpM);&6I@!vSHWTkdl(QZk>afi7Sdu}^RvDxx z48MAZr!XwPnJv6*F2^OVxm7Q-Z_O2WS8%};y5}w|gt>fZq9-Cd#v=O^ul=UOJ-#W~ zGcIp?)Qyjl5A%-gtijZZzwpuZZ=7w{^t&zklZ!g`(9?|PIM?@1@EI{E8OAD?@3peUZTHn_ba*aF8gk! zIdaTWT-g4a#9JI(*2Kd9n4u;7LFD2rKTO?H80-Yox5KvBgck!MNh= zXap*GXj`%HLw0Hvc25l)F>XBlqedl*aup+{hy|X?2_G|0^_=vPbBm}EYy>HR zbeKmD*rvek$ZVp^biMr@?k!*^3tL2cElm9R#_;!fIcQ0QngtEU1@#iAP&Z8zn6fqe zhxE0$#Y+>8AL#CnAgn` zl(du56w#V~99F~k88eDt=PxSv3`#2Q^w0Yn<#)2SQ@>Ylp=vYYhi_e)+7+&sX;!Ow zMd7CX3O#pyNV-mky-$w3biPaS+c;jNPtmL<@!RmNB2IR;@95r;#>C$%+h01F8z3=b z=kR!m%=L>+gM@-lnY(=D@EU$3Mk!upYRW$OPUjEL8>MH{Lbk5R0LM~-nWxG(_yuvH z_|g0kAnsvnBYMT9lC0*Je|5k;vp_-RKVFNFyp{LQ6IM>D#c!czKC6&qJv1vhxnY4aBxch#Guesk;nU5( z@rjtQ=zansxVnEQ&PK1btVI)5C$MmTkY|QzKup-eGSiT{B9s8PxY5n+S6Z_g9DIbW z30BlbJ4~s*?CE6kfUSQML!2yYFf}sRlqtM@+q9awdG&<1=Y$cMRMsvJkLGokgc0=Y zDqy4sGk1vkgV{*(3?KdspY4dLJ9b*Dt!pclIjJWZwDZTACw=sL;KP3WP}Z?{58Ld* z;|Hxd{x4p1Clu~%+CR&Sx&(DSW#&W%86V1ycwr%E=+1Zvwp~1VmD3^Yx^J^~G8!ZQ z>FpkBQL33WvtujulCPZ;);iT%k_9?Jcl%`4m$P2{j8A6Pty`IyMd2P~E%M-c>AAzV zyuPy@;vW(|gih4(&xY)7M8QRe%F75W!jWQ7(17lBB1ClI^q!t-bKR82->yt-MQJ_Z z;P`p+9Yxs$q2asH3sGe}cQNd~uWs4E{H*==zT(i_n445mnIU3qbWsRrf@)?7JGfcj z<}GrnjSZNk(}<;E%WTy9+6v~!t{O&j=V8!nb@WV;*##Z%#?H@J`5NVgP{6TXuL#;V z|54yRXe!BKYIOaT;v7+i6k3zuMT@yA73iaV%)k8jwj24aU;SBloaXqhyFN{3iE~DC zo$sKe{YLT(CUsY6p~6gs^DfNrvm$j&WnFHn*g4xizfKxY$Sv%8p2^JJqI{EG8OQ4s z`=GAQlWh+ReKH6f*OjIoQEg6$QI(dxC=v@9-1{7*B;pt;7D!PzFuX)iOXp@245Nkkn7sp%MyUb^C^yA4kLf1raNc!26B?4h%sqayW!`_-jW#pT z&=&A+(++q}-_@J!--jlO=yQrW8088^?COoa_I@u$ufai1Bv}_4mnq^ zHDj!=ewMliK4P;WJE+vJGpdp#UkLx4$*XhDr}EKLIxp%TWi_czc|*uuj#ECRCLS6Q zINK7Vb4Xk+BrUM<@oc+BUp&Y{u_CmkGRNg~1MWMe=t9-&qp%lm_TSNOq2JY&EySBw z2UU&;@+teT-p{ioBe&7sX!8e(?B<;|vW9l(Mjy5$H|GkC6n6w^r1$U{k!pUvVR=Rw z?%SU+ZydelPx9I^eKI3?Zr87gqK&Mh+#FVfw*{foU~rO~InuUD_jFuM3w%JY6STi9 zF~vSPXfdNH$;w1pmZ^$*8W>nlFMF(RSC5ZFL#~+xUK&|#Z?5)f-*e#9-g?hk@WH{m z`*uXFXUliq34>wVtSnu;AWF4ds_xNns^)XpVWNaoem%=R z_HowQEamCf=zF&2`Bvo-P%wLoF|ICiorSQG>V^vGd-P$tRIuqTqBp7}E|T%Y7k?KV z7{`mClVnd%L(FRX%bb0l2lnjk;Z>0rg(en%4hWM`O)`1_3EOGeQ| zHi7Qz(I4RivU2CN1|S^U2VV9J$n`qtl*SD4V_}}x>>HX1YDFUNF9G|0LyYvk8r`xD zI0JY;CF_+;B6D@U?UoX-0#WmcVCn_fxzGUU{cKFaoQ>YCwplkkX28*0ybJMxcL4|w zWI!H5+T$hetbCucPwmc)U zhB>mnXZx71Sh4%Y(yc}&9!#;CJR=#@WNZOfJI!xDUN$>R6dD ztLi8U`7Doc8J1oFrRyKtBaDC78YzmQ>{mWwOLPRcTd4+2vnDOwEotF3#IzCjdBuY*s9z& z_?1J%Zrla<8hl2ecf)PP%0!N-zm?iDTQSMi60sX>IX3S@q~mz!JYPp9zE^78xKwl^ z$aAWf#2Iu#=z~4e30fWW&;?{SEi_hbn@!1_PLFvTXwTo~yByBKRov<_4CWmq$QT!7 zenU=V-zC6IFFXyiG3v8z8rpo}8;eq@A%uxeqj`shoH>V@s^2lQM+{T%ZCeCqT{J7mqoG|X5fdMW zG}bR&u{N5%If#Jy+#;jFSox_gOJ7<+bJglQqWcqP3RlGsCE%sQxxGo*PH~aS;r)bM zMl%COyQR%PFGhK{rJ;|N3@D((W8tj3^my8}(uHh((1P)P=)}2W(dxlaAS~lgxH?Yu zpI1;$ZawTb7IrwuL^q5Nxt(=oY!BeHp8O@v$w?VeW$iO76_6d@SJ}+4&Bh)PPITmY zM|iSLHQGWlUm?x9@^s4-S>3C0f7_=~->JB1L?69bPDZwzpUl}8(LDyA?7%A~akJ&?QX8+|^gFKSFqpmXHjJ<;6vMB2R8)tsR<@Aq=$O%@67QM18sN z%%LTB!yMo&(d25RA8T@V{n9)9MW2sLY{=EQ%Qr3xlgF;9m!+k=2MswB5Hk^%$!m+O zWBiXV;PclzhBbcByjuXfh8GYGGo75YF&!lnZ9p8Nmtv~n<96vaA!tz;xply5tQ+lI zl4KntyqwcbGQzoqDJPsGw=*qIKkBkBR82h~A21S^M-ygZ7xG19g&`};2Wfc(*Mn&x ztu?3I2-_eD2SV`S*Zaxz8iN+@PXQHrG=VS3Yq*A0oXDYG7mjrG5sbW@WC-u_X!@B* zGr;9dDTMW(6^^tIa9v3S{O8eTfO5E!lW&Y{{?=C&3R>iCqA!uN6yt*uZE<#HufPxZ zX5hmmfk{igGQ&NE&!d0$Z0^{#!Y_5eL&=1AN8`9D)qtmO;4&xnP$_-%P-UC0;`+y# z`9e8%P(btXOg^pCHG1F$X4o&n{*XCkeyLmAiS+@-hVw;>!st`&Zuc)#VeM1lzcEJ^n7@}JbG%WDcBw14yY_wDDn;y^rdw*1nyO!pB|-kkIok0^&0l5@9ceUW0S;_QYEOsGYhp0LwErwzm(Y)BD*+jV%@d3-HZ}XYD!hX?t)`bO zep+0AhTQI}?N_m%uW6xqc1cZThMrYmb~Q>jd-y=;QcL$C)Pyol<~MjxU37~w5Y&zy z5o=!SY5a07DGxzlmC5Y)!8XUN3$r8V^2gls?_R#gp1)RJ#&SP6Qvw(fy5_UBC_X0< z#(qR9&f(R)jwzktJh^Jp4KLdC4^W#*50S4mdz?)2^MV4+p16Sn|T zxa6d8P^o%f6>yg{M~>P_3mB9Uq}MUCa}Z~r-Mz7se8*+%qjPckAwqB zIB@&hwGQ=x)1)A0_>mnJipTTYSzD?5Qfv*v;X;CYVjj)2rBGaUk{`7XWfwSOR(Xqr zE9Mmv*5z)-4I$rNYJFeC_YX;G=rx&kM!<^2HsVtr!Mqwh-5%yU19K9$r!DbJ?DWWD zYQiw{cM;lHW2rjyu<-e!k6B?4f8_S$>fT!{g5^bwXZ~nKxq+bfd$cxS4A;ozj}<-! zZY97U%$*`@zVi0>N;kGs=`2@z=s>2q@v5jz3-9x23>#kjN@u~j)~R~8-_Yq8x~u^< zZ8r9U4Y3P7yXTI>mzhAQ{{|PTK8N3^%S(^$HXw2#m#BTUq;QI!Nu>=LE}V86X0i#^ z)L-9Bf{09-N6e9mxAVNtjL1_Ov%4#j3qDtfeY2Z)2`?Uq<1xDf=C;WPG?g<499WpU zi4#4}V?_q20ebF3Aj_x=;?G+oeO38gYKhrvvV?@|+do}C)iGrx{_;3V_NC8=@t%*x zYZW2j+ttON_m?VBUdwDEXiVn zV+1K4E`_D)i+%jr0DmupGH&z07vafggbNBG%kH0T=x~`n*2CRVeCt=Hf~D34Q;e@m z=)onNlKaYthNNEEKrKfqGR(!F&-ml*@&*y$Nq)i^>M>&R_HhGGmw*XaSxw=9Gn+G_ z?Zy=ip<&$$m%pm|FV!QY`g5IO?+)@#%Y(H=%{kTulh>-kfaUp7Ha4@`(5fAg9$3zE z*dT`B{7e3rkh$mb$PvHK+-ZHe`8y`{6-hka6!i<~*RaOUj^J`a#D;gQhh1MmJ95h( zIED3*I|j^byb{?|-$0$sg4M7<;b}DvLG-HdyX7-&r|8AVAtq-yGp$G9$>KM(4ZGnt zdyYmAfR_ik-nIGT4ynQ|Xv6{|o$KlSlOq8<<-{ znd!Ez#kWbl!jT7c(k|TXtGmX*U6zgFw*tJDRf_ztvark7XBD*;6*izjHQNkxiG(GE zJt9a2BbJnQQQXax8x@!wyDA{`ZsZufrF%<71tV9JQun!GI#0Nr!xz;# zhuqbn7oF@ew%$)DbIJ}dK)k#%&(wMr1{|yyUF}4joi)FrBf1vRodbNc0?r*4%SEyh zlY8a>zJZs1;zrWVz|uj3AQrdCsH0S96xO7Jw6`#ENy3sgd-2m6`#u!( zj$~nb#t`Ry7M#1UrXtI^>Fw&6hCGqhFqA#)=Bha-x7ZNmo@u6FWx8i)&CPq_Ork%$ zI4Y$wHu`J92T2w-^DmH()Zq67?@fGcGb;Jg^wr`!R&3hac_laB-AwnUKUV>U6SnG# z`GuR-`S_3;Md42OEs^GjSKP-WS*#(+N~LD8QLHFzc|pGpw_L?SxI@}YTgNpIShe)5 zc#!FCkM=4YIm}+~V{TGb0rtMA)MqX-#ephfe<*)FCoT~oJLi|Ym`zc!p2$`YISa*2 z%g28ab?CJgEsCt|2K3|+lD&=Rq(Ya-Xx(gtBV*v*>E=1G{3s~uXkcP#%ln-~R@n0) z^iJP_wxj2q`2CFLlYaUk8H}Z>JKbpUH&qJ8w8r{3DxW~uYB4JpGOIr9#%MYwe0Rrd zBfGAeNFKQt%VVy9$`6CrYK45WZ=#U8U+N#EM@7pc@l)58!p4@yLlSi>rKlKRVOs>w zilrbKlyNV%Q7S@rM#emzFO%&a2(8~_F)UiOY>g>Ve=&r=S8jwSJeaWgch6(k#}1)GcOf zWmX^*VV}`t$qqh1O3>>IqRo=VK(`ySyApq~U;FCXR7P|57s6kRf3KL&kB(CEBT85( zN(TLm+{o&%DX%qf&&p#Vk&;8tYu#MiI@}e)aCJ8W|N3aM=c={^Jb1icqckm&@hn_W zz`{Q#oeUSpXwED)DCC>?`!@_2d3n$*aQ21uXcpKd$XW_PNL)G0v1+skVt7!Z8@a1{tTsP^L zHn5voet3-XrPDIB9XeNg9^#}s2zGk@WeJ>IMhoGqzvYyFkbc+kN1S@agey6u65AWs z=$ReZMQjC1cthW*0xnNwDQP2}@qQ(R-_^(JE{KrPw;>=Y3N zc|ub7W@JEdVK9FLw52ic^MpI?Xm1bs1p=wjv0B?R5MdSvirN9KHE4>997hJ<{^X3Z z?k~IBc@}O@PR?k0Ezv)YQ-2?FtXCsPwg#+ds&<6rknV}7Qo0XfdaJBu04i@Gw%&Q@ z2CeB(r;C>uFb$U9JnFFL2~rKcNK76xNuU;w^beORZX z?on6O;z~(iPBf6vWpq5R;~u!}zBTSNfC@_yrSxl5`P>b^tzp6=bBA@T3L^%U^sjbX z)^A#bcfHfy-7)GLCSYX0S#gkYt+oEh-6~mHvZqx?_G2&->4Jz(xI2)7^-RG(uiutg zZmGo%H?Pw8Jy8H77_y^F1p+~HPen7bQOihU&!o^nN4aO>g~DL&W-6!l1=6u+zY^{%* z>&h3;l|wXc8~)9xqZ?!?WXS^+tX5&R?hXg<&irm#gUj04s-zW%*fX|4TVs((`1NFAbZYOfQ{>X;5Sq7dZ+#SQhBEmn~oNTggg=idtVHFcR+%@7ITwe5Q-(xI3cTur$^U9i66TBWfSQkfT&TDzfD>@BL3 zeLdsMk9HAVA03!ig4N;tS9X6ic_7H9lojAHtZj{?0>b|5(4e9D13NpWW7Dh@h&1_4 z;~2O&tB(G8s@8n0?V|#JKFE`YliJ_WWTh=5uAG!m6TyA7#>6;b~v|AY;ogLHq;2zXCo@Q(_p6 z)Y(OMqhgdDh~UXVOdUY+D@8_E@%WRM#ctl0Ak`IipndkUNnOm08xpd>}%=B z?vw$b4B?O;=SmXGVGC3;C@xakd_?9DJ(j&8HoH(JquiMsY3_M3_No(VSKxE!D?ayz z&X=g9%k(jPHccb$WiIM{_;v>2YTSPI$ol2Zm*XRw!ZIahmx4T6J6qtBF9IL0TVehX z`@}j2sqP{9;}N056>q5?P9}FWPN#L%0(kgs=nTr&}a?AN>1 z{BByFX{sp4xu~S+c@Whz;7J{3>Jk5VL_Y?FaDK z$QX#nd$XnWi@!6BMVuFAK|V%KXW9Lwx6V|7n+Z(IUJUaB!*#$7rI|b9D%IVW_NyR{ zca(boy%TVhm7E9mGJ8AMoP4)Z;pyGl_v>}#hp3@XbAx64W0zp;5mPke2(kIP7w5iw zBJQ07v;Cwu1bkZXD(W8 zr{&UO^^Un2gUl{uiAT9{^PSa~%YEjYJJkc}zVPB2rkNW*(sO-XqwdSd6n?h>Blh<9 z0Y*DNVGg2H0&iGT3gMjaNM0nFB?VtH=^D5+{F~x2(bHST3=LV3un-1teB(|7Xk8bn z#3o9w@G>Uk_Gkv^NCz|Qty>6~$}PByd1&tyZ}ib_pU?h%%GuR<*2LxKrZom!YXEcM z?y4cR&6FDow!R7Tn*Qc8gHP)!v+jIhG!uX{X4Yaq6`5%|O<`rs!O#=0T+?(Z-}!S5 zKsazcTf=Y6N7-4{zc-_Cr|}~ukg?s;?c$W9MoEyLnF+d?Pb;%1JY$0P@*#VsCJo#$ zkxATwM|@~*EN_-QlVftsQ4Y#!RJwKWIT>Mi-eyLI+V7yC;oH!YMl`=|KTxr*qie?; z_nId0k|kqp@HGEy`WjC`PjqbgPPP9AL}<1YjOZ(@zVY;9gRVg2ydeoFf&7!*=Y*Eg zkMp_0lNIA9G`d923x)vbCPfWVeqpN$2W5_saHIapTvtKM=w-iTEUEUr&m*=p-;N`ImG^o2x*w-K5BGW=KbcSPhu&kZvw%Zm52zMS z;RXFheTOOprHf#arXMBtgd(`-gTZFAv^BLjUO;un%B@u(6=jo>mjvdvTs}Ab#%c9o zyUW_SjjU7I{_BQdS3OslJtvc*>30@AYK$k6Z+Hg{J*tcL=mTGC9kW@V>`pm%2_pJf zhxY8unlW}=>mRd$PGCAW%y@5Fo|^NGKh5ajT?!^L`+O!J?{C0`$&|A5*$#KyN>61# zQ{ekzHWLGU^}5YatMDMW+Aen$zlvDg(n-5Y1oZ9+QL-Emca1)kQ16c-tRJp(jz%LR z^+U3~h>QC0Jyg-g0MGmA`8%GEo0#c32rqSUNyvHP?&*t00SYy~`@o>j{C2e> zH1_U^645ANf3l5sQhETi2lp!fDAGzvYZ++6fj%UYUtJhJR*+))#P@UVp5jZ;r%z=X z%Hzofhh5W9lYJiM?$_1;Bvar)5G}UEtgJ(dZ6`0t!<*}T;*EzLiM}&GrrWG>e`S?y za`Gn$!=Fx;7hq_Yowr5-!i!iMajMv$( z+BF;)a=Vbj1*h@|l>tjuh-<{xXOwr#3YJ5X(ZZ({VSoG`gEL}lGDz!)ul{Pl=S8~j zWOl>WdOne+A8d}iPVQ+c=LdQyPOUY=8QE#qna`XjANL^E;P2>@^gRd&L&%)EF#9s( zD_uPYz;K>!9?bnA+!Ys@93i$EI*cHh!*u%~8E+^Jo4X;jZ(L=#N6CbW)fAr8W8e#) zz(6OG(@{EaX(gT;lF^DkDNwpc{2h(hF6id;(`D3wMYo&+$FNTXgoFmNkD))av%(8FJ0ExhNNdolb!?Z(?Wq;h!cCG zk88;JD=d**V#iZA=J~%&s$f^!%1lXR>p{CK77{#1n@;tK1BfIwzmC0s2`<7;3<=YN zqs=0o)~6<2y(*~mQ?uI0T6{1C(|&iSFDP(y(P+i!^H&`RMeNuE@-zcOX(DjiTdV*|Em)R_Q;lJ8wT;*!DZu5LUC=2j)UV`|mc5NDyIlBBj@B<-F3jT*ceZ?_Lh? z`H84&3u6VR-%N|l)rgRpP_tFuav*59ug$6g%kDcGQLEu+ub^F`uwJxsoHidWT?cwG z^`JPuNkYYMgJh69hg&SfvEkN>2|gp-pr8$7t!1D$(vI*wUi&^TnHEO;INEWb8o&jF z?@+bbjh^R7UddKcRO)b&>0A59QxNW}`=kS?+{*^d$dtXhWecVg)0NUeR;R5z+h#U7 zi^TEu8T~*ZTguk25yXmZFYIU-BVE_?r&CLtOO-tv77Yh9c<9*CQn1O1q?mD-2?1&L1C`l)2*vKeF=Rn$wZ&d!9gCm-OT@1rlR z;NM3TBexqug70DA$jI#B3$vSKZD## z<(3IXj+78>-V>wM-yK{ONdncQ7K)gZ7ALRr>FUP7hW)}N+c$z%qtx!Si>DAA-%$Fo zbNr+F)g2eoyuDqbdn!=P=H61TO|PwGT9v!N`{=><&A>_8C-82;Sda2&uXxk>&_nQ^ zLw_M>t5x2(`IMJ_(yPND<8>1#9&_CT-(}|al>$zv@~3mH{e+~w%IBO+a{7FfvS3aU zm})j;0%4hGwA)v4&#=8q3b4Hs8?*-%MM=Fit2v|BJPR?KKfCLEH(}F1+Dm^8|3q+L zA>$kh1&Xr%uHNO6BLnKG(b|SI6W}=RAoVttr|D0tJt%j8euX2j=Q5z0y0SH@KbIB- z_ht_}O-iylzBBR)Ratz$WWGT!LIPy4BPS?wU=OzIHK?T<@x2Vk>xFFpk;*u56h<-j zct;|bVk}a2*D!03U?4KI?k|yKr~z6PlN+E4*Oqe9mga5xff%b+jdpL0Q&a3&Gz~3{ z$12tHw?lJhM#AQm95w-P_9yQ2h`KY7mch-)*XS{ZvfyQt;n?ZJVLu%Pv*GPG2F%L^ z;=QJA-k<49*@+4UN5Zo8n9qUWOIPUv^%@4<@qWh0az!<0QZuP+3$;FBvr|NsY$Dit zKX|lvQTd>Dx>5x+z+t}cr|$+%*VV@Q5i}?`5g=Gdj<_h$o)%*HSEN}wT9Ft#G^jqt z7@jYkTMe3&!&xKl@zwN)EzGYfIg|9xWCW=Zp_>L8`1r0DBXC;h?cn0dfW*p^5SP$J zVJVKn9u@(`=#Q2MZvWC;Fx2vcF#@sd`Q~k`C+6qkcv!4%g_+<$j;?=<^RZ*5X;y9K z3ky*FO3;Kgh_zazz9_S=U&#!nOD|4V#*-dO>2IL|kU*=FwqM6ZKgFyPvkPA`v#=N~ zhxQES(erxg<;&RiY#<|M{KarJUpvf9rp9x1;aU;?;#bqPC--+*C`OHLcCzg$nfI2@ zrq4TS7@2%I+A*8MeFl~vmaEl*q+8p4$Lapcj(%t3l04M4n>Yegh7`QmU7@R-GeNV4 zMy!5`$a(9@hh)fnTT?76=lHGoiYDPYRv?3qO+d4a-uCMp|Kgcl4ES1{Y7v@nE)R{qfUbuHgg z@c4e4Zqlf&?2UO}s@^9=5rx=x7#%5Au`kQovFAk6{rpM4eXD~Casb~~VV1f=A` zwIJnhr;DbWpzmRcgVasq>c0UX#TE`f2;?U%P--Jz9r#KDe*BN9#yWgGZ8s zo5dOBRlH_K}hy}Ok&!WCcekf3Og5j4xz~u#(oh6 zC!TZnFryrD6Z#Fe4=0R=ddryiRU;8py-A}$oiVEOD%v{4-$q(uoh=ETH z2^?AFS3sGY$6q9INm3S1a#YK~97c+XahwceDkb3-r0mF#nY(KYSS$x=s*6W>k;y&Q zL$~V7Gb36wGa^m4R!?JA$I>pzKc#)fE3x!=*N=&7+7YA;ECm7J=bL^2ReW6jI_JKY zw9W93LdKo8fSY|J-0m9=i{t%0v@B9@C^wa6B*mU8Y|1=dF|FEl*f++ihj0XD%ei^}RGkk4!K16XA`K4Ahj>o%~1m{b#E~~~Z7)fJ+^0e1Kd|vMRXqZZ4 z&s3?u9sV%V(_SGGq%<3v#jp#a7rUKOg5MN*e2Stx{Btzt0zTA2oPJi*((dyqW%1AU z`Q%patsq(%;6eL!!yczWIq{8l^>?M^AwSf53*k#!*B;ybDR;KTTqe_{ z)gajH%vjibqg(hR_NMV`7Q){R=*jOJSu8s3OBvzzNdUAt3ycRz0W#9OCO_6#oWOjP z>#&ttS)5yLcWrQ5QM+_oUg5;j_b9?hwNB5#3jl3POes2nI*s@i)r3+9Ft@puIH@PR zO{_1X)$zE<*bAW6iHik82KS9|*?hpkFhCbql}RqxT;qDE56i|o^h5_)NwVHDLFy?G z9}s}!Zv;$2_$n^)=!dYNr8jkuBUlV|Pf11Y8>N@NQd5X?U&FKs$&KIL+Pb9Y8r!E) z+IF2G(&f5lGK=Vx-FcYM^Q+;aXl?C-Tc~p_>rFq@80@TpI+J@Wm-;VU(f@s{(soy&rD%QLuIg@oJLr3Lq1YU7b2`NQ4T0aYvt5Mz6 z(Yf;F49rnEqTA}|r@9ASFMOMvQ=xpMsWxVR29^!Za_~nV@{^Vf8uK_K7RC%iliZj} zClREKEe-lRNbP0AqR!6d%m9V78F91Vkk@kSP`gjD5oAR@<=OmLd^6JnBU~0e$)kye zph1q7_8QE#i)!45`}Ii1#WVOlFoa~#-n%XL&cQbTX_1R$_NO^58`u%b5kJZy)Q%s@ zDt^iyPeoUNg430{>V!#s*X-X6iIymZKUu5jzw%F)911M;I2-(MZ237!{r5iI^e41O zznsJQyU{~GYR~p0MTE7E2ItdAWn;3XGkB0wPISkX@$9^ufhTUm*|pH)=KSVMMx*Qh z3VYRwFMr!Zsu%6wc6JN(_|JRm;(42Z0($=s*-5)oo_wynUjESwx0<06ebiW~vs1;m zL#JRdf$Dj3)lJbVj zD)28XaWQng@F^xQ#J>lS>5S^wNLXCtsQtVVC4J0;4B3~n7=mnF3B0Rd36u_R-ew4K zfy+N+MI5aiL`et?B#wjb0ohH&wOi3mzGtqr>eEiW{Ia#J^v$#PreH_A%9U4n1YAr= zy6!wyJJk%xbkQQV&tp6e$3-r5M~Ag}OU}zRR-A8D_xpTr!zKX4w*a8vKfMwq7l{~0 zq1h;#eLcI(MbG}~%HcSkDa?Bxjob5hk_jYCQSjjq0?xQXPZR~Fr-UFbk{@s01Jri^ zHkS}7O6*kz2uUpd`dj%3;J8VYC-ws&e@9i!>2!uUuz!~^x7VV8%GcvAD6BChDcG>Zex=LK4{qi zED^Xn0E&~8_Kh<;B-%HV{{&=~F2_0y!X@cJ^}Rn59c~3%(%;5K%7Rslnqk3~lY}Pz zg0oZ~ZF8hO*3FsY;%Bv!!7lrx`W~E-KKvifle|;Qy#o@@Wby##8Ivg&WLrY>}{^lbb0se9Y)&cpUtFsYX3xkNudu)xIvK;09kjZSE*KG)u zFgyX;`7p`adJMi-Mjm(#@Ui~G6{|b6v*v&Bz5YFmPp3?ud#@N@5tj&^t_aR{@{U^ot zPs0Ex)B$eVfB2#;sptOpPzV24isD@AA=7jJ2y{;)ep%~Zirzo({vY{B`GwWkhEXB8 zec^{S`B1bxS31GX3y7eV=DNT?tjv2^tw%U3T+xF3gvfOv!t*QeZCXXF6xAN1`$wRW zBkKBpK6lveZ(_Zyh@H>|dZ6fcftx9>{__LVfVf*#YJYdD=7;GBMXq$B|J~L<|NMXP zG3>UFJCyy8ib<49+$sR{_HWkhy{w;@xP$jnhW;q;pQptUV9ETq@n`4$&ou5<=867L z3$duxF=Y*+m<2@%`;^z_;uHJIxNECSbMu6~EB#Xh^apF0|J`d^Od zUB?4*HR1&{adi>T5V5hIqG_;r&k9mB<0)PgGQ?{bM#O*Uf?T28F5SaS(R>OjR_-d- zFJ@Tx%|vii(Dj`D2*-nEAfn_7!YQ9!-H21tx=11LcTuWk|D9{eXz2~N!E!hG{WD<+ zNosyfX9XpwBe{Hm{fMIDhuSJYT${M`UZ~A?dbINC;lG;~+nl|tM%4%1QXfj2#{vvP z#v2|W5dU0UB!U>~a&qJE%be^(E>qXD#{mM9@NnF$CPh2Hvrdaq@D7wbX*3oAJ^?u5 z5p<)#DeJN#k_gm*3-E3M{Jo+MRpM8mtoQGgFMpbPE7<*6cGb&c`NHf+kqPmA3w5PU z_@5bEE&3mFsEm7-%?ZK+z=?bx7Gs0~xPGxEu|?>#z-P#V34F|2A4>3Esl@IjOaUxh z3E@{Y4(9qwP8VaxGaJjijLajVdnPW5NBT;+)_%F4!X?KA%^+pjZ~qX=ikb6{P4(TG zOHtDsKmJcLd7QvWm&-(`$KdCnk?o91+?P-azumkz%H-U6XxvI*^3*_0ecX6+6o3_+f^4M{~vNOY)&&|{Fa%O(N39rCD=@l6_ zPDCUWd6*_CFBlJ2sGSYOZNWHwmwkqneql_X(TO;DY!>^o2>{S44ZIU0qAc0D$O+>1 zkE`$VJUVXeR{BNo6as+xtFXrNmqfD-*8h{qRBBIN=+9msEctm~VV47(vGk3RpU)K$bOwU)9P-rTEs&ZM56?~rJ;u% zcY7<@VLW9fUGszw+aFm)}+Fv=EO(wC?VJG*`~)SQxWU zT|SWV2nLgtD;aMboZ>gchpk_vbF!AdGPCVj(U4?LfpY%x6M!vG_Cnb3~5eK@a!8V zJjofO*+(4B#ouM19+*k9GJRWrHO&7iEdlz~?6~@C<%taIiKq|J2`slWq2g$beBmRb ze>v{;c6Lu2`jR@HntA@{OZyV;aq?ITw*hKgFZT?br4mUb18A<*Ma&e-gQLZ`Q{@~N zii=GVzL3jf&IdVb^A)xO9+sIbXbNDFy^4!Y~~?U==hH%oQZ(VnT{&)zHr#mtk*v zH;y2iDAwjmjK1OItknLi6{IQt9}yH5Q>_NBT22BR(RYhTw45R$AltHscZH@`GYqio zl%)S|`0n_!RO{u$sEnl5%g#B2W!?ic1BY4v>#MOi7^F?}szG#FEEGI)S=GOd7@mU; zU0SqzF=KSu!%Awg=z#+zM2m=oNKzb)Vmz)l8&#}TtXkLD64(Gw`2Qg6%fq3J+PGB| zsf0>p3klhxvM*5)ibQs!NXEVl!x%$}kS%3N_9T@p%-CnNU@S3HmeGtY+t|j~W(H&V zp3(ch-*tWee1E#k^PK10=RWuSJNNQC#(@=g$@kxChxeFi_uw2KI0;v*l_aQ(#Lrk@ z2!OHLNQ7sX9{C6yHr(n1Mt(N;P#AY2VE^MKD9`&b(DBt{0?*f8?5BlzI(qScya{y| zfm*vul8>-yMz)$*zA!9 zqx-^tkyFSKcJDn*$FkfkGKc@~A8OwObYlQQ87k;c5PCyBY?jR7=i-oKM^!(cERSMAqHi$b$_WeErYhg+!14h@*}UM?|QO=W*{gbC2g5GqDt> zNE#_;1ra1l^){Jk64Az7u?7z zM1Gg9fDI;-TI{;Y!}#lPwz?^pM{eR*U9eOqzyShc5BrnUdX^dAN|V|l@lVxj`faBT zWL+lErX!i;Zc`YwrdPoNqxO<6@K$O`ybnT~r??LZIRa8IzhL|_F<)JmFv{}ejaKcF z307NS5f{u=8)^PHxD35!_dV)rtr&_spJc^=_XFWz02|S9az$|7Rf)KqG=< z6MA&kzZP+ys1vdHA!&c?ue_T1_Qu*Ww>njsf8V;#RZJJk>x%3D7VK_7oJnGNER43OF}VJ`gJ z8ehBoWh0^NeF`AM$(^|eXVL#%FSkyH%Kz=>0{wnderyT z>)UEr*N3V%6s@TY#g`^^YkXY02>PulB?Pn@-#0U=D3{j{>UW&0O1iT^I3#7i5a5L!F5orC6=kfN+)`MsjVa^$j#&Nv!hY6_ck5R6 zdZuY?JFAgaEBsTw_HaZXLDsOdIzYY3SdsQtfL(jk#vj|=5hk<8oB|x4yHWK*wzk&I z_LHKX)4}mq$~&gon{s`rU15O}t*QXxU-J0Tem{3qczt_%Ijx`e8AAsa@yS&1rZdx~ zR8gtP{<&z}GaM5p>0tIo)(8;k2>)G(*7wz+*rnLxLpUcc;>m_f_OMhx(zbnu4QX3H zFTNKM;>*1|XeJNFt&N{LzmY48JAouW&a^nEIBxvDQ7~@Ik#={KYWvRVQ~oyP@q`P@ z_J<&^A+e}+{D>?ei@Ue(lqfOdRY0>QuUi_~T#t#(@yx`O;JApvnh))431s)UotJ9s zT6xyrgPup!_S>IgHO@!lZ9t+Yk;RfUb>okTx^vnB5+!OurJb*ii|qh_TAQ0uIHm4Z zfDU;0s-Xh8!h9dzsJLqJx09W6c$e_liF5BWQdysInCL(1?|znW5u6b{!6O9_i~P;v zcQ0Mh0@w#Yn4$W%#>=ykBUy)ro{b9C)kye{!J=dQKXs+ZI1}QF6*qkYyCR_u4nITu zA^r+z&T?6TP#RROWPaB@XGk?w>nu4NHwQ#v6sdEAxn_%fgf;bWAiTTQku$L!eNEA9 z``9o~Xun&q|Ere*+>+1pR;k-u?37(MO-lF?);BJ3KAi!Iq!ept1~G=e>#SH?Cf+_p zRnt9(KfxL%0yOeYr#Owo_?$rmUqJh|ZPb4drT)kuDf&2LNHSfG&sg7EJatbG{$fyc z+n`nsj4f(~Dn3ttAjnI*6)O3KzB-|N9xLAWnE!`Hb5GnQQm+;txiS*kX}=%lXt9!s z4H$tc>Jx(cD(mz!73`7@6SdJzM?T_*%O|7tSKcaeCrt*sMnY)?h~$^l+8Ffly>r?A zUt9$fK4hku7&rn|rg}l^Rom4@p^e8(KmK;=^VqeWpBp7wK>FF&E}XXj;D_=Rsm~Y@ zC~O*~rZTtbmA=cfqFc9wLO_3K)uh0?b%+@$=E9z&lUl#mM)hrKIX+w{swFc*1+9L% zx(hSJv@OH`EG*s1#sV1obC`V{sH4(HeYkbz;wbO05MYz*L;IN*(H72(qlbyJs&37@ zC@e2azS&Zde?@$$&7;*qq0Nrd7bdd@w|1^+$n<9L+?v?VoZncCD<^kz5H923&4{{` zNmY{4=&uqjMtC!NB3Po?#=TiRL^teD=We(m8yvnn%wQ~IMr20o)z-JpfpvLg^_YCT z%)-%`v(n#G2U<&T@Z}Ge(B#7teK2hbU!_^x=b&)gr`AnqWKxT5PW8pnxsY<#`%b<3 znuJ%%!jc;Xu}($7pL0WDBCkeI9wz=Rp=2QByiV^?;eFLJmIKALq0Vn=_CJ$-n!l}e z17A)DDZW}1VG0f_OWI_l`PDPur2hs8LC6N|GbT3{?5p#4ZbU+tr|5q#7W36LuYmWn zgzz8_uny%0m<9o4Prchd(Ah}(s$Z@4@Zs1j0c8AolLD`%Ag_;^^fbWxvu!CT$A`=a zLn9VE5>Wq`tTb=*ExvXi_)~X*lc>5Ph(2iCq+mi>dFB{Nx8SM=vRw41m^$k9Vyw{u=eIU+5j(T2xJ&_l0}FsH93)~F z2&Sw7L6yzD5R2~iVsJBa)(ikSW&kK0Jw`TPmMw#9vFNJYwsN6K-FK(fS-(C{vzRKm zA`A2u$?kqm9^IN&)n$cXwm#~-U_qwc+qL8Ox9@A0k0aV5p&_-1x`&hSjyi(^4N{XQ9;Kts&^RhmK+MMC|7N`HM#_w*^cVDh;= z2|Ss9`ftcW_6bVV6H@EK3yX0tsqr8Ie@N+04|2t^XydLP{N6i1jBLtAfn}z+N*+H_ z7U#e)p6o#pu98PODnTQK>0^1B`HrbP6yF*i)r2jg%bu}i8`IcZ&9!v*J1J4 z2^xgg{<*~WZ_R*MEGA`i2Evc*N^FA`NG=2ZBb8^CRn6GJ8_T${$~Q6}QhNngA5;d{ zr{(557BmWUa6hqmk@q>?<{3W{cW*x(VE-E+I>FBWQ;;x?Jsn2)0QuPE4L&{1Q+To* z>-$XZCfybOi{A0!7C6tFBB4(hT}LIYqbW8ZYuknpN9_3J#uaM0?JKJ}XOz2ZIr-}P zukZr5K>`UYXizoqFIr{xV|RIYcN+-hTCcO$uDQv8btfAn#;xAcc0F9$A((PAE@B$c zq+=9}>TnKo2nH}ouSwW3UJUa>8tK~|u|~y(GVc)EsrV(}aB4xgh1W6hKDL4AqQ-PT zPx&VinR5+?*(n5<0z}-*Sv~l3zKpIiEUzaO$oZ?VPila^2iE5HTu8RBev))DxMkh} z?vMynvba@aJDZ0x`p6CCv+s(S(~k?b-#3XezH6ct_Ffxuf>-2eaa8_xRV4HV0iuI! zl!o`j(0LUAyYF$m%jQ|e`jBJJjnWFwlBx?{3 zc+=s@B$byTLV9{f%vz^`-&Zi5#9L<}dQxifZjUX7AIW1O4y_$fNFn`Kerl|gjZ;j% z<9@Fjl(LPo#k@wR1t1^kB8HZ4Ap`00`!MHBP>{vZa0szQw+@) z4K|;rNcxS=CjOhB7^`00NmufQn?);RQS+i{tGr%~K3AROfN2R?!)kdva#HMkg+eNM z@OsN~ov`}ZhymL^Dc37`b?m}0$p3`DPALaY?p8t}x@aqWz-?VS) zIx~xM-IG#5>)CtaL5Bs9UHbaCeiup&u1TL~bkBwkQTAcrdEd!Ym)@?cpa!F%Uro82 zXSs-MoB?O} zg6|6efZ#Cv0E+KnW4paLH0}=x-HwG%l3bqu(lSdSdhkX8p&fc{vwX9ndE*cy^(4?d+&F$en{$j7+#nk_ zXg_hXpsw8FHS7F7{6*dH>v ze(ku9KOlv#0F#%*Z7NqIxhEcMAL${GJ0IeZe&gEWxQ{@U#I&ioHrAx2e08P12L{9i zBLj4G^#g_T^fw0_DlB(+o8chCiJ}C@8C=kZ9^;nskY%Tlh-4c)0Y;0cAaPFY^(sfw zrN+099~haf-v=ZX_l{V4OmhXzhFe?5fi{D=+iow~Ix3=ud{S~I(oBCV`qxH{xIc7U zcK#L?moHqYw%?b1Hq)?Es|+WhLtg(C;@CnZXmQXpBT7pw78RPTHSRB(uzI3_aR6S< zjA&@5Q##3M?%Tf8qSto0+1LQW$k^Co-dByM?erIp@IETe7uE}UQ5+^mgVbI6vsi|exTf2uujzA*u3tN3 zDq`_5NjkSFsf{G-(s@Gk@8#Q`jL)POBt(+gw@si|m^B1PYGVquhNlcgU7Pv8)5v?z z49m@u009Q>Nv^opkXs9JJ-aJKDEXR>Jht|b4cI#K6|}a;^vzDO3kuPb*8VF5?&`3C z)-hhm^|`H(2$|^OCBWQs^ps1b zLxjX_GyhTnp`~(D<|_ zjvAvske% zKLGvMu?eI}6}oTBOM&;F++ZV7$WTDX{< zg6=*ig@Cm4MZ*6ugYNL|-fBWCLz@ZJJ)d`geOK8qeR!4QM~a&2dGLYtEoo6AwU*~5 zv*JGYleT9Zl(U6p`iU7I#IP3em85-y5Jkx&qVgv6%-M{V$;MR9LlCNt32TX`=BYyKR$E76B_3Prq+!=?I{MuHEJs76 zl{B( zmn~WSb_e}9Qc-l4aqQn8M%MybFBJwijT>Vc+~A3k&?ELA%#BBb$@L(%r2;_YOJ{&H zznsceMV z#l#m`$c8W3$nm6#&flvGf(gFfX4#yijtL;O7yK;v#uobh!Zyb@#%M%yBks;9>G5GA z`j{I!*c#p;J2eL%*1FM0XYmM!IPbNkhD`kg4hF*wUk@rATmp#P#@)EGPHs;BkxYVm zjxR*t)1LuolTX_L`K;U5xjx(TziG@zOFgy@zntTIxZcRKQ6RLKE$0eks`!x_{!{Dl@*5xg1edRt%ZtA&54%$_6xH%7I@Fk2oM;OF-aN6nankPYeisd3@H*<#Bh+7f_Z2xZ3WN{Yu6l@Ho#0LVxpGZm;y4a9@-F|Zc7Dvm@5DpN?;g= zeg9|xgzzu-aU1Z~{W-r3afV_MPCU6O12G=N>JGd@iJ8?ZvWwo+7z;sVMzn}p01z!_=2r7($iK*vJD z_U7`Ul!gm((|%ETT-hZl)AM!))kRNOFZhGSxR0<&JrG7gRCm6&_7BJ@ZmUqQ?X0bO zl5|dVAG6YxC(WtI(Y4giV`y0Q-R;~B`2e4uO#ivsl7O75&x}3Y76aMgdlso??leD> zzM$qln%j7Fj_q$_VDDuD=9+LJ1p!!Wg7g>R+3Ns}tM4o%DavGgvB+^hUeaVm!h!bE zb%(4%lmJiElk9ipJ+=C&y|LA#rN2TcD*E10V>UeGU4?LKV$i4*i(k>Gc_kb3@%8^h zu;})Z*XfUp3voSHbP1b^NOPN`r;2rea$!FAG4&&7iV2Nm00n=Qit0)snB;i@xr(v# zu4&uhOuQYlUNxb~=7DBa=ci(l{B>Nk$+(D7BYOb^flImb7Ze)JtH@LJTs?ad7!TU(oC*ll}v0ql)Ke*rAU!uv1RIZ?B~<-9%6LP0}B?dYJGjXNJ)*K@^;dTjfdbG315yLbWbT-N#D?57D( z&#RxgkG$m&;dvb4QJsD^@bNX+MfBmTac^EHPX6k(J?bo*O+xHAWcRQ1!O+8!x3(iX z^TzAfKHTgaii!nXp;V@zL3$oVvEt@~MoavNW&}=jNz$L;0KN}&r1n8JZrrt-Lazwi z#<-tF1T?bB7RkQmwI$52LKwL&+w!Sbq9BiKjac3;@_zR{# z9!?&qSA#Ejo4!sA47JN_vEHa@RqphO9_E-|KUC^DOxj}Ql^X#s0;G49|10dJ`0eiAw5Uh}fm>ZcEJnaTg2i z*bx9)xq<2`7QX!I`4^}QrGTQ!aaZFKA7sb5-5|dGIPj?ELc3!&s3@but+mX^JIPQQ zG}YTWp&`|!P|EQMl;YqY^#cCs`u9GQte2dT1 zfB!)Lp8o^fC(2Jg5_(76nt#g1%a1>QsqiUwng{+T%@j1pij{$315e={> zNl2Rk!5ZbtobO&qc_xa>bG~`&v2Xa8*U&nUML9ueWR^lo(#64M<`{dM z;@vsINc0r?~gD3cum2ki3)o078)~6l;dAJbe#TqBM zO4-yY`ffzAQ$76H)3RnHMez=N5M+bT6?66%cmYK1NzP_3<}BcO&DOaVi`T~WAGvB{ zD4dIso0jaGCMr)ZJaClUv*T=RAa0=g9;-WT&P~w<;-n92FZ{l$*Ao?jDh*Zav9Jf@ zjDi3%hW!zJ9zb^=Te357-rQZ(@=adv4e!@MCi`_xwf|aixCq9Mhf;~tFFqO<+I#VT z@!2~U@2!J9z!W8y$p`~-6XLPB3D@qSvhulI_vovLry_CHSaVUL#h1m`OK%^M8(b1| zj-GRiwK;AL)NHba@L@L9w-}T)lNQJh;K&3xiS;$MxwG%Q8iZzNgPZ@rjI^*Pc`Lh& z!+dv-?jN}GpQ{A`bp$ebqYFwpB_{84(|?hFcWv+{Lc)`7N-w4l^#bG{tgANeCv7%g zAf;Jt#hCupdc8o&*ELfxWE-b7K-9^aSp^b%g={Eo)&gHEzRlspn`&s%9J0TOi!AgaFjOilU#kS&uL_~^fH z`wS+NPFjz*W|04lU$ba(g;4vJU9yLF_dRR60V7_AR38tSGJ7w8byM<%3mP_iVxvFB ztVP&v^r^=EylR>u4q0{EKTtB}!A>!uG6@$Ka|_!@s~c4sdsR>CM_}%Hn&nE$-?H-< z<3Y{^!%}^fUvz>%d0(xv-wu_lz&>9wmMb^%zw^kYBgj450#+Pm;a48`*#YhzL=ZP@ zO@0A41xMVBr0><-g{VylFqz!*a9o~>9y5-elB)V0if&p&xYM4(_GvQ-+8K{Z`fb?d zfbs)_q3#Q0!vy6XVF^~D!Et0oI$y4d)?(KA$I4-k?uM0xUF7xv4o^V0__-0?FFnl? zU(iQ*{CTQEVPTem5)ZUxeQQ^%~%u@Yk!4v=V6e#L0>bi#SG7vGE>a0}=J7z~0d=*#K=C_WeBd1N;lbdk3UiWg z*IF1+sKUx1^)UqEwhGsb#qZQT83Lj3HTbktnnm0O{4@vZhPTjAlT{w1voZP1yyd9v zYm3DY?YCrCVcJ5a%N2LDwrgm(;0P!l+8uA^JaCV%e>&~TP6F|js}9)-7@jSP?qNfk zh=SBUoGX0G=^drQCeT-ML2-iy5Zf}>;cmTmY|l`q5!A|^%}J-$RXq~ zME^6W3Y)$lTftH9^MfaDZhkEoBLrc&eik4$K8E7~q+pUpiJGo&I4Lx?u|IbzSuYP;?pj-nV&=)K?`jvyG zUJ&b5bRiB_>$Law_`?*XdUR-kwxy|l-TPFbvMEoRj2+yoxs}ta=v>_F?epa>H)u9z zYHO_`+~4p2V+baQS2{wEx^El9<@ORQtXIP~IZ6~dpWnl%3Aj+i14=$(-lW58h|h7KtqYcmUd-$Xw=%>daI}!i3;po^@H5E z7Iu+DSdC<&=XQ}@W+f-3D4LGnvA2z)H>YkNRNWgjO2y$bp5AJmBzF&Ko^Xvw7RPtZ znh#0NlvR-TM|e(tyKkm6=plc$Eb8g!wc)q2?=xN~Uk6w0k4sQIn8F^V(mVb-Hww?s z8_XP9(n-gc8xe_NjQcA`hede3#6aX?~I3n10Jr1y~Z0C^s3baGN&2i@xNZZCOR&siG{1Aeb#Men1jkB35f|}OFQAX*r zc21=xuhZiwT2c6c(0(0ihMh-3ov;E!HDXoOiGqEDz1-Qa^)fB}leecsF$aqNnlOtf zfq_Pt0diBf!i4AhRyKakNFQ0lfEb*sxwv9U;Q%x&yv*UJ?ODbI{^)*ol2IMkeu=}0 z_ql4y4twuHvD0EmoH5ueWw7J~QO6erO73RV%Y_r1FTNTGSpJaIgV0c%clg~PQ=zPZ zoAJGKgpumLt>8&1>9z)5oGZWt6Hj>)&K`{P65{dcB!7Dvq;M{em1_K|qn{Nwd%X8; z6x4g{y-Z5&vU7*Nin%eNLF8Yq(fuWc=F9COJ^dg+Va`!c)*>yp=6a=`48r zjnn0U3fC>Q;Pg32R88T``5eYcqE9igXtyvbnIhY}Pn|NfqH7o0K77_1Q8$o#k5na7FU+*aI#Pcouac%P!ArJF_3 zk57CvAE?XbC@(~>x-=wVx+BA63MiD=dm4*UPC{5pY^0&`)lLht7@?_7e)*T1VHEiz zn0q2y|HkSBW>u$ymu{<88wE{A`(#Z$kqj<%npaHRSKJm=Q{;St>Q5pc_O87ge}kHr z{@}(J;Y2%2Xm&aMq!;Sjc=kmZgT;BcKr_##4i>wzeJi{~XVmY7>e`xckC`nvXSwz* zgxMx|$)YoXt8@^GTlK$4W!NYH&T3ZmRNlws?kiFBj*vkMXJ2D56SxNEBNp7Q+RRII z6GLtf)0;B3`dCI{Y^*AE_{$CsJo9T+&QCsOB{jNSG2OxJRM)9I1@e@Kz}q5TBhP(j z3puOLQBM-fIw&PaOHUA7WGDpNF*wXb&hVbS7Z!1^F>Z{KAakqgNzbd%cQV{0xiEWu zO=_doDDBbHG?MChr2Xr{wZgS5S!BORzd*C15)n-k{(BmF0+hUUa8yog&||+wIL^*Y zcRhGM)nIwryhKNSVj{}-19&5>^u_%0>!V5815UVA-~8gAc>f-89BJM=9(!B#vU3zP z*P0ZuTz4h-u6npHxmQ4y+yf1MrCig}Rq~ou zx9F|E@ob19Wj}L@xNJHEgH2}xzR&X&=ymkYAXM$7p(i?N3EH`@%ir*2>j|+WLSGkL%-eIw5 z=vMB=^Lw{TTpl{aZddhRa*KY$A+E<=*($s!-6);Ys<9qw+YeKYE(gikyPTi+d#D zeWr5DZwjV!SR^6OmoZ*TntGxY$PT^fgSn!smdtmWFPQ6RkmATy4f&Yrg-)hNIUcGR z+PZ4f$;;hv&nQpX8f<8k2}}od{t))dGo@2$!mDhG;)8bjni2G!?v_z6oeLY98dgpY zIuT^MQQNZSwB*pzgjG*$M{~hFBeJFD&K*EaMrVZl+o?wjj^h2b>bR)d%a{M-6$MJ~ zJXJm)ZD0Qdaxq1y{*A=fZ4JZgj~_aia37Zt;gLkj|ArvSC}m}YXL82USZRY|QU5D$ zrwB?fu8e?B^AqK{sG)#~Q1^w66hjV0Bint8yyIvC0Z=#F!B-iaHHE!`X~So?Je3M^ zES~puqLLg@69#3NkkzsTEmtw4+`P7>v1;@8JZYbbmgAbAR%S&*qM#pwZjUjZ;}6;_ zg)AvI?S3^o3G58Oe>!|ok>Hiole7{}ky1Z>$fP>3?53S&aEYX$P;8ls^m~YuXXk#+ zV>UC&xdLQGd8E8)4mW|MBpF?&>YbOV64WlOCpqcyq&&izL-f8Xm=i@7 zxveoETG$Ef{{D&WltZoYqtOGKRd{;vE!XXB8XMe>hnHyhw;?RtvMIPhJ(cDE0l+4% zHez}9Gp-A}^{0$;&ZaiJb|;4{7~@lc$kH-iKZ^whzw$=agvinQELpQ)~f+G9y(%(jdU#sUR7<%}2I(NHciLPc5v|g^aA$>uVlHR9t7azk-$Hdw7}47mT+S z5H1hA?zRv0tfS9thWj0{&z9jO`evN)0&IKp0oXO-y5=G;QC=p;SLE@QCav;a@z1`A zA&h<_ea+At&P(LVr}(4*PWjBj}8kupUx(HkYP$X!P0 znvE!P((hoRP^M`qfrR%{mTq#?DYWS7y!4>w>iuTo%tlhINGEz^bz}WR%!p-Lea`EORL^M%O@3<&)=_PlUzhsK!QE#u557bRP7>KGKHIwf}#f;B%_T@?Es z3a;M@d@>24TzFa79Fu!)M&|OV0^;RSvxt@r80ga^eUzb!qbBZrtNf@PbXT)>1HMSD ziFg*S7*4ilnz%ZI&pb}f^NTgBg)Jn7lrG!w=}_B>b*4Ws|9stGh^01@i-A1tKPBJS z@E&^N6~yA$IizA_He;qNak;~7HT$21Sn35)?iQ@Y;B^K z_d2tSPovm59@&<=bjKMN90)aa8J~jt26!)R3F0JFx5ECbh_YYLy}O4KP~9|qmViJ(H)~28oD{Y7rPR@N{7w~{=w`Q?zNSfF z@v6VQ-MstJV+cL6Q;ur38Pi*A4}1Y|v;F}`ouxZg$;%v+{#Q@biP}Q{oMfF&l$m1n;zCQxY zwxa&`;lRm_{*ddJ-|vaEJpH=K_Jumd^zgIKSfaSfNvTiikvwwhjFh2`iDmNkY}C#2M`BpNNm9)kbuQge(uAM9Xe(&FTIj1uef79*c`NE9GFSjn2!DI%f${537_d&P5y*b{LSrA6NnoYH4*fy0e0*ExiE(sd5)Es4Hh-GiGWYYT8Eo8)d1--u2W-S&0iM7bJhU$g4wA@Vt- z@KXxnoUL4nSG@zYzMP!hYp>QHcfMhuXvVKXvE1Z4amCJ6{<}%+_;kn;hl!5lUMnxB z_u#X}6@vLtMaE`=>C<<)&gIy3JAv1mP6BRaUEJ8anYt`5Tqj=Y`yyh6X zcq)hgvVjLJ{R}_xVnzJDe*TPpPhHT$nji|MFu#kE|6Gw)UN|rf`Xw~~bG{FgK-KiV zMd1ZUv~57l*I<8+dPU=P?2n~2pD0KKj(PNN(NKzz!}mK@IlU{e@VM=w(BT^C(jd@J za>rXgatMA>Jv-TF-JONF+4PxLnvOB)Ftd8}TOne&yx8aX%ZBIDQu%++s)X?t8wuo; zZH~7-1dRwJl*@vyw0$hL(E(kdr=DrF>8+Wf&~8GTVteMLWg_br-qkq=R@W*{@AF30 zyPf+a_XcgMyI+_v>%9b>ijN?3Bqn=t_ZPdK{{bN2uBtP{k$kBkrtp% z{8I|Q!gt0YrPj32Khn-FJ67g~v!(5+d4-I5LvY?n&I3;pX@A*N0(Z10k6N>q zW{}u=yz!xbXcV+!2$X~GG5=oaAbT{LW7Y;?J~OwircR5E{!$oy-%8oaS*zB&Gy!r8 z+d-|RBRn-jqoJmh6iBp2q(+Up!sP*ANFPKEsE0W)@87^%{P(=oGi&0(H~h&`2tf^$I5pzqB5#L~)Mv*1 z*pNQbzyX9KY#Q-B&rU&B|AX}0RM6_GtX(UoE({w64goTT&+(!1t|k(Mz_J8L7`B7A zJ%*21#>vQ4nWx!g!Nrj-Z(px(+K#t|&EppLTLC*Ob|epdO)^CxE$g6CF$|Qk zgL5~{?zu@J?81Z%G5I@Yjm^PA;h1rqg3nxVM*sA45Ds2rcpnWlw0oz~ails*0KzLRVW@`hJxsnVRL zsnSYA@lTAtJ@44a0N+9E&(%)T9pk%L?=IdyNnyF9X*nd?G~TO=B3J4X7?wWlb`I&= zN#4gJ>C;ia5hU{46Uz1V12FM{X!`xX&-cpkzW(|M#4GvWCk0Ah_$9fYUzxAm;Crt$ zY-r@JVApkX4f8%wzBy2CVrBx-*_ZwaRiQFUx<_&<+b0OG^)$Z7^GxIYD_{m}MmO{< z(`fmbs2~q!jCjTHbOH2e8MQ^AFF3mIY)bJLU7tU~Y*_~ij@khf7WWo%DI-Y;Z{NIbk-An%fOT@t@jKkrje;qvA9UObXB$-J^sZws zkRLxfue!CiXOE8k@U~s7Y_D#7aKEJ-qGRHN%^q~vN-L~b{T@|Qx7PkhVQZhlv*g>i zW#1&V8u{AnGK+WDQ{v)g>-dX-`h_P)9lqoW8heUmQXGD5?xuc{0g#L&lBQNAY8LLK zFu(jrr2F7C@MR@$b8vjfkIj!8$dvfow>epLh;=~p*Kgb~*Y`UdClFBjU23Ruy z;aoBUJH47jUx5KtBpF>@E_p!U4mhv?|9x{1cnb!PiMk?`@dgB74}~n?V;1qU*IB`K zV3-GIfz!fQtQ3PYTI4^v2{qK*M#h{N(b7v=FM8nrDbp|Lf*gehgjj4>A^kibO z2CDqJJn5nUqmoXI7f{qm>tF`E=-UrPSNW<0hJ_p^1$F%e(7FPr(L&)6@C}N=i2UX+ zS|*Og|GrhG2Lv{G>mhjluD!nfyZ$NXKkFLC1ks4C<=(J3YLiGt1Y>ddfu>``0$V7^ zt8n0NC~dc*c}D|23Kn#*KDB8guW8xC?54s<$mUR={n?fHQx(@YG6dCBw7k6FfVe*J zVPA%5^%;NQ^saT2UcdJAkV35_uX+?Sf0IQq3qc3Zd!_zCwI2}A9QrPJ8)k|j;Thg(y(-}F zbi&-)Vxyy*II`7B{4F)fe;=@Reg|6wn+}~}3r)gXMRb?9-|yZ#34T^-TB!-S&U@M-3OaNQ)oYGN zsnpgGysaifDBE`M&8)z`RA z;fl5IgBwNd;C6V#twW-Hn~1q)kMN3@La)csv`<5+K}k3F&2LveNLAN|gW)m63dVEd zm$mR@+G3-kr|ONq;0gp*t}&*(%p(s4Qbatv8DyGx%8-?6^Lp0J?R}F!HO269u>ovd z-}&Q#tM|do{WpSmj3B|{g=)ZsEmpw;@MtKm%AylR{}FuyY|j~SM1MV7ZSI^6AaJ+9 z@U`xdTGzi>Xdt4U7jBVviGQ?fTc@Mr&0KlZwHJ0v`f`eTxy%47+86S#{x^WB9G6 zlEme+trTm`mF^SUv~>-i7^0QEPOTy?xniOLsT=C54rFrJ9te7Yz>IdXj6UgC$0?i% zCuPv6Cd9XB&JWw2b9-+?W7W}I`|p3EcHi=JuqFU74+9|i)-{abu3pDrBg{@B#H@#!wA3&ENq z{aG#H840@$TyR>mxl|;9a-Ju4Y=@(6&c{U#eG)B9$dg4*gs+hCaC8i-KyUD1mH-h_ z^D9TP-(4x|&~u@9H4#4IXN&JdT3akfu=4P}1YzT@wZVs|^`vC>^q!g75!d2!38-cN zA1%UGqALd96R^H>B#A@uI$Tuk3eY=rBv;l+@&2q(0Y&Fwm@{tGxuuHvfDbW;1C}(8 zGvT}64A-}jO*sb$vYLqj>C~XpHSkj&W*6J8oMCcO3Mi;6-K2skd2>&5vI&*Zq$^jT zJ+_^SP!IIJMC<63Mo!#0RjXUkJOAk>kP4v44LLgPfd1LkuGSksy#I5hh1Ktgr*nAv z(6%Ta(LdmjrEsN?>qJkzX5-=sJ2^PXCCab@zvu|<(LhG@8Bh$VyWGa=eCio%!b*2K z`D1PZ`{aUe=XbNq-Mqxbi8jm|X55@=4SN9-`qtSaUhW7)gWnLiD|_ZH0@Wv_-YyYu zNhp>vFyJY|$N%~biJ}kE+uE0;D?JhOR)Xn+)UxeAuYO-=2Fz?q#y|9~IN(2?iWTjo z_#!~*i{Z*apYc8O%b1Fyzj}Kc=ryo`g5S3VfC(#gl*`5&eL}>m0haNi6PY|Y!%1>5vEzq_C(wN#c=#fz+DHQYD^)5N| zs6A;k^j-ki=@z&2LM|q;&KRe zUp^4!Gtr_22_v$YRf{op;TfrJgJuUzPnMu*_0%khw~&i3)M7&tF48Bb>}mDdF(-(N zStS*Mht$_}e2tbQ8I4mRxUXw2%mC@*lEpgianwqBqrAFrFecoSi(=*RY9PGRELD8Z z_jXN3LlX?F$mTgcYWa6gjN~?-DiGEz-yvPDbDkACQ5F0rHvqLg*vUjZ)VFU^IMiNW z&&E^WsHiRksCQuXAR8VMUDNt;6Er7fz?I4IFg07<@7)5g%#gj)sWf$RP#H!fg=qXR zK)S%u*+}5J-)R*n1)k zJ<>STj$I$DWZTDntI64Z{9GX@kKw=N<5p0H``-0=L=MBd=QPO+(KV2U#Hn^G@-0`( zft-6m(*0*T&2eutH2o}=8{W0kn$!cD(s~N->rj(fjw|P0&nLsc1A0sw*U)#JfTGBf zr%6>0db+jSlZ{`ivh1)3oQDmaj8FQ7L5&^Z)`=ghikDW}2bqYCsk0 z9<&{NVl_$cy~|+{Rde1c{ds84RQ|Ke?m&(XUtZw);kq4aAA*NG_!)=DXmD_zICqX3;|ye7R1 z_4pDfk}%E)JQasp=r7Qi7;CfKIk9Tio4s!1QRABG30*y4as0+InQX ze3m)mwKwQj2LNjo^AfFH)yUKJB0j8_YMR}zVMLku@zyuyKV1@9iVcF-ZOUxdLA@}W}K;Wlp zss&{;3dJ1e<@56#b4@aNtB-XBRvrT)6M8}5FQQ-FEBpBOVUSZq6K=3Pvqny~9e~;S zi9IN@9DOEl0b&FUw1cE?a__-+VD3Qu!zq&@&N!LdKI%V#p9&pKOk57X0iNHMWr=ie z@Gmj=$Ii}Yvy(cm+fcjCbYZc%PLNn&U^R^w10sUbE6EToR$a({js#V&;@lne?X_Zp zMsM3Js>m^({{WWetI@A(xH2%MRv`+Xi&JdgS~0q~yE$Ls#N~E}1^Y!;*P5rdw^ngZ zKYQn?80KQ{sErN*xQe||cgL5D0>J;--lqS<)tkpd-A4W6x?3nAk``Nq5JOt*OOd67 z>^ntjn8q^pT_Fi0vSe>awyce@FD0f-wk)Hu7Gp4m7)Ffcca84*d7kgDKYDqk`OLYl z&$-TZmiKud4nftRYNUC&U2uTG+Uk1^KU+I}Co1M2#=8e6l+7lRby8YDS71_6l8X6z z6`oP%46&5~*44NWFdARW7Ilt{;=`+ zWa;E0u5D9uzR7cBfE^E#1lnb|HUH*=zDX-Qe+-`k#LeE?^cs(zY~t8Y6ws&(^kMAh z#k!03$Yj*q8B~)X&u1+oOX`+f$}1b6(-5Hzeh=;%lC`XEPz?fVeYs@klmfgm+c?&b zKjnL=_rUq)eR~?rwLw>eSL%4fLph+gnf$wL zp=&K$_AC^Ygx{t&(Zq_#^1SM-<~Az-V3r7)SesV>3)i1vf#X(`EgU6|C|?mqdyepp zshIKXDLE4p?E0LRgG)le@6D7kpxo|evs!3ZBeev?gHYiQB1u`_g>E}FnVEQIx*g7H zujo&bCj$Re7aG`XkNqeP>7!N^8|RL$G+aQhWfKyqZVf6P)*upRyu6BsE7y(E?W(N7 z&)rAH-qK!~1J7~WL)iP)Mwz3_OUV=!`(HnbRBqoE{4RlZyon*T^9@&fTbR|HNAt%) zDog8=)Z1}N-beO~3V6no-MpZbAG;C_K+ehcU*rnPj}hp{i5{^on3q!bd%tAnD252| z^!RTGph{)s@zo=Dt+j8d2SD$Bd95^J9QzORUTn|QO6Z&Vy2~f!vbuYs0hpah$?l&$ zL|ugqEw{7iR9-$KV}VP3RO`Z^JNptR`l}Lgma-gOJu9Q)H(e;u!4nha^sCjK47eHi zb1x1YzMr9oPqy_efHX>&>2c%O&vSNez~lLMf32rwY?&sa@Ei*OBp7CD#Jl?|-b?5r z(3d?M3`~DqN@I5-<7q=TAniJt0=CEgGm=chTyptYcEaz@`L{?gYDoq8$qvH$zEZLO z`{~FM_ZwTy1I%HK>`vLN;WvRoXC|V!g#_CawmEP(7!9a6bI5eck=pW>cpv@4kZgC?jM7?&as2-g_4%ClY}_@Kk9N5_7WZ; zAoU@gLjV1<4ZD|S2n6F90buD5!!!kfWCIptK}uo#z%p%P?L#*5OR}tTz_com70EEb z73=xi29&Ow=u^H20P9$wNgD*&h)wMgQVMyDeeVrtvVOUJ;oTYwrNmukE5z4Bne>!E z37I(ta*OT*)IU=?IloFgmDK5jqF=3ip4Ni|Q|M^wMLMNsVrSVZQTEqH^_t(#vBaig zdtA`^m~SY{eTYPn0M5U8gt%M%jf>v7)+H|5$3OB8zD&-4tF)&H$Sx>}Wzt9-dy=qx z{*P;Xz@o0>q&V0pd9gFtU%L|CY1s4(abu%6Xam3PI3cAH;i3&O8l_+&_7V%N3Qxct+|-La*=oQ# zSa)+11P@HFeaD*!dLFGa^L3|CDw1dsn)l>$=tP<=Z{qllsabkf^gl1)HjJ{4>WonP1E9-C|qo3ZDDtn}%}%TcGV3hSmjI z{0;xDjYi45G1)%#?3$tEP}~CFVLKca?ySsxaQr;(cS_+Rgrwt|+#j;)^DR!`ugpcj z9m+I=#%sA!zQPN4*`ff9GpxN%V#>L#%o-?cR3t_yzd%coex(nQ*jc?#%u#AF#R9qi zR!UZ*j}oLb3Wz`Xs`R;6we(Vs0ud2-`_wT+&er(eQIZ#V6@gIKpe`0SuqXjv+=O%J&u z8_!1rOXd+$bzZZ1sLU4GyBO1(ees#D(>;As`eMSAz=9!X*SvA@QZZ)wqkwK=CtV9# zB2_9xi-P6W*VrruD&1~)iC~o0x?fA7O4+lR*`tjGf}C=o2c+fXdPYY#ZwGU4mT|WC zBrAW>gY{-;B<9_8nSj_{{T9u4n%4pfN)D<6yMDl5%&2I#W)B8Wim6zCWyV7J*=)UV zg?)|H^ZMuJZ9jYZbO5A2m~FtSC-V!U;FzJB1++sc|F*YWQa)r?6 zxma8EYGCnZ`$lD(SDCt!&mk(>8x?%y*SheM8%a#^iuNI$vmdL5nX{CDKP=OGfvRww?9Sn#~d8%__TxP{c%C zL1vCcsb=&enfSj4!J#6LghVQ;S6@9Aa-bsOdGt9{sp5e1nHSA>ne~X{O@VT+Mwq`14XD+AM*~&#+uQBNT$Qm@OI=EonwguPGqf4!pR*`0< z)CN(i30$e=aL*t$X*JTba183Lw)`N{s>y$+SJQAb-m!0jcuC#bVMF0LF+*r){DTfK zG+`Sgll9}-v?3OiiWEAy>)c041z!5sP5bTk#OcRBjG;H`}$gQ2bScw+%Ep@lo<5@KyWH zHrTQYp&jSr8;)6q&gc(RaR$m8STD7HYmhlUxPdRyhQ>|c8Y2X#2)_K*=nc8mo zQ+kU>tGMP-63b1oB25 zY7@t?QvN_!#F-xj9i`NgE`f;cQnY*_bzBZaM%()XN%nbci-VcSjF!T%2Yd%(?0 zxF11Olt>u$>?H^)OX>4CcJU2KXQ@(~7nakaif`<(RzNdSI9ppfPUG48St%n`y?I|* zmu+Z#pO1jM_n$geQcsk~#hXxjGvQ4UElqG;gZJi~4r03xLvt6Cy##rhETJ51@G4|#EkbJbCOBz$xFqbSvP zCQRKr{=Aa!+6W@^##wBw-Te$;e1`5Q=Sa&IUQwrC{{1So^!kxK|LD4!9t!@`LRaCJ z_Rv*JJ4^)au6|?f4yqARnK|uz2li8YUDBU)gd(Y=Vp)uYN39#fn)Cxv`tL<*`-_uo zJGpk3yf^zkb0JoW_>p$q|0tG!SlgIc0h?KKjRBAJR})nqF$4q2;h^AI*J<6)89}qk zL%HGVb34FJ4SnlB*G*!f;974mTsCm7x1Eo`wOLaWZk+qj3eV?r)`X@;t1Hc9BeOn9 zs;73N#%`D2zn?DDMGa_DGmUSrnDKAMNECGZh8$C;P9m3Y84R$hRC zQq0Nw0d~wx*yo4mU7ZQ^m$^qzq5Sh-))a9&k&(3E!YPk->=QYsj~`3H z>yhoylQ(F#tVe)jyXWz|lM(ZDL_a<}cTV^p$AITtD<22Xmx|oYY8#FpsYciZs#w)% zPfOi+ZtYX7S|jxC=W{CVsk)NZrXs=S@&uyx(WTq0KNAnMP@yEEWUPo!^#t&N7Uf$+ z(7_*Wz)c#^22+nP<-N2=sLizvQjbGfZ<%HSmeMEtht643AcCGco0?8anAH{848DOX zjhSetJ{EXhvZ4TaK?v$X0Y>R4L`TLMcvvro;PashpNMcHezIqOB3P z2YmA8(o!NKG!-BYV@4X)aQ~=}Tvi#s^LsMc$pFP)(?=H$r8U>R6l(v_%K*LK$XscC z#d;WSyU?t1%5zb4886eRzKOZ~Gcb&27LQQi9>oEErJ$TAZ`8#-xRYjG4$;9S?V$*CC( zjT$OaL^sYG`s-CTrmlnWzEeK<89Md1tmSJCqTSL)5W1=Qa=Ks(Z=G$*eDx9%;Chsj zTpt*qg3H8}yG1mCNct@$DV$=v488IF% zSr&Xu7n}^oHw7iXUg)D;8f&0uVC;tnXF0n<(N?xee{Sw!Fu=6U4U#Kogjh^Xw*FF;NZ~vtLgqEqA zL4M=q^?d13or}W+sDeV{2id;4K={fQy|1h;ix*H?6(8&jRk00+} zzQFDcG?~N(GF73dZ3|4A@1~Q&r+bY0m&6SN;UQFqV;n0M~7e=Z0c#nV00{d(o(gB=L5Ns6+9b)`ybaJ`>)M| zDHKcNO-iDuZ4)VS<@BjK+7}ckJIhA&%qd47cvb#?9wTzz!if6vA8!eDKup`R`-j@2 z7T-Oxekh!PD17cQxC`c{-gMq_-uxLQ*nI=q!oJkh{AQ|MV|X!@Pjw&Dz5R_dqlf4= z+Le3pul@jMIF*Y~jUax@9kFlpTaIcqdy9F{*b)z`E9$Gg8=cVUCiVL@r%->i!~Wss z!ut-2L@%hT)K(@tj!zUASf)4oHRER!b{q1+{!F$-apV{o$al)Oq`U{&F*mitcbm*( z{%kKcEbSb7tFflgr5Ot~6Rt`z9jGwAH@($o7j>b@K1OUw{K^G%)(edh-VqcSoWcHyZ`3^=ryaZdU)U51s-;F%i}e$~}I6^+M%N;62Y_ zABx*+p=JDdTAyjMwI0D%SiHa@EFJH+K0&@(U?f$;RC9Z~UkRi(N^di|-Wr5YOIjBD z3taCd3Y$VS9YbcUg1O42B+{&WRtVHp^ItPn|i!WLmX< znF0Od$QyWdqabfsr3hN2O~qi%@^N7HP=UBZUpYKl?$}i|*BaVHI-;gH!z@GL)$)ZSAkG}VNInc$1wK! z`n?pMb<1bv(@!_EStl;)goMS#oC#w)LdjG;dary+xM5GV;_X0(q4%);qT+H%<30Fc z3g3@RoMptrb7Nc|>r|zs4@3IS6mUgU9H`D~wzcDud$%|%p0|eeu2m_$aSkB)>r^!Z zu#__}10L{W`AF`_C*>DQeASfT47Hf_qxN490_L6hm(LE~o60?OFC0L)s63MXnGYCt z;qkg~54udelQPIkNx9y&QBJ*{pmJVb%KkXP@VAu(7W&KR7UyNN4zh&!?rLQO#`wKo zlOyFTSm-CL4M{5j&og}OBgK0r<5+c-L>+)kwHWLgow`(fXus1xz%F%bLa}dNrfz-v zn;m4KP}8p#)7y9=FwCdE88-2BHmNJc5Cr%PGy4PXKsedDvO^_VpxZhVf-Afq>v&?A&G33ll_c;BjH$>ffrGLe$%2eI5JP<@O<1kX=&_osQ zn@dM1RWW`;k=at0o7nCDtR?+0hhk|i*_aYp=_Bx_0=!Jqm)BQa55rkqD54-&t&sSQ zUSV>+a*AH8>y5>qOltev=K=R-DJOtCpq16 z<*2o}I9I2ma5+Z1+n}{t3JB<)s0MLV54__bjrds4gMxo8!Snp6Kjo!5Q@QCdPCg}) zAG0ipGWF$KHQXh!QnN3EV!ga{cMmo9+ydr2f6ub`b&E-ltK4oK}SfxYM3{Nm0rfsy!IaFtM3zlY-$y|%-Y^NHK zmCF(7spM6^lZxxp)TVCD<{Jz3!5`{RIG0jOCYCq}Ay0LUDa!NPhQ|p_aQue-?kW8$}CbThAYOm^>m{AM|tT#+{h8Rs-JJAx2$Ce}&9R=sf5*Azo=Nj^;>a zMz}n9B}M31F)RLN5#ec&=IFMoS|yJ!2u3Fb*}+y%V=F8_^X2!jz%PJujFPqxX}~n3 z{_9e*SKE-z%MRWSsU=Ui-g$gS<;$1eDx2h8>!L;%xnkIesgK;F;BbuBO(=cFU6%{* z%CnA9QB7W358YiK^tcuz)V$EqR5*5XJ??qqgri`qct;~gXLjEoRsoKc)^;WvGRU4X ziEbQch3hVq1gqtFc5J+MEQ5CSlPjN`s=4Dvl`Lec%v2S?Im4AJ?r{B9*CeI$a%{F?%dt-3W%fbTr}L#V>xes%*r8^wOdIc<RWfd?Xm*YzT68_BC-L;m!}J9h-R)*JFVJkzYkMCAo8S;+h+48W$#)xR_nPLw+D zZCGQl*a4YTPSMqbHZ&W*GrUoeb)tTbW|I;Zs{y43hF{?%2>h1QM=o?37$}fFT*zu! zYd%;GD*Vg6dV`rRLaKF64us>$F~*NG$eovEwVym%Ztl?|zSeGx;J%<+J!ZHlLwUYl z;wCoj)3Au*b}SRB`N~P~%ucbM^gGsX-}IAo^xyaCzlZ4W)%a$+iCNfva2P8Q;^uxq zy+K3<;^Y_*`#SK){v%c)*4!i$K@9+4-k8CVi*DW5>)seuy5wDAOIH;+VI1LSPZ{y~ zxjoN)QG|H4hvLKg%stI2<7(Dfb$ai9?)`gP(0WUNk#Wbc0ame?#3@GiK(`H4i7hfO z0#fUF)tiSOb8I=)A@_e>CJVf7fG^7Q(#^Mhu$e=<4>S(^99ywZV7B19Cxv?sN-#ZT zn^}3cAShGwW%+hu{v#J+tar9`baqN%n$@F@{lvj{3W(c$>!ingZ9F2Rt4*#+btp(g~5}(ymh~be5LDBTRrB~^JneIg=b&+e*kE(a$w5NA#(tl3!&2E9-;vo3Ke_?~XQxb& zezc{Dz66DItCncL*xn@#Ec5mSNp}Zs=dH!hnZ3uxh3(_PJu&(~HJ_&i{?4my+=<-oAshw9LB0o!}Gh6AdC5+A&vI%eOtmDD<2 zvR*Y>#+LM;FB5O7c)s#k1SVy0)3$shg&94c{KIbiX|tE=CK;Ovq2ykVp*)Y*D0w_e zvQ8k_6!JyTE9?b*sf$l%cWPgaXe-&pu|a3n*OZF4d4eDJE(wSFJ_~Wwz{XUv;I=Z* z3qx)_yI71>NZlQY$lE-|`X_!m!A6#*J~wktZH;3Z3KE^)q|P0w7-mjTYgPqy`ao zy=wXU=wbLz7UH=%fcXrp5I>xk@))T;#`(4>2Ue<*pV-+-H(m>zK+_`K4uFU}VwVb9 z;-uCFt}o%wgLZ4d2Ft5|NjrWNi1+3h_2_L1I=U>=M_#XN#U=+m z{w?DYonf&x4s6^NB(v^O`Mr%|Ag0v=x7A1fDG@F3N|}UwoM!og!Y&qowB!Zdw<%YB z(eJ3v4`obsuk^aa*9d_o@OTPqjBJGLQTGqW2^t%VOG9VT+qO9uA3It*He2DQjZVkH zb)ZVgfOZ}-E{{+O?ib(9w#LT#HNI%^+(vqCo?R-C@4+2~v+nK96C|@tm zG6G;Yt$KsD>iQANYZu3_NkT5x-^x|Ef6RI9Q}}+FLtgxxDyGYWNm&J74Ua@TyWbYUxol9=y#D^Vv)OM}fM$|rpJj>^{b6-E78 z7S#@*f+pzwd~<&8wuJDbf2*hrUW@VU-s}%!9+NflUmhfxI2yjt0M9GTCs_-V-EPHi zleUIr*20{Ug=u%$N1tOp8cB4P%Ad_y^kC>|F%?)!f&sMfDxJcZ_b?b4AvK(5ulRdpe^nFVH9;Z@{3g0CNb%(cb$C zS=DX%tX_x*l=ys9Ui;)T`_x--W}tvINIRL|!B*Ez2;_nzFe1P2p4m5{W3|#Cdo9X` zENAMIFl2b01TX(uSh+P+)_{paev94TRedN@a=Q1KthBCuC#Rm9J_H@vUfMZ0U242< z?jh1``n@L9`Z}pOGX6=3gU~x(!yj8ADdPlDMS%h26^49tTKrn@_$Omk&ro`5<2FAw zeB_DefxTSkpyP~D1l&$RBo zO}5!}hvf|ByEZYsG&fVMVzv1Q7jyDA|K%oC!ADn}q6Ll5>or++fwuLgRo_t9<9z$T zH&aitSN$6@c0n^~%8@<8;4X%O-{+Qo5xiWu8{Eeyq}^i*#SSp%3;9X|s8X;bieDrp z#B;OBQftl^dh0~{f>nf1?5Q<330BdTB@;1^$<9AYtH-(R#BLS3+nokxfp z3uQb5W33}n<7VGLQZAmjr>ZqD1E1ZZU-;HS6#c84LPy{^f2`^N2js$5o#RQ&#S%pQ z-8l8K3zJui?6jQ+rOrB>zI8DKjdtu#}_!+Y&&dIh*l^0}ItE&!jxtF}Rrw>t)|X#ewW}Ne!rmXgTv+8rxI2%IA#1t#LCfeWCz6 zQPAK?5u8>nr+e$CO^Ft|kkptJYjb8Zu9a?%o{SAP?uvKz-r_b&>0S)q+)AY%QgH$t zA@s{4DyBa#du&F9{)sA|QK)U(g&76%we#0VEGpIfAnl&W3@!Japai`-{#Q4Tq9{-D z-m&Kg>a(GEp53)%*D=FD=DL%j zfO6E*>NPdby<9Y(CA1+gjryyuElt$-vL|Vre}wPaeP^2(voc%BSj}?GfsNtF$FRGK zzOjxsbtHV9o0;YyC#8cB!o@QrodM znk+L;l4C|PMrJoj=#5rvxSwh*NtAu>ayZUHIpt(SZcOdWds@xzqPti><@#jIDhH=I zS5lveIH&5O0Fb;oUTVK)Gy5%PYYtU^xJs)Jz*Q4wiA{0-)=fQpX?CCP*(=1U8=T%N z4bjeP!zMf!>|24LyBZN_s;;CqHrxCTSEn}2`{!l2iq#X24e<_nF&7B}=%5>bMUJ*) zT|@(BjkBlLV%AR|pJBp=|H(SIx1E{l&<5&jX6@BUgKR|e9N?Ki;1l}FLUpcV5&mHc z$@s{pPD>^ze&z&;xQ{f%53{Io@;n6Z;+WJ^Ev}X6tmZ`t75bWyF#M+L|s6FwHxG zum*i$^hAokvnQi)3B` z^_yqSn7$5NuC#H*qCmvxKH)!qj|Qk?vgJ?gAyn@jRJZ1mReJ`#aYj(hgw)4XKXfBb zsIH`+H+5|)#bc5;8BdnK+^Dpj(ry<;e&JM<EEgRwa9>v<8ZPwb7@klwzfsf^Pox zN<^t_$E+WuoKh6GvIsX5waePBb)()>)A-L8tm(fKZ~8*7*qvA!Co9_dzabi{Tnu$V zkCAG(x5jyrB{FU)0IVVZB}1~>`oKv~??C@;Q+tD8We7=hWu<7h_qWfk>?!RAJO zLlgYW;Gswp-m%od%jx#858>B32tkumA0cY6J>7hN9g1at*nJIBS*-6Ukqo-(6{9|o zd+nQ$b^9Gja!1^Jzd70>Ke^i?ulsDLd;!6>Ly#op8J-Bp2jrBGkc^_0PfN%RbTszZ zXa^||W^IY#OjTWt|2;-p3D^7Y#d8pG`_59ml!$(VTxO~2OMWM}#-*&FIbCmF0(WhD z;#A>T1&}eUaa?QX``#_cAFeG~H-;ZnUQ33}tp+e?HXOsA)7z^3WiF2fuli9YDKEm+ z6?@;+bn$V7If4Xa>%qg6QSlt5*KdTx0`l6ThVs@86sqb7LoXJxja?^#-(7Yj26Tdp z>=@xU^KGHrCjvQ&?IXA%tc2S3haUemqX4BecExDdcB8Ub?OXpl(Rk~3&vG_P;O5!oOs7bbv@K=nnQ0$kOuxX; zKgKiYw5fgW-$3H9_U>LO@jBEkiM*n~L8D)WocJV!E!j{9{CO$HXK(u3f-j~#j-=5~ zbJ5?2x&e|N2ZbiLv!Ck(IdpCu*b4~$1$48(XY=H^lni>D9_mlHWr73mWrF~x# zED`G{IUD@7618cfWtdqhu)TDZS>OMqQ#!f#?Yc;o6=#};Q^$c6;}`yo>^N2J%KA7f zY|hSVLHUgH$QV2B6N;tBD#P>4H2|+9ZZB%}j^(d$tZomi+q@DgK4H7CdM-62ob}(D zC_@-oQFPkt*M~%3$Dc><`v2OYoqhS?fOU5QujiNBuD8$~AJZJY5Z@p$&P>nW^{S-e zxPKt=N31rxcSzt@O&XOXsKYBeW1T*fe^mcn~9*HI*;O>nCWxS{Yh34lz zUCXhh*a@qt6XlQwVHwTH$a=z3&%a_-_hyUb1hC_O10+=3lgLsaLYQN3M!MGV5rZU+?(q?{>K*FrmJ{~L_`%$F;l zfnOdrwQtYCV#HjB04%*Ta$9X%bsLr@%QBq{vc|F`gu>Gw7GCj#C;MLyTmTJ{a#%Er zejlfZyTR!upJo@m0RE9wPaBdCs5F&s%Q-b1vjCDIng1t_m1qmU zE_HsmDyb{sv!l8a>#(Gk^_nGA;xa*;<5$wb56sE2T=Lq!9Lbg$y#g**6#j1Yo}sXL z+3U0OHRyhTIJ6-iu)lqi21v-TI8@<++%2n?@n|yP>`Qad!GG1@hie_7>rVpw{#cqgv?aUd!RJ}u#GYR828%L8@zu8iKC^yrKZdTT&-U?)R!nZ z8+`%9fJ`QJ(m8om{K3H=0+-&MN4uo}o})-i_dyZj3)=N~m9@do<=@?*&GQ2Yfjcs+ zc4K!T4Of9nmBLN%01S09au{Y@Y?YORHPtc6Z#>kIOW;XluU*x|=#}cHr>@#Bajfl! zt+h{dj54KU2Xu6<4eebMpw6w%+kJ}T$PBAr4B)Ui3BJXKLuU0alJ@|x7Cf~@p8J5UcSR+0p9FJb$nx>Mg)*zBPISR_x$a+2>}Lh z&wvw682OUwA4J4}(m$-I-~*hB^2CX#=9ZnNlq9+;j)DsC9(}#}IVWD+u_YOWsHTzE zeHWMjmcZuM-xB1bk7puj!;!g)Lrn>1E?SeH$7`0X4un33ato;{F#GG3HYpJ@Xrv$Mq*FPV~o=lawwM2fHn4SJxwp~;oT=7u~`c>ey zPfE&rcP9L?+QrV62la``P_&<}H7m{nE)pjF5Ca_fmje63+g(RdEB?pjLUye)+GW_z zRlC<%R+u*dke$5Y7kRgSWpNT_FR$33eT;E{iU)G5?0uwa6hFSBnl=}asrYjpmW8Em z`e}}DchV8B%;DJq$9ZKc^bI{1EpFE|w}jY-!dkBE?o7Q3)S!0|VZps;mcKqj`LHL0 zRuNc)v4Gk!g8O0YEe7x*^B|0qfRx0%t{(O-xCYp2NCf}fvAV9cWZMT()S!;;xAXDw z{Ma>lQekA(1>LF4RIkEsz@e*A(;6zY9j9f=-aLu7leOb31Z*&r0k0R0;oywQYa}T? zxNSl!KA6F*5Ic9ntNA-Qo43N+3E8W453W7%*cKY7WNkpN3q6(pySI7S6kzS~c@APnD6s1oBEh9CJH7pM4p$e>UHJ-Y2pA^!#lvw+8%DOy+A1rFR|* z4{x%9qt+dPzeQ`_2(Bk2-s|6JRq*iMznbWTaXW-R)S+?D_F3VNs$QQQ#QgD7rNMa< zFWjChwB|00g_b@ zV_g=!R=nWFS71U7qcD#QRBFt`G(>#iF5rzel(%~%aimFZ_Z$#ND$L4+#{ZP0kyI?=POC-4`y#YA4h;%$UqPu2NQeXUe&zS(p?HX(QEn(qbLlB{)jq| z!;apKQ!>9vWM(W^vFa(4H0elBjfnv zuvSG;!w*g<{`{r8xA~cUWApjuK5kpAg$~LZ==+L3DaAJte_%haJt?@*YdCTV)R z)eWb1QEGZsv1lPaj>*jb)E__zN{q=#XBMC9&el7sn%OU~G)(dB_Qb$sFJAfV{`c3oLUYS{(vP8`=ft$B>`n8ll!^R6zROGiN%?7!%R=SpX5Fy14HRt zA~^emJL`N8)$zdUIhvl^#RMyb^~mZ{mzuJD%GdKRAeot>RQ|JQm|inxLc*D9d?`XJ z3*r4`Y`JUhsuiwX>|wI$>A^gN0$?&I2`{}oZ=x$pV)r*lL+PYs8Y^jK>8jS`7MXT) zHAO1!y2N+RE{P-90FZ3*^ZRQTc^RBfz$rUUSY%aNQ7}l)!CHaK*(_B4kYCc-+0)!y zfAbPtD3y~O5wDeg3cReOSr8R^M}Dfw0fXzVo^+BPZ28Cp_-rV)t?`FbH?V<+U|zo? zpYn2l>$BYFzy`qlF7Xx|v#dXk>{W%KwgJhB zvlrpwi{_A5m^E)pa}!xc6%u$FuI_)m7^xf38i5-Fd9Mm|ik<-4?kh6qhRrmQogq8H z^$M?8wJ+@b?|;FU_hQtQRMqF^M(2Wsj{_D$3Ba*!YGwF?iD{K1j4Hk7AAl7AODR=o z-iAYjfM0>{>Tdbl4iq8#m(T4D>pXd)prAr!$rr)P6t=U^edll34G!-f;kC;b!yg7p zAK93$?*eE819-w%A$Zc{(|{AllG*;x*MWael|WxRMr!a~Qf2EbD!TTiaHgaibS>%; znC;bVq>x+emo zZuAVa4ROVQ2BGICPukyMGzD3*QigomoyqESQcmd3s(z2B6K=d{Lz< zrHm(Y|Dh(JyXn>wrEZ{4v!A{xh#G4rdl^r5E%l#x1phx0vQYL4cSAlOU1Y@W+r}iB z-YkEMd0R69SP*WDE{_2sm&ByHX0bPGEki|B8bx|0W@N)3e74G_P=gzsT`n0rGZgoW zY^4+8Ry)gJyb~y0r?w+pb5hO6nu{g_(4_?5YPk2d6fZu1eQZ3v7m35$!4#q&4cclD zCEvktJ#|0dzz67DBOWdI`7Wxp`Uu^Ot{#d#|Gj5gJ)b8z82jT46A#z5J9i2)|7I>2 zunO8$6lJ~OZ7yIOI@Lw_GEx=OT^jv6=tiEup+m81%gKJ!=T*_DV+1bJIPy;oX1ta^ zblo>}-4yL{r;f9|WcsmWNj+@DMS=gIw_{(NIm3mN zQTIXirreQK`r+&^cEfdUws1nsPvmBG=7rP`f0AyR+NMjknd(zwsVO`Jk5E6Qt38Pd zTB2A=FFBN66H32*$odRP*6I*Ua8PxBPk(^5x04KIYnUKcx$wFdnUl6&ITI*I?|AAa z|KKC&Zuc4U$mu#kXufqUN4R%gDm1=Z2RgB%+Qdfb-@WO?z;FHsagd$@(!KES)y92M zZudGN0y-=EUO!SI-30`w1<_(R9^Y0ievCX*qZ3zEcjp)b-wzN&H2ge{ybO!}{yZ`; zslww0QL-YdT)AMl;^w;QZ`E!&L92h{b^oOFA}?FYcmANiJrQktfdyVIvqU`=EzqC& z4FKVMdGM~G63LxM;_m|+B4)1zDq6W!43yh{5S!H1(`#B^%x+GY8+KSiAN6IDDMle5go74LuC?D^X-YnwchfhUY`f&Ph@A{~cq&qGeJa+h3Q5ngNV*f7}0f{vdHePkwucEjNff7I}O z0;2qnEGO4b#~sGriG5dH;d+>oSzedC=%@p=p0xVi5i}~meYxO58mOKcl4Q6>*YGRR zc)(t7zEZm&T#57J-!TP# z93aXW3D+Hpz!aDXP?GuDxP!MXo&$XXUb59m@VER|jfpoeOOuXjh|5X7x#bY})Rv!TE9W3jo6Nc=_Z9$V zfOj(2Sl;WSqVSJ$!PT_$Xw%fl;Bh=?BRCi;Wq#r4R^4S(Y2)w`E5inZ0a<^6ME-xa zPRHG{Is7Rl^IKmtz!Z8OLE4m^;B>wJxDWny|q@CMBLY_Q?Wp zBCK!z3R&Ng-0!S!(!JlNqsP-8Md|spRzj%^1`RKb9-Pd_*J!iP!gkha(>-Td;Wfri z)v6G(7y^lTU1UgF<_%Oi?3J?D7n`rDF@pr*I^M{8dQgcMX2e{t z?z)&Jo21VoU%uC}<8%!4zur%nf4)*x(VL>|ljMFl{N&L2LI7k%``@aP>)d)RaFdu9 zhNLTLS%T80lk0m89I=rEL@KkVs~n*P0}iJSi+If%HQy!@#rZJ2rXP@@7ZPu(CG<8+ z{lAC({??WG_WdVsPhxJqx%q(aj!a7d(NmvPn>o?H`{EB=v!^(uDTn+$NUyFL1wX@8 zUJ73rLuRk}rxj8T9)9_u zj%#XaNwu-lTGMH+AEL(y(*Hjdnx)4w&Aj2|O~lJUo<~&Xwf1BQ0ZmWd<{lC%+D>d^ zG-NXU3kec9q$(6vCqEYTJ@hmBvSY$B3#B}_N(57`us2fqKO1&``pQ4U%$iUMBkRl7 z4odI#-++c5{C^d9m~`czusjwDo`}04$S~3Wd!*V1dRUm)0sl?wlqZOg>Dg_JF?;_b z!owgfCihFIEZ`AtsoHwXGECHer?KrD_$bdwLUrMS61xK9Q~w>5Vd@pdgJ*kGowUrj6RWLbN zVt$gJ->&I#>(Z-Ch@*XbSYzSGPWS7ZA|8`7(xnP5Q}?;bA`Q z^J?MwY9)>#v^{wqbL+1>S~;EdltH|(MNG9nMgS?QIve=7OfM!)z&g&fZJ=Fr{W zZ5;N=C0$Cs+-aroO@MD#-2t=^IWvIoifSf__O85R!r-HETFZ&v5qQ|8PIp0hBN&j@ zil5yAq8n{q$F2>J;#TlaNVJ{Up_d5ZX0roT-}Rx1jn$K}37;;OuV+Qoo94_Trppb3 zi_LMVwpS}6Ty8oEn`G^%gNYD*pTHcK2u%OuLViLv=(mQj4yt_*9++wOI%{R-U8|>X z#12kvj)8a6Yek7Vb_MomX=^+$!Jl(~)ZC!K+s=cra(90KmOWaIOXZ?a%BH-p ztY5Ku)jMP#V_T5*(mihf@%pjEuZe>fNmX^KF-9>K1>}mOHCuy?@;Z4UwL_4f3x&|G z9HWPKQi|W_SJiw(s4VX1_+pVk+LUa+rcnQb{`GTkG10cHrpppqqFQQ>WAH@=2s(XT zdj2>Whp;xXuKQpBsbNOmuYVmVI3D`fRJ<=O&lDlGEoN@ZW@0Cv5nbErww>jIN0XCh z8z&%3#BB9lx?cKHfSnoP+7PnJT8g#?GkBu%&bz)$?l{Kzc{aTM2qmzEu9gz2Qfaf~ zqgN#Tee-DfcUjd`>HR!=S9|SQ`LBYQS9Tfi0GRwRMa*90s_u*5)o0^4+j+iLZzAfC zG^K>aJvAkJ+^o%ett(PB{N0MYDBdrmn|ISU%kT{TP|y3WlEGBXN^;MN6>b%yz)ES< z*L;-P$rj;fd_ELur3Z7`uXr_~d44uzZ2FRsF=) z>(1vRCU`h;Ysuf>{X2msG)@M-epFN{UNi)Fks#*=!UZZ&^5 zn1}fG*V#6$vB5{koj%Huo33pkEx#Do^CV(YICNNlb9glbAKk=Dj|GXC6KT9(6Y%-x zA3$6J=Ou1+YW0<_XG*n^`MHWg(fNV1U0F3hfVLN$^03raS{h*x#wH9bUnv<&s(||1 z@(^58RSkk&T?12oih{0`!UbM-ZE(y>C=!p-HU}&hyRMJbWL*cu}75B@|)&~|P1oEZ6<<`oM&qVjHW zP_ut&OJafLgK^pUdhr}jRoDx030F}}8^T*mJ7i+Q5jhsg_1Q0@5T`Z zMBJ)aDP_L2T#IsLRkhwFDR}ztuK)Rh?iBZVC~X9_R>r>*doN-lVw?5ut;zKK-P0Vc zr$lwf5W*n)(qj#$sq(|9+$c)0bM&GPj5G$~Y+Btzg^=^8?cJytu|5BDsLKWgQ9kw4 zb6yF1mD@k9;qK6WZWjcSG!jkujnLT7Rey(vkS%cGv2olue1-hIPOpjBfhnnJK2zQE zJ2Sr>|aM3=)||yh#$$7;!4f zN*x0>X&ft1@qO;TLz!2LJ?zmpc!)XxqkId+o#O|pB{{4O)fNuomswyL8|l0(U_Vvo z0r66*d3)Vxy%R#thEh~2^)Bbb5+O6k9A`!$>ji zdS2Ik-QCypq9=wWB$JPCxgcEg#ga~5|hYbe^H)Q26xeeyqI z402S0`y9yymV-0P=;DgT*7|-zO+>n1Ewg_gy|x}-K}c0X7Q1!;T_IfBS}nF~pLB}g zcT?ClbK%L8xN9%w&u*49vOo8+{N_@}y}^{<6~9!Aw8rbgPbWY6RIFBFckY_ek)e;> z6UU=l4(+Pgo8y+ZHyDu#L*9GT;8Wu_VFnTtUGu)DKe^Y5T^rHE!46{oav7oPY-h&4 zI=TyN=)<+VmWyf?Ds1(;Aphu)qlPiQ)u$|`1}WIW8;tOHR(Wmt$;sfPzPy3Afqy?^ z%w0YGfInwsY-FX#1#Hlx56xSw^muWQ^)|-n4v|K4!9~09zuY9w{bg6-W3H{|~pA8~pE3c`{DABrHa6UC}x*SH!x~}p(9r5@gTp})ap;~AJongEJ zm;J!PK9M(k`IUh?#%kGc@nUiOH-)RHX~uD35wQ0XvJWc9%I~z{TR$(oO~)S?C{kcB z5e~K{IIGF~%)03V<@5R{0zR@ocUz=K1I0vn!i)YjK%d_#?0?_z1#m@tlq|mp(`)o& zn?fq(QQx_+;1=IOz0VX+=}gDfGy##IXPvgfsu6})t*83$EapH@`N4GZ77hs!WY9Dh zjPI=;*a~_Hx_N2Z)%0_|zIN>hOF55o)A zQh%TZk;BaY^e6u(^-tt7HEi4J0j=TVD%wIM1w5k6;CWht6N?~X@X|8s+!8Q`AZSqH z7FTd2rXz;kS76Kj$$^9TMj<9?Yi%arOkegugT*Q?}j0 zfAZUK!|reAR%bEN+;-}t@+dc)&zW!Fj)8vifq44y8)-Ys(7+@$cZb&?@3wB*z|095 z+)S`!*gMVOL+|&D-w>Wy8a_;ctRChQ=`FI1BA=8`TeE?|)$86}wAWd~O(RaDBKId| z;$+LY`yFyr%ZKzwA1wbenM2}qQ@g|SGpp(d+YSV$S3o^wsFq{SSqaqh-d6Wl5B-Y} zqXZvTmnW{y_z9>5x4r4~g!YrSYF4_^8xXUgG#B2BT@PHokQMx|*)!WdVaTeF86})g z%)1N(v$ZUQkTLvcoj?B{DR1Q~Wf>!`2viHqQ}D|q1vV_T5_a%zssyna3OjLDUF#8*|C z0&O&2i6oA@|JU)`#{pLUyi>F0y4uBn1FM*)2od!^=Z+i!-9L6`vTG%C&%MMsZ;)#2 zn75BmvhTCTZ9!&kqefy%b1so-^a$yz#ESjGW~ah`=s&hxm9Sqhp|rl=t0n+m4Br5Nsr2sFy``ttDi^Slt~ zT8DpzIQm~htOSRKV%761#QZTM?n#}*r#(N9>^AzFnxN@@*VSrI?Dj5=3j;3azSR*s zqF*+x7FQHW3;Unr(mq`?B_8@DE@eJtPxzkDGk42C>g0u~19asz%!)mQE{tzKnaOi5 zD{y3Uh*KK*((dYv+%K&Gmu*U{4Yu@emktL}2{5_^E#^QOW?Ns{{$cp#;+x4gpD99? z++6R=eKS^1wpwZ@$YA4c>h;KL+>eTrGgM1X?C$Z4j9ceJXfCWf`cWaTPQTZzJ8C$n zT1MenN{ZdXMbzK82JaN(Fud*`H;2q!!FzCFmQmC3;hSz-+rYlWq=_;mZn;EN zjkw;5LZ8!HQf;R{jSolpUR5$Z9-5UHA_>rIGM_7*wniilK5!oAbV-z+AeVNt(WyXr zO~fTR+gm@&+P|&0z}!FhGIXH+Q?^+pSn+11wD-CKu0s#Rb1iE-bLwvU%*|t_RZ`ScktBeBkTT<7?IMs zy3PDQHcsK}{l78e|92^gvzdde%tyd9wzpo}VICzSg0T{E{i&Nt+misVTy7F_PTHQe z`orTJ0C!=|Wp8TZ(LP?)?d1+;wdOJz9m}q&hm=SAIFacpq5LKK9L}WqN&Dr@++fI5 z6g1NxyEq&N=f5)PNpOkA4lP2~xNeWv7c#2-ne0u6c)QBr1s)+{eKX!f)LW7~WIZkM ztid!6;*C~n3-xQ0Y7zGGW}J#7s(4<%FV9`RJ2bk3$%la0DqwD*1S)vhiCCrwEQawd zN1ZCoEPIhx!A)i`1S|=$+*7~4I{kz6%h=)_TWOUXXm)Jr@5DaWY&X(iaOj8JXCb2rM{Ps7tVrgQQT6m5;csq9Tc0hHV7yB7JSG(vZlr$wWvWz-o&3QqBj zDqir9&KE1=?{zM*RL$GQMVb1#9g5FqsPv}G_X|er6>Gx-k%Bp_NVk9)jJ`I?+0$45 z(|vKt$%(R$*Z?zwxbdq~b&T*kuDa(QMt@iF(be8Rd|ZqwlvqdYc>U5ew|LG0v1w@U z==!g31j=zkG#vjb7UInpr|w!i>4PMP;ZW8AtZEI$GgJd=MeV42;sr)(49xHKLXm#J zYzV$BXW9jZwj+KSt#dq#?-<}VCviMYQt%1f&4y*DwM-#+hT?OW6rnwJPVA;pR{HN{ z0l&y|WtIea)TSLnV?y4_W zGIs!-*>5C@xU3Z#&$)zr@Y?*2K9gSG<+RJSv7=bEen`a=r^9<-QG@py)T8X}o+uHx z4!H7@xw`@bD3!k=yWP?vwo;ZyFRw%lnDQ<7tl@Y5M2hChozBXb33|;0hW!a`927CN z)z|vG$}SIkAD=k>!t7|#d%R|u+p;TvZmczV-Lv2+36((O18zE2?pUdL>(=T^njPz? z!6gB6&@5yYkD)SiA{L@5dwL@fk7?6Cvu&at#YODbcIeZOK0Hd|uoAWg?(QVtW5ITi z*`Dcm;`G%9h69tIaD;E=DXYmmX=x7;@jMIctphHJ)Ca!{~`%Pr{BLC92bs=_2jw7pZ9qM8i_GmmeLuy&8rz+?MA!(5OwwXfYvcyhD}?pbf50iAv?Faz-$%7 z;VZS#n|M9)2$wZkp-cRR6~4#&=X6@7^%u#pTB5d9-SliU+C6H?Fw39za91dO+E;J_q-yA0Rx*W!-H$kWxxR>frap18&RFgemtjq7%J9r2@Jz;QUX|Px z+8n>csPEmwq1!^bcE8%;ab4^Kck{y0d9RU$8lDbXwGBzBio$hpdVSLPi28f{w1BqP zqwRHC1n5co8(8NDU2t}=cd^tL^XaH*E|uFEdkVgN`1YX<73FEyVb|)^JIRL+-)LBN zlo1(k#p&ATFir_$#7$pAhhos7ZsJn}LL1J)WT?iF(rbXV zYn)#)`#ee_edP+pUVOr6Fie!mt9vZHxt)8Bg47Z!i{3W^$Ma8H^Lu{>hKOHo0Z*e# zYvC*ukzu)+yWwtL^l8faog933qGAtayt)0}G!{8kew_GIp8W(9%o`T4KCxrJ#Y)Yj z&FiCknl(J$s7glR`B^6i4CrC#!}esx@~4&aR!msjR%cmXpUV`7Z1Os)UOo3T@PK%Ezjww$JP@3fY0EG5FRM2lY=M`?+_x|pf6OYTr!n9M z22TGPfHf-bU(2mmG+Bv!Fk>s$=kynpa{A3C)yqz#d_LVZd>vK5wOz4(WENxV^dZ(a zTZK|7PB!9Nu4Ni`M61ljgbTh}3b)Tv>W0f-c6W;322aS`i1jz;q+ovQ-5?il#vHCd zXODh^&A)AL>Viit?VxEhTzMj4^E>Js27*=3PKD;5H6U5j!U!7HB{8vo_gqy^jaHe_ zh=fh2F|F;UP}zc4{wuGPbXVxVE2jkmPv9@m#gU!$4oGtsqBHuMR_{~ zTaCVhuk2;5eU(UsHn5}Va&@RJ{obO?7}U0tK8K$ljhLn+&R>*{Z!3+w*3IsEM5l@W zK&Y7Gos>)+%cTp;pF=5hqqBn9oe6_aL4_#EY-{wHUqzJAw!QJ3Uv*Onqs!7#xr^=V zs26QwhPt`DP79L`gYiDJv@L38)mRt@CC$Cj;RSJj)rc~;d>AXb3(=IJa?si?in4Ur z_?v@B118Dh9ma&}GriuUjkVlnMw#^*Uq4s$p~|);3{RiGjOC7{J%jw75vO-`lr`M$ zQ{q#A4%CKRCn>zhoAj?bVI#_v+Q`3rWNrUcWkzfMi{s1!XyvD6r%k4k+N|DO3*_{g zSk&IRODAT!sxb$f^O`^F@5|kbNoOfT59wO|l7i8WUyX!zNHEG9qBw8JTk1JbtY^=1 zUs#`@;`&bIj+_F2kDfXKRAQ+nqKI;X5vy zHcTVU>Bt)O&P>T4d7`tx+9)Vw^Y9{@nK{r&2QxWe?kJl!#>k!I?G@UWl%5sF@2?>p zS4Bk(FY-;dy$#%@v%{CNvrN2}aD`}Ad-&skBGG$iCJDo?#Vf}lIbEGi%%taQv>tfht{pNIY0KyhjSpu+Ud&j=np$PQK;O@&cwJgkADHc?KE&W! zFw0c=2~vZ%2y=P>u}{>4zQ`!!@Ci@2h711-6_j-`P&mg=|5R&03M5 zabQH+x|L+)gu8{a|2BJ5d>TS%9rixPp%)eZ)pPuFww~;MOkVc`#vt}fS?g%eyk`Fu5(#B$i ze$mtULP2*lsAOlev$H1{i1w%{ThnUA^*q&bDfePY*5cqqgoPYt%G>KdA zP8xAdaQ!lNWdZ8|0mk{__uev1_askjv@dBk#H{*L89Ye&=$x;6O(8C{_MO*L)iq(} z0d0Duc2xP<({kn=dJ)Yh7A`_r>k*@lo*t8m54{^?VmUzFA6;=Ai7ALPysGp- zqO*M9X3f{d)}NiJhqPFXXxqE(IBD0w7&h>moKirh|Sw@dbc?_T4ZF_KKhGb>c#5S+I73SBYsa|@*F zn6ihz*=Oy9zG7f*8&iXiKUJbNk^DOzrt|<+ey3%oKacZWbsbfqkp6V>ax22_b z-}p1qwS@E0m#PPdCAK4uWoYvNvpa87EE+H}p?47@ia)E{?IJc-%v7AcsIgh3NUd`_ z;Oh*%cn&42uuS((iA`~lT$jv(_lfInlWVttz1~UCFK*YCtFIbdTfY)^vjgN_WL&_` z6d27<8jYFddNx-nEKoRq5x9~5xyH!uc#d}d_VsaFxBhz|?*vwt6s#RhC^)sEhiX+N zA3|=AI!&0$WrFOrufchoWVX|5r~$rrU*TKbNO#3w^bB4Z>-juGx0B40Cm?=YSRKX^Xl&~|L$^U=lIKl?duJ7DH|#nYd=P4@J%C^a8HdJF5X?A2T*|}XX*1o zryl440Mi)W5YAuFe%Jk-n4pM>4oXc=fab9gx6tP%4T;hacfvQmfIAfEkLfGaG85v= zoA(I8qkfsfrZt#h3e1x}$&LEmEFs7EYA9U?};y9C|87l{SAg@{&FMo&N?M$<~IL#Sp!7 z{*Ww+_7%6bSvG@G&OEQvi6KJNY_rVjqEXj1jh1mc(E%nxBul1PyAzZCscM7U+phG1 zey&j8uR<8n{qxO~T&5K%DZQgN_Eg9F3)os$hqR}b%3Q}4|8g>FLYe!*HUsVdn4+D! zVA_+6Yxm7?6{?}{Z1Z`6obOuem(I$LB^TFG3DGqvL=`4^{!%1#+Lp1PaGfp-wN8x(q~hN%FiS#SlnB+9e7q-A!Fg`sLw9aH>S{%v<}kLu%==}38oLx9-oKU04T#cu%I%5*RK8)uX2*E3E47L3 ze7e*ZKGQGhp{E#i5iiZpHl0c2A4^?A?W7Z~|5;tvW7#SOU5mU+MGs=OoPi~4+tocZ zC=rAm3*su#8gyj?POeB~m*~~3#jeXyn%6tbRl643lXZEok7sWRx-JwH z&5wT}eMxn0{Uo-ueH*gAcKy`M?40|%27@+s)Jg_7rB7BXZ8j)QaGip^gElkwaCe6{V)E=)1b8BFOx)uHHck4Rr;iI~BqijVplGYX{fsSMx z-XtsfMc%;zxS<v3DVDaySF29O{qH&%gBb+YUS@wzLo1V&H$e7 z&jz}U{LJV?_6u%)l=SlsziY48o$6nIZe9DLmW=00uK}@yPv;Om?`a&VlI|Ebp^TyY z{3F^dOe}fEu`yHXU(*8qh^-BvyE%9wnX${DhBTQ2^&@>fO?xOs*F8Xn?Hj2IZgYK` z=Mhu?N~7i5V#T^O*Oza6+ddD}{+BrSY*5d(U_1Ssk)1Si8L448Wq;2Cj8@`U!k7G5caps=uWTn4fmz1Nswe1WM9L7_^rm?#g-jA9Y~xxo&BOtg5`lNK~`NY z@w#TBni>k6(1f{yM1!8pdtv_vtZ!^5X6OkBPtXQs<~Ao}^; zR}z77?o;XYqp7RhVMIO(o5_8c%l~L-RjW$Fx_6X~H@6~qJv(ha88t7|OsCU{NnxQZ zfa8lC3fcg|k;l~@BrEkQf(qAHSB`UPw)xV{a(n(_@3?*Ok}t5&j^__ROag|j*RcBy z`&>%J*+66r@A%EKTj2A-NR`cIGoI_I206FCqh>$hh`Z8+%j{Zfv!NnaF?V_EdkHtS z2PF)@*UfH{VCM>_o=I86i7Ys>zO+2EUyi!w+A8&xmJM6P1&7pMTv_@XJiMO(c-a~%QR0WIoNuDh&Ivdp%cuA%no(_*Q5Adozs>~{jWQe zgu9I`d2grL{J!U-uGSPG6$W14$qmji-#HscMxS!uX!bgbBDmv1b&=p2vKkAil7Q!L zH{WvxVs zoW=dOCyiZC$}$xRqW60_3a%h)_tUghrJ7+OwGN1VpU;URgp=a5Rnu7*cQp*HRI%5Z zh0$UjpwJWX0cxGgz?q@YJF;Nxa@-ZSLj+P-j4ET*(jNeBRegP|fX5gmVew|ol#3+n zZ(}no!YZC4yg!LO)|xg%EEpY^pLK@ZuuYb^;-;*d8mW;Qive!}?Q)vl&2Ah2cSbRT z7!-PF?SNIbKp5VLqC7A@Hf)+Dkp$c##$W?_MPr`K+QyxT^Ei?pEy%Q4iJ<(&lQMfHg9zN5hT@Tcz5aO5 z;}%&VHO?s2EYfU1ZnqHN2JPI;zK@eI+Q6owv^$G~HPc4}iW0+bccN+@`m-L6u$JCQ zr||F1NZp4Unsr+uw|OV#cLliZbR*p{>hkO^WNxvemu&pR2Id6_akFG{ro3yEO^Ra) z(7|#Y@g~GNDk`X-cr?`B!+I?2P{~cQ!VRt6-Jfs*f;CHMo`XpHghWxtdq9+es?W*( z)BJ}L>A43@XgB8LuLBSea^P9_chts5u`9kxvOhx1ip%U{d?W_2U(>-}6J&zzt27V- zOrV=!i6X5}2yv>+Z>#K#?oRfHdwnJ*NP9}*w3$5F!j$6k9tvPPUa<%^7gr|i zq$iGPKez`vF%L|M6WsPFtaHYnsQ=~6(&vEzz)C+o*|V;(4`!0OP}^d^OD4{GryDKW z%Le;8KK%A%iRn}s=RJ9u(s%oneW${WDi-=N#MJg&hr2N=%jC z+8?RB8{4Png~85-4Fmve3zH^&qwRMsBo~47qZpB)+jIb^a{03%8BwA1XIw$)0lSNr zs|HrmLfVuQHU^~wFGTfFR2u0H%2wO@&s!=z%H3Xy8s8Ypzk?|?I`|5!Vm}mGy#ciE%1Jb2&AiLLgH|M8fCOEXNqBNFv}S@KD_ zta0Ob{v$IhM}e0~^V&2e^5g&VlElv)bKI11^Pf*Rn<|EY)h`5$U?8f?EgGu?X9qlt z2pJt`BLzHmW$0CB*W5_Rn|iZbKxr-gMnx8L4`GmPEE(U%K3V&<5x4)<(}oIj6z@9Q zhSQp+g|CkVf#NBxQ5gx6nSsiiW=DfvH@+HjzP*p3uIjnlMj7B50X$$&@hTZk6Fn@7H$PDc>kl;Us-KYj zZJ5ZN8PeKZq4s$OZ>?E*p(%FKf=?{`MB_+{hu!6B)%(!&dx03Lc<2R%|Kn>Cj{tcf zhW)rMUUkuq1mfBY+kc-Ugx*1n9~+eDU@-c|_gJ$1^|a-znrG?Gm%0$vACULA9`(KX zvgTk6D|$-FnN)Q#2d%6={+zx%v$8y6Ud+JTn3GI9YTc_5u4%75Ii73^E-XIKykVIk zkDPYqcw)#AL#G9i(xYtCbFnZsiAe#W+%iP$ie}03cHhQR8;atS{%~&wVQ+l6SOnty zk9QzcCve`Ns4G@1*I)LDf~t}ywNFwbp*I?RBfg#X?O*>acq#u4SwuJ0(CX4;Pp39F ziK$!M5jvGHXz1>8{&k&gw#sNTCoD6!N8V^9W7K@9(LAO`)y_R_8M{8}1>A5*yu`yw zjuMru)0WICEP;-&B0H#{U z4~N>{jGv7byRC4g=YhTp%Im#rBNG7E^o%PF#hLAhNz)KF{~e>Ga3k*lP4!2#8|TA- z66vTW>o$%n9dE{%>+I$?Gom>2oU>hnMR@_4b0;m~#1a8m$SA zdf%5gAGH*_(qs1=w!c%=4o6v}oHwp4VzmY9WVFAB;yrGEUnC_IAL^+8#_oU1t5r28 z70=KviMaCf0V4>InWWZIXd`!dVbS_FP`xWCdCCF24QfStQw85#b{;+di?Xre$NrCg zKz|JffhbmeyEVl7oQBgw6G?ymuy(?<6wGCZ1?R`F1D)j3;Q8MTrm+n$^tqZItqm3Q zRMR1SnQ!BU3a9C@gd4_<-;3{e1c##BS*G;*5^M4jYK{S=Si;-_^P=xmUk}8FI z`XZjuFU#*BEPVFFiZZLSs{*k)a_4Pdz;RhfLyrg$x-v9 zjuxgR(w9>Ig9Q8{zHgWJ{vGl9>Ofzu?kK0SU+fzEW>GW!ox3d{VY{M2mA#XZsof8# z9Jn~AGOi($G=2V>w@@Mv`ExzDDjE)#iBg4x7_oUd%JFR(f6z-peF!sy(S+KKQ93z;8H!YZfpk6FCIuxSl>l361 z8C?j=49fsfrkb0a`Oj zxYCd;i<(TBc#~1byA{{->AU=3VJ|2Z=mx+Ql-d=`qv;YA@b~>KjoWDtMVX2R=sWuah=J(Gulm6mks8M+3x@2{rd}=gj$pMT#VI728yHcx1W~Q_erL$_sRDr& zLwBJ#exb{i^#N;h^*tn1g=wJIVeD>>@AtXhK1!$>U;N!JMPVJir_g4b{48Z)v3`ug z#>&*x4-(9J#z5Fy`)U^B2_@>fd+<`s9lEc3^^k97Fp5=|wCHE_F%y_G!lcd8ET9|g z|C*89j@{c_lmkbUmlgfZd-_II^>gqT#E|#i2zf)RAR4wdp3~HO#6wRw-?JK`CT$x&b4%9@#Zv!hJP&G$)EbbNfXDpVo?Brx$} zwF){jlX5X_=W9}a``}7Gm5n7R<%T@{ew$h{?`WPj=^xKI=>LZcho`CgJIR?JWe#To znB@bffYcVQM&I@ZS0MOY?T(D+ltcpr#~wX6y@EqY7eVy?(AVB)&?zoPjW~b1+m;IZ zFqg!E`kRG2qK3wVl2x@=7$F-9UR8V|%OFW2o}&RQ58-5FSn#JmFaMkkMueJ$p+TnB zK?$Kc5HZw@=M2npn6#o{O*0&VteP1W98X-8l(_F(eTP(JB&{uK-vl{kqf+BJWsQ^g zKePK@TLPGquuIu0Yv$ev^Us9tn?n|t^yMHUQUkN7wKAvmIE6wuh%5xx$~|J;8%^ff(W@@9 zsCIILTxO>83P?SJV20LH!y2CY06chOtu0Wg9$|+R^sQ~ASEopX0pdK-a;}w#xE=99 zhZe&&5rEkiN|+XAtQEW~`7 zQzrgNCWOv2#v5{Sbvj z+x;@zeXfSnZdWUWklsnX$_=R&ry`Wj(1{9DJ@@FQ-nIJC2fUQa5edHy&-rH^jhcSU zy3bbFP9jM?S36)1!qqU+;<(YzB`aD}SPNV>v@Hr2{hD(;TVqw`;y_4$$g)>alB-kO z{(vZ5uVw)PapGiwyxx%xPpfQzzw7$qbFLFjzg#v{ud+s*rrtQn1Y!WY`+}Ek%xz_S z1$EKQ8h%^wVd<-DEW!cqxB4EkzRIljv0;hJ@TwOG=PfLn%}TnPK(Nv|D=c`F@pNTE$@6M#tOek8M;FIi~T+YUwRY)5ve*#?+1|) z3}r+bo+g0A(+xYXf++=m&R2qN-;1S;v$I3b`Ni$8YHvQCsw*BLL>w!q@biB$@ zR)2zgXw3x+CH~nud#(z*34;n(iTJW4y$QpZ4b{wLY_S`1S{%p4^4O7eO@K``^?sy@ae4|I5_|W z7_I(fXyh*TE%DxYLu@xkqJsO(c1b%Y>?pcU6Y+a}VnOgEe~V`~H|R)Ktl*LPp+93B zmU4B@R$=ge8+@cn*}yHJ1l#&lew*DTVD09%jKpXZW=PBNi3q}Vnp&s3ahBpdH)V1* zs<;(?5o=^{&b2Y7gJ*tdex@K*O}Gp}IhxQ;+);nX`5Ef|8#Mi`?^R`e6OFM2ou>5x zZYM$!da|DPMTZ&W{a)wID06$&t2}A1M}PL`eH6l)paFNE(~;8N=ROL6EU7}8zD1^@ zn()y@aA{twoSDxZklk04Wy?fBDci4hH6k6d1t7h-FWVI$ilFT{mkm|nXd;YckIt33 zV17O|O04%YYUEO%`#Rn$QjP&WVl5baH8bL?{j-QIY53ekkg| zPghzp3L^Qe#Pmw7B5#~m{D53IK_#mdr85kH*=I9e{?qte-FhQ(m+a`FDBC|A7?jha z>`Mt(s*`$wy^u5Tu2T;q<7rDEJ1Y88_I?~S45;nlwvuDJ`oj0nl8@hUb4mVW1d$a6 zB;`n4Ci4IYOO08|4n{F^T$$M2>8t>CNED znDP+QgaN6za=}T+HlJHz0O9Z8ep~!WcwsVb2CV>ARpZWoWFrOiq8Fpeb|FiSD@M*p z=9Pm}h~>5tvG*)$%!hhzH7LxJHEa%kPb24v|GGHmks6j>tx8p>?|}LWA!OmS29y96 z>yf$X{W?f2SLTe5p)=pb_SScqEj~8b9y^tx*A^ygxl%-WVJ=;CIDZlo3`b%u)j={5 zo@qT7-3(cuin+*7u_fsIgOS~GfrHpoG)-Ru@p`w>jUmPR7@kKZ&LEugsy-<`9FeQj z?RWK*NKEnK@x5Vn<9&@au>%e+Gm87w8}gi26KcRV|FJ|5gxHGJ`Y{%I$2u;sL;QE( z*)~&3M>)UZZ&uru%$vP>RGW1Xr$n%Mww3jHz20=W;B5Q;W1OepM-=l^iCk-GqN~oi zsmC*|zXb9xuzz%c=4a~K)pQ@AR&M9|GI&tne?%3rU{|h*=n~VSe literal 0 HcmV?d00001 From 2227f2cc626c457cf8a782155080d9975eeb3405 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Mon, 5 Aug 2019 08:47:39 -0700 Subject: [PATCH 179/523] Update Kubernetes from v1.15.1 to v1.15.2 * https://github.com/kubernetes/kubernetes/blob/master/CHANGELOG-1.15.md#v1152 --- CHANGES.md | 1 + README.md | 10 +++++----- aws/container-linux/kubernetes/README.md | 2 +- aws/container-linux/kubernetes/bootkube.tf | 2 +- .../kubernetes/cl/controller.yaml.tmpl | 2 +- .../kubernetes/workers/cl/worker.yaml.tmpl | 4 ++-- aws/fedora-coreos/kubernetes/README.md | 2 +- aws/fedora-coreos/kubernetes/bootkube.tf | 2 +- aws/fedora-coreos/kubernetes/fcc/controller.yaml | 2 +- .../kubernetes/workers/fcc/worker.yaml | 2 +- azure/container-linux/kubernetes/README.md | 2 +- azure/container-linux/kubernetes/bootkube.tf | 2 +- .../kubernetes/cl/controller.yaml.tmpl | 2 +- .../kubernetes/workers/cl/worker.yaml.tmpl | 4 ++-- bare-metal/container-linux/kubernetes/README.md | 2 +- bare-metal/container-linux/kubernetes/bootkube.tf | 2 +- .../kubernetes/cl/controller.yaml.tmpl | 2 +- .../container-linux/kubernetes/cl/worker.yaml.tmpl | 2 +- bare-metal/fedora-coreos/kubernetes/README.md | 2 +- bare-metal/fedora-coreos/kubernetes/bootkube.tf | 2 +- .../fedora-coreos/kubernetes/fcc/controller.yaml | 2 +- .../fedora-coreos/kubernetes/fcc/worker.yaml | 2 +- digital-ocean/container-linux/kubernetes/README.md | 2 +- .../container-linux/kubernetes/bootkube.tf | 2 +- .../kubernetes/cl/controller.yaml.tmpl | 2 +- .../container-linux/kubernetes/cl/worker.yaml.tmpl | 4 ++-- docs/advanced/worker-pools.md | 14 +++++++------- docs/cl/aws.md | 10 +++++----- docs/cl/azure.md | 10 +++++----- docs/cl/bare-metal.md | 14 +++++++------- docs/cl/digital-ocean.md | 10 +++++----- docs/cl/google-cloud.md | 10 +++++----- docs/fedora-coreos/aws.md | 8 ++++---- docs/fedora-coreos/bare-metal.md | 8 ++++---- docs/index.md | 10 +++++----- docs/topics/maintenance.md | 10 +++++----- google-cloud/container-linux/kubernetes/README.md | 2 +- .../container-linux/kubernetes/bootkube.tf | 2 +- .../kubernetes/cl/controller.yaml.tmpl | 2 +- .../kubernetes/workers/cl/worker.yaml.tmpl | 4 ++-- 40 files changed, 90 insertions(+), 89 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index a45a9ffc9..cad5bf951 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -4,6 +4,7 @@ Notable changes between versions. ## Latest +* Kubernetes [v1.15.2](https://github.com/kubernetes/kubernetes/blob/master/CHANGELOG-1.15.md#v1152) * Update Calico from v3.8.0 to [v3.8.1](https://docs.projectcalico.org/v3.8/release-notes/) ## v1.15.1 diff --git a/README.md b/README.md index 4b5a41286..f75961f85 100644 --- a/README.md +++ b/README.md @@ -11,7 +11,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features -* Kubernetes v1.15.1 (upstream, via [kubernetes-incubator/bootkube](https://github.com/kubernetes-incubator/bootkube)) +* Kubernetes v1.15.2 (upstream, via [kubernetes-incubator/bootkube](https://github.com/kubernetes-incubator/bootkube)) * Single or multi-master, [Calico](https://www.projectcalico.org/) or [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/) * Advanced features like [worker pools](https://typhoon.psdn.io/advanced/worker-pools/), [preemptible](https://typhoon.psdn.io/cl/google-cloud/#preemption) workers, and [snippets](https://typhoon.psdn.io/advanced/customization/#container-linux) customization @@ -48,7 +48,7 @@ Define a Kubernetes cluster by using the Terraform module for your chosen platfo ```tf module "google-cloud-yavin" { - source = "git::https://github.com/poseidon/typhoon//google-cloud/container-linux/kubernetes?ref=v1.15.1" + source = "git::https://github.com/poseidon/typhoon//google-cloud/container-linux/kubernetes?ref=v1.15.2" # Google Cloud cluster_name = "yavin" @@ -82,9 +82,9 @@ In 4-8 minutes (varies by platform), the cluster will be ready. This Google Clou $ export KUBECONFIG=/home/user/.secrets/clusters/yavin/auth/kubeconfig $ kubectl get nodes NAME ROLES STATUS AGE VERSION -yavin-controller-0.c.example-com.internal controller,master Ready 6m v1.15.1 -yavin-worker-jrbf.c.example-com.internal node Ready 5m v1.15.1 -yavin-worker-mzdm.c.example-com.internal node Ready 5m v1.15.1 +yavin-controller-0.c.example-com.internal controller,master Ready 6m v1.15.2 +yavin-worker-jrbf.c.example-com.internal node Ready 5m v1.15.2 +yavin-worker-mzdm.c.example-com.internal node Ready 5m v1.15.2 ``` List the pods. diff --git a/aws/container-linux/kubernetes/README.md b/aws/container-linux/kubernetes/README.md index bb7645fac..cd5e3aa09 100644 --- a/aws/container-linux/kubernetes/README.md +++ b/aws/container-linux/kubernetes/README.md @@ -11,7 +11,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features -* Kubernetes v1.15.1 (upstream, via [kubernetes-incubator/bootkube](https://github.com/kubernetes-incubator/bootkube)) +* Kubernetes v1.15.2 (upstream, via [kubernetes-incubator/bootkube](https://github.com/kubernetes-incubator/bootkube)) * Single or multi-master, [Calico](https://www.projectcalico.org/) or [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/) * Advanced features like [worker pools](https://typhoon.psdn.io/advanced/worker-pools/), [spot](https://typhoon.psdn.io/cl/aws/#spot) workers, and [snippets](https://typhoon.psdn.io/advanced/customization/#container-linux) customization diff --git a/aws/container-linux/kubernetes/bootkube.tf b/aws/container-linux/kubernetes/bootkube.tf index ea1721113..d467b2b3e 100644 --- a/aws/container-linux/kubernetes/bootkube.tf +++ b/aws/container-linux/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=83dd5a7cfc5daa9afc259fd4c3ceb22dec9437da" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=c21da0224984493e92dd2dc7bb3b755c564852fc" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/aws/container-linux/kubernetes/cl/controller.yaml.tmpl b/aws/container-linux/kubernetes/cl/controller.yaml.tmpl index 09c3d9a06..c83f7ad48 100644 --- a/aws/container-linux/kubernetes/cl/controller.yaml.tmpl +++ b/aws/container-linux/kubernetes/cl/controller.yaml.tmpl @@ -125,7 +125,7 @@ storage: contents: inline: | KUBELET_IMAGE_URL=docker://k8s.gcr.io/hyperkube - KUBELET_IMAGE_TAG=v1.15.1 + KUBELET_IMAGE_TAG=v1.15.2 - path: /etc/sysctl.d/max-user-watches.conf filesystem: root contents: diff --git a/aws/container-linux/kubernetes/workers/cl/worker.yaml.tmpl b/aws/container-linux/kubernetes/workers/cl/worker.yaml.tmpl index a2bd65db4..74ed194a3 100644 --- a/aws/container-linux/kubernetes/workers/cl/worker.yaml.tmpl +++ b/aws/container-linux/kubernetes/workers/cl/worker.yaml.tmpl @@ -95,7 +95,7 @@ storage: contents: inline: | KUBELET_IMAGE_URL=docker://k8s.gcr.io/hyperkube - KUBELET_IMAGE_TAG=v1.15.1 + KUBELET_IMAGE_TAG=v1.15.2 - path: /etc/sysctl.d/max-user-watches.conf filesystem: root contents: @@ -113,7 +113,7 @@ storage: --volume config,kind=host,source=/etc/kubernetes \ --mount volume=config,target=/etc/kubernetes \ --insecure-options=image \ - docker://k8s.gcr.io/hyperkube:v1.15.1 \ + docker://k8s.gcr.io/hyperkube:v1.15.2 \ --net=host \ --dns=host \ --exec=/kubectl -- --kubeconfig=/etc/kubernetes/kubeconfig delete node $(hostname) diff --git a/aws/fedora-coreos/kubernetes/README.md b/aws/fedora-coreos/kubernetes/README.md index 375287bc1..c70661fc3 100644 --- a/aws/fedora-coreos/kubernetes/README.md +++ b/aws/fedora-coreos/kubernetes/README.md @@ -11,7 +11,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features -* Kubernetes v1.15.1 (upstream, via [kubernetes-incubator/bootkube](https://github.com/kubernetes-incubator/bootkube)) +* Kubernetes v1.15.2 (upstream, via [kubernetes-incubator/bootkube](https://github.com/kubernetes-incubator/bootkube)) * Single or multi-master, [Calico](https://www.projectcalico.org/) or [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/) * Advanced features like [worker pools](https://typhoon.psdn.io/advanced/worker-pools/), [spot](https://typhoon.psdn.io/cl/aws/#spot) workers, and [snippets](https://typhoon.psdn.io/advanced/customization/#container-linux) customization diff --git a/aws/fedora-coreos/kubernetes/bootkube.tf b/aws/fedora-coreos/kubernetes/bootkube.tf index 1a2d53dc4..8b476e834 100644 --- a/aws/fedora-coreos/kubernetes/bootkube.tf +++ b/aws/fedora-coreos/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=83dd5a7cfc5daa9afc259fd4c3ceb22dec9437da" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=c21da0224984493e92dd2dc7bb3b755c564852fc" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/aws/fedora-coreos/kubernetes/fcc/controller.yaml b/aws/fedora-coreos/kubernetes/fcc/controller.yaml index b98424180..32313e7fc 100644 --- a/aws/fedora-coreos/kubernetes/fcc/controller.yaml +++ b/aws/fedora-coreos/kubernetes/fcc/controller.yaml @@ -80,7 +80,7 @@ systemd: --volume /var/run:/var/run \ --volume /var/run/lock:/var/run/lock:z \ --volume /opt/cni/bin:/opt/cni/bin:z \ - k8s.gcr.io/hyperkube:v1.15.1 /hyperkube kubelet \ + k8s.gcr.io/hyperkube:v1.15.2 /hyperkube kubelet \ --anonymous-auth=false \ --authentication-token-webhook \ --authorization-mode=Webhook \ diff --git a/aws/fedora-coreos/kubernetes/workers/fcc/worker.yaml b/aws/fedora-coreos/kubernetes/workers/fcc/worker.yaml index 3940ef6e8..d1e3915e4 100644 --- a/aws/fedora-coreos/kubernetes/workers/fcc/worker.yaml +++ b/aws/fedora-coreos/kubernetes/workers/fcc/worker.yaml @@ -50,7 +50,7 @@ systemd: --volume /var/run:/var/run \ --volume /var/run/lock:/var/run/lock:z \ --volume /opt/cni/bin:/opt/cni/bin:z \ - k8s.gcr.io/hyperkube:v1.15.1 /hyperkube kubelet \ + k8s.gcr.io/hyperkube:v1.15.2 /hyperkube kubelet \ --anonymous-auth=false \ --authentication-token-webhook \ --authorization-mode=Webhook \ diff --git a/azure/container-linux/kubernetes/README.md b/azure/container-linux/kubernetes/README.md index 24a90b2b0..6e00a0c23 100644 --- a/azure/container-linux/kubernetes/README.md +++ b/azure/container-linux/kubernetes/README.md @@ -11,7 +11,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features -* Kubernetes v1.15.1 (upstream, via [kubernetes-incubator/bootkube](https://github.com/kubernetes-incubator/bootkube)) +* Kubernetes v1.15.2 (upstream, via [kubernetes-incubator/bootkube](https://github.com/kubernetes-incubator/bootkube)) * Single or multi-master, [Calico](https://www.projectcalico.org/) or [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/) * Advanced features like [worker pools](https://typhoon.psdn.io/advanced/worker-pools/), [low-priority](https://typhoon.psdn.io/cl/azure/#low-priority) workers, and [snippets](https://typhoon.psdn.io/advanced/customization/#container-linux) customization diff --git a/azure/container-linux/kubernetes/bootkube.tf b/azure/container-linux/kubernetes/bootkube.tf index 1899ac55d..b1a924c1d 100644 --- a/azure/container-linux/kubernetes/bootkube.tf +++ b/azure/container-linux/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=83dd5a7cfc5daa9afc259fd4c3ceb22dec9437da" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=c21da0224984493e92dd2dc7bb3b755c564852fc" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/azure/container-linux/kubernetes/cl/controller.yaml.tmpl b/azure/container-linux/kubernetes/cl/controller.yaml.tmpl index a6eea4a4a..ee172434d 100644 --- a/azure/container-linux/kubernetes/cl/controller.yaml.tmpl +++ b/azure/container-linux/kubernetes/cl/controller.yaml.tmpl @@ -123,7 +123,7 @@ storage: contents: inline: | KUBELET_IMAGE_URL=docker://k8s.gcr.io/hyperkube - KUBELET_IMAGE_TAG=v1.15.1 + KUBELET_IMAGE_TAG=v1.15.2 - path: /etc/sysctl.d/max-user-watches.conf filesystem: root contents: diff --git a/azure/container-linux/kubernetes/workers/cl/worker.yaml.tmpl b/azure/container-linux/kubernetes/workers/cl/worker.yaml.tmpl index b94e23d18..05768f969 100644 --- a/azure/container-linux/kubernetes/workers/cl/worker.yaml.tmpl +++ b/azure/container-linux/kubernetes/workers/cl/worker.yaml.tmpl @@ -93,7 +93,7 @@ storage: contents: inline: | KUBELET_IMAGE_URL=docker://k8s.gcr.io/hyperkube - KUBELET_IMAGE_TAG=v1.15.1 + KUBELET_IMAGE_TAG=v1.15.2 - path: /etc/sysctl.d/max-user-watches.conf filesystem: root contents: @@ -111,7 +111,7 @@ storage: --volume config,kind=host,source=/etc/kubernetes \ --mount volume=config,target=/etc/kubernetes \ --insecure-options=image \ - docker://k8s.gcr.io/hyperkube:v1.15.1 \ + docker://k8s.gcr.io/hyperkube:v1.15.2 \ --net=host \ --dns=host \ --exec=/kubectl -- --kubeconfig=/etc/kubernetes/kubeconfig delete node $(hostname | tr '[:upper:]' '[:lower:]') diff --git a/bare-metal/container-linux/kubernetes/README.md b/bare-metal/container-linux/kubernetes/README.md index 001a140b2..6bf2d62b0 100644 --- a/bare-metal/container-linux/kubernetes/README.md +++ b/bare-metal/container-linux/kubernetes/README.md @@ -11,7 +11,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features -* Kubernetes v1.15.1 (upstream, via [kubernetes-incubator/bootkube](https://github.com/kubernetes-incubator/bootkube)) +* Kubernetes v1.15.2 (upstream, via [kubernetes-incubator/bootkube](https://github.com/kubernetes-incubator/bootkube)) * Single or multi-master, [Calico](https://www.projectcalico.org/) or [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/) * Advanced features like [snippets](https://typhoon.psdn.io/advanced/customization/#container-linux) customization diff --git a/bare-metal/container-linux/kubernetes/bootkube.tf b/bare-metal/container-linux/kubernetes/bootkube.tf index c8a68e184..8dd33d0fc 100644 --- a/bare-metal/container-linux/kubernetes/bootkube.tf +++ b/bare-metal/container-linux/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=83dd5a7cfc5daa9afc259fd4c3ceb22dec9437da" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=c21da0224984493e92dd2dc7bb3b755c564852fc" cluster_name = var.cluster_name api_servers = [var.k8s_domain_name] diff --git a/bare-metal/container-linux/kubernetes/cl/controller.yaml.tmpl b/bare-metal/container-linux/kubernetes/cl/controller.yaml.tmpl index 7528c3cac..b37afdecd 100644 --- a/bare-metal/container-linux/kubernetes/cl/controller.yaml.tmpl +++ b/bare-metal/container-linux/kubernetes/cl/controller.yaml.tmpl @@ -130,7 +130,7 @@ storage: contents: inline: | KUBELET_IMAGE_URL=docker://k8s.gcr.io/hyperkube - KUBELET_IMAGE_TAG=v1.15.1 + KUBELET_IMAGE_TAG=v1.15.2 - path: /etc/hostname filesystem: root mode: 0644 diff --git a/bare-metal/container-linux/kubernetes/cl/worker.yaml.tmpl b/bare-metal/container-linux/kubernetes/cl/worker.yaml.tmpl index 26d826efc..314dbb1dd 100644 --- a/bare-metal/container-linux/kubernetes/cl/worker.yaml.tmpl +++ b/bare-metal/container-linux/kubernetes/cl/worker.yaml.tmpl @@ -91,7 +91,7 @@ storage: contents: inline: | KUBELET_IMAGE_URL=docker://k8s.gcr.io/hyperkube - KUBELET_IMAGE_TAG=v1.15.1 + KUBELET_IMAGE_TAG=v1.15.2 - path: /etc/hostname filesystem: root mode: 0644 diff --git a/bare-metal/fedora-coreos/kubernetes/README.md b/bare-metal/fedora-coreos/kubernetes/README.md index 7d6ac2382..6e7edb46b 100644 --- a/bare-metal/fedora-coreos/kubernetes/README.md +++ b/bare-metal/fedora-coreos/kubernetes/README.md @@ -11,7 +11,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features -* Kubernetes v1.15.1 (upstream, via [kubernetes-incubator/bootkube](https://github.com/kubernetes-incubator/bootkube)) +* Kubernetes v1.15.2 (upstream, via [kubernetes-incubator/bootkube](https://github.com/kubernetes-incubator/bootkube)) * Single or multi-master, [Calico](https://www.projectcalico.org/) or [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/) * Advanced features like [snippets](https://typhoon.psdn.io/advanced/customization/#container-linux) customization diff --git a/bare-metal/fedora-coreos/kubernetes/bootkube.tf b/bare-metal/fedora-coreos/kubernetes/bootkube.tf index 9188500a1..209bf2514 100644 --- a/bare-metal/fedora-coreos/kubernetes/bootkube.tf +++ b/bare-metal/fedora-coreos/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=83dd5a7cfc5daa9afc259fd4c3ceb22dec9437da" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=c21da0224984493e92dd2dc7bb3b755c564852fc" cluster_name = var.cluster_name api_servers = [var.k8s_domain_name] diff --git a/bare-metal/fedora-coreos/kubernetes/fcc/controller.yaml b/bare-metal/fedora-coreos/kubernetes/fcc/controller.yaml index 22d20b81e..933adaae8 100644 --- a/bare-metal/fedora-coreos/kubernetes/fcc/controller.yaml +++ b/bare-metal/fedora-coreos/kubernetes/fcc/controller.yaml @@ -81,7 +81,7 @@ systemd: --volume /opt/cni/bin:/opt/cni/bin:z \ --volume /etc/iscsi:/etc/iscsi \ --volume /sbin/iscsiadm:/sbin/iscsiadm \ - k8s.gcr.io/hyperkube:v1.15.1 /hyperkube kubelet \ + k8s.gcr.io/hyperkube:v1.15.2 /hyperkube kubelet \ --anonymous-auth=false \ --authentication-token-webhook \ --authorization-mode=Webhook \ diff --git a/bare-metal/fedora-coreos/kubernetes/fcc/worker.yaml b/bare-metal/fedora-coreos/kubernetes/fcc/worker.yaml index 5fa9c25ca..5f4483fb7 100644 --- a/bare-metal/fedora-coreos/kubernetes/fcc/worker.yaml +++ b/bare-metal/fedora-coreos/kubernetes/fcc/worker.yaml @@ -51,7 +51,7 @@ systemd: --volume /opt/cni/bin:/opt/cni/bin:z \ --volume /etc/iscsi:/etc/iscsi \ --volume /sbin/iscsiadm:/sbin/iscsiadm \ - k8s.gcr.io/hyperkube:v1.15.1 /hyperkube kubelet \ + k8s.gcr.io/hyperkube:v1.15.2 /hyperkube kubelet \ --anonymous-auth=false \ --authentication-token-webhook \ --authorization-mode=Webhook \ diff --git a/digital-ocean/container-linux/kubernetes/README.md b/digital-ocean/container-linux/kubernetes/README.md index ea079a815..ca047181a 100644 --- a/digital-ocean/container-linux/kubernetes/README.md +++ b/digital-ocean/container-linux/kubernetes/README.md @@ -11,7 +11,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features -* Kubernetes v1.15.1 (upstream, via [kubernetes-incubator/bootkube](https://github.com/kubernetes-incubator/bootkube)) +* Kubernetes v1.15.2 (upstream, via [kubernetes-incubator/bootkube](https://github.com/kubernetes-incubator/bootkube)) * Single or multi-master, [Calico](https://www.projectcalico.org/) or [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/) * Advanced features like [snippets](https://typhoon.psdn.io/advanced/customization/#container-linux) customization diff --git a/digital-ocean/container-linux/kubernetes/bootkube.tf b/digital-ocean/container-linux/kubernetes/bootkube.tf index e9e09078a..d9d6d9b83 100644 --- a/digital-ocean/container-linux/kubernetes/bootkube.tf +++ b/digital-ocean/container-linux/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=83dd5a7cfc5daa9afc259fd4c3ceb22dec9437da" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=c21da0224984493e92dd2dc7bb3b755c564852fc" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/digital-ocean/container-linux/kubernetes/cl/controller.yaml.tmpl b/digital-ocean/container-linux/kubernetes/cl/controller.yaml.tmpl index 97c7ae26a..343fd6fe4 100644 --- a/digital-ocean/container-linux/kubernetes/cl/controller.yaml.tmpl +++ b/digital-ocean/container-linux/kubernetes/cl/controller.yaml.tmpl @@ -129,7 +129,7 @@ storage: contents: inline: | KUBELET_IMAGE_URL=docker://k8s.gcr.io/hyperkube - KUBELET_IMAGE_TAG=v1.15.1 + KUBELET_IMAGE_TAG=v1.15.2 - path: /etc/sysctl.d/max-user-watches.conf filesystem: root contents: diff --git a/digital-ocean/container-linux/kubernetes/cl/worker.yaml.tmpl b/digital-ocean/container-linux/kubernetes/cl/worker.yaml.tmpl index 57276d074..2f80b247d 100644 --- a/digital-ocean/container-linux/kubernetes/cl/worker.yaml.tmpl +++ b/digital-ocean/container-linux/kubernetes/cl/worker.yaml.tmpl @@ -99,7 +99,7 @@ storage: contents: inline: | KUBELET_IMAGE_URL=docker://k8s.gcr.io/hyperkube - KUBELET_IMAGE_TAG=v1.15.1 + KUBELET_IMAGE_TAG=v1.15.2 - path: /etc/sysctl.d/max-user-watches.conf filesystem: root contents: @@ -117,7 +117,7 @@ storage: --volume config,kind=host,source=/etc/kubernetes \ --mount volume=config,target=/etc/kubernetes \ --insecure-options=image \ - docker://k8s.gcr.io/hyperkube:v1.15.1 \ + docker://k8s.gcr.io/hyperkube:v1.15.2 \ --net=host \ --dns=host \ --exec=/kubectl -- --kubeconfig=/etc/kubernetes/kubeconfig delete node $(hostname) diff --git a/docs/advanced/worker-pools.md b/docs/advanced/worker-pools.md index 650abcaa2..8ec39ce31 100644 --- a/docs/advanced/worker-pools.md +++ b/docs/advanced/worker-pools.md @@ -76,7 +76,7 @@ Create a cluster following the Azure [tutorial](../cl/azure.md#cluster). Define ```tf module "ramius-worker-pool" { - source = "git::https://github.com/poseidon/typhoon//azure/container-linux/kubernetes/workers?ref=v1.15.1" + source = "git::https://github.com/poseidon/typhoon//azure/container-linux/kubernetes/workers?ref=v1.15.2" # Azure region = module.azure-ramius.region @@ -142,7 +142,7 @@ Create a cluster following the Google Cloud [tutorial](../cl/google-cloud.md#clu ```tf module "yavin-worker-pool" { - source = "git::https://github.com/poseidon/typhoon//google-cloud/container-linux/kubernetes/workers?ref=v1.15.1" + source = "git::https://github.com/poseidon/typhoon//google-cloud/container-linux/kubernetes/workers?ref=v1.15.2" # Google Cloud region = "europe-west2" @@ -173,11 +173,11 @@ Verify a managed instance group of workers joins the cluster within a few minute ``` $ kubectl get nodes NAME STATUS AGE VERSION -yavin-controller-0.c.example-com.internal Ready 6m v1.15.1 -yavin-worker-jrbf.c.example-com.internal Ready 5m v1.15.1 -yavin-worker-mzdm.c.example-com.internal Ready 5m v1.15.1 -yavin-16x-worker-jrbf.c.example-com.internal Ready 3m v1.15.1 -yavin-16x-worker-mzdm.c.example-com.internal Ready 3m v1.15.1 +yavin-controller-0.c.example-com.internal Ready 6m v1.15.2 +yavin-worker-jrbf.c.example-com.internal Ready 5m v1.15.2 +yavin-worker-mzdm.c.example-com.internal Ready 5m v1.15.2 +yavin-16x-worker-jrbf.c.example-com.internal Ready 3m v1.15.2 +yavin-16x-worker-mzdm.c.example-com.internal Ready 3m v1.15.2 ``` ### Variables diff --git a/docs/cl/aws.md b/docs/cl/aws.md index edf0b0de0..68bf9f1d4 100644 --- a/docs/cl/aws.md +++ b/docs/cl/aws.md @@ -1,6 +1,6 @@ # AWS -In this tutorial, we'll create a Kubernetes v1.15.1 cluster on AWS with Container Linux. +In this tutorial, we'll create a Kubernetes v1.15.2 cluster on AWS with Container Linux. We'll declare a Kubernetes cluster using the Typhoon Terraform module. Then apply the changes to create a VPC, gateway, subnets, security groups, controller instances, worker auto-scaling group, network load balancer, and TLS assets. @@ -70,7 +70,7 @@ Define a Kubernetes cluster using the module `aws/container-linux/kubernetes`. ```tf module "aws-tempest" { - source = "git::https://github.com/poseidon/typhoon//aws/container-linux/kubernetes?ref=v1.15.1" + source = "git::https://github.com/poseidon/typhoon//aws/container-linux/kubernetes?ref=v1.15.2" # AWS cluster_name = "tempest" @@ -135,9 +135,9 @@ In 4-8 minutes, the Kubernetes cluster will be ready. $ export KUBECONFIG=/home/user/.secrets/clusters/tempest/auth/kubeconfig $ kubectl get nodes NAME STATUS ROLES AGE VERSION -ip-10-0-3-155 Ready controller,master 10m v1.15.1 -ip-10-0-26-65 Ready node 10m v1.15.1 -ip-10-0-41-21 Ready node 10m v1.15.1 +ip-10-0-3-155 Ready controller,master 10m v1.15.2 +ip-10-0-26-65 Ready node 10m v1.15.2 +ip-10-0-41-21 Ready node 10m v1.15.2 ``` List the pods. diff --git a/docs/cl/azure.md b/docs/cl/azure.md index 84777f205..f68b11e01 100644 --- a/docs/cl/azure.md +++ b/docs/cl/azure.md @@ -3,7 +3,7 @@ !!! danger Typhoon for Azure is alpha. For production, use AWS, Google Cloud, or bare-metal. As Azure matures, check [errata](https://github.com/poseidon/typhoon/wiki/Errata) for known shortcomings. -In this tutorial, we'll create a Kubernetes v1.15.1 cluster on Azure with Container Linux. +In this tutorial, we'll create a Kubernetes v1.15.2 cluster on Azure with Container Linux. We'll declare a Kubernetes cluster using the Typhoon Terraform module. Then apply the changes to create a resource group, virtual network, subnets, security groups, controller availability set, worker scale set, load balancer, and TLS assets. @@ -66,7 +66,7 @@ Define a Kubernetes cluster using the module `azure/container-linux/kubernetes`. ```tf module "azure-ramius" { - source = "git::https://github.com/poseidon/typhoon//azure/container-linux/kubernetes?ref=v1.15.1" + source = "git::https://github.com/poseidon/typhoon//azure/container-linux/kubernetes?ref=v1.15.2" # Azure cluster_name = "ramius" @@ -132,9 +132,9 @@ In 4-8 minutes, the Kubernetes cluster will be ready. $ export KUBECONFIG=/home/user/.secrets/clusters/ramius/auth/kubeconfig $ kubectl get nodes NAME STATUS ROLES AGE VERSION -ramius-controller-0 Ready controller,master 24m v1.15.1 -ramius-worker-000001 Ready node 25m v1.15.1 -ramius-worker-000002 Ready node 24m v1.15.1 +ramius-controller-0 Ready controller,master 24m v1.15.2 +ramius-worker-000001 Ready node 25m v1.15.2 +ramius-worker-000002 Ready node 24m v1.15.2 ``` List the pods. diff --git a/docs/cl/bare-metal.md b/docs/cl/bare-metal.md index 76c85a3f1..70c24919e 100644 --- a/docs/cl/bare-metal.md +++ b/docs/cl/bare-metal.md @@ -1,6 +1,6 @@ # Bare-Metal -In this tutorial, we'll network boot and provision a Kubernetes v1.15.1 cluster on bare-metal with Container Linux. +In this tutorial, we'll network boot and provision a Kubernetes v1.15.2 cluster on bare-metal with Container Linux. First, we'll deploy a [Matchbox](https://github.com/poseidon/matchbox) service and setup a network boot environment. Then, we'll declare a Kubernetes cluster using the Typhoon Terraform module and power on machines. On PXE boot, machines will install Container Linux to disk, reboot into the disk install, and provision themselves as Kubernetes controllers or workers via Ignition. @@ -160,7 +160,7 @@ Define a Kubernetes cluster using the module `bare-metal/container-linux/kuberne ```tf module "bare-metal-mercury" { - source = "git::https://github.com/poseidon/typhoon//bare-metal/container-linux/kubernetes?ref=v1.15.1" + source = "git::https://github.com/poseidon/typhoon//bare-metal/container-linux/kubernetes?ref=v1.15.2" # bare-metal cluster_name = "mercury" @@ -265,9 +265,9 @@ Apply complete! Resources: 55 added, 0 changed, 0 destroyed. To watch the install to disk (until machines reboot from disk), SSH to port 2222. ``` -# before v1.15.1 +# before v1.15.2 $ ssh debug@node1.example.com -# after v1.15.1 +# after v1.15.2 $ ssh -p 2222 core@node1.example.com ``` @@ -292,9 +292,9 @@ bootkube[5]: Tearing down temporary bootstrap control plane... $ export KUBECONFIG=/home/user/.secrets/clusters/mercury/auth/kubeconfig $ kubectl get nodes NAME STATUS ROLES AGE VERSION -node1.example.com Ready controller,master 10m v1.15.1 -node2.example.com Ready node 10m v1.15.1 -node3.example.com Ready node 10m v1.15.1 +node1.example.com Ready controller,master 10m v1.15.2 +node2.example.com Ready node 10m v1.15.2 +node3.example.com Ready node 10m v1.15.2 ``` List the pods. diff --git a/docs/cl/digital-ocean.md b/docs/cl/digital-ocean.md index 71cf4b2ac..4c971a7ef 100644 --- a/docs/cl/digital-ocean.md +++ b/docs/cl/digital-ocean.md @@ -1,6 +1,6 @@ # Digital Ocean -In this tutorial, we'll create a Kubernetes v1.15.1 cluster on DigitalOcean with Container Linux. +In this tutorial, we'll create a Kubernetes v1.15.2 cluster on DigitalOcean with Container Linux. We'll declare a Kubernetes cluster using the Typhoon Terraform module. Then apply the changes to create controller droplets, worker droplets, DNS records, tags, and TLS assets. @@ -65,7 +65,7 @@ Define a Kubernetes cluster using the module `digital-ocean/container-linux/kube ```tf module "digital-ocean-nemo" { - source = "git::https://github.com/poseidon/typhoon//digital-ocean/container-linux/kubernetes?ref=v1.15.1" + source = "git::https://github.com/poseidon/typhoon//digital-ocean/container-linux/kubernetes?ref=v1.15.2" # Digital Ocean cluster_name = "nemo" @@ -130,9 +130,9 @@ In 3-6 minutes, the Kubernetes cluster will be ready. $ export KUBECONFIG=/home/user/.secrets/clusters/nemo/auth/kubeconfig $ kubectl get nodes NAME STATUS ROLES AGE VERSION -10.132.110.130 Ready controller,master 10m v1.15.1 -10.132.115.81 Ready node 10m v1.15.1 -10.132.124.107 Ready node 10m v1.15.1 +10.132.110.130 Ready controller,master 10m v1.15.2 +10.132.115.81 Ready node 10m v1.15.2 +10.132.124.107 Ready node 10m v1.15.2 ``` List the pods. diff --git a/docs/cl/google-cloud.md b/docs/cl/google-cloud.md index d6aef86d3..160097967 100644 --- a/docs/cl/google-cloud.md +++ b/docs/cl/google-cloud.md @@ -1,6 +1,6 @@ # Google Cloud -In this tutorial, we'll create a Kubernetes v1.15.1 cluster on Google Compute Engine with Container Linux. +In this tutorial, we'll create a Kubernetes v1.15.2 cluster on Google Compute Engine with Container Linux. We'll declare a Kubernetes cluster using the Typhoon Terraform module. Then apply the changes to create a network, firewall rules, health checks, controller instances, worker managed instance group, load balancers, and TLS assets. @@ -71,7 +71,7 @@ Define a Kubernetes cluster using the module `google-cloud/container-linux/kuber ```tf module "google-cloud-yavin" { - source = "git::https://github.com/poseidon/typhoon//google-cloud/container-linux/kubernetes?ref=v1.15.1" + source = "git::https://github.com/poseidon/typhoon//google-cloud/container-linux/kubernetes?ref=v1.15.2" # Google Cloud cluster_name = "yavin" @@ -138,9 +138,9 @@ In 4-8 minutes, the Kubernetes cluster will be ready. $ export KUBECONFIG=/home/user/.secrets/clusters/yavin/auth/kubeconfig $ kubectl get nodes NAME ROLES STATUS AGE VERSION -yavin-controller-0.c.example-com.internal controller,master Ready 6m v1.15.1 -yavin-worker-jrbf.c.example-com.internal node Ready 5m v1.15.1 -yavin-worker-mzdm.c.example-com.internal node Ready 5m v1.15.1 +yavin-controller-0.c.example-com.internal controller,master Ready 6m v1.15.2 +yavin-worker-jrbf.c.example-com.internal node Ready 5m v1.15.2 +yavin-worker-mzdm.c.example-com.internal node Ready 5m v1.15.2 ``` List the pods. diff --git a/docs/fedora-coreos/aws.md b/docs/fedora-coreos/aws.md index 0060c6525..37ce42e7e 100644 --- a/docs/fedora-coreos/aws.md +++ b/docs/fedora-coreos/aws.md @@ -3,7 +3,7 @@ !!! danger Typhoon for Fedora CoreOS is an early preview! Fedora CoreOS itself is a preview! Expect bugs and design shifts. Please help both projects solve problems. Report Fedora CoreOS bugs to [Fedora](https://github.com/coreos/fedora-coreos-tracker/issues). Report Typhoon issues to Typhoon. -In this tutorial, we'll create a Kubernetes v1.15.1 cluster on AWS with Fedora CoreOS. +In this tutorial, we'll create a Kubernetes v1.15.2 cluster on AWS with Fedora CoreOS. We'll declare a Kubernetes cluster using the Typhoon Terraform module. Then apply the changes to create a VPC, gateway, subnets, security groups, controller instances, worker auto-scaling group, network load balancer, and TLS assets. @@ -138,9 +138,9 @@ In 4-8 minutes, the Kubernetes cluster will be ready. $ export KUBECONFIG=/home/user/.secrets/clusters/tempest/auth/kubeconfig $ kubectl get nodes NAME STATUS ROLES AGE VERSION -ip-10-0-3-155 Ready controller,master 10m v1.15.1 -ip-10-0-26-65 Ready node 10m v1.15.1 -ip-10-0-41-21 Ready node 10m v1.15.1 +ip-10-0-3-155 Ready controller,master 10m v1.15.2 +ip-10-0-26-65 Ready node 10m v1.15.2 +ip-10-0-41-21 Ready node 10m v1.15.2 ``` List the pods. diff --git a/docs/fedora-coreos/bare-metal.md b/docs/fedora-coreos/bare-metal.md index 5401130e2..7aee0fcf0 100644 --- a/docs/fedora-coreos/bare-metal.md +++ b/docs/fedora-coreos/bare-metal.md @@ -3,7 +3,7 @@ !!! danger Typhoon for Fedora CoreOS is an early preview! Fedora CoreOS itself is a preview! Expect bugs and design shifts. Please help both projects solve problems. Report Fedora CoreOS bugs to [Fedora](https://github.com/coreos/fedora-coreos-tracker/issues). Report Typhoon issues to Typhoon. -In this tutorial, we'll network boot and provision a Kubernetes v1.15.1 cluster on bare-metal with Fedora CoreOS. +In this tutorial, we'll network boot and provision a Kubernetes v1.15.2 cluster on bare-metal with Fedora CoreOS. First, we'll deploy a [Matchbox](https://github.com/poseidon/matchbox) service and setup a network boot environment. Then, we'll declare a Kubernetes cluster using the Typhoon Terraform module and power on machines. On PXE boot, machines will install Fedora CoreOS to disk, reboot into the disk install, and provision themselves as Kubernetes controllers or workers via Ignition. @@ -284,9 +284,9 @@ bootkube[5]: Tearing down temporary bootstrap control plane... $ export KUBECONFIG=/home/user/.secrets/clusters/mercury/auth/kubeconfig $ kubectl get nodes NAME STATUS ROLES AGE VERSION -node1.example.com Ready controller,master 10m v1.15.1 -node2.example.com Ready node 10m v1.15.1 -node3.example.com Ready node 10m v1.15.1 +node1.example.com Ready controller,master 10m v1.15.2 +node2.example.com Ready node 10m v1.15.2 +node3.example.com Ready node 10m v1.15.2 ``` List the pods. diff --git a/docs/index.md b/docs/index.md index cdd5bb00d..dbe2a1e06 100644 --- a/docs/index.md +++ b/docs/index.md @@ -11,7 +11,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features -* Kubernetes v1.15.1 (upstream, via [kubernetes-incubator/bootkube](https://github.com/kubernetes-incubator/bootkube)) +* Kubernetes v1.15.2 (upstream, via [kubernetes-incubator/bootkube](https://github.com/kubernetes-incubator/bootkube)) * Single or multi-master, [Calico](https://www.projectcalico.org/) or [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/) * Advanced features like [worker pools](advanced/worker-pools/), [preemptible](cl/google-cloud/#preemption) workers, and [snippets](advanced/customization/#container-linux) customization @@ -47,7 +47,7 @@ Define a Kubernetes cluster by using the Terraform module for your chosen platfo ```tf module "google-cloud-yavin" { - source = "git::https://github.com/poseidon/typhoon//google-cloud/container-linux/kubernetes?ref=v1.15.1" + source = "git::https://github.com/poseidon/typhoon//google-cloud/container-linux/kubernetes?ref=v1.15.2" # Google Cloud cluster_name = "yavin" @@ -80,9 +80,9 @@ In 4-8 minutes (varies by platform), the cluster will be ready. This Google Clou $ export KUBECONFIG=/home/user/.secrets/clusters/yavin/auth/kubeconfig $ kubectl get nodes NAME ROLES STATUS AGE VERSION -yavin-controller-0.c.example-com.internal controller,master Ready 6m v1.15.1 -yavin-worker-jrbf.c.example-com.internal node Ready 5m v1.15.1 -yavin-worker-mzdm.c.example-com.internal node Ready 5m v1.15.1 +yavin-controller-0.c.example-com.internal controller,master Ready 6m v1.15.2 +yavin-worker-jrbf.c.example-com.internal node Ready 5m v1.15.2 +yavin-worker-mzdm.c.example-com.internal node Ready 5m v1.15.2 ``` List the pods. diff --git a/docs/topics/maintenance.md b/docs/topics/maintenance.md index 49253a4c8..5eac215fe 100644 --- a/docs/topics/maintenance.md +++ b/docs/topics/maintenance.md @@ -18,7 +18,7 @@ module "google-cloud-yavin" { } module "bare-metal-mercury" { - source = "git::https://github.com/poseidon/typhoon//bare-metal/container-linux/kubernetes?ref=v1.15.1" + source = "git::https://github.com/poseidon/typhoon//bare-metal/container-linux/kubernetes?ref=v1.15.2" ... } ``` @@ -279,15 +279,15 @@ Typhoon modules have been adapted for Terraform v0.12. Provider plugins requirem | Typhoon Release | Terraform version | |-------------------|---------------------| -| v1.15.1 - ? | v0.12.x | -| v1.10.3 - v1.15.1 | v0.11.x | +| v1.15.2 - ? | v0.12.x | +| v1.10.3 - v1.15.2 | v0.11.x | | v1.9.2 - v1.10.2 | v0.10.4+ or v0.11.x | | v1.7.3 - v1.9.1 | v0.10.x | | v1.6.4 - v1.7.2 | v0.9.x | ### New users -New users can start with Terraform v0.12.x and follow the docs for Typhoon v1.15.1+ without issue. +New users can start with Terraform v0.12.x and follow the docs for Typhoon v1.15.2+ without issue. ### Existing users @@ -404,7 +404,7 @@ tree . └── infraB <- new Terraform v0.12.x configs ``` -Define Typhoon clusters in the new config directory using Terraform v0.12 syntax. Follow the Typhoon v1.15.1+ docs (e.g. use `terraform12` in the `infraB` dir). See [AWS](/cl/aws), [Azure](/cl/azure), [Bare-Metal](/cl/bare-metal), [Digital Ocean](/cl/digital-ocean), or [Google-Cloud](/cl/google-cloud)) to create new clusters. Follow the usual [upgrade](/topics/maintenance/#upgrades) process to apply workloads and shift traffic. Later, switch back to the old config directory and deprovision clusters with Terraform v0.11. +Define Typhoon clusters in the new config directory using Terraform v0.12 syntax. Follow the Typhoon v1.15.2+ docs (e.g. use `terraform12` in the `infraB` dir). See [AWS](/cl/aws), [Azure](/cl/azure), [Bare-Metal](/cl/bare-metal), [Digital Ocean](/cl/digital-ocean), or [Google-Cloud](/cl/google-cloud)) to create new clusters. Follow the usual [upgrade](/topics/maintenance/#upgrades) process to apply workloads and shift traffic. Later, switch back to the old config directory and deprovision clusters with Terraform v0.11. ```shell terraform12 init diff --git a/google-cloud/container-linux/kubernetes/README.md b/google-cloud/container-linux/kubernetes/README.md index 2c996e92c..9d148ba74 100644 --- a/google-cloud/container-linux/kubernetes/README.md +++ b/google-cloud/container-linux/kubernetes/README.md @@ -11,7 +11,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features -* Kubernetes v1.15.1 (upstream, via [kubernetes-incubator/bootkube](https://github.com/kubernetes-incubator/bootkube)) +* Kubernetes v1.15.2 (upstream, via [kubernetes-incubator/bootkube](https://github.com/kubernetes-incubator/bootkube)) * Single or multi-master, [Calico](https://www.projectcalico.org/) or [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/) * Advanced features like [worker pools](https://typhoon.psdn.io/advanced/worker-pools/), [preemptible](https://typhoon.psdn.io/cl/google-cloud/#preemption) workers, and [snippets](https://typhoon.psdn.io/advanced/customization/#container-linux) customization diff --git a/google-cloud/container-linux/kubernetes/bootkube.tf b/google-cloud/container-linux/kubernetes/bootkube.tf index 56845e09e..d83104b74 100644 --- a/google-cloud/container-linux/kubernetes/bootkube.tf +++ b/google-cloud/container-linux/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=83dd5a7cfc5daa9afc259fd4c3ceb22dec9437da" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=c21da0224984493e92dd2dc7bb3b755c564852fc" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/google-cloud/container-linux/kubernetes/cl/controller.yaml.tmpl b/google-cloud/container-linux/kubernetes/cl/controller.yaml.tmpl index 9bf32eb27..61ee6d538 100644 --- a/google-cloud/container-linux/kubernetes/cl/controller.yaml.tmpl +++ b/google-cloud/container-linux/kubernetes/cl/controller.yaml.tmpl @@ -124,7 +124,7 @@ storage: contents: inline: | KUBELET_IMAGE_URL=docker://k8s.gcr.io/hyperkube - KUBELET_IMAGE_TAG=v1.15.1 + KUBELET_IMAGE_TAG=v1.15.2 - path: /etc/sysctl.d/max-user-watches.conf filesystem: root contents: diff --git a/google-cloud/container-linux/kubernetes/workers/cl/worker.yaml.tmpl b/google-cloud/container-linux/kubernetes/workers/cl/worker.yaml.tmpl index 4ce7668e8..aefa0d8de 100644 --- a/google-cloud/container-linux/kubernetes/workers/cl/worker.yaml.tmpl +++ b/google-cloud/container-linux/kubernetes/workers/cl/worker.yaml.tmpl @@ -94,7 +94,7 @@ storage: contents: inline: | KUBELET_IMAGE_URL=docker://k8s.gcr.io/hyperkube - KUBELET_IMAGE_TAG=v1.15.1 + KUBELET_IMAGE_TAG=v1.15.2 - path: /etc/sysctl.d/max-user-watches.conf filesystem: root contents: @@ -112,7 +112,7 @@ storage: --volume config,kind=host,source=/etc/kubernetes \ --mount volume=config,target=/etc/kubernetes \ --insecure-options=image \ - docker://k8s.gcr.io/hyperkube:v1.15.1 \ + docker://k8s.gcr.io/hyperkube:v1.15.2 \ --net=host \ --dns=host \ --exec=/kubectl -- --kubeconfig=/etc/kubernetes/kubeconfig delete node $(hostname) From 10d4d9e5658ab81591ebbe96534d2e69f1ef3836 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Mon, 5 Aug 2019 22:45:47 -0700 Subject: [PATCH 180/523] Add Grafana dashboards for CoreDNS and Nginx Ingress Controller * Add a CoreDNS dashboard originally based on an upstream dashboard, but now customized according to preferences * Add an Nginx Ingress Controller based on an upstream dashboard, but customized according to preferences --- CHANGES.md | 5 + addons/grafana/dashboards-coredns.yaml | 1035 +++++++++++++++++ addons/grafana/dashboards-nginx-ingress.yaml | 1058 ++++++++++++++++++ addons/grafana/deployment.yaml | 10 + 4 files changed, 2108 insertions(+) create mode 100644 addons/grafana/dashboards-coredns.yaml create mode 100644 addons/grafana/dashboards-nginx-ingress.yaml diff --git a/CHANGES.md b/CHANGES.md index cad5bf951..623b6ddaa 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -6,6 +6,11 @@ Notable changes between versions. * Kubernetes [v1.15.2](https://github.com/kubernetes/kubernetes/blob/master/CHANGELOG-1.15.md#v1152) * Update Calico from v3.8.0 to [v3.8.1](https://docs.projectcalico.org/v3.8/release-notes/) +* Publish new load balancing, TCP/UDP, and firewall [docs](https://typhoon.psdn.io/architecture/aws/) ([#523](https://github.com/poseidon/typhoon/pull/523)) + +#### Addons + +* Add new Grafana dashboards for CoreDNS and Nginx Ingress Controller ([#525](https://github.com/poseidon/typhoon/pull/525)) ## v1.15.1 diff --git a/addons/grafana/dashboards-coredns.yaml b/addons/grafana/dashboards-coredns.yaml new file mode 100644 index 000000000..f2313cf61 --- /dev/null +++ b/addons/grafana/dashboards-coredns.yaml @@ -0,0 +1,1035 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: grafana-dashboards-coredns + namespace: monitoring +data: + coredns.json: |- + { + "__inputs": [ + + ], + "__requires": [ + + ], + "annotations": { + "list": [ + + ] + }, + "description": "CoreDNS", + "editable": false, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "id": null, + "links": [ + + ], + "refresh": "", + "rows": [ + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "gridPos": { + + }, + "id": 2, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": "true", + "total": false, + "values": "true" + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(coredns_dns_request_count_total{instance=~\"$instance\"}[5m])) by (proto)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{proto}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Requests (proto)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "rps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "rps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "gridPos": { + + }, + "id": 3, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": "true", + "total": false, + "values": "true" + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(coredns_dns_request_type_count_total{instance=~\"$instance\"}[5m])) by (type)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{type}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Requests (type)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "rps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "rps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "gridPos": { + + }, + "id": 4, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": "true", + "total": false, + "values": "true" + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(coredns_dns_request_count_total{instance=~\"$instance\"}[5m])) by (zone)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{zone}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Requests (zone)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "rps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "rps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "gridPos": { + + }, + "id": 5, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": "true", + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(coredns_dns_request_duration_seconds_bucket{instance=~\"$instance\"}[5m])) by (le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "99%", + "refId": "A" + }, + { + "expr": "histogram_quantile(0.90, sum(rate(coredns_dns_request_duration_seconds_bucket{instance=~\"$instance\"}[5m])) by (le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "90%", + "refId": "B" + }, + { + "expr": "histogram_quantile(0.50, sum(rate(coredns_dns_request_duration_seconds_bucket{instance=~\"$instance\"}[5m])) by (le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "50%", + "refId": "C" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Latency", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "gridPos": { + + }, + "id": 6, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": "true", + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(coredns_dns_response_rcode_count_total{instance=~\"$instance\"}[5m])) by (rcode)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{rcode}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Response Code", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "rps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "rps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "gridPos": { + + }, + "id": 7, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": "true", + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(coredns_dns_request_size_bytes_bucket{instance=~\"$instance\"}[5m])) by (le,proto))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "%99", + "refId": "A" + }, + { + "expr": "histogram_quantile(0.90, sum(rate(coredns_dns_request_size_bytes_bucket{instance=~\"$instance\"}[5m])) by (le,proto))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "%90", + "refId": "B" + }, + { + "expr": "histogram_quantile(0.50, sum(rate(coredns_dns_request_size_bytes_bucket{instance=~\"$instance\"}[5m])) by (le,proto))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "%50", + "refId": "C" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Request Size", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "gridPos": { + + }, + "id": 8, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": "true", + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(coredns_dns_response_size_bytes_bucket{instance=~\"$instance\"}[5m])) by (le,proto))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "%99", + "refId": "A" + }, + { + "expr": "histogram_quantile(0.90, sum(rate(coredns_dns_response_size_bytes_bucket{instance=~\"$instance\"}[5m])) by (le,proto))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "%90", + "refId": "B" + }, + { + "expr": "histogram_quantile(0.50, sum(rate(coredns_dns_response_size_bytes_bucket{instance=~\"$instance\"}[5m])) by (le,proto))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "%50", + "refId": "C" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Response Size", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "gridPos": { + + }, + "id": 9, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": "true", + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(coredns_cache_size{instance=~\"$instance\"}) by (type)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{type}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Cache Size", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "gridPos": { + + }, + "id": 10, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": "true", + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(coredns_cache_hits_total{instance=~\"$instance\"}[5m])) by (type)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{type}} hits", + "refId": "A" + }, + { + "expr": "sum(rate(coredns_cache_misses_total{instance=~\"$instance\"}[5m])) by (type)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{type}} misses", + "refId": "B" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Cache Hit Rate", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + } + ], + "schemaVersion": 14, + "style": "dark", + "tags": [ + "coredns" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": null, + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": true, + "label": null, + "multi": false, + "name": "instance", + "options": [ + + ], + "query": "label_values(coredns_build_info{job=\"coredns\"}, instance)", + "refresh": 2, + "regex": "", + "sort": 0, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "browser", + "title": "CoreDNS", + "version": 0 + } diff --git a/addons/grafana/dashboards-nginx-ingress.yaml b/addons/grafana/dashboards-nginx-ingress.yaml new file mode 100644 index 000000000..4a993e6c3 --- /dev/null +++ b/addons/grafana/dashboards-nginx-ingress.yaml @@ -0,0 +1,1058 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: grafana-dashboards-nginx-ingress + namespace: monitoring +data: + nginx.json: |- + { + "__inputs": [ + + ], + "__requires": [ + + ], + "annotations": { + "list": [ + + ] + }, + "description": "Nginx Ingress Controller", + "editable": false, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "id": null, + "links": [ + + ], + "refresh": "", + "rows": [ + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "$datasource", + "format": "rps", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + + }, + "height": "100px", + "id": 2, + "interval": null, + "links": [ + + ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "span": 4, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": "true", + "lineColor": "rgb(31, 120, 193)", + "show": "true" + }, + "tableColumn": "", + "targets": [ + { + "expr": "round(sum(irate(nginx_ingress_controller_requests{controller_pod=~\"$controller\",controller_class=~\"$controller_class\",namespace=~\"$namespace\"}[2m])), 0.01)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": "", + "title": "Request Volume", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "$datasource", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + + }, + "height": "100px", + "id": 3, + "interval": null, + "links": [ + + ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "span": 4, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": "true", + "lineColor": "rgb(31, 120, 193)", + "show": "true" + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum(avg_over_time(nginx_ingress_controller_nginx_process_connections{controller_pod=~\"$controller\",controller_class=~\"$controller_class\",controller_namespace=~\"$namespace\"}[2m]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": "", + "title": "Connections", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "$datasource", + "format": "percentunit", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + + }, + "height": "100px", + "id": 4, + "interval": null, + "links": [ + + ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "span": 4, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": "true", + "lineColor": "rgb(31, 120, 193)", + "show": "true" + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum(rate(nginx_ingress_controller_requests{controller_pod=~\"$controller\",controller_class=~\"$controller_class\",namespace=~\"$namespace\",status!~\"[4-5].*\"}[2m])) / sum(rate(nginx_ingress_controller_requests{controller_pod=~\"$controller\",controller_class=~\"$controller_class\",namespace=~\"$namespace\"}[2m]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": "", + "title": "Success Rate", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "gridPos": { + + }, + "id": 5, + "legend": { + "alignAsTable": "true", + "avg": false, + "current": "true", + "max": false, + "min": false, + "rightSide": "true", + "show": "true", + "total": false, + "values": "true" + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "round(sum(irate(nginx_ingress_controller_requests{controller_pod=~\"$controller\",controller_class=~\"$controller_class\",controller_namespace=~\"$namespace\",ingress=~\"$ingress\"}[2m])) by (ingress), 0.01)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{ingress}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Request Rate", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "rps", + "label": null, + "logBase": 2, + "max": null, + "min": null, + "show": true + }, + { + "format": "rps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "gridPos": { + + }, + "id": 6, + "legend": { + "alignAsTable": "true", + "avg": false, + "current": "true", + "max": false, + "min": false, + "rightSide": "true", + "show": "true", + "total": false, + "values": "true" + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(nginx_ingress_controller_requests{controller_pod=~\"$controller\",controller_class=~\"$controller_class\",namespace=~\"$namespace\",ingress=~\"$ingress\",status!~\"[4-5].*\"}[2m])) by (ingress) / sum(rate(nginx_ingress_controller_requests{controller_pod=~\"$controller\",controller_class=~\"$controller_class\",namespace=~\"$namespace\",ingress=~\"$ingress\"}[2m])) by (ingress)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{ingress}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Success Rate", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": 1, + "min": null, + "show": true + }, + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": 1, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "gridPos": { + + }, + "id": 7, + "legend": { + "alignAsTable": "true", + "avg": false, + "current": "true", + "max": false, + "min": false, + "rightSide": "true", + "show": "true", + "total": false, + "values": "true" + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(nginx_ingress_controller_request_duration_seconds_bucket{ingress!=\"\",controller_pod=~\"$controller\",controller_class=~\"$controller_class\",controller_namespace=~\"$namespace\",ingress=~\"$ingress\"}[2m])) by (le, ingress))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{ingress}} 99%", + "refId": "A" + }, + { + "expr": "histogram_quantile(0.90, sum(rate(nginx_ingress_controller_request_duration_seconds_bucket{ingress!=\"\",controller_pod=~\"$controller\",controller_class=~\"$controller_class\",controller_namespace=~\"$namespace\",ingress=~\"$ingress\"}[2m])) by (le, ingress))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{ingress}} 90%", + "refId": "B" + }, + { + "expr": "histogram_quantile(0.50, sum(rate(nginx_ingress_controller_request_duration_seconds_bucket{ingress!=\"\",controller_pod=~\"$controller\",controller_class=~\"$controller_class\",controller_namespace=~\"$namespace\",ingress=~\"$ingress\"}[2m])) by (le, ingress))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{ingress}} 50%", + "refId": "C" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Latency", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 2, + "max": null, + "min": null, + "show": true + }, + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "gridPos": { + + }, + "id": 8, + "legend": { + "alignAsTable": "true", + "avg": false, + "current": "true", + "max": false, + "min": false, + "rightSide": false, + "show": "true", + "total": false, + "values": "true" + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum (irate (nginx_ingress_controller_request_size_sum{controller_pod=~\"$controller\",controller_class=~\"$controller_class\",controller_namespace=~\"$namespace\"}[2m]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "received", + "refId": "A" + }, + { + "expr": "sum (irate (nginx_ingress_controller_response_size_sum{controller_pod=~\"$controller\",controller_class=~\"$controller_class\",controller_namespace=~\"$namespace\"}[2m]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "sent", + "refId": "B" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Network I/O Pressure", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "gridPos": { + + }, + "id": 9, + "legend": { + "alignAsTable": "true", + "avg": false, + "current": "true", + "max": false, + "min": false, + "rightSide": false, + "show": "true", + "total": false, + "values": "true" + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "avg(nginx_ingress_controller_nginx_process_resident_memory_bytes{controller_pod=~\"$controller\",controller_class=~\"$controller_class\",controller_namespace=~\"$namespace\"}) by (controller_pod)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{controller_pod}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Avg Memory Usage", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "gridPos": { + + }, + "id": 10, + "legend": { + "alignAsTable": "true", + "avg": false, + "current": "true", + "max": false, + "min": false, + "rightSide": false, + "show": "true", + "total": false, + "values": "true" + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(nginx_ingress_controller_nginx_process_cpu_seconds_total{controller_pod=~\"$controller\",controller_class=~\"$controller_class\",controller_namespace=~\"$namespace\"}[2m])) by (controller_pod)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{controller_pod}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Avg CPU Usage", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + } + ], + "schemaVersion": 14, + "style": "dark", + "tags": [ + "nginx-ingress" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": null, + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": ".*", + "current": { + + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": true, + "label": null, + "multi": false, + "name": "namespace", + "options": [ + + ], + "query": "label_values(nginx_ingress_controller_config_hash, controller_namespace)", + "refresh": 2, + "regex": "", + "sort": 0, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": ".*", + "current": { + + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": true, + "label": null, + "multi": false, + "name": "controller_class", + "options": [ + + ], + "query": "label_values(nginx_ingress_controller_config_hash{namespace=~\"$namespace\"}, controller_class)", + "refresh": 2, + "regex": "", + "sort": 0, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": ".*", + "current": { + + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": true, + "label": null, + "multi": false, + "name": "controller", + "options": [ + + ], + "query": "label_values(nginx_ingress_controller_config_hash{namespace=~\"$namespace\",controller_class=~\"$controller_class\"}, controller_pod)", + "refresh": 2, + "regex": "", + "sort": 0, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": ".*", + "current": { + + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": true, + "label": null, + "multi": false, + "name": "ingress", + "options": [ + + ], + "query": "label_values(nginx_ingress_controller_requests{namespace=~\"$namespace\",controller_class=~\"$controller_class\",controller=~\"$controller\"}, ingress)", + "refresh": 2, + "regex": "", + "sort": 0, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "browser", + "title": "Nginx Ingress Controller", + "version": 0 + } diff --git a/addons/grafana/deployment.yaml b/addons/grafana/deployment.yaml index bdc2d891f..758c190ca 100644 --- a/addons/grafana/deployment.yaml +++ b/addons/grafana/deployment.yaml @@ -64,6 +64,10 @@ spec: mountPath: /etc/grafana/dashboards/k8s-nodes - name: dashboards-k8s-resources mountPath: /etc/grafana/dashboards/k8s-resources + - name: dashboards-coredns + mountPath: /etc/grafana/dashboards/coredns + - name: dashboards-nginx-ingress + mountPath: /etc/grafana/dashboards/nginx-ingress volumes: - name: config configMap: @@ -89,4 +93,10 @@ spec: - name: dashboards-k8s-resources configMap: name: grafana-dashboards-k8s-resources + - name: dashboards-coredns + configMap: + name: grafana-dashboards-coredns + - name: dashboards-nginx-ingress + configMap: + name: grafana-dashboards-nginx-ingress From f79568c02a4a4727fc8efa410d2941d299159419 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Tue, 6 Aug 2019 09:01:22 -0700 Subject: [PATCH 181/523] Add CHANGES section for v1.15.2 release --- CHANGES.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGES.md b/CHANGES.md index 623b6ddaa..10813304d 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -4,6 +4,8 @@ Notable changes between versions. ## Latest +## v1.15.2 + * Kubernetes [v1.15.2](https://github.com/kubernetes/kubernetes/blob/master/CHANGELOG-1.15.md#v1152) * Update Calico from v3.8.0 to [v3.8.1](https://docs.projectcalico.org/v3.8/release-notes/) * Publish new load balancing, TCP/UDP, and firewall [docs](https://typhoon.psdn.io/architecture/aws/) ([#523](https://github.com/poseidon/typhoon/pull/523)) From 457ad18daa94752a9519322fff879800bfe295ea Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Wed, 7 Aug 2019 19:57:38 -0700 Subject: [PATCH 182/523] Update kube-state-metrics from v1.7.1 to v1.7.2 * Add a separate liveness and readiness probe * https://github.com/kubernetes/kube-state-metrics/releases/tag/v1.7.2 --- CHANGES.md | 4 ++++ .../exporters/kube-state-metrics/deployment.yaml | 10 ++++++++-- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 10813304d..c61128886 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -4,6 +4,10 @@ Notable changes between versions. ## Latest +#### Addons + +* Update kube-state-metrics from v1.7.1 to v1.7.2 + ## v1.15.2 * Kubernetes [v1.15.2](https://github.com/kubernetes/kubernetes/blob/master/CHANGELOG-1.15.md#v1152) diff --git a/addons/prometheus/exporters/kube-state-metrics/deployment.yaml b/addons/prometheus/exporters/kube-state-metrics/deployment.yaml index 52c654454..74639bec2 100644 --- a/addons/prometheus/exporters/kube-state-metrics/deployment.yaml +++ b/addons/prometheus/exporters/kube-state-metrics/deployment.yaml @@ -24,16 +24,22 @@ spec: serviceAccountName: kube-state-metrics containers: - name: kube-state-metrics - image: quay.io/coreos/kube-state-metrics:v1.7.1 + image: quay.io/coreos/kube-state-metrics:v1.7.2 ports: - name: metrics containerPort: 8080 - readinessProbe: + livenessProbe: httpGet: path: /healthz port: 8080 initialDelaySeconds: 5 timeoutSeconds: 5 + readinessProbe: + httpGet: + path: / + port: 8080 + initialDelaySeconds: 5 + timeoutSeconds: 5 - name: addon-resizer image: k8s.gcr.io/addon-resizer:1.8.5 resources: From eaea4d37a270ed503ed09cfe2598f3813a41cc82 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Wed, 7 Aug 2019 20:01:18 -0700 Subject: [PATCH 183/523] Update Grafana from v6.2.5 to v6.3.2 * https://github.com/grafana/grafana/releases/tag/v6.3.2 * https://github.com/grafana/grafana/releases/tag/v6.3.1 * https://github.com/grafana/grafana/releases/tag/v6.3.0 --- CHANGES.md | 1 + addons/grafana/deployment.yaml | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index c61128886..6c6768d65 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -7,6 +7,7 @@ Notable changes between versions. #### Addons * Update kube-state-metrics from v1.7.1 to v1.7.2 +* Update Grafana from v6.2.5 to v6.3.2 ## v1.15.2 diff --git a/addons/grafana/deployment.yaml b/addons/grafana/deployment.yaml index 758c190ca..80a2e3c90 100644 --- a/addons/grafana/deployment.yaml +++ b/addons/grafana/deployment.yaml @@ -23,7 +23,7 @@ spec: spec: containers: - name: grafana - image: docker.io/grafana/grafana:6.2.5 + image: docker.io/grafana/grafana:6.3.2 env: - name: GF_PATHS_CONFIG value: "/etc/grafana/custom.ini" From cad12804c8501c3b1cdb79bc2e3118cfecd887a1 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Wed, 7 Aug 2019 20:42:40 -0700 Subject: [PATCH 184/523] Refresh terraform provider versions used in docs * Sync terraform provider versions with those tested against --- docs/cl/aws.md | 6 +++--- docs/cl/azure.md | 6 +++--- docs/cl/bare-metal.md | 2 +- docs/cl/digital-ocean.md | 4 ++-- docs/cl/google-cloud.md | 4 ++-- docs/fedora-coreos/aws.md | 4 ++-- docs/fedora-coreos/bare-metal.md | 2 +- 7 files changed, 14 insertions(+), 14 deletions(-) diff --git a/docs/cl/aws.md b/docs/cl/aws.md index 68bf9f1d4..ea5f61b9f 100644 --- a/docs/cl/aws.md +++ b/docs/cl/aws.md @@ -18,7 +18,7 @@ Install [Terraform](https://www.terraform.io/downloads.html) v0.12.x on your sys ```sh $ terraform version -Terraform v0.12.2 +Terraform v0.12.6 ``` Add the [terraform-provider-ct](https://github.com/poseidon/terraform-provider-ct) plugin binary for your system to `~/.terraform.d/plugins/`, noting the final name. @@ -49,7 +49,7 @@ Configure the AWS provider to use your access key credentials in a `providers.tf ```tf provider "aws" { - version = "2.15.0" + version = "2.23.0" region = "eu-central-1" shared_credentials_file = "/home/user/.config/aws/credentials" } @@ -69,7 +69,7 @@ Additional configuration options are described in the `aws` provider [docs](http Define a Kubernetes cluster using the module `aws/container-linux/kubernetes`. ```tf -module "aws-tempest" { +module "tempest" { source = "git::https://github.com/poseidon/typhoon//aws/container-linux/kubernetes?ref=v1.15.2" # AWS diff --git a/docs/cl/azure.md b/docs/cl/azure.md index f68b11e01..e63b690b5 100644 --- a/docs/cl/azure.md +++ b/docs/cl/azure.md @@ -21,7 +21,7 @@ Install [Terraform](https://www.terraform.io/downloads.html) v0.12.x on your sys ```sh $ terraform version -Terraform v0.12.2 +Terraform v0.12.6 ``` Add the [terraform-provider-ct](https://github.com/poseidon/terraform-provider-ct) plugin binary for your system to `~/.terraform.d/plugins/`, noting the final name. @@ -50,7 +50,7 @@ Configure the Azure provider in a `providers.tf` file. ```tf provider "azurerm" { - version = "1.30.1" + version = "1.32.1" } provider "ct" { @@ -65,7 +65,7 @@ Additional configuration options are described in the `azurerm` provider [docs]( Define a Kubernetes cluster using the module `azure/container-linux/kubernetes`. ```tf -module "azure-ramius" { +module "ramius" { source = "git::https://github.com/poseidon/typhoon//azure/container-linux/kubernetes?ref=v1.15.2" # Azure diff --git a/docs/cl/bare-metal.md b/docs/cl/bare-metal.md index 70c24919e..725a2e96c 100644 --- a/docs/cl/bare-metal.md +++ b/docs/cl/bare-metal.md @@ -111,7 +111,7 @@ Install [Terraform](https://www.terraform.io/downloads.html) v0.12.x on your sys ```sh $ terraform version -Terraform v0.12.2 +Terraform v0.12.6 ``` Add the [terraform-provider-matchbox](https://github.com/poseidon/terraform-provider-matchbox) plugin binary for your system to `~/.terraform.d/plugins/`, noting the final name. diff --git a/docs/cl/digital-ocean.md b/docs/cl/digital-ocean.md index 4c971a7ef..62f36a321 100644 --- a/docs/cl/digital-ocean.md +++ b/docs/cl/digital-ocean.md @@ -18,7 +18,7 @@ Install [Terraform](https://www.terraform.io/downloads.html) v0.12.x on your sys ```sh $ terraform version -Terraform v0.12.2 +Terraform v0.12.6 ``` Add the [terraform-provider-ct](https://github.com/poseidon/terraform-provider-ct) plugin binary for your system to `~/.terraform.d/plugins/`, noting the final name. @@ -50,7 +50,7 @@ Configure the DigitalOcean provider to use your token in a `providers.tf` file. ```tf provider "digitalocean" { - version = "1.4.0" + version = "1.6.0" token = "${chomp(file("~/.config/digital-ocean/token"))}" } diff --git a/docs/cl/google-cloud.md b/docs/cl/google-cloud.md index 160097967..15b0b25e1 100644 --- a/docs/cl/google-cloud.md +++ b/docs/cl/google-cloud.md @@ -18,7 +18,7 @@ Install [Terraform](https://www.terraform.io/downloads.html) v0.12.x on your sys ```sh $ terraform version -Terraform v0.12.2 +Terraform v0.12.6 ``` Add the [terraform-provider-ct](https://github.com/poseidon/terraform-provider-ct) plugin binary for your system to `~/.terraform.d/plugins/`, noting the final name. @@ -49,7 +49,7 @@ Configure the Google Cloud provider to use your service account key, project-id, ```tf provider "google" { - version = "2.9.0" + version = "2.12.0" project = "project-id" region = "us-central1" credentials = "${file("~/.config/google-cloud/terraform.json")}" diff --git a/docs/fedora-coreos/aws.md b/docs/fedora-coreos/aws.md index 37ce42e7e..4ab27d4a2 100644 --- a/docs/fedora-coreos/aws.md +++ b/docs/fedora-coreos/aws.md @@ -21,7 +21,7 @@ Install [Terraform](https://www.terraform.io/downloads.html) v0.12.x on your sys ```sh $ terraform version -Terraform v0.12.2 +Terraform v0.12.6 ``` Add the [terraform-provider-ct](https://github.com/poseidon/terraform-provider-ct) plugin binary for your system to `~/.terraform.d/plugins/`, noting the final name. @@ -52,7 +52,7 @@ Configure the AWS provider to use your access key credentials in a `providers.tf ```tf provider "aws" { - version = "2.19.0" + version = "2.23.0" region = "us-east-1" # MUST be us-east-1 right now! shared_credentials_file = "/home/user/.config/aws/credentials" } diff --git a/docs/fedora-coreos/bare-metal.md b/docs/fedora-coreos/bare-metal.md index 7aee0fcf0..305d17636 100644 --- a/docs/fedora-coreos/bare-metal.md +++ b/docs/fedora-coreos/bare-metal.md @@ -114,7 +114,7 @@ Install [Terraform](https://www.terraform.io/downloads.html) v0.12.x on your sys ```sh $ terraform version -Terraform v0.12.2 +Terraform v0.12.6 ``` Add the [terraform-provider-matchbox](https://github.com/poseidon/terraform-provider-matchbox) plugin binary for your system to `~/.terraform.d/plugins/`, noting the final name. From 6db11d5908ff3e41a291e534598e6216af8eeb49 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Wed, 7 Aug 2019 20:56:55 -0700 Subject: [PATCH 185/523] Enable AWS root block device encryption by default * terraform-provider-aws v2.23.0 allows AWS root block devices to enable encryption by default. * Require updating terraform-provider-aws to v2.23.0 or higher * Enable root EBS device encryption by default for controller instances and worker instances in auto-scaling groups For comparison: * Google Cloud persistent disks have been encrypted by default for years * Azure managed disk encryption is not ready yet (#486) --- CHANGES.md | 5 +++++ aws/container-linux/kubernetes/controllers.tf | 1 + aws/container-linux/kubernetes/versions.tf | 2 +- aws/container-linux/kubernetes/workers/workers.tf | 1 + aws/fedora-coreos/kubernetes/controllers.tf | 1 + aws/fedora-coreos/kubernetes/versions.tf | 2 +- aws/fedora-coreos/kubernetes/workers/workers.tf | 1 + 7 files changed, 11 insertions(+), 2 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 6c6768d65..70b62cda8 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -4,6 +4,11 @@ Notable changes between versions. ## Latest +#### AWS + +* Enable root block device encryption by default ([#527](https://github.com/poseidon/typhoon/pull/527)) + * Require `terraform-provider-aws` v2.23+ (**action required**) + #### Addons * Update kube-state-metrics from v1.7.1 to v1.7.2 diff --git a/aws/container-linux/kubernetes/controllers.tf b/aws/container-linux/kubernetes/controllers.tf index c0553eb9f..75e28c83e 100644 --- a/aws/container-linux/kubernetes/controllers.tf +++ b/aws/container-linux/kubernetes/controllers.tf @@ -31,6 +31,7 @@ resource "aws_instance" "controllers" { volume_type = var.disk_type volume_size = var.disk_size iops = var.disk_iops + encrypted = true } # network diff --git a/aws/container-linux/kubernetes/versions.tf b/aws/container-linux/kubernetes/versions.tf index 62ed9ebfe..5f5ac8a23 100644 --- a/aws/container-linux/kubernetes/versions.tf +++ b/aws/container-linux/kubernetes/versions.tf @@ -3,7 +3,7 @@ terraform { required_version = "~> 0.12.0" required_providers { - aws = "~> 2.7" + aws = "~> 2.23" ct = "~> 0.3" template = "~> 2.1" null = "~> 2.1" diff --git a/aws/container-linux/kubernetes/workers/workers.tf b/aws/container-linux/kubernetes/workers/workers.tf index d470b5407..5effb7890 100644 --- a/aws/container-linux/kubernetes/workers/workers.tf +++ b/aws/container-linux/kubernetes/workers/workers.tf @@ -56,6 +56,7 @@ resource "aws_launch_configuration" "worker" { volume_type = var.disk_type volume_size = var.disk_size iops = var.disk_iops + encrypted = true } # network diff --git a/aws/fedora-coreos/kubernetes/controllers.tf b/aws/fedora-coreos/kubernetes/controllers.tf index 821ad6484..97e792d81 100644 --- a/aws/fedora-coreos/kubernetes/controllers.tf +++ b/aws/fedora-coreos/kubernetes/controllers.tf @@ -31,6 +31,7 @@ resource "aws_instance" "controllers" { volume_type = var.disk_type volume_size = var.disk_size iops = var.disk_iops + encrypted = true } # network diff --git a/aws/fedora-coreos/kubernetes/versions.tf b/aws/fedora-coreos/kubernetes/versions.tf index 5694dd11c..83532feb4 100644 --- a/aws/fedora-coreos/kubernetes/versions.tf +++ b/aws/fedora-coreos/kubernetes/versions.tf @@ -3,7 +3,7 @@ terraform { required_version = "~> 0.12.0" required_providers { - aws = "~> 2.7" + aws = "~> 2.23" ct = "~> 0.4" template = "~> 2.1" null = "~> 2.1" diff --git a/aws/fedora-coreos/kubernetes/workers/workers.tf b/aws/fedora-coreos/kubernetes/workers/workers.tf index f59e114f7..4ea1dec08 100644 --- a/aws/fedora-coreos/kubernetes/workers/workers.tf +++ b/aws/fedora-coreos/kubernetes/workers/workers.tf @@ -56,6 +56,7 @@ resource "aws_launch_configuration" "worker" { volume_type = var.disk_type volume_size = var.disk_size iops = var.disk_iops + encrypted = true } # network From 09eb2365196ed076206b0fe1d9ec52612bdfb024 Mon Sep 17 00:00:00 2001 From: Bob Henkel Date: Wed, 14 Aug 2019 23:25:38 -0500 Subject: [PATCH 186/523] Fix worker_preemptible spelling in GCP docs (#529) --- docs/cl/google-cloud.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/cl/google-cloud.md b/docs/cl/google-cloud.md index 15b0b25e1..c1029b99b 100644 --- a/docs/cl/google-cloud.md +++ b/docs/cl/google-cloud.md @@ -227,5 +227,5 @@ Check the list of valid [machine types](https://cloud.google.com/compute/docs/ma #### Preemption -Add `worker_preemeptible = "true"` to allow worker nodes to be [preempted](https://cloud.google.com/compute/docs/instances/preemptible) at random, but pay [significantly](https://cloud.google.com/compute/pricing) less. Clusters tolerate stopping instances fairly well (reschedules pods, but cannot drain) and preemption provides a nice reward for running fault-tolerant cluster systems.` +Add `worker_preemptible = "true"` to allow worker nodes to be [preempted](https://cloud.google.com/compute/docs/instances/preemptible) at random, but pay [significantly](https://cloud.google.com/compute/pricing) less. Clusters tolerate stopping instances fairly well (reschedules pods, but cannot drain) and preemption provides a nice reward for running fault-tolerant cluster systems.` From 7bc5633c389671a6574e4111db7331ed8c7464da Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Wed, 14 Aug 2019 21:08:31 -0700 Subject: [PATCH 187/523] Update nginx-ingress from v0.25.0 to v0.25.1 * https://github.com/kubernetes/ingress-nginx/releases/tag/nginx-0.25.1 --- CHANGES.md | 2 ++ addons/nginx-ingress/aws/deployment.yaml | 2 +- addons/nginx-ingress/azure/deployment.yaml | 2 +- addons/nginx-ingress/bare-metal/deployment.yaml | 2 +- addons/nginx-ingress/digital-ocean/daemonset.yaml | 2 +- addons/nginx-ingress/google-cloud/deployment.yaml | 2 +- 6 files changed, 7 insertions(+), 5 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 70b62cda8..9e413500c 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -13,6 +13,8 @@ Notable changes between versions. * Update kube-state-metrics from v1.7.1 to v1.7.2 * Update Grafana from v6.2.5 to v6.3.2 +* Update nginx-ingress from v0.25.0 to [v0.25.1](https://github.com/kubernetes/ingress-nginx/releases/tag/nginx-0.25.1) + * Fix Nginx security advisories ## v1.15.2 diff --git a/addons/nginx-ingress/aws/deployment.yaml b/addons/nginx-ingress/aws/deployment.yaml index fdb9cc317..16dce5a39 100644 --- a/addons/nginx-ingress/aws/deployment.yaml +++ b/addons/nginx-ingress/aws/deployment.yaml @@ -24,7 +24,7 @@ spec: node-role.kubernetes.io/node: "" containers: - name: nginx-ingress-controller - image: quay.io/kubernetes-ingress-controller/nginx-ingress-controller:0.25.0 + image: quay.io/kubernetes-ingress-controller/nginx-ingress-controller:0.25.1 args: - /nginx-ingress-controller - --ingress-class=public diff --git a/addons/nginx-ingress/azure/deployment.yaml b/addons/nginx-ingress/azure/deployment.yaml index fdb9cc317..16dce5a39 100644 --- a/addons/nginx-ingress/azure/deployment.yaml +++ b/addons/nginx-ingress/azure/deployment.yaml @@ -24,7 +24,7 @@ spec: node-role.kubernetes.io/node: "" containers: - name: nginx-ingress-controller - image: quay.io/kubernetes-ingress-controller/nginx-ingress-controller:0.25.0 + image: quay.io/kubernetes-ingress-controller/nginx-ingress-controller:0.25.1 args: - /nginx-ingress-controller - --ingress-class=public diff --git a/addons/nginx-ingress/bare-metal/deployment.yaml b/addons/nginx-ingress/bare-metal/deployment.yaml index d45bbc263..2e5c42f54 100644 --- a/addons/nginx-ingress/bare-metal/deployment.yaml +++ b/addons/nginx-ingress/bare-metal/deployment.yaml @@ -22,7 +22,7 @@ spec: spec: containers: - name: nginx-ingress-controller - image: quay.io/kubernetes-ingress-controller/nginx-ingress-controller:0.25.0 + image: quay.io/kubernetes-ingress-controller/nginx-ingress-controller:0.25.1 args: - /nginx-ingress-controller - --ingress-class=public diff --git a/addons/nginx-ingress/digital-ocean/daemonset.yaml b/addons/nginx-ingress/digital-ocean/daemonset.yaml index ab43187e2..fff2970c8 100644 --- a/addons/nginx-ingress/digital-ocean/daemonset.yaml +++ b/addons/nginx-ingress/digital-ocean/daemonset.yaml @@ -24,7 +24,7 @@ spec: node-role.kubernetes.io/node: "" containers: - name: nginx-ingress-controller - image: quay.io/kubernetes-ingress-controller/nginx-ingress-controller:0.25.0 + image: quay.io/kubernetes-ingress-controller/nginx-ingress-controller:0.25.1 args: - /nginx-ingress-controller - --ingress-class=public diff --git a/addons/nginx-ingress/google-cloud/deployment.yaml b/addons/nginx-ingress/google-cloud/deployment.yaml index fdb9cc317..16dce5a39 100644 --- a/addons/nginx-ingress/google-cloud/deployment.yaml +++ b/addons/nginx-ingress/google-cloud/deployment.yaml @@ -24,7 +24,7 @@ spec: node-role.kubernetes.io/node: "" containers: - name: nginx-ingress-controller - image: quay.io/kubernetes-ingress-controller/nginx-ingress-controller:0.25.0 + image: quay.io/kubernetes-ingress-controller/nginx-ingress-controller:0.25.1 args: - /nginx-ingress-controller - --ingress-class=public From 976452825eaef9e47b263badfa11df0611760d8c Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Wed, 14 Aug 2019 21:17:02 -0700 Subject: [PATCH 188/523] Update Prometheus from v2.11.0 to v2.11.2 * https://github.com/prometheus/prometheus/releases/tag/v2.11.2 --- CHANGES.md | 3 ++- addons/prometheus/deployment.yaml | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 9e413500c..1c94bb130 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -11,7 +11,8 @@ Notable changes between versions. #### Addons -* Update kube-state-metrics from v1.7.1 to v1.7.2 +* Update Prometheus from v2.11.0 to [v2.11.2](https://github.com/prometheus/prometheus/releases/tag/v2.11.2) + * Update kube-state-metrics from v1.7.1 to v1.7.2 * Update Grafana from v6.2.5 to v6.3.2 * Update nginx-ingress from v0.25.0 to [v0.25.1](https://github.com/kubernetes/ingress-nginx/releases/tag/nginx-0.25.1) * Fix Nginx security advisories diff --git a/addons/prometheus/deployment.yaml b/addons/prometheus/deployment.yaml index 94b096004..abc543ca5 100644 --- a/addons/prometheus/deployment.yaml +++ b/addons/prometheus/deployment.yaml @@ -20,7 +20,7 @@ spec: serviceAccountName: prometheus containers: - name: prometheus - image: quay.io/prometheus/prometheus:v2.11.0 + image: quay.io/prometheus/prometheus:v2.11.2 args: - --web.listen-address=0.0.0.0:9090 - --config.file=/etc/prometheus/prometheus.yaml From 0c45cd0f06383190c5e34ba543fb564d45338baf Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Fri, 16 Aug 2019 14:40:47 -0700 Subject: [PATCH 189/523] Update Grafana from v6.3.2 to v6.3.3 * https://github.com/grafana/grafana/releases/tag/v6.3.3 --- CHANGES.md | 2 +- addons/grafana/deployment.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 1c94bb130..7bd97f127 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -13,7 +13,7 @@ Notable changes between versions. * Update Prometheus from v2.11.0 to [v2.11.2](https://github.com/prometheus/prometheus/releases/tag/v2.11.2) * Update kube-state-metrics from v1.7.1 to v1.7.2 -* Update Grafana from v6.2.5 to v6.3.2 +* Update Grafana from v6.2.5 to v6.3.3 * Update nginx-ingress from v0.25.0 to [v0.25.1](https://github.com/kubernetes/ingress-nginx/releases/tag/nginx-0.25.1) * Fix Nginx security advisories diff --git a/addons/grafana/deployment.yaml b/addons/grafana/deployment.yaml index 80a2e3c90..a4dbef818 100644 --- a/addons/grafana/deployment.yaml +++ b/addons/grafana/deployment.yaml @@ -23,7 +23,7 @@ spec: spec: containers: - name: grafana - image: docker.io/grafana/grafana:6.3.2 + image: docker.io/grafana/grafana:6.3.3 env: - name: GF_PATHS_CONFIG value: "/etc/grafana/custom.ini" From 3c3708d58eaef76c126ceab7a1b1eb3548986a15 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Fri, 16 Aug 2019 15:38:23 -0700 Subject: [PATCH 190/523] Update Calico from v3.8.1 to v3.8.2 * https://docs.projectcalico.org/v3.8/release-notes/ --- CHANGES.md | 2 ++ aws/container-linux/kubernetes/bootkube.tf | 2 +- aws/fedora-coreos/kubernetes/bootkube.tf | 2 +- azure/container-linux/kubernetes/bootkube.tf | 2 +- bare-metal/container-linux/kubernetes/bootkube.tf | 2 +- bare-metal/fedora-coreos/kubernetes/bootkube.tf | 2 +- digital-ocean/container-linux/kubernetes/bootkube.tf | 2 +- google-cloud/container-linux/kubernetes/bootkube.tf | 2 +- 8 files changed, 9 insertions(+), 7 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 7bd97f127..4f69d655d 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -4,6 +4,8 @@ Notable changes between versions. ## Latest +* Update Calico from v3.8.1 to [v3.8.2](https://docs.projectcalico.org/v3.8/release-notes/) + #### AWS * Enable root block device encryption by default ([#527](https://github.com/poseidon/typhoon/pull/527)) diff --git a/aws/container-linux/kubernetes/bootkube.tf b/aws/container-linux/kubernetes/bootkube.tf index d467b2b3e..d9289ff57 100644 --- a/aws/container-linux/kubernetes/bootkube.tf +++ b/aws/container-linux/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=c21da0224984493e92dd2dc7bb3b755c564852fc" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=8b3738b2cccf3cd9830beac170a53c4fbec26ed5" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/aws/fedora-coreos/kubernetes/bootkube.tf b/aws/fedora-coreos/kubernetes/bootkube.tf index 8b476e834..ea8164ad4 100644 --- a/aws/fedora-coreos/kubernetes/bootkube.tf +++ b/aws/fedora-coreos/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=c21da0224984493e92dd2dc7bb3b755c564852fc" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=8b3738b2cccf3cd9830beac170a53c4fbec26ed5" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/azure/container-linux/kubernetes/bootkube.tf b/azure/container-linux/kubernetes/bootkube.tf index b1a924c1d..7f185f16d 100644 --- a/azure/container-linux/kubernetes/bootkube.tf +++ b/azure/container-linux/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=c21da0224984493e92dd2dc7bb3b755c564852fc" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=8b3738b2cccf3cd9830beac170a53c4fbec26ed5" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/bare-metal/container-linux/kubernetes/bootkube.tf b/bare-metal/container-linux/kubernetes/bootkube.tf index 8dd33d0fc..37198b072 100644 --- a/bare-metal/container-linux/kubernetes/bootkube.tf +++ b/bare-metal/container-linux/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=c21da0224984493e92dd2dc7bb3b755c564852fc" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=8b3738b2cccf3cd9830beac170a53c4fbec26ed5" cluster_name = var.cluster_name api_servers = [var.k8s_domain_name] diff --git a/bare-metal/fedora-coreos/kubernetes/bootkube.tf b/bare-metal/fedora-coreos/kubernetes/bootkube.tf index 209bf2514..97db7c6eb 100644 --- a/bare-metal/fedora-coreos/kubernetes/bootkube.tf +++ b/bare-metal/fedora-coreos/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=c21da0224984493e92dd2dc7bb3b755c564852fc" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=8b3738b2cccf3cd9830beac170a53c4fbec26ed5" cluster_name = var.cluster_name api_servers = [var.k8s_domain_name] diff --git a/digital-ocean/container-linux/kubernetes/bootkube.tf b/digital-ocean/container-linux/kubernetes/bootkube.tf index d9d6d9b83..5e733d592 100644 --- a/digital-ocean/container-linux/kubernetes/bootkube.tf +++ b/digital-ocean/container-linux/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=c21da0224984493e92dd2dc7bb3b755c564852fc" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=8b3738b2cccf3cd9830beac170a53c4fbec26ed5" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/google-cloud/container-linux/kubernetes/bootkube.tf b/google-cloud/container-linux/kubernetes/bootkube.tf index d83104b74..033d01a12 100644 --- a/google-cloud/container-linux/kubernetes/bootkube.tf +++ b/google-cloud/container-linux/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=c21da0224984493e92dd2dc7bb3b755c564852fc" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=8b3738b2cccf3cd9830beac170a53c4fbec26ed5" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] From 99990e3cbb482cc3c3bad77fc98ecadce721d0b2 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Sun, 18 Aug 2019 11:46:10 -0700 Subject: [PATCH 191/523] Use stable IDs for etcd, CoreDNS, and Ngnix dashboards * Use unique dashboard ID so that multiple replicas of Grafana serve dashboards with uniform paths * Fix issue where refreshing a dashboard served by one replica could show a 404 unless the request went to the same replica --- CHANGES.md | 1 + addons/grafana/dashboards-coredns.yaml | 3 ++- addons/grafana/dashboards-etcd.yaml | 3 ++- addons/grafana/dashboards-nginx-ingress.yaml | 3 ++- 4 files changed, 7 insertions(+), 3 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 4f69d655d..0ead71b25 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -16,6 +16,7 @@ Notable changes between versions. * Update Prometheus from v2.11.0 to [v2.11.2](https://github.com/prometheus/prometheus/releases/tag/v2.11.2) * Update kube-state-metrics from v1.7.1 to v1.7.2 * Update Grafana from v6.2.5 to v6.3.3 + * Use stable IDs for etcd, CoreDNS, and Nginx Ingress dashboards * Update nginx-ingress from v0.25.0 to [v0.25.1](https://github.com/kubernetes/ingress-nginx/releases/tag/nginx-0.25.1) * Fix Nginx security advisories diff --git a/addons/grafana/dashboards-coredns.yaml b/addons/grafana/dashboards-coredns.yaml index f2313cf61..44520b7d4 100644 --- a/addons/grafana/dashboards-coredns.yaml +++ b/addons/grafana/dashboards-coredns.yaml @@ -1029,7 +1029,8 @@ data: "30d" ] }, - "timezone": "browser", + "timezone": "", "title": "CoreDNS", + "uid": "2f3f749259235f58698ea949170d3bd5", "version": 0 } diff --git a/addons/grafana/dashboards-etcd.yaml b/addons/grafana/dashboards-etcd.yaml index 94bcaf06d..21339a16b 100644 --- a/addons/grafana/dashboards-etcd.yaml +++ b/addons/grafana/dashboards-etcd.yaml @@ -1224,7 +1224,8 @@ data: "30d" ] }, - "timezone": "browser", + "timezone": "", "title": "etcd", + "uid": "c2f4e12cdf69feb95caa41a5a1b423d9", "version": 215 } diff --git a/addons/grafana/dashboards-nginx-ingress.yaml b/addons/grafana/dashboards-nginx-ingress.yaml index 4a993e6c3..0f7595206 100644 --- a/addons/grafana/dashboards-nginx-ingress.yaml +++ b/addons/grafana/dashboards-nginx-ingress.yaml @@ -1052,7 +1052,8 @@ data: "30d" ] }, - "timezone": "browser", + "timezone": "", "title": "Nginx Ingress Controller", + "uid": "f4af03eca476c08ecf2b5cf15fd60168", "version": 0 } From 4ef2eb7e6b5312a16a8f8cd4175bc9bb0166105f Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Sun, 18 Aug 2019 20:59:44 -0700 Subject: [PATCH 192/523] Update Prometheus from v2.11.2 to v2.12.0 * https://github.com/prometheus/prometheus/releases/tag/v2.12.0 --- CHANGES.md | 2 +- addons/prometheus/deployment.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 0ead71b25..4125d78de 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -13,7 +13,7 @@ Notable changes between versions. #### Addons -* Update Prometheus from v2.11.0 to [v2.11.2](https://github.com/prometheus/prometheus/releases/tag/v2.11.2) +* Update Prometheus from v2.11.0 to [v2.12.0](https://github.com/prometheus/prometheus/releases/tag/v2.12.0) * Update kube-state-metrics from v1.7.1 to v1.7.2 * Update Grafana from v6.2.5 to v6.3.3 * Use stable IDs for etcd, CoreDNS, and Nginx Ingress dashboards diff --git a/addons/prometheus/deployment.yaml b/addons/prometheus/deployment.yaml index abc543ca5..0b21c6114 100644 --- a/addons/prometheus/deployment.yaml +++ b/addons/prometheus/deployment.yaml @@ -20,7 +20,7 @@ spec: serviceAccountName: prometheus containers: - name: prometheus - image: quay.io/prometheus/prometheus:v2.11.2 + image: quay.io/prometheus/prometheus:v2.12.0 args: - --web.listen-address=0.0.0.0:9090 - --config.file=/etc/prometheus/prometheus.yaml From 8f412e2f092efeb634ac4a86926e9e5164498ec4 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Sun, 18 Aug 2019 21:05:06 -0700 Subject: [PATCH 193/523] Update etcd from v3.3.13 to v3.3.14 * https://github.com/etcd-io/etcd/releases/tag/v3.3.14 --- CHANGES.md | 1 + aws/container-linux/kubernetes/cl/controller.yaml.tmpl | 2 +- aws/fedora-coreos/kubernetes/fcc/controller.yaml | 2 +- azure/container-linux/kubernetes/cl/controller.yaml.tmpl | 2 +- bare-metal/container-linux/kubernetes/cl/controller.yaml.tmpl | 2 +- bare-metal/fedora-coreos/kubernetes/fcc/controller.yaml | 2 +- .../container-linux/kubernetes/cl/controller.yaml.tmpl | 2 +- google-cloud/container-linux/kubernetes/cl/controller.yaml.tmpl | 2 +- 8 files changed, 8 insertions(+), 7 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 4125d78de..d4f169e3d 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -5,6 +5,7 @@ Notable changes between versions. ## Latest * Update Calico from v3.8.1 to [v3.8.2](https://docs.projectcalico.org/v3.8/release-notes/) +* Update etcd from v3.3.13 to [v3.3.14](https://github.com/etcd-io/etcd/releases/tag/v3.3.14) #### AWS diff --git a/aws/container-linux/kubernetes/cl/controller.yaml.tmpl b/aws/container-linux/kubernetes/cl/controller.yaml.tmpl index c83f7ad48..6867a3744 100644 --- a/aws/container-linux/kubernetes/cl/controller.yaml.tmpl +++ b/aws/container-linux/kubernetes/cl/controller.yaml.tmpl @@ -7,7 +7,7 @@ systemd: - name: 40-etcd-cluster.conf contents: | [Service] - Environment="ETCD_IMAGE_TAG=v3.3.13" + Environment="ETCD_IMAGE_TAG=v3.3.14" Environment="ETCD_NAME=${etcd_name}" Environment="ETCD_ADVERTISE_CLIENT_URLS=https://${etcd_domain}:2379" Environment="ETCD_INITIAL_ADVERTISE_PEER_URLS=https://${etcd_domain}:2380" diff --git a/aws/fedora-coreos/kubernetes/fcc/controller.yaml b/aws/fedora-coreos/kubernetes/fcc/controller.yaml index 32313e7fc..52b8caf96 100644 --- a/aws/fedora-coreos/kubernetes/fcc/controller.yaml +++ b/aws/fedora-coreos/kubernetes/fcc/controller.yaml @@ -28,7 +28,7 @@ systemd: --network host \ --volume /var/lib/etcd:/var/lib/etcd:rw,Z \ --volume /etc/ssl/etcd:/etc/ssl/certs:ro,Z \ - quay.io/coreos/etcd:v3.3.13 + quay.io/coreos/etcd:v3.3.14 ExecStop=/usr/bin/podman stop etcd [Install] WantedBy=multi-user.target diff --git a/azure/container-linux/kubernetes/cl/controller.yaml.tmpl b/azure/container-linux/kubernetes/cl/controller.yaml.tmpl index ee172434d..29f0c7930 100644 --- a/azure/container-linux/kubernetes/cl/controller.yaml.tmpl +++ b/azure/container-linux/kubernetes/cl/controller.yaml.tmpl @@ -7,7 +7,7 @@ systemd: - name: 40-etcd-cluster.conf contents: | [Service] - Environment="ETCD_IMAGE_TAG=v3.3.13" + Environment="ETCD_IMAGE_TAG=v3.3.14" Environment="ETCD_NAME=${etcd_name}" Environment="ETCD_ADVERTISE_CLIENT_URLS=https://${etcd_domain}:2379" Environment="ETCD_INITIAL_ADVERTISE_PEER_URLS=https://${etcd_domain}:2380" diff --git a/bare-metal/container-linux/kubernetes/cl/controller.yaml.tmpl b/bare-metal/container-linux/kubernetes/cl/controller.yaml.tmpl index b37afdecd..e023f4e02 100644 --- a/bare-metal/container-linux/kubernetes/cl/controller.yaml.tmpl +++ b/bare-metal/container-linux/kubernetes/cl/controller.yaml.tmpl @@ -7,7 +7,7 @@ systemd: - name: 40-etcd-cluster.conf contents: | [Service] - Environment="ETCD_IMAGE_TAG=v3.3.13" + Environment="ETCD_IMAGE_TAG=v3.3.14" Environment="ETCD_NAME=${etcd_name}" Environment="ETCD_ADVERTISE_CLIENT_URLS=https://${domain_name}:2379" Environment="ETCD_INITIAL_ADVERTISE_PEER_URLS=https://${domain_name}:2380" diff --git a/bare-metal/fedora-coreos/kubernetes/fcc/controller.yaml b/bare-metal/fedora-coreos/kubernetes/fcc/controller.yaml index 933adaae8..622633c6c 100644 --- a/bare-metal/fedora-coreos/kubernetes/fcc/controller.yaml +++ b/bare-metal/fedora-coreos/kubernetes/fcc/controller.yaml @@ -28,7 +28,7 @@ systemd: --network host \ --volume /var/lib/etcd:/var/lib/etcd:rw,Z \ --volume /etc/ssl/etcd:/etc/ssl/certs:ro,Z \ - quay.io/coreos/etcd:v3.3.13 + quay.io/coreos/etcd:v3.3.14 ExecStop=/usr/bin/podman stop etcd [Install] WantedBy=multi-user.target diff --git a/digital-ocean/container-linux/kubernetes/cl/controller.yaml.tmpl b/digital-ocean/container-linux/kubernetes/cl/controller.yaml.tmpl index 343fd6fe4..ad37c44c7 100644 --- a/digital-ocean/container-linux/kubernetes/cl/controller.yaml.tmpl +++ b/digital-ocean/container-linux/kubernetes/cl/controller.yaml.tmpl @@ -7,7 +7,7 @@ systemd: - name: 40-etcd-cluster.conf contents: | [Service] - Environment="ETCD_IMAGE_TAG=v3.3.13" + Environment="ETCD_IMAGE_TAG=v3.3.14" Environment="ETCD_NAME=${etcd_name}" Environment="ETCD_ADVERTISE_CLIENT_URLS=https://${etcd_domain}:2379" Environment="ETCD_INITIAL_ADVERTISE_PEER_URLS=https://${etcd_domain}:2380" diff --git a/google-cloud/container-linux/kubernetes/cl/controller.yaml.tmpl b/google-cloud/container-linux/kubernetes/cl/controller.yaml.tmpl index 61ee6d538..9cf183862 100644 --- a/google-cloud/container-linux/kubernetes/cl/controller.yaml.tmpl +++ b/google-cloud/container-linux/kubernetes/cl/controller.yaml.tmpl @@ -7,7 +7,7 @@ systemd: - name: 40-etcd-cluster.conf contents: | [Service] - Environment="ETCD_IMAGE_TAG=v3.3.13" + Environment="ETCD_IMAGE_TAG=v3.3.14" Environment="ETCD_NAME=${etcd_name}" Environment="ETCD_ADVERTISE_CLIENT_URLS=https://${etcd_domain}:2379" Environment="ETCD_INITIAL_ADVERTISE_PEER_URLS=https://${etcd_domain}:2380" From 2067356ae92b748aa13decf50ceaa073590b5900 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Sun, 18 Aug 2019 21:46:59 -0700 Subject: [PATCH 194/523] Update Fedora CoreOS to testing 30.20190801.0 --- aws/fedora-coreos/kubernetes/ami.tf | 2 +- aws/fedora-coreos/kubernetes/workers/ami.tf | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/aws/fedora-coreos/kubernetes/ami.tf b/aws/fedora-coreos/kubernetes/ami.tf index 7d27a2908..71322d17a 100644 --- a/aws/fedora-coreos/kubernetes/ami.tf +++ b/aws/fedora-coreos/kubernetes/ami.tf @@ -16,6 +16,6 @@ data "aws_ami" "fedora-coreos" { // pin on known ok versions as preview matures filter { name = "name" - values = ["fedora-coreos-30.20190725.0-hvm"] + values = ["fedora-coreos-30.20190801.0-hvm"] } } diff --git a/aws/fedora-coreos/kubernetes/workers/ami.tf b/aws/fedora-coreos/kubernetes/workers/ami.tf index 7d27a2908..71322d17a 100644 --- a/aws/fedora-coreos/kubernetes/workers/ami.tf +++ b/aws/fedora-coreos/kubernetes/workers/ami.tf @@ -16,6 +16,6 @@ data "aws_ami" "fedora-coreos" { // pin on known ok versions as preview matures filter { name = "name" - values = ["fedora-coreos-30.20190725.0-hvm"] + values = ["fedora-coreos-30.20190801.0-hvm"] } } From 35c2763ab021d4c707792ed7f945cbdc588ca225 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Mon, 19 Aug 2019 14:49:24 -0700 Subject: [PATCH 195/523] Update Kubernetes from v1.15.2 to v1.15.3 * https://github.com/kubernetes/kubernetes/blob/master/CHANGELOG-1.15.md/#v1153 --- CHANGES.md | 7 +++++-- README.md | 10 +++++----- aws/container-linux/kubernetes/README.md | 2 +- aws/container-linux/kubernetes/bootkube.tf | 2 +- .../kubernetes/cl/controller.yaml.tmpl | 2 +- .../kubernetes/workers/cl/worker.yaml.tmpl | 4 ++-- aws/fedora-coreos/kubernetes/README.md | 2 +- aws/fedora-coreos/kubernetes/bootkube.tf | 2 +- aws/fedora-coreos/kubernetes/fcc/controller.yaml | 2 +- .../kubernetes/workers/fcc/worker.yaml | 2 +- azure/container-linux/kubernetes/README.md | 2 +- azure/container-linux/kubernetes/bootkube.tf | 2 +- .../kubernetes/cl/controller.yaml.tmpl | 2 +- .../kubernetes/workers/cl/worker.yaml.tmpl | 4 ++-- bare-metal/container-linux/kubernetes/README.md | 2 +- bare-metal/container-linux/kubernetes/bootkube.tf | 2 +- .../kubernetes/cl/controller.yaml.tmpl | 2 +- .../container-linux/kubernetes/cl/worker.yaml.tmpl | 2 +- bare-metal/fedora-coreos/kubernetes/README.md | 2 +- bare-metal/fedora-coreos/kubernetes/bootkube.tf | 2 +- .../fedora-coreos/kubernetes/fcc/controller.yaml | 2 +- .../fedora-coreos/kubernetes/fcc/worker.yaml | 2 +- digital-ocean/container-linux/kubernetes/README.md | 2 +- .../container-linux/kubernetes/bootkube.tf | 2 +- .../kubernetes/cl/controller.yaml.tmpl | 2 +- .../container-linux/kubernetes/cl/worker.yaml.tmpl | 4 ++-- docs/advanced/worker-pools.md | 14 +++++++------- docs/cl/aws.md | 10 +++++----- docs/cl/azure.md | 10 +++++----- docs/cl/bare-metal.md | 14 +++++++------- docs/cl/digital-ocean.md | 10 +++++----- docs/cl/google-cloud.md | 10 +++++----- docs/fedora-coreos/aws.md | 8 ++++---- docs/fedora-coreos/bare-metal.md | 8 ++++---- docs/index.md | 10 +++++----- docs/topics/maintenance.md | 10 +++++----- google-cloud/container-linux/kubernetes/README.md | 2 +- .../container-linux/kubernetes/bootkube.tf | 2 +- .../kubernetes/cl/controller.yaml.tmpl | 2 +- .../kubernetes/workers/cl/worker.yaml.tmpl | 4 ++-- 40 files changed, 94 insertions(+), 91 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index d4f169e3d..e1997b45a 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -4,8 +4,11 @@ Notable changes between versions. ## Latest -* Update Calico from v3.8.1 to [v3.8.2](https://docs.projectcalico.org/v3.8/release-notes/) +## v1.15.3 + +* Kubernetes [v1.15.3](https://github.com/kubernetes/kubernetes/blob/master/CHANGELOG-1.15.md#v1153) * Update etcd from v3.3.13 to [v3.3.14](https://github.com/etcd-io/etcd/releases/tag/v3.3.14) +* Update Calico from v3.8.1 to [v3.8.2](https://docs.projectcalico.org/v3.8/release-notes/) #### AWS @@ -17,7 +20,7 @@ Notable changes between versions. * Update Prometheus from v2.11.0 to [v2.12.0](https://github.com/prometheus/prometheus/releases/tag/v2.12.0) * Update kube-state-metrics from v1.7.1 to v1.7.2 * Update Grafana from v6.2.5 to v6.3.3 - * Use stable IDs for etcd, CoreDNS, and Nginx Ingress dashboards + * Use stable IDs for etcd, CoreDNS, and Nginx Ingress dashboards ([#530](https://github.com/poseidon/typhoon/pull/530)) * Update nginx-ingress from v0.25.0 to [v0.25.1](https://github.com/kubernetes/ingress-nginx/releases/tag/nginx-0.25.1) * Fix Nginx security advisories diff --git a/README.md b/README.md index f75961f85..2421a808e 100644 --- a/README.md +++ b/README.md @@ -11,7 +11,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features -* Kubernetes v1.15.2 (upstream, via [kubernetes-incubator/bootkube](https://github.com/kubernetes-incubator/bootkube)) +* Kubernetes v1.15.3 (upstream, via [kubernetes-incubator/bootkube](https://github.com/kubernetes-incubator/bootkube)) * Single or multi-master, [Calico](https://www.projectcalico.org/) or [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/) * Advanced features like [worker pools](https://typhoon.psdn.io/advanced/worker-pools/), [preemptible](https://typhoon.psdn.io/cl/google-cloud/#preemption) workers, and [snippets](https://typhoon.psdn.io/advanced/customization/#container-linux) customization @@ -48,7 +48,7 @@ Define a Kubernetes cluster by using the Terraform module for your chosen platfo ```tf module "google-cloud-yavin" { - source = "git::https://github.com/poseidon/typhoon//google-cloud/container-linux/kubernetes?ref=v1.15.2" + source = "git::https://github.com/poseidon/typhoon//google-cloud/container-linux/kubernetes?ref=v1.15.3" # Google Cloud cluster_name = "yavin" @@ -82,9 +82,9 @@ In 4-8 minutes (varies by platform), the cluster will be ready. This Google Clou $ export KUBECONFIG=/home/user/.secrets/clusters/yavin/auth/kubeconfig $ kubectl get nodes NAME ROLES STATUS AGE VERSION -yavin-controller-0.c.example-com.internal controller,master Ready 6m v1.15.2 -yavin-worker-jrbf.c.example-com.internal node Ready 5m v1.15.2 -yavin-worker-mzdm.c.example-com.internal node Ready 5m v1.15.2 +yavin-controller-0.c.example-com.internal controller,master Ready 6m v1.15.3 +yavin-worker-jrbf.c.example-com.internal node Ready 5m v1.15.3 +yavin-worker-mzdm.c.example-com.internal node Ready 5m v1.15.3 ``` List the pods. diff --git a/aws/container-linux/kubernetes/README.md b/aws/container-linux/kubernetes/README.md index cd5e3aa09..220911a97 100644 --- a/aws/container-linux/kubernetes/README.md +++ b/aws/container-linux/kubernetes/README.md @@ -11,7 +11,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features -* Kubernetes v1.15.2 (upstream, via [kubernetes-incubator/bootkube](https://github.com/kubernetes-incubator/bootkube)) +* Kubernetes v1.15.3 (upstream, via [kubernetes-incubator/bootkube](https://github.com/kubernetes-incubator/bootkube)) * Single or multi-master, [Calico](https://www.projectcalico.org/) or [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/) * Advanced features like [worker pools](https://typhoon.psdn.io/advanced/worker-pools/), [spot](https://typhoon.psdn.io/cl/aws/#spot) workers, and [snippets](https://typhoon.psdn.io/advanced/customization/#container-linux) customization diff --git a/aws/container-linux/kubernetes/bootkube.tf b/aws/container-linux/kubernetes/bootkube.tf index d9289ff57..b2f577e2b 100644 --- a/aws/container-linux/kubernetes/bootkube.tf +++ b/aws/container-linux/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=8b3738b2cccf3cd9830beac170a53c4fbec26ed5" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=248675e7a95660e5c8a65f8b7ce86be89ef55a0e" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/aws/container-linux/kubernetes/cl/controller.yaml.tmpl b/aws/container-linux/kubernetes/cl/controller.yaml.tmpl index 6867a3744..c44c31c6d 100644 --- a/aws/container-linux/kubernetes/cl/controller.yaml.tmpl +++ b/aws/container-linux/kubernetes/cl/controller.yaml.tmpl @@ -125,7 +125,7 @@ storage: contents: inline: | KUBELET_IMAGE_URL=docker://k8s.gcr.io/hyperkube - KUBELET_IMAGE_TAG=v1.15.2 + KUBELET_IMAGE_TAG=v1.15.3 - path: /etc/sysctl.d/max-user-watches.conf filesystem: root contents: diff --git a/aws/container-linux/kubernetes/workers/cl/worker.yaml.tmpl b/aws/container-linux/kubernetes/workers/cl/worker.yaml.tmpl index 74ed194a3..6da106809 100644 --- a/aws/container-linux/kubernetes/workers/cl/worker.yaml.tmpl +++ b/aws/container-linux/kubernetes/workers/cl/worker.yaml.tmpl @@ -95,7 +95,7 @@ storage: contents: inline: | KUBELET_IMAGE_URL=docker://k8s.gcr.io/hyperkube - KUBELET_IMAGE_TAG=v1.15.2 + KUBELET_IMAGE_TAG=v1.15.3 - path: /etc/sysctl.d/max-user-watches.conf filesystem: root contents: @@ -113,7 +113,7 @@ storage: --volume config,kind=host,source=/etc/kubernetes \ --mount volume=config,target=/etc/kubernetes \ --insecure-options=image \ - docker://k8s.gcr.io/hyperkube:v1.15.2 \ + docker://k8s.gcr.io/hyperkube:v1.15.3 \ --net=host \ --dns=host \ --exec=/kubectl -- --kubeconfig=/etc/kubernetes/kubeconfig delete node $(hostname) diff --git a/aws/fedora-coreos/kubernetes/README.md b/aws/fedora-coreos/kubernetes/README.md index c70661fc3..c249ea1a7 100644 --- a/aws/fedora-coreos/kubernetes/README.md +++ b/aws/fedora-coreos/kubernetes/README.md @@ -11,7 +11,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features -* Kubernetes v1.15.2 (upstream, via [kubernetes-incubator/bootkube](https://github.com/kubernetes-incubator/bootkube)) +* Kubernetes v1.15.3 (upstream, via [kubernetes-incubator/bootkube](https://github.com/kubernetes-incubator/bootkube)) * Single or multi-master, [Calico](https://www.projectcalico.org/) or [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/) * Advanced features like [worker pools](https://typhoon.psdn.io/advanced/worker-pools/), [spot](https://typhoon.psdn.io/cl/aws/#spot) workers, and [snippets](https://typhoon.psdn.io/advanced/customization/#container-linux) customization diff --git a/aws/fedora-coreos/kubernetes/bootkube.tf b/aws/fedora-coreos/kubernetes/bootkube.tf index ea8164ad4..8cde2eeae 100644 --- a/aws/fedora-coreos/kubernetes/bootkube.tf +++ b/aws/fedora-coreos/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=8b3738b2cccf3cd9830beac170a53c4fbec26ed5" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=248675e7a95660e5c8a65f8b7ce86be89ef55a0e" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/aws/fedora-coreos/kubernetes/fcc/controller.yaml b/aws/fedora-coreos/kubernetes/fcc/controller.yaml index 52b8caf96..1ef51bb16 100644 --- a/aws/fedora-coreos/kubernetes/fcc/controller.yaml +++ b/aws/fedora-coreos/kubernetes/fcc/controller.yaml @@ -80,7 +80,7 @@ systemd: --volume /var/run:/var/run \ --volume /var/run/lock:/var/run/lock:z \ --volume /opt/cni/bin:/opt/cni/bin:z \ - k8s.gcr.io/hyperkube:v1.15.2 /hyperkube kubelet \ + k8s.gcr.io/hyperkube:v1.15.3 /hyperkube kubelet \ --anonymous-auth=false \ --authentication-token-webhook \ --authorization-mode=Webhook \ diff --git a/aws/fedora-coreos/kubernetes/workers/fcc/worker.yaml b/aws/fedora-coreos/kubernetes/workers/fcc/worker.yaml index d1e3915e4..c5f1aeffc 100644 --- a/aws/fedora-coreos/kubernetes/workers/fcc/worker.yaml +++ b/aws/fedora-coreos/kubernetes/workers/fcc/worker.yaml @@ -50,7 +50,7 @@ systemd: --volume /var/run:/var/run \ --volume /var/run/lock:/var/run/lock:z \ --volume /opt/cni/bin:/opt/cni/bin:z \ - k8s.gcr.io/hyperkube:v1.15.2 /hyperkube kubelet \ + k8s.gcr.io/hyperkube:v1.15.3 /hyperkube kubelet \ --anonymous-auth=false \ --authentication-token-webhook \ --authorization-mode=Webhook \ diff --git a/azure/container-linux/kubernetes/README.md b/azure/container-linux/kubernetes/README.md index 6e00a0c23..ea10af3b0 100644 --- a/azure/container-linux/kubernetes/README.md +++ b/azure/container-linux/kubernetes/README.md @@ -11,7 +11,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features -* Kubernetes v1.15.2 (upstream, via [kubernetes-incubator/bootkube](https://github.com/kubernetes-incubator/bootkube)) +* Kubernetes v1.15.3 (upstream, via [kubernetes-incubator/bootkube](https://github.com/kubernetes-incubator/bootkube)) * Single or multi-master, [Calico](https://www.projectcalico.org/) or [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/) * Advanced features like [worker pools](https://typhoon.psdn.io/advanced/worker-pools/), [low-priority](https://typhoon.psdn.io/cl/azure/#low-priority) workers, and [snippets](https://typhoon.psdn.io/advanced/customization/#container-linux) customization diff --git a/azure/container-linux/kubernetes/bootkube.tf b/azure/container-linux/kubernetes/bootkube.tf index 7f185f16d..9e1ab679c 100644 --- a/azure/container-linux/kubernetes/bootkube.tf +++ b/azure/container-linux/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=8b3738b2cccf3cd9830beac170a53c4fbec26ed5" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=248675e7a95660e5c8a65f8b7ce86be89ef55a0e" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/azure/container-linux/kubernetes/cl/controller.yaml.tmpl b/azure/container-linux/kubernetes/cl/controller.yaml.tmpl index 29f0c7930..8d760c387 100644 --- a/azure/container-linux/kubernetes/cl/controller.yaml.tmpl +++ b/azure/container-linux/kubernetes/cl/controller.yaml.tmpl @@ -123,7 +123,7 @@ storage: contents: inline: | KUBELET_IMAGE_URL=docker://k8s.gcr.io/hyperkube - KUBELET_IMAGE_TAG=v1.15.2 + KUBELET_IMAGE_TAG=v1.15.3 - path: /etc/sysctl.d/max-user-watches.conf filesystem: root contents: diff --git a/azure/container-linux/kubernetes/workers/cl/worker.yaml.tmpl b/azure/container-linux/kubernetes/workers/cl/worker.yaml.tmpl index 05768f969..7db9c32f1 100644 --- a/azure/container-linux/kubernetes/workers/cl/worker.yaml.tmpl +++ b/azure/container-linux/kubernetes/workers/cl/worker.yaml.tmpl @@ -93,7 +93,7 @@ storage: contents: inline: | KUBELET_IMAGE_URL=docker://k8s.gcr.io/hyperkube - KUBELET_IMAGE_TAG=v1.15.2 + KUBELET_IMAGE_TAG=v1.15.3 - path: /etc/sysctl.d/max-user-watches.conf filesystem: root contents: @@ -111,7 +111,7 @@ storage: --volume config,kind=host,source=/etc/kubernetes \ --mount volume=config,target=/etc/kubernetes \ --insecure-options=image \ - docker://k8s.gcr.io/hyperkube:v1.15.2 \ + docker://k8s.gcr.io/hyperkube:v1.15.3 \ --net=host \ --dns=host \ --exec=/kubectl -- --kubeconfig=/etc/kubernetes/kubeconfig delete node $(hostname | tr '[:upper:]' '[:lower:]') diff --git a/bare-metal/container-linux/kubernetes/README.md b/bare-metal/container-linux/kubernetes/README.md index 6bf2d62b0..e902faa81 100644 --- a/bare-metal/container-linux/kubernetes/README.md +++ b/bare-metal/container-linux/kubernetes/README.md @@ -11,7 +11,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features -* Kubernetes v1.15.2 (upstream, via [kubernetes-incubator/bootkube](https://github.com/kubernetes-incubator/bootkube)) +* Kubernetes v1.15.3 (upstream, via [kubernetes-incubator/bootkube](https://github.com/kubernetes-incubator/bootkube)) * Single or multi-master, [Calico](https://www.projectcalico.org/) or [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/) * Advanced features like [snippets](https://typhoon.psdn.io/advanced/customization/#container-linux) customization diff --git a/bare-metal/container-linux/kubernetes/bootkube.tf b/bare-metal/container-linux/kubernetes/bootkube.tf index 37198b072..f98964fee 100644 --- a/bare-metal/container-linux/kubernetes/bootkube.tf +++ b/bare-metal/container-linux/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=8b3738b2cccf3cd9830beac170a53c4fbec26ed5" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=248675e7a95660e5c8a65f8b7ce86be89ef55a0e" cluster_name = var.cluster_name api_servers = [var.k8s_domain_name] diff --git a/bare-metal/container-linux/kubernetes/cl/controller.yaml.tmpl b/bare-metal/container-linux/kubernetes/cl/controller.yaml.tmpl index e023f4e02..678bb8afd 100644 --- a/bare-metal/container-linux/kubernetes/cl/controller.yaml.tmpl +++ b/bare-metal/container-linux/kubernetes/cl/controller.yaml.tmpl @@ -130,7 +130,7 @@ storage: contents: inline: | KUBELET_IMAGE_URL=docker://k8s.gcr.io/hyperkube - KUBELET_IMAGE_TAG=v1.15.2 + KUBELET_IMAGE_TAG=v1.15.3 - path: /etc/hostname filesystem: root mode: 0644 diff --git a/bare-metal/container-linux/kubernetes/cl/worker.yaml.tmpl b/bare-metal/container-linux/kubernetes/cl/worker.yaml.tmpl index 314dbb1dd..2af4d66a8 100644 --- a/bare-metal/container-linux/kubernetes/cl/worker.yaml.tmpl +++ b/bare-metal/container-linux/kubernetes/cl/worker.yaml.tmpl @@ -91,7 +91,7 @@ storage: contents: inline: | KUBELET_IMAGE_URL=docker://k8s.gcr.io/hyperkube - KUBELET_IMAGE_TAG=v1.15.2 + KUBELET_IMAGE_TAG=v1.15.3 - path: /etc/hostname filesystem: root mode: 0644 diff --git a/bare-metal/fedora-coreos/kubernetes/README.md b/bare-metal/fedora-coreos/kubernetes/README.md index 6e7edb46b..938a4732a 100644 --- a/bare-metal/fedora-coreos/kubernetes/README.md +++ b/bare-metal/fedora-coreos/kubernetes/README.md @@ -11,7 +11,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features -* Kubernetes v1.15.2 (upstream, via [kubernetes-incubator/bootkube](https://github.com/kubernetes-incubator/bootkube)) +* Kubernetes v1.15.3 (upstream, via [kubernetes-incubator/bootkube](https://github.com/kubernetes-incubator/bootkube)) * Single or multi-master, [Calico](https://www.projectcalico.org/) or [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/) * Advanced features like [snippets](https://typhoon.psdn.io/advanced/customization/#container-linux) customization diff --git a/bare-metal/fedora-coreos/kubernetes/bootkube.tf b/bare-metal/fedora-coreos/kubernetes/bootkube.tf index 97db7c6eb..7264478b0 100644 --- a/bare-metal/fedora-coreos/kubernetes/bootkube.tf +++ b/bare-metal/fedora-coreos/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=8b3738b2cccf3cd9830beac170a53c4fbec26ed5" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=248675e7a95660e5c8a65f8b7ce86be89ef55a0e" cluster_name = var.cluster_name api_servers = [var.k8s_domain_name] diff --git a/bare-metal/fedora-coreos/kubernetes/fcc/controller.yaml b/bare-metal/fedora-coreos/kubernetes/fcc/controller.yaml index 622633c6c..9c4e2b707 100644 --- a/bare-metal/fedora-coreos/kubernetes/fcc/controller.yaml +++ b/bare-metal/fedora-coreos/kubernetes/fcc/controller.yaml @@ -81,7 +81,7 @@ systemd: --volume /opt/cni/bin:/opt/cni/bin:z \ --volume /etc/iscsi:/etc/iscsi \ --volume /sbin/iscsiadm:/sbin/iscsiadm \ - k8s.gcr.io/hyperkube:v1.15.2 /hyperkube kubelet \ + k8s.gcr.io/hyperkube:v1.15.3 /hyperkube kubelet \ --anonymous-auth=false \ --authentication-token-webhook \ --authorization-mode=Webhook \ diff --git a/bare-metal/fedora-coreos/kubernetes/fcc/worker.yaml b/bare-metal/fedora-coreos/kubernetes/fcc/worker.yaml index 5f4483fb7..8016a6e04 100644 --- a/bare-metal/fedora-coreos/kubernetes/fcc/worker.yaml +++ b/bare-metal/fedora-coreos/kubernetes/fcc/worker.yaml @@ -51,7 +51,7 @@ systemd: --volume /opt/cni/bin:/opt/cni/bin:z \ --volume /etc/iscsi:/etc/iscsi \ --volume /sbin/iscsiadm:/sbin/iscsiadm \ - k8s.gcr.io/hyperkube:v1.15.2 /hyperkube kubelet \ + k8s.gcr.io/hyperkube:v1.15.3 /hyperkube kubelet \ --anonymous-auth=false \ --authentication-token-webhook \ --authorization-mode=Webhook \ diff --git a/digital-ocean/container-linux/kubernetes/README.md b/digital-ocean/container-linux/kubernetes/README.md index ca047181a..08b6373c3 100644 --- a/digital-ocean/container-linux/kubernetes/README.md +++ b/digital-ocean/container-linux/kubernetes/README.md @@ -11,7 +11,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features -* Kubernetes v1.15.2 (upstream, via [kubernetes-incubator/bootkube](https://github.com/kubernetes-incubator/bootkube)) +* Kubernetes v1.15.3 (upstream, via [kubernetes-incubator/bootkube](https://github.com/kubernetes-incubator/bootkube)) * Single or multi-master, [Calico](https://www.projectcalico.org/) or [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/) * Advanced features like [snippets](https://typhoon.psdn.io/advanced/customization/#container-linux) customization diff --git a/digital-ocean/container-linux/kubernetes/bootkube.tf b/digital-ocean/container-linux/kubernetes/bootkube.tf index 5e733d592..9755976fb 100644 --- a/digital-ocean/container-linux/kubernetes/bootkube.tf +++ b/digital-ocean/container-linux/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=8b3738b2cccf3cd9830beac170a53c4fbec26ed5" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=248675e7a95660e5c8a65f8b7ce86be89ef55a0e" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/digital-ocean/container-linux/kubernetes/cl/controller.yaml.tmpl b/digital-ocean/container-linux/kubernetes/cl/controller.yaml.tmpl index ad37c44c7..5e3eabe91 100644 --- a/digital-ocean/container-linux/kubernetes/cl/controller.yaml.tmpl +++ b/digital-ocean/container-linux/kubernetes/cl/controller.yaml.tmpl @@ -129,7 +129,7 @@ storage: contents: inline: | KUBELET_IMAGE_URL=docker://k8s.gcr.io/hyperkube - KUBELET_IMAGE_TAG=v1.15.2 + KUBELET_IMAGE_TAG=v1.15.3 - path: /etc/sysctl.d/max-user-watches.conf filesystem: root contents: diff --git a/digital-ocean/container-linux/kubernetes/cl/worker.yaml.tmpl b/digital-ocean/container-linux/kubernetes/cl/worker.yaml.tmpl index 2f80b247d..130deed24 100644 --- a/digital-ocean/container-linux/kubernetes/cl/worker.yaml.tmpl +++ b/digital-ocean/container-linux/kubernetes/cl/worker.yaml.tmpl @@ -99,7 +99,7 @@ storage: contents: inline: | KUBELET_IMAGE_URL=docker://k8s.gcr.io/hyperkube - KUBELET_IMAGE_TAG=v1.15.2 + KUBELET_IMAGE_TAG=v1.15.3 - path: /etc/sysctl.d/max-user-watches.conf filesystem: root contents: @@ -117,7 +117,7 @@ storage: --volume config,kind=host,source=/etc/kubernetes \ --mount volume=config,target=/etc/kubernetes \ --insecure-options=image \ - docker://k8s.gcr.io/hyperkube:v1.15.2 \ + docker://k8s.gcr.io/hyperkube:v1.15.3 \ --net=host \ --dns=host \ --exec=/kubectl -- --kubeconfig=/etc/kubernetes/kubeconfig delete node $(hostname) diff --git a/docs/advanced/worker-pools.md b/docs/advanced/worker-pools.md index 8ec39ce31..8a1f71024 100644 --- a/docs/advanced/worker-pools.md +++ b/docs/advanced/worker-pools.md @@ -76,7 +76,7 @@ Create a cluster following the Azure [tutorial](../cl/azure.md#cluster). Define ```tf module "ramius-worker-pool" { - source = "git::https://github.com/poseidon/typhoon//azure/container-linux/kubernetes/workers?ref=v1.15.2" + source = "git::https://github.com/poseidon/typhoon//azure/container-linux/kubernetes/workers?ref=v1.15.3" # Azure region = module.azure-ramius.region @@ -142,7 +142,7 @@ Create a cluster following the Google Cloud [tutorial](../cl/google-cloud.md#clu ```tf module "yavin-worker-pool" { - source = "git::https://github.com/poseidon/typhoon//google-cloud/container-linux/kubernetes/workers?ref=v1.15.2" + source = "git::https://github.com/poseidon/typhoon//google-cloud/container-linux/kubernetes/workers?ref=v1.15.3" # Google Cloud region = "europe-west2" @@ -173,11 +173,11 @@ Verify a managed instance group of workers joins the cluster within a few minute ``` $ kubectl get nodes NAME STATUS AGE VERSION -yavin-controller-0.c.example-com.internal Ready 6m v1.15.2 -yavin-worker-jrbf.c.example-com.internal Ready 5m v1.15.2 -yavin-worker-mzdm.c.example-com.internal Ready 5m v1.15.2 -yavin-16x-worker-jrbf.c.example-com.internal Ready 3m v1.15.2 -yavin-16x-worker-mzdm.c.example-com.internal Ready 3m v1.15.2 +yavin-controller-0.c.example-com.internal Ready 6m v1.15.3 +yavin-worker-jrbf.c.example-com.internal Ready 5m v1.15.3 +yavin-worker-mzdm.c.example-com.internal Ready 5m v1.15.3 +yavin-16x-worker-jrbf.c.example-com.internal Ready 3m v1.15.3 +yavin-16x-worker-mzdm.c.example-com.internal Ready 3m v1.15.3 ``` ### Variables diff --git a/docs/cl/aws.md b/docs/cl/aws.md index ea5f61b9f..bf9e58dc8 100644 --- a/docs/cl/aws.md +++ b/docs/cl/aws.md @@ -1,6 +1,6 @@ # AWS -In this tutorial, we'll create a Kubernetes v1.15.2 cluster on AWS with Container Linux. +In this tutorial, we'll create a Kubernetes v1.15.3 cluster on AWS with Container Linux. We'll declare a Kubernetes cluster using the Typhoon Terraform module. Then apply the changes to create a VPC, gateway, subnets, security groups, controller instances, worker auto-scaling group, network load balancer, and TLS assets. @@ -70,7 +70,7 @@ Define a Kubernetes cluster using the module `aws/container-linux/kubernetes`. ```tf module "tempest" { - source = "git::https://github.com/poseidon/typhoon//aws/container-linux/kubernetes?ref=v1.15.2" + source = "git::https://github.com/poseidon/typhoon//aws/container-linux/kubernetes?ref=v1.15.3" # AWS cluster_name = "tempest" @@ -135,9 +135,9 @@ In 4-8 minutes, the Kubernetes cluster will be ready. $ export KUBECONFIG=/home/user/.secrets/clusters/tempest/auth/kubeconfig $ kubectl get nodes NAME STATUS ROLES AGE VERSION -ip-10-0-3-155 Ready controller,master 10m v1.15.2 -ip-10-0-26-65 Ready node 10m v1.15.2 -ip-10-0-41-21 Ready node 10m v1.15.2 +ip-10-0-3-155 Ready controller,master 10m v1.15.3 +ip-10-0-26-65 Ready node 10m v1.15.3 +ip-10-0-41-21 Ready node 10m v1.15.3 ``` List the pods. diff --git a/docs/cl/azure.md b/docs/cl/azure.md index e63b690b5..773967cdc 100644 --- a/docs/cl/azure.md +++ b/docs/cl/azure.md @@ -3,7 +3,7 @@ !!! danger Typhoon for Azure is alpha. For production, use AWS, Google Cloud, or bare-metal. As Azure matures, check [errata](https://github.com/poseidon/typhoon/wiki/Errata) for known shortcomings. -In this tutorial, we'll create a Kubernetes v1.15.2 cluster on Azure with Container Linux. +In this tutorial, we'll create a Kubernetes v1.15.3 cluster on Azure with Container Linux. We'll declare a Kubernetes cluster using the Typhoon Terraform module. Then apply the changes to create a resource group, virtual network, subnets, security groups, controller availability set, worker scale set, load balancer, and TLS assets. @@ -66,7 +66,7 @@ Define a Kubernetes cluster using the module `azure/container-linux/kubernetes`. ```tf module "ramius" { - source = "git::https://github.com/poseidon/typhoon//azure/container-linux/kubernetes?ref=v1.15.2" + source = "git::https://github.com/poseidon/typhoon//azure/container-linux/kubernetes?ref=v1.15.3" # Azure cluster_name = "ramius" @@ -132,9 +132,9 @@ In 4-8 minutes, the Kubernetes cluster will be ready. $ export KUBECONFIG=/home/user/.secrets/clusters/ramius/auth/kubeconfig $ kubectl get nodes NAME STATUS ROLES AGE VERSION -ramius-controller-0 Ready controller,master 24m v1.15.2 -ramius-worker-000001 Ready node 25m v1.15.2 -ramius-worker-000002 Ready node 24m v1.15.2 +ramius-controller-0 Ready controller,master 24m v1.15.3 +ramius-worker-000001 Ready node 25m v1.15.3 +ramius-worker-000002 Ready node 24m v1.15.3 ``` List the pods. diff --git a/docs/cl/bare-metal.md b/docs/cl/bare-metal.md index 725a2e96c..a8a4a3e17 100644 --- a/docs/cl/bare-metal.md +++ b/docs/cl/bare-metal.md @@ -1,6 +1,6 @@ # Bare-Metal -In this tutorial, we'll network boot and provision a Kubernetes v1.15.2 cluster on bare-metal with Container Linux. +In this tutorial, we'll network boot and provision a Kubernetes v1.15.3 cluster on bare-metal with Container Linux. First, we'll deploy a [Matchbox](https://github.com/poseidon/matchbox) service and setup a network boot environment. Then, we'll declare a Kubernetes cluster using the Typhoon Terraform module and power on machines. On PXE boot, machines will install Container Linux to disk, reboot into the disk install, and provision themselves as Kubernetes controllers or workers via Ignition. @@ -160,7 +160,7 @@ Define a Kubernetes cluster using the module `bare-metal/container-linux/kuberne ```tf module "bare-metal-mercury" { - source = "git::https://github.com/poseidon/typhoon//bare-metal/container-linux/kubernetes?ref=v1.15.2" + source = "git::https://github.com/poseidon/typhoon//bare-metal/container-linux/kubernetes?ref=v1.15.3" # bare-metal cluster_name = "mercury" @@ -265,9 +265,9 @@ Apply complete! Resources: 55 added, 0 changed, 0 destroyed. To watch the install to disk (until machines reboot from disk), SSH to port 2222. ``` -# before v1.15.2 +# before v1.15.3 $ ssh debug@node1.example.com -# after v1.15.2 +# after v1.15.3 $ ssh -p 2222 core@node1.example.com ``` @@ -292,9 +292,9 @@ bootkube[5]: Tearing down temporary bootstrap control plane... $ export KUBECONFIG=/home/user/.secrets/clusters/mercury/auth/kubeconfig $ kubectl get nodes NAME STATUS ROLES AGE VERSION -node1.example.com Ready controller,master 10m v1.15.2 -node2.example.com Ready node 10m v1.15.2 -node3.example.com Ready node 10m v1.15.2 +node1.example.com Ready controller,master 10m v1.15.3 +node2.example.com Ready node 10m v1.15.3 +node3.example.com Ready node 10m v1.15.3 ``` List the pods. diff --git a/docs/cl/digital-ocean.md b/docs/cl/digital-ocean.md index 62f36a321..fcb28cf7e 100644 --- a/docs/cl/digital-ocean.md +++ b/docs/cl/digital-ocean.md @@ -1,6 +1,6 @@ # Digital Ocean -In this tutorial, we'll create a Kubernetes v1.15.2 cluster on DigitalOcean with Container Linux. +In this tutorial, we'll create a Kubernetes v1.15.3 cluster on DigitalOcean with Container Linux. We'll declare a Kubernetes cluster using the Typhoon Terraform module. Then apply the changes to create controller droplets, worker droplets, DNS records, tags, and TLS assets. @@ -65,7 +65,7 @@ Define a Kubernetes cluster using the module `digital-ocean/container-linux/kube ```tf module "digital-ocean-nemo" { - source = "git::https://github.com/poseidon/typhoon//digital-ocean/container-linux/kubernetes?ref=v1.15.2" + source = "git::https://github.com/poseidon/typhoon//digital-ocean/container-linux/kubernetes?ref=v1.15.3" # Digital Ocean cluster_name = "nemo" @@ -130,9 +130,9 @@ In 3-6 minutes, the Kubernetes cluster will be ready. $ export KUBECONFIG=/home/user/.secrets/clusters/nemo/auth/kubeconfig $ kubectl get nodes NAME STATUS ROLES AGE VERSION -10.132.110.130 Ready controller,master 10m v1.15.2 -10.132.115.81 Ready node 10m v1.15.2 -10.132.124.107 Ready node 10m v1.15.2 +10.132.110.130 Ready controller,master 10m v1.15.3 +10.132.115.81 Ready node 10m v1.15.3 +10.132.124.107 Ready node 10m v1.15.3 ``` List the pods. diff --git a/docs/cl/google-cloud.md b/docs/cl/google-cloud.md index c1029b99b..ae7e892a0 100644 --- a/docs/cl/google-cloud.md +++ b/docs/cl/google-cloud.md @@ -1,6 +1,6 @@ # Google Cloud -In this tutorial, we'll create a Kubernetes v1.15.2 cluster on Google Compute Engine with Container Linux. +In this tutorial, we'll create a Kubernetes v1.15.3 cluster on Google Compute Engine with Container Linux. We'll declare a Kubernetes cluster using the Typhoon Terraform module. Then apply the changes to create a network, firewall rules, health checks, controller instances, worker managed instance group, load balancers, and TLS assets. @@ -71,7 +71,7 @@ Define a Kubernetes cluster using the module `google-cloud/container-linux/kuber ```tf module "google-cloud-yavin" { - source = "git::https://github.com/poseidon/typhoon//google-cloud/container-linux/kubernetes?ref=v1.15.2" + source = "git::https://github.com/poseidon/typhoon//google-cloud/container-linux/kubernetes?ref=v1.15.3" # Google Cloud cluster_name = "yavin" @@ -138,9 +138,9 @@ In 4-8 minutes, the Kubernetes cluster will be ready. $ export KUBECONFIG=/home/user/.secrets/clusters/yavin/auth/kubeconfig $ kubectl get nodes NAME ROLES STATUS AGE VERSION -yavin-controller-0.c.example-com.internal controller,master Ready 6m v1.15.2 -yavin-worker-jrbf.c.example-com.internal node Ready 5m v1.15.2 -yavin-worker-mzdm.c.example-com.internal node Ready 5m v1.15.2 +yavin-controller-0.c.example-com.internal controller,master Ready 6m v1.15.3 +yavin-worker-jrbf.c.example-com.internal node Ready 5m v1.15.3 +yavin-worker-mzdm.c.example-com.internal node Ready 5m v1.15.3 ``` List the pods. diff --git a/docs/fedora-coreos/aws.md b/docs/fedora-coreos/aws.md index 4ab27d4a2..1870bdacb 100644 --- a/docs/fedora-coreos/aws.md +++ b/docs/fedora-coreos/aws.md @@ -3,7 +3,7 @@ !!! danger Typhoon for Fedora CoreOS is an early preview! Fedora CoreOS itself is a preview! Expect bugs and design shifts. Please help both projects solve problems. Report Fedora CoreOS bugs to [Fedora](https://github.com/coreos/fedora-coreos-tracker/issues). Report Typhoon issues to Typhoon. -In this tutorial, we'll create a Kubernetes v1.15.2 cluster on AWS with Fedora CoreOS. +In this tutorial, we'll create a Kubernetes v1.15.3 cluster on AWS with Fedora CoreOS. We'll declare a Kubernetes cluster using the Typhoon Terraform module. Then apply the changes to create a VPC, gateway, subnets, security groups, controller instances, worker auto-scaling group, network load balancer, and TLS assets. @@ -138,9 +138,9 @@ In 4-8 minutes, the Kubernetes cluster will be ready. $ export KUBECONFIG=/home/user/.secrets/clusters/tempest/auth/kubeconfig $ kubectl get nodes NAME STATUS ROLES AGE VERSION -ip-10-0-3-155 Ready controller,master 10m v1.15.2 -ip-10-0-26-65 Ready node 10m v1.15.2 -ip-10-0-41-21 Ready node 10m v1.15.2 +ip-10-0-3-155 Ready controller,master 10m v1.15.3 +ip-10-0-26-65 Ready node 10m v1.15.3 +ip-10-0-41-21 Ready node 10m v1.15.3 ``` List the pods. diff --git a/docs/fedora-coreos/bare-metal.md b/docs/fedora-coreos/bare-metal.md index 305d17636..8a1c2a831 100644 --- a/docs/fedora-coreos/bare-metal.md +++ b/docs/fedora-coreos/bare-metal.md @@ -3,7 +3,7 @@ !!! danger Typhoon for Fedora CoreOS is an early preview! Fedora CoreOS itself is a preview! Expect bugs and design shifts. Please help both projects solve problems. Report Fedora CoreOS bugs to [Fedora](https://github.com/coreos/fedora-coreos-tracker/issues). Report Typhoon issues to Typhoon. -In this tutorial, we'll network boot and provision a Kubernetes v1.15.2 cluster on bare-metal with Fedora CoreOS. +In this tutorial, we'll network boot and provision a Kubernetes v1.15.3 cluster on bare-metal with Fedora CoreOS. First, we'll deploy a [Matchbox](https://github.com/poseidon/matchbox) service and setup a network boot environment. Then, we'll declare a Kubernetes cluster using the Typhoon Terraform module and power on machines. On PXE boot, machines will install Fedora CoreOS to disk, reboot into the disk install, and provision themselves as Kubernetes controllers or workers via Ignition. @@ -284,9 +284,9 @@ bootkube[5]: Tearing down temporary bootstrap control plane... $ export KUBECONFIG=/home/user/.secrets/clusters/mercury/auth/kubeconfig $ kubectl get nodes NAME STATUS ROLES AGE VERSION -node1.example.com Ready controller,master 10m v1.15.2 -node2.example.com Ready node 10m v1.15.2 -node3.example.com Ready node 10m v1.15.2 +node1.example.com Ready controller,master 10m v1.15.3 +node2.example.com Ready node 10m v1.15.3 +node3.example.com Ready node 10m v1.15.3 ``` List the pods. diff --git a/docs/index.md b/docs/index.md index dbe2a1e06..49740b34b 100644 --- a/docs/index.md +++ b/docs/index.md @@ -11,7 +11,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features -* Kubernetes v1.15.2 (upstream, via [kubernetes-incubator/bootkube](https://github.com/kubernetes-incubator/bootkube)) +* Kubernetes v1.15.3 (upstream, via [kubernetes-incubator/bootkube](https://github.com/kubernetes-incubator/bootkube)) * Single or multi-master, [Calico](https://www.projectcalico.org/) or [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/) * Advanced features like [worker pools](advanced/worker-pools/), [preemptible](cl/google-cloud/#preemption) workers, and [snippets](advanced/customization/#container-linux) customization @@ -47,7 +47,7 @@ Define a Kubernetes cluster by using the Terraform module for your chosen platfo ```tf module "google-cloud-yavin" { - source = "git::https://github.com/poseidon/typhoon//google-cloud/container-linux/kubernetes?ref=v1.15.2" + source = "git::https://github.com/poseidon/typhoon//google-cloud/container-linux/kubernetes?ref=v1.15.3" # Google Cloud cluster_name = "yavin" @@ -80,9 +80,9 @@ In 4-8 minutes (varies by platform), the cluster will be ready. This Google Clou $ export KUBECONFIG=/home/user/.secrets/clusters/yavin/auth/kubeconfig $ kubectl get nodes NAME ROLES STATUS AGE VERSION -yavin-controller-0.c.example-com.internal controller,master Ready 6m v1.15.2 -yavin-worker-jrbf.c.example-com.internal node Ready 5m v1.15.2 -yavin-worker-mzdm.c.example-com.internal node Ready 5m v1.15.2 +yavin-controller-0.c.example-com.internal controller,master Ready 6m v1.15.3 +yavin-worker-jrbf.c.example-com.internal node Ready 5m v1.15.3 +yavin-worker-mzdm.c.example-com.internal node Ready 5m v1.15.3 ``` List the pods. diff --git a/docs/topics/maintenance.md b/docs/topics/maintenance.md index 5eac215fe..d1597d68a 100644 --- a/docs/topics/maintenance.md +++ b/docs/topics/maintenance.md @@ -18,7 +18,7 @@ module "google-cloud-yavin" { } module "bare-metal-mercury" { - source = "git::https://github.com/poseidon/typhoon//bare-metal/container-linux/kubernetes?ref=v1.15.2" + source = "git::https://github.com/poseidon/typhoon//bare-metal/container-linux/kubernetes?ref=v1.15.3" ... } ``` @@ -279,15 +279,15 @@ Typhoon modules have been adapted for Terraform v0.12. Provider plugins requirem | Typhoon Release | Terraform version | |-------------------|---------------------| -| v1.15.2 - ? | v0.12.x | -| v1.10.3 - v1.15.2 | v0.11.x | +| v1.15.3 - ? | v0.12.x | +| v1.10.3 - v1.15.3 | v0.11.x | | v1.9.2 - v1.10.2 | v0.10.4+ or v0.11.x | | v1.7.3 - v1.9.1 | v0.10.x | | v1.6.4 - v1.7.2 | v0.9.x | ### New users -New users can start with Terraform v0.12.x and follow the docs for Typhoon v1.15.2+ without issue. +New users can start with Terraform v0.12.x and follow the docs for Typhoon v1.15.3+ without issue. ### Existing users @@ -404,7 +404,7 @@ tree . └── infraB <- new Terraform v0.12.x configs ``` -Define Typhoon clusters in the new config directory using Terraform v0.12 syntax. Follow the Typhoon v1.15.2+ docs (e.g. use `terraform12` in the `infraB` dir). See [AWS](/cl/aws), [Azure](/cl/azure), [Bare-Metal](/cl/bare-metal), [Digital Ocean](/cl/digital-ocean), or [Google-Cloud](/cl/google-cloud)) to create new clusters. Follow the usual [upgrade](/topics/maintenance/#upgrades) process to apply workloads and shift traffic. Later, switch back to the old config directory and deprovision clusters with Terraform v0.11. +Define Typhoon clusters in the new config directory using Terraform v0.12 syntax. Follow the Typhoon v1.15.3+ docs (e.g. use `terraform12` in the `infraB` dir). See [AWS](/cl/aws), [Azure](/cl/azure), [Bare-Metal](/cl/bare-metal), [Digital Ocean](/cl/digital-ocean), or [Google-Cloud](/cl/google-cloud)) to create new clusters. Follow the usual [upgrade](/topics/maintenance/#upgrades) process to apply workloads and shift traffic. Later, switch back to the old config directory and deprovision clusters with Terraform v0.11. ```shell terraform12 init diff --git a/google-cloud/container-linux/kubernetes/README.md b/google-cloud/container-linux/kubernetes/README.md index 9d148ba74..3a69bef35 100644 --- a/google-cloud/container-linux/kubernetes/README.md +++ b/google-cloud/container-linux/kubernetes/README.md @@ -11,7 +11,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features -* Kubernetes v1.15.2 (upstream, via [kubernetes-incubator/bootkube](https://github.com/kubernetes-incubator/bootkube)) +* Kubernetes v1.15.3 (upstream, via [kubernetes-incubator/bootkube](https://github.com/kubernetes-incubator/bootkube)) * Single or multi-master, [Calico](https://www.projectcalico.org/) or [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/) * Advanced features like [worker pools](https://typhoon.psdn.io/advanced/worker-pools/), [preemptible](https://typhoon.psdn.io/cl/google-cloud/#preemption) workers, and [snippets](https://typhoon.psdn.io/advanced/customization/#container-linux) customization diff --git a/google-cloud/container-linux/kubernetes/bootkube.tf b/google-cloud/container-linux/kubernetes/bootkube.tf index 033d01a12..904c2d5af 100644 --- a/google-cloud/container-linux/kubernetes/bootkube.tf +++ b/google-cloud/container-linux/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=8b3738b2cccf3cd9830beac170a53c4fbec26ed5" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=248675e7a95660e5c8a65f8b7ce86be89ef55a0e" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/google-cloud/container-linux/kubernetes/cl/controller.yaml.tmpl b/google-cloud/container-linux/kubernetes/cl/controller.yaml.tmpl index 9cf183862..af767c0cd 100644 --- a/google-cloud/container-linux/kubernetes/cl/controller.yaml.tmpl +++ b/google-cloud/container-linux/kubernetes/cl/controller.yaml.tmpl @@ -124,7 +124,7 @@ storage: contents: inline: | KUBELET_IMAGE_URL=docker://k8s.gcr.io/hyperkube - KUBELET_IMAGE_TAG=v1.15.2 + KUBELET_IMAGE_TAG=v1.15.3 - path: /etc/sysctl.d/max-user-watches.conf filesystem: root contents: diff --git a/google-cloud/container-linux/kubernetes/workers/cl/worker.yaml.tmpl b/google-cloud/container-linux/kubernetes/workers/cl/worker.yaml.tmpl index aefa0d8de..ab4d84474 100644 --- a/google-cloud/container-linux/kubernetes/workers/cl/worker.yaml.tmpl +++ b/google-cloud/container-linux/kubernetes/workers/cl/worker.yaml.tmpl @@ -94,7 +94,7 @@ storage: contents: inline: | KUBELET_IMAGE_URL=docker://k8s.gcr.io/hyperkube - KUBELET_IMAGE_TAG=v1.15.2 + KUBELET_IMAGE_TAG=v1.15.3 - path: /etc/sysctl.d/max-user-watches.conf filesystem: root contents: @@ -112,7 +112,7 @@ storage: --volume config,kind=host,source=/etc/kubernetes \ --mount volume=config,target=/etc/kubernetes \ --insecure-options=image \ - docker://k8s.gcr.io/hyperkube:v1.15.2 \ + docker://k8s.gcr.io/hyperkube:v1.15.3 \ --net=host \ --dns=host \ --exec=/kubectl -- --kubeconfig=/etc/kubernetes/kubeconfig delete node $(hostname) From c42139beaa1d14917b4b53e1c0597c637bde6266 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Mon, 19 Aug 2019 14:55:06 -0700 Subject: [PATCH 196/523] Update etcd from v3.3.14 to v3.3.15 * No functional changes, just changes to vendoring tools (go modules -> glide). Still, update to v3.3.15 anyway * https://github.com/etcd-io/etcd/compare/v3.3.14...v3.3.15 --- CHANGES.md | 2 +- aws/container-linux/kubernetes/cl/controller.yaml.tmpl | 2 +- aws/fedora-coreos/kubernetes/fcc/controller.yaml | 2 +- azure/container-linux/kubernetes/cl/controller.yaml.tmpl | 2 +- bare-metal/container-linux/kubernetes/cl/controller.yaml.tmpl | 2 +- bare-metal/fedora-coreos/kubernetes/fcc/controller.yaml | 2 +- .../container-linux/kubernetes/cl/controller.yaml.tmpl | 2 +- google-cloud/container-linux/kubernetes/cl/controller.yaml.tmpl | 2 +- 8 files changed, 8 insertions(+), 8 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index e1997b45a..0012cbebc 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -7,7 +7,7 @@ Notable changes between versions. ## v1.15.3 * Kubernetes [v1.15.3](https://github.com/kubernetes/kubernetes/blob/master/CHANGELOG-1.15.md#v1153) -* Update etcd from v3.3.13 to [v3.3.14](https://github.com/etcd-io/etcd/releases/tag/v3.3.14) +* Update etcd from v3.3.13 to [v3.3.15](https://github.com/etcd-io/etcd/releases/tag/v3.3.15) * Update Calico from v3.8.1 to [v3.8.2](https://docs.projectcalico.org/v3.8/release-notes/) #### AWS diff --git a/aws/container-linux/kubernetes/cl/controller.yaml.tmpl b/aws/container-linux/kubernetes/cl/controller.yaml.tmpl index c44c31c6d..0d5c3482f 100644 --- a/aws/container-linux/kubernetes/cl/controller.yaml.tmpl +++ b/aws/container-linux/kubernetes/cl/controller.yaml.tmpl @@ -7,7 +7,7 @@ systemd: - name: 40-etcd-cluster.conf contents: | [Service] - Environment="ETCD_IMAGE_TAG=v3.3.14" + Environment="ETCD_IMAGE_TAG=v3.3.15" Environment="ETCD_NAME=${etcd_name}" Environment="ETCD_ADVERTISE_CLIENT_URLS=https://${etcd_domain}:2379" Environment="ETCD_INITIAL_ADVERTISE_PEER_URLS=https://${etcd_domain}:2380" diff --git a/aws/fedora-coreos/kubernetes/fcc/controller.yaml b/aws/fedora-coreos/kubernetes/fcc/controller.yaml index 1ef51bb16..6a8ece453 100644 --- a/aws/fedora-coreos/kubernetes/fcc/controller.yaml +++ b/aws/fedora-coreos/kubernetes/fcc/controller.yaml @@ -28,7 +28,7 @@ systemd: --network host \ --volume /var/lib/etcd:/var/lib/etcd:rw,Z \ --volume /etc/ssl/etcd:/etc/ssl/certs:ro,Z \ - quay.io/coreos/etcd:v3.3.14 + quay.io/coreos/etcd:v3.3.15 ExecStop=/usr/bin/podman stop etcd [Install] WantedBy=multi-user.target diff --git a/azure/container-linux/kubernetes/cl/controller.yaml.tmpl b/azure/container-linux/kubernetes/cl/controller.yaml.tmpl index 8d760c387..0b1e6679c 100644 --- a/azure/container-linux/kubernetes/cl/controller.yaml.tmpl +++ b/azure/container-linux/kubernetes/cl/controller.yaml.tmpl @@ -7,7 +7,7 @@ systemd: - name: 40-etcd-cluster.conf contents: | [Service] - Environment="ETCD_IMAGE_TAG=v3.3.14" + Environment="ETCD_IMAGE_TAG=v3.3.15" Environment="ETCD_NAME=${etcd_name}" Environment="ETCD_ADVERTISE_CLIENT_URLS=https://${etcd_domain}:2379" Environment="ETCD_INITIAL_ADVERTISE_PEER_URLS=https://${etcd_domain}:2380" diff --git a/bare-metal/container-linux/kubernetes/cl/controller.yaml.tmpl b/bare-metal/container-linux/kubernetes/cl/controller.yaml.tmpl index 678bb8afd..70e5c641a 100644 --- a/bare-metal/container-linux/kubernetes/cl/controller.yaml.tmpl +++ b/bare-metal/container-linux/kubernetes/cl/controller.yaml.tmpl @@ -7,7 +7,7 @@ systemd: - name: 40-etcd-cluster.conf contents: | [Service] - Environment="ETCD_IMAGE_TAG=v3.3.14" + Environment="ETCD_IMAGE_TAG=v3.3.15" Environment="ETCD_NAME=${etcd_name}" Environment="ETCD_ADVERTISE_CLIENT_URLS=https://${domain_name}:2379" Environment="ETCD_INITIAL_ADVERTISE_PEER_URLS=https://${domain_name}:2380" diff --git a/bare-metal/fedora-coreos/kubernetes/fcc/controller.yaml b/bare-metal/fedora-coreos/kubernetes/fcc/controller.yaml index 9c4e2b707..f8f83d91c 100644 --- a/bare-metal/fedora-coreos/kubernetes/fcc/controller.yaml +++ b/bare-metal/fedora-coreos/kubernetes/fcc/controller.yaml @@ -28,7 +28,7 @@ systemd: --network host \ --volume /var/lib/etcd:/var/lib/etcd:rw,Z \ --volume /etc/ssl/etcd:/etc/ssl/certs:ro,Z \ - quay.io/coreos/etcd:v3.3.14 + quay.io/coreos/etcd:v3.3.15 ExecStop=/usr/bin/podman stop etcd [Install] WantedBy=multi-user.target diff --git a/digital-ocean/container-linux/kubernetes/cl/controller.yaml.tmpl b/digital-ocean/container-linux/kubernetes/cl/controller.yaml.tmpl index 5e3eabe91..f54d26ee6 100644 --- a/digital-ocean/container-linux/kubernetes/cl/controller.yaml.tmpl +++ b/digital-ocean/container-linux/kubernetes/cl/controller.yaml.tmpl @@ -7,7 +7,7 @@ systemd: - name: 40-etcd-cluster.conf contents: | [Service] - Environment="ETCD_IMAGE_TAG=v3.3.14" + Environment="ETCD_IMAGE_TAG=v3.3.15" Environment="ETCD_NAME=${etcd_name}" Environment="ETCD_ADVERTISE_CLIENT_URLS=https://${etcd_domain}:2379" Environment="ETCD_INITIAL_ADVERTISE_PEER_URLS=https://${etcd_domain}:2380" diff --git a/google-cloud/container-linux/kubernetes/cl/controller.yaml.tmpl b/google-cloud/container-linux/kubernetes/cl/controller.yaml.tmpl index af767c0cd..aeaec248e 100644 --- a/google-cloud/container-linux/kubernetes/cl/controller.yaml.tmpl +++ b/google-cloud/container-linux/kubernetes/cl/controller.yaml.tmpl @@ -7,7 +7,7 @@ systemd: - name: 40-etcd-cluster.conf contents: | [Service] - Environment="ETCD_IMAGE_TAG=v3.3.14" + Environment="ETCD_IMAGE_TAG=v3.3.15" Environment="ETCD_NAME=${etcd_name}" Environment="ETCD_ADVERTISE_CLIENT_URLS=https://${etcd_domain}:2379" Environment="ETCD_INITIAL_ADVERTISE_PEER_URLS=https://${etcd_domain}:2380" From d95bf2d1ea34ff6a844cde2661a650534ebf6f9f Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Tue, 27 Aug 2019 21:57:20 -0700 Subject: [PATCH 197/523] Update mkdocs-material from v4.4.0 to v4.4.2 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index f655243f5..0fe02e3b2 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ mkdocs==1.0.4 -mkdocs-material==4.4.0 +mkdocs-material==4.4.2 pygments==2.2.0 pymdown-extensions==5.0.0 From e7d805d9a4a69054a5a7767655714e3f4749d6dd Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Tue, 27 Aug 2019 22:00:08 -0700 Subject: [PATCH 198/523] Sync recommended versions of Terraform providers for clouds * Align Terraform provider plugin versions with those tested against --- docs/cl/aws.md | 2 +- docs/cl/azure.md | 2 +- docs/cl/digital-ocean.md | 2 +- docs/cl/google-cloud.md | 2 +- docs/fedora-coreos/aws.md | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/cl/aws.md b/docs/cl/aws.md index bf9e58dc8..86e58021b 100644 --- a/docs/cl/aws.md +++ b/docs/cl/aws.md @@ -49,7 +49,7 @@ Configure the AWS provider to use your access key credentials in a `providers.tf ```tf provider "aws" { - version = "2.23.0" + version = "2.25.0" region = "eu-central-1" shared_credentials_file = "/home/user/.config/aws/credentials" } diff --git a/docs/cl/azure.md b/docs/cl/azure.md index 773967cdc..99a1e2330 100644 --- a/docs/cl/azure.md +++ b/docs/cl/azure.md @@ -50,7 +50,7 @@ Configure the Azure provider in a `providers.tf` file. ```tf provider "azurerm" { - version = "1.32.1" + version = "1.33.0" } provider "ct" { diff --git a/docs/cl/digital-ocean.md b/docs/cl/digital-ocean.md index fcb28cf7e..5f0f0c47f 100644 --- a/docs/cl/digital-ocean.md +++ b/docs/cl/digital-ocean.md @@ -50,7 +50,7 @@ Configure the DigitalOcean provider to use your token in a `providers.tf` file. ```tf provider "digitalocean" { - version = "1.6.0" + version = "1.7.0" token = "${chomp(file("~/.config/digital-ocean/token"))}" } diff --git a/docs/cl/google-cloud.md b/docs/cl/google-cloud.md index ae7e892a0..ea033d5c6 100644 --- a/docs/cl/google-cloud.md +++ b/docs/cl/google-cloud.md @@ -49,7 +49,7 @@ Configure the Google Cloud provider to use your service account key, project-id, ```tf provider "google" { - version = "2.12.0" + version = "2.13.0" project = "project-id" region = "us-central1" credentials = "${file("~/.config/google-cloud/terraform.json")}" diff --git a/docs/fedora-coreos/aws.md b/docs/fedora-coreos/aws.md index 1870bdacb..724c4ad45 100644 --- a/docs/fedora-coreos/aws.md +++ b/docs/fedora-coreos/aws.md @@ -52,7 +52,7 @@ Configure the AWS provider to use your access key credentials in a `providers.tf ```tf provider "aws" { - version = "2.23.0" + version = "2.25.0" region = "us-east-1" # MUST be us-east-1 right now! shared_credentials_file = "/home/user/.config/aws/credentials" } From 4d5f962d761ebe1791f610ab0b228f1131d2048b Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Sat, 31 Aug 2019 15:24:10 -0700 Subject: [PATCH 199/523] Update CoreDNS from v1.5.0 to v1.6.2 * https://coredns.io/2019/06/26/coredns-1.5.1-release/ * https://coredns.io/2019/07/03/coredns-1.5.2-release/ * https://coredns.io/2019/07/28/coredns-1.6.0-release/ * https://coredns.io/2019/08/02/coredns-1.6.1-release/ * https://coredns.io/2019/08/13/coredns-1.6.2-release/ --- CHANGES.md | 2 ++ aws/container-linux/kubernetes/bootkube.tf | 2 +- aws/fedora-coreos/kubernetes/bootkube.tf | 2 +- azure/container-linux/kubernetes/bootkube.tf | 2 +- bare-metal/container-linux/kubernetes/bootkube.tf | 2 +- bare-metal/fedora-coreos/kubernetes/bootkube.tf | 2 +- digital-ocean/container-linux/kubernetes/bootkube.tf | 2 +- google-cloud/container-linux/kubernetes/bootkube.tf | 2 +- 8 files changed, 9 insertions(+), 7 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 0012cbebc..35fd715e7 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -4,6 +4,8 @@ Notable changes between versions. ## Latest +* Update CoreDNS from v1.5.0 to v1.6.2 ([#535](https://github.com/poseidon/typhoon/pull/535)) + ## v1.15.3 * Kubernetes [v1.15.3](https://github.com/kubernetes/kubernetes/blob/master/CHANGELOG-1.15.md#v1153) diff --git a/aws/container-linux/kubernetes/bootkube.tf b/aws/container-linux/kubernetes/bootkube.tf index b2f577e2b..f49cc8158 100644 --- a/aws/container-linux/kubernetes/bootkube.tf +++ b/aws/container-linux/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=248675e7a95660e5c8a65f8b7ce86be89ef55a0e" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=98cc19f80f2c4c3ddc63fc7aea6320e74bec561a" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/aws/fedora-coreos/kubernetes/bootkube.tf b/aws/fedora-coreos/kubernetes/bootkube.tf index 8cde2eeae..d651b51d9 100644 --- a/aws/fedora-coreos/kubernetes/bootkube.tf +++ b/aws/fedora-coreos/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=248675e7a95660e5c8a65f8b7ce86be89ef55a0e" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=98cc19f80f2c4c3ddc63fc7aea6320e74bec561a" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/azure/container-linux/kubernetes/bootkube.tf b/azure/container-linux/kubernetes/bootkube.tf index 9e1ab679c..f1d7d8e41 100644 --- a/azure/container-linux/kubernetes/bootkube.tf +++ b/azure/container-linux/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=248675e7a95660e5c8a65f8b7ce86be89ef55a0e" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=98cc19f80f2c4c3ddc63fc7aea6320e74bec561a" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/bare-metal/container-linux/kubernetes/bootkube.tf b/bare-metal/container-linux/kubernetes/bootkube.tf index f98964fee..f19b57f3b 100644 --- a/bare-metal/container-linux/kubernetes/bootkube.tf +++ b/bare-metal/container-linux/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=248675e7a95660e5c8a65f8b7ce86be89ef55a0e" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=98cc19f80f2c4c3ddc63fc7aea6320e74bec561a" cluster_name = var.cluster_name api_servers = [var.k8s_domain_name] diff --git a/bare-metal/fedora-coreos/kubernetes/bootkube.tf b/bare-metal/fedora-coreos/kubernetes/bootkube.tf index 7264478b0..ea43e5bc7 100644 --- a/bare-metal/fedora-coreos/kubernetes/bootkube.tf +++ b/bare-metal/fedora-coreos/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=248675e7a95660e5c8a65f8b7ce86be89ef55a0e" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=98cc19f80f2c4c3ddc63fc7aea6320e74bec561a" cluster_name = var.cluster_name api_servers = [var.k8s_domain_name] diff --git a/digital-ocean/container-linux/kubernetes/bootkube.tf b/digital-ocean/container-linux/kubernetes/bootkube.tf index 9755976fb..4c4fa1906 100644 --- a/digital-ocean/container-linux/kubernetes/bootkube.tf +++ b/digital-ocean/container-linux/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=248675e7a95660e5c8a65f8b7ce86be89ef55a0e" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=98cc19f80f2c4c3ddc63fc7aea6320e74bec561a" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/google-cloud/container-linux/kubernetes/bootkube.tf b/google-cloud/container-linux/kubernetes/bootkube.tf index 904c2d5af..867d0cc0e 100644 --- a/google-cloud/container-linux/kubernetes/bootkube.tf +++ b/google-cloud/container-linux/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=248675e7a95660e5c8a65f8b7ce86be89ef55a0e" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=98cc19f80f2c4c3ddc63fc7aea6320e74bec561a" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] From 45bc52d15627f27717507629f8a971a0b12a0970 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Sat, 31 Aug 2019 15:59:13 -0700 Subject: [PATCH 200/523] Update Grafana from v6.3.3 to v6.3.4 * https://github.com/grafana/grafana/releases/tag/v6.3.4 --- CHANGES.md | 4 ++++ addons/grafana/deployment.yaml | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index 35fd715e7..68a10c97c 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -6,6 +6,10 @@ Notable changes between versions. * Update CoreDNS from v1.5.0 to v1.6.2 ([#535](https://github.com/poseidon/typhoon/pull/535)) +#### Addons + +* Update Grafana from v6.3.3 to v6.3.4 + ## v1.15.3 * Kubernetes [v1.15.3](https://github.com/kubernetes/kubernetes/blob/master/CHANGELOG-1.15.md#v1153) diff --git a/addons/grafana/deployment.yaml b/addons/grafana/deployment.yaml index a4dbef818..83a1c305f 100644 --- a/addons/grafana/deployment.yaml +++ b/addons/grafana/deployment.yaml @@ -23,7 +23,7 @@ spec: spec: containers: - name: grafana - image: docker.io/grafana/grafana:6.3.3 + image: docker.io/grafana/grafana:6.3.4 env: - name: GF_PATHS_CONFIG value: "/etc/grafana/custom.ini" From b74f47070121ba5231563a3e449d2c4cd1579dfe Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Sat, 31 Aug 2019 16:07:22 -0700 Subject: [PATCH 201/523] Recommend updating terraform-provider-ct from v0.3.2 to v0.4.0 * v0.4.0 adds a "strict" mode we'll start using in future and also adds support for Fedora CoreOS * https://github.com/poseidon/terraform-provider-ct/releases/tag/v0.4.0 --- CHANGES.md | 1 + docs/cl/aws.md | 10 +++++----- docs/cl/azure.md | 10 +++++----- docs/cl/bare-metal.md | 10 +++++----- docs/cl/digital-ocean.md | 10 +++++----- docs/cl/google-cloud.md | 10 +++++----- docs/fedora-coreos/aws.md | 2 +- docs/fedora-coreos/bare-metal.md | 2 +- 8 files changed, 28 insertions(+), 27 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 68a10c97c..67f315fff 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -5,6 +5,7 @@ Notable changes between versions. ## Latest * Update CoreDNS from v1.5.0 to v1.6.2 ([#535](https://github.com/poseidon/typhoon/pull/535)) +* Recommend updating `terraform-provider-ct` plugin from v0.3.2 to [v0.4.0](https://github.com/poseidon/terraform-provider-ct/releases/tag/v0.4.0) #### Addons diff --git a/docs/cl/aws.md b/docs/cl/aws.md index 86e58021b..2b447faf1 100644 --- a/docs/cl/aws.md +++ b/docs/cl/aws.md @@ -18,15 +18,15 @@ Install [Terraform](https://www.terraform.io/downloads.html) v0.12.x on your sys ```sh $ terraform version -Terraform v0.12.6 +Terraform v0.12.7 ``` Add the [terraform-provider-ct](https://github.com/poseidon/terraform-provider-ct) plugin binary for your system to `~/.terraform.d/plugins/`, noting the final name. ```sh -wget https://github.com/poseidon/terraform-provider-ct/releases/download/v0.3.2/terraform-provider-ct-v0.3.2-linux-amd64.tar.gz -tar xzf terraform-provider-ct-v0.3.2-linux-amd64.tar.gz -mv terraform-provider-ct-v0.3.2-linux-amd64/terraform-provider-ct ~/.terraform.d/plugins/terraform-provider-ct_v0.3.2 +wget https://github.com/poseidon/terraform-provider-ct/releases/download/v0.4.0/terraform-provider-ct-v0.4.0-linux-amd64.tar.gz +tar xzf terraform-provider-ct-v0.4.0-linux-amd64.tar.gz +mv terraform-provider-ct-v0.4.0-linux-amd64/terraform-provider-ct ~/.terraform.d/plugins/terraform-provider-ct_v0.4.0 ``` Read [concepts](/architecture/concepts/) to learn about Terraform, modules, and organizing resources. Change to your infrastructure repository (e.g. `infra`). @@ -55,7 +55,7 @@ provider "aws" { } provider "ct" { - version = "0.3.2" + version = "0.4.0" } ``` diff --git a/docs/cl/azure.md b/docs/cl/azure.md index 99a1e2330..19ff805e5 100644 --- a/docs/cl/azure.md +++ b/docs/cl/azure.md @@ -21,15 +21,15 @@ Install [Terraform](https://www.terraform.io/downloads.html) v0.12.x on your sys ```sh $ terraform version -Terraform v0.12.6 +Terraform v0.12.7 ``` Add the [terraform-provider-ct](https://github.com/poseidon/terraform-provider-ct) plugin binary for your system to `~/.terraform.d/plugins/`, noting the final name. ```sh -wget https://github.com/poseidon/terraform-provider-ct/releases/download/v0.3.2/terraform-provider-ct-v0.3.2-linux-amd64.tar.gz -tar xzf terraform-provider-ct-v0.3.2-linux-amd64.tar.gz -mv terraform-provider-ct-v0.3.2-linux-amd64/terraform-provider-ct ~/.terraform.d/plugins/terraform-provider-ct_v0.3.2 +wget https://github.com/poseidon/terraform-provider-ct/releases/download/v0.4.0/terraform-provider-ct-v0.4.0-linux-amd64.tar.gz +tar xzf terraform-provider-ct-v0.4.0-linux-amd64.tar.gz +mv terraform-provider-ct-v0.4.0-linux-amd64/terraform-provider-ct ~/.terraform.d/plugins/terraform-provider-ct_v0.4.0 ``` Read [concepts](/architecture/concepts/) to learn about Terraform, modules, and organizing resources. Change to your infrastructure repository (e.g. `infra`). @@ -54,7 +54,7 @@ provider "azurerm" { } provider "ct" { - version = "0.3.2" + version = "0.4.0" } ``` diff --git a/docs/cl/bare-metal.md b/docs/cl/bare-metal.md index a8a4a3e17..3e73d4c0a 100644 --- a/docs/cl/bare-metal.md +++ b/docs/cl/bare-metal.md @@ -111,7 +111,7 @@ Install [Terraform](https://www.terraform.io/downloads.html) v0.12.x on your sys ```sh $ terraform version -Terraform v0.12.6 +Terraform v0.12.7 ``` Add the [terraform-provider-matchbox](https://github.com/poseidon/terraform-provider-matchbox) plugin binary for your system to `~/.terraform.d/plugins/`, noting the final name. @@ -125,9 +125,9 @@ mv terraform-provider-matchbox-v0.3.0-linux-amd64/terraform-provider-matchbox ~/ Add the [terraform-provider-ct](https://github.com/poseidon/terraform-provider-ct) plugin binary for your system to `~/.terraform.d/plugins/`, noting the final name. ```sh -wget https://github.com/poseidon/terraform-provider-ct/releases/download/v0.3.2/terraform-provider-ct-v0.3.2-linux-amd64.tar.gz -tar xzf terraform-provider-ct-v0.3.2-linux-amd64.tar.gz -mv terraform-provider-ct-v0.3.2-linux-amd64/terraform-provider-ct ~/.terraform.d/plugins/terraform-provider-ct_v0.3.2 +wget https://github.com/poseidon/terraform-provider-ct/releases/download/v0.4.0/terraform-provider-ct-v0.4.0-linux-amd64.tar.gz +tar xzf terraform-provider-ct-v0.4.0-linux-amd64.tar.gz +mv terraform-provider-ct-v0.4.0-linux-amd64/terraform-provider-ct ~/.terraform.d/plugins/terraform-provider-ct_v0.4.0 ``` Read [concepts](/architecture/concepts/) to learn about Terraform, modules, and organizing resources. Change to your infrastructure repository (e.g. `infra`). @@ -150,7 +150,7 @@ provider "matchbox" { } provider "ct" { - version = "0.3.2" + version = "0.4.0" } ``` diff --git a/docs/cl/digital-ocean.md b/docs/cl/digital-ocean.md index 5f0f0c47f..f2b4b3314 100644 --- a/docs/cl/digital-ocean.md +++ b/docs/cl/digital-ocean.md @@ -18,15 +18,15 @@ Install [Terraform](https://www.terraform.io/downloads.html) v0.12.x on your sys ```sh $ terraform version -Terraform v0.12.6 +Terraform v0.12.7 ``` Add the [terraform-provider-ct](https://github.com/poseidon/terraform-provider-ct) plugin binary for your system to `~/.terraform.d/plugins/`, noting the final name. ```sh -wget https://github.com/poseidon/terraform-provider-ct/releases/download/v0.3.2/terraform-provider-ct-v0.3.2-linux-amd64.tar.gz -tar xzf terraform-provider-ct-v0.3.2-linux-amd64.tar.gz -mv terraform-provider-ct-v0.3.2-linux-amd64/terraform-provider-ct ~/.terraform.d/plugins/terraform-provider-ct_v0.3.2 +wget https://github.com/poseidon/terraform-provider-ct/releases/download/v0.4.0/terraform-provider-ct-v0.4.0-linux-amd64.tar.gz +tar xzf terraform-provider-ct-v0.4.0-linux-amd64.tar.gz +mv terraform-provider-ct-v0.4.0-linux-amd64/terraform-provider-ct ~/.terraform.d/plugins/terraform-provider-ct_v0.4.0 ``` Read [concepts](/architecture/concepts/) to learn about Terraform, modules, and organizing resources. Change to your infrastructure repository (e.g. `infra`). @@ -55,7 +55,7 @@ provider "digitalocean" { } provider "ct" { - version = "0.3.2" + version = "0.4.0" } ``` diff --git a/docs/cl/google-cloud.md b/docs/cl/google-cloud.md index ea033d5c6..7d1e8a530 100644 --- a/docs/cl/google-cloud.md +++ b/docs/cl/google-cloud.md @@ -18,15 +18,15 @@ Install [Terraform](https://www.terraform.io/downloads.html) v0.12.x on your sys ```sh $ terraform version -Terraform v0.12.6 +Terraform v0.12.7 ``` Add the [terraform-provider-ct](https://github.com/poseidon/terraform-provider-ct) plugin binary for your system to `~/.terraform.d/plugins/`, noting the final name. ```sh -wget https://github.com/poseidon/terraform-provider-ct/releases/download/v0.3.2/terraform-provider-ct-v0.3.2-linux-amd64.tar.gz -tar xzf terraform-provider-ct-v0.3.2-linux-amd64.tar.gz -mv terraform-provider-ct-v0.3.2-linux-amd64/terraform-provider-ct ~/.terraform.d/plugins/terraform-provider-ct_v0.3.2 +wget https://github.com/poseidon/terraform-provider-ct/releases/download/v0.4.0/terraform-provider-ct-v0.4.0-linux-amd64.tar.gz +tar xzf terraform-provider-ct-v0.4.0-linux-amd64.tar.gz +mv terraform-provider-ct-v0.4.0-linux-amd64/terraform-provider-ct ~/.terraform.d/plugins/terraform-provider-ct_v0.4.0 ``` Read [concepts](/architecture/concepts/) to learn about Terraform, modules, and organizing resources. Change to your infrastructure repository (e.g. `infra`). @@ -56,7 +56,7 @@ provider "google" { } provider "ct" { - version = "0.3.2" + version = "0.4.0" } ``` diff --git a/docs/fedora-coreos/aws.md b/docs/fedora-coreos/aws.md index 724c4ad45..ab7d59060 100644 --- a/docs/fedora-coreos/aws.md +++ b/docs/fedora-coreos/aws.md @@ -21,7 +21,7 @@ Install [Terraform](https://www.terraform.io/downloads.html) v0.12.x on your sys ```sh $ terraform version -Terraform v0.12.6 +Terraform v0.12.7 ``` Add the [terraform-provider-ct](https://github.com/poseidon/terraform-provider-ct) plugin binary for your system to `~/.terraform.d/plugins/`, noting the final name. diff --git a/docs/fedora-coreos/bare-metal.md b/docs/fedora-coreos/bare-metal.md index 8a1c2a831..0fead4c5f 100644 --- a/docs/fedora-coreos/bare-metal.md +++ b/docs/fedora-coreos/bare-metal.md @@ -114,7 +114,7 @@ Install [Terraform](https://www.terraform.io/downloads.html) v0.12.x on your sys ```sh $ terraform version -Terraform v0.12.6 +Terraform v0.12.7 ``` Add the [terraform-provider-matchbox](https://github.com/poseidon/terraform-provider-matchbox) plugin binary for your system to `~/.terraform.d/plugins/`, noting the final name. From e8d586f3b3dd5d21e81c9881d23b6c28ef407664 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Tue, 3 Sep 2019 22:45:06 -0700 Subject: [PATCH 202/523] Enable QoS on Fedora CoreOS controllers * Kubelet race should be fixed in Kubernetes v1.15.1 * https://github.com/kubernetes/kubernetes/issues/79046 * Reverts temporary mitigation https://github.com/poseidon/typhoon/pull/515 --- aws/fedora-coreos/kubernetes/fcc/controller.yaml | 4 ++-- bare-metal/fedora-coreos/kubernetes/fcc/controller.yaml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/aws/fedora-coreos/kubernetes/fcc/controller.yaml b/aws/fedora-coreos/kubernetes/fcc/controller.yaml index 6a8ece453..423ea62be 100644 --- a/aws/fedora-coreos/kubernetes/fcc/controller.yaml +++ b/aws/fedora-coreos/kubernetes/fcc/controller.yaml @@ -85,8 +85,8 @@ systemd: --authentication-token-webhook \ --authorization-mode=Webhook \ --cgroup-driver=systemd \ - --cgroups-per-qos=false \ - --enforce-node-allocatable="" \ + --cgroups-per-qos=true \ + --enforce-node-allocatable=pods \ --client-ca-file=/etc/kubernetes/ca.crt \ --cluster_dns=${cluster_dns_service_ip} \ --cluster_domain=${cluster_domain_suffix} \ diff --git a/bare-metal/fedora-coreos/kubernetes/fcc/controller.yaml b/bare-metal/fedora-coreos/kubernetes/fcc/controller.yaml index f8f83d91c..c26db823d 100644 --- a/bare-metal/fedora-coreos/kubernetes/fcc/controller.yaml +++ b/bare-metal/fedora-coreos/kubernetes/fcc/controller.yaml @@ -86,8 +86,8 @@ systemd: --authentication-token-webhook \ --authorization-mode=Webhook \ --cgroup-driver=systemd \ - --cgroups-per-qos=false \ - --enforce-node-allocatable="" \ + --cgroups-per-qos=true \ + --enforce-node-allocatable=pods \ --client-ca-file=/etc/kubernetes/ca.crt \ --cluster_dns=${cluster_dns_service_ip} \ --cluster_domain=${cluster_domain_suffix} \ From efb9a2d09a4b325f776c751a818ea761e2b9fcb8 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Wed, 4 Sep 2019 21:11:22 -0700 Subject: [PATCH 203/523] Update Fedora CoreOS bare-metal docs for 30.20190801.0 --- docs/fedora-coreos/bare-metal.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/fedora-coreos/bare-metal.md b/docs/fedora-coreos/bare-metal.md index 0fead4c5f..e49a4a56c 100644 --- a/docs/fedora-coreos/bare-metal.md +++ b/docs/fedora-coreos/bare-metal.md @@ -169,8 +169,8 @@ module "bare-metal-mercury" { cluster_name = "mercury" matchbox_http_endpoint = "http://matchbox.example.com" os_stream = "testing" - os_version = "30.20190716.1" - cached_install = false + os_version = "30.20190801.0" + cached_install = "true" # configuration k8s_domain_name = "node1.example.com" From dc436b8fe9fa8420d222a5a0292a4ac8654060db Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Sat, 7 Sep 2019 14:21:59 -0700 Subject: [PATCH 204/523] Update Grafana from v6.3.4 to v6.3.5 * https://github.com/grafana/grafana/releases/tag/v6.3.5 --- CHANGES.md | 2 +- addons/grafana/deployment.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 67f315fff..ffc8399de 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -9,7 +9,7 @@ Notable changes between versions. #### Addons -* Update Grafana from v6.3.3 to v6.3.4 +* Update Grafana from v6.3.3 to v6.3.5 ## v1.15.3 diff --git a/addons/grafana/deployment.yaml b/addons/grafana/deployment.yaml index 83a1c305f..5c2671b64 100644 --- a/addons/grafana/deployment.yaml +++ b/addons/grafana/deployment.yaml @@ -23,7 +23,7 @@ spec: spec: containers: - name: grafana - image: docker.io/grafana/grafana:6.3.4 + image: docker.io/grafana/grafana:6.3.5 env: - name: GF_PATHS_CONFIG value: "/etc/grafana/custom.ini" From c20683067d04bd5d247e550c8fd75a03afea07e0 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Sun, 8 Sep 2019 12:57:40 -0700 Subject: [PATCH 205/523] Update etcd from v3.3.15 to v3.4.0 * https://github.com/etcd-io/etcd/releases/tag/v3.4.0 --- CHANGES.md | 1 + aws/container-linux/kubernetes/cl/controller.yaml.tmpl | 2 +- aws/fedora-coreos/kubernetes/fcc/controller.yaml | 2 +- azure/container-linux/kubernetes/cl/controller.yaml.tmpl | 2 +- bare-metal/container-linux/kubernetes/cl/controller.yaml.tmpl | 2 +- bare-metal/fedora-coreos/kubernetes/fcc/controller.yaml | 2 +- .../container-linux/kubernetes/cl/controller.yaml.tmpl | 2 +- google-cloud/container-linux/kubernetes/cl/controller.yaml.tmpl | 2 +- 8 files changed, 8 insertions(+), 7 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index ffc8399de..90551d080 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -5,6 +5,7 @@ Notable changes between versions. ## Latest * Update CoreDNS from v1.5.0 to v1.6.2 ([#535](https://github.com/poseidon/typhoon/pull/535)) +* Update etcd from v3.3.15 to [v3.4.0](https://github.com/etcd-io/etcd/releases/tag/v3.4.0) * Recommend updating `terraform-provider-ct` plugin from v0.3.2 to [v0.4.0](https://github.com/poseidon/terraform-provider-ct/releases/tag/v0.4.0) #### Addons diff --git a/aws/container-linux/kubernetes/cl/controller.yaml.tmpl b/aws/container-linux/kubernetes/cl/controller.yaml.tmpl index 0d5c3482f..6fdf12f3b 100644 --- a/aws/container-linux/kubernetes/cl/controller.yaml.tmpl +++ b/aws/container-linux/kubernetes/cl/controller.yaml.tmpl @@ -7,7 +7,7 @@ systemd: - name: 40-etcd-cluster.conf contents: | [Service] - Environment="ETCD_IMAGE_TAG=v3.3.15" + Environment="ETCD_IMAGE_TAG=v3.4.0" Environment="ETCD_NAME=${etcd_name}" Environment="ETCD_ADVERTISE_CLIENT_URLS=https://${etcd_domain}:2379" Environment="ETCD_INITIAL_ADVERTISE_PEER_URLS=https://${etcd_domain}:2380" diff --git a/aws/fedora-coreos/kubernetes/fcc/controller.yaml b/aws/fedora-coreos/kubernetes/fcc/controller.yaml index 423ea62be..047cf1b0e 100644 --- a/aws/fedora-coreos/kubernetes/fcc/controller.yaml +++ b/aws/fedora-coreos/kubernetes/fcc/controller.yaml @@ -28,7 +28,7 @@ systemd: --network host \ --volume /var/lib/etcd:/var/lib/etcd:rw,Z \ --volume /etc/ssl/etcd:/etc/ssl/certs:ro,Z \ - quay.io/coreos/etcd:v3.3.15 + quay.io/coreos/etcd:v3.4.0 ExecStop=/usr/bin/podman stop etcd [Install] WantedBy=multi-user.target diff --git a/azure/container-linux/kubernetes/cl/controller.yaml.tmpl b/azure/container-linux/kubernetes/cl/controller.yaml.tmpl index 0b1e6679c..1d0981b9b 100644 --- a/azure/container-linux/kubernetes/cl/controller.yaml.tmpl +++ b/azure/container-linux/kubernetes/cl/controller.yaml.tmpl @@ -7,7 +7,7 @@ systemd: - name: 40-etcd-cluster.conf contents: | [Service] - Environment="ETCD_IMAGE_TAG=v3.3.15" + Environment="ETCD_IMAGE_TAG=v3.4.0" Environment="ETCD_NAME=${etcd_name}" Environment="ETCD_ADVERTISE_CLIENT_URLS=https://${etcd_domain}:2379" Environment="ETCD_INITIAL_ADVERTISE_PEER_URLS=https://${etcd_domain}:2380" diff --git a/bare-metal/container-linux/kubernetes/cl/controller.yaml.tmpl b/bare-metal/container-linux/kubernetes/cl/controller.yaml.tmpl index 70e5c641a..234c00f5a 100644 --- a/bare-metal/container-linux/kubernetes/cl/controller.yaml.tmpl +++ b/bare-metal/container-linux/kubernetes/cl/controller.yaml.tmpl @@ -7,7 +7,7 @@ systemd: - name: 40-etcd-cluster.conf contents: | [Service] - Environment="ETCD_IMAGE_TAG=v3.3.15" + Environment="ETCD_IMAGE_TAG=v3.4.0" Environment="ETCD_NAME=${etcd_name}" Environment="ETCD_ADVERTISE_CLIENT_URLS=https://${domain_name}:2379" Environment="ETCD_INITIAL_ADVERTISE_PEER_URLS=https://${domain_name}:2380" diff --git a/bare-metal/fedora-coreos/kubernetes/fcc/controller.yaml b/bare-metal/fedora-coreos/kubernetes/fcc/controller.yaml index c26db823d..abe358359 100644 --- a/bare-metal/fedora-coreos/kubernetes/fcc/controller.yaml +++ b/bare-metal/fedora-coreos/kubernetes/fcc/controller.yaml @@ -28,7 +28,7 @@ systemd: --network host \ --volume /var/lib/etcd:/var/lib/etcd:rw,Z \ --volume /etc/ssl/etcd:/etc/ssl/certs:ro,Z \ - quay.io/coreos/etcd:v3.3.15 + quay.io/coreos/etcd:v3.4.0 ExecStop=/usr/bin/podman stop etcd [Install] WantedBy=multi-user.target diff --git a/digital-ocean/container-linux/kubernetes/cl/controller.yaml.tmpl b/digital-ocean/container-linux/kubernetes/cl/controller.yaml.tmpl index f54d26ee6..653a9899e 100644 --- a/digital-ocean/container-linux/kubernetes/cl/controller.yaml.tmpl +++ b/digital-ocean/container-linux/kubernetes/cl/controller.yaml.tmpl @@ -7,7 +7,7 @@ systemd: - name: 40-etcd-cluster.conf contents: | [Service] - Environment="ETCD_IMAGE_TAG=v3.3.15" + Environment="ETCD_IMAGE_TAG=v3.4.0" Environment="ETCD_NAME=${etcd_name}" Environment="ETCD_ADVERTISE_CLIENT_URLS=https://${etcd_domain}:2379" Environment="ETCD_INITIAL_ADVERTISE_PEER_URLS=https://${etcd_domain}:2380" diff --git a/google-cloud/container-linux/kubernetes/cl/controller.yaml.tmpl b/google-cloud/container-linux/kubernetes/cl/controller.yaml.tmpl index aeaec248e..63ca9547f 100644 --- a/google-cloud/container-linux/kubernetes/cl/controller.yaml.tmpl +++ b/google-cloud/container-linux/kubernetes/cl/controller.yaml.tmpl @@ -7,7 +7,7 @@ systemd: - name: 40-etcd-cluster.conf contents: | [Service] - Environment="ETCD_IMAGE_TAG=v3.3.15" + Environment="ETCD_IMAGE_TAG=v3.4.0" Environment="ETCD_NAME=${etcd_name}" Environment="ETCD_ADVERTISE_CLIENT_URLS=https://${etcd_domain}:2379" Environment="ETCD_INITIAL_ADVERTISE_PEER_URLS=https://${etcd_domain}:2380" From 4a7083d94a6e199091f248f669516109fdbffb47 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Mon, 9 Sep 2019 22:24:10 -0700 Subject: [PATCH 206/523] Change Azure default controller_type and worker_type * Change default controller_type to Standard_B2s. A B2s is cheaper by $17/month and provides 2 vCPU, 4GB RAM (vs 1 vCPU, 3.5GB RAM) * Change default worker_type to Standard_DS1_v2. F1 was the previous generation. The DS1_v2 is newer, similar cost, more memory, and still supports Low Priority mode, if desired --- CHANGES.md | 7 +++++++ azure/container-linux/kubernetes/variables.tf | 4 ++-- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 90551d080..61cab0149 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -8,6 +8,13 @@ Notable changes between versions. * Update etcd from v3.3.15 to [v3.4.0](https://github.com/etcd-io/etcd/releases/tag/v3.4.0) * Recommend updating `terraform-provider-ct` plugin from v0.3.2 to [v0.4.0](https://github.com/poseidon/terraform-provider-ct/releases/tag/v0.4.0) +#### Azure + +* Change default `controller_type` to `Standard_B2s` ([#539](https://github.com/poseidon/typhoon/pull/539)) + * `B2s` is cheaper by $17/month and provides 2 vCPU, 4GB RAM +* Change default `worker_type` to `Standard_DS1_v2` ([#539](https://github.com/poseidon/typhoon/pull/539)) + * `F1` is previous generation. `DS1_v2` is newer, similar cost, and supports Low Priority mode + #### Addons * Update Grafana from v6.3.3 to v6.3.5 diff --git a/azure/container-linux/kubernetes/variables.tf b/azure/container-linux/kubernetes/variables.tf index 5ff72471a..74b2dde83 100644 --- a/azure/container-linux/kubernetes/variables.tf +++ b/azure/container-linux/kubernetes/variables.tf @@ -36,13 +36,13 @@ variable "worker_count" { variable "controller_type" { type = string - default = "Standard_DS1_v2" + default = "Standard_B2s" description = "Machine type for controllers (see `az vm list-skus --location centralus`)" } variable "worker_type" { type = string - default = "Standard_F1" + default = "Standard_DS1_v2" description = "Machine type for workers (see `az vm list-skus --location centralus`)" } From b60a2ecdf76d88573a021c4f8673ddb19c0d42c0 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Mon, 2 Sep 2019 21:10:30 -0700 Subject: [PATCH 207/523] Migrate Fedora CoreOS AWS to a static pod control plane * Run a kube-apiserver, kube-scheduler, and kube-controller-manager static pod on each controller node. Previously, kube-apiserver was self-hosted as a DaemonSet across controllers and kube-scheduler and kube-controller-manager were a Deployment (with 2 or controller_count many replicas). * Remove bootkube bootstrap and pivot to self-hosted * Remove pod-checkpointer manifests (no longer needed) --- aws/fedora-coreos/kubernetes/README.md | 2 +- aws/fedora-coreos/kubernetes/bootkube.tf | 2 +- .../kubernetes/fcc/controller.yaml | 39 +++++++++++++------ aws/fedora-coreos/kubernetes/security.tf | 22 +++++++++++ aws/fedora-coreos/kubernetes/ssh.tf | 32 ++++++++------- .../kubernetes/workers/fcc/worker.yaml | 1 - docs/fedora-coreos/aws.md | 38 ++++++++---------- 7 files changed, 87 insertions(+), 49 deletions(-) diff --git a/aws/fedora-coreos/kubernetes/README.md b/aws/fedora-coreos/kubernetes/README.md index c249ea1a7..33421e84e 100644 --- a/aws/fedora-coreos/kubernetes/README.md +++ b/aws/fedora-coreos/kubernetes/README.md @@ -11,7 +11,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features -* Kubernetes v1.15.3 (upstream, via [kubernetes-incubator/bootkube](https://github.com/kubernetes-incubator/bootkube)) +* Kubernetes v1.15.3 (upstream) * Single or multi-master, [Calico](https://www.projectcalico.org/) or [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/) * Advanced features like [worker pools](https://typhoon.psdn.io/advanced/worker-pools/), [spot](https://typhoon.psdn.io/cl/aws/#spot) workers, and [snippets](https://typhoon.psdn.io/advanced/customization/#container-linux) customization diff --git a/aws/fedora-coreos/kubernetes/bootkube.tf b/aws/fedora-coreos/kubernetes/bootkube.tf index d651b51d9..b11e6cf85 100644 --- a/aws/fedora-coreos/kubernetes/bootkube.tf +++ b/aws/fedora-coreos/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=98cc19f80f2c4c3ddc63fc7aea6320e74bec561a" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=6e59af71138bc5f784453873074de16e7ee150eb" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/aws/fedora-coreos/kubernetes/fcc/controller.yaml b/aws/fedora-coreos/kubernetes/fcc/controller.yaml index 047cf1b0e..667583c7f 100644 --- a/aws/fedora-coreos/kubernetes/fcc/controller.yaml +++ b/aws/fedora-coreos/kubernetes/fcc/controller.yaml @@ -107,33 +107,48 @@ systemd: RestartSec=10 [Install] WantedBy=multi-user.target - - name: bootkube.service + - name: bootstrap.service contents: | [Unit] - Description=Bootstrap a Kubernetes control plane - ConditionPathExists=!/opt/bootkube/init_bootkube.done + Description=Bootstrap Kubernetes control plane + ConditionPathExists=!/opt/bootstrap/bootstrap.done [Service] Type=oneshot RemainAfterExit=true - WorkingDirectory=/opt/bootkube - ExecStart=/usr/bin/bash -c 'set -x && \ - [ -n "$(ls /opt/bootkube/assets/manifests-*/* 2>/dev/null)" ] && mv /opt/bootkube/assets/manifests-*/* /opt/bootkube/assets/manifests && rm -rf /opt/bootkube/assets/manifests-* && exec podman run --name bootkube --privileged \ + WorkingDirectory=/opt/bootstrap + ExecStartPre=-/usr/bin/bash -c 'set -x && [ -n "$(ls /opt/bootstrap/assets/manifests-*/* 2>/dev/null)" ] && mv /opt/bootstrap/assets/manifests-*/* /opt/bootstrap/assets/manifests && rm -rf /opt/bootstrap/assets/manifests-*' + ExecStart=/usr/bin/podman run --name bootstrap \ --network host \ - --volume /opt/bootkube/assets:/assets \ - --volume /etc/kubernetes:/etc/kubernetes \ - quay.io/coreos/bootkube:v0.14.0 \ - /bootkube start --asset-dir=/assets' - ExecStartPost=/bin/touch /opt/bootkube/init_bootkube.done + --volume /opt/bootstrap/assets:/assets:ro,Z \ + --volume /opt/bootstrap/apply:/apply:ro,Z \ + k8s.gcr.io/hyperkube:v1.15.3 \ + /apply + ExecStartPost=/bin/touch /opt/bootstrap/bootstrap.done + ExecStartPost=-/usr/bin/podman stop bootstrap storage: directories: - path: /etc/kubernetes - - path: /opt/bootkube + - path: /opt/bootstrap files: - path: /etc/kubernetes/kubeconfig mode: 0644 contents: inline: | ${kubeconfig} + - path: /opt/bootstrap/apply + mode: 0544 + contents: + inline: | + #!/bin/bash -e + export KUBECONFIG=/assets/auth/kubeconfig + until kubectl version; do + echo "Waiting for static pod control plane" + sleep 5 + done + until kubectl apply -f /assets/manifests -R; do + echo "Retry applying manifests" + sleep 5 + done - path: /etc/sysctl.d/reverse-path-filter.conf contents: inline: | diff --git a/aws/fedora-coreos/kubernetes/security.tf b/aws/fedora-coreos/kubernetes/security.tf index aa2f84cb8..ddc4e8e47 100644 --- a/aws/fedora-coreos/kubernetes/security.tf +++ b/aws/fedora-coreos/kubernetes/security.tf @@ -44,6 +44,28 @@ resource "aws_security_group_rule" "controller-etcd-metrics" { source_security_group_id = aws_security_group.worker.id } +# Allow Prometheus to scrape kube-scheduler +resource "aws_security_group_rule" "controller-scheduler-metrics" { + security_group_id = aws_security_group.controller.id + + type = "ingress" + protocol = "tcp" + from_port = 10251 + to_port = 10251 + source_security_group_id = aws_security_group.worker.id +} + +# Allow Prometheus to scrape kube-controller-manager +resource "aws_security_group_rule" "controller-manager-metrics" { + security_group_id = aws_security_group.controller.id + + type = "ingress" + protocol = "tcp" + from_port = 10252 + to_port = 10252 + source_security_group_id = aws_security_group.worker.id +} + resource "aws_security_group_rule" "controller-vxlan" { count = var.networking == "flannel" ? 1 : 0 diff --git a/aws/fedora-coreos/kubernetes/ssh.tf b/aws/fedora-coreos/kubernetes/ssh.tf index 09e31c192..2e39779e2 100644 --- a/aws/fedora-coreos/kubernetes/ssh.tf +++ b/aws/fedora-coreos/kubernetes/ssh.tf @@ -1,6 +1,10 @@ -# Secure copy etcd TLS assets to controllers. +# Secure copy assets to controllers. resource "null_resource" "copy-controller-secrets" { count = var.controller_count + + depends_on = [ + module.bootkube, + ] connection { type = "ssh" @@ -44,6 +48,11 @@ resource "null_resource" "copy-controller-secrets" { destination = "$HOME/etcd-peer.key" } + provisioner "file" { + source = var.asset_dir + destination = "$HOME/assets" + } + provisioner "remote-exec" { inline = [ "sudo mkdir -p /etc/ssl/etcd/etcd", @@ -56,18 +65,21 @@ resource "null_resource" "copy-controller-secrets" { "sudo mv etcd-peer.key /etc/ssl/etcd/etcd/peer.key", "sudo chown -R etcd:etcd /etc/ssl/etcd", "sudo chmod -R 500 /etc/ssl/etcd", + "sudo mv $HOME/assets /opt/bootstrap/assets", + "sudo mkdir -p /etc/kubernetes/bootstrap-secrets", + "sudo cp -r /opt/bootstrap/assets/tls/* /etc/kubernetes/bootstrap-secrets/", + "sudo cp /opt/bootstrap/assets/auth/kubeconfig /etc/kubernetes/bootstrap-secrets/", + "sudo cp -r /opt/bootstrap/assets/static-manifests/* /etc/kubernetes/manifests/" ] } } -# Secure copy bootkube assets to ONE controller and start bootkube to perform -# one-time self-hosted cluster bootstrapping. -resource "null_resource" "bootkube-start" { +# Connect to a controller to perform one-time cluster bootstrap. +resource "null_resource" "bootstrap" { depends_on = [ - module.bootkube, + null_resource.copy-controller-secrets, module.workers, aws_route53_record.apiserver, - null_resource.copy-controller-secrets, ] connection { @@ -77,15 +89,9 @@ resource "null_resource" "bootkube-start" { timeout = "15m" } - provisioner "file" { - source = var.asset_dir - destination = "$HOME/assets" - } - provisioner "remote-exec" { inline = [ - "sudo mv $HOME/assets /opt/bootkube", - "sudo systemctl start bootkube", + "sudo systemctl start bootstrap", ] } } diff --git a/aws/fedora-coreos/kubernetes/workers/fcc/worker.yaml b/aws/fedora-coreos/kubernetes/workers/fcc/worker.yaml index c5f1aeffc..4cdcd4627 100644 --- a/aws/fedora-coreos/kubernetes/workers/fcc/worker.yaml +++ b/aws/fedora-coreos/kubernetes/workers/fcc/worker.yaml @@ -78,7 +78,6 @@ systemd: storage: directories: - path: /etc/kubernetes - - path: /opt/bootkube files: - path: /etc/kubernetes/kubeconfig mode: 0644 diff --git a/docs/fedora-coreos/aws.md b/docs/fedora-coreos/aws.md index ab7d59060..67466c828 100644 --- a/docs/fedora-coreos/aws.md +++ b/docs/fedora-coreos/aws.md @@ -7,7 +7,7 @@ In this tutorial, we'll create a Kubernetes v1.15.3 cluster on AWS with Fedora C We'll declare a Kubernetes cluster using the Typhoon Terraform module. Then apply the changes to create a VPC, gateway, subnets, security groups, controller instances, worker auto-scaling group, network load balancer, and TLS assets. -Controllers are provisioned to run an `etcd-member` peer and a `kubelet` service. Workers run just a `kubelet` service. A one-time [bootkube](https://github.com/kubernetes-incubator/bootkube) bootstrap schedules the `apiserver`, `scheduler`, `controller-manager`, and `coredns` on controllers and schedules `kube-proxy` and `calico` (or `flannel`) on every node. A generated `kubeconfig` provides `kubectl` access to the cluster. +Controllers hosts are provisioned to run an `etcd-member` peer and a `kubelet` service. Worker hosts run a `kubelet` service. Controller nodes run `kube-apiserver`, `kube-scheduler`, `kube-controller-manager`, and `coredns`, while `kube-proxy` and `calico` (or `flannel`) run on every node. A generated `kubeconfig` provides `kubectl` access to the cluster. ## Requirements @@ -94,7 +94,7 @@ Reference the [variables docs](#variables) or the [variables.tf](https://github. ## ssh-agent -Initial bootstrapping requires `bootkube.service` be started on one controller node. Terraform uses `ssh-agent` to automate this step. Add your SSH private key to `ssh-agent`. +Initial bootstrapping requires `bootstrap.service` be started on one controller node. Terraform uses `ssh-agent` to automate this step. Add your SSH private key to `ssh-agent`. ```sh ssh-add ~/.ssh/id_rsa @@ -121,9 +121,9 @@ Apply the changes to create the cluster. ```sh $ terraform apply ... -module.aws-tempest.null_resource.bootkube-start: Still creating... (4m50s elapsed) -module.aws-tempest.null_resource.bootkube-start: Still creating... (5m0s elapsed) -module.aws-tempest.null_resource.bootkube-start: Creation complete after 11m8s (ID: 3961816482286168143) +module.aws-tempest.null_resource.bootstrap: Still creating... (4m50s elapsed) +module.aws-tempest.null_resource.bootstrap: Still creating... (5m0s elapsed) +module.aws-tempest.null_resource.bootstrap: Creation complete after 5m8s (ID: 3961816482286168143) Apply complete! Resources: 98 added, 0 changed, 0 destroyed. ``` @@ -147,22 +147,18 @@ List the pods. ``` $ kubectl get pods --all-namespaces -NAMESPACE NAME READY STATUS RESTARTS AGE -kube-system calico-node-1m5bf 2/2 Running 0 34m -kube-system calico-node-7jmr1 2/2 Running 0 34m -kube-system calico-node-bknc8 2/2 Running 0 34m -kube-system coredns-1187388186-wx1lg 1/1 Running 0 34m -kube-system coredns-1187388186-qjnvp 1/1 Running 0 34m -kube-system kube-apiserver-4mjbk 1/1 Running 0 34m -kube-system kube-controller-manager-3597210155-j2jbt 1/1 Running 1 34m -kube-system kube-controller-manager-3597210155-j7g7x 1/1 Running 0 34m -kube-system kube-proxy-14wxv 1/1 Running 0 34m -kube-system kube-proxy-9vxh2 1/1 Running 0 34m -kube-system kube-proxy-sbbsh 1/1 Running 0 34m -kube-system kube-scheduler-3359497473-5plhf 1/1 Running 0 34m -kube-system kube-scheduler-3359497473-r7zg7 1/1 Running 1 34m -kube-system pod-checkpointer-4kxtl 1/1 Running 0 34m -kube-system pod-checkpointer-4kxtl-ip-10-0-3-155 1/1 Running 0 33m +NAMESPACE NAME READY STATUS RESTARTS AGE +kube-system calico-node-1m5bf 2/2 Running 0 34m +kube-system calico-node-7jmr1 2/2 Running 0 34m +kube-system calico-node-bknc8 2/2 Running 0 34m +kube-system coredns-1187388186-wx1lg 1/1 Running 0 34m +kube-system coredns-1187388186-qjnvp 1/1 Running 0 34m +kube-system kube-apiserver-4mjbk 1/1 Running 0 34m +kube-system kube-controller-manager-ip-10-0-3-155 1/1 Running 0 34m +kube-system kube-proxy-14wxv 1/1 Running 0 34m +kube-system kube-proxy-9vxh2 1/1 Running 0 34m +kube-system kube-proxy-sbbsh 1/1 Running 0 34m +kube-system kube-scheduler-ip-10-0-3-155 1/1 Running 1 34m ``` ## Going Further From 74780fb09fcd8019f53cbabe096764b8bd6b934a Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Tue, 3 Sep 2019 22:00:34 -0700 Subject: [PATCH 208/523] Migrate Fedora CoreOS bare-metal to static pod control plane * Run a kube-apiserver, kube-scheduler, and kube-controller-manager static pod on each controller node. Previously, kube-apiserver was self-hosted as a DaemonSet across controllers and kube-scheduler and kube-controller-manager were a Deployment (with 2 or controller_count many replicas). * Remove bootkube bootstrap and pivot to self-hosted * Remove pod-checkpointer manifests (no longer needed) --- .../kubernetes/fcc/controller.yaml | 2 +- bare-metal/fedora-coreos/kubernetes/README.md | 2 +- .../fedora-coreos/kubernetes/bootkube.tf | 2 +- .../kubernetes/fcc/controller.yaml | 39 ++++++++++++------ .../fedora-coreos/kubernetes/fcc/worker.yaml | 1 - bare-metal/fedora-coreos/kubernetes/ssh.tf | 26 +++++++----- docs/fedora-coreos/aws.md | 4 +- docs/fedora-coreos/bare-metal.md | 41 ++++++++----------- 8 files changed, 65 insertions(+), 52 deletions(-) diff --git a/aws/fedora-coreos/kubernetes/fcc/controller.yaml b/aws/fedora-coreos/kubernetes/fcc/controller.yaml index 667583c7f..f309a2981 100644 --- a/aws/fedora-coreos/kubernetes/fcc/controller.yaml +++ b/aws/fedora-coreos/kubernetes/fcc/controller.yaml @@ -110,7 +110,7 @@ systemd: - name: bootstrap.service contents: | [Unit] - Description=Bootstrap Kubernetes control plane + Description=Kubernetes control plane ConditionPathExists=!/opt/bootstrap/bootstrap.done [Service] Type=oneshot diff --git a/bare-metal/fedora-coreos/kubernetes/README.md b/bare-metal/fedora-coreos/kubernetes/README.md index 938a4732a..8292c8c62 100644 --- a/bare-metal/fedora-coreos/kubernetes/README.md +++ b/bare-metal/fedora-coreos/kubernetes/README.md @@ -11,7 +11,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features -* Kubernetes v1.15.3 (upstream, via [kubernetes-incubator/bootkube](https://github.com/kubernetes-incubator/bootkube)) +* Kubernetes v1.15.3 (upstream) * Single or multi-master, [Calico](https://www.projectcalico.org/) or [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/) * Advanced features like [snippets](https://typhoon.psdn.io/advanced/customization/#container-linux) customization diff --git a/bare-metal/fedora-coreos/kubernetes/bootkube.tf b/bare-metal/fedora-coreos/kubernetes/bootkube.tf index ea43e5bc7..a9c0bb605 100644 --- a/bare-metal/fedora-coreos/kubernetes/bootkube.tf +++ b/bare-metal/fedora-coreos/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=98cc19f80f2c4c3ddc63fc7aea6320e74bec561a" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=6e59af71138bc5f784453873074de16e7ee150eb" cluster_name = var.cluster_name api_servers = [var.k8s_domain_name] diff --git a/bare-metal/fedora-coreos/kubernetes/fcc/controller.yaml b/bare-metal/fedora-coreos/kubernetes/fcc/controller.yaml index abe358359..9ebbf6916 100644 --- a/bare-metal/fedora-coreos/kubernetes/fcc/controller.yaml +++ b/bare-metal/fedora-coreos/kubernetes/fcc/controller.yaml @@ -118,33 +118,48 @@ systemd: PathExists=/etc/kubernetes/kubeconfig [Install] WantedBy=multi-user.target - - name: bootkube.service + - name: bootstrap.service contents: | [Unit] - Description=Bootstrap a Kubernetes control plane - ConditionPathExists=!/opt/bootkube/init_bootkube.done + Description=Kubernetes control plane + ConditionPathExists=!/opt/bootstrap/bootstrap.done [Service] Type=oneshot RemainAfterExit=true - WorkingDirectory=/opt/bootkube - ExecStart=/usr/bin/bash -c 'set -x && \ - [ -n "$(ls /opt/bootkube/assets/manifests-*/* 2>/dev/null)" ] && mv /opt/bootkube/assets/manifests-*/* /opt/bootkube/assets/manifests && rm -rf /opt/bootkube/assets/manifests-* && exec podman run --name bootkube --privileged \ + WorkingDirectory=/opt/bootstrap + ExecStartPre=-/usr/bin/bash -c 'set -x && [ -n "$(ls /opt/bootstrap/assets/manifests-*/* 2>/dev/null)" ] && mv /opt/bootstrap/assets/manifests-*/* /opt/bootstrap/assets/manifests && rm -rf /opt/bootstrap/assets/manifests-*' + ExecStart=/usr/bin/podman run --name bootstrap \ --network host \ - --volume /opt/bootkube/assets:/assets \ - --volume /etc/kubernetes:/etc/kubernetes \ - quay.io/coreos/bootkube:v0.14.0 \ - /bootkube start --asset-dir=/assets' - ExecStartPost=/bin/touch /opt/bootkube/init_bootkube.done + --volume /opt/bootstrap/assets:/assets:ro,Z \ + --volume /opt/bootstrap/apply:/apply:ro,Z \ + k8s.gcr.io/hyperkube:v1.15.3 \ + /apply + ExecStartPost=/bin/touch /opt/bootstrap/bootstrap.done + ExecStartPost=-/usr/bin/podman stop bootstrap storage: directories: - path: /etc/kubernetes - - path: /opt/bootkube + - path: /opt/bootstrap files: - path: /etc/hostname mode: 0644 contents: inline: ${domain_name} + - path: /opt/bootstrap/apply + mode: 0544 + contents: + inline: | + #!/bin/bash -e + export KUBECONFIG=/assets/auth/kubeconfig + until kubectl version; do + echo "Waiting for static pod control plane" + sleep 5 + done + until kubectl apply -f /assets/manifests -R; do + echo "Retry applying manifests" + sleep 5 + done - path: /etc/sysctl.d/reverse-path-filter.conf contents: inline: | diff --git a/bare-metal/fedora-coreos/kubernetes/fcc/worker.yaml b/bare-metal/fedora-coreos/kubernetes/fcc/worker.yaml index 8016a6e04..fc4bb600c 100644 --- a/bare-metal/fedora-coreos/kubernetes/fcc/worker.yaml +++ b/bare-metal/fedora-coreos/kubernetes/fcc/worker.yaml @@ -89,7 +89,6 @@ systemd: storage: directories: - path: /etc/kubernetes - - path: /opt/bootkube files: - path: /etc/hostname mode: 0644 diff --git a/bare-metal/fedora-coreos/kubernetes/ssh.tf b/bare-metal/fedora-coreos/kubernetes/ssh.tf index da8329abc..0a10bf619 100644 --- a/bare-metal/fedora-coreos/kubernetes/ssh.tf +++ b/bare-metal/fedora-coreos/kubernetes/ssh.tf @@ -1,4 +1,4 @@ -# Secure copy etcd TLS assets and kubeconfig to controllers. Activates kubelet.service +# Secure copy assets to controllers. Activates kubelet.service resource "null_resource" "copy-controller-secrets" { count = length(var.controller_names) @@ -7,6 +7,7 @@ resource "null_resource" "copy-controller-secrets" { depends_on = [ matchbox_group.controller, matchbox_group.worker, + module.bootkube, ] connection { @@ -55,6 +56,11 @@ resource "null_resource" "copy-controller-secrets" { content = module.bootkube.etcd_peer_key destination = "$HOME/etcd-peer.key" } + + provisioner "file" { + source = var.asset_dir + destination = "$HOME/assets" + } provisioner "remote-exec" { inline = [ @@ -67,6 +73,11 @@ resource "null_resource" "copy-controller-secrets" { "sudo mv etcd-peer.crt /etc/ssl/etcd/etcd/peer.crt", "sudo mv etcd-peer.key /etc/ssl/etcd/etcd/peer.key", "sudo mv $HOME/kubeconfig /etc/kubernetes/kubeconfig", + "sudo mv $HOME/assets /opt/bootstrap/assets", + "sudo mkdir -p /etc/kubernetes/bootstrap-secrets", + "sudo cp -r /opt/bootstrap/assets/tls/* /etc/kubernetes/bootstrap-secrets/", + "sudo cp /opt/bootstrap/assets/auth/kubeconfig /etc/kubernetes/bootstrap-secrets/", + "sudo cp -r /opt/bootstrap/assets/static-manifests/* /etc/kubernetes/manifests/" ] } } @@ -101,9 +112,8 @@ resource "null_resource" "copy-worker-secrets" { } } -# Secure copy bootkube assets to ONE controller and start bootkube to perform -# one-time self-hosted cluster bootstrapping. -resource "null_resource" "bootkube-start" { +# Connect to a controller to perform one-time cluster bootstrap. +resource "null_resource" "bootstrap" { # Without depends_on, this remote-exec may start before the kubeconfig copy. # Terraform only does one task at a time, so it would try to bootstrap # while no Kubelets are running. @@ -119,15 +129,9 @@ resource "null_resource" "bootkube-start" { timeout = "15m" } - provisioner "file" { - source = var.asset_dir - destination = "$HOME/assets" - } - provisioner "remote-exec" { inline = [ - "sudo mv $HOME/assets /opt/bootkube", - "sudo systemctl start bootkube", + "sudo systemctl start bootstrap", ] } } diff --git a/docs/fedora-coreos/aws.md b/docs/fedora-coreos/aws.md index 67466c828..bff3f9b61 100644 --- a/docs/fedora-coreos/aws.md +++ b/docs/fedora-coreos/aws.md @@ -7,7 +7,7 @@ In this tutorial, we'll create a Kubernetes v1.15.3 cluster on AWS with Fedora C We'll declare a Kubernetes cluster using the Typhoon Terraform module. Then apply the changes to create a VPC, gateway, subnets, security groups, controller instances, worker auto-scaling group, network load balancer, and TLS assets. -Controllers hosts are provisioned to run an `etcd-member` peer and a `kubelet` service. Worker hosts run a `kubelet` service. Controller nodes run `kube-apiserver`, `kube-scheduler`, `kube-controller-manager`, and `coredns`, while `kube-proxy` and `calico` (or `flannel`) run on every node. A generated `kubeconfig` provides `kubectl` access to the cluster. +Controller hosts are provisioned to run an `etcd-member` peer and a `kubelet` service. Worker hosts run a `kubelet` service. Controller nodes run `kube-apiserver`, `kube-scheduler`, `kube-controller-manager`, and `coredns`, while `kube-proxy` and `calico` (or `flannel`) run on every node. A generated `kubeconfig` provides `kubectl` access to the cluster. ## Requirements @@ -153,7 +153,7 @@ kube-system calico-node-7jmr1 2/2 Running 0 kube-system calico-node-bknc8 2/2 Running 0 34m kube-system coredns-1187388186-wx1lg 1/1 Running 0 34m kube-system coredns-1187388186-qjnvp 1/1 Running 0 34m -kube-system kube-apiserver-4mjbk 1/1 Running 0 34m +kube-system kube-apiserver-ip-10-0-3-155 1/1 Running 0 34m kube-system kube-controller-manager-ip-10-0-3-155 1/1 Running 0 34m kube-system kube-proxy-14wxv 1/1 Running 0 34m kube-system kube-proxy-9vxh2 1/1 Running 0 34m diff --git a/docs/fedora-coreos/bare-metal.md b/docs/fedora-coreos/bare-metal.md index e49a4a56c..f5a517cdf 100644 --- a/docs/fedora-coreos/bare-metal.md +++ b/docs/fedora-coreos/bare-metal.md @@ -7,7 +7,7 @@ In this tutorial, we'll network boot and provision a Kubernetes v1.15.3 cluster First, we'll deploy a [Matchbox](https://github.com/poseidon/matchbox) service and setup a network boot environment. Then, we'll declare a Kubernetes cluster using the Typhoon Terraform module and power on machines. On PXE boot, machines will install Fedora CoreOS to disk, reboot into the disk install, and provision themselves as Kubernetes controllers or workers via Ignition. -Controllers are provisioned to run an `etcd-member` peer and a `kubelet` service. Workers run just a `kubelet` service. A one-time [bootkube](https://github.com/kubernetes-incubator/bootkube) bootstrap schedules the `apiserver`, `scheduler`, `controller-manager`, and `coredns` on controllers and schedules `kube-proxy` and `calico` (or `flannel`) on every node. A generated `kubeconfig` provides `kubectl` access to the cluster. +Controller hosts are provisioned to run an `etcd-member` peer and a `kubelet` service. Worker hosts run a `kubelet` service. Controller nodes run `kube-apiserver`, `kube-scheduler`, `kube-controller-manager`, and `coredns`, while `kube-proxy` and `calico` (or `flannel`) run on every node. A generated `kubeconfig` provides `kubectl` access to the cluster. ## Requirements @@ -200,7 +200,7 @@ Reference the [variables docs](#variables) or the [variables.tf](https://github. ## ssh-agent -Initial bootstrapping requires `bootkube.service` be started on one controller node. Terraform uses `ssh-agent` to automate this step. Add your SSH private key to `ssh-agent`. +Initial bootstrapping requires `bootstrap.service` be started on one controller node. Terraform uses `ssh-agent` to automate this step. Add your SSH private key to `ssh-agent`. ```sh ssh-add ~/.ssh/id_rsa @@ -222,7 +222,7 @@ $ terraform plan Plan: 55 to add, 0 to change, 0 to destroy. ``` -Apply the changes. Terraform will generate bootkube assets to `asset_dir` and create Matchbox profiles (e.g. controller, worker) and matching rules via the Matchbox API. +Apply the changes. Terraform will generate bootstrap assets to `asset_dir` and create Matchbox profiles (e.g. controller, worker) and matching rules via the Matchbox API. ```sh $ terraform apply @@ -251,14 +251,14 @@ Machines will network boot, install Fedora CoreOS to disk, reboot into the disk ### Bootstrap -Wait for the `bootkube-start` step to finish bootstrapping the Kubernetes control plane. This may take 5-15 minutes depending on your network. +Wait for the `bootstrap` step to finish bootstrapping the Kubernetes control plane. This may take 5-15 minutes depending on your network. ``` -module.bare-metal-mercury.null_resource.bootkube-start: Still creating... (6m10s elapsed) -module.bare-metal-mercury.null_resource.bootkube-start: Still creating... (6m20s elapsed) -module.bare-metal-mercury.null_resource.bootkube-start: Still creating... (6m30s elapsed) -module.bare-metal-mercury.null_resource.bootkube-start: Still creating... (6m40s elapsed) -module.bare-metal-mercury.null_resource.bootkube-start: Creation complete (ID: 5441741360626669024) +module.bare-metal-mercury.null_resource.bootstrap: Still creating... (6m10s elapsed) +module.bare-metal-mercury.null_resource.bootstrap: Still creating... (6m20s elapsed) +module.bare-metal-mercury.null_resource.bootstrap: Still creating... (6m30s elapsed) +module.bare-metal-mercury.null_resource.bootstrap: Still creating... (6m40s elapsed) +module.bare-metal-mercury.null_resource.bootstrap: Creation complete (ID: 5441741360626669024) Apply complete! Resources: 55 added, 0 changed, 0 destroyed. ``` @@ -267,13 +267,12 @@ To watch the bootstrap process in detail, SSH to the first controller and journa ``` $ ssh core@node1.example.com -$ journalctl -f -u bootkube -bootkube[5]: Pod Status: pod-checkpointer Running -bootkube[5]: Pod Status: kube-apiserver Running -bootkube[5]: Pod Status: kube-scheduler Running -bootkube[5]: Pod Status: kube-controller-manager Running -bootkube[5]: All self-hosted control plane components successfully started -bootkube[5]: Tearing down temporary bootstrap control plane... +$ journalctl -f -u bootstrap +podman[1750]: The connection to the server cluster.example.com:6443 was refused - did you specify the right host or port? +podman[1750]: Waiting for static pod control plane +... +podman[1750]: serviceaccount/calico-node unchanged +systemd[1]: Started Kubernetes control plane. ``` ## Verify @@ -299,16 +298,12 @@ kube-system calico-node-gnjrm 2/2 Running 0 kube-system calico-node-llbgt 2/2 Running 0 11m kube-system coredns-1187388186-dj3pd 1/1 Running 0 11m kube-system coredns-1187388186-mx9rt 1/1 Running 0 11m -kube-system kube-apiserver-7336w 1/1 Running 0 11m -kube-system kube-controller-manager-3271970485-b9chx 1/1 Running 0 11m -kube-system kube-controller-manager-3271970485-v30js 1/1 Running 1 11m +kube-system kube-apiserver-node1.example.com 1/1 Running 0 11m +kube-system kube-controller-manager-node1.example.com 1/1 Running 1 11m kube-system kube-proxy-50sd4 1/1 Running 0 11m kube-system kube-proxy-bczhp 1/1 Running 0 11m kube-system kube-proxy-mp2fw 1/1 Running 0 11m -kube-system kube-scheduler-3895335239-fd3l7 1/1 Running 1 11m -kube-system kube-scheduler-3895335239-hfjv0 1/1 Running 0 11m -kube-system pod-checkpointer-wf65d 1/1 Running 0 11m -kube-system pod-checkpointer-wf65d-node1.example.com 1/1 Running 0 11m +kube-system kube-scheduler-node1.example.com 1/1 Running 0 11m ``` ## Going Further From 21632c667443f0928d083d1a8a51f61b83f4dc4b Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Wed, 4 Sep 2019 22:05:29 -0700 Subject: [PATCH 209/523] Migrate Container Linux bare-metal to static pod control plane * Run a kube-apiserver, kube-scheduler, and kube-controller-manager static pod on each controller node. Previously, kube-apiserver was self-hosted as a DaemonSet across controllers and kube-scheduler and kube-controller-manager were a Deployment (with 2 or controller_count many replicas). * Remove bootkube bootstrap and pivot to self-hosted * Remove pod-checkpointer manifests (no longer needed) --- .../container-linux/kubernetes/README.md | 2 +- .../container-linux/kubernetes/bootkube.tf | 4 +- .../kubernetes/cl/controller.yaml.tmpl | 63 ++++++++++--------- bare-metal/container-linux/kubernetes/ssh.tf | 30 +++++---- docs/cl/bare-metal.md | 47 ++++++-------- 5 files changed, 73 insertions(+), 73 deletions(-) diff --git a/bare-metal/container-linux/kubernetes/README.md b/bare-metal/container-linux/kubernetes/README.md index e902faa81..4869b1595 100644 --- a/bare-metal/container-linux/kubernetes/README.md +++ b/bare-metal/container-linux/kubernetes/README.md @@ -11,7 +11,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features -* Kubernetes v1.15.3 (upstream, via [kubernetes-incubator/bootkube](https://github.com/kubernetes-incubator/bootkube)) +* Kubernetes v1.15.3 (upstream) * Single or multi-master, [Calico](https://www.projectcalico.org/) or [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/) * Advanced features like [snippets](https://typhoon.psdn.io/advanced/customization/#container-linux) customization diff --git a/bare-metal/container-linux/kubernetes/bootkube.tf b/bare-metal/container-linux/kubernetes/bootkube.tf index f19b57f3b..67efc0c7d 100644 --- a/bare-metal/container-linux/kubernetes/bootkube.tf +++ b/bare-metal/container-linux/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ -# Self-hosted Kubernetes assets (kubeconfig, manifests) +# Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=98cc19f80f2c4c3ddc63fc7aea6320e74bec561a" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=6e59af71138bc5f784453873074de16e7ee150eb" cluster_name = var.cluster_name api_servers = [var.k8s_domain_name] diff --git a/bare-metal/container-linux/kubernetes/cl/controller.yaml.tmpl b/bare-metal/container-linux/kubernetes/cl/controller.yaml.tmpl index 234c00f5a..e3aeeac82 100644 --- a/bare-metal/container-linux/kubernetes/cl/controller.yaml.tmpl +++ b/bare-metal/container-linux/kubernetes/cl/controller.yaml.tmpl @@ -111,17 +111,30 @@ systemd: RestartSec=10 [Install] WantedBy=multi-user.target - - name: bootkube.service + - name: bootstrap.service contents: | [Unit] - Description=Bootstrap a Kubernetes control plane with a temp api-server - ConditionPathExists=!/opt/bootkube/init_bootkube.done + Description=Kubernetes control plane + ConditionPathExists=!/opt/bootstrap/bootstrap.done [Service] Type=oneshot RemainAfterExit=true - WorkingDirectory=/opt/bootkube - ExecStart=/opt/bootkube/bootkube-start - ExecStartPost=/bin/touch /opt/bootkube/init_bootkube.done + WorkingDirectory=/opt/bootstrap + ExecStartPre=-/usr/bin/bash -c 'set -x && [ -n "$(ls /opt/bootstrap/assets/manifests-*/* 2>/dev/null)" ] && mv /opt/bootstrap/assets/manifests-*/* /opt/bootstrap/assets/manifests && rm -rf /opt/bootstrap/assets/manifests-*' + ExecStart=/usr/bin/rkt run \ + --trust-keys-from-https \ + --volume assets,kind=host,source=/opt/bootstrap/assets \ + --mount volume=assets,target=/assets \ + --volume script,kind=host,source=/opt/bootstrap/apply \ + --mount volume=script,target=/apply \ + --insecure-options=image \ + docker://k8s.gcr.io/hyperkube:v1.15.3 \ + --net=host \ + --dns=host \ + --exec=/apply + ExecStartPost=/bin/touch /opt/bootstrap/bootstrap.done + [Install] + WantedBy=multi-user.target storage: files: - path: /etc/kubernetes/kubelet.env @@ -137,36 +150,26 @@ storage: contents: inline: ${domain_name} - - path: /etc/sysctl.d/max-user-watches.conf + - path: /opt/bootstrap/apply filesystem: root + mode: 0544 contents: inline: | - fs.inotify.max_user_watches=16184 - - path: /opt/bootkube/bootkube-start + #!/bin/bash -e + export KUBECONFIG=/assets/auth/kubeconfig + until kubectl version; do + echo "Waiting for static pod control plane" + sleep 5 + done + until kubectl apply -f /assets/manifests -R; do + echo "Retry applying manifests" + sleep 5 + done + - path: /etc/sysctl.d/max-user-watches.conf filesystem: root - mode: 0544 - user: - id: 500 - group: - id: 500 contents: inline: | - #!/bin/bash - # Wrapper for bootkube start - set -e - # Move experimental manifests - [ -n "$(ls /opt/bootkube/assets/manifests-*/* 2>/dev/null)" ] && mv /opt/bootkube/assets/manifests-*/* /opt/bootkube/assets/manifests && rm -rf /opt/bootkube/assets/manifests-* - exec /usr/bin/rkt run \ - --trust-keys-from-https \ - --volume assets,kind=host,source=/opt/bootkube/assets \ - --mount volume=assets,target=/assets \ - --volume bootstrap,kind=host,source=/etc/kubernetes \ - --mount volume=bootstrap,target=/etc/kubernetes \ - $${RKT_OPTS} \ - quay.io/coreos/bootkube:v0.14.0 \ - --net=host \ - --dns=host \ - --exec=/bootkube -- start --asset-dir=/assets "$@" + fs.inotify.max_user_watches=16184 passwd: users: - name: core diff --git a/bare-metal/container-linux/kubernetes/ssh.tf b/bare-metal/container-linux/kubernetes/ssh.tf index 72f3c58fd..4a8f80ea6 100644 --- a/bare-metal/container-linux/kubernetes/ssh.tf +++ b/bare-metal/container-linux/kubernetes/ssh.tf @@ -1,4 +1,4 @@ -# Secure copy etcd TLS assets and kubeconfig to controllers. Activates kubelet.service +# Secure copy assets to controllers. Activates kubelet.service resource "null_resource" "copy-controller-secrets" { count = length(var.controller_names) @@ -8,11 +8,12 @@ resource "null_resource" "copy-controller-secrets" { matchbox_group.install, matchbox_group.controller, matchbox_group.worker, + module.bootkube, ] connection { type = "ssh" - host = element(var.controller_domains, count.index) + host = var.controller_domains[count.index] user = "core" timeout = "60m" } @@ -56,6 +57,11 @@ resource "null_resource" "copy-controller-secrets" { content = module.bootkube.etcd_peer_key destination = "$HOME/etcd-peer.key" } + + provisioner "file" { + source = var.asset_dir + destination = "$HOME/assets" + } provisioner "remote-exec" { inline = [ @@ -70,6 +76,11 @@ resource "null_resource" "copy-controller-secrets" { "sudo chown -R etcd:etcd /etc/ssl/etcd", "sudo chmod -R 500 /etc/ssl/etcd", "sudo mv $HOME/kubeconfig /etc/kubernetes/kubeconfig", + "sudo mv $HOME/assets /opt/bootstrap/assets", + "sudo mkdir -p /etc/kubernetes/bootstrap-secrets", + "sudo cp -r /opt/bootstrap/assets/tls/* /etc/kubernetes/bootstrap-secrets/", + "sudo cp /opt/bootstrap/assets/auth/kubeconfig /etc/kubernetes/bootstrap-secrets/", + "sudo cp -r /opt/bootstrap/assets/static-manifests/* /etc/kubernetes/manifests/", ] } } @@ -105,9 +116,8 @@ resource "null_resource" "copy-worker-secrets" { } } -# Secure copy bootkube assets to ONE controller and start bootkube to perform -# one-time self-hosted cluster bootstrapping. -resource "null_resource" "bootkube-start" { +# Connect to a controller to perform one-time cluster bootstrap. +resource "null_resource" "bootstrap" { # Without depends_on, this remote-exec may start before the kubeconfig copy. # Terraform only does one task at a time, so it would try to bootstrap # while no Kubelets are running. @@ -118,20 +128,14 @@ resource "null_resource" "bootkube-start" { connection { type = "ssh" - host = element(var.controller_domains, 0) + host = var.controller_domains[0] user = "core" timeout = "15m" } - provisioner "file" { - source = var.asset_dir - destination = "$HOME/assets" - } - provisioner "remote-exec" { inline = [ - "sudo mv $HOME/assets /opt/bootkube", - "sudo systemctl start bootkube", + "sudo systemctl start bootstrap", ] } } diff --git a/docs/cl/bare-metal.md b/docs/cl/bare-metal.md index 3e73d4c0a..c608608ee 100644 --- a/docs/cl/bare-metal.md +++ b/docs/cl/bare-metal.md @@ -4,7 +4,7 @@ In this tutorial, we'll network boot and provision a Kubernetes v1.15.3 cluster First, we'll deploy a [Matchbox](https://github.com/poseidon/matchbox) service and setup a network boot environment. Then, we'll declare a Kubernetes cluster using the Typhoon Terraform module and power on machines. On PXE boot, machines will install Container Linux to disk, reboot into the disk install, and provision themselves as Kubernetes controllers or workers via Ignition. -Controllers are provisioned to run an `etcd-member` peer and a `kubelet` service. Workers run just a `kubelet` service. A one-time [bootkube](https://github.com/kubernetes-incubator/bootkube) bootstrap schedules the `apiserver`, `scheduler`, `controller-manager`, and `coredns` on controllers and schedules `kube-proxy` and `calico` (or `flannel`) on every node. A generated `kubeconfig` provides `kubectl` access to the cluster. +Controller hosts are provisioned to run an `etcd-member` peer and a `kubelet` service. Worker hosts run a `kubelet` service. Controller nodes run `kube-apiserver`, `kube-scheduler`, `kube-controller-manager`, and `coredns` while `kube-proxy` and `calico` (or `flannel`) run on every node. A generated `kubeconfig` provides `kubectl` access to the cluster. ## Requirements @@ -199,7 +199,7 @@ Reference the [variables docs](#variables) or the [variables.tf](https://github. ## ssh-agent -Initial bootstrapping requires `bootkube.service` be started on one controller node. Terraform uses `ssh-agent` to automate this step. Add your SSH private key to `ssh-agent`. +Initial bootstrapping requires `bootstrap.service` be started on one controller node. Terraform uses `ssh-agent` to automate this step. Add your SSH private key to `ssh-agent`. ```sh ssh-add ~/.ssh/id_rsa @@ -221,14 +221,12 @@ $ terraform plan Plan: 55 to add, 0 to change, 0 to destroy. ``` -Apply the changes. Terraform will generate bootkube assets to `asset_dir` and create Matchbox profiles (e.g. controller, worker) and matching rules via the Matchbox API. +Apply the changes. Terraform will generate bootstrap assets to `asset_dir` and create Matchbox profiles (e.g. controller, worker) and matching rules via the Matchbox API. ```sh $ terraform apply -module.bare-metal-mercury.null_resource.copy-kubeconfig.0: Provisioning with 'file'... -module.bare-metal-mercury.null_resource.copy-etcd-secrets.0: Provisioning with 'file'... -module.bare-metal-mercury.null_resource.copy-kubeconfig.0: Still creating... (10s elapsed) -module.bare-metal-mercury.null_resource.copy-etcd-secrets.0: Still creating... (10s elapsed) +module.bare-metal-mercury.null_resource.copy-controller-secrets.0: Still creating... (10s elapsed) +module.bare-metal-mercury.null_resource.copy-worker-secrets.0: Still creating... (10s elapsed) ... ``` @@ -250,14 +248,14 @@ Machines will network boot, install Container Linux to disk, reboot into the dis ### Bootstrap -Wait for the `bootkube-start` step to finish bootstrapping the Kubernetes control plane. This may take 5-15 minutes depending on your network. +Wait for the `bootstrap` step to finish bootstrapping the Kubernetes control plane. This may take 5-15 minutes depending on your network. ``` -module.bare-metal-mercury.null_resource.bootkube-start: Still creating... (6m10s elapsed) -module.bare-metal-mercury.null_resource.bootkube-start: Still creating... (6m20s elapsed) -module.bare-metal-mercury.null_resource.bootkube-start: Still creating... (6m30s elapsed) -module.bare-metal-mercury.null_resource.bootkube-start: Still creating... (6m40s elapsed) -module.bare-metal-mercury.null_resource.bootkube-start: Creation complete (ID: 5441741360626669024) +module.bare-metal-mercury.null_resource.bootstrap: Still creating... (6m10s elapsed) +module.bare-metal-mercury.null_resource.bootstrap: Still creating... (6m20s elapsed) +module.bare-metal-mercury.null_resource.bootstrap: Still creating... (6m30s elapsed) +module.bare-metal-mercury.null_resource.bootstrap: Still creating... (6m40s elapsed) +module.bare-metal-mercury.null_resource.bootstrap: Creation complete (ID: 5441741360626669024) Apply complete! Resources: 55 added, 0 changed, 0 destroyed. ``` @@ -275,13 +273,12 @@ To watch the bootstrap process in detail, SSH to the first controller and journa ``` $ ssh core@node1.example.com -$ journalctl -f -u bootkube -bootkube[5]: Pod Status: pod-checkpointer Running -bootkube[5]: Pod Status: kube-apiserver Running -bootkube[5]: Pod Status: kube-scheduler Running -bootkube[5]: Pod Status: kube-controller-manager Running -bootkube[5]: All self-hosted control plane components successfully started -bootkube[5]: Tearing down temporary bootstrap control plane... +$ journalctl -f -u bootstrap +podman[1750]: The connection to the server cluster.example.com:6443 was refused - did you specify the right host or port? +podman[1750]: Waiting for static pod control plane +... +podman[1750]: serviceaccount/calico-node unchanged +systemd[1]: Started Kubernetes control plane. ``` ## Verify @@ -307,16 +304,12 @@ kube-system calico-node-gnjrm 2/2 Running 0 kube-system calico-node-llbgt 2/2 Running 0 11m kube-system coredns-1187388186-dj3pd 1/1 Running 0 11m kube-system coredns-1187388186-mx9rt 1/1 Running 0 11m -kube-system kube-apiserver-7336w 1/1 Running 0 11m -kube-system kube-controller-manager-3271970485-b9chx 1/1 Running 0 11m -kube-system kube-controller-manager-3271970485-v30js 1/1 Running 1 11m +kube-system kube-apiserver-node1.example.com 1/1 Running 0 11m +kube-system kube-controller-node1.example.com 1/1 Running 1 11m kube-system kube-proxy-50sd4 1/1 Running 0 11m kube-system kube-proxy-bczhp 1/1 Running 0 11m kube-system kube-proxy-mp2fw 1/1 Running 0 11m -kube-system kube-scheduler-3895335239-fd3l7 1/1 Running 1 11m -kube-system kube-scheduler-3895335239-hfjv0 1/1 Running 0 11m -kube-system pod-checkpointer-wf65d 1/1 Running 0 11m -kube-system pod-checkpointer-wf65d-node1.example.com 1/1 Running 0 11m +kube-system kube-scheduler-node1.example.com 1/1 Running 0 11m ``` ## Going Further From c933bdfc269b6f796940e05447a2df8a79aeed72 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Wed, 4 Sep 2019 22:20:36 -0700 Subject: [PATCH 210/523] Migrate Container Linux AWS to static pod control plane * Run a kube-apiserver, kube-scheduler, and kube-controller-manager static pod on each controller node. Previously, kube-apiserver was self-hosted as a DaemonSet across controllers and kube-scheduler and kube-controller-manager were a Deployment (with 2 or controller_count many replicas). * Remove bootkube bootstrap and pivot to self-hosted * Remove pod-checkpointer manifests (no longer needed) --- aws/container-linux/kubernetes/README.md | 2 +- aws/container-linux/kubernetes/bootkube.tf | 2 +- .../kubernetes/cl/controller.yaml.tmpl | 61 ++++++++++--------- aws/container-linux/kubernetes/security.tf | 22 +++++++ aws/container-linux/kubernetes/ssh.tf | 34 ++++++----- docs/cl/aws.md | 20 +++--- 6 files changed, 83 insertions(+), 58 deletions(-) diff --git a/aws/container-linux/kubernetes/README.md b/aws/container-linux/kubernetes/README.md index 220911a97..8dc9fd85a 100644 --- a/aws/container-linux/kubernetes/README.md +++ b/aws/container-linux/kubernetes/README.md @@ -11,7 +11,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features -* Kubernetes v1.15.3 (upstream, via [kubernetes-incubator/bootkube](https://github.com/kubernetes-incubator/bootkube)) +* Kubernetes v1.15.3 (upstream) * Single or multi-master, [Calico](https://www.projectcalico.org/) or [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/) * Advanced features like [worker pools](https://typhoon.psdn.io/advanced/worker-pools/), [spot](https://typhoon.psdn.io/cl/aws/#spot) workers, and [snippets](https://typhoon.psdn.io/advanced/customization/#container-linux) customization diff --git a/aws/container-linux/kubernetes/bootkube.tf b/aws/container-linux/kubernetes/bootkube.tf index f49cc8158..6db18ccf6 100644 --- a/aws/container-linux/kubernetes/bootkube.tf +++ b/aws/container-linux/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=98cc19f80f2c4c3ddc63fc7aea6320e74bec561a" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=6e59af71138bc5f784453873074de16e7ee150eb" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/aws/container-linux/kubernetes/cl/controller.yaml.tmpl b/aws/container-linux/kubernetes/cl/controller.yaml.tmpl index 6fdf12f3b..e1776eb36 100644 --- a/aws/container-linux/kubernetes/cl/controller.yaml.tmpl +++ b/aws/container-linux/kubernetes/cl/controller.yaml.tmpl @@ -98,17 +98,28 @@ systemd: RestartSec=10 [Install] WantedBy=multi-user.target - - name: bootkube.service + - name: bootstrap.service contents: | [Unit] - Description=Bootstrap a Kubernetes cluster - ConditionPathExists=!/opt/bootkube/init_bootkube.done + Description=Kubernetes control plane + ConditionPathExists=!/opt/bootstrap/bootstrap.done [Service] Type=oneshot RemainAfterExit=true - WorkingDirectory=/opt/bootkube - ExecStart=/opt/bootkube/bootkube-start - ExecStartPost=/bin/touch /opt/bootkube/init_bootkube.done + WorkingDirectory=/opt/bootstrap + ExecStartPre=-/usr/bin/bash -c 'set -x && [ -n "$(ls /opt/bootstrap/assets/manifests-*/* 2>/dev/null)" ] && mv /opt/bootstrap/assets/manifests-*/* /opt/bootstrap/assets/manifests && rm -rf /opt/bootstrap/assets/manifests-*' + ExecStart=/usr/bin/rkt run \ + --trust-keys-from-https \ + --volume assets,kind=host,source=/opt/bootstrap/assets \ + --mount volume=assets,target=/assets \ + --volume script,kind=host,source=/opt/bootstrap/apply \ + --mount volume=script,target=/apply \ + --insecure-options=image \ + docker://k8s.gcr.io/hyperkube:v1.15.3 \ + --net=host \ + --dns=host \ + --exec=/apply + ExecStartPost=/bin/touch /opt/bootstrap/bootstrap.done [Install] WantedBy=multi-user.target storage: @@ -126,36 +137,26 @@ storage: inline: | KUBELET_IMAGE_URL=docker://k8s.gcr.io/hyperkube KUBELET_IMAGE_TAG=v1.15.3 - - path: /etc/sysctl.d/max-user-watches.conf + - path: /opt/bootstrap/apply filesystem: root + mode: 0544 contents: inline: | - fs.inotify.max_user_watches=16184 - - path: /opt/bootkube/bootkube-start + #!/bin/bash -e + export KUBECONFIG=/assets/auth/kubeconfig + until kubectl version; do + echo "Waiting for static pod control plane" + sleep 5 + done + until kubectl apply -f /assets/manifests -R; do + echo "Retry applying manifests" + sleep 5 + done + - path: /etc/sysctl.d/max-user-watches.conf filesystem: root - mode: 0544 - user: - id: 500 - group: - id: 500 contents: inline: | - #!/bin/bash - # Wrapper for bootkube start - set -e - # Move experimental manifests - [ -n "$(ls /opt/bootkube/assets/manifests-*/* 2>/dev/null)" ] && mv /opt/bootkube/assets/manifests-*/* /opt/bootkube/assets/manifests && rm -rf /opt/bootkube/assets/manifests-* - exec /usr/bin/rkt run \ - --trust-keys-from-https \ - --volume assets,kind=host,source=/opt/bootkube/assets \ - --mount volume=assets,target=/assets \ - --volume bootstrap,kind=host,source=/etc/kubernetes \ - --mount volume=bootstrap,target=/etc/kubernetes \ - $${RKT_OPTS} \ - quay.io/coreos/bootkube:v0.14.0 \ - --net=host \ - --dns=host \ - --exec=/bootkube -- start --asset-dir=/assets "$@" + fs.inotify.max_user_watches=16184 passwd: users: - name: core diff --git a/aws/container-linux/kubernetes/security.tf b/aws/container-linux/kubernetes/security.tf index aa2f84cb8..706f80f87 100644 --- a/aws/container-linux/kubernetes/security.tf +++ b/aws/container-linux/kubernetes/security.tf @@ -33,6 +33,28 @@ resource "aws_security_group_rule" "controller-etcd" { self = true } +# Allow Prometheus to scrape kube-scheduler +resource "aws_security_group_rule" "controller-scheduler-metrics" { + security_group_id = aws_security_group.controller.id + + type = "ingress" + protocol = "tcp" + from_port = 10251 + to_port = 10251 + source_security_group_id = aws_security_group.worker.id +} + +# Allow Prometheus to scrape kube-controller-manager +resource "aws_security_group_rule" "controller-manager-metrics" { + security_group_id = aws_security_group.controller.id + + type = "ingress" + protocol = "tcp" + from_port = 10252 + to_port = 10252 + source_security_group_id = aws_security_group.worker.id +} + # Allow Prometheus to scrape etcd metrics resource "aws_security_group_rule" "controller-etcd-metrics" { security_group_id = aws_security_group.controller.id diff --git a/aws/container-linux/kubernetes/ssh.tf b/aws/container-linux/kubernetes/ssh.tf index 22b7a5d98..5f027d17f 100644 --- a/aws/container-linux/kubernetes/ssh.tf +++ b/aws/container-linux/kubernetes/ssh.tf @@ -1,10 +1,14 @@ -# Secure copy etcd TLS assets to controllers. +# Secure copy assets to controllers. resource "null_resource" "copy-controller-secrets" { count = var.controller_count + + depends_on = [ + module.bootkube, + ] connection { type = "ssh" - host = element(aws_instance.controllers.*.public_ip, count.index) + host = aws_instance.controllers.*.public_ip[count.index] user = "core" timeout = "15m" } @@ -43,6 +47,11 @@ resource "null_resource" "copy-controller-secrets" { content = module.bootkube.etcd_peer_key destination = "$HOME/etcd-peer.key" } + + provisioner "file" { + source = var.asset_dir + destination = "$HOME/assets" + } provisioner "remote-exec" { inline = [ @@ -56,18 +65,21 @@ resource "null_resource" "copy-controller-secrets" { "sudo mv etcd-peer.key /etc/ssl/etcd/etcd/peer.key", "sudo chown -R etcd:etcd /etc/ssl/etcd", "sudo chmod -R 500 /etc/ssl/etcd", + "sudo mv $HOME/assets /opt/bootstrap/assets", + "sudo mkdir -p /etc/kubernetes/bootstrap-secrets", + "sudo cp -r /opt/bootstrap/assets/tls/* /etc/kubernetes/bootstrap-secrets/", + "sudo cp /opt/bootstrap/assets/auth/kubeconfig /etc/kubernetes/bootstrap-secrets/", + "sudo cp -r /opt/bootstrap/assets/static-manifests/* /etc/kubernetes/manifests/", ] } } -# Secure copy bootkube assets to ONE controller and start bootkube to perform -# one-time self-hosted cluster bootstrapping. -resource "null_resource" "bootkube-start" { +# Connect to a controller to perform one-time cluster bootstrap. +resource "null_resource" "bootstrap" { depends_on = [ - module.bootkube, + null_resource.copy-controller-secrets, module.workers, aws_route53_record.apiserver, - null_resource.copy-controller-secrets, ] connection { @@ -77,15 +89,9 @@ resource "null_resource" "bootkube-start" { timeout = "15m" } - provisioner "file" { - source = var.asset_dir - destination = "$HOME/assets" - } - provisioner "remote-exec" { inline = [ - "sudo mv $HOME/assets /opt/bootkube", - "sudo systemctl start bootkube", + "sudo systemctl start bootstrap", ] } } diff --git a/docs/cl/aws.md b/docs/cl/aws.md index 2b447faf1..90f4f579e 100644 --- a/docs/cl/aws.md +++ b/docs/cl/aws.md @@ -4,7 +4,7 @@ In this tutorial, we'll create a Kubernetes v1.15.3 cluster on AWS with Containe We'll declare a Kubernetes cluster using the Typhoon Terraform module. Then apply the changes to create a VPC, gateway, subnets, security groups, controller instances, worker auto-scaling group, network load balancer, and TLS assets. -Controllers are provisioned to run an `etcd-member` peer and a `kubelet` service. Workers run just a `kubelet` service. A one-time [bootkube](https://github.com/kubernetes-incubator/bootkube) bootstrap schedules the `apiserver`, `scheduler`, `controller-manager`, and `coredns` on controllers and schedules `kube-proxy` and `calico` (or `flannel`) on every node. A generated `kubeconfig` provides `kubectl` access to the cluster. +Controller hosts are provisioned to run an `etcd-member` peer and a `kubelet` service. Worker hosts run a `kubelet` service. Controller nodes run `kube-apiserver`, `kube-scheduler`, `kube-controller-manager`, and `coredns`, while `kube-proxy` and `calico` (or `flannel`) run on every node. A generated `kubeconfig` provides `kubectl` access to the cluster. ## Requirements @@ -91,7 +91,7 @@ Reference the [variables docs](#variables) or the [variables.tf](https://github. ## ssh-agent -Initial bootstrapping requires `bootkube.service` be started on one controller node. Terraform uses `ssh-agent` to automate this step. Add your SSH private key to `ssh-agent`. +Initial bootstrapping requires `bootstrap.service` be started on one controller node. Terraform uses `ssh-agent` to automate this step. Add your SSH private key to `ssh-agent`. ```sh ssh-add ~/.ssh/id_rsa @@ -118,9 +118,9 @@ Apply the changes to create the cluster. ```sh $ terraform apply ... -module.aws-tempest.null_resource.bootkube-start: Still creating... (4m50s elapsed) -module.aws-tempest.null_resource.bootkube-start: Still creating... (5m0s elapsed) -module.aws-tempest.null_resource.bootkube-start: Creation complete after 11m8s (ID: 3961816482286168143) +module.aws-tempest.null_resource.bootstrap: Still creating... (4m50s elapsed) +module.aws-tempest.null_resource.bootstrap: Still creating... (5m0s elapsed) +module.aws-tempest.null_resource.bootstrap: Creation complete after 11m8s (ID: 3961816482286168143) Apply complete! Resources: 98 added, 0 changed, 0 destroyed. ``` @@ -150,16 +150,12 @@ kube-system calico-node-7jmr1 2/2 Running 0 kube-system calico-node-bknc8 2/2 Running 0 34m kube-system coredns-1187388186-wx1lg 1/1 Running 0 34m kube-system coredns-1187388186-qjnvp 1/1 Running 0 34m -kube-system kube-apiserver-4mjbk 1/1 Running 0 34m -kube-system kube-controller-manager-3597210155-j2jbt 1/1 Running 1 34m -kube-system kube-controller-manager-3597210155-j7g7x 1/1 Running 0 34m +kube-system kube-apiserver-ip-10-0-3-155 1/1 Running 0 34m +kube-system kube-controller-manager-ip-10-0-3-155 1/1 Running 0 34m kube-system kube-proxy-14wxv 1/1 Running 0 34m kube-system kube-proxy-9vxh2 1/1 Running 0 34m kube-system kube-proxy-sbbsh 1/1 Running 0 34m -kube-system kube-scheduler-3359497473-5plhf 1/1 Running 0 34m -kube-system kube-scheduler-3359497473-r7zg7 1/1 Running 1 34m -kube-system pod-checkpointer-4kxtl 1/1 Running 0 34m -kube-system pod-checkpointer-4kxtl-ip-10-0-3-155 1/1 Running 0 33m +kube-system kube-scheduler-ip-10-0-3-155 1/1 Running 1 34m ``` ## Going Further From db947537d1bb6d2f65de9b61115d519f55aa1c7b Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Thu, 5 Sep 2019 23:12:09 -0700 Subject: [PATCH 211/523] Migrate GCP, DO, Azure to static pod control plane * Run a kube-apiserver, kube-scheduler, and kube-controller-manager static pod on each controller node. Previously, kube-apiserver was self-hosted as a DaemonSet across controllers and kube-scheduler and kube-controller-manager were a Deployment (with 2 or controller_count many replicas). * Remove bootkube bootstrap and pivot to self-hosted * Remove pod-checkpointer manifests (no longer needed) --- README.md | 10 +-- azure/container-linux/kubernetes/README.md | 2 +- azure/container-linux/kubernetes/bootkube.tf | 2 +- .../kubernetes/cl/controller.yaml.tmpl | 61 ++++++++++--------- azure/container-linux/kubernetes/security.tf | 16 +++++ azure/container-linux/kubernetes/ssh.tf | 35 ++++++----- bare-metal/container-linux/kubernetes/ssh.tf | 2 +- .../container-linux/kubernetes/README.md | 2 +- .../container-linux/kubernetes/bootkube.tf | 2 +- .../kubernetes/cl/controller.yaml.tmpl | 61 ++++++++++--------- .../container-linux/kubernetes/network.tf | 11 +++- .../container-linux/kubernetes/ssh.tf | 31 +++++----- docs/cl/azure.md | 22 +++---- docs/cl/digital-ocean.md | 24 +++----- docs/cl/google-cloud.md | 22 +++---- docs/index.md | 10 +-- .../container-linux/kubernetes/README.md | 2 +- .../container-linux/kubernetes/bootkube.tf | 2 +- .../kubernetes/cl/controller.yaml.tmpl | 61 ++++++++++--------- .../container-linux/kubernetes/network.tf | 14 +++++ .../container-linux/kubernetes/ssh.tf | 36 ++++++----- 21 files changed, 232 insertions(+), 196 deletions(-) diff --git a/README.md b/README.md index 2421a808e..06157cf02 100644 --- a/README.md +++ b/README.md @@ -97,16 +97,12 @@ kube-system calico-node-d1l5b 2/2 Running 0 kube-system calico-node-sp9ps 2/2 Running 0 6m kube-system coredns-1187388186-zj5dl 1/1 Running 0 6m kube-system coredns-1187388186-dkh3o 1/1 Running 0 6m -kube-system kube-apiserver-zppls 1/1 Running 0 6m -kube-system kube-controller-manager-3271970485-gh9kt 1/1 Running 0 6m -kube-system kube-controller-manager-3271970485-h90v8 1/1 Running 1 6m +kube-system kube-apiserver-controller-0 1/1 Running 0 6m +kube-system kube-controller-manager-controller-0 1/1 Running 0 6m kube-system kube-proxy-117v6 1/1 Running 0 6m kube-system kube-proxy-9886n 1/1 Running 0 6m kube-system kube-proxy-njn47 1/1 Running 0 6m -kube-system kube-scheduler-3895335239-5x87r 1/1 Running 0 6m -kube-system kube-scheduler-3895335239-bzrrt 1/1 Running 1 6m -kube-system pod-checkpointer-l6lrt 1/1 Running 0 6m -kube-system pod-checkpointer-l6lrt-controller-0 1/1 Running 0 6m +kube-system kube-scheduler-controller-0 1/1 Running 0 6m ``` ## Non-Goals diff --git a/azure/container-linux/kubernetes/README.md b/azure/container-linux/kubernetes/README.md index ea10af3b0..56d2f9e5c 100644 --- a/azure/container-linux/kubernetes/README.md +++ b/azure/container-linux/kubernetes/README.md @@ -11,7 +11,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features -* Kubernetes v1.15.3 (upstream, via [kubernetes-incubator/bootkube](https://github.com/kubernetes-incubator/bootkube)) +* Kubernetes v1.15.3 (upstream) * Single or multi-master, [Calico](https://www.projectcalico.org/) or [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/) * Advanced features like [worker pools](https://typhoon.psdn.io/advanced/worker-pools/), [low-priority](https://typhoon.psdn.io/cl/azure/#low-priority) workers, and [snippets](https://typhoon.psdn.io/advanced/customization/#container-linux) customization diff --git a/azure/container-linux/kubernetes/bootkube.tf b/azure/container-linux/kubernetes/bootkube.tf index f1d7d8e41..8cefa6b2c 100644 --- a/azure/container-linux/kubernetes/bootkube.tf +++ b/azure/container-linux/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=98cc19f80f2c4c3ddc63fc7aea6320e74bec561a" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=6e59af71138bc5f784453873074de16e7ee150eb" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/azure/container-linux/kubernetes/cl/controller.yaml.tmpl b/azure/container-linux/kubernetes/cl/controller.yaml.tmpl index 1d0981b9b..362189a0e 100644 --- a/azure/container-linux/kubernetes/cl/controller.yaml.tmpl +++ b/azure/container-linux/kubernetes/cl/controller.yaml.tmpl @@ -96,17 +96,28 @@ systemd: RestartSec=10 [Install] WantedBy=multi-user.target - - name: bootkube.service + - name: bootstrap.service contents: | [Unit] - Description=Bootstrap a Kubernetes cluster - ConditionPathExists=!/opt/bootkube/init_bootkube.done + Description=Kubernetes control plane + ConditionPathExists=!/opt/bootstrap/bootstrap.done [Service] Type=oneshot RemainAfterExit=true - WorkingDirectory=/opt/bootkube - ExecStart=/opt/bootkube/bootkube-start - ExecStartPost=/bin/touch /opt/bootkube/init_bootkube.done + WorkingDirectory=/opt/bootstrap + ExecStartPre=-/usr/bin/bash -c 'set -x && [ -n "$(ls /opt/bootstrap/assets/manifests-*/* 2>/dev/null)" ] && mv /opt/bootstrap/assets/manifests-*/* /opt/bootstrap/assets/manifests && rm -rf /opt/bootstrap/assets/manifests-*' + ExecStart=/usr/bin/rkt run \ + --trust-keys-from-https \ + --volume assets,kind=host,source=/opt/bootstrap/assets \ + --mount volume=assets,target=/assets \ + --volume script,kind=host,source=/opt/bootstrap/apply \ + --mount volume=script,target=/apply \ + --insecure-options=image \ + docker://k8s.gcr.io/hyperkube:v1.15.3 \ + --net=host \ + --dns=host \ + --exec=/apply + ExecStartPost=/bin/touch /opt/bootstrap/bootstrap.done [Install] WantedBy=multi-user.target storage: @@ -124,36 +135,26 @@ storage: inline: | KUBELET_IMAGE_URL=docker://k8s.gcr.io/hyperkube KUBELET_IMAGE_TAG=v1.15.3 - - path: /etc/sysctl.d/max-user-watches.conf + - path: /opt/bootstrap/apply filesystem: root + mode: 0544 contents: inline: | - fs.inotify.max_user_watches=16184 - - path: /opt/bootkube/bootkube-start + #!/bin/bash -e + export KUBECONFIG=/assets/auth/kubeconfig + until kubectl version; do + echo "Waiting for static pod control plane" + sleep 5 + done + until kubectl apply -f /assets/manifests -R; do + echo "Retry applying manifests" + sleep 5 + done + - path: /etc/sysctl.d/max-user-watches.conf filesystem: root - mode: 0544 - user: - id: 500 - group: - id: 500 contents: inline: | - #!/bin/bash - # Wrapper for bootkube start - set -e - # Move experimental manifests - [ -n "$(ls /opt/bootkube/assets/manifests-*/* 2>/dev/null)" ] && mv /opt/bootkube/assets/manifests-*/* /opt/bootkube/assets/manifests && rm -rf /opt/bootkube/assets/manifests-* - exec /usr/bin/rkt run \ - --trust-keys-from-https \ - --volume assets,kind=host,source=/opt/bootkube/assets \ - --mount volume=assets,target=/assets \ - --volume bootstrap,kind=host,source=/etc/kubernetes \ - --mount volume=bootstrap,target=/etc/kubernetes \ - $${RKT_OPTS} \ - quay.io/coreos/bootkube:v0.14.0 \ - --net=host \ - --dns=host \ - --exec=/bootkube -- start --asset-dir=/assets "$@" + fs.inotify.max_user_watches=16184 passwd: users: - name: core diff --git a/azure/container-linux/kubernetes/security.tf b/azure/container-linux/kubernetes/security.tf index b9fd1c65f..c2e97307e 100644 --- a/azure/container-linux/kubernetes/security.tf +++ b/azure/container-linux/kubernetes/security.tf @@ -53,6 +53,22 @@ resource "azurerm_network_security_rule" "controller-etcd-metrics" { destination_address_prefix = azurerm_subnet.controller.address_prefix } +# Allow Prometheus to scrape kube-scheduler and kube-controller-manager metrics +resource "azurerm_network_security_rule" "controller-kube-metrics" { + resource_group_name = azurerm_resource_group.cluster.name + + name = "allow-kube-metrics" + network_security_group_name = azurerm_network_security_group.controller.name + priority = "2011" + access = "Allow" + direction = "Inbound" + protocol = "Tcp" + source_port_range = "*" + destination_port_range = "10251-10252" + source_address_prefix = azurerm_subnet.worker.address_prefix + destination_address_prefix = azurerm_subnet.controller.address_prefix +} + resource "azurerm_network_security_rule" "controller-apiserver" { resource_group_name = azurerm_resource_group.cluster.name diff --git a/azure/container-linux/kubernetes/ssh.tf b/azure/container-linux/kubernetes/ssh.tf index 85a5d7b46..961cb1dfd 100644 --- a/azure/container-linux/kubernetes/ssh.tf +++ b/azure/container-linux/kubernetes/ssh.tf @@ -1,12 +1,15 @@ -# Secure copy etcd TLS assets to controllers. +# Secure copy assets to controllers. resource "null_resource" "copy-controller-secrets" { count = var.controller_count - depends_on = [azurerm_virtual_machine.controllers] + depends_on = [ + module.bootkube, + azurerm_virtual_machine.controllers + ] connection { type = "ssh" - host = element(azurerm_public_ip.controllers.*.ip_address, count.index) + host = azurerm_public_ip.controllers.*.ip_address[count.index] user = "core" timeout = "15m" } @@ -45,6 +48,11 @@ resource "null_resource" "copy-controller-secrets" { content = module.bootkube.etcd_peer_key destination = "$HOME/etcd-peer.key" } + + provisioner "file" { + source = var.asset_dir + destination = "$HOME/assets" + } provisioner "remote-exec" { inline = [ @@ -58,18 +66,21 @@ resource "null_resource" "copy-controller-secrets" { "sudo mv etcd-peer.key /etc/ssl/etcd/etcd/peer.key", "sudo chown -R etcd:etcd /etc/ssl/etcd", "sudo chmod -R 500 /etc/ssl/etcd", + "sudo mv $HOME/assets /opt/bootstrap/assets", + "sudo mkdir -p /etc/kubernetes/bootstrap-secrets", + "sudo cp -r /opt/bootstrap/assets/tls/* /etc/kubernetes/bootstrap-secrets/", + "sudo cp /opt/bootstrap/assets/auth/kubeconfig /etc/kubernetes/bootstrap-secrets/", + "sudo cp -r /opt/bootstrap/assets/static-manifests/* /etc/kubernetes/manifests/", ] } } -# Secure copy bootkube assets to ONE controller and start bootkube to perform -# one-time self-hosted cluster bootstrapping. -resource "null_resource" "bootkube-start" { +# Connect to a controller to perform one-time cluster bootstrap. +resource "null_resource" "bootstrap" { depends_on = [ - module.bootkube, + null_resource.copy-controller-secrets, module.workers, azurerm_dns_a_record.apiserver, - null_resource.copy-controller-secrets, ] connection { @@ -79,15 +90,9 @@ resource "null_resource" "bootkube-start" { timeout = "15m" } - provisioner "file" { - source = var.asset_dir - destination = "$HOME/assets" - } - provisioner "remote-exec" { inline = [ - "sudo mv $HOME/assets /opt/bootkube", - "sudo systemctl start bootkube", + "sudo systemctl start bootstrap", ] } } diff --git a/bare-metal/container-linux/kubernetes/ssh.tf b/bare-metal/container-linux/kubernetes/ssh.tf index 4a8f80ea6..30e0be7a9 100644 --- a/bare-metal/container-linux/kubernetes/ssh.tf +++ b/bare-metal/container-linux/kubernetes/ssh.tf @@ -99,7 +99,7 @@ resource "null_resource" "copy-worker-secrets" { connection { type = "ssh" - host = element(var.worker_domains, count.index) + host = var.worker_domains[count.index] user = "core" timeout = "60m" } diff --git a/digital-ocean/container-linux/kubernetes/README.md b/digital-ocean/container-linux/kubernetes/README.md index 08b6373c3..0d5146cbc 100644 --- a/digital-ocean/container-linux/kubernetes/README.md +++ b/digital-ocean/container-linux/kubernetes/README.md @@ -11,7 +11,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features -* Kubernetes v1.15.3 (upstream, via [kubernetes-incubator/bootkube](https://github.com/kubernetes-incubator/bootkube)) +* Kubernetes v1.15.3 (upstream) * Single or multi-master, [Calico](https://www.projectcalico.org/) or [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/) * Advanced features like [snippets](https://typhoon.psdn.io/advanced/customization/#container-linux) customization diff --git a/digital-ocean/container-linux/kubernetes/bootkube.tf b/digital-ocean/container-linux/kubernetes/bootkube.tf index 4c4fa1906..e279fa9d8 100644 --- a/digital-ocean/container-linux/kubernetes/bootkube.tf +++ b/digital-ocean/container-linux/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=98cc19f80f2c4c3ddc63fc7aea6320e74bec561a" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=6e59af71138bc5f784453873074de16e7ee150eb" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/digital-ocean/container-linux/kubernetes/cl/controller.yaml.tmpl b/digital-ocean/container-linux/kubernetes/cl/controller.yaml.tmpl index 653a9899e..066780600 100644 --- a/digital-ocean/container-linux/kubernetes/cl/controller.yaml.tmpl +++ b/digital-ocean/container-linux/kubernetes/cl/controller.yaml.tmpl @@ -108,17 +108,28 @@ systemd: RestartSec=10 [Install] WantedBy=multi-user.target - - name: bootkube.service + - name: bootstrap.service contents: | [Unit] - Description=Bootstrap a Kubernetes cluster - ConditionPathExists=!/opt/bootkube/init_bootkube.done + Description=Kubernetes control plane + ConditionPathExists=!/opt/bootstrap/bootstrap.done [Service] Type=oneshot RemainAfterExit=true - WorkingDirectory=/opt/bootkube - ExecStart=/opt/bootkube/bootkube-start - ExecStartPost=/bin/touch /opt/bootkube/init_bootkube.done + WorkingDirectory=/opt/bootstrap + ExecStartPre=-/usr/bin/bash -c 'set -x && [ -n "$(ls /opt/bootstrap/assets/manifests-*/* 2>/dev/null)" ] && mv /opt/bootstrap/assets/manifests-*/* /opt/bootstrap/assets/manifests && rm -rf /opt/bootstrap/assets/manifests-*' + ExecStart=/usr/bin/rkt run \ + --trust-keys-from-https \ + --volume assets,kind=host,source=/opt/bootstrap/assets \ + --mount volume=assets,target=/assets \ + --volume script,kind=host,source=/opt/bootstrap/apply \ + --mount volume=script,target=/apply \ + --insecure-options=image \ + docker://k8s.gcr.io/hyperkube:v1.15.3 \ + --net=host \ + --dns=host \ + --exec=/apply + ExecStartPost=/bin/touch /opt/bootstrap/bootstrap.done [Install] WantedBy=multi-user.target storage: @@ -130,33 +141,23 @@ storage: inline: | KUBELET_IMAGE_URL=docker://k8s.gcr.io/hyperkube KUBELET_IMAGE_TAG=v1.15.3 - - path: /etc/sysctl.d/max-user-watches.conf + - path: /opt/bootstrap/apply filesystem: root + mode: 0544 contents: inline: | - fs.inotify.max_user_watches=16184 - - path: /opt/bootkube/bootkube-start + #!/bin/bash -e + export KUBECONFIG=/assets/auth/kubeconfig + until kubectl version; do + echo "Waiting for static pod control plane" + sleep 5 + done + until kubectl apply -f /assets/manifests -R; do + echo "Retry applying manifests" + sleep 5 + done + - path: /etc/sysctl.d/max-user-watches.conf filesystem: root - mode: 0544 - user: - id: 500 - group: - id: 500 contents: inline: | - #!/bin/bash - # Wrapper for bootkube start - set -e - # Move experimental manifests - [ -n "$(ls /opt/bootkube/assets/manifests-*/* 2>/dev/null)" ] && mv /opt/bootkube/assets/manifests-*/* /opt/bootkube/assets/manifests && rm -rf /opt/bootkube/assets/manifests-* - exec /usr/bin/rkt run \ - --trust-keys-from-https \ - --volume assets,kind=host,source=/opt/bootkube/assets \ - --mount volume=assets,target=/assets \ - --volume bootstrap,kind=host,source=/etc/kubernetes \ - --mount volume=bootstrap,target=/etc/kubernetes \ - $${RKT_OPTS} \ - quay.io/coreos/bootkube:v0.14.0 \ - --net=host \ - --dns=host \ - --exec=/bootkube -- start --asset-dir=/assets "$@" + fs.inotify.max_user_watches=16184 diff --git a/digital-ocean/container-linux/kubernetes/network.tf b/digital-ocean/container-linux/kubernetes/network.tf index b145cdf30..700a929b8 100644 --- a/digital-ocean/container-linux/kubernetes/network.tf +++ b/digital-ocean/container-linux/kubernetes/network.tf @@ -53,24 +53,33 @@ resource "digitalocean_firewall" "controllers" { tags = ["${var.cluster_name}-controller"] - # etcd, kube-apiserver, kubelet + # etcd inbound_rule { protocol = "tcp" port_range = "2379-2380" source_tags = [digitalocean_tag.controllers.name] } + # etcd metrics inbound_rule { protocol = "tcp" port_range = "2381" source_tags = [digitalocean_tag.workers.name] } + # kube-apiserver inbound_rule { protocol = "tcp" port_range = "6443" source_addresses = ["0.0.0.0/0", "::/0"] } + + # kube-scheduler metrics, kube-controller-manager metrics + inbound_rule { + protocol = "tcp" + port_range = "10251-10252" + source_tags = [digitalocean_tag.workers.name] + } } resource "digitalocean_firewall" "workers" { diff --git a/digital-ocean/container-linux/kubernetes/ssh.tf b/digital-ocean/container-linux/kubernetes/ssh.tf index cc9385d27..1d334ccb2 100644 --- a/digital-ocean/container-linux/kubernetes/ssh.tf +++ b/digital-ocean/container-linux/kubernetes/ssh.tf @@ -1,14 +1,15 @@ -# Secure copy etcd TLS assets and kubeconfig to controllers. Activates kubelet.service +# Secure copy assets to controllers. Activates kubelet.service resource "null_resource" "copy-controller-secrets" { count = var.controller_count depends_on = [ + module.bootkube, digitalocean_firewall.rules ] connection { type = "ssh" - host = element(digitalocean_droplet.controllers.*.ipv4_address, count.index) + host = digitalocean_droplet.controllers.*.ipv4_address[count.index] user = "core" timeout = "15m" } @@ -52,6 +53,11 @@ resource "null_resource" "copy-controller-secrets" { content = module.bootkube.etcd_peer_key destination = "$HOME/etcd-peer.key" } + + provisioner "file" { + source = var.asset_dir + destination = "$HOME/assets" + } provisioner "remote-exec" { inline = [ @@ -66,6 +72,11 @@ resource "null_resource" "copy-controller-secrets" { "sudo chown -R etcd:etcd /etc/ssl/etcd", "sudo chmod -R 500 /etc/ssl/etcd", "sudo mv $HOME/kubeconfig /etc/kubernetes/kubeconfig", + "sudo mv $HOME/assets /opt/bootstrap/assets", + "sudo mkdir -p /etc/kubernetes/bootstrap-secrets", + "sudo cp -r /opt/bootstrap/assets/tls/* /etc/kubernetes/bootstrap-secrets/", + "sudo cp /opt/bootstrap/assets/auth/kubeconfig /etc/kubernetes/bootstrap-secrets/", + "sudo cp -r /opt/bootstrap/assets/static-manifests/* /etc/kubernetes/manifests/", ] } } @@ -76,7 +87,7 @@ resource "null_resource" "copy-worker-secrets" { connection { type = "ssh" - host = element(digitalocean_droplet.workers.*.ipv4_address, count.index) + host = digitalocean_droplet.workers.*.ipv4_address[count.index] user = "core" timeout = "15m" } @@ -93,11 +104,9 @@ resource "null_resource" "copy-worker-secrets" { } } -# Secure copy bootkube assets to ONE controller and start bootkube to perform -# one-time self-hosted cluster bootstrapping. -resource "null_resource" "bootkube-start" { +# Connect to a controller to perform one-time cluster bootstrap. +resource "null_resource" "bootstrap" { depends_on = [ - module.bootkube, null_resource.copy-controller-secrets, null_resource.copy-worker-secrets, ] @@ -109,15 +118,9 @@ resource "null_resource" "bootkube-start" { timeout = "15m" } - provisioner "file" { - source = var.asset_dir - destination = "$HOME/assets" - } - provisioner "remote-exec" { inline = [ - "sudo mv $HOME/assets /opt/bootkube", - "sudo systemctl start bootkube", + "sudo systemctl start bootstrap", ] } } diff --git a/docs/cl/azure.md b/docs/cl/azure.md index 19ff805e5..a02611843 100644 --- a/docs/cl/azure.md +++ b/docs/cl/azure.md @@ -7,7 +7,7 @@ In this tutorial, we'll create a Kubernetes v1.15.3 cluster on Azure with Contai We'll declare a Kubernetes cluster using the Typhoon Terraform module. Then apply the changes to create a resource group, virtual network, subnets, security groups, controller availability set, worker scale set, load balancer, and TLS assets. -Controllers are provisioned to run an `etcd-member` peer and a `kubelet` service. Workers run just a `kubelet` service. A one-time [bootkube](https://github.com/kubernetes-incubator/bootkube) bootstrap schedules the `apiserver`, `scheduler`, `controller-manager`, and `coredns` on controllers and schedules `kube-proxy` and `flannel` on every node. A generated `kubeconfig` provides `kubectl` access to the cluster. +Controller hosts are provisioned to run an `etcd-member` peer and a `kubelet` service. Worker hosts run a `kubelet` service. Controller nodes run `kube-apiserver`, `kube-scheduler`, `kube-controller-manager`, and `coredns`, while `kube-proxy` and `calico` (or `flannel`) run on every node. A generated `kubeconfig` provides `kubectl` access to the cluster. ## Requirements @@ -88,7 +88,7 @@ Reference the [variables docs](#variables) or the [variables.tf](https://github. ## ssh-agent -Initial bootstrapping requires `bootkube.service` be started on one controller node. Terraform uses `ssh-agent` to automate this step. Add your SSH private key to `ssh-agent`. +Initial bootstrapping requires `bootstrap.service` be started on one controller node. Terraform uses `ssh-agent` to automate this step. Add your SSH private key to `ssh-agent`. ```sh ssh-add ~/.ssh/id_rsa @@ -115,9 +115,9 @@ Apply the changes to create the cluster. ```sh $ terraform apply ... -module.azure-ramius.null_resource.bootkube-start: Still creating... (6m50s elapsed) -module.azure-ramius.null_resource.bootkube-start: Still creating... (7m0s elapsed) -module.azure-ramius.null_resource.bootkube-start: Creation complete after 7m8s (ID: 3961816482286168143) +module.azure-ramius.null_resource.bootstrap: Still creating... (6m50s elapsed) +module.azure-ramius.null_resource.bootstrap: Still creating... (7m0s elapsed) +module.azure-ramius.null_resource.bootstrap: Creation complete after 7m8s (ID: 3961816482286168143) Apply complete! Resources: 86 added, 0 changed, 0 destroyed. ``` @@ -144,19 +144,15 @@ $ kubectl get pods --all-namespaces NAMESPACE NAME READY STATUS RESTARTS AGE kube-system coredns-7c6fbb4f4b-b6qzx 1/1 Running 0 26m kube-system coredns-7c6fbb4f4b-j2k3d 1/1 Running 0 26m -kube-system flannel-bwf24 2/2 Running 2 26m +kube-system flannel-bwf24 2/2 Running 0 26m kube-system flannel-ks5qb 2/2 Running 0 26m kube-system flannel-tq2wg 2/2 Running 0 26m -kube-system kube-apiserver-hxgsx 1/1 Running 3 26m -kube-system kube-controller-manager-5ff9cd7bb6-b942n 1/1 Running 0 26m -kube-system kube-controller-manager-5ff9cd7bb6-bbr6w 1/1 Running 0 26m +kube-system kube-apiserver-ramius-controller-0 1/1 Running 0 26m +kube-system kube-controller-manager-ramius-controller-0 1/1 Running 0 26m kube-system kube-proxy-j4vpq 1/1 Running 0 26m kube-system kube-proxy-jxr5d 1/1 Running 0 26m kube-system kube-proxy-lbdw5 1/1 Running 0 26m -kube-system kube-scheduler-5f76d69686-s4fbx 1/1 Running 0 26m -kube-system kube-scheduler-5f76d69686-vgdgn 1/1 Running 0 26m -kube-system pod-checkpointer-cnqdg 1/1 Running 0 26m -kube-system pod-checkpointer-cnqdg-ramius-controller-0 1/1 Running 0 25m +kube-system kube-scheduler-ramius-controller-0 1/1 Running 0 26m ``` ## Going Further diff --git a/docs/cl/digital-ocean.md b/docs/cl/digital-ocean.md index f2b4b3314..3681ff791 100644 --- a/docs/cl/digital-ocean.md +++ b/docs/cl/digital-ocean.md @@ -4,7 +4,7 @@ In this tutorial, we'll create a Kubernetes v1.15.3 cluster on DigitalOcean with We'll declare a Kubernetes cluster using the Typhoon Terraform module. Then apply the changes to create controller droplets, worker droplets, DNS records, tags, and TLS assets. -Controllers are provisioned to run an `etcd-member` peer and a `kubelet` service. Workers run just a `kubelet` service. A one-time [bootkube](https://github.com/kubernetes-incubator/bootkube) bootstrap schedules the `apiserver`, `scheduler`, `controller-manager`, and `coredns` on controllers and schedules `kube-proxy` and `flannel` on every node. A generated `kubeconfig` provides `kubectl` access to the cluster. +Controller hosts are provisioned to run an `etcd-member` peer and a `kubelet` service. Worker hosts run a `kubelet` service. Controller nodes run `kube-apiserver`, `kube-scheduler`, `kube-controller-manager`, and `coredns`, while `kube-proxy` and `calico` (or `flannel`) run on every node. A generated `kubeconfig` provides `kubectl` access to the cluster. ## Requirements @@ -85,7 +85,7 @@ Reference the [variables docs](#variables) or the [variables.tf](https://github. ## ssh-agent -Initial bootstrapping requires `bootkube.service` be started on one controller node. Terraform uses `ssh-agent` to automate this step. Add your SSH private key to `ssh-agent`. +Initial bootstrapping requires `bootstrap.service` be started on one controller node. Terraform uses `ssh-agent` to automate this step. Add your SSH private key to `ssh-agent`. ```sh ssh-add ~/.ssh/id_rsa @@ -111,11 +111,11 @@ Apply the changes to create the cluster. ```sh $ terraform apply -module.digital-ocean-nemo.null_resource.bootkube-start: Still creating... (30s elapsed) -module.digital-ocean-nemo.null_resource.bootkube-start: Provisioning with 'remote-exec'... +module.digital-ocean-nemo.null_resource.bootstrap: Still creating... (30s elapsed) +module.digital-ocean-nemo.null_resource.bootstrap: Provisioning with 'remote-exec'... ... -module.digital-ocean-nemo.null_resource.bootkube-start: Still creating... (6m20s elapsed) -module.digital-ocean-nemo.null_resource.bootkube-start: Creation complete (ID: 7599298447329218468) +module.digital-ocean-nemo.null_resource.bootstrap: Still creating... (6m20s elapsed) +module.digital-ocean-nemo.null_resource.bootstrap: Creation complete (ID: 7599298447329218468) Apply complete! Resources: 54 added, 0 changed, 0 destroyed. ``` @@ -142,18 +142,14 @@ NAMESPACE NAME READY STATUS RES kube-system coredns-1187388186-ld1j7 1/1 Running 0 11m kube-system coredns-1187388186-rdhf7 1/1 Running 0 11m kube-system flannel-1cq1v 2/2 Running 0 11m -kube-system flannel-hq9t0 2/2 Running 1 11m +kube-system flannel-hq9t0 2/2 Running 0 11m kube-system flannel-v0g9w 2/2 Running 0 11m -kube-system kube-apiserver-n10qr 1/1 Running 0 11m -kube-system kube-controller-manager-3271970485-37gtw 1/1 Running 1 11m -kube-system kube-controller-manager-3271970485-p52t5 1/1 Running 0 11m +kube-system kube-apiserver-ip-10.132.115.81 1/1 Running 0 11m +kube-system kube-controller-manager-ip-10.132.115.81 1/1 Running 0 11m kube-system kube-proxy-6kxjf 1/1 Running 0 11m kube-system kube-proxy-fh3td 1/1 Running 0 11m kube-system kube-proxy-k35rc 1/1 Running 0 11m -kube-system kube-scheduler-3895335239-2bc4c 1/1 Running 0 11m -kube-system kube-scheduler-3895335239-b7q47 1/1 Running 1 11m -kube-system pod-checkpointer-pr1lq 1/1 Running 0 11m -kube-system pod-checkpointer-pr1lq-10.132.115.81 1/1 Running 0 10m +kube-system kube-scheduler-ip-10.132.115.81 1/1 Running 0 11m ``` ## Going Further diff --git a/docs/cl/google-cloud.md b/docs/cl/google-cloud.md index 7d1e8a530..10e318711 100644 --- a/docs/cl/google-cloud.md +++ b/docs/cl/google-cloud.md @@ -4,7 +4,7 @@ In this tutorial, we'll create a Kubernetes v1.15.3 cluster on Google Compute En We'll declare a Kubernetes cluster using the Typhoon Terraform module. Then apply the changes to create a network, firewall rules, health checks, controller instances, worker managed instance group, load balancers, and TLS assets. -Controllers are provisioned to run an `etcd-member` peer and a `kubelet` service. Workers run just a `kubelet` service. A one-time [bootkube](https://github.com/kubernetes-incubator/bootkube) bootstrap schedules the `apiserver`, `scheduler`, `controller-manager`, and `coredns` on controllers and schedules `kube-proxy` and `calico` (or `flannel`) on every node. A generated `kubeconfig` provides `kubectl` access to the cluster. +Controller hosts are provisioned to run an `etcd-member` peer and a `kubelet` service. Worker hosts run a `kubelet` service. Controller nodes run `kube-apiserver`, `kube-scheduler`, `kube-controller-manager`, and `coredns`, while `kube-proxy` and `calico` (or `flannel`) run on every node. A generated `kubeconfig` provides `kubectl` access to the cluster. ## Requirements @@ -92,7 +92,7 @@ Reference the [variables docs](#variables) or the [variables.tf](https://github. ## ssh-agent -Initial bootstrapping requires `bootkube.service` be started on one controller node. Terraform uses `ssh-agent` to automate this step. Add your SSH private key to `ssh-agent`. +Initial bootstrapping requires `bootstrap.service` be started on one controller node. Terraform uses `ssh-agent` to automate this step. Add your SSH private key to `ssh-agent`. ```sh ssh-add ~/.ssh/id_rsa @@ -118,12 +118,11 @@ Apply the changes to create the cluster. ```sh $ terraform apply -module.google-cloud-yavin.null_resource.bootkube-start: Still creating... (10s elapsed) +module.google-cloud-yavin.null_resource.bootstrap: Still creating... (10s elapsed) ... - -module.google-cloud-yavin.null_resource.bootkube-start: Still creating... (5m30s elapsed) -module.google-cloud-yavin.null_resource.bootkube-start: Still creating... (5m40s elapsed) -module.google-cloud-yavin.null_resource.bootkube-start: Creation complete (ID: 5768638456220583358) +module.google-cloud-yavin.null_resource.bootstrap: Still creating... (5m30s elapsed) +module.google-cloud-yavin.null_resource.bootstrap: Still creating... (5m40s elapsed) +module.google-cloud-yavin.null_resource.bootstrap: Creation complete (ID: 5768638456220583358) Apply complete! Resources: 64 added, 0 changed, 0 destroyed. ``` @@ -153,15 +152,12 @@ kube-system calico-node-d1l5b 2/2 Running 0 kube-system calico-node-sp9ps 2/2 Running 0 6m kube-system coredns-1187388186-dkh3o 1/1 Running 0 6m kube-system coredns-1187388186-zj5dl 1/1 Running 0 6m -kube-system kube-apiserver-zppls 1/1 Running 0 6m -kube-system kube-controller-manager-3271970485-gh9kt 1/1 Running 0 6m -kube-system kube-controller-manager-3271970485-h90v8 1/1 Running 1 6m +kube-system kube-apiserver-controller-0 1/1 Running 0 6m +kube-system kube-controller-manager-controller-0 1/1 Running 0 6m kube-system kube-proxy-117v6 1/1 Running 0 6m kube-system kube-proxy-9886n 1/1 Running 0 6m kube-system kube-proxy-njn47 1/1 Running 0 6m -kube-system kube-scheduler-3895335239-5x87r 1/1 Running 0 6m -kube-system kube-scheduler-3895335239-bzrrt 1/1 Running 1 6m -kube-system pod-checkpointer-l6lrt 1/1 Running 0 6m +kube-system kube-scheduler-controller-0 1/1 Running 0 6m ``` ## Going Further diff --git a/docs/index.md b/docs/index.md index 49740b34b..b78c9dded 100644 --- a/docs/index.md +++ b/docs/index.md @@ -95,16 +95,12 @@ kube-system calico-node-d1l5b 2/2 Running 0 kube-system calico-node-sp9ps 2/2 Running 0 6m kube-system coredns-1187388186-dkh3o 1/1 Running 0 6m kube-system coredns-1187388186-zj5dl 1/1 Running 0 6m -kube-system kube-apiserver-zppls 1/1 Running 0 6m -kube-system kube-controller-manager-3271970485-gh9kt 1/1 Running 0 6m -kube-system kube-controller-manager-3271970485-h90v8 1/1 Running 1 6m +kube-system kube-apiserver-controller-0 1/1 Running 0 6m +kube-system kube-controller-manager-controller-0 1/1 Running 0 6m kube-system kube-proxy-117v6 1/1 Running 0 6m kube-system kube-proxy-9886n 1/1 Running 0 6m kube-system kube-proxy-njn47 1/1 Running 0 6m -kube-system kube-scheduler-3895335239-5x87r 1/1 Running 0 6m -kube-system kube-scheduler-3895335239-bzrrt 1/1 Running 1 6m -kube-system pod-checkpointer-l6lrt 1/1 Running 0 6m -kube-system pod-checkpointer-l6lrt-controller-0 1/1 Running 0 6m +kube-system kube-scheduler-controller-0 1/1 Running 0 6m ``` ## Help diff --git a/google-cloud/container-linux/kubernetes/README.md b/google-cloud/container-linux/kubernetes/README.md index 3a69bef35..11f6c2bd8 100644 --- a/google-cloud/container-linux/kubernetes/README.md +++ b/google-cloud/container-linux/kubernetes/README.md @@ -11,7 +11,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features -* Kubernetes v1.15.3 (upstream, via [kubernetes-incubator/bootkube](https://github.com/kubernetes-incubator/bootkube)) +* Kubernetes v1.15.3 (upstream) * Single or multi-master, [Calico](https://www.projectcalico.org/) or [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/) * Advanced features like [worker pools](https://typhoon.psdn.io/advanced/worker-pools/), [preemptible](https://typhoon.psdn.io/cl/google-cloud/#preemption) workers, and [snippets](https://typhoon.psdn.io/advanced/customization/#container-linux) customization diff --git a/google-cloud/container-linux/kubernetes/bootkube.tf b/google-cloud/container-linux/kubernetes/bootkube.tf index 867d0cc0e..76b31172e 100644 --- a/google-cloud/container-linux/kubernetes/bootkube.tf +++ b/google-cloud/container-linux/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=98cc19f80f2c4c3ddc63fc7aea6320e74bec561a" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=6e59af71138bc5f784453873074de16e7ee150eb" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/google-cloud/container-linux/kubernetes/cl/controller.yaml.tmpl b/google-cloud/container-linux/kubernetes/cl/controller.yaml.tmpl index 63ca9547f..3a6fe3e58 100644 --- a/google-cloud/container-linux/kubernetes/cl/controller.yaml.tmpl +++ b/google-cloud/container-linux/kubernetes/cl/controller.yaml.tmpl @@ -97,17 +97,28 @@ systemd: RestartSec=10 [Install] WantedBy=multi-user.target - - name: bootkube.service + - name: bootstrap.service contents: | [Unit] - Description=Bootstrap a Kubernetes cluster - ConditionPathExists=!/opt/bootkube/init_bootkube.done + Description=Kubernetes control plane + ConditionPathExists=!/opt/bootstrap/bootstrap.done [Service] Type=oneshot RemainAfterExit=true - WorkingDirectory=/opt/bootkube - ExecStart=/opt/bootkube/bootkube-start - ExecStartPost=/bin/touch /opt/bootkube/init_bootkube.done + WorkingDirectory=/opt/bootstrap + ExecStartPre=-/usr/bin/bash -c 'set -x && [ -n "$(ls /opt/bootstrap/assets/manifests-*/* 2>/dev/null)" ] && mv /opt/bootstrap/assets/manifests-*/* /opt/bootstrap/assets/manifests && rm -rf /opt/bootstrap/assets/manifests-*' + ExecStart=/usr/bin/rkt run \ + --trust-keys-from-https \ + --volume assets,kind=host,source=/opt/bootstrap/assets \ + --mount volume=assets,target=/assets \ + --volume script,kind=host,source=/opt/bootstrap/apply \ + --mount volume=script,target=/apply \ + --insecure-options=image \ + docker://k8s.gcr.io/hyperkube:v1.15.3 \ + --net=host \ + --dns=host \ + --exec=/apply + ExecStartPost=/bin/touch /opt/bootstrap/bootstrap.done [Install] WantedBy=multi-user.target storage: @@ -125,36 +136,26 @@ storage: inline: | KUBELET_IMAGE_URL=docker://k8s.gcr.io/hyperkube KUBELET_IMAGE_TAG=v1.15.3 - - path: /etc/sysctl.d/max-user-watches.conf + - path: /opt/bootstrap/apply filesystem: root + mode: 0544 contents: inline: | - fs.inotify.max_user_watches=16184 - - path: /opt/bootkube/bootkube-start + #!/bin/bash -e + export KUBECONFIG=/assets/auth/kubeconfig + until kubectl version; do + echo "Waiting for static pod control plane" + sleep 5 + done + until kubectl apply -f /assets/manifests -R; do + echo "Retry applying manifests" + sleep 5 + done + - path: /etc/sysctl.d/max-user-watches.conf filesystem: root - mode: 0544 - user: - id: 500 - group: - id: 500 contents: inline: | - #!/bin/bash - # Wrapper for bootkube start - set -e - # Move experimental manifests - [ -n "$(ls /opt/bootkube/assets/manifests-*/* 2>/dev/null)" ] && mv /opt/bootkube/assets/manifests-*/* /opt/bootkube/assets/manifests && rm -rf /opt/bootkube/assets/manifests-* - exec /usr/bin/rkt run \ - --trust-keys-from-https \ - --volume assets,kind=host,source=/opt/bootkube/assets \ - --mount volume=assets,target=/assets \ - --volume bootstrap,kind=host,source=/etc/kubernetes \ - --mount volume=bootstrap,target=/etc/kubernetes \ - $${RKT_OPTS} \ - quay.io/coreos/bootkube:v0.14.0 \ - --net=host \ - --dns=host \ - --exec=/bootkube -- start --asset-dir=/assets "$@" + fs.inotify.max_user_watches=16184 passwd: users: - name: core diff --git a/google-cloud/container-linux/kubernetes/network.tf b/google-cloud/container-linux/kubernetes/network.tf index 735fa3a84..8d0bbefc8 100644 --- a/google-cloud/container-linux/kubernetes/network.tf +++ b/google-cloud/container-linux/kubernetes/network.tf @@ -48,6 +48,20 @@ resource "google_compute_firewall" "internal-etcd-metrics" { target_tags = ["${var.cluster_name}-controller"] } +# Allow Prometheus to scrape kube-scheduler and kube-controller-manager metrics +resource "google_compute_firewall" "internal-kube-metrics" { + name = "${var.cluster_name}-internal-kube-metrics" + network = google_compute_network.network.name + + allow { + protocol = "tcp" + ports = [10251, 10252] + } + + source_tags = ["${var.cluster_name}-worker"] + target_tags = ["${var.cluster_name}-controller"] +} + resource "google_compute_firewall" "allow-apiserver" { name = "${var.cluster_name}-allow-apiserver" network = google_compute_network.network.name diff --git a/google-cloud/container-linux/kubernetes/ssh.tf b/google-cloud/container-linux/kubernetes/ssh.tf index ba298ea39..f0e101340 100644 --- a/google-cloud/container-linux/kubernetes/ssh.tf +++ b/google-cloud/container-linux/kubernetes/ssh.tf @@ -1,10 +1,14 @@ -# Secure copy etcd TLS assets to controllers. +# Secure copy assets to controllers. resource "null_resource" "copy-controller-secrets" { count = var.controller_count + + depends_on = [ + module.bootkube, + ] connection { type = "ssh" - host = element(local.controllers_ipv4_public, count.index) + host = local.controllers_ipv4_public[count.index] user = "core" timeout = "15m" } @@ -43,6 +47,11 @@ resource "null_resource" "copy-controller-secrets" { content = module.bootkube.etcd_peer_key destination = "$HOME/etcd-peer.key" } + + provisioner "file" { + source = var.asset_dir + destination = "$HOME/assets" + } provisioner "remote-exec" { inline = [ @@ -56,36 +65,33 @@ resource "null_resource" "copy-controller-secrets" { "sudo mv etcd-peer.key /etc/ssl/etcd/etcd/peer.key", "sudo chown -R etcd:etcd /etc/ssl/etcd", "sudo chmod -R 500 /etc/ssl/etcd", + "sudo mv $HOME/assets /opt/bootstrap/assets", + "sudo mkdir -p /etc/kubernetes/bootstrap-secrets", + "sudo cp -r /opt/bootstrap/assets/tls/* /etc/kubernetes/bootstrap-secrets/", + "sudo cp /opt/bootstrap/assets/auth/kubeconfig /etc/kubernetes/bootstrap-secrets/", + "sudo cp -r /opt/bootstrap/assets/static-manifests/* /etc/kubernetes/manifests/", ] } } -# Secure copy bootkube assets to ONE controller and start bootkube to perform -# one-time self-hosted cluster bootstrapping. -resource "null_resource" "bootkube-start" { +# Connect to a controller to perform one-time cluster bootstrap. +resource "null_resource" "bootstrap" { depends_on = [ - module.bootkube, + null_resource.copy-controller-secrets, module.workers, google_dns_record_set.apiserver, - null_resource.copy-controller-secrets, ] connection { type = "ssh" - host = element(local.controllers_ipv4_public, 0) + host = local.controllers_ipv4_public[0] user = "core" timeout = "15m" } - provisioner "file" { - source = var.asset_dir - destination = "$HOME/assets" - } - provisioner "remote-exec" { inline = [ - "sudo mv $HOME/assets /opt/bootkube", - "sudo systemctl start bootkube", + "sudo systemctl start bootstrap", ] } } From b15c60fa2f8a1cca153b03bbf22a13eae74a4bac Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Sat, 7 Sep 2019 13:38:51 -0700 Subject: [PATCH 212/523] Update CHANGES for control plane static pod switch * Remove old references to bootkube / self-hosted --- CHANGES.md | 4 ++++ README.md | 2 +- aws/container-linux/kubernetes/bootkube.tf | 2 +- aws/fedora-coreos/kubernetes/bootkube.tf | 2 +- azure/container-linux/kubernetes/bootkube.tf | 2 +- bare-metal/fedora-coreos/kubernetes/bootkube.tf | 2 +- digital-ocean/container-linux/kubernetes/bootkube.tf | 2 +- docs/architecture/operating-systems.md | 2 +- docs/index.md | 2 +- docs/topics/maintenance.md | 2 +- google-cloud/container-linux/kubernetes/bootkube.tf | 2 +- 11 files changed, 14 insertions(+), 10 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 61cab0149..70b48218c 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -4,6 +4,10 @@ Notable changes between versions. ## Latest +* Migrate control plane from self-hosted to static pods ([#536](https://github.com/poseidon/typhoon/pull/536)) + * Run `kube-apiserver`, `kube-scheduler`, and `kube-controller-manager` as static pods on each controller + * `kubectl` edits to `kube-apiserver`, `kube-scheduler`, and `kube-controller-manager` are no longer possible (change) + * Remove [bootkube](https://github.com/kubernetes-incubator/bootkube), self-hosted pivot, and `pod-checkpointer` * Update CoreDNS from v1.5.0 to v1.6.2 ([#535](https://github.com/poseidon/typhoon/pull/535)) * Update etcd from v3.3.15 to [v3.4.0](https://github.com/etcd-io/etcd/releases/tag/v3.4.0) * Recommend updating `terraform-provider-ct` plugin from v0.3.2 to [v0.4.0](https://github.com/poseidon/terraform-provider-ct/releases/tag/v0.4.0) diff --git a/README.md b/README.md index 06157cf02..dee650356 100644 --- a/README.md +++ b/README.md @@ -11,7 +11,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features -* Kubernetes v1.15.3 (upstream, via [kubernetes-incubator/bootkube](https://github.com/kubernetes-incubator/bootkube)) +* Kubernetes v1.15.3 (upstream) * Single or multi-master, [Calico](https://www.projectcalico.org/) or [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/) * Advanced features like [worker pools](https://typhoon.psdn.io/advanced/worker-pools/), [preemptible](https://typhoon.psdn.io/cl/google-cloud/#preemption) workers, and [snippets](https://typhoon.psdn.io/advanced/customization/#container-linux) customization diff --git a/aws/container-linux/kubernetes/bootkube.tf b/aws/container-linux/kubernetes/bootkube.tf index 6db18ccf6..9390a7522 100644 --- a/aws/container-linux/kubernetes/bootkube.tf +++ b/aws/container-linux/kubernetes/bootkube.tf @@ -1,4 +1,4 @@ -# Self-hosted Kubernetes assets (kubeconfig, manifests) +# Kubernetes assets (kubeconfig, manifests) module "bootkube" { source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=6e59af71138bc5f784453873074de16e7ee150eb" diff --git a/aws/fedora-coreos/kubernetes/bootkube.tf b/aws/fedora-coreos/kubernetes/bootkube.tf index b11e6cf85..10002329c 100644 --- a/aws/fedora-coreos/kubernetes/bootkube.tf +++ b/aws/fedora-coreos/kubernetes/bootkube.tf @@ -1,4 +1,4 @@ -# Self-hosted Kubernetes assets (kubeconfig, manifests) +# Kubernetes assets (kubeconfig, manifests) module "bootkube" { source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=6e59af71138bc5f784453873074de16e7ee150eb" diff --git a/azure/container-linux/kubernetes/bootkube.tf b/azure/container-linux/kubernetes/bootkube.tf index 8cefa6b2c..70c889b37 100644 --- a/azure/container-linux/kubernetes/bootkube.tf +++ b/azure/container-linux/kubernetes/bootkube.tf @@ -1,4 +1,4 @@ -# Self-hosted Kubernetes assets (kubeconfig, manifests) +# Kubernetes assets (kubeconfig, manifests) module "bootkube" { source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=6e59af71138bc5f784453873074de16e7ee150eb" diff --git a/bare-metal/fedora-coreos/kubernetes/bootkube.tf b/bare-metal/fedora-coreos/kubernetes/bootkube.tf index a9c0bb605..a977981bb 100644 --- a/bare-metal/fedora-coreos/kubernetes/bootkube.tf +++ b/bare-metal/fedora-coreos/kubernetes/bootkube.tf @@ -1,4 +1,4 @@ -# Self-hosted Kubernetes assets (kubeconfig, manifests) +# Kubernetes assets (kubeconfig, manifests) module "bootkube" { source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=6e59af71138bc5f784453873074de16e7ee150eb" diff --git a/digital-ocean/container-linux/kubernetes/bootkube.tf b/digital-ocean/container-linux/kubernetes/bootkube.tf index e279fa9d8..b7711aea5 100644 --- a/digital-ocean/container-linux/kubernetes/bootkube.tf +++ b/digital-ocean/container-linux/kubernetes/bootkube.tf @@ -1,4 +1,4 @@ -# Self-hosted Kubernetes assets (kubeconfig, manifests) +# Kubernetes assets (kubeconfig, manifests) module "bootkube" { source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=6e59af71138bc5f784453873074de16e7ee150eb" diff --git a/docs/architecture/operating-systems.md b/docs/architecture/operating-systems.md index 3a3b8e4c5..9db886494 100644 --- a/docs/architecture/operating-systems.md +++ b/docs/architecture/operating-systems.md @@ -30,7 +30,7 @@ Together, they diversify Typhoon to support a range of container technologies. |-------------------|-----------------|---------------| | single-master | all platforms | all platforms | | multi-master | all platforms | all platforms | -| control plane | self-hosted | self-hosted | +| control plane | static pods | static pods | | kubelet image | upstream hyperkube | upstream hyperkube | | control plane images | upstream hyperkube | upstream hyperkube | | on-host etcd | rkt-fly | podman | diff --git a/docs/index.md b/docs/index.md index b78c9dded..b0d064c93 100644 --- a/docs/index.md +++ b/docs/index.md @@ -11,7 +11,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features -* Kubernetes v1.15.3 (upstream, via [kubernetes-incubator/bootkube](https://github.com/kubernetes-incubator/bootkube)) +* Kubernetes v1.15.3 (upstream) * Single or multi-master, [Calico](https://www.projectcalico.org/) or [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/) * Advanced features like [worker pools](advanced/worker-pools/), [preemptible](cl/google-cloud/#preemption) workers, and [snippets](advanced/customization/#container-linux) customization diff --git a/docs/topics/maintenance.md b/docs/topics/maintenance.md index d1597d68a..2d605085b 100644 --- a/docs/topics/maintenance.md +++ b/docs/topics/maintenance.md @@ -110,7 +110,7 @@ Apply complete! Resources: 0 added, 0 changed, 55 destroyed. #### In-place Edits -Typhoon uses a self-hosted Kubernetes control plane which allows certain manifest upgrades to be performed in-place. Components like `apiserver`, `controller-manager`, `scheduler`, `flannel`/`calico`, `coredns`, and `kube-proxy` are run on Kubernetes itself and can be edited via `kubectl`. If you're interested, see the bootkube [upgrade docs](https://github.com/kubernetes-incubator/bootkube/blob/master/Documentation/upgrading.md). +Typhoon uses a static pod Kubernetes control plane which allows certain manifest upgrades to be performed in-place. Components like `kube-apiserver`, `kube-controller-manager`, and `kube-scheduler` are run as static pods. Components `flannel`/`calico`, `coredns`, and `kube-proxy` are scheduled on Kubernetes and can be edited via `kubectl`. In certain scenarios, in-place edits can be useful for quickly rolling out security patches (e.g. bumping `coredns`) or prioritizing speed over the safety of a proper cluster re-provision and transition. diff --git a/google-cloud/container-linux/kubernetes/bootkube.tf b/google-cloud/container-linux/kubernetes/bootkube.tf index 76b31172e..4ad35381a 100644 --- a/google-cloud/container-linux/kubernetes/bootkube.tf +++ b/google-cloud/container-linux/kubernetes/bootkube.tf @@ -1,4 +1,4 @@ -# Self-hosted Kubernetes assets (kubeconfig, manifests) +# Kubernetes assets (kubeconfig, manifests) module "bootkube" { source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=6e59af71138bc5f784453873074de16e7ee150eb" From 96b646cf6d4bfb416fb64eeaf3fc2742bc528d69 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Sat, 14 Sep 2019 16:24:32 -0700 Subject: [PATCH 213/523] Rename bootkube modules to bootstrap * Rename render module from bootkube to bootstrap. Avoid confusion with the kubernetes-incubator/bootkube tool since it is no longer used * Use the poseidon/terraform-render-bootstrap Terraform module (formerly poseidon/terraform-render-bootkube) * https://github.com/poseidon/terraform-render-bootkube/pull/149 --- aws/container-linux/kubernetes/bootkube.tf | 4 ++-- aws/container-linux/kubernetes/controllers.tf | 2 +- aws/container-linux/kubernetes/outputs.tf | 4 ++-- aws/container-linux/kubernetes/ssh.tf | 16 +++++++-------- aws/container-linux/kubernetes/workers.tf | 2 +- aws/fedora-coreos/kubernetes/bootkube.tf | 4 ++-- aws/fedora-coreos/kubernetes/controllers.tf | 2 +- aws/fedora-coreos/kubernetes/outputs.tf | 4 ++-- aws/fedora-coreos/kubernetes/ssh.tf | 16 +++++++-------- aws/fedora-coreos/kubernetes/workers.tf | 2 +- azure/container-linux/kubernetes/bootkube.tf | 4 ++-- .../container-linux/kubernetes/controllers.tf | 2 +- azure/container-linux/kubernetes/outputs.tf | 4 ++-- azure/container-linux/kubernetes/ssh.tf | 16 +++++++-------- azure/container-linux/kubernetes/workers.tf | 2 +- .../container-linux/kubernetes/bootkube.tf | 4 ++-- .../container-linux/kubernetes/outputs.tf | 2 +- .../container-linux/kubernetes/profiles.tf | 4 ++-- bare-metal/container-linux/kubernetes/ssh.tf | 20 +++++++++---------- .../fedora-coreos/kubernetes/bootkube.tf | 4 ++-- .../fedora-coreos/kubernetes/outputs.tf | 2 +- .../fedora-coreos/kubernetes/profiles.tf | 4 ++-- bare-metal/fedora-coreos/kubernetes/ssh.tf | 20 +++++++++---------- .../container-linux/kubernetes/bootkube.tf | 4 ++-- .../container-linux/kubernetes/outputs.tf | 2 +- .../container-linux/kubernetes/ssh.tf | 20 +++++++++---------- docs/advanced/customization.md | 2 +- .../container-linux/kubernetes/bootkube.tf | 4 ++-- .../container-linux/kubernetes/controllers.tf | 2 +- .../container-linux/kubernetes/outputs.tf | 4 ++-- .../container-linux/kubernetes/ssh.tf | 16 +++++++-------- .../container-linux/kubernetes/workers.tf | 2 +- 32 files changed, 100 insertions(+), 100 deletions(-) diff --git a/aws/container-linux/kubernetes/bootkube.tf b/aws/container-linux/kubernetes/bootkube.tf index 9390a7522..53a45edd1 100644 --- a/aws/container-linux/kubernetes/bootkube.tf +++ b/aws/container-linux/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) -module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=6e59af71138bc5f784453873074de16e7ee150eb" +module "bootstrap" { + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=6e59af71138bc5f784453873074de16e7ee150eb" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/aws/container-linux/kubernetes/controllers.tf b/aws/container-linux/kubernetes/controllers.tf index 75e28c83e..282679455 100644 --- a/aws/container-linux/kubernetes/controllers.tf +++ b/aws/container-linux/kubernetes/controllers.tf @@ -71,7 +71,7 @@ data "template_file" "controller-configs" { # etcd0=https://cluster-etcd0.example.com,etcd1=https://cluster-etcd1.example.com,... etcd_initial_cluster = join(",", data.template_file.etcds.*.rendered) cgroup_driver = local.flavor == "flatcar" && local.channel == "edge" ? "systemd" : "cgroupfs" - kubeconfig = indent(10, module.bootkube.kubeconfig-kubelet) + kubeconfig = indent(10, module.bootstrap.kubeconfig-kubelet) ssh_authorized_key = var.ssh_authorized_key cluster_dns_service_ip = cidrhost(var.service_cidr, 10) cluster_domain_suffix = var.cluster_domain_suffix diff --git a/aws/container-linux/kubernetes/outputs.tf b/aws/container-linux/kubernetes/outputs.tf index 471c3300e..d9afc7bd3 100644 --- a/aws/container-linux/kubernetes/outputs.tf +++ b/aws/container-linux/kubernetes/outputs.tf @@ -1,5 +1,5 @@ output "kubeconfig-admin" { - value = module.bootkube.kubeconfig-admin + value = module.bootstrap.kubeconfig-admin } # Outputs for Kubernetes Ingress @@ -32,7 +32,7 @@ output "worker_security_groups" { } output "kubeconfig" { - value = module.bootkube.kubeconfig-kubelet + value = module.bootstrap.kubeconfig-kubelet } # Outputs for custom load balancing diff --git a/aws/container-linux/kubernetes/ssh.tf b/aws/container-linux/kubernetes/ssh.tf index 5f027d17f..4ebd5f240 100644 --- a/aws/container-linux/kubernetes/ssh.tf +++ b/aws/container-linux/kubernetes/ssh.tf @@ -3,7 +3,7 @@ resource "null_resource" "copy-controller-secrets" { count = var.controller_count depends_on = [ - module.bootkube, + module.bootstrap, ] connection { @@ -14,37 +14,37 @@ resource "null_resource" "copy-controller-secrets" { } provisioner "file" { - content = module.bootkube.etcd_ca_cert + content = module.bootstrap.etcd_ca_cert destination = "$HOME/etcd-client-ca.crt" } provisioner "file" { - content = module.bootkube.etcd_client_cert + content = module.bootstrap.etcd_client_cert destination = "$HOME/etcd-client.crt" } provisioner "file" { - content = module.bootkube.etcd_client_key + content = module.bootstrap.etcd_client_key destination = "$HOME/etcd-client.key" } provisioner "file" { - content = module.bootkube.etcd_server_cert + content = module.bootstrap.etcd_server_cert destination = "$HOME/etcd-server.crt" } provisioner "file" { - content = module.bootkube.etcd_server_key + content = module.bootstrap.etcd_server_key destination = "$HOME/etcd-server.key" } provisioner "file" { - content = module.bootkube.etcd_peer_cert + content = module.bootstrap.etcd_peer_cert destination = "$HOME/etcd-peer.crt" } provisioner "file" { - content = module.bootkube.etcd_peer_key + content = module.bootstrap.etcd_peer_key destination = "$HOME/etcd-peer.key" } diff --git a/aws/container-linux/kubernetes/workers.tf b/aws/container-linux/kubernetes/workers.tf index 5b3b787d2..b5df22138 100644 --- a/aws/container-linux/kubernetes/workers.tf +++ b/aws/container-linux/kubernetes/workers.tf @@ -14,7 +14,7 @@ module "workers" { target_groups = var.worker_target_groups # configuration - kubeconfig = module.bootkube.kubeconfig-kubelet + kubeconfig = module.bootstrap.kubeconfig-kubelet ssh_authorized_key = var.ssh_authorized_key service_cidr = var.service_cidr cluster_domain_suffix = var.cluster_domain_suffix diff --git a/aws/fedora-coreos/kubernetes/bootkube.tf b/aws/fedora-coreos/kubernetes/bootkube.tf index 10002329c..26ea352e3 100644 --- a/aws/fedora-coreos/kubernetes/bootkube.tf +++ b/aws/fedora-coreos/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) -module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=6e59af71138bc5f784453873074de16e7ee150eb" +module "bootstrap" { + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=6e59af71138bc5f784453873074de16e7ee150eb" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/aws/fedora-coreos/kubernetes/controllers.tf b/aws/fedora-coreos/kubernetes/controllers.tf index 97e792d81..e4bd9ddba 100644 --- a/aws/fedora-coreos/kubernetes/controllers.tf +++ b/aws/fedora-coreos/kubernetes/controllers.tf @@ -67,7 +67,7 @@ data "template_file" "controller-configs" { etcd_domain = "${var.cluster_name}-etcd${count.index}.${var.dns_zone}" # etcd0=https://cluster-etcd0.example.com,etcd1=https://cluster-etcd1.example.com,... etcd_initial_cluster = join(",", data.template_file.etcds.*.rendered) - kubeconfig = indent(10, module.bootkube.kubeconfig-kubelet) + kubeconfig = indent(10, module.bootstrap.kubeconfig-kubelet) ssh_authorized_key = var.ssh_authorized_key cluster_dns_service_ip = cidrhost(var.service_cidr, 10) cluster_domain_suffix = var.cluster_domain_suffix diff --git a/aws/fedora-coreos/kubernetes/outputs.tf b/aws/fedora-coreos/kubernetes/outputs.tf index 471c3300e..d9afc7bd3 100644 --- a/aws/fedora-coreos/kubernetes/outputs.tf +++ b/aws/fedora-coreos/kubernetes/outputs.tf @@ -1,5 +1,5 @@ output "kubeconfig-admin" { - value = module.bootkube.kubeconfig-admin + value = module.bootstrap.kubeconfig-admin } # Outputs for Kubernetes Ingress @@ -32,7 +32,7 @@ output "worker_security_groups" { } output "kubeconfig" { - value = module.bootkube.kubeconfig-kubelet + value = module.bootstrap.kubeconfig-kubelet } # Outputs for custom load balancing diff --git a/aws/fedora-coreos/kubernetes/ssh.tf b/aws/fedora-coreos/kubernetes/ssh.tf index 2e39779e2..2e2cc3fe7 100644 --- a/aws/fedora-coreos/kubernetes/ssh.tf +++ b/aws/fedora-coreos/kubernetes/ssh.tf @@ -3,7 +3,7 @@ resource "null_resource" "copy-controller-secrets" { count = var.controller_count depends_on = [ - module.bootkube, + module.bootstrap, ] connection { @@ -14,37 +14,37 @@ resource "null_resource" "copy-controller-secrets" { } provisioner "file" { - content = module.bootkube.etcd_ca_cert + content = module.bootstrap.etcd_ca_cert destination = "$HOME/etcd-client-ca.crt" } provisioner "file" { - content = module.bootkube.etcd_client_cert + content = module.bootstrap.etcd_client_cert destination = "$HOME/etcd-client.crt" } provisioner "file" { - content = module.bootkube.etcd_client_key + content = module.bootstrap.etcd_client_key destination = "$HOME/etcd-client.key" } provisioner "file" { - content = module.bootkube.etcd_server_cert + content = module.bootstrap.etcd_server_cert destination = "$HOME/etcd-server.crt" } provisioner "file" { - content = module.bootkube.etcd_server_key + content = module.bootstrap.etcd_server_key destination = "$HOME/etcd-server.key" } provisioner "file" { - content = module.bootkube.etcd_peer_cert + content = module.bootstrap.etcd_peer_cert destination = "$HOME/etcd-peer.crt" } provisioner "file" { - content = module.bootkube.etcd_peer_key + content = module.bootstrap.etcd_peer_key destination = "$HOME/etcd-peer.key" } diff --git a/aws/fedora-coreos/kubernetes/workers.tf b/aws/fedora-coreos/kubernetes/workers.tf index af74d8bb8..4a2e50764 100644 --- a/aws/fedora-coreos/kubernetes/workers.tf +++ b/aws/fedora-coreos/kubernetes/workers.tf @@ -14,7 +14,7 @@ module "workers" { target_groups = var.worker_target_groups # configuration - kubeconfig = module.bootkube.kubeconfig-kubelet + kubeconfig = module.bootstrap.kubeconfig-kubelet ssh_authorized_key = var.ssh_authorized_key service_cidr = var.service_cidr cluster_domain_suffix = var.cluster_domain_suffix diff --git a/azure/container-linux/kubernetes/bootkube.tf b/azure/container-linux/kubernetes/bootkube.tf index 70c889b37..d5983f850 100644 --- a/azure/container-linux/kubernetes/bootkube.tf +++ b/azure/container-linux/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) -module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=6e59af71138bc5f784453873074de16e7ee150eb" +module "bootstrap" { + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=6e59af71138bc5f784453873074de16e7ee150eb" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/azure/container-linux/kubernetes/controllers.tf b/azure/container-linux/kubernetes/controllers.tf index c57f42da3..36518d180 100644 --- a/azure/container-linux/kubernetes/controllers.tf +++ b/azure/container-linux/kubernetes/controllers.tf @@ -155,7 +155,7 @@ data "template_file" "controller-configs" { etcd_domain = "${var.cluster_name}-etcd${count.index}.${var.dns_zone}" # etcd0=https://cluster-etcd0.example.com,etcd1=https://cluster-etcd1.example.com,... etcd_initial_cluster = join(",", data.template_file.etcds.*.rendered) - kubeconfig = indent(10, module.bootkube.kubeconfig-kubelet) + kubeconfig = indent(10, module.bootstrap.kubeconfig-kubelet) ssh_authorized_key = var.ssh_authorized_key cluster_dns_service_ip = cidrhost(var.service_cidr, 10) cluster_domain_suffix = var.cluster_domain_suffix diff --git a/azure/container-linux/kubernetes/outputs.tf b/azure/container-linux/kubernetes/outputs.tf index 79a936a80..39169d8df 100644 --- a/azure/container-linux/kubernetes/outputs.tf +++ b/azure/container-linux/kubernetes/outputs.tf @@ -1,5 +1,5 @@ output "kubeconfig-admin" { - value = module.bootkube.kubeconfig-admin + value = module.bootstrap.kubeconfig-admin } # Outputs for Kubernetes Ingress @@ -28,7 +28,7 @@ output "security_group_id" { } output "kubeconfig" { - value = module.bootkube.kubeconfig-kubelet + value = module.bootstrap.kubeconfig-kubelet } # Outputs for custom firewalling diff --git a/azure/container-linux/kubernetes/ssh.tf b/azure/container-linux/kubernetes/ssh.tf index 961cb1dfd..f8012c270 100644 --- a/azure/container-linux/kubernetes/ssh.tf +++ b/azure/container-linux/kubernetes/ssh.tf @@ -3,7 +3,7 @@ resource "null_resource" "copy-controller-secrets" { count = var.controller_count depends_on = [ - module.bootkube, + module.bootstrap, azurerm_virtual_machine.controllers ] @@ -15,37 +15,37 @@ resource "null_resource" "copy-controller-secrets" { } provisioner "file" { - content = module.bootkube.etcd_ca_cert + content = module.bootstrap.etcd_ca_cert destination = "$HOME/etcd-client-ca.crt" } provisioner "file" { - content = module.bootkube.etcd_client_cert + content = module.bootstrap.etcd_client_cert destination = "$HOME/etcd-client.crt" } provisioner "file" { - content = module.bootkube.etcd_client_key + content = module.bootstrap.etcd_client_key destination = "$HOME/etcd-client.key" } provisioner "file" { - content = module.bootkube.etcd_server_cert + content = module.bootstrap.etcd_server_cert destination = "$HOME/etcd-server.crt" } provisioner "file" { - content = module.bootkube.etcd_server_key + content = module.bootstrap.etcd_server_key destination = "$HOME/etcd-server.key" } provisioner "file" { - content = module.bootkube.etcd_peer_cert + content = module.bootstrap.etcd_peer_cert destination = "$HOME/etcd-peer.crt" } provisioner "file" { - content = module.bootkube.etcd_peer_key + content = module.bootstrap.etcd_peer_key destination = "$HOME/etcd-peer.key" } diff --git a/azure/container-linux/kubernetes/workers.tf b/azure/container-linux/kubernetes/workers.tf index 0f40da907..5775f55c9 100644 --- a/azure/container-linux/kubernetes/workers.tf +++ b/azure/container-linux/kubernetes/workers.tf @@ -15,7 +15,7 @@ module "workers" { priority = var.worker_priority # configuration - kubeconfig = module.bootkube.kubeconfig-kubelet + kubeconfig = module.bootstrap.kubeconfig-kubelet ssh_authorized_key = var.ssh_authorized_key service_cidr = var.service_cidr cluster_domain_suffix = var.cluster_domain_suffix diff --git a/bare-metal/container-linux/kubernetes/bootkube.tf b/bare-metal/container-linux/kubernetes/bootkube.tf index 67efc0c7d..9c3bc841f 100644 --- a/bare-metal/container-linux/kubernetes/bootkube.tf +++ b/bare-metal/container-linux/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) -module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=6e59af71138bc5f784453873074de16e7ee150eb" +module "bootstrap" { + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=6e59af71138bc5f784453873074de16e7ee150eb" cluster_name = var.cluster_name api_servers = [var.k8s_domain_name] diff --git a/bare-metal/container-linux/kubernetes/outputs.tf b/bare-metal/container-linux/kubernetes/outputs.tf index 1fd43af63..8e81ca1e5 100644 --- a/bare-metal/container-linux/kubernetes/outputs.tf +++ b/bare-metal/container-linux/kubernetes/outputs.tf @@ -1,4 +1,4 @@ output "kubeconfig-admin" { - value = module.bootkube.kubeconfig-admin + value = module.bootstrap.kubeconfig-admin } diff --git a/bare-metal/container-linux/kubernetes/profiles.tf b/bare-metal/container-linux/kubernetes/profiles.tf index 400ef6ed3..0a6ba49e3 100644 --- a/bare-metal/container-linux/kubernetes/profiles.tf +++ b/bare-metal/container-linux/kubernetes/profiles.tf @@ -160,7 +160,7 @@ data "template_file" "controller-configs" { etcd_name = element(var.controller_names, count.index) etcd_initial_cluster = join(",", formatlist("%s=https://%s:2380", var.controller_names, var.controller_domains)) cgroup_driver = var.os_channel == "flatcar-edge" ? "systemd" : "cgroupfs" - cluster_dns_service_ip = module.bootkube.cluster_dns_service_ip + cluster_dns_service_ip = module.bootstrap.cluster_dns_service_ip cluster_domain_suffix = var.cluster_domain_suffix ssh_authorized_key = var.ssh_authorized_key } @@ -188,7 +188,7 @@ data "template_file" "worker-configs" { vars = { domain_name = element(var.worker_domains, count.index) cgroup_driver = var.os_channel == "flatcar-edge" ? "systemd" : "cgroupfs" - cluster_dns_service_ip = module.bootkube.cluster_dns_service_ip + cluster_dns_service_ip = module.bootstrap.cluster_dns_service_ip cluster_domain_suffix = var.cluster_domain_suffix ssh_authorized_key = var.ssh_authorized_key } diff --git a/bare-metal/container-linux/kubernetes/ssh.tf b/bare-metal/container-linux/kubernetes/ssh.tf index 30e0be7a9..2a85da6c6 100644 --- a/bare-metal/container-linux/kubernetes/ssh.tf +++ b/bare-metal/container-linux/kubernetes/ssh.tf @@ -8,7 +8,7 @@ resource "null_resource" "copy-controller-secrets" { matchbox_group.install, matchbox_group.controller, matchbox_group.worker, - module.bootkube, + module.bootstrap, ] connection { @@ -19,42 +19,42 @@ resource "null_resource" "copy-controller-secrets" { } provisioner "file" { - content = module.bootkube.kubeconfig-kubelet + content = module.bootstrap.kubeconfig-kubelet destination = "$HOME/kubeconfig" } provisioner "file" { - content = module.bootkube.etcd_ca_cert + content = module.bootstrap.etcd_ca_cert destination = "$HOME/etcd-client-ca.crt" } provisioner "file" { - content = module.bootkube.etcd_client_cert + content = module.bootstrap.etcd_client_cert destination = "$HOME/etcd-client.crt" } provisioner "file" { - content = module.bootkube.etcd_client_key + content = module.bootstrap.etcd_client_key destination = "$HOME/etcd-client.key" } provisioner "file" { - content = module.bootkube.etcd_server_cert + content = module.bootstrap.etcd_server_cert destination = "$HOME/etcd-server.crt" } provisioner "file" { - content = module.bootkube.etcd_server_key + content = module.bootstrap.etcd_server_key destination = "$HOME/etcd-server.key" } provisioner "file" { - content = module.bootkube.etcd_peer_cert + content = module.bootstrap.etcd_peer_cert destination = "$HOME/etcd-peer.crt" } provisioner "file" { - content = module.bootkube.etcd_peer_key + content = module.bootstrap.etcd_peer_key destination = "$HOME/etcd-peer.key" } @@ -105,7 +105,7 @@ resource "null_resource" "copy-worker-secrets" { } provisioner "file" { - content = module.bootkube.kubeconfig-kubelet + content = module.bootstrap.kubeconfig-kubelet destination = "$HOME/kubeconfig" } diff --git a/bare-metal/fedora-coreos/kubernetes/bootkube.tf b/bare-metal/fedora-coreos/kubernetes/bootkube.tf index a977981bb..0c49932fe 100644 --- a/bare-metal/fedora-coreos/kubernetes/bootkube.tf +++ b/bare-metal/fedora-coreos/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) -module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=6e59af71138bc5f784453873074de16e7ee150eb" +module "bootstrap" { + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=6e59af71138bc5f784453873074de16e7ee150eb" cluster_name = var.cluster_name api_servers = [var.k8s_domain_name] diff --git a/bare-metal/fedora-coreos/kubernetes/outputs.tf b/bare-metal/fedora-coreos/kubernetes/outputs.tf index 1fd43af63..8e81ca1e5 100644 --- a/bare-metal/fedora-coreos/kubernetes/outputs.tf +++ b/bare-metal/fedora-coreos/kubernetes/outputs.tf @@ -1,4 +1,4 @@ output "kubeconfig-admin" { - value = module.bootkube.kubeconfig-admin + value = module.bootstrap.kubeconfig-admin } diff --git a/bare-metal/fedora-coreos/kubernetes/profiles.tf b/bare-metal/fedora-coreos/kubernetes/profiles.tf index b17a655a5..80771c9b7 100644 --- a/bare-metal/fedora-coreos/kubernetes/profiles.tf +++ b/bare-metal/fedora-coreos/kubernetes/profiles.tf @@ -56,7 +56,7 @@ data "template_file" "controller-configs" { domain_name = var.controller_domains[count.index] etcd_name = var.controller_names[count.index] etcd_initial_cluster = join(",", formatlist("%s=https://%s:2380", var.controller_names, var.controller_domains)) - cluster_dns_service_ip = module.bootkube.cluster_dns_service_ip + cluster_dns_service_ip = module.bootstrap.cluster_dns_service_ip cluster_domain_suffix = var.cluster_domain_suffix ssh_authorized_key = var.ssh_authorized_key } @@ -89,7 +89,7 @@ data "template_file" "worker-configs" { template = file("${path.module}/fcc/worker.yaml") vars = { domain_name = var.worker_domains[count.index] - cluster_dns_service_ip = module.bootkube.cluster_dns_service_ip + cluster_dns_service_ip = module.bootstrap.cluster_dns_service_ip cluster_domain_suffix = var.cluster_domain_suffix ssh_authorized_key = var.ssh_authorized_key } diff --git a/bare-metal/fedora-coreos/kubernetes/ssh.tf b/bare-metal/fedora-coreos/kubernetes/ssh.tf index 0a10bf619..002faf1fd 100644 --- a/bare-metal/fedora-coreos/kubernetes/ssh.tf +++ b/bare-metal/fedora-coreos/kubernetes/ssh.tf @@ -7,7 +7,7 @@ resource "null_resource" "copy-controller-secrets" { depends_on = [ matchbox_group.controller, matchbox_group.worker, - module.bootkube, + module.bootstrap, ] connection { @@ -18,42 +18,42 @@ resource "null_resource" "copy-controller-secrets" { } provisioner "file" { - content = module.bootkube.kubeconfig-kubelet + content = module.bootstrap.kubeconfig-kubelet destination = "$HOME/kubeconfig" } provisioner "file" { - content = module.bootkube.etcd_ca_cert + content = module.bootstrap.etcd_ca_cert destination = "$HOME/etcd-client-ca.crt" } provisioner "file" { - content = module.bootkube.etcd_client_cert + content = module.bootstrap.etcd_client_cert destination = "$HOME/etcd-client.crt" } provisioner "file" { - content = module.bootkube.etcd_client_key + content = module.bootstrap.etcd_client_key destination = "$HOME/etcd-client.key" } provisioner "file" { - content = module.bootkube.etcd_server_cert + content = module.bootstrap.etcd_server_cert destination = "$HOME/etcd-server.crt" } provisioner "file" { - content = module.bootkube.etcd_server_key + content = module.bootstrap.etcd_server_key destination = "$HOME/etcd-server.key" } provisioner "file" { - content = module.bootkube.etcd_peer_cert + content = module.bootstrap.etcd_peer_cert destination = "$HOME/etcd-peer.crt" } provisioner "file" { - content = module.bootkube.etcd_peer_key + content = module.bootstrap.etcd_peer_key destination = "$HOME/etcd-peer.key" } @@ -101,7 +101,7 @@ resource "null_resource" "copy-worker-secrets" { } provisioner "file" { - content = module.bootkube.kubeconfig-kubelet + content = module.bootstrap.kubeconfig-kubelet destination = "$HOME/kubeconfig" } diff --git a/digital-ocean/container-linux/kubernetes/bootkube.tf b/digital-ocean/container-linux/kubernetes/bootkube.tf index b7711aea5..e3ed992eb 100644 --- a/digital-ocean/container-linux/kubernetes/bootkube.tf +++ b/digital-ocean/container-linux/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) -module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=6e59af71138bc5f784453873074de16e7ee150eb" +module "bootstrap" { + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=6e59af71138bc5f784453873074de16e7ee150eb" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/digital-ocean/container-linux/kubernetes/outputs.tf b/digital-ocean/container-linux/kubernetes/outputs.tf index bf05b4a0d..fe5338efd 100644 --- a/digital-ocean/container-linux/kubernetes/outputs.tf +++ b/digital-ocean/container-linux/kubernetes/outputs.tf @@ -1,5 +1,5 @@ output "kubeconfig-admin" { - value = module.bootkube.kubeconfig-admin + value = module.bootstrap.kubeconfig-admin } output "controllers_dns" { diff --git a/digital-ocean/container-linux/kubernetes/ssh.tf b/digital-ocean/container-linux/kubernetes/ssh.tf index 1d334ccb2..a1b734d1f 100644 --- a/digital-ocean/container-linux/kubernetes/ssh.tf +++ b/digital-ocean/container-linux/kubernetes/ssh.tf @@ -3,7 +3,7 @@ resource "null_resource" "copy-controller-secrets" { count = var.controller_count depends_on = [ - module.bootkube, + module.bootstrap, digitalocean_firewall.rules ] @@ -15,42 +15,42 @@ resource "null_resource" "copy-controller-secrets" { } provisioner "file" { - content = module.bootkube.kubeconfig-kubelet + content = module.bootstrap.kubeconfig-kubelet destination = "$HOME/kubeconfig" } provisioner "file" { - content = module.bootkube.etcd_ca_cert + content = module.bootstrap.etcd_ca_cert destination = "$HOME/etcd-client-ca.crt" } provisioner "file" { - content = module.bootkube.etcd_client_cert + content = module.bootstrap.etcd_client_cert destination = "$HOME/etcd-client.crt" } provisioner "file" { - content = module.bootkube.etcd_client_key + content = module.bootstrap.etcd_client_key destination = "$HOME/etcd-client.key" } provisioner "file" { - content = module.bootkube.etcd_server_cert + content = module.bootstrap.etcd_server_cert destination = "$HOME/etcd-server.crt" } provisioner "file" { - content = module.bootkube.etcd_server_key + content = module.bootstrap.etcd_server_key destination = "$HOME/etcd-server.key" } provisioner "file" { - content = module.bootkube.etcd_peer_cert + content = module.bootstrap.etcd_peer_cert destination = "$HOME/etcd-peer.crt" } provisioner "file" { - content = module.bootkube.etcd_peer_key + content = module.bootstrap.etcd_peer_key destination = "$HOME/etcd-peer.key" } @@ -93,7 +93,7 @@ resource "null_resource" "copy-worker-secrets" { } provisioner "file" { - content = module.bootkube.kubeconfig-kubelet + content = module.bootstrap.kubeconfig-kubelet destination = "$HOME/kubeconfig" } diff --git a/docs/advanced/customization.md b/docs/advanced/customization.md index e4ff1a6aa..e9ad8de74 100644 --- a/docs/advanced/customization.md +++ b/docs/advanced/customization.md @@ -147,5 +147,5 @@ module "digital-ocean-nemo" { } ``` -To customize lower-level Kubernetes control plane bootstrapping, see the [poseidon/terraform-render-bootkube](https://github.com/poseidon/terraform-render-bootkube) Terraform module. +To customize low-level Kubernetes control plane bootstrapping, see the [poseidon/terraform-render-bootstrap](https://github.com/poseidon/terraform-render-bootstrap) Terraform module. diff --git a/google-cloud/container-linux/kubernetes/bootkube.tf b/google-cloud/container-linux/kubernetes/bootkube.tf index 4ad35381a..d3c81bf05 100644 --- a/google-cloud/container-linux/kubernetes/bootkube.tf +++ b/google-cloud/container-linux/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) -module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=6e59af71138bc5f784453873074de16e7ee150eb" +module "bootstrap" { + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=6e59af71138bc5f784453873074de16e7ee150eb" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/google-cloud/container-linux/kubernetes/controllers.tf b/google-cloud/container-linux/kubernetes/controllers.tf index 06a921a66..893a324b2 100644 --- a/google-cloud/container-linux/kubernetes/controllers.tf +++ b/google-cloud/container-linux/kubernetes/controllers.tf @@ -85,7 +85,7 @@ data "template_file" "controller-configs" { etcd_domain = "${var.cluster_name}-etcd${count.index}.${var.dns_zone}" # etcd0=https://cluster-etcd0.example.com,etcd1=https://cluster-etcd1.example.com,... etcd_initial_cluster = join(",", data.template_file.etcds.*.rendered) - kubeconfig = indent(10, module.bootkube.kubeconfig-kubelet) + kubeconfig = indent(10, module.bootstrap.kubeconfig-kubelet) ssh_authorized_key = var.ssh_authorized_key cluster_dns_service_ip = cidrhost(var.service_cidr, 10) cluster_domain_suffix = var.cluster_domain_suffix diff --git a/google-cloud/container-linux/kubernetes/outputs.tf b/google-cloud/container-linux/kubernetes/outputs.tf index 5c6c889bd..0de725025 100644 --- a/google-cloud/container-linux/kubernetes/outputs.tf +++ b/google-cloud/container-linux/kubernetes/outputs.tf @@ -1,5 +1,5 @@ output "kubeconfig-admin" { - value = module.bootkube.kubeconfig-admin + value = module.bootstrap.kubeconfig-admin } # Outputs for Kubernetes Ingress @@ -21,7 +21,7 @@ output "network_name" { } output "kubeconfig" { - value = module.bootkube.kubeconfig-kubelet + value = module.bootstrap.kubeconfig-kubelet } # Outputs for custom firewalling diff --git a/google-cloud/container-linux/kubernetes/ssh.tf b/google-cloud/container-linux/kubernetes/ssh.tf index f0e101340..b4229674b 100644 --- a/google-cloud/container-linux/kubernetes/ssh.tf +++ b/google-cloud/container-linux/kubernetes/ssh.tf @@ -3,7 +3,7 @@ resource "null_resource" "copy-controller-secrets" { count = var.controller_count depends_on = [ - module.bootkube, + module.bootstrap, ] connection { @@ -14,37 +14,37 @@ resource "null_resource" "copy-controller-secrets" { } provisioner "file" { - content = module.bootkube.etcd_ca_cert + content = module.bootstrap.etcd_ca_cert destination = "$HOME/etcd-client-ca.crt" } provisioner "file" { - content = module.bootkube.etcd_client_cert + content = module.bootstrap.etcd_client_cert destination = "$HOME/etcd-client.crt" } provisioner "file" { - content = module.bootkube.etcd_client_key + content = module.bootstrap.etcd_client_key destination = "$HOME/etcd-client.key" } provisioner "file" { - content = module.bootkube.etcd_server_cert + content = module.bootstrap.etcd_server_cert destination = "$HOME/etcd-server.crt" } provisioner "file" { - content = module.bootkube.etcd_server_key + content = module.bootstrap.etcd_server_key destination = "$HOME/etcd-server.key" } provisioner "file" { - content = module.bootkube.etcd_peer_cert + content = module.bootstrap.etcd_peer_cert destination = "$HOME/etcd-peer.crt" } provisioner "file" { - content = module.bootkube.etcd_peer_key + content = module.bootstrap.etcd_peer_key destination = "$HOME/etcd-peer.key" } diff --git a/google-cloud/container-linux/kubernetes/workers.tf b/google-cloud/container-linux/kubernetes/workers.tf index 0c76ae712..eef186687 100644 --- a/google-cloud/container-linux/kubernetes/workers.tf +++ b/google-cloud/container-linux/kubernetes/workers.tf @@ -13,7 +13,7 @@ module "workers" { preemptible = var.worker_preemptible # configuration - kubeconfig = module.bootkube.kubeconfig-kubelet + kubeconfig = module.bootstrap.kubeconfig-kubelet ssh_authorized_key = var.ssh_authorized_key service_cidr = var.service_cidr cluster_domain_suffix = var.cluster_domain_suffix From fd12f3612bc63e5782b1f36c8f01e61203f30b5c Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Sat, 14 Sep 2019 16:54:08 -0700 Subject: [PATCH 214/523] Rename CA organization from bootkube to typhoon * Rename the organization in generated CA certificates from bootkube to typhoon. Avoid confusion with the bootkube project * https://github.com/poseidon/terraform-render-bootstrap/pull/149 --- aws/container-linux/kubernetes/bootkube.tf | 2 +- aws/fedora-coreos/kubernetes/bootkube.tf | 2 +- azure/container-linux/kubernetes/bootkube.tf | 2 +- bare-metal/container-linux/kubernetes/bootkube.tf | 2 +- bare-metal/fedora-coreos/kubernetes/bootkube.tf | 2 +- digital-ocean/container-linux/kubernetes/bootkube.tf | 2 +- google-cloud/container-linux/kubernetes/bootkube.tf | 2 +- 7 files changed, 7 insertions(+), 7 deletions(-) diff --git a/aws/container-linux/kubernetes/bootkube.tf b/aws/container-linux/kubernetes/bootkube.tf index 53a45edd1..0a87eb4a7 100644 --- a/aws/container-linux/kubernetes/bootkube.tf +++ b/aws/container-linux/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=6e59af71138bc5f784453873074de16e7ee150eb" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=d6206abedd69aca2e362bf533f73b13805ea344c" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/aws/fedora-coreos/kubernetes/bootkube.tf b/aws/fedora-coreos/kubernetes/bootkube.tf index 26ea352e3..21ceb94a9 100644 --- a/aws/fedora-coreos/kubernetes/bootkube.tf +++ b/aws/fedora-coreos/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=6e59af71138bc5f784453873074de16e7ee150eb" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=d6206abedd69aca2e362bf533f73b13805ea344c" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/azure/container-linux/kubernetes/bootkube.tf b/azure/container-linux/kubernetes/bootkube.tf index d5983f850..d42f20981 100644 --- a/azure/container-linux/kubernetes/bootkube.tf +++ b/azure/container-linux/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=6e59af71138bc5f784453873074de16e7ee150eb" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=d6206abedd69aca2e362bf533f73b13805ea344c" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/bare-metal/container-linux/kubernetes/bootkube.tf b/bare-metal/container-linux/kubernetes/bootkube.tf index 9c3bc841f..71a14f907 100644 --- a/bare-metal/container-linux/kubernetes/bootkube.tf +++ b/bare-metal/container-linux/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=6e59af71138bc5f784453873074de16e7ee150eb" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=d6206abedd69aca2e362bf533f73b13805ea344c" cluster_name = var.cluster_name api_servers = [var.k8s_domain_name] diff --git a/bare-metal/fedora-coreos/kubernetes/bootkube.tf b/bare-metal/fedora-coreos/kubernetes/bootkube.tf index 0c49932fe..453dbeab8 100644 --- a/bare-metal/fedora-coreos/kubernetes/bootkube.tf +++ b/bare-metal/fedora-coreos/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=6e59af71138bc5f784453873074de16e7ee150eb" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=d6206abedd69aca2e362bf533f73b13805ea344c" cluster_name = var.cluster_name api_servers = [var.k8s_domain_name] diff --git a/digital-ocean/container-linux/kubernetes/bootkube.tf b/digital-ocean/container-linux/kubernetes/bootkube.tf index e3ed992eb..1df36a63f 100644 --- a/digital-ocean/container-linux/kubernetes/bootkube.tf +++ b/digital-ocean/container-linux/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=6e59af71138bc5f784453873074de16e7ee150eb" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=d6206abedd69aca2e362bf533f73b13805ea344c" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/google-cloud/container-linux/kubernetes/bootkube.tf b/google-cloud/container-linux/kubernetes/bootkube.tf index d3c81bf05..94ad8e494 100644 --- a/google-cloud/container-linux/kubernetes/bootkube.tf +++ b/google-cloud/container-linux/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=6e59af71138bc5f784453873074de16e7ee150eb" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=d6206abedd69aca2e362bf533f73b13805ea344c" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] From 9da3725738bd3cae694878ba938adec5a8672ea7 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Tue, 17 Sep 2019 21:24:30 -0700 Subject: [PATCH 215/523] Update Kubernetes from v1.15.3 to v1.16.0 * Drop `node-role.kubernetes.io/master` and `node-role.kubernetes.io/node` node labels * Kubelet (v1.16) now rejects the node labels used in the kubectl get nodes ROLES output * https://github.com/kubernetes/kubernetes/issues/75457 --- CHANGES.md | 7 ++++++- README.md | 10 +++++----- addons/nginx-ingress/aws/deployment.yaml | 2 -- addons/nginx-ingress/azure/deployment.yaml | 2 -- addons/nginx-ingress/digital-ocean/daemonset.yaml | 2 -- addons/nginx-ingress/google-cloud/deployment.yaml | 2 -- aws/container-linux/kubernetes/README.md | 2 +- aws/container-linux/kubernetes/bootkube.tf | 2 +- .../kubernetes/cl/controller.yaml.tmpl | 8 ++++---- .../kubernetes/workers/cl/worker.yaml.tmpl | 6 +++--- aws/fedora-coreos/kubernetes/README.md | 2 +- aws/fedora-coreos/kubernetes/bootkube.tf | 2 +- aws/fedora-coreos/kubernetes/fcc/controller.yaml | 8 ++++---- .../kubernetes/workers/fcc/worker.yaml | 4 ++-- azure/container-linux/kubernetes/README.md | 2 +- azure/container-linux/kubernetes/bootkube.tf | 2 +- .../kubernetes/cl/controller.yaml.tmpl | 8 ++++---- .../kubernetes/workers/cl/worker.yaml.tmpl | 6 +++--- bare-metal/container-linux/kubernetes/README.md | 2 +- bare-metal/container-linux/kubernetes/bootkube.tf | 2 +- .../kubernetes/cl/controller.yaml.tmpl | 8 ++++---- .../container-linux/kubernetes/cl/worker.yaml.tmpl | 4 ++-- bare-metal/fedora-coreos/kubernetes/README.md | 2 +- bare-metal/fedora-coreos/kubernetes/bootkube.tf | 2 +- .../fedora-coreos/kubernetes/fcc/controller.yaml | 8 ++++---- .../fedora-coreos/kubernetes/fcc/worker.yaml | 4 ++-- digital-ocean/container-linux/kubernetes/README.md | 2 +- .../container-linux/kubernetes/bootkube.tf | 2 +- .../kubernetes/cl/controller.yaml.tmpl | 8 ++++---- .../container-linux/kubernetes/cl/worker.yaml.tmpl | 6 +++--- docs/advanced/worker-pools.md | 14 +++++++------- docs/cl/aws.md | 10 +++++----- docs/cl/azure.md | 10 +++++----- docs/cl/bare-metal.md | 14 +++++++------- docs/cl/digital-ocean.md | 10 +++++----- docs/cl/google-cloud.md | 10 +++++----- docs/fedora-coreos/aws.md | 8 ++++---- docs/fedora-coreos/bare-metal.md | 8 ++++---- docs/index.md | 10 +++++----- docs/topics/maintenance.md | 10 +++++----- google-cloud/container-linux/kubernetes/README.md | 2 +- .../container-linux/kubernetes/bootkube.tf | 2 +- .../kubernetes/cl/controller.yaml.tmpl | 8 ++++---- .../kubernetes/workers/cl/worker.yaml.tmpl | 6 +++--- 44 files changed, 123 insertions(+), 126 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 70b48218c..51a229dcc 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -4,10 +4,15 @@ Notable changes between versions. ## Latest +## v1.16.0 + +* Kubernetes [v1.16.0](https://github.com/kubernetes/kubernetes/blob/master/CHANGELOG-1.16.md#v1160) ([#543](https://github.com/poseidon/typhoon/pull/543)) + * Read about several Kubernetes API [deprecations](https://github.com/kubernetes/kubernetes/blob/master/CHANGELOG-1.16.md#deprecations-and-removals)! + * Rename `node-role.kubernetes.io` labels for `master` and `node` roles (no longer shown in `kubectl get nodes`) * Migrate control plane from self-hosted to static pods ([#536](https://github.com/poseidon/typhoon/pull/536)) * Run `kube-apiserver`, `kube-scheduler`, and `kube-controller-manager` as static pods on each controller * `kubectl` edits to `kube-apiserver`, `kube-scheduler`, and `kube-controller-manager` are no longer possible (change) - * Remove [bootkube](https://github.com/kubernetes-incubator/bootkube), self-hosted pivot, and `pod-checkpointer` + * Remove bootkube, self-hosted pivot, and `pod-checkpointer` * Update CoreDNS from v1.5.0 to v1.6.2 ([#535](https://github.com/poseidon/typhoon/pull/535)) * Update etcd from v3.3.15 to [v3.4.0](https://github.com/etcd-io/etcd/releases/tag/v3.4.0) * Recommend updating `terraform-provider-ct` plugin from v0.3.2 to [v0.4.0](https://github.com/poseidon/terraform-provider-ct/releases/tag/v0.4.0) diff --git a/README.md b/README.md index dee650356..130b06b61 100644 --- a/README.md +++ b/README.md @@ -11,7 +11,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features -* Kubernetes v1.15.3 (upstream) +* Kubernetes v1.16.0 (upstream) * Single or multi-master, [Calico](https://www.projectcalico.org/) or [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/) * Advanced features like [worker pools](https://typhoon.psdn.io/advanced/worker-pools/), [preemptible](https://typhoon.psdn.io/cl/google-cloud/#preemption) workers, and [snippets](https://typhoon.psdn.io/advanced/customization/#container-linux) customization @@ -48,7 +48,7 @@ Define a Kubernetes cluster by using the Terraform module for your chosen platfo ```tf module "google-cloud-yavin" { - source = "git::https://github.com/poseidon/typhoon//google-cloud/container-linux/kubernetes?ref=v1.15.3" + source = "git::https://github.com/poseidon/typhoon//google-cloud/container-linux/kubernetes?ref=v1.16.0" # Google Cloud cluster_name = "yavin" @@ -82,9 +82,9 @@ In 4-8 minutes (varies by platform), the cluster will be ready. This Google Clou $ export KUBECONFIG=/home/user/.secrets/clusters/yavin/auth/kubeconfig $ kubectl get nodes NAME ROLES STATUS AGE VERSION -yavin-controller-0.c.example-com.internal controller,master Ready 6m v1.15.3 -yavin-worker-jrbf.c.example-com.internal node Ready 5m v1.15.3 -yavin-worker-mzdm.c.example-com.internal node Ready 5m v1.15.3 +yavin-controller-0.c.example-com.internal controller,master Ready 6m v1.16.0 +yavin-worker-jrbf.c.example-com.internal node Ready 5m v1.16.0 +yavin-worker-mzdm.c.example-com.internal node Ready 5m v1.16.0 ``` List the pods. diff --git a/addons/nginx-ingress/aws/deployment.yaml b/addons/nginx-ingress/aws/deployment.yaml index 16dce5a39..621c0e4d2 100644 --- a/addons/nginx-ingress/aws/deployment.yaml +++ b/addons/nginx-ingress/aws/deployment.yaml @@ -20,8 +20,6 @@ spec: annotations: seccomp.security.alpha.kubernetes.io/pod: 'docker/default' spec: - nodeSelector: - node-role.kubernetes.io/node: "" containers: - name: nginx-ingress-controller image: quay.io/kubernetes-ingress-controller/nginx-ingress-controller:0.25.1 diff --git a/addons/nginx-ingress/azure/deployment.yaml b/addons/nginx-ingress/azure/deployment.yaml index 16dce5a39..621c0e4d2 100644 --- a/addons/nginx-ingress/azure/deployment.yaml +++ b/addons/nginx-ingress/azure/deployment.yaml @@ -20,8 +20,6 @@ spec: annotations: seccomp.security.alpha.kubernetes.io/pod: 'docker/default' spec: - nodeSelector: - node-role.kubernetes.io/node: "" containers: - name: nginx-ingress-controller image: quay.io/kubernetes-ingress-controller/nginx-ingress-controller:0.25.1 diff --git a/addons/nginx-ingress/digital-ocean/daemonset.yaml b/addons/nginx-ingress/digital-ocean/daemonset.yaml index fff2970c8..21abeb716 100644 --- a/addons/nginx-ingress/digital-ocean/daemonset.yaml +++ b/addons/nginx-ingress/digital-ocean/daemonset.yaml @@ -20,8 +20,6 @@ spec: annotations: seccomp.security.alpha.kubernetes.io/pod: 'docker/default' spec: - nodeSelector: - node-role.kubernetes.io/node: "" containers: - name: nginx-ingress-controller image: quay.io/kubernetes-ingress-controller/nginx-ingress-controller:0.25.1 diff --git a/addons/nginx-ingress/google-cloud/deployment.yaml b/addons/nginx-ingress/google-cloud/deployment.yaml index 16dce5a39..621c0e4d2 100644 --- a/addons/nginx-ingress/google-cloud/deployment.yaml +++ b/addons/nginx-ingress/google-cloud/deployment.yaml @@ -20,8 +20,6 @@ spec: annotations: seccomp.security.alpha.kubernetes.io/pod: 'docker/default' spec: - nodeSelector: - node-role.kubernetes.io/node: "" containers: - name: nginx-ingress-controller image: quay.io/kubernetes-ingress-controller/nginx-ingress-controller:0.25.1 diff --git a/aws/container-linux/kubernetes/README.md b/aws/container-linux/kubernetes/README.md index 8dc9fd85a..bb495cb39 100644 --- a/aws/container-linux/kubernetes/README.md +++ b/aws/container-linux/kubernetes/README.md @@ -11,7 +11,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features -* Kubernetes v1.15.3 (upstream) +* Kubernetes v1.16.0 (upstream) * Single or multi-master, [Calico](https://www.projectcalico.org/) or [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/) * Advanced features like [worker pools](https://typhoon.psdn.io/advanced/worker-pools/), [spot](https://typhoon.psdn.io/cl/aws/#spot) workers, and [snippets](https://typhoon.psdn.io/advanced/customization/#container-linux) customization diff --git a/aws/container-linux/kubernetes/bootkube.tf b/aws/container-linux/kubernetes/bootkube.tf index 0a87eb4a7..53783725c 100644 --- a/aws/container-linux/kubernetes/bootkube.tf +++ b/aws/container-linux/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=d6206abedd69aca2e362bf533f73b13805ea344c" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=539b725093c8cd94ba46603adb25ac5280562ec8" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/aws/container-linux/kubernetes/cl/controller.yaml.tmpl b/aws/container-linux/kubernetes/cl/controller.yaml.tmpl index e1776eb36..4128ecf5c 100644 --- a/aws/container-linux/kubernetes/cl/controller.yaml.tmpl +++ b/aws/container-linux/kubernetes/cl/controller.yaml.tmpl @@ -87,8 +87,8 @@ systemd: --kubeconfig=/etc/kubernetes/kubeconfig \ --lock-file=/var/run/lock/kubelet.lock \ --network-plugin=cni \ - --node-labels=node-role.kubernetes.io/master \ - --node-labels=node-role.kubernetes.io/controller="true" \ + --node-labels=node.kubernetes.io/master \ + --node-labels=node.kubernetes.io/controller="true" \ --pod-manifest-path=/etc/kubernetes/manifests \ --read-only-port=0 \ --register-with-taints=node-role.kubernetes.io/master=:NoSchedule \ @@ -115,7 +115,7 @@ systemd: --volume script,kind=host,source=/opt/bootstrap/apply \ --mount volume=script,target=/apply \ --insecure-options=image \ - docker://k8s.gcr.io/hyperkube:v1.15.3 \ + docker://k8s.gcr.io/hyperkube:v1.16.0 \ --net=host \ --dns=host \ --exec=/apply @@ -136,7 +136,7 @@ storage: contents: inline: | KUBELET_IMAGE_URL=docker://k8s.gcr.io/hyperkube - KUBELET_IMAGE_TAG=v1.15.3 + KUBELET_IMAGE_TAG=v1.16.0 - path: /opt/bootstrap/apply filesystem: root mode: 0544 diff --git a/aws/container-linux/kubernetes/workers/cl/worker.yaml.tmpl b/aws/container-linux/kubernetes/workers/cl/worker.yaml.tmpl index 6da106809..01dd15557 100644 --- a/aws/container-linux/kubernetes/workers/cl/worker.yaml.tmpl +++ b/aws/container-linux/kubernetes/workers/cl/worker.yaml.tmpl @@ -60,7 +60,7 @@ systemd: --kubeconfig=/etc/kubernetes/kubeconfig \ --lock-file=/var/run/lock/kubelet.lock \ --network-plugin=cni \ - --node-labels=node-role.kubernetes.io/node \ + --node-labels=node.kubernetes.io/node \ --pod-manifest-path=/etc/kubernetes/manifests \ --read-only-port=0 \ --volume-plugin-dir=/var/lib/kubelet/volumeplugins @@ -95,7 +95,7 @@ storage: contents: inline: | KUBELET_IMAGE_URL=docker://k8s.gcr.io/hyperkube - KUBELET_IMAGE_TAG=v1.15.3 + KUBELET_IMAGE_TAG=v1.16.0 - path: /etc/sysctl.d/max-user-watches.conf filesystem: root contents: @@ -113,7 +113,7 @@ storage: --volume config,kind=host,source=/etc/kubernetes \ --mount volume=config,target=/etc/kubernetes \ --insecure-options=image \ - docker://k8s.gcr.io/hyperkube:v1.15.3 \ + docker://k8s.gcr.io/hyperkube:v1.16.0 \ --net=host \ --dns=host \ --exec=/kubectl -- --kubeconfig=/etc/kubernetes/kubeconfig delete node $(hostname) diff --git a/aws/fedora-coreos/kubernetes/README.md b/aws/fedora-coreos/kubernetes/README.md index 33421e84e..11361bfb2 100644 --- a/aws/fedora-coreos/kubernetes/README.md +++ b/aws/fedora-coreos/kubernetes/README.md @@ -11,7 +11,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features -* Kubernetes v1.15.3 (upstream) +* Kubernetes v1.16.0 (upstream) * Single or multi-master, [Calico](https://www.projectcalico.org/) or [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/) * Advanced features like [worker pools](https://typhoon.psdn.io/advanced/worker-pools/), [spot](https://typhoon.psdn.io/cl/aws/#spot) workers, and [snippets](https://typhoon.psdn.io/advanced/customization/#container-linux) customization diff --git a/aws/fedora-coreos/kubernetes/bootkube.tf b/aws/fedora-coreos/kubernetes/bootkube.tf index 21ceb94a9..4469d5146 100644 --- a/aws/fedora-coreos/kubernetes/bootkube.tf +++ b/aws/fedora-coreos/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=d6206abedd69aca2e362bf533f73b13805ea344c" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=539b725093c8cd94ba46603adb25ac5280562ec8" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/aws/fedora-coreos/kubernetes/fcc/controller.yaml b/aws/fedora-coreos/kubernetes/fcc/controller.yaml index f309a2981..b0333e939 100644 --- a/aws/fedora-coreos/kubernetes/fcc/controller.yaml +++ b/aws/fedora-coreos/kubernetes/fcc/controller.yaml @@ -80,7 +80,7 @@ systemd: --volume /var/run:/var/run \ --volume /var/run/lock:/var/run/lock:z \ --volume /opt/cni/bin:/opt/cni/bin:z \ - k8s.gcr.io/hyperkube:v1.15.3 /hyperkube kubelet \ + k8s.gcr.io/hyperkube:v1.16.0 /hyperkube kubelet \ --anonymous-auth=false \ --authentication-token-webhook \ --authorization-mode=Webhook \ @@ -95,8 +95,8 @@ systemd: --kubeconfig=/etc/kubernetes/kubeconfig \ --lock-file=/var/run/lock/kubelet.lock \ --network-plugin=cni \ - --node-labels=node-role.kubernetes.io/master \ - --node-labels=node-role.kubernetes.io/controller="true" \ + --node-labels=node.kubernetes.io/master \ + --node-labels=node.kubernetes.io/controller="true" \ --pod-manifest-path=/etc/kubernetes/manifests \ --read-only-port=0 \ --register-with-taints=node-role.kubernetes.io/master=:NoSchedule \ @@ -121,7 +121,7 @@ systemd: --network host \ --volume /opt/bootstrap/assets:/assets:ro,Z \ --volume /opt/bootstrap/apply:/apply:ro,Z \ - k8s.gcr.io/hyperkube:v1.15.3 \ + k8s.gcr.io/hyperkube:v1.16.0 \ /apply ExecStartPost=/bin/touch /opt/bootstrap/bootstrap.done ExecStartPost=-/usr/bin/podman stop bootstrap diff --git a/aws/fedora-coreos/kubernetes/workers/fcc/worker.yaml b/aws/fedora-coreos/kubernetes/workers/fcc/worker.yaml index 4cdcd4627..6973fbf08 100644 --- a/aws/fedora-coreos/kubernetes/workers/fcc/worker.yaml +++ b/aws/fedora-coreos/kubernetes/workers/fcc/worker.yaml @@ -50,7 +50,7 @@ systemd: --volume /var/run:/var/run \ --volume /var/run/lock:/var/run/lock:z \ --volume /opt/cni/bin:/opt/cni/bin:z \ - k8s.gcr.io/hyperkube:v1.15.3 /hyperkube kubelet \ + k8s.gcr.io/hyperkube:v1.16.0 /hyperkube kubelet \ --anonymous-auth=false \ --authentication-token-webhook \ --authorization-mode=Webhook \ @@ -65,7 +65,7 @@ systemd: --kubeconfig=/etc/kubernetes/kubeconfig \ --lock-file=/var/run/lock/kubelet.lock \ --network-plugin=cni \ - --node-labels=node-role.kubernetes.io/node \ + --node-labels=node.kubernetes.io/node \ --pod-manifest-path=/etc/kubernetes/manifests \ --read-only-port=0 \ --volume-plugin-dir=/var/lib/kubelet/volumeplugins diff --git a/azure/container-linux/kubernetes/README.md b/azure/container-linux/kubernetes/README.md index 56d2f9e5c..873615197 100644 --- a/azure/container-linux/kubernetes/README.md +++ b/azure/container-linux/kubernetes/README.md @@ -11,7 +11,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features -* Kubernetes v1.15.3 (upstream) +* Kubernetes v1.16.0 (upstream) * Single or multi-master, [Calico](https://www.projectcalico.org/) or [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/) * Advanced features like [worker pools](https://typhoon.psdn.io/advanced/worker-pools/), [low-priority](https://typhoon.psdn.io/cl/azure/#low-priority) workers, and [snippets](https://typhoon.psdn.io/advanced/customization/#container-linux) customization diff --git a/azure/container-linux/kubernetes/bootkube.tf b/azure/container-linux/kubernetes/bootkube.tf index d42f20981..52cf18671 100644 --- a/azure/container-linux/kubernetes/bootkube.tf +++ b/azure/container-linux/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=d6206abedd69aca2e362bf533f73b13805ea344c" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=539b725093c8cd94ba46603adb25ac5280562ec8" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/azure/container-linux/kubernetes/cl/controller.yaml.tmpl b/azure/container-linux/kubernetes/cl/controller.yaml.tmpl index 362189a0e..8261ffae8 100644 --- a/azure/container-linux/kubernetes/cl/controller.yaml.tmpl +++ b/azure/container-linux/kubernetes/cl/controller.yaml.tmpl @@ -85,8 +85,8 @@ systemd: --kubeconfig=/etc/kubernetes/kubeconfig \ --lock-file=/var/run/lock/kubelet.lock \ --network-plugin=cni \ - --node-labels=node-role.kubernetes.io/master \ - --node-labels=node-role.kubernetes.io/controller="true" \ + --node-labels=node.kubernetes.io/master \ + --node-labels=node.kubernetes.io/controller="true" \ --pod-manifest-path=/etc/kubernetes/manifests \ --read-only-port=0 \ --register-with-taints=node-role.kubernetes.io/master=:NoSchedule \ @@ -113,7 +113,7 @@ systemd: --volume script,kind=host,source=/opt/bootstrap/apply \ --mount volume=script,target=/apply \ --insecure-options=image \ - docker://k8s.gcr.io/hyperkube:v1.15.3 \ + docker://k8s.gcr.io/hyperkube:v1.16.0 \ --net=host \ --dns=host \ --exec=/apply @@ -134,7 +134,7 @@ storage: contents: inline: | KUBELET_IMAGE_URL=docker://k8s.gcr.io/hyperkube - KUBELET_IMAGE_TAG=v1.15.3 + KUBELET_IMAGE_TAG=v1.16.0 - path: /opt/bootstrap/apply filesystem: root mode: 0544 diff --git a/azure/container-linux/kubernetes/workers/cl/worker.yaml.tmpl b/azure/container-linux/kubernetes/workers/cl/worker.yaml.tmpl index 7db9c32f1..20dcf2d31 100644 --- a/azure/container-linux/kubernetes/workers/cl/worker.yaml.tmpl +++ b/azure/container-linux/kubernetes/workers/cl/worker.yaml.tmpl @@ -58,7 +58,7 @@ systemd: --kubeconfig=/etc/kubernetes/kubeconfig \ --lock-file=/var/run/lock/kubelet.lock \ --network-plugin=cni \ - --node-labels=node-role.kubernetes.io/node \ + --node-labels=node.kubernetes.io/node \ --pod-manifest-path=/etc/kubernetes/manifests \ --read-only-port=0 \ --volume-plugin-dir=/var/lib/kubelet/volumeplugins @@ -93,7 +93,7 @@ storage: contents: inline: | KUBELET_IMAGE_URL=docker://k8s.gcr.io/hyperkube - KUBELET_IMAGE_TAG=v1.15.3 + KUBELET_IMAGE_TAG=v1.16.0 - path: /etc/sysctl.d/max-user-watches.conf filesystem: root contents: @@ -111,7 +111,7 @@ storage: --volume config,kind=host,source=/etc/kubernetes \ --mount volume=config,target=/etc/kubernetes \ --insecure-options=image \ - docker://k8s.gcr.io/hyperkube:v1.15.3 \ + docker://k8s.gcr.io/hyperkube:v1.16.0 \ --net=host \ --dns=host \ --exec=/kubectl -- --kubeconfig=/etc/kubernetes/kubeconfig delete node $(hostname | tr '[:upper:]' '[:lower:]') diff --git a/bare-metal/container-linux/kubernetes/README.md b/bare-metal/container-linux/kubernetes/README.md index 4869b1595..571bd9af7 100644 --- a/bare-metal/container-linux/kubernetes/README.md +++ b/bare-metal/container-linux/kubernetes/README.md @@ -11,7 +11,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features -* Kubernetes v1.15.3 (upstream) +* Kubernetes v1.16.0 (upstream) * Single or multi-master, [Calico](https://www.projectcalico.org/) or [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/) * Advanced features like [snippets](https://typhoon.psdn.io/advanced/customization/#container-linux) customization diff --git a/bare-metal/container-linux/kubernetes/bootkube.tf b/bare-metal/container-linux/kubernetes/bootkube.tf index 71a14f907..06fb187dc 100644 --- a/bare-metal/container-linux/kubernetes/bootkube.tf +++ b/bare-metal/container-linux/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=d6206abedd69aca2e362bf533f73b13805ea344c" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=539b725093c8cd94ba46603adb25ac5280562ec8" cluster_name = var.cluster_name api_servers = [var.k8s_domain_name] diff --git a/bare-metal/container-linux/kubernetes/cl/controller.yaml.tmpl b/bare-metal/container-linux/kubernetes/cl/controller.yaml.tmpl index e3aeeac82..476a08511 100644 --- a/bare-metal/container-linux/kubernetes/cl/controller.yaml.tmpl +++ b/bare-metal/container-linux/kubernetes/cl/controller.yaml.tmpl @@ -100,8 +100,8 @@ systemd: --kubeconfig=/etc/kubernetes/kubeconfig \ --lock-file=/var/run/lock/kubelet.lock \ --network-plugin=cni \ - --node-labels=node-role.kubernetes.io/master \ - --node-labels=node-role.kubernetes.io/controller="true" \ + --node-labels=node.kubernetes.io/master \ + --node-labels=node.kubernetes.io/controller="true" \ --pod-manifest-path=/etc/kubernetes/manifests \ --read-only-port=0 \ --register-with-taints=node-role.kubernetes.io/master=:NoSchedule \ @@ -128,7 +128,7 @@ systemd: --volume script,kind=host,source=/opt/bootstrap/apply \ --mount volume=script,target=/apply \ --insecure-options=image \ - docker://k8s.gcr.io/hyperkube:v1.15.3 \ + docker://k8s.gcr.io/hyperkube:v1.16.0 \ --net=host \ --dns=host \ --exec=/apply @@ -143,7 +143,7 @@ storage: contents: inline: | KUBELET_IMAGE_URL=docker://k8s.gcr.io/hyperkube - KUBELET_IMAGE_TAG=v1.15.3 + KUBELET_IMAGE_TAG=v1.16.0 - path: /etc/hostname filesystem: root mode: 0644 diff --git a/bare-metal/container-linux/kubernetes/cl/worker.yaml.tmpl b/bare-metal/container-linux/kubernetes/cl/worker.yaml.tmpl index 2af4d66a8..c991e9ec4 100644 --- a/bare-metal/container-linux/kubernetes/cl/worker.yaml.tmpl +++ b/bare-metal/container-linux/kubernetes/cl/worker.yaml.tmpl @@ -73,7 +73,7 @@ systemd: --kubeconfig=/etc/kubernetes/kubeconfig \ --lock-file=/var/run/lock/kubelet.lock \ --network-plugin=cni \ - --node-labels=node-role.kubernetes.io/node \ + --node-labels=node.kubernetes.io/node \ --pod-manifest-path=/etc/kubernetes/manifests \ --read-only-port=0 \ --volume-plugin-dir=/var/lib/kubelet/volumeplugins @@ -91,7 +91,7 @@ storage: contents: inline: | KUBELET_IMAGE_URL=docker://k8s.gcr.io/hyperkube - KUBELET_IMAGE_TAG=v1.15.3 + KUBELET_IMAGE_TAG=v1.16.0 - path: /etc/hostname filesystem: root mode: 0644 diff --git a/bare-metal/fedora-coreos/kubernetes/README.md b/bare-metal/fedora-coreos/kubernetes/README.md index 8292c8c62..f28a9898d 100644 --- a/bare-metal/fedora-coreos/kubernetes/README.md +++ b/bare-metal/fedora-coreos/kubernetes/README.md @@ -11,7 +11,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features -* Kubernetes v1.15.3 (upstream) +* Kubernetes v1.16.0 (upstream) * Single or multi-master, [Calico](https://www.projectcalico.org/) or [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/) * Advanced features like [snippets](https://typhoon.psdn.io/advanced/customization/#container-linux) customization diff --git a/bare-metal/fedora-coreos/kubernetes/bootkube.tf b/bare-metal/fedora-coreos/kubernetes/bootkube.tf index 453dbeab8..cf3c9b505 100644 --- a/bare-metal/fedora-coreos/kubernetes/bootkube.tf +++ b/bare-metal/fedora-coreos/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=d6206abedd69aca2e362bf533f73b13805ea344c" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=539b725093c8cd94ba46603adb25ac5280562ec8" cluster_name = var.cluster_name api_servers = [var.k8s_domain_name] diff --git a/bare-metal/fedora-coreos/kubernetes/fcc/controller.yaml b/bare-metal/fedora-coreos/kubernetes/fcc/controller.yaml index 9ebbf6916..68bcc4c0f 100644 --- a/bare-metal/fedora-coreos/kubernetes/fcc/controller.yaml +++ b/bare-metal/fedora-coreos/kubernetes/fcc/controller.yaml @@ -81,7 +81,7 @@ systemd: --volume /opt/cni/bin:/opt/cni/bin:z \ --volume /etc/iscsi:/etc/iscsi \ --volume /sbin/iscsiadm:/sbin/iscsiadm \ - k8s.gcr.io/hyperkube:v1.15.3 /hyperkube kubelet \ + k8s.gcr.io/hyperkube:v1.16.0 /hyperkube kubelet \ --anonymous-auth=false \ --authentication-token-webhook \ --authorization-mode=Webhook \ @@ -97,8 +97,8 @@ systemd: --kubeconfig=/etc/kubernetes/kubeconfig \ --lock-file=/var/run/lock/kubelet.lock \ --network-plugin=cni \ - --node-labels=node-role.kubernetes.io/master \ - --node-labels=node-role.kubernetes.io/controller="true" \ + --node-labels=node.kubernetes.io/master \ + --node-labels=node.kubernetes.io/controller="true" \ --pod-manifest-path=/etc/kubernetes/manifests \ --read-only-port=0 \ --register-with-taints=node-role.kubernetes.io/master=:NoSchedule \ @@ -132,7 +132,7 @@ systemd: --network host \ --volume /opt/bootstrap/assets:/assets:ro,Z \ --volume /opt/bootstrap/apply:/apply:ro,Z \ - k8s.gcr.io/hyperkube:v1.15.3 \ + k8s.gcr.io/hyperkube:v1.16.0 \ /apply ExecStartPost=/bin/touch /opt/bootstrap/bootstrap.done ExecStartPost=-/usr/bin/podman stop bootstrap diff --git a/bare-metal/fedora-coreos/kubernetes/fcc/worker.yaml b/bare-metal/fedora-coreos/kubernetes/fcc/worker.yaml index fc4bb600c..1c1af1fea 100644 --- a/bare-metal/fedora-coreos/kubernetes/fcc/worker.yaml +++ b/bare-metal/fedora-coreos/kubernetes/fcc/worker.yaml @@ -51,7 +51,7 @@ systemd: --volume /opt/cni/bin:/opt/cni/bin:z \ --volume /etc/iscsi:/etc/iscsi \ --volume /sbin/iscsiadm:/sbin/iscsiadm \ - k8s.gcr.io/hyperkube:v1.15.3 /hyperkube kubelet \ + k8s.gcr.io/hyperkube:v1.16.0 /hyperkube kubelet \ --anonymous-auth=false \ --authentication-token-webhook \ --authorization-mode=Webhook \ @@ -67,7 +67,7 @@ systemd: --kubeconfig=/etc/kubernetes/kubeconfig \ --lock-file=/var/run/lock/kubelet.lock \ --network-plugin=cni \ - --node-labels=node-role.kubernetes.io/node \ + --node-labels=node.kubernetes.io/node \ --pod-manifest-path=/etc/kubernetes/manifests \ --read-only-port=0 \ --volume-plugin-dir=/var/lib/kubelet/volumeplugins diff --git a/digital-ocean/container-linux/kubernetes/README.md b/digital-ocean/container-linux/kubernetes/README.md index 0d5146cbc..ad096cf95 100644 --- a/digital-ocean/container-linux/kubernetes/README.md +++ b/digital-ocean/container-linux/kubernetes/README.md @@ -11,7 +11,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features -* Kubernetes v1.15.3 (upstream) +* Kubernetes v1.16.0 (upstream) * Single or multi-master, [Calico](https://www.projectcalico.org/) or [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/) * Advanced features like [snippets](https://typhoon.psdn.io/advanced/customization/#container-linux) customization diff --git a/digital-ocean/container-linux/kubernetes/bootkube.tf b/digital-ocean/container-linux/kubernetes/bootkube.tf index 1df36a63f..ee794a92e 100644 --- a/digital-ocean/container-linux/kubernetes/bootkube.tf +++ b/digital-ocean/container-linux/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=d6206abedd69aca2e362bf533f73b13805ea344c" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=539b725093c8cd94ba46603adb25ac5280562ec8" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/digital-ocean/container-linux/kubernetes/cl/controller.yaml.tmpl b/digital-ocean/container-linux/kubernetes/cl/controller.yaml.tmpl index 066780600..9fe9aaace 100644 --- a/digital-ocean/container-linux/kubernetes/cl/controller.yaml.tmpl +++ b/digital-ocean/container-linux/kubernetes/cl/controller.yaml.tmpl @@ -97,8 +97,8 @@ systemd: --kubeconfig=/etc/kubernetes/kubeconfig \ --lock-file=/var/run/lock/kubelet.lock \ --network-plugin=cni \ - --node-labels=node-role.kubernetes.io/master \ - --node-labels=node-role.kubernetes.io/controller="true" \ + --node-labels=node.kubernetes.io/master \ + --node-labels=node.kubernetes.io/controller="true" \ --pod-manifest-path=/etc/kubernetes/manifests \ --read-only-port=0 \ --register-with-taints=node-role.kubernetes.io/master=:NoSchedule \ @@ -125,7 +125,7 @@ systemd: --volume script,kind=host,source=/opt/bootstrap/apply \ --mount volume=script,target=/apply \ --insecure-options=image \ - docker://k8s.gcr.io/hyperkube:v1.15.3 \ + docker://k8s.gcr.io/hyperkube:v1.16.0 \ --net=host \ --dns=host \ --exec=/apply @@ -140,7 +140,7 @@ storage: contents: inline: | KUBELET_IMAGE_URL=docker://k8s.gcr.io/hyperkube - KUBELET_IMAGE_TAG=v1.15.3 + KUBELET_IMAGE_TAG=v1.16.0 - path: /opt/bootstrap/apply filesystem: root mode: 0544 diff --git a/digital-ocean/container-linux/kubernetes/cl/worker.yaml.tmpl b/digital-ocean/container-linux/kubernetes/cl/worker.yaml.tmpl index 130deed24..2dc571cd8 100644 --- a/digital-ocean/container-linux/kubernetes/cl/worker.yaml.tmpl +++ b/digital-ocean/container-linux/kubernetes/cl/worker.yaml.tmpl @@ -70,7 +70,7 @@ systemd: --kubeconfig=/etc/kubernetes/kubeconfig \ --lock-file=/var/run/lock/kubelet.lock \ --network-plugin=cni \ - --node-labels=node-role.kubernetes.io/node \ + --node-labels=node.kubernetes.io/node \ --pod-manifest-path=/etc/kubernetes/manifests \ --read-only-port=0 \ --volume-plugin-dir=/var/lib/kubelet/volumeplugins @@ -99,7 +99,7 @@ storage: contents: inline: | KUBELET_IMAGE_URL=docker://k8s.gcr.io/hyperkube - KUBELET_IMAGE_TAG=v1.15.3 + KUBELET_IMAGE_TAG=v1.16.0 - path: /etc/sysctl.d/max-user-watches.conf filesystem: root contents: @@ -117,7 +117,7 @@ storage: --volume config,kind=host,source=/etc/kubernetes \ --mount volume=config,target=/etc/kubernetes \ --insecure-options=image \ - docker://k8s.gcr.io/hyperkube:v1.15.3 \ + docker://k8s.gcr.io/hyperkube:v1.16.0 \ --net=host \ --dns=host \ --exec=/kubectl -- --kubeconfig=/etc/kubernetes/kubeconfig delete node $(hostname) diff --git a/docs/advanced/worker-pools.md b/docs/advanced/worker-pools.md index 8a1f71024..f3430cc48 100644 --- a/docs/advanced/worker-pools.md +++ b/docs/advanced/worker-pools.md @@ -76,7 +76,7 @@ Create a cluster following the Azure [tutorial](../cl/azure.md#cluster). Define ```tf module "ramius-worker-pool" { - source = "git::https://github.com/poseidon/typhoon//azure/container-linux/kubernetes/workers?ref=v1.15.3" + source = "git::https://github.com/poseidon/typhoon//azure/container-linux/kubernetes/workers?ref=v1.16.0" # Azure region = module.azure-ramius.region @@ -142,7 +142,7 @@ Create a cluster following the Google Cloud [tutorial](../cl/google-cloud.md#clu ```tf module "yavin-worker-pool" { - source = "git::https://github.com/poseidon/typhoon//google-cloud/container-linux/kubernetes/workers?ref=v1.15.3" + source = "git::https://github.com/poseidon/typhoon//google-cloud/container-linux/kubernetes/workers?ref=v1.16.0" # Google Cloud region = "europe-west2" @@ -173,11 +173,11 @@ Verify a managed instance group of workers joins the cluster within a few minute ``` $ kubectl get nodes NAME STATUS AGE VERSION -yavin-controller-0.c.example-com.internal Ready 6m v1.15.3 -yavin-worker-jrbf.c.example-com.internal Ready 5m v1.15.3 -yavin-worker-mzdm.c.example-com.internal Ready 5m v1.15.3 -yavin-16x-worker-jrbf.c.example-com.internal Ready 3m v1.15.3 -yavin-16x-worker-mzdm.c.example-com.internal Ready 3m v1.15.3 +yavin-controller-0.c.example-com.internal Ready 6m v1.16.0 +yavin-worker-jrbf.c.example-com.internal Ready 5m v1.16.0 +yavin-worker-mzdm.c.example-com.internal Ready 5m v1.16.0 +yavin-16x-worker-jrbf.c.example-com.internal Ready 3m v1.16.0 +yavin-16x-worker-mzdm.c.example-com.internal Ready 3m v1.16.0 ``` ### Variables diff --git a/docs/cl/aws.md b/docs/cl/aws.md index 90f4f579e..6577feb65 100644 --- a/docs/cl/aws.md +++ b/docs/cl/aws.md @@ -1,6 +1,6 @@ # AWS -In this tutorial, we'll create a Kubernetes v1.15.3 cluster on AWS with Container Linux. +In this tutorial, we'll create a Kubernetes v1.16.0 cluster on AWS with Container Linux. We'll declare a Kubernetes cluster using the Typhoon Terraform module. Then apply the changes to create a VPC, gateway, subnets, security groups, controller instances, worker auto-scaling group, network load balancer, and TLS assets. @@ -70,7 +70,7 @@ Define a Kubernetes cluster using the module `aws/container-linux/kubernetes`. ```tf module "tempest" { - source = "git::https://github.com/poseidon/typhoon//aws/container-linux/kubernetes?ref=v1.15.3" + source = "git::https://github.com/poseidon/typhoon//aws/container-linux/kubernetes?ref=v1.16.0" # AWS cluster_name = "tempest" @@ -135,9 +135,9 @@ In 4-8 minutes, the Kubernetes cluster will be ready. $ export KUBECONFIG=/home/user/.secrets/clusters/tempest/auth/kubeconfig $ kubectl get nodes NAME STATUS ROLES AGE VERSION -ip-10-0-3-155 Ready controller,master 10m v1.15.3 -ip-10-0-26-65 Ready node 10m v1.15.3 -ip-10-0-41-21 Ready node 10m v1.15.3 +ip-10-0-3-155 Ready controller,master 10m v1.16.0 +ip-10-0-26-65 Ready node 10m v1.16.0 +ip-10-0-41-21 Ready node 10m v1.16.0 ``` List the pods. diff --git a/docs/cl/azure.md b/docs/cl/azure.md index a02611843..70ef1f2c9 100644 --- a/docs/cl/azure.md +++ b/docs/cl/azure.md @@ -3,7 +3,7 @@ !!! danger Typhoon for Azure is alpha. For production, use AWS, Google Cloud, or bare-metal. As Azure matures, check [errata](https://github.com/poseidon/typhoon/wiki/Errata) for known shortcomings. -In this tutorial, we'll create a Kubernetes v1.15.3 cluster on Azure with Container Linux. +In this tutorial, we'll create a Kubernetes v1.16.0 cluster on Azure with Container Linux. We'll declare a Kubernetes cluster using the Typhoon Terraform module. Then apply the changes to create a resource group, virtual network, subnets, security groups, controller availability set, worker scale set, load balancer, and TLS assets. @@ -66,7 +66,7 @@ Define a Kubernetes cluster using the module `azure/container-linux/kubernetes`. ```tf module "ramius" { - source = "git::https://github.com/poseidon/typhoon//azure/container-linux/kubernetes?ref=v1.15.3" + source = "git::https://github.com/poseidon/typhoon//azure/container-linux/kubernetes?ref=v1.16.0" # Azure cluster_name = "ramius" @@ -132,9 +132,9 @@ In 4-8 minutes, the Kubernetes cluster will be ready. $ export KUBECONFIG=/home/user/.secrets/clusters/ramius/auth/kubeconfig $ kubectl get nodes NAME STATUS ROLES AGE VERSION -ramius-controller-0 Ready controller,master 24m v1.15.3 -ramius-worker-000001 Ready node 25m v1.15.3 -ramius-worker-000002 Ready node 24m v1.15.3 +ramius-controller-0 Ready controller,master 24m v1.16.0 +ramius-worker-000001 Ready node 25m v1.16.0 +ramius-worker-000002 Ready node 24m v1.16.0 ``` List the pods. diff --git a/docs/cl/bare-metal.md b/docs/cl/bare-metal.md index c608608ee..d579004a6 100644 --- a/docs/cl/bare-metal.md +++ b/docs/cl/bare-metal.md @@ -1,6 +1,6 @@ # Bare-Metal -In this tutorial, we'll network boot and provision a Kubernetes v1.15.3 cluster on bare-metal with Container Linux. +In this tutorial, we'll network boot and provision a Kubernetes v1.16.0 cluster on bare-metal with Container Linux. First, we'll deploy a [Matchbox](https://github.com/poseidon/matchbox) service and setup a network boot environment. Then, we'll declare a Kubernetes cluster using the Typhoon Terraform module and power on machines. On PXE boot, machines will install Container Linux to disk, reboot into the disk install, and provision themselves as Kubernetes controllers or workers via Ignition. @@ -160,7 +160,7 @@ Define a Kubernetes cluster using the module `bare-metal/container-linux/kuberne ```tf module "bare-metal-mercury" { - source = "git::https://github.com/poseidon/typhoon//bare-metal/container-linux/kubernetes?ref=v1.15.3" + source = "git::https://github.com/poseidon/typhoon//bare-metal/container-linux/kubernetes?ref=v1.16.0" # bare-metal cluster_name = "mercury" @@ -263,9 +263,9 @@ Apply complete! Resources: 55 added, 0 changed, 0 destroyed. To watch the install to disk (until machines reboot from disk), SSH to port 2222. ``` -# before v1.15.3 +# before v1.16.0 $ ssh debug@node1.example.com -# after v1.15.3 +# after v1.16.0 $ ssh -p 2222 core@node1.example.com ``` @@ -289,9 +289,9 @@ systemd[1]: Started Kubernetes control plane. $ export KUBECONFIG=/home/user/.secrets/clusters/mercury/auth/kubeconfig $ kubectl get nodes NAME STATUS ROLES AGE VERSION -node1.example.com Ready controller,master 10m v1.15.3 -node2.example.com Ready node 10m v1.15.3 -node3.example.com Ready node 10m v1.15.3 +node1.example.com Ready controller,master 10m v1.16.0 +node2.example.com Ready node 10m v1.16.0 +node3.example.com Ready node 10m v1.16.0 ``` List the pods. diff --git a/docs/cl/digital-ocean.md b/docs/cl/digital-ocean.md index 3681ff791..73a6653fd 100644 --- a/docs/cl/digital-ocean.md +++ b/docs/cl/digital-ocean.md @@ -1,6 +1,6 @@ # Digital Ocean -In this tutorial, we'll create a Kubernetes v1.15.3 cluster on DigitalOcean with Container Linux. +In this tutorial, we'll create a Kubernetes v1.16.0 cluster on DigitalOcean with Container Linux. We'll declare a Kubernetes cluster using the Typhoon Terraform module. Then apply the changes to create controller droplets, worker droplets, DNS records, tags, and TLS assets. @@ -65,7 +65,7 @@ Define a Kubernetes cluster using the module `digital-ocean/container-linux/kube ```tf module "digital-ocean-nemo" { - source = "git::https://github.com/poseidon/typhoon//digital-ocean/container-linux/kubernetes?ref=v1.15.3" + source = "git::https://github.com/poseidon/typhoon//digital-ocean/container-linux/kubernetes?ref=v1.16.0" # Digital Ocean cluster_name = "nemo" @@ -130,9 +130,9 @@ In 3-6 minutes, the Kubernetes cluster will be ready. $ export KUBECONFIG=/home/user/.secrets/clusters/nemo/auth/kubeconfig $ kubectl get nodes NAME STATUS ROLES AGE VERSION -10.132.110.130 Ready controller,master 10m v1.15.3 -10.132.115.81 Ready node 10m v1.15.3 -10.132.124.107 Ready node 10m v1.15.3 +10.132.110.130 Ready controller,master 10m v1.16.0 +10.132.115.81 Ready node 10m v1.16.0 +10.132.124.107 Ready node 10m v1.16.0 ``` List the pods. diff --git a/docs/cl/google-cloud.md b/docs/cl/google-cloud.md index 10e318711..b3268dd1b 100644 --- a/docs/cl/google-cloud.md +++ b/docs/cl/google-cloud.md @@ -1,6 +1,6 @@ # Google Cloud -In this tutorial, we'll create a Kubernetes v1.15.3 cluster on Google Compute Engine with Container Linux. +In this tutorial, we'll create a Kubernetes v1.16.0 cluster on Google Compute Engine with Container Linux. We'll declare a Kubernetes cluster using the Typhoon Terraform module. Then apply the changes to create a network, firewall rules, health checks, controller instances, worker managed instance group, load balancers, and TLS assets. @@ -71,7 +71,7 @@ Define a Kubernetes cluster using the module `google-cloud/container-linux/kuber ```tf module "google-cloud-yavin" { - source = "git::https://github.com/poseidon/typhoon//google-cloud/container-linux/kubernetes?ref=v1.15.3" + source = "git::https://github.com/poseidon/typhoon//google-cloud/container-linux/kubernetes?ref=v1.16.0" # Google Cloud cluster_name = "yavin" @@ -137,9 +137,9 @@ In 4-8 minutes, the Kubernetes cluster will be ready. $ export KUBECONFIG=/home/user/.secrets/clusters/yavin/auth/kubeconfig $ kubectl get nodes NAME ROLES STATUS AGE VERSION -yavin-controller-0.c.example-com.internal controller,master Ready 6m v1.15.3 -yavin-worker-jrbf.c.example-com.internal node Ready 5m v1.15.3 -yavin-worker-mzdm.c.example-com.internal node Ready 5m v1.15.3 +yavin-controller-0.c.example-com.internal controller,master Ready 6m v1.16.0 +yavin-worker-jrbf.c.example-com.internal node Ready 5m v1.16.0 +yavin-worker-mzdm.c.example-com.internal node Ready 5m v1.16.0 ``` List the pods. diff --git a/docs/fedora-coreos/aws.md b/docs/fedora-coreos/aws.md index bff3f9b61..ad6cd5f0b 100644 --- a/docs/fedora-coreos/aws.md +++ b/docs/fedora-coreos/aws.md @@ -3,7 +3,7 @@ !!! danger Typhoon for Fedora CoreOS is an early preview! Fedora CoreOS itself is a preview! Expect bugs and design shifts. Please help both projects solve problems. Report Fedora CoreOS bugs to [Fedora](https://github.com/coreos/fedora-coreos-tracker/issues). Report Typhoon issues to Typhoon. -In this tutorial, we'll create a Kubernetes v1.15.3 cluster on AWS with Fedora CoreOS. +In this tutorial, we'll create a Kubernetes v1.16.0 cluster on AWS with Fedora CoreOS. We'll declare a Kubernetes cluster using the Typhoon Terraform module. Then apply the changes to create a VPC, gateway, subnets, security groups, controller instances, worker auto-scaling group, network load balancer, and TLS assets. @@ -138,9 +138,9 @@ In 4-8 minutes, the Kubernetes cluster will be ready. $ export KUBECONFIG=/home/user/.secrets/clusters/tempest/auth/kubeconfig $ kubectl get nodes NAME STATUS ROLES AGE VERSION -ip-10-0-3-155 Ready controller,master 10m v1.15.3 -ip-10-0-26-65 Ready node 10m v1.15.3 -ip-10-0-41-21 Ready node 10m v1.15.3 +ip-10-0-3-155 Ready controller,master 10m v1.16.0 +ip-10-0-26-65 Ready node 10m v1.16.0 +ip-10-0-41-21 Ready node 10m v1.16.0 ``` List the pods. diff --git a/docs/fedora-coreos/bare-metal.md b/docs/fedora-coreos/bare-metal.md index f5a517cdf..2e991cc2c 100644 --- a/docs/fedora-coreos/bare-metal.md +++ b/docs/fedora-coreos/bare-metal.md @@ -3,7 +3,7 @@ !!! danger Typhoon for Fedora CoreOS is an early preview! Fedora CoreOS itself is a preview! Expect bugs and design shifts. Please help both projects solve problems. Report Fedora CoreOS bugs to [Fedora](https://github.com/coreos/fedora-coreos-tracker/issues). Report Typhoon issues to Typhoon. -In this tutorial, we'll network boot and provision a Kubernetes v1.15.3 cluster on bare-metal with Fedora CoreOS. +In this tutorial, we'll network boot and provision a Kubernetes v1.16.0 cluster on bare-metal with Fedora CoreOS. First, we'll deploy a [Matchbox](https://github.com/poseidon/matchbox) service and setup a network boot environment. Then, we'll declare a Kubernetes cluster using the Typhoon Terraform module and power on machines. On PXE boot, machines will install Fedora CoreOS to disk, reboot into the disk install, and provision themselves as Kubernetes controllers or workers via Ignition. @@ -283,9 +283,9 @@ systemd[1]: Started Kubernetes control plane. $ export KUBECONFIG=/home/user/.secrets/clusters/mercury/auth/kubeconfig $ kubectl get nodes NAME STATUS ROLES AGE VERSION -node1.example.com Ready controller,master 10m v1.15.3 -node2.example.com Ready node 10m v1.15.3 -node3.example.com Ready node 10m v1.15.3 +node1.example.com Ready controller,master 10m v1.16.0 +node2.example.com Ready node 10m v1.16.0 +node3.example.com Ready node 10m v1.16.0 ``` List the pods. diff --git a/docs/index.md b/docs/index.md index b0d064c93..e9e15f456 100644 --- a/docs/index.md +++ b/docs/index.md @@ -11,7 +11,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features -* Kubernetes v1.15.3 (upstream) +* Kubernetes v1.16.0 (upstream) * Single or multi-master, [Calico](https://www.projectcalico.org/) or [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/) * Advanced features like [worker pools](advanced/worker-pools/), [preemptible](cl/google-cloud/#preemption) workers, and [snippets](advanced/customization/#container-linux) customization @@ -47,7 +47,7 @@ Define a Kubernetes cluster by using the Terraform module for your chosen platfo ```tf module "google-cloud-yavin" { - source = "git::https://github.com/poseidon/typhoon//google-cloud/container-linux/kubernetes?ref=v1.15.3" + source = "git::https://github.com/poseidon/typhoon//google-cloud/container-linux/kubernetes?ref=v1.16.0" # Google Cloud cluster_name = "yavin" @@ -80,9 +80,9 @@ In 4-8 minutes (varies by platform), the cluster will be ready. This Google Clou $ export KUBECONFIG=/home/user/.secrets/clusters/yavin/auth/kubeconfig $ kubectl get nodes NAME ROLES STATUS AGE VERSION -yavin-controller-0.c.example-com.internal controller,master Ready 6m v1.15.3 -yavin-worker-jrbf.c.example-com.internal node Ready 5m v1.15.3 -yavin-worker-mzdm.c.example-com.internal node Ready 5m v1.15.3 +yavin-controller-0.c.example-com.internal controller,master Ready 6m v1.16.0 +yavin-worker-jrbf.c.example-com.internal node Ready 5m v1.16.0 +yavin-worker-mzdm.c.example-com.internal node Ready 5m v1.16.0 ``` List the pods. diff --git a/docs/topics/maintenance.md b/docs/topics/maintenance.md index 2d605085b..a8d2a608c 100644 --- a/docs/topics/maintenance.md +++ b/docs/topics/maintenance.md @@ -18,7 +18,7 @@ module "google-cloud-yavin" { } module "bare-metal-mercury" { - source = "git::https://github.com/poseidon/typhoon//bare-metal/container-linux/kubernetes?ref=v1.15.3" + source = "git::https://github.com/poseidon/typhoon//bare-metal/container-linux/kubernetes?ref=v1.16.0" ... } ``` @@ -279,15 +279,15 @@ Typhoon modules have been adapted for Terraform v0.12. Provider plugins requirem | Typhoon Release | Terraform version | |-------------------|---------------------| -| v1.15.3 - ? | v0.12.x | -| v1.10.3 - v1.15.3 | v0.11.x | +| v1.16.0 - ? | v0.12.x | +| v1.10.3 - v1.16.0 | v0.11.x | | v1.9.2 - v1.10.2 | v0.10.4+ or v0.11.x | | v1.7.3 - v1.9.1 | v0.10.x | | v1.6.4 - v1.7.2 | v0.9.x | ### New users -New users can start with Terraform v0.12.x and follow the docs for Typhoon v1.15.3+ without issue. +New users can start with Terraform v0.12.x and follow the docs for Typhoon v1.16.0+ without issue. ### Existing users @@ -404,7 +404,7 @@ tree . └── infraB <- new Terraform v0.12.x configs ``` -Define Typhoon clusters in the new config directory using Terraform v0.12 syntax. Follow the Typhoon v1.15.3+ docs (e.g. use `terraform12` in the `infraB` dir). See [AWS](/cl/aws), [Azure](/cl/azure), [Bare-Metal](/cl/bare-metal), [Digital Ocean](/cl/digital-ocean), or [Google-Cloud](/cl/google-cloud)) to create new clusters. Follow the usual [upgrade](/topics/maintenance/#upgrades) process to apply workloads and shift traffic. Later, switch back to the old config directory and deprovision clusters with Terraform v0.11. +Define Typhoon clusters in the new config directory using Terraform v0.12 syntax. Follow the Typhoon v1.16.0+ docs (e.g. use `terraform12` in the `infraB` dir). See [AWS](/cl/aws), [Azure](/cl/azure), [Bare-Metal](/cl/bare-metal), [Digital Ocean](/cl/digital-ocean), or [Google-Cloud](/cl/google-cloud)) to create new clusters. Follow the usual [upgrade](/topics/maintenance/#upgrades) process to apply workloads and shift traffic. Later, switch back to the old config directory and deprovision clusters with Terraform v0.11. ```shell terraform12 init diff --git a/google-cloud/container-linux/kubernetes/README.md b/google-cloud/container-linux/kubernetes/README.md index 11f6c2bd8..6cbdfce1c 100644 --- a/google-cloud/container-linux/kubernetes/README.md +++ b/google-cloud/container-linux/kubernetes/README.md @@ -11,7 +11,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features -* Kubernetes v1.15.3 (upstream) +* Kubernetes v1.16.0 (upstream) * Single or multi-master, [Calico](https://www.projectcalico.org/) or [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/) * Advanced features like [worker pools](https://typhoon.psdn.io/advanced/worker-pools/), [preemptible](https://typhoon.psdn.io/cl/google-cloud/#preemption) workers, and [snippets](https://typhoon.psdn.io/advanced/customization/#container-linux) customization diff --git a/google-cloud/container-linux/kubernetes/bootkube.tf b/google-cloud/container-linux/kubernetes/bootkube.tf index 94ad8e494..490174060 100644 --- a/google-cloud/container-linux/kubernetes/bootkube.tf +++ b/google-cloud/container-linux/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=d6206abedd69aca2e362bf533f73b13805ea344c" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=539b725093c8cd94ba46603adb25ac5280562ec8" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/google-cloud/container-linux/kubernetes/cl/controller.yaml.tmpl b/google-cloud/container-linux/kubernetes/cl/controller.yaml.tmpl index 3a6fe3e58..208bcd8f3 100644 --- a/google-cloud/container-linux/kubernetes/cl/controller.yaml.tmpl +++ b/google-cloud/container-linux/kubernetes/cl/controller.yaml.tmpl @@ -86,8 +86,8 @@ systemd: --kubeconfig=/etc/kubernetes/kubeconfig \ --lock-file=/var/run/lock/kubelet.lock \ --network-plugin=cni \ - --node-labels=node-role.kubernetes.io/master \ - --node-labels=node-role.kubernetes.io/controller="true" \ + --node-labels=node.kubernetes.io/master \ + --node-labels=node.kubernetes.io/controller="true" \ --pod-manifest-path=/etc/kubernetes/manifests \ --register-with-taints=node-role.kubernetes.io/master=:NoSchedule \ --read-only-port=0 \ @@ -114,7 +114,7 @@ systemd: --volume script,kind=host,source=/opt/bootstrap/apply \ --mount volume=script,target=/apply \ --insecure-options=image \ - docker://k8s.gcr.io/hyperkube:v1.15.3 \ + docker://k8s.gcr.io/hyperkube:v1.16.0 \ --net=host \ --dns=host \ --exec=/apply @@ -135,7 +135,7 @@ storage: contents: inline: | KUBELET_IMAGE_URL=docker://k8s.gcr.io/hyperkube - KUBELET_IMAGE_TAG=v1.15.3 + KUBELET_IMAGE_TAG=v1.16.0 - path: /opt/bootstrap/apply filesystem: root mode: 0544 diff --git a/google-cloud/container-linux/kubernetes/workers/cl/worker.yaml.tmpl b/google-cloud/container-linux/kubernetes/workers/cl/worker.yaml.tmpl index ab4d84474..d68bd23f2 100644 --- a/google-cloud/container-linux/kubernetes/workers/cl/worker.yaml.tmpl +++ b/google-cloud/container-linux/kubernetes/workers/cl/worker.yaml.tmpl @@ -59,7 +59,7 @@ systemd: --kubeconfig=/etc/kubernetes/kubeconfig \ --lock-file=/var/run/lock/kubelet.lock \ --network-plugin=cni \ - --node-labels=node-role.kubernetes.io/node \ + --node-labels=node.kubernetes.io/node \ --pod-manifest-path=/etc/kubernetes/manifests \ --read-only-port=0 \ --volume-plugin-dir=/var/lib/kubelet/volumeplugins @@ -94,7 +94,7 @@ storage: contents: inline: | KUBELET_IMAGE_URL=docker://k8s.gcr.io/hyperkube - KUBELET_IMAGE_TAG=v1.15.3 + KUBELET_IMAGE_TAG=v1.16.0 - path: /etc/sysctl.d/max-user-watches.conf filesystem: root contents: @@ -112,7 +112,7 @@ storage: --volume config,kind=host,source=/etc/kubernetes \ --mount volume=config,target=/etc/kubernetes \ --insecure-options=image \ - docker://k8s.gcr.io/hyperkube:v1.15.3 \ + docker://k8s.gcr.io/hyperkube:v1.16.0 \ --net=host \ --dns=host \ --exec=/kubectl -- --kubeconfig=/etc/kubernetes/kubeconfig delete node $(hostname) From b951aca66fe728d9c382f3cb5c637bac71311c73 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Wed, 18 Sep 2019 23:56:17 -0700 Subject: [PATCH 216/523] Create /etc/kubernetes/manifests before asset copy * Fix issue (present since bootkube->bootstrap switch) where controller asset copy could fail if /etc/kubernetes/manifests wasn't created in time on platforms using path activation for the Kubelet (observed on DigitalOcean, also possible on bare-metal) --- aws/container-linux/kubernetes/ssh.tf | 1 + aws/fedora-coreos/kubernetes/ssh.tf | 1 + azure/container-linux/kubernetes/ssh.tf | 1 + bare-metal/container-linux/kubernetes/ssh.tf | 3 ++- bare-metal/fedora-coreos/kubernetes/ssh.tf | 3 ++- digital-ocean/container-linux/kubernetes/ssh.tf | 3 ++- google-cloud/container-linux/kubernetes/ssh.tf | 1 + 7 files changed, 10 insertions(+), 3 deletions(-) diff --git a/aws/container-linux/kubernetes/ssh.tf b/aws/container-linux/kubernetes/ssh.tf index 4ebd5f240..1821e195b 100644 --- a/aws/container-linux/kubernetes/ssh.tf +++ b/aws/container-linux/kubernetes/ssh.tf @@ -66,6 +66,7 @@ resource "null_resource" "copy-controller-secrets" { "sudo chown -R etcd:etcd /etc/ssl/etcd", "sudo chmod -R 500 /etc/ssl/etcd", "sudo mv $HOME/assets /opt/bootstrap/assets", + "sudo mkdir -p /etc/kubernetes/manifests", "sudo mkdir -p /etc/kubernetes/bootstrap-secrets", "sudo cp -r /opt/bootstrap/assets/tls/* /etc/kubernetes/bootstrap-secrets/", "sudo cp /opt/bootstrap/assets/auth/kubeconfig /etc/kubernetes/bootstrap-secrets/", diff --git a/aws/fedora-coreos/kubernetes/ssh.tf b/aws/fedora-coreos/kubernetes/ssh.tf index 2e2cc3fe7..339e95964 100644 --- a/aws/fedora-coreos/kubernetes/ssh.tf +++ b/aws/fedora-coreos/kubernetes/ssh.tf @@ -66,6 +66,7 @@ resource "null_resource" "copy-controller-secrets" { "sudo chown -R etcd:etcd /etc/ssl/etcd", "sudo chmod -R 500 /etc/ssl/etcd", "sudo mv $HOME/assets /opt/bootstrap/assets", + "sudo mkdir -p /etc/kubernetes/manifests", "sudo mkdir -p /etc/kubernetes/bootstrap-secrets", "sudo cp -r /opt/bootstrap/assets/tls/* /etc/kubernetes/bootstrap-secrets/", "sudo cp /opt/bootstrap/assets/auth/kubeconfig /etc/kubernetes/bootstrap-secrets/", diff --git a/azure/container-linux/kubernetes/ssh.tf b/azure/container-linux/kubernetes/ssh.tf index f8012c270..159d74f7a 100644 --- a/azure/container-linux/kubernetes/ssh.tf +++ b/azure/container-linux/kubernetes/ssh.tf @@ -67,6 +67,7 @@ resource "null_resource" "copy-controller-secrets" { "sudo chown -R etcd:etcd /etc/ssl/etcd", "sudo chmod -R 500 /etc/ssl/etcd", "sudo mv $HOME/assets /opt/bootstrap/assets", + "sudo mkdir -p /etc/kubernetes/manifests", "sudo mkdir -p /etc/kubernetes/bootstrap-secrets", "sudo cp -r /opt/bootstrap/assets/tls/* /etc/kubernetes/bootstrap-secrets/", "sudo cp /opt/bootstrap/assets/auth/kubeconfig /etc/kubernetes/bootstrap-secrets/", diff --git a/bare-metal/container-linux/kubernetes/ssh.tf b/bare-metal/container-linux/kubernetes/ssh.tf index 2a85da6c6..a9674f98c 100644 --- a/bare-metal/container-linux/kubernetes/ssh.tf +++ b/bare-metal/container-linux/kubernetes/ssh.tf @@ -75,9 +75,10 @@ resource "null_resource" "copy-controller-secrets" { "sudo mv etcd-peer.key /etc/ssl/etcd/etcd/peer.key", "sudo chown -R etcd:etcd /etc/ssl/etcd", "sudo chmod -R 500 /etc/ssl/etcd", - "sudo mv $HOME/kubeconfig /etc/kubernetes/kubeconfig", "sudo mv $HOME/assets /opt/bootstrap/assets", + "sudo mkdir -p /etc/kubernetes/manifests" "sudo mkdir -p /etc/kubernetes/bootstrap-secrets", + "sudo mv $HOME/kubeconfig /etc/kubernetes/kubeconfig", "sudo cp -r /opt/bootstrap/assets/tls/* /etc/kubernetes/bootstrap-secrets/", "sudo cp /opt/bootstrap/assets/auth/kubeconfig /etc/kubernetes/bootstrap-secrets/", "sudo cp -r /opt/bootstrap/assets/static-manifests/* /etc/kubernetes/manifests/", diff --git a/bare-metal/fedora-coreos/kubernetes/ssh.tf b/bare-metal/fedora-coreos/kubernetes/ssh.tf index 002faf1fd..931801959 100644 --- a/bare-metal/fedora-coreos/kubernetes/ssh.tf +++ b/bare-metal/fedora-coreos/kubernetes/ssh.tf @@ -72,9 +72,10 @@ resource "null_resource" "copy-controller-secrets" { "sudo cp /etc/ssl/etcd/etcd-client-ca.crt /etc/ssl/etcd/etcd/peer-ca.crt", "sudo mv etcd-peer.crt /etc/ssl/etcd/etcd/peer.crt", "sudo mv etcd-peer.key /etc/ssl/etcd/etcd/peer.key", - "sudo mv $HOME/kubeconfig /etc/kubernetes/kubeconfig", "sudo mv $HOME/assets /opt/bootstrap/assets", + "sudo mkdir -p /etc/kubernetes/manifests" "sudo mkdir -p /etc/kubernetes/bootstrap-secrets", + "sudo mv $HOME/kubeconfig /etc/kubernetes/kubeconfig", "sudo cp -r /opt/bootstrap/assets/tls/* /etc/kubernetes/bootstrap-secrets/", "sudo cp /opt/bootstrap/assets/auth/kubeconfig /etc/kubernetes/bootstrap-secrets/", "sudo cp -r /opt/bootstrap/assets/static-manifests/* /etc/kubernetes/manifests/" diff --git a/digital-ocean/container-linux/kubernetes/ssh.tf b/digital-ocean/container-linux/kubernetes/ssh.tf index a1b734d1f..fbc9ebf7d 100644 --- a/digital-ocean/container-linux/kubernetes/ssh.tf +++ b/digital-ocean/container-linux/kubernetes/ssh.tf @@ -71,9 +71,10 @@ resource "null_resource" "copy-controller-secrets" { "sudo mv etcd-peer.key /etc/ssl/etcd/etcd/peer.key", "sudo chown -R etcd:etcd /etc/ssl/etcd", "sudo chmod -R 500 /etc/ssl/etcd", - "sudo mv $HOME/kubeconfig /etc/kubernetes/kubeconfig", "sudo mv $HOME/assets /opt/bootstrap/assets", + "sudo mkdir -p /etc/kubernetes/manifests" "sudo mkdir -p /etc/kubernetes/bootstrap-secrets", + "sudo mv $HOME/kubeconfig /etc/kubernetes/kubeconfig", "sudo cp -r /opt/bootstrap/assets/tls/* /etc/kubernetes/bootstrap-secrets/", "sudo cp /opt/bootstrap/assets/auth/kubeconfig /etc/kubernetes/bootstrap-secrets/", "sudo cp -r /opt/bootstrap/assets/static-manifests/* /etc/kubernetes/manifests/", diff --git a/google-cloud/container-linux/kubernetes/ssh.tf b/google-cloud/container-linux/kubernetes/ssh.tf index b4229674b..d86fad75b 100644 --- a/google-cloud/container-linux/kubernetes/ssh.tf +++ b/google-cloud/container-linux/kubernetes/ssh.tf @@ -66,6 +66,7 @@ resource "null_resource" "copy-controller-secrets" { "sudo chown -R etcd:etcd /etc/ssl/etcd", "sudo chmod -R 500 /etc/ssl/etcd", "sudo mv $HOME/assets /opt/bootstrap/assets", + "sudo mkdir -p /etc/kubernetes/manifests", "sudo mkdir -p /etc/kubernetes/bootstrap-secrets", "sudo cp -r /opt/bootstrap/assets/tls/* /etc/kubernetes/bootstrap-secrets/", "sudo cp /opt/bootstrap/assets/auth/kubeconfig /etc/kubernetes/bootstrap-secrets/", From 5b06e0e86912345f5809775af840df8b52234b8e Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Thu, 19 Sep 2019 00:15:39 -0700 Subject: [PATCH 217/523] Organize and cleanup Kubelet ExecStartPre * Sort Kubelet ExecStartPre mkdir commands * Remove unused inactive-manifests and checkpoint-secrets directories (were used by bootkube self-hosting) --- aws/container-linux/kubernetes/cl/controller.yaml.tmpl | 6 ++---- aws/container-linux/kubernetes/workers/cl/worker.yaml.tmpl | 2 +- aws/fedora-coreos/kubernetes/fcc/controller.yaml | 2 +- aws/fedora-coreos/kubernetes/workers/fcc/worker.yaml | 2 +- azure/container-linux/kubernetes/cl/controller.yaml.tmpl | 4 +--- .../container-linux/kubernetes/workers/cl/worker.yaml.tmpl | 2 +- .../container-linux/kubernetes/cl/controller.yaml.tmpl | 6 ++---- bare-metal/container-linux/kubernetes/cl/worker.yaml.tmpl | 4 ++-- bare-metal/fedora-coreos/kubernetes/fcc/controller.yaml | 2 +- bare-metal/fedora-coreos/kubernetes/fcc/worker.yaml | 2 +- .../container-linux/kubernetes/cl/controller.yaml.tmpl | 6 ++---- .../container-linux/kubernetes/cl/worker.yaml.tmpl | 4 ++-- .../container-linux/kubernetes/cl/controller.yaml.tmpl | 6 ++---- .../container-linux/kubernetes/workers/cl/worker.yaml.tmpl | 4 ++-- 14 files changed, 21 insertions(+), 31 deletions(-) diff --git a/aws/container-linux/kubernetes/cl/controller.yaml.tmpl b/aws/container-linux/kubernetes/cl/controller.yaml.tmpl index 4128ecf5c..60e8ab31e 100644 --- a/aws/container-linux/kubernetes/cl/controller.yaml.tmpl +++ b/aws/container-linux/kubernetes/cl/controller.yaml.tmpl @@ -64,11 +64,9 @@ systemd: --mount volume=var-log,target=/var/log \ --insecure-options=image" Environment=KUBELET_CGROUP_DRIVER=${cgroup_driver} - ExecStartPre=/bin/mkdir -p /opt/cni/bin - ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d - ExecStartPre=/bin/mkdir -p /etc/kubernetes/checkpoint-secrets - ExecStartPre=/bin/mkdir -p /etc/kubernetes/inactive-manifests + ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests + ExecStartPre=/bin/mkdir -p /opt/cni/bin ExecStartPre=/bin/mkdir -p /var/lib/cni ExecStartPre=/bin/mkdir -p /var/lib/calico ExecStartPre=/bin/mkdir -p /var/lib/kubelet/volumeplugins diff --git a/aws/container-linux/kubernetes/workers/cl/worker.yaml.tmpl b/aws/container-linux/kubernetes/workers/cl/worker.yaml.tmpl index 01dd15557..a54411410 100644 --- a/aws/container-linux/kubernetes/workers/cl/worker.yaml.tmpl +++ b/aws/container-linux/kubernetes/workers/cl/worker.yaml.tmpl @@ -39,9 +39,9 @@ systemd: --mount volume=var-log,target=/var/log \ --insecure-options=image" Environment=KUBELET_CGROUP_DRIVER=${cgroup_driver} - ExecStartPre=/bin/mkdir -p /opt/cni/bin ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d + ExecStartPre=/bin/mkdir -p /opt/cni/bin ExecStartPre=/bin/mkdir -p /var/lib/cni ExecStartPre=/bin/mkdir -p /var/lib/calico ExecStartPre=/bin/mkdir -p /var/lib/kubelet/volumeplugins diff --git a/aws/fedora-coreos/kubernetes/fcc/controller.yaml b/aws/fedora-coreos/kubernetes/fcc/controller.yaml index b0333e939..3d55f0452 100644 --- a/aws/fedora-coreos/kubernetes/fcc/controller.yaml +++ b/aws/fedora-coreos/kubernetes/fcc/controller.yaml @@ -56,9 +56,9 @@ systemd: [Service] ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests + ExecStartPre=/bin/mkdir -p /opt/cni/bin ExecStartPre=/bin/mkdir -p /var/lib/calico ExecStartPre=/bin/mkdir -p /var/lib/kubelet/volumeplugins - ExecStartPre=/bin/mkdir -p /opt/cni/bin ExecStartPre=/usr/bin/bash -c "grep 'certificate-authority-data' /etc/kubernetes/kubeconfig | awk '{print $2}' | base64 -d > /etc/kubernetes/ca.crt" ExecStartPre=-/usr/bin/podman rm kubelet ExecStart=/usr/bin/podman run --name kubelet \ diff --git a/aws/fedora-coreos/kubernetes/workers/fcc/worker.yaml b/aws/fedora-coreos/kubernetes/workers/fcc/worker.yaml index 6973fbf08..79ab90652 100644 --- a/aws/fedora-coreos/kubernetes/workers/fcc/worker.yaml +++ b/aws/fedora-coreos/kubernetes/workers/fcc/worker.yaml @@ -26,9 +26,9 @@ systemd: [Service] ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests + ExecStartPre=/bin/mkdir -p /opt/cni/bin ExecStartPre=/bin/mkdir -p /var/lib/calico ExecStartPre=/bin/mkdir -p /var/lib/kubelet/volumeplugins - ExecStartPre=/bin/mkdir -p /opt/cni/bin ExecStartPre=/usr/bin/bash -c "grep 'certificate-authority-data' /etc/kubernetes/kubeconfig | awk '{print $2}' | base64 -d > /etc/kubernetes/ca.crt" ExecStartPre=-/usr/bin/podman rm kubelet ExecStart=/usr/bin/podman run --name kubelet \ diff --git a/azure/container-linux/kubernetes/cl/controller.yaml.tmpl b/azure/container-linux/kubernetes/cl/controller.yaml.tmpl index 8261ffae8..0b1bc8e9f 100644 --- a/azure/container-linux/kubernetes/cl/controller.yaml.tmpl +++ b/azure/container-linux/kubernetes/cl/controller.yaml.tmpl @@ -63,11 +63,9 @@ systemd: --volume var-log,kind=host,source=/var/log \ --mount volume=var-log,target=/var/log \ --insecure-options=image" - ExecStartPre=/bin/mkdir -p /opt/cni/bin ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d - ExecStartPre=/bin/mkdir -p /etc/kubernetes/checkpoint-secrets - ExecStartPre=/bin/mkdir -p /etc/kubernetes/inactive-manifests + ExecStartPre=/bin/mkdir -p /opt/cni/bin ExecStartPre=/bin/mkdir -p /var/lib/cni ExecStartPre=/bin/mkdir -p /var/lib/calico ExecStartPre=/bin/mkdir -p /var/lib/kubelet/volumeplugins diff --git a/azure/container-linux/kubernetes/workers/cl/worker.yaml.tmpl b/azure/container-linux/kubernetes/workers/cl/worker.yaml.tmpl index 20dcf2d31..7c3a68115 100644 --- a/azure/container-linux/kubernetes/workers/cl/worker.yaml.tmpl +++ b/azure/container-linux/kubernetes/workers/cl/worker.yaml.tmpl @@ -38,9 +38,9 @@ systemd: --volume var-log,kind=host,source=/var/log \ --mount volume=var-log,target=/var/log \ --insecure-options=image" - ExecStartPre=/bin/mkdir -p /opt/cni/bin ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d + ExecStartPre=/bin/mkdir -p /opt/cni/bin ExecStartPre=/bin/mkdir -p /var/lib/cni ExecStartPre=/bin/mkdir -p /var/lib/calico ExecStartPre=/bin/mkdir -p /var/lib/kubelet/volumeplugins diff --git a/bare-metal/container-linux/kubernetes/cl/controller.yaml.tmpl b/bare-metal/container-linux/kubernetes/cl/controller.yaml.tmpl index 476a08511..356c7bff4 100644 --- a/bare-metal/container-linux/kubernetes/cl/controller.yaml.tmpl +++ b/bare-metal/container-linux/kubernetes/cl/controller.yaml.tmpl @@ -76,11 +76,9 @@ systemd: --mount volume=iscsiadm,target=/sbin/iscsiadm \ --insecure-options=image" Environment=KUBELET_CGROUP_DRIVER=${cgroup_driver} - ExecStartPre=/bin/mkdir -p /opt/cni/bin - ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d - ExecStartPre=/bin/mkdir -p /etc/kubernetes/checkpoint-secrets - ExecStartPre=/bin/mkdir -p /etc/kubernetes/inactive-manifests + ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests + ExecStartPre=/bin/mkdir -p /opt/cni/bin ExecStartPre=/bin/mkdir -p /var/lib/cni ExecStartPre=/bin/mkdir -p /var/lib/calico ExecStartPre=/bin/mkdir -p /var/lib/kubelet/volumeplugins diff --git a/bare-metal/container-linux/kubernetes/cl/worker.yaml.tmpl b/bare-metal/container-linux/kubernetes/cl/worker.yaml.tmpl index c991e9ec4..4397dd2f0 100644 --- a/bare-metal/container-linux/kubernetes/cl/worker.yaml.tmpl +++ b/bare-metal/container-linux/kubernetes/cl/worker.yaml.tmpl @@ -51,9 +51,9 @@ systemd: --mount volume=iscsiadm,target=/sbin/iscsiadm \ --insecure-options=image" Environment=KUBELET_CGROUP_DRIVER=${cgroup_driver} - ExecStartPre=/bin/mkdir -p /opt/cni/bin - ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d + ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests + ExecStartPre=/bin/mkdir -p /opt/cni/bin ExecStartPre=/bin/mkdir -p /var/lib/cni ExecStartPre=/bin/mkdir -p /var/lib/calico ExecStartPre=/bin/mkdir -p /var/lib/kubelet/volumeplugins diff --git a/bare-metal/fedora-coreos/kubernetes/fcc/controller.yaml b/bare-metal/fedora-coreos/kubernetes/fcc/controller.yaml index 68bcc4c0f..7ecf846a3 100644 --- a/bare-metal/fedora-coreos/kubernetes/fcc/controller.yaml +++ b/bare-metal/fedora-coreos/kubernetes/fcc/controller.yaml @@ -55,9 +55,9 @@ systemd: [Service] ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests + ExecStartPre=/bin/mkdir -p /opt/cni/bin ExecStartPre=/bin/mkdir -p /var/lib/calico ExecStartPre=/bin/mkdir -p /var/lib/kubelet/volumeplugins - ExecStartPre=/bin/mkdir -p /opt/cni/bin ExecStartPre=/usr/bin/bash -c "grep 'certificate-authority-data' /etc/kubernetes/kubeconfig | awk '{print $2}' | base64 -d > /etc/kubernetes/ca.crt" ExecStartPre=-/usr/bin/podman rm kubelet ExecStart=/usr/bin/podman run --name kubelet \ diff --git a/bare-metal/fedora-coreos/kubernetes/fcc/worker.yaml b/bare-metal/fedora-coreos/kubernetes/fcc/worker.yaml index 1c1af1fea..1e3736a2b 100644 --- a/bare-metal/fedora-coreos/kubernetes/fcc/worker.yaml +++ b/bare-metal/fedora-coreos/kubernetes/fcc/worker.yaml @@ -25,9 +25,9 @@ systemd: [Service] ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests + ExecStartPre=/bin/mkdir -p /opt/cni/bin ExecStartPre=/bin/mkdir -p /var/lib/calico ExecStartPre=/bin/mkdir -p /var/lib/kubelet/volumeplugins - ExecStartPre=/bin/mkdir -p /opt/cni/bin ExecStartPre=/usr/bin/bash -c "grep 'certificate-authority-data' /etc/kubernetes/kubeconfig | awk '{print $2}' | base64 -d > /etc/kubernetes/ca.crt" ExecStartPre=-/usr/bin/podman rm kubelet ExecStart=/usr/bin/podman run --name kubelet \ diff --git a/digital-ocean/container-linux/kubernetes/cl/controller.yaml.tmpl b/digital-ocean/container-linux/kubernetes/cl/controller.yaml.tmpl index 9fe9aaace..801268c80 100644 --- a/digital-ocean/container-linux/kubernetes/cl/controller.yaml.tmpl +++ b/digital-ocean/container-linux/kubernetes/cl/controller.yaml.tmpl @@ -74,11 +74,9 @@ systemd: --volume var-log,kind=host,source=/var/log \ --mount volume=var-log,target=/var/log \ --insecure-options=image" - ExecStartPre=/bin/mkdir -p /opt/cni/bin - ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d - ExecStartPre=/bin/mkdir -p /etc/kubernetes/checkpoint-secrets - ExecStartPre=/bin/mkdir -p /etc/kubernetes/inactive-manifests + ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests + ExecStartPre=/bin/mkdir -p /opt/cni/bin ExecStartPre=/bin/mkdir -p /var/lib/cni ExecStartPre=/bin/mkdir -p /var/lib/calico ExecStartPre=/bin/mkdir -p /var/lib/kubelet/volumeplugins diff --git a/digital-ocean/container-linux/kubernetes/cl/worker.yaml.tmpl b/digital-ocean/container-linux/kubernetes/cl/worker.yaml.tmpl index 2dc571cd8..da19b6aa1 100644 --- a/digital-ocean/container-linux/kubernetes/cl/worker.yaml.tmpl +++ b/digital-ocean/container-linux/kubernetes/cl/worker.yaml.tmpl @@ -49,9 +49,9 @@ systemd: --volume var-log,kind=host,source=/var/log \ --mount volume=var-log,target=/var/log \ --insecure-options=image" - ExecStartPre=/bin/mkdir -p /opt/cni/bin - ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d + ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests + ExecStartPre=/bin/mkdir -p /opt/cni/bin ExecStartPre=/bin/mkdir -p /var/lib/cni ExecStartPre=/bin/mkdir -p /var/lib/calico ExecStartPre=/bin/mkdir -p /var/lib/kubelet/volumeplugins diff --git a/google-cloud/container-linux/kubernetes/cl/controller.yaml.tmpl b/google-cloud/container-linux/kubernetes/cl/controller.yaml.tmpl index 208bcd8f3..3d0bdbdb7 100644 --- a/google-cloud/container-linux/kubernetes/cl/controller.yaml.tmpl +++ b/google-cloud/container-linux/kubernetes/cl/controller.yaml.tmpl @@ -64,11 +64,9 @@ systemd: --mount volume=var-log,target=/var/log \ --hosts-entry=host \ --insecure-options=image" - ExecStartPre=/bin/mkdir -p /opt/cni/bin - ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d - ExecStartPre=/bin/mkdir -p /etc/kubernetes/checkpoint-secrets - ExecStartPre=/bin/mkdir -p /etc/kubernetes/inactive-manifests + ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests + ExecStartPre=/bin/mkdir -p /opt/cni/bin ExecStartPre=/bin/mkdir -p /var/lib/cni ExecStartPre=/bin/mkdir -p /var/lib/calico ExecStartPre=/bin/mkdir -p /var/lib/kubelet/volumeplugins diff --git a/google-cloud/container-linux/kubernetes/workers/cl/worker.yaml.tmpl b/google-cloud/container-linux/kubernetes/workers/cl/worker.yaml.tmpl index d68bd23f2..ddc78680c 100644 --- a/google-cloud/container-linux/kubernetes/workers/cl/worker.yaml.tmpl +++ b/google-cloud/container-linux/kubernetes/workers/cl/worker.yaml.tmpl @@ -39,9 +39,9 @@ systemd: --mount volume=var-log,target=/var/log \ --hosts-entry=host \ --insecure-options=image" - ExecStartPre=/bin/mkdir -p /opt/cni/bin - ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d + ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests + ExecStartPre=/bin/mkdir -p /opt/cni/bin ExecStartPre=/bin/mkdir -p /var/lib/cni ExecStartPre=/bin/mkdir -p /var/lib/calico ExecStartPre=/bin/mkdir -p /var/lib/kubelet/volumeplugins From 81a1ae38e604c0dd4ac594076ab61519b38537ce Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Sun, 22 Sep 2019 17:14:30 -0700 Subject: [PATCH 218/523] Update Terraform provider plugin versions * Recommend provider plugin versions that Typhoon authors use --- docs/cl/aws.md | 2 +- docs/cl/azure.md | 2 +- docs/cl/google-cloud.md | 2 +- docs/fedora-coreos/aws.md | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/cl/aws.md b/docs/cl/aws.md index 6577feb65..77c63adaa 100644 --- a/docs/cl/aws.md +++ b/docs/cl/aws.md @@ -49,7 +49,7 @@ Configure the AWS provider to use your access key credentials in a `providers.tf ```tf provider "aws" { - version = "2.25.0" + version = "2.29.0" region = "eu-central-1" shared_credentials_file = "/home/user/.config/aws/credentials" } diff --git a/docs/cl/azure.md b/docs/cl/azure.md index 70ef1f2c9..49be1bf72 100644 --- a/docs/cl/azure.md +++ b/docs/cl/azure.md @@ -50,7 +50,7 @@ Configure the Azure provider in a `providers.tf` file. ```tf provider "azurerm" { - version = "1.33.0" + version = "1.34.0" } provider "ct" { diff --git a/docs/cl/google-cloud.md b/docs/cl/google-cloud.md index b3268dd1b..2cbaf4902 100644 --- a/docs/cl/google-cloud.md +++ b/docs/cl/google-cloud.md @@ -49,7 +49,7 @@ Configure the Google Cloud provider to use your service account key, project-id, ```tf provider "google" { - version = "2.13.0" + version = "2.15.0" project = "project-id" region = "us-central1" credentials = "${file("~/.config/google-cloud/terraform.json")}" diff --git a/docs/fedora-coreos/aws.md b/docs/fedora-coreos/aws.md index ad6cd5f0b..dc54b1832 100644 --- a/docs/fedora-coreos/aws.md +++ b/docs/fedora-coreos/aws.md @@ -52,7 +52,7 @@ Configure the AWS provider to use your access key credentials in a `providers.tf ```tf provider "aws" { - version = "2.25.0" + version = "2.29.0" region = "us-east-1" # MUST be us-east-1 right now! shared_credentials_file = "/home/user/.config/aws/credentials" } From 078f08422015f5316ccdfde26e7c702e8d6c48fb Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Sun, 22 Sep 2019 17:37:23 -0700 Subject: [PATCH 219/523] Update CHANGES and docs for v1.16.0 release --- CHANGES.md | 3 ++- README.md | 10 +++++----- docs/cl/aws.md | 8 ++++---- docs/cl/azure.md | 8 ++++---- docs/cl/bare-metal.md | 8 ++++---- docs/cl/digital-ocean.md | 8 ++++---- docs/cl/google-cloud.md | 8 ++++---- docs/fedora-coreos/aws.md | 8 ++++---- docs/fedora-coreos/bare-metal.md | 8 ++++---- docs/index.md | 8 ++++---- 10 files changed, 39 insertions(+), 38 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 51a229dcc..a76236281 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -8,7 +8,8 @@ Notable changes between versions. * Kubernetes [v1.16.0](https://github.com/kubernetes/kubernetes/blob/master/CHANGELOG-1.16.md#v1160) ([#543](https://github.com/poseidon/typhoon/pull/543)) * Read about several Kubernetes API [deprecations](https://github.com/kubernetes/kubernetes/blob/master/CHANGELOG-1.16.md#deprecations-and-removals)! - * Rename `node-role.kubernetes.io` labels for `master` and `node` roles (no longer shown in `kubectl get nodes`) + * Remove legacy node role labels (no longer shown in `kubectl get nodes`) + * Rename node labels to `node.kubernetes.io/master` and `node.kubernetes.io/node` (migratory) * Migrate control plane from self-hosted to static pods ([#536](https://github.com/poseidon/typhoon/pull/536)) * Run `kube-apiserver`, `kube-scheduler`, and `kube-controller-manager` as static pods on each controller * `kubectl` edits to `kube-apiserver`, `kube-scheduler`, and `kube-controller-manager` are no longer possible (change) diff --git a/README.md b/README.md index 130b06b61..66ccb339a 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# Typhoon [![IRC](https://img.shields.io/badge/freenode-%23typhoon-0099ef.svg)]() +# Typhoon Typhoon is a minimal and free Kubernetes distribution. @@ -81,10 +81,10 @@ In 4-8 minutes (varies by platform), the cluster will be ready. This Google Clou ```sh $ export KUBECONFIG=/home/user/.secrets/clusters/yavin/auth/kubeconfig $ kubectl get nodes -NAME ROLES STATUS AGE VERSION -yavin-controller-0.c.example-com.internal controller,master Ready 6m v1.16.0 -yavin-worker-jrbf.c.example-com.internal node Ready 5m v1.16.0 -yavin-worker-mzdm.c.example-com.internal node Ready 5m v1.16.0 +NAME ROLES STATUS AGE VERSION +yavin-controller-0.c.example-com.internal Ready 6m v1.16.0 +yavin-worker-jrbf.c.example-com.internal Ready 5m v1.16.0 +yavin-worker-mzdm.c.example-com.internal Ready 5m v1.16.0 ``` List the pods. diff --git a/docs/cl/aws.md b/docs/cl/aws.md index 77c63adaa..200314c53 100644 --- a/docs/cl/aws.md +++ b/docs/cl/aws.md @@ -134,10 +134,10 @@ In 4-8 minutes, the Kubernetes cluster will be ready. ``` $ export KUBECONFIG=/home/user/.secrets/clusters/tempest/auth/kubeconfig $ kubectl get nodes -NAME STATUS ROLES AGE VERSION -ip-10-0-3-155 Ready controller,master 10m v1.16.0 -ip-10-0-26-65 Ready node 10m v1.16.0 -ip-10-0-41-21 Ready node 10m v1.16.0 +NAME STATUS ROLES AGE VERSION +ip-10-0-3-155 Ready 10m v1.16.0 +ip-10-0-26-65 Ready 10m v1.16.0 +ip-10-0-41-21 Ready 10m v1.16.0 ``` List the pods. diff --git a/docs/cl/azure.md b/docs/cl/azure.md index 49be1bf72..566d439ad 100644 --- a/docs/cl/azure.md +++ b/docs/cl/azure.md @@ -131,10 +131,10 @@ In 4-8 minutes, the Kubernetes cluster will be ready. ``` $ export KUBECONFIG=/home/user/.secrets/clusters/ramius/auth/kubeconfig $ kubectl get nodes -NAME STATUS ROLES AGE VERSION -ramius-controller-0 Ready controller,master 24m v1.16.0 -ramius-worker-000001 Ready node 25m v1.16.0 -ramius-worker-000002 Ready node 24m v1.16.0 +NAME STATUS ROLES AGE VERSION +ramius-controller-0 Ready 24m v1.16.0 +ramius-worker-000001 Ready 25m v1.16.0 +ramius-worker-000002 Ready 24m v1.16.0 ``` List the pods. diff --git a/docs/cl/bare-metal.md b/docs/cl/bare-metal.md index d579004a6..ba8012140 100644 --- a/docs/cl/bare-metal.md +++ b/docs/cl/bare-metal.md @@ -288,10 +288,10 @@ systemd[1]: Started Kubernetes control plane. ``` $ export KUBECONFIG=/home/user/.secrets/clusters/mercury/auth/kubeconfig $ kubectl get nodes -NAME STATUS ROLES AGE VERSION -node1.example.com Ready controller,master 10m v1.16.0 -node2.example.com Ready node 10m v1.16.0 -node3.example.com Ready node 10m v1.16.0 +NAME STATUS ROLES AGE VERSION +node1.example.com Ready 10m v1.16.0 +node2.example.com Ready 10m v1.16.0 +node3.example.com Ready 10m v1.16.0 ``` List the pods. diff --git a/docs/cl/digital-ocean.md b/docs/cl/digital-ocean.md index 73a6653fd..fbbe30cb2 100644 --- a/docs/cl/digital-ocean.md +++ b/docs/cl/digital-ocean.md @@ -129,10 +129,10 @@ In 3-6 minutes, the Kubernetes cluster will be ready. ``` $ export KUBECONFIG=/home/user/.secrets/clusters/nemo/auth/kubeconfig $ kubectl get nodes -NAME STATUS ROLES AGE VERSION -10.132.110.130 Ready controller,master 10m v1.16.0 -10.132.115.81 Ready node 10m v1.16.0 -10.132.124.107 Ready node 10m v1.16.0 +NAME STATUS ROLES AGE VERSION +10.132.110.130 Ready 10m v1.16.0 +10.132.115.81 Ready 10m v1.16.0 +10.132.124.107 Ready 10m v1.16.0 ``` List the pods. diff --git a/docs/cl/google-cloud.md b/docs/cl/google-cloud.md index 2cbaf4902..da809973f 100644 --- a/docs/cl/google-cloud.md +++ b/docs/cl/google-cloud.md @@ -136,10 +136,10 @@ In 4-8 minutes, the Kubernetes cluster will be ready. ``` $ export KUBECONFIG=/home/user/.secrets/clusters/yavin/auth/kubeconfig $ kubectl get nodes -NAME ROLES STATUS AGE VERSION -yavin-controller-0.c.example-com.internal controller,master Ready 6m v1.16.0 -yavin-worker-jrbf.c.example-com.internal node Ready 5m v1.16.0 -yavin-worker-mzdm.c.example-com.internal node Ready 5m v1.16.0 +NAME ROLES STATUS AGE VERSION +yavin-controller-0.c.example-com.internal Ready 6m v1.16.0 +yavin-worker-jrbf.c.example-com.internal Ready 5m v1.16.0 +yavin-worker-mzdm.c.example-com.internal Ready 5m v1.16.0 ``` List the pods. diff --git a/docs/fedora-coreos/aws.md b/docs/fedora-coreos/aws.md index dc54b1832..694db1237 100644 --- a/docs/fedora-coreos/aws.md +++ b/docs/fedora-coreos/aws.md @@ -137,10 +137,10 @@ In 4-8 minutes, the Kubernetes cluster will be ready. ``` $ export KUBECONFIG=/home/user/.secrets/clusters/tempest/auth/kubeconfig $ kubectl get nodes -NAME STATUS ROLES AGE VERSION -ip-10-0-3-155 Ready controller,master 10m v1.16.0 -ip-10-0-26-65 Ready node 10m v1.16.0 -ip-10-0-41-21 Ready node 10m v1.16.0 +NAME STATUS ROLES AGE VERSION +ip-10-0-3-155 Ready 10m v1.16.0 +ip-10-0-26-65 Ready 10m v1.16.0 +ip-10-0-41-21 Ready 10m v1.16.0 ``` List the pods. diff --git a/docs/fedora-coreos/bare-metal.md b/docs/fedora-coreos/bare-metal.md index 2e991cc2c..ef4238513 100644 --- a/docs/fedora-coreos/bare-metal.md +++ b/docs/fedora-coreos/bare-metal.md @@ -282,10 +282,10 @@ systemd[1]: Started Kubernetes control plane. ``` $ export KUBECONFIG=/home/user/.secrets/clusters/mercury/auth/kubeconfig $ kubectl get nodes -NAME STATUS ROLES AGE VERSION -node1.example.com Ready controller,master 10m v1.16.0 -node2.example.com Ready node 10m v1.16.0 -node3.example.com Ready node 10m v1.16.0 +NAME STATUS ROLES AGE VERSION +node1.example.com Ready 10m v1.16.0 +node2.example.com Ready 10m v1.16.0 +node3.example.com Ready 10m v1.16.0 ``` List the pods. diff --git a/docs/index.md b/docs/index.md index e9e15f456..beffd63cb 100644 --- a/docs/index.md +++ b/docs/index.md @@ -79,10 +79,10 @@ In 4-8 minutes (varies by platform), the cluster will be ready. This Google Clou ``` $ export KUBECONFIG=/home/user/.secrets/clusters/yavin/auth/kubeconfig $ kubectl get nodes -NAME ROLES STATUS AGE VERSION -yavin-controller-0.c.example-com.internal controller,master Ready 6m v1.16.0 -yavin-worker-jrbf.c.example-com.internal node Ready 5m v1.16.0 -yavin-worker-mzdm.c.example-com.internal node Ready 5m v1.16.0 +NAME ROLES STATUS AGE VERSION +yavin-controller-0.c.example-com.internal Ready 6m v1.16.0 +yavin-worker-jrbf.c.example-com.internal Ready 5m v1.16.0 +yavin-worker-mzdm.c.example-com.internal Ready 5m v1.16.0 ``` List the pods. From 8703f2c3c57597e5c60832198bfa2ae7971cff87 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Mon, 23 Sep 2019 10:56:22 -0700 Subject: [PATCH 220/523] Fix missing comma separator on bare-metal and DO * Introduced in bare-metal and DigitalOcean in #544 while addressing possible ordering race, but after the v1.16 upgrade validation --- CHANGES.md | 8 ++++++++ bare-metal/container-linux/kubernetes/ssh.tf | 2 +- bare-metal/fedora-coreos/kubernetes/ssh.tf | 2 +- digital-ocean/container-linux/kubernetes/ssh.tf | 2 +- 4 files changed, 11 insertions(+), 3 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index a76236281..cfb5ec8ba 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -4,6 +4,14 @@ Notable changes between versions. ## Latest +#### Bare-Metal + +* Fix Terraform missing comma error ([#549](https://github.com/poseidon/typhoon/pull/549)) + +#### DigitalOcean + +* Fix Terraform missing comma error ([#549](https://github.com/poseidon/typhoon/pull/549)) + ## v1.16.0 * Kubernetes [v1.16.0](https://github.com/kubernetes/kubernetes/blob/master/CHANGELOG-1.16.md#v1160) ([#543](https://github.com/poseidon/typhoon/pull/543)) diff --git a/bare-metal/container-linux/kubernetes/ssh.tf b/bare-metal/container-linux/kubernetes/ssh.tf index a9674f98c..899f76474 100644 --- a/bare-metal/container-linux/kubernetes/ssh.tf +++ b/bare-metal/container-linux/kubernetes/ssh.tf @@ -76,7 +76,7 @@ resource "null_resource" "copy-controller-secrets" { "sudo chown -R etcd:etcd /etc/ssl/etcd", "sudo chmod -R 500 /etc/ssl/etcd", "sudo mv $HOME/assets /opt/bootstrap/assets", - "sudo mkdir -p /etc/kubernetes/manifests" + "sudo mkdir -p /etc/kubernetes/manifests", "sudo mkdir -p /etc/kubernetes/bootstrap-secrets", "sudo mv $HOME/kubeconfig /etc/kubernetes/kubeconfig", "sudo cp -r /opt/bootstrap/assets/tls/* /etc/kubernetes/bootstrap-secrets/", diff --git a/bare-metal/fedora-coreos/kubernetes/ssh.tf b/bare-metal/fedora-coreos/kubernetes/ssh.tf index 931801959..0feeaceef 100644 --- a/bare-metal/fedora-coreos/kubernetes/ssh.tf +++ b/bare-metal/fedora-coreos/kubernetes/ssh.tf @@ -73,7 +73,7 @@ resource "null_resource" "copy-controller-secrets" { "sudo mv etcd-peer.crt /etc/ssl/etcd/etcd/peer.crt", "sudo mv etcd-peer.key /etc/ssl/etcd/etcd/peer.key", "sudo mv $HOME/assets /opt/bootstrap/assets", - "sudo mkdir -p /etc/kubernetes/manifests" + "sudo mkdir -p /etc/kubernetes/manifests", "sudo mkdir -p /etc/kubernetes/bootstrap-secrets", "sudo mv $HOME/kubeconfig /etc/kubernetes/kubeconfig", "sudo cp -r /opt/bootstrap/assets/tls/* /etc/kubernetes/bootstrap-secrets/", diff --git a/digital-ocean/container-linux/kubernetes/ssh.tf b/digital-ocean/container-linux/kubernetes/ssh.tf index fbc9ebf7d..958c9ad81 100644 --- a/digital-ocean/container-linux/kubernetes/ssh.tf +++ b/digital-ocean/container-linux/kubernetes/ssh.tf @@ -72,7 +72,7 @@ resource "null_resource" "copy-controller-secrets" { "sudo chown -R etcd:etcd /etc/ssl/etcd", "sudo chmod -R 500 /etc/ssl/etcd", "sudo mv $HOME/assets /opt/bootstrap/assets", - "sudo mkdir -p /etc/kubernetes/manifests" + "sudo mkdir -p /etc/kubernetes/manifests", "sudo mkdir -p /etc/kubernetes/bootstrap-secrets", "sudo mv $HOME/kubeconfig /etc/kubernetes/kubeconfig", "sudo cp -r /opt/bootstrap/assets/tls/* /etc/kubernetes/bootstrap-secrets/", From 99ab81f79c600d747439a0646b181b5885be44ed Mon Sep 17 00:00:00 2001 From: Valer Cara Date: Sun, 29 Sep 2019 00:59:24 +0300 Subject: [PATCH 221/523] Add node_labels variable in workers modules to set initial node labels (#550) * Also add `worker_node_labels` variable in `kubernetes` modules to set initial node labels for the default workers --- aws/container-linux/kubernetes/variables.tf | 6 ++++++ aws/container-linux/kubernetes/workers.tf | 1 + aws/container-linux/kubernetes/workers/cl/worker.yaml.tmpl | 3 +++ aws/container-linux/kubernetes/workers/variables.tf | 5 +++++ aws/container-linux/kubernetes/workers/workers.tf | 1 + aws/fedora-coreos/kubernetes/variables.tf | 5 +++++ aws/fedora-coreos/kubernetes/workers.tf | 1 + aws/fedora-coreos/kubernetes/workers/fcc/worker.yaml | 3 +++ aws/fedora-coreos/kubernetes/workers/variables.tf | 5 +++++ aws/fedora-coreos/kubernetes/workers/workers.tf | 1 + azure/container-linux/kubernetes/variables.tf | 6 ++++++ azure/container-linux/kubernetes/workers.tf | 1 + .../container-linux/kubernetes/workers/cl/worker.yaml.tmpl | 3 +++ azure/container-linux/kubernetes/workers/variables.tf | 5 +++++ azure/container-linux/kubernetes/workers/workers.tf | 1 + google-cloud/container-linux/kubernetes/variables.tf | 6 ++++++ google-cloud/container-linux/kubernetes/workers.tf | 1 + .../container-linux/kubernetes/workers/cl/worker.yaml.tmpl | 3 +++ .../container-linux/kubernetes/workers/variables.tf | 6 ++++++ google-cloud/container-linux/kubernetes/workers/workers.tf | 1 + 20 files changed, 64 insertions(+) diff --git a/aws/container-linux/kubernetes/variables.tf b/aws/container-linux/kubernetes/variables.tf index 7005e63b6..b76f3d54e 100644 --- a/aws/container-linux/kubernetes/variables.tf +++ b/aws/container-linux/kubernetes/variables.tf @@ -154,3 +154,9 @@ variable "enable_aggregation" { default = "false" } +variable "worker_node_labels" { + description = "List of additional labels to add to worker nodes" + type = list + default = [] +} + diff --git a/aws/container-linux/kubernetes/workers.tf b/aws/container-linux/kubernetes/workers.tf index b5df22138..9bebffdeb 100644 --- a/aws/container-linux/kubernetes/workers.tf +++ b/aws/container-linux/kubernetes/workers.tf @@ -19,5 +19,6 @@ module "workers" { service_cidr = var.service_cidr cluster_domain_suffix = var.cluster_domain_suffix clc_snippets = var.worker_clc_snippets + node_labels = var.worker_node_labels } diff --git a/aws/container-linux/kubernetes/workers/cl/worker.yaml.tmpl b/aws/container-linux/kubernetes/workers/cl/worker.yaml.tmpl index a54411410..7d18b1631 100644 --- a/aws/container-linux/kubernetes/workers/cl/worker.yaml.tmpl +++ b/aws/container-linux/kubernetes/workers/cl/worker.yaml.tmpl @@ -61,6 +61,9 @@ systemd: --lock-file=/var/run/lock/kubelet.lock \ --network-plugin=cni \ --node-labels=node.kubernetes.io/node \ + %{ for label in split(",", node_labels) } + --node-labels=${label} \ + %{ endfor ~} --pod-manifest-path=/etc/kubernetes/manifests \ --read-only-port=0 \ --volume-plugin-dir=/var/lib/kubelet/volumeplugins diff --git a/aws/container-linux/kubernetes/workers/variables.tf b/aws/container-linux/kubernetes/workers/variables.tf index e5c5b6082..830a480cd 100644 --- a/aws/container-linux/kubernetes/workers/variables.tf +++ b/aws/container-linux/kubernetes/workers/variables.tf @@ -105,3 +105,8 @@ variable "cluster_domain_suffix" { default = "cluster.local" } +variable "node_labels" { + description = "List of additional labels to add to worker nodes" + type = list + default = [] +} diff --git a/aws/container-linux/kubernetes/workers/workers.tf b/aws/container-linux/kubernetes/workers/workers.tf index 5effb7890..79729bef0 100644 --- a/aws/container-linux/kubernetes/workers/workers.tf +++ b/aws/container-linux/kubernetes/workers/workers.tf @@ -86,6 +86,7 @@ data "template_file" "worker-config" { cluster_dns_service_ip = cidrhost(var.service_cidr, 10) cluster_domain_suffix = var.cluster_domain_suffix cgroup_driver = local.flavor == "flatcar" && local.channel == "edge" ? "systemd" : "cgroupfs" + node_labels = join(",", var.node_labels) } } diff --git a/aws/fedora-coreos/kubernetes/variables.tf b/aws/fedora-coreos/kubernetes/variables.tf index 8df68d856..da2643fd6 100644 --- a/aws/fedora-coreos/kubernetes/variables.tf +++ b/aws/fedora-coreos/kubernetes/variables.tf @@ -154,3 +154,8 @@ variable "enable_aggregation" { default = "false" } +variable "worker_node_labels" { + description = "List of additional labels to add to worker nodes" + type = list + default = [] +} diff --git a/aws/fedora-coreos/kubernetes/workers.tf b/aws/fedora-coreos/kubernetes/workers.tf index 4a2e50764..e8b57e620 100644 --- a/aws/fedora-coreos/kubernetes/workers.tf +++ b/aws/fedora-coreos/kubernetes/workers.tf @@ -19,5 +19,6 @@ module "workers" { service_cidr = var.service_cidr cluster_domain_suffix = var.cluster_domain_suffix snippets = var.worker_snippets + node_labels = var.worker_node_labels } diff --git a/aws/fedora-coreos/kubernetes/workers/fcc/worker.yaml b/aws/fedora-coreos/kubernetes/workers/fcc/worker.yaml index 79ab90652..549749410 100644 --- a/aws/fedora-coreos/kubernetes/workers/fcc/worker.yaml +++ b/aws/fedora-coreos/kubernetes/workers/fcc/worker.yaml @@ -66,6 +66,9 @@ systemd: --lock-file=/var/run/lock/kubelet.lock \ --network-plugin=cni \ --node-labels=node.kubernetes.io/node \ + %{ for label in split(",", node_labels) } + --node-labels=${label} \ + %{ endfor ~} --pod-manifest-path=/etc/kubernetes/manifests \ --read-only-port=0 \ --volume-plugin-dir=/var/lib/kubelet/volumeplugins diff --git a/aws/fedora-coreos/kubernetes/workers/variables.tf b/aws/fedora-coreos/kubernetes/workers/variables.tf index a1de562fb..90700e004 100644 --- a/aws/fedora-coreos/kubernetes/workers/variables.tf +++ b/aws/fedora-coreos/kubernetes/workers/variables.tf @@ -105,3 +105,8 @@ variable "cluster_domain_suffix" { default = "cluster.local" } +variable "node_labels" { + description = "List of additional labels to add to worker nodes" + type = list + default = [] +} diff --git a/aws/fedora-coreos/kubernetes/workers/workers.tf b/aws/fedora-coreos/kubernetes/workers/workers.tf index 4ea1dec08..f63fd2b2d 100644 --- a/aws/fedora-coreos/kubernetes/workers/workers.tf +++ b/aws/fedora-coreos/kubernetes/workers/workers.tf @@ -85,6 +85,7 @@ data "template_file" "worker-config" { ssh_authorized_key = var.ssh_authorized_key cluster_dns_service_ip = cidrhost(var.service_cidr, 10) cluster_domain_suffix = var.cluster_domain_suffix + node_labels = join(",", var.node_labels) } } diff --git a/azure/container-linux/kubernetes/variables.tf b/azure/container-linux/kubernetes/variables.tf index 74b2dde83..77bfdb4c4 100644 --- a/azure/container-linux/kubernetes/variables.tf +++ b/azure/container-linux/kubernetes/variables.tf @@ -135,3 +135,9 @@ variable "enable_aggregation" { default = "false" } +variable "worker_node_labels" { + description = "List of additional labels to add to worker nodes" + type = list + default = [] +} + diff --git a/azure/container-linux/kubernetes/workers.tf b/azure/container-linux/kubernetes/workers.tf index 5775f55c9..11b77c50f 100644 --- a/azure/container-linux/kubernetes/workers.tf +++ b/azure/container-linux/kubernetes/workers.tf @@ -20,5 +20,6 @@ module "workers" { service_cidr = var.service_cidr cluster_domain_suffix = var.cluster_domain_suffix clc_snippets = var.worker_clc_snippets + node_labels = var.worker_node_labels } diff --git a/azure/container-linux/kubernetes/workers/cl/worker.yaml.tmpl b/azure/container-linux/kubernetes/workers/cl/worker.yaml.tmpl index 7c3a68115..f12be17ea 100644 --- a/azure/container-linux/kubernetes/workers/cl/worker.yaml.tmpl +++ b/azure/container-linux/kubernetes/workers/cl/worker.yaml.tmpl @@ -59,6 +59,9 @@ systemd: --lock-file=/var/run/lock/kubelet.lock \ --network-plugin=cni \ --node-labels=node.kubernetes.io/node \ + %{ for label in split(",", node_labels) } + --node-labels=${label} \ + %{ endfor ~} --pod-manifest-path=/etc/kubernetes/manifests \ --read-only-port=0 \ --volume-plugin-dir=/var/lib/kubelet/volumeplugins diff --git a/azure/container-linux/kubernetes/workers/variables.tf b/azure/container-linux/kubernetes/workers/variables.tf index a430a94a6..026222ac9 100644 --- a/azure/container-linux/kubernetes/workers/variables.tf +++ b/azure/container-linux/kubernetes/workers/variables.tf @@ -91,3 +91,8 @@ variable "cluster_domain_suffix" { default = "cluster.local" } +variable "node_labels" { + description = "List of additional labels to add to worker nodes" + type = list + default = [] +} diff --git a/azure/container-linux/kubernetes/workers/workers.tf b/azure/container-linux/kubernetes/workers/workers.tf index 9e32c4b68..d6d6d3c27 100644 --- a/azure/container-linux/kubernetes/workers/workers.tf +++ b/azure/container-linux/kubernetes/workers/workers.tf @@ -111,6 +111,7 @@ data "template_file" "worker-config" { ssh_authorized_key = var.ssh_authorized_key cluster_dns_service_ip = cidrhost(var.service_cidr, 10) cluster_domain_suffix = var.cluster_domain_suffix + node_labels = join(",", var.node_labels) } } diff --git a/google-cloud/container-linux/kubernetes/variables.tf b/google-cloud/container-linux/kubernetes/variables.tf index 874deecb8..e6c4120a2 100644 --- a/google-cloud/container-linux/kubernetes/variables.tf +++ b/google-cloud/container-linux/kubernetes/variables.tf @@ -129,3 +129,9 @@ variable "enable_aggregation" { default = "false" } +variable "worker_node_labels" { + description = "List of additional labels to add to worker nodes" + type = list + default = [] +} + diff --git a/google-cloud/container-linux/kubernetes/workers.tf b/google-cloud/container-linux/kubernetes/workers.tf index eef186687..00e34c5f1 100644 --- a/google-cloud/container-linux/kubernetes/workers.tf +++ b/google-cloud/container-linux/kubernetes/workers.tf @@ -18,5 +18,6 @@ module "workers" { service_cidr = var.service_cidr cluster_domain_suffix = var.cluster_domain_suffix clc_snippets = var.worker_clc_snippets + node_labels = var.worker_node_labels } diff --git a/google-cloud/container-linux/kubernetes/workers/cl/worker.yaml.tmpl b/google-cloud/container-linux/kubernetes/workers/cl/worker.yaml.tmpl index ddc78680c..29e6cd9a2 100644 --- a/google-cloud/container-linux/kubernetes/workers/cl/worker.yaml.tmpl +++ b/google-cloud/container-linux/kubernetes/workers/cl/worker.yaml.tmpl @@ -60,6 +60,9 @@ systemd: --lock-file=/var/run/lock/kubelet.lock \ --network-plugin=cni \ --node-labels=node.kubernetes.io/node \ + %{ for label in split(",", node_labels) } + --node-labels=${label} \ + %{ endfor ~} --pod-manifest-path=/etc/kubernetes/manifests \ --read-only-port=0 \ --volume-plugin-dir=/var/lib/kubelet/volumeplugins diff --git a/google-cloud/container-linux/kubernetes/workers/variables.tf b/google-cloud/container-linux/kubernetes/workers/variables.tf index a6c6aba33..292af163f 100644 --- a/google-cloud/container-linux/kubernetes/workers/variables.tf +++ b/google-cloud/container-linux/kubernetes/workers/variables.tf @@ -81,6 +81,12 @@ variable "cluster_domain_suffix" { default = "cluster.local" } +variable "node_labels" { + description = "List of additional labels to add to worker nodes" + type = list + default = [] +} + variable "clc_snippets" { type = list(string) description = "Container Linux Config snippets" diff --git a/google-cloud/container-linux/kubernetes/workers/workers.tf b/google-cloud/container-linux/kubernetes/workers/workers.tf index 18b938f74..83eea9484 100644 --- a/google-cloud/container-linux/kubernetes/workers/workers.tf +++ b/google-cloud/container-linux/kubernetes/workers/workers.tf @@ -82,6 +82,7 @@ data "template_file" "worker-config" { ssh_authorized_key = var.ssh_authorized_key cluster_dns_service_ip = cidrhost(var.service_cidr, 10) cluster_domain_suffix = var.cluster_domain_suffix + node_labels = join(",", var.node_labels) } } From 9bfb1c5faf5de5fd2373985b2460525bde9d6a8c Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Sat, 28 Sep 2019 15:01:14 -0700 Subject: [PATCH 222/523] Update docs and variable types for worker node_labels * Document worker pools `node_labels` variable to set the initial node labels for a homogeneous set of workers * Document `worker_node_labels` convenience variable to set the initial node labels for default worker nodes --- CHANGES.md | 12 ++++++++++++ aws/container-linux/kubernetes/variables.tf | 4 ++-- .../kubernetes/workers/variables.tf | 4 ++-- aws/fedora-coreos/kubernetes/variables.tf | 4 ++-- aws/fedora-coreos/kubernetes/workers/variables.tf | 4 ++-- azure/container-linux/kubernetes/variables.tf | 4 ++-- .../kubernetes/workers/variables.tf | 4 ++-- docs/advanced/worker-pools.md | 5 +++++ docs/cl/aws.md | 1 + docs/cl/azure.md | 1 + docs/cl/google-cloud.md | 1 + docs/fedora-coreos/aws.md | 1 + .../container-linux/kubernetes/variables.tf | 4 ++-- .../kubernetes/workers/variables.tf | 14 +++++++------- 14 files changed, 42 insertions(+), 21 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index cfb5ec8ba..16e197e5b 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -4,6 +4,18 @@ Notable changes between versions. ## Latest +#### AWS + +* Add `node_labels` variable to internal `workers` module ([#550](https://github.com/poseidon/typhoon/pull/550)) + +#### Azure + +* Add `node_labels` variable to internal `workers` module ([#550](https://github.com/poseidon/typhoon/pull/550)) + +#### Google Cloud + +* Add `node_labels` variable to internal `workers` module ([#550](https://github.com/poseidon/typhoon/pull/550)) + #### Bare-Metal * Fix Terraform missing comma error ([#549](https://github.com/poseidon/typhoon/pull/549)) diff --git a/aws/container-linux/kubernetes/variables.tf b/aws/container-linux/kubernetes/variables.tf index b76f3d54e..2809f880e 100644 --- a/aws/container-linux/kubernetes/variables.tf +++ b/aws/container-linux/kubernetes/variables.tf @@ -155,8 +155,8 @@ variable "enable_aggregation" { } variable "worker_node_labels" { - description = "List of additional labels to add to worker nodes" - type = list + type = list(string) + description = "List of initial worker node labels" default = [] } diff --git a/aws/container-linux/kubernetes/workers/variables.tf b/aws/container-linux/kubernetes/workers/variables.tf index 830a480cd..cb733db25 100644 --- a/aws/container-linux/kubernetes/workers/variables.tf +++ b/aws/container-linux/kubernetes/workers/variables.tf @@ -106,7 +106,7 @@ variable "cluster_domain_suffix" { } variable "node_labels" { - description = "List of additional labels to add to worker nodes" - type = list + type = list(string) + description = "List of initial node labels" default = [] } diff --git a/aws/fedora-coreos/kubernetes/variables.tf b/aws/fedora-coreos/kubernetes/variables.tf index da2643fd6..bb19230bf 100644 --- a/aws/fedora-coreos/kubernetes/variables.tf +++ b/aws/fedora-coreos/kubernetes/variables.tf @@ -155,7 +155,7 @@ variable "enable_aggregation" { } variable "worker_node_labels" { - description = "List of additional labels to add to worker nodes" - type = list + type = list(string) + description = "List of initial worker node labels" default = [] } diff --git a/aws/fedora-coreos/kubernetes/workers/variables.tf b/aws/fedora-coreos/kubernetes/workers/variables.tf index 90700e004..8979d9c29 100644 --- a/aws/fedora-coreos/kubernetes/workers/variables.tf +++ b/aws/fedora-coreos/kubernetes/workers/variables.tf @@ -106,7 +106,7 @@ variable "cluster_domain_suffix" { } variable "node_labels" { - description = "List of additional labels to add to worker nodes" - type = list + type = list(string) + description = "List of initial node labels" default = [] } diff --git a/azure/container-linux/kubernetes/variables.tf b/azure/container-linux/kubernetes/variables.tf index 77bfdb4c4..31328e39c 100644 --- a/azure/container-linux/kubernetes/variables.tf +++ b/azure/container-linux/kubernetes/variables.tf @@ -136,8 +136,8 @@ variable "enable_aggregation" { } variable "worker_node_labels" { - description = "List of additional labels to add to worker nodes" - type = list + type = list(string) + description = "List of initial worker node labels" default = [] } diff --git a/azure/container-linux/kubernetes/workers/variables.tf b/azure/container-linux/kubernetes/workers/variables.tf index 026222ac9..2de845be2 100644 --- a/azure/container-linux/kubernetes/workers/variables.tf +++ b/azure/container-linux/kubernetes/workers/variables.tf @@ -92,7 +92,7 @@ variable "cluster_domain_suffix" { } variable "node_labels" { - description = "List of additional labels to add to worker nodes" - type = list + type = list(string) + description = "List of initial node labels" default = [] } diff --git a/docs/advanced/worker-pools.md b/docs/advanced/worker-pools.md index f3430cc48..dd82649ec 100644 --- a/docs/advanced/worker-pools.md +++ b/docs/advanced/worker-pools.md @@ -65,8 +65,10 @@ The AWS internal `workers` module supports a number of [variables](https://githu | os_image | AMI channel for a Container Linux derivative | coreos-stable | coreos-stable, coreos-beta, coreos-alpha, flatcar-stable, flatcar-beta, flatcar-alpha | | disk_size | Size of the disk in GB | 40 | 100 | | spot_price | Spot price in USD for workers. Leave as default empty string for regular on-demand instances | "" | "0.10" | +| clc_snippets | Container Linux Config snippets | [] | [example](/advanced/customization/#usage) | | service_cidr | Must match `service_cidr` of cluster | "10.3.0.0/16" | "10.3.0.0/24" | | cluster_domain_suffix | Must match `cluster_domain_suffix` of cluster | "cluster.local" | "k8s.example.com" | +| node_labels | List of initial node labels | [] | ["worker-pool=foo"] | Check the list of valid [instance types](https://aws.amazon.com/ec2/instance-types/) or per-region and per-type [spot prices](https://aws.amazon.com/ec2/spot/pricing/). @@ -133,6 +135,7 @@ The Azure internal `workers` module supports a number of [variables](https://git | clc_snippets | Container Linux Config snippets | [] | [example](/advanced/customization/#usage) | | service_cidr | CIDR IPv4 range to assign to Kubernetes services | "10.3.0.0/16" | "10.3.0.0/24" | | cluster_domain_suffix | FQDN suffix for Kubernetes services answered by coredns. | "cluster.local" | "k8s.example.com" | +| node_labels | List of initial node labels | [] | ["worker-pool=foo"] | Check the list of valid [machine types](https://azure.microsoft.com/en-us/pricing/details/virtual-machines/linux/) and their [specs](https://docs.microsoft.com/en-us/azure/virtual-machines/linux/sizes-general). Use `az vm list-skus` to get the identifier. @@ -206,8 +209,10 @@ Check the list of regions [docs](https://cloud.google.com/compute/docs/regions-z | os_image | Container Linux image for compute instances | "coreos-stable" | "coreos-alpha", "coreos-beta" | | disk_size | Size of the disk in GB | 40 | 100 | | preemptible | If true, Compute Engine will terminate instances randomly within 24 hours | false | true | +| clc_snippets | Container Linux Config snippets | [] | [example](/advanced/customization/#usage) | | service_cidr | Must match `service_cidr` of cluster | "10.3.0.0/16" | "10.3.0.0/24" | | cluster_domain_suffix | Must match `cluster_domain_suffix` of cluster | "cluster.local" | "k8s.example.com" | +| node_labels | List of initial node labels | [] | ["worker-pool=foo"] | Check the list of valid [machine types](https://cloud.google.com/compute/docs/machine-types). diff --git a/docs/cl/aws.md b/docs/cl/aws.md index 200314c53..dc887b73c 100644 --- a/docs/cl/aws.md +++ b/docs/cl/aws.md @@ -209,6 +209,7 @@ Reference the DNS zone id with `"${aws_route53_zone.zone-for-clusters.zone_id}"` | disk_type | Type of the EBS volume | "gp2" | standard, gp2, io1 | | disk_iops | IOPS of the EBS volume | "0" (i.e. auto) | "400" | | worker_target_groups | Target group ARNs to which worker instances should be added | [] | ["${aws_lb_target_group.app.id}"] | +| worker_node_labels | List of initial worker node labels | [] | ["worker-pool=default"] | | worker_price | Spot price in USD for workers. Leave as default empty string for regular on-demand instances | "" | "0.10" | | controller_clc_snippets | Controller Container Linux Config snippets | [] | [example](/advanced/customization/) | | worker_clc_snippets | Worker Container Linux Config snippets | [] | [example](/advanced/customization/) | diff --git a/docs/cl/azure.md b/docs/cl/azure.md index 566d439ad..96bc13926 100644 --- a/docs/cl/azure.md +++ b/docs/cl/azure.md @@ -217,6 +217,7 @@ Reference the DNS zone with `"${azurerm_dns_zone.clusters.name}"` and its resour | worker_type | Machine type for workers | "Standard_F1" | See below | | os_image | Channel for a Container Linux derivative | coreos-stable | coreos-stable, coreos-beta, coreos-alpha | | disk_size | Size of the disk in GB | "40" | "100" | +| worker_node_labels | List of initial worker node labels | [] | ["worker-pool=default"] | | worker_priority | Set priority to Low to use reduced cost surplus capacity, with the tradeoff that instances can be deallocated at any time | Regular | Low | | controller_clc_snippets | Controller Container Linux Config snippets | [] | [example](/advanced/customization/#usage) | | worker_clc_snippets | Worker Container Linux Config snippets | [] | [example](/advanced/customization/#usage) | diff --git a/docs/cl/google-cloud.md b/docs/cl/google-cloud.md index da809973f..a33b4cd71 100644 --- a/docs/cl/google-cloud.md +++ b/docs/cl/google-cloud.md @@ -211,6 +211,7 @@ resource "google_dns_managed_zone" "zone-for-clusters" { | worker_type | Machine type for workers | "n1-standard-1" | See below | | os_image | Container Linux image for compute instances | "coreos-stable" | "coreos-stable-1632-3-0-v20180215" | | disk_size | Size of the disk in GB | 40 | 100 | +| worker_node_labels | List of initial worker node labels | [] | ["worker-pool=default"] | | worker_preemptible | If enabled, Compute Engine will terminate workers randomly within 24 hours | false | true | | controller_clc_snippets | Controller Container Linux Config snippets | [] | [example](/advanced/customization/) | | worker_clc_snippets | Worker Container Linux Config snippets | [] | [example](/advanced/customization/) | diff --git a/docs/fedora-coreos/aws.md b/docs/fedora-coreos/aws.md index 694db1237..f8bf2313c 100644 --- a/docs/fedora-coreos/aws.md +++ b/docs/fedora-coreos/aws.md @@ -209,6 +209,7 @@ Reference the DNS zone id with `"${aws_route53_zone.zone-for-clusters.zone_id}"` | disk_type | Type of the EBS volume | "gp2" | standard, gp2, io1 | | disk_iops | IOPS of the EBS volume | "0" (i.e. auto) | "400" | | worker_target_groups | Target group ARNs to which worker instances should be added | [] | ["${aws_lb_target_group.app.id}"] | +| worker_node_labels | List of initial worker node labels | [] | ["worker-pool=default"] | | worker_price | Spot price in USD for workers. Leave as default empty string for regular on-demand instances | "" | "0.10" | | controller_snippets | Controller Fedora CoreOS Config snippets | [] | UNSUPPORTED | | worker_clc_snippets | Worker Fedora CoreOS Config snippets | [] | UNSUPPORTED | diff --git a/google-cloud/container-linux/kubernetes/variables.tf b/google-cloud/container-linux/kubernetes/variables.tf index e6c4120a2..e767a8fdf 100644 --- a/google-cloud/container-linux/kubernetes/variables.tf +++ b/google-cloud/container-linux/kubernetes/variables.tf @@ -130,8 +130,8 @@ variable "enable_aggregation" { } variable "worker_node_labels" { - description = "List of additional labels to add to worker nodes" - type = list + type = list(string) + description = "List of initial worker node labels" default = [] } diff --git a/google-cloud/container-linux/kubernetes/workers/variables.tf b/google-cloud/container-linux/kubernetes/workers/variables.tf index 292af163f..b6f58d389 100644 --- a/google-cloud/container-linux/kubernetes/workers/variables.tf +++ b/google-cloud/container-linux/kubernetes/workers/variables.tf @@ -52,6 +52,12 @@ variable "preemptible" { description = "If enabled, Compute Engine will terminate instances randomly within 24 hours" } +variable "clc_snippets" { + type = list(string) + description = "Container Linux Config snippets" + default = [] +} + # configuration variable "kubeconfig" { @@ -82,14 +88,8 @@ variable "cluster_domain_suffix" { } variable "node_labels" { - description = "List of additional labels to add to worker nodes" - type = list - default = [] -} - -variable "clc_snippets" { type = list(string) - description = "Container Linux Config snippets" + description = "List of initial node labels" default = [] } From 3e34fb075b3350b310971260e042c5ff23436ca3 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Sat, 28 Sep 2019 15:09:57 -0700 Subject: [PATCH 223/523] Update etcd from v3.4.0 to v3.4.1 * https://github.com/etcd-io/etcd/releases/tag/v3.4.1 --- CHANGES.md | 11 +++++++---- .../kubernetes/cl/controller.yaml.tmpl | 2 +- aws/fedora-coreos/kubernetes/fcc/controller.yaml | 2 +- .../kubernetes/cl/controller.yaml.tmpl | 2 +- .../kubernetes/cl/controller.yaml.tmpl | 2 +- .../fedora-coreos/kubernetes/fcc/controller.yaml | 2 +- .../kubernetes/cl/controller.yaml.tmpl | 2 +- .../kubernetes/cl/controller.yaml.tmpl | 2 +- 8 files changed, 14 insertions(+), 11 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 16e197e5b..1918c1732 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -4,6 +4,8 @@ Notable changes between versions. ## Latest +* Update etcd from v3.3.15 to [v3.4.1](https://github.com/etcd-io/etcd/releases/tag/v3.4.1) + #### AWS * Add `node_labels` variable to internal `workers` module ([#550](https://github.com/poseidon/typhoon/pull/550)) @@ -12,10 +14,6 @@ Notable changes between versions. * Add `node_labels` variable to internal `workers` module ([#550](https://github.com/poseidon/typhoon/pull/550)) -#### Google Cloud - -* Add `node_labels` variable to internal `workers` module ([#550](https://github.com/poseidon/typhoon/pull/550)) - #### Bare-Metal * Fix Terraform missing comma error ([#549](https://github.com/poseidon/typhoon/pull/549)) @@ -24,6 +22,11 @@ Notable changes between versions. * Fix Terraform missing comma error ([#549](https://github.com/poseidon/typhoon/pull/549)) +#### Google Cloud + +* Add `node_labels` variable to internal `workers` module ([#550](https://github.com/poseidon/typhoon/pull/550)) + + ## v1.16.0 * Kubernetes [v1.16.0](https://github.com/kubernetes/kubernetes/blob/master/CHANGELOG-1.16.md#v1160) ([#543](https://github.com/poseidon/typhoon/pull/543)) diff --git a/aws/container-linux/kubernetes/cl/controller.yaml.tmpl b/aws/container-linux/kubernetes/cl/controller.yaml.tmpl index 60e8ab31e..086ced6ba 100644 --- a/aws/container-linux/kubernetes/cl/controller.yaml.tmpl +++ b/aws/container-linux/kubernetes/cl/controller.yaml.tmpl @@ -7,7 +7,7 @@ systemd: - name: 40-etcd-cluster.conf contents: | [Service] - Environment="ETCD_IMAGE_TAG=v3.4.0" + Environment="ETCD_IMAGE_TAG=v3.4.1" Environment="ETCD_NAME=${etcd_name}" Environment="ETCD_ADVERTISE_CLIENT_URLS=https://${etcd_domain}:2379" Environment="ETCD_INITIAL_ADVERTISE_PEER_URLS=https://${etcd_domain}:2380" diff --git a/aws/fedora-coreos/kubernetes/fcc/controller.yaml b/aws/fedora-coreos/kubernetes/fcc/controller.yaml index 3d55f0452..5d3887264 100644 --- a/aws/fedora-coreos/kubernetes/fcc/controller.yaml +++ b/aws/fedora-coreos/kubernetes/fcc/controller.yaml @@ -28,7 +28,7 @@ systemd: --network host \ --volume /var/lib/etcd:/var/lib/etcd:rw,Z \ --volume /etc/ssl/etcd:/etc/ssl/certs:ro,Z \ - quay.io/coreos/etcd:v3.4.0 + quay.io/coreos/etcd:v3.4.1 ExecStop=/usr/bin/podman stop etcd [Install] WantedBy=multi-user.target diff --git a/azure/container-linux/kubernetes/cl/controller.yaml.tmpl b/azure/container-linux/kubernetes/cl/controller.yaml.tmpl index 0b1bc8e9f..2007857ea 100644 --- a/azure/container-linux/kubernetes/cl/controller.yaml.tmpl +++ b/azure/container-linux/kubernetes/cl/controller.yaml.tmpl @@ -7,7 +7,7 @@ systemd: - name: 40-etcd-cluster.conf contents: | [Service] - Environment="ETCD_IMAGE_TAG=v3.4.0" + Environment="ETCD_IMAGE_TAG=v3.4.1" Environment="ETCD_NAME=${etcd_name}" Environment="ETCD_ADVERTISE_CLIENT_URLS=https://${etcd_domain}:2379" Environment="ETCD_INITIAL_ADVERTISE_PEER_URLS=https://${etcd_domain}:2380" diff --git a/bare-metal/container-linux/kubernetes/cl/controller.yaml.tmpl b/bare-metal/container-linux/kubernetes/cl/controller.yaml.tmpl index 356c7bff4..e63ba3783 100644 --- a/bare-metal/container-linux/kubernetes/cl/controller.yaml.tmpl +++ b/bare-metal/container-linux/kubernetes/cl/controller.yaml.tmpl @@ -7,7 +7,7 @@ systemd: - name: 40-etcd-cluster.conf contents: | [Service] - Environment="ETCD_IMAGE_TAG=v3.4.0" + Environment="ETCD_IMAGE_TAG=v3.4.1" Environment="ETCD_NAME=${etcd_name}" Environment="ETCD_ADVERTISE_CLIENT_URLS=https://${domain_name}:2379" Environment="ETCD_INITIAL_ADVERTISE_PEER_URLS=https://${domain_name}:2380" diff --git a/bare-metal/fedora-coreos/kubernetes/fcc/controller.yaml b/bare-metal/fedora-coreos/kubernetes/fcc/controller.yaml index 7ecf846a3..f9f4fa546 100644 --- a/bare-metal/fedora-coreos/kubernetes/fcc/controller.yaml +++ b/bare-metal/fedora-coreos/kubernetes/fcc/controller.yaml @@ -28,7 +28,7 @@ systemd: --network host \ --volume /var/lib/etcd:/var/lib/etcd:rw,Z \ --volume /etc/ssl/etcd:/etc/ssl/certs:ro,Z \ - quay.io/coreos/etcd:v3.4.0 + quay.io/coreos/etcd:v3.4.1 ExecStop=/usr/bin/podman stop etcd [Install] WantedBy=multi-user.target diff --git a/digital-ocean/container-linux/kubernetes/cl/controller.yaml.tmpl b/digital-ocean/container-linux/kubernetes/cl/controller.yaml.tmpl index 801268c80..28e4c7a8c 100644 --- a/digital-ocean/container-linux/kubernetes/cl/controller.yaml.tmpl +++ b/digital-ocean/container-linux/kubernetes/cl/controller.yaml.tmpl @@ -7,7 +7,7 @@ systemd: - name: 40-etcd-cluster.conf contents: | [Service] - Environment="ETCD_IMAGE_TAG=v3.4.0" + Environment="ETCD_IMAGE_TAG=v3.4.1" Environment="ETCD_NAME=${etcd_name}" Environment="ETCD_ADVERTISE_CLIENT_URLS=https://${etcd_domain}:2379" Environment="ETCD_INITIAL_ADVERTISE_PEER_URLS=https://${etcd_domain}:2380" diff --git a/google-cloud/container-linux/kubernetes/cl/controller.yaml.tmpl b/google-cloud/container-linux/kubernetes/cl/controller.yaml.tmpl index 3d0bdbdb7..f864c22ab 100644 --- a/google-cloud/container-linux/kubernetes/cl/controller.yaml.tmpl +++ b/google-cloud/container-linux/kubernetes/cl/controller.yaml.tmpl @@ -7,7 +7,7 @@ systemd: - name: 40-etcd-cluster.conf contents: | [Service] - Environment="ETCD_IMAGE_TAG=v3.4.0" + Environment="ETCD_IMAGE_TAG=v3.4.1" Environment="ETCD_NAME=${etcd_name}" Environment="ETCD_ADVERTISE_CLIENT_URLS=https://${etcd_domain}:2379" Environment="ETCD_INITIAL_ADVERTISE_PEER_URLS=https://${etcd_domain}:2380" From f453c54956840be8bf9bfc684428094728d4524e Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Sat, 28 Sep 2019 15:13:46 -0700 Subject: [PATCH 224/523] Update Grafana from v6.3.5 to v6.3.6 * https://github.com/grafana/grafana/releases/tag/v6.3.6 --- CHANGES.md | 3 +++ addons/grafana/deployment.yaml | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index 1918c1732..e87d24678 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -26,6 +26,9 @@ Notable changes between versions. * Add `node_labels` variable to internal `workers` module ([#550](https://github.com/poseidon/typhoon/pull/550)) +#### Addons + +* Update Grafana from v6.3.5 to [v6.3.6](https://github.com/grafana/grafana/releases/tag/v6.3.6) ## v1.16.0 diff --git a/addons/grafana/deployment.yaml b/addons/grafana/deployment.yaml index 5c2671b64..6bef7bdb6 100644 --- a/addons/grafana/deployment.yaml +++ b/addons/grafana/deployment.yaml @@ -23,7 +23,7 @@ spec: spec: containers: - name: grafana - image: docker.io/grafana/grafana:6.3.5 + image: docker.io/grafana/grafana:6.3.6 env: - name: GF_PATHS_CONFIG value: "/etc/grafana/custom.ini" From a407ff72dff1d2ca0f6afd8ac03a1989d17c4a0d Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Sun, 29 Sep 2019 11:14:36 -0700 Subject: [PATCH 225/523] Add stricter types for AWS modules and update docs * Review variables available in AWS kubernetes and workers modules and documentation * Switching between spot and on-demand has worked since Terraform v0.12 * Generally, there are too many knobs. Less useful ones should be de-emphasized or removed * Remove `cluster_domain_suffix` documentation --- CHANGES.md | 1 + aws/container-linux/kubernetes/variables.tf | 68 +++++++++---------- .../kubernetes/workers/variables.tf | 30 ++++---- .../kubernetes/workers/workers.tf | 2 +- aws/fedora-coreos/kubernetes/variables.tf | 65 ++++++++++-------- .../kubernetes/workers/variables.tf | 30 ++++---- .../kubernetes/workers/workers.tf | 2 +- docs/advanced/worker-pools.md | 6 +- docs/cl/aws.md | 15 ++-- docs/fedora-coreos/aws.md | 13 ++-- 10 files changed, 118 insertions(+), 114 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index e87d24678..cb4ec108b 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -8,6 +8,7 @@ Notable changes between versions. #### AWS +* Add Terraform v0.12 variables types ([#553](https://github.com/poseidon/typhoon/pull/553)) * Add `node_labels` variable to internal `workers` module ([#550](https://github.com/poseidon/typhoon/pull/550)) #### Azure diff --git a/aws/container-linux/kubernetes/variables.tf b/aws/container-linux/kubernetes/variables.tf index 2809f880e..ba9c172c7 100644 --- a/aws/container-linux/kubernetes/variables.tf +++ b/aws/container-linux/kubernetes/variables.tf @@ -18,57 +18,57 @@ variable "dns_zone_id" { # instances variable "controller_count" { - type = string - default = "1" + type = number description = "Number of controllers (i.e. masters)" + default = 1 } variable "worker_count" { - type = string - default = "1" + type = number description = "Number of workers" + default = 1 } variable "controller_type" { type = string - default = "t3.small" description = "EC2 instance type for controllers" + default = "t3.small" } variable "worker_type" { type = string - default = "t3.small" description = "EC2 instance type for workers" + default = "t3.small" } variable "os_image" { type = string - default = "coreos-stable" description = "AMI channel for a Container Linux derivative (coreos-stable, coreos-beta, coreos-alpha, flatcar-stable, flatcar-beta, flatcar-alpha, flatcar-edge)" + default = "coreos-stable" } variable "disk_size" { - type = string - default = "40" + type = number description = "Size of the EBS volume in GB" + default = 40 } variable "disk_type" { type = string - default = "gp2" description = "Type of the EBS volume (e.g. standard, gp2, io1)" + default = "gp2" } variable "disk_iops" { - type = string - default = "0" + type = number description = "IOPS of the EBS volume (e.g. 100)" + default = 0 } variable "worker_price" { - type = string - default = "" - description = "Spot price in USD for autoscaling group spot instances. Leave as default empty string for autoscaling group to use on-demand instances. Note, switching in-place from spot to on-demand is not possible: https://github.com/terraform-providers/terraform-provider-aws/issues/4320" + type = number + description = "Spot price in USD for worker instances or 0 to use on-demand instances" + default = 0 } variable "worker_target_groups" { @@ -97,61 +97,53 @@ variable "ssh_authorized_key" { } variable "asset_dir" { - description = "Path to a directory where generated assets should be placed (contains secrets)" type = string + description = "Absolute path to a directory where generated assets should be placed (contains secrets)" } variable "networking" { - description = "Choice of networking provider (calico or flannel)" type = string + description = "Choice of networking provider (calico or flannel)" default = "calico" } variable "network_mtu" { + type = number description = "CNI interface MTU (applies to calico only). Use 8981 if using instances types with Jumbo frames." - type = string - default = "1480" + default = 1480 } variable "host_cidr" { - description = "CIDR IPv4 range to assign to EC2 nodes" type = string + description = "CIDR IPv4 range to assign to EC2 nodes" default = "10.0.0.0/16" } variable "pod_cidr" { - description = "CIDR IPv4 range to assign Kubernetes pods" type = string + description = "CIDR IPv4 range to assign Kubernetes pods" default = "10.2.0.0/16" } variable "service_cidr" { + type = string description = < 0 ? var.spot_price : null enable_monitoring = false user_data = data.ct_config.worker-ignition.rendered diff --git a/aws/fedora-coreos/kubernetes/variables.tf b/aws/fedora-coreos/kubernetes/variables.tf index bb19230bf..6b8d3e760 100644 --- a/aws/fedora-coreos/kubernetes/variables.tf +++ b/aws/fedora-coreos/kubernetes/variables.tf @@ -18,57 +18,57 @@ variable "dns_zone_id" { # instances variable "controller_count" { - type = string - default = "1" + type = number description = "Number of controllers (i.e. masters)" + default = 1 } variable "worker_count" { - type = string - default = "1" + type = number description = "Number of workers" + default = 1 } variable "controller_type" { type = string - default = "t3.small" description = "EC2 instance type for controllers" + default = "t3.small" } variable "worker_type" { type = string - default = "t3.small" description = "EC2 instance type for workers" + default = "t3.small" } variable "os_image" { type = string - default = "coreos-stable" description = "AMI channel for Fedora CoreOS (not yet used)" + default = "coreos-stable" } variable "disk_size" { - type = string - default = "40" + type = number description = "Size of the EBS volume in GB" + default = 40 } variable "disk_type" { type = string - default = "gp2" description = "Type of the EBS volume (e.g. standard, gp2, io1)" + default = "gp2" } variable "disk_iops" { - type = string - default = "0" + type = number description = "IOPS of the EBS volume (e.g. 100)" + default = 0 } variable "worker_price" { - type = string - default = "" - description = "Spot price in USD for autoscaling group spot instances. Leave as default empty string for autoscaling group to use on-demand instances. Note, switching in-place from spot to on-demand is not possible: https://github.com/terraform-providers/terraform-provider-aws/issues/4320" + type = number + description = "Spot price in USD for worker instances or 0 to use on-demand instances" + default = 0 } variable "worker_target_groups" { @@ -97,61 +97,59 @@ variable "ssh_authorized_key" { } variable "asset_dir" { - description = "Path to a directory where generated assets should be placed (contains secrets)" type = string + description = "Absolute path to a directory where generated assets should be placed (contains secrets)" } variable "networking" { - description = "Choice of networking provider (calico or flannel)" type = string + description = "Choice of networking provider (calico or flannel)" default = "calico" } variable "network_mtu" { + type = number description = "CNI interface MTU (applies to calico only). Use 8981 if using instances types with Jumbo frames." - type = string - default = "1480" + default = 1480 } variable "host_cidr" { - description = "CIDR IPv4 range to assign to EC2 nodes" type = string + description = "CIDR IPv4 range to assign to EC2 nodes" default = "10.0.0.0/16" } variable "pod_cidr" { - description = "CIDR IPv4 range to assign Kubernetes pods" type = string + description = "CIDR IPv4 range to assign Kubernetes pods" default = "10.2.0.0/16" } variable "service_cidr" { + type = string description = < 0 ? var.spot_price : null enable_monitoring = false user_data = data.ct_config.worker-ignition.rendered diff --git a/docs/advanced/worker-pools.md b/docs/advanced/worker-pools.md index dd82649ec..a689011e5 100644 --- a/docs/advanced/worker-pools.md +++ b/docs/advanced/worker-pools.md @@ -63,8 +63,10 @@ The AWS internal `workers` module supports a number of [variables](https://githu | worker_count | Number of instances | 1 | 3 | | instance_type | EC2 instance type | "t3.small" | "t3.medium" | | os_image | AMI channel for a Container Linux derivative | coreos-stable | coreos-stable, coreos-beta, coreos-alpha, flatcar-stable, flatcar-beta, flatcar-alpha | -| disk_size | Size of the disk in GB | 40 | 100 | -| spot_price | Spot price in USD for workers. Leave as default empty string for regular on-demand instances | "" | "0.10" | +| disk_size | Size of the EBS volume in GB | 40 | 100 | +| disk_type | Type of the EBS volume | "gp2" | standard, gp2, io1 | +| disk_iops | IOPS of the EBS volume | 0 (i.e. auto) | 400 | +| spot_price | Spot price in USD for worker instances or 0 to use on-demand instances | 0 | 0.10 | | clc_snippets | Container Linux Config snippets | [] | [example](/advanced/customization/#usage) | | service_cidr | Must match `service_cidr` of cluster | "10.3.0.0/16" | "10.3.0.0/24" | | cluster_domain_suffix | Must match `cluster_domain_suffix` of cluster | "cluster.local" | "k8s.example.com" | diff --git a/docs/cl/aws.md b/docs/cl/aws.md index dc887b73c..dc96fac7b 100644 --- a/docs/cl/aws.md +++ b/docs/cl/aws.md @@ -177,7 +177,7 @@ Check the [variables.tf](https://github.com/poseidon/typhoon/blob/master/aws/con | dns_zone | AWS Route53 DNS zone | "aws.example.com" | | dns_zone_id | AWS Route53 DNS zone id | "Z3PAABBCFAKEC0" | | ssh_authorized_key | SSH public key for user 'core' | "ssh-rsa AAAAB3NZ..." | -| asset_dir | Path to a directory where generated assets should be placed (contains secrets) | "/home/user/.secrets/clusters/tempest" | +| asset_dir | Absolute path to a directory where generated assets should be placed (contains secrets) | "/home/user/.secrets/clusters/tempest" | #### DNS Zone @@ -191,7 +191,7 @@ resource "aws_route53_zone" "zone-for-clusters" { } ``` -Reference the DNS zone id with `"${aws_route53_zone.zone-for-clusters.zone_id}"`. +Reference the DNS zone id with `aws_route53_zone.zone-for-clusters.zone_id`. !!! tip "" If you have an existing domain name with a zone file elsewhere, just delegate a subdomain that can be managed on Route53 (e.g. aws.mydomain.com) and [update nameservers](http://docs.aws.amazon.com/Route53/latest/DeveloperGuide/SOA-NSrecords.html). @@ -205,12 +205,11 @@ Reference the DNS zone id with `"${aws_route53_zone.zone-for-clusters.zone_id}"` | controller_type | EC2 instance type for controllers | "t3.small" | See below | | worker_type | EC2 instance type for workers | "t3.small" | See below | | os_image | AMI channel for a Container Linux derivative | coreos-stable | coreos-stable, coreos-beta, coreos-alpha, flatcar-stable, flatcar-beta, flatcar-alpha, flatcar-edge | -| disk_size | Size of the EBS volume in GB | "40" | "100" | +| disk_size | Size of the EBS volume in GB | 40 | 100 | | disk_type | Type of the EBS volume | "gp2" | standard, gp2, io1 | -| disk_iops | IOPS of the EBS volume | "0" (i.e. auto) | "400" | -| worker_target_groups | Target group ARNs to which worker instances should be added | [] | ["${aws_lb_target_group.app.id}"] | -| worker_node_labels | List of initial worker node labels | [] | ["worker-pool=default"] | -| worker_price | Spot price in USD for workers. Leave as default empty string for regular on-demand instances | "" | "0.10" | +| disk_iops | IOPS of the EBS volume | 0 (i.e. auto) | 400 | +| worker_target_groups | Target group ARNs to which worker instances should be added | [] | [aws_lb_target_group.app.id] | +| worker_price | Spot price in USD for worker instances or 0 to use on-demand instances | 0/null | 0.10 | | controller_clc_snippets | Controller Container Linux Config snippets | [] | [example](/advanced/customization/) | | worker_clc_snippets | Worker Container Linux Config snippets | [] | [example](/advanced/customization/) | | networking | Choice of networking provider | "calico" | "calico" or "flannel" | @@ -218,7 +217,7 @@ Reference the DNS zone id with `"${aws_route53_zone.zone-for-clusters.zone_id}"` | host_cidr | CIDR IPv4 range to assign to EC2 instances | "10.0.0.0/16" | "10.1.0.0/16" | | pod_cidr | CIDR IPv4 range to assign to Kubernetes pods | "10.2.0.0/16" | "10.22.0.0/16" | | service_cidr | CIDR IPv4 range to assign to Kubernetes services | "10.3.0.0/16" | "10.3.0.0/24" | -| cluster_domain_suffix | FQDN suffix for Kubernetes services answered by coredns. | "cluster.local" | "k8s.example.com" | +| worker_node_labels | List of initial worker node labels | [] | ["worker-pool=default"] | Check the list of valid [instance types](https://aws.amazon.com/ec2/instance-types/). diff --git a/docs/fedora-coreos/aws.md b/docs/fedora-coreos/aws.md index f8bf2313c..bfb36c2f1 100644 --- a/docs/fedora-coreos/aws.md +++ b/docs/fedora-coreos/aws.md @@ -191,7 +191,7 @@ resource "aws_route53_zone" "zone-for-clusters" { } ``` -Reference the DNS zone id with `"${aws_route53_zone.zone-for-clusters.zone_id}"`. +Reference the DNS zone id with `aws_route53_zone.zone-for-clusters.zone_id`. !!! tip "" If you have an existing domain name with a zone file elsewhere, just delegate a subdomain that can be managed on Route53 (e.g. aws.mydomain.com) and [update nameservers](http://docs.aws.amazon.com/Route53/latest/DeveloperGuide/SOA-NSrecords.html). @@ -205,12 +205,11 @@ Reference the DNS zone id with `"${aws_route53_zone.zone-for-clusters.zone_id}"` | controller_type | EC2 instance type for controllers | "t3.small" | See below | | worker_type | EC2 instance type for workers | "t3.small" | See below | | os_image | AMI channel for Fedora CoreOS | not yet used | ? | -| disk_size | Size of the EBS volume in GB | "40" | "100" | +| disk_size | Size of the EBS volume in GB | 40 | 100 | | disk_type | Type of the EBS volume | "gp2" | standard, gp2, io1 | -| disk_iops | IOPS of the EBS volume | "0" (i.e. auto) | "400" | -| worker_target_groups | Target group ARNs to which worker instances should be added | [] | ["${aws_lb_target_group.app.id}"] | -| worker_node_labels | List of initial worker node labels | [] | ["worker-pool=default"] | -| worker_price | Spot price in USD for workers. Leave as default empty string for regular on-demand instances | "" | "0.10" | +| disk_iops | IOPS of the EBS volume | 0 (i.e. auto) | 400 | +| worker_target_groups | Target group ARNs to which worker instances should be added | [] | [aws_lb_target_group.app.id] | +| worker_price | Spot price in USD for worker instances or 0 to use on-demand instances | 0 | 0.10 | | controller_snippets | Controller Fedora CoreOS Config snippets | [] | UNSUPPORTED | | worker_clc_snippets | Worker Fedora CoreOS Config snippets | [] | UNSUPPORTED | | networking | Choice of networking provider | "calico" | "calico" or "flannel" | @@ -218,7 +217,7 @@ Reference the DNS zone id with `"${aws_route53_zone.zone-for-clusters.zone_id}"` | host_cidr | CIDR IPv4 range to assign to EC2 instances | "10.0.0.0/16" | "10.1.0.0/16" | | pod_cidr | CIDR IPv4 range to assign to Kubernetes pods | "10.2.0.0/16" | "10.22.0.0/16" | | service_cidr | CIDR IPv4 range to assign to Kubernetes services | "10.3.0.0/16" | "10.3.0.0/24" | -| cluster_domain_suffix | FQDN suffix for Kubernetes services answered by coredns. | "cluster.local" | "k8s.example.com" | +| worker_node_labels | List of initial worker node labels | [] | ["worker-pool=default"] | Check the list of valid [instance types](https://aws.amazon.com/ec2/instance-types/). From 96afa6a531423c76e4345358287a99ca27754d96 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Sun, 29 Sep 2019 11:22:53 -0700 Subject: [PATCH 226/523] Update Calico from v3.8.2 to v3.9.1 * https://docs.projectcalico.org/v3.9/release-notes/ --- CHANGES.md | 1 + aws/container-linux/kubernetes/bootkube.tf | 2 +- aws/fedora-coreos/kubernetes/bootkube.tf | 2 +- azure/container-linux/kubernetes/bootkube.tf | 2 +- bare-metal/container-linux/kubernetes/bootkube.tf | 2 +- bare-metal/fedora-coreos/kubernetes/bootkube.tf | 2 +- digital-ocean/container-linux/kubernetes/bootkube.tf | 2 +- google-cloud/container-linux/kubernetes/bootkube.tf | 2 +- 8 files changed, 8 insertions(+), 7 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index cb4ec108b..fb9631c65 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -5,6 +5,7 @@ Notable changes between versions. ## Latest * Update etcd from v3.3.15 to [v3.4.1](https://github.com/etcd-io/etcd/releases/tag/v3.4.1) +* Update Calico from v3.8.2 to [v3.9.1](https://docs.projectcalico.org/v3.9/release-notes/) #### AWS diff --git a/aws/container-linux/kubernetes/bootkube.tf b/aws/container-linux/kubernetes/bootkube.tf index 53783725c..70cbe67aa 100644 --- a/aws/container-linux/kubernetes/bootkube.tf +++ b/aws/container-linux/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=539b725093c8cd94ba46603adb25ac5280562ec8" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=18b7a74d309da7290474e35701bc6668f0dd035e" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/aws/fedora-coreos/kubernetes/bootkube.tf b/aws/fedora-coreos/kubernetes/bootkube.tf index 4469d5146..54797f2f4 100644 --- a/aws/fedora-coreos/kubernetes/bootkube.tf +++ b/aws/fedora-coreos/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=539b725093c8cd94ba46603adb25ac5280562ec8" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=18b7a74d309da7290474e35701bc6668f0dd035e" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/azure/container-linux/kubernetes/bootkube.tf b/azure/container-linux/kubernetes/bootkube.tf index 52cf18671..09e436ef7 100644 --- a/azure/container-linux/kubernetes/bootkube.tf +++ b/azure/container-linux/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=539b725093c8cd94ba46603adb25ac5280562ec8" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=18b7a74d309da7290474e35701bc6668f0dd035e" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/bare-metal/container-linux/kubernetes/bootkube.tf b/bare-metal/container-linux/kubernetes/bootkube.tf index 06fb187dc..7cd04c643 100644 --- a/bare-metal/container-linux/kubernetes/bootkube.tf +++ b/bare-metal/container-linux/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=539b725093c8cd94ba46603adb25ac5280562ec8" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=18b7a74d309da7290474e35701bc6668f0dd035e" cluster_name = var.cluster_name api_servers = [var.k8s_domain_name] diff --git a/bare-metal/fedora-coreos/kubernetes/bootkube.tf b/bare-metal/fedora-coreos/kubernetes/bootkube.tf index cf3c9b505..8ee902b4d 100644 --- a/bare-metal/fedora-coreos/kubernetes/bootkube.tf +++ b/bare-metal/fedora-coreos/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=539b725093c8cd94ba46603adb25ac5280562ec8" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=18b7a74d309da7290474e35701bc6668f0dd035e" cluster_name = var.cluster_name api_servers = [var.k8s_domain_name] diff --git a/digital-ocean/container-linux/kubernetes/bootkube.tf b/digital-ocean/container-linux/kubernetes/bootkube.tf index ee794a92e..ec1c8588d 100644 --- a/digital-ocean/container-linux/kubernetes/bootkube.tf +++ b/digital-ocean/container-linux/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=539b725093c8cd94ba46603adb25ac5280562ec8" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=18b7a74d309da7290474e35701bc6668f0dd035e" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/google-cloud/container-linux/kubernetes/bootkube.tf b/google-cloud/container-linux/kubernetes/bootkube.tf index 490174060..94abfd0b7 100644 --- a/google-cloud/container-linux/kubernetes/bootkube.tf +++ b/google-cloud/container-linux/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=539b725093c8cd94ba46603adb25ac5280562ec8" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=18b7a74d309da7290474e35701bc6668f0dd035e" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] From a6de245d8ab890942512eed381d1cb425d2881a1 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Sun, 29 Sep 2019 11:30:49 -0700 Subject: [PATCH 227/523] Rename bootkube.tf to bootstrap.tf * Typhoon no longer uses the bootkube project --- aws/container-linux/kubernetes/{bootkube.tf => bootstrap.tf} | 0 aws/fedora-coreos/kubernetes/{bootkube.tf => bootstrap.tf} | 0 azure/container-linux/kubernetes/{bootkube.tf => bootstrap.tf} | 0 .../container-linux/kubernetes/{bootkube.tf => bootstrap.tf} | 0 bare-metal/fedora-coreos/kubernetes/{bootkube.tf => bootstrap.tf} | 0 .../container-linux/kubernetes/{bootkube.tf => bootstrap.tf} | 0 .../container-linux/kubernetes/{bootkube.tf => bootstrap.tf} | 0 7 files changed, 0 insertions(+), 0 deletions(-) rename aws/container-linux/kubernetes/{bootkube.tf => bootstrap.tf} (100%) rename aws/fedora-coreos/kubernetes/{bootkube.tf => bootstrap.tf} (100%) rename azure/container-linux/kubernetes/{bootkube.tf => bootstrap.tf} (100%) rename bare-metal/container-linux/kubernetes/{bootkube.tf => bootstrap.tf} (100%) rename bare-metal/fedora-coreos/kubernetes/{bootkube.tf => bootstrap.tf} (100%) rename digital-ocean/container-linux/kubernetes/{bootkube.tf => bootstrap.tf} (100%) rename google-cloud/container-linux/kubernetes/{bootkube.tf => bootstrap.tf} (100%) diff --git a/aws/container-linux/kubernetes/bootkube.tf b/aws/container-linux/kubernetes/bootstrap.tf similarity index 100% rename from aws/container-linux/kubernetes/bootkube.tf rename to aws/container-linux/kubernetes/bootstrap.tf diff --git a/aws/fedora-coreos/kubernetes/bootkube.tf b/aws/fedora-coreos/kubernetes/bootstrap.tf similarity index 100% rename from aws/fedora-coreos/kubernetes/bootkube.tf rename to aws/fedora-coreos/kubernetes/bootstrap.tf diff --git a/azure/container-linux/kubernetes/bootkube.tf b/azure/container-linux/kubernetes/bootstrap.tf similarity index 100% rename from azure/container-linux/kubernetes/bootkube.tf rename to azure/container-linux/kubernetes/bootstrap.tf diff --git a/bare-metal/container-linux/kubernetes/bootkube.tf b/bare-metal/container-linux/kubernetes/bootstrap.tf similarity index 100% rename from bare-metal/container-linux/kubernetes/bootkube.tf rename to bare-metal/container-linux/kubernetes/bootstrap.tf diff --git a/bare-metal/fedora-coreos/kubernetes/bootkube.tf b/bare-metal/fedora-coreos/kubernetes/bootstrap.tf similarity index 100% rename from bare-metal/fedora-coreos/kubernetes/bootkube.tf rename to bare-metal/fedora-coreos/kubernetes/bootstrap.tf diff --git a/digital-ocean/container-linux/kubernetes/bootkube.tf b/digital-ocean/container-linux/kubernetes/bootstrap.tf similarity index 100% rename from digital-ocean/container-linux/kubernetes/bootkube.tf rename to digital-ocean/container-linux/kubernetes/bootstrap.tf diff --git a/google-cloud/container-linux/kubernetes/bootkube.tf b/google-cloud/container-linux/kubernetes/bootstrap.tf similarity index 100% rename from google-cloud/container-linux/kubernetes/bootkube.tf rename to google-cloud/container-linux/kubernetes/bootstrap.tf From 78bfff0afe03550eea5f1683de299809bac66ec2 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Sun, 29 Sep 2019 11:34:31 -0700 Subject: [PATCH 228/523] Update Fedora CoreOS to testing 30.20190905.0 * Fix duplicated cluster_domain_suffix variable --- aws/fedora-coreos/kubernetes/ami.tf | 2 +- aws/fedora-coreos/kubernetes/variables.tf | 6 ------ aws/fedora-coreos/kubernetes/workers/ami.tf | 2 +- 3 files changed, 2 insertions(+), 8 deletions(-) diff --git a/aws/fedora-coreos/kubernetes/ami.tf b/aws/fedora-coreos/kubernetes/ami.tf index 71322d17a..626b1f834 100644 --- a/aws/fedora-coreos/kubernetes/ami.tf +++ b/aws/fedora-coreos/kubernetes/ami.tf @@ -16,6 +16,6 @@ data "aws_ami" "fedora-coreos" { // pin on known ok versions as preview matures filter { name = "name" - values = ["fedora-coreos-30.20190801.0-hvm"] + values = ["fedora-coreos-30.20190905.0-hvm"] } } diff --git a/aws/fedora-coreos/kubernetes/variables.tf b/aws/fedora-coreos/kubernetes/variables.tf index 6b8d3e760..dc25a4152 100644 --- a/aws/fedora-coreos/kubernetes/variables.tf +++ b/aws/fedora-coreos/kubernetes/variables.tf @@ -134,12 +134,6 @@ EOD default = "10.3.0.0/16" } -variable "cluster_domain_suffix" { - type = string - description = "Queries for domains with the suffix will be answered by coredns. Default is cluster.local (e.g. foo.default.svc.cluster.local) " - default = "cluster.local" -} - variable "enable_reporting" { type = bool description = "Enable usage or analytics reporting to upstreams (Calico)" diff --git a/aws/fedora-coreos/kubernetes/workers/ami.tf b/aws/fedora-coreos/kubernetes/workers/ami.tf index 71322d17a..626b1f834 100644 --- a/aws/fedora-coreos/kubernetes/workers/ami.tf +++ b/aws/fedora-coreos/kubernetes/workers/ami.tf @@ -16,6 +16,6 @@ data "aws_ami" "fedora-coreos" { // pin on known ok versions as preview matures filter { name = "name" - values = ["fedora-coreos-30.20190801.0-hvm"] + values = ["fedora-coreos-30.20190905.0-hvm"] } } From 7bcf2d7831a70f8689aff07598e194f7dc11f459 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Sun, 29 Sep 2019 11:56:31 -0700 Subject: [PATCH 229/523] Update nginx-ingress from v0.25.1 to v0.26.1 * Add lifecycle hook to allow draining connections for up to 5 minutes --- CHANGES.md | 2 ++ addons/nginx-ingress/aws/deployment.yaml | 9 +++++++-- addons/nginx-ingress/azure/deployment.yaml | 9 +++++++-- addons/nginx-ingress/bare-metal/deployment.yaml | 9 +++++++-- addons/nginx-ingress/digital-ocean/daemonset.yaml | 9 +++++++-- addons/nginx-ingress/google-cloud/deployment.yaml | 9 +++++++-- 6 files changed, 37 insertions(+), 10 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index fb9631c65..6263277bb 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -30,6 +30,8 @@ Notable changes between versions. #### Addons +* Update nginx-ingress from v0.25.1 to [v0.26.0](https://github.com/kubernetes/ingress-nginx/releases/tag/nginx-0.26.0) ([#555](https://github.com/poseidon/typhoon/pull/555)) + * Add lifecycle hook to allow draining for up to 5 minutes * Update Grafana from v6.3.5 to [v6.3.6](https://github.com/grafana/grafana/releases/tag/v6.3.6) ## v1.16.0 diff --git a/addons/nginx-ingress/aws/deployment.yaml b/addons/nginx-ingress/aws/deployment.yaml index 621c0e4d2..3d1382e66 100644 --- a/addons/nginx-ingress/aws/deployment.yaml +++ b/addons/nginx-ingress/aws/deployment.yaml @@ -22,7 +22,7 @@ spec: spec: containers: - name: nginx-ingress-controller - image: quay.io/kubernetes-ingress-controller/nginx-ingress-controller:0.25.1 + image: quay.io/kubernetes-ingress-controller/nginx-ingress-controller:0.26.1 args: - /nginx-ingress-controller - --ingress-class=public @@ -65,6 +65,11 @@ spec: periodSeconds: 10 successThreshold: 1 timeoutSeconds: 5 + lifecycle: + preStop: + exec: + command: + - /wait-shutdown securityContext: capabilities: add: @@ -73,4 +78,4 @@ spec: - ALL runAsUser: 33 # www-data restartPolicy: Always - terminationGracePeriodSeconds: 60 + terminationGracePeriodSeconds: 300 diff --git a/addons/nginx-ingress/azure/deployment.yaml b/addons/nginx-ingress/azure/deployment.yaml index 621c0e4d2..3d1382e66 100644 --- a/addons/nginx-ingress/azure/deployment.yaml +++ b/addons/nginx-ingress/azure/deployment.yaml @@ -22,7 +22,7 @@ spec: spec: containers: - name: nginx-ingress-controller - image: quay.io/kubernetes-ingress-controller/nginx-ingress-controller:0.25.1 + image: quay.io/kubernetes-ingress-controller/nginx-ingress-controller:0.26.1 args: - /nginx-ingress-controller - --ingress-class=public @@ -65,6 +65,11 @@ spec: periodSeconds: 10 successThreshold: 1 timeoutSeconds: 5 + lifecycle: + preStop: + exec: + command: + - /wait-shutdown securityContext: capabilities: add: @@ -73,4 +78,4 @@ spec: - ALL runAsUser: 33 # www-data restartPolicy: Always - terminationGracePeriodSeconds: 60 + terminationGracePeriodSeconds: 300 diff --git a/addons/nginx-ingress/bare-metal/deployment.yaml b/addons/nginx-ingress/bare-metal/deployment.yaml index 2e5c42f54..38b85c865 100644 --- a/addons/nginx-ingress/bare-metal/deployment.yaml +++ b/addons/nginx-ingress/bare-metal/deployment.yaml @@ -22,7 +22,7 @@ spec: spec: containers: - name: nginx-ingress-controller - image: quay.io/kubernetes-ingress-controller/nginx-ingress-controller:0.25.1 + image: quay.io/kubernetes-ingress-controller/nginx-ingress-controller:0.26.1 args: - /nginx-ingress-controller - --ingress-class=public @@ -62,6 +62,11 @@ spec: successThreshold: 1 failureThreshold: 3 timeoutSeconds: 5 + lifecycle: + preStop: + exec: + command: + - /wait-shutdown securityContext: capabilities: add: @@ -70,5 +75,5 @@ spec: - ALL runAsUser: 33 # www-data restartPolicy: Always - terminationGracePeriodSeconds: 60 + terminationGracePeriodSeconds: 300 diff --git a/addons/nginx-ingress/digital-ocean/daemonset.yaml b/addons/nginx-ingress/digital-ocean/daemonset.yaml index 21abeb716..3c2bb74de 100644 --- a/addons/nginx-ingress/digital-ocean/daemonset.yaml +++ b/addons/nginx-ingress/digital-ocean/daemonset.yaml @@ -22,7 +22,7 @@ spec: spec: containers: - name: nginx-ingress-controller - image: quay.io/kubernetes-ingress-controller/nginx-ingress-controller:0.25.1 + image: quay.io/kubernetes-ingress-controller/nginx-ingress-controller:0.26.1 args: - /nginx-ingress-controller - --ingress-class=public @@ -65,6 +65,11 @@ spec: periodSeconds: 10 successThreshold: 1 timeoutSeconds: 5 + lifecycle: + preStop: + exec: + command: + - /wait-shutdown securityContext: capabilities: add: @@ -73,4 +78,4 @@ spec: - ALL runAsUser: 33 # www-data restartPolicy: Always - terminationGracePeriodSeconds: 60 + terminationGracePeriodSeconds: 300 diff --git a/addons/nginx-ingress/google-cloud/deployment.yaml b/addons/nginx-ingress/google-cloud/deployment.yaml index 621c0e4d2..3d1382e66 100644 --- a/addons/nginx-ingress/google-cloud/deployment.yaml +++ b/addons/nginx-ingress/google-cloud/deployment.yaml @@ -22,7 +22,7 @@ spec: spec: containers: - name: nginx-ingress-controller - image: quay.io/kubernetes-ingress-controller/nginx-ingress-controller:0.25.1 + image: quay.io/kubernetes-ingress-controller/nginx-ingress-controller:0.26.1 args: - /nginx-ingress-controller - --ingress-class=public @@ -65,6 +65,11 @@ spec: periodSeconds: 10 successThreshold: 1 timeoutSeconds: 5 + lifecycle: + preStop: + exec: + command: + - /wait-shutdown securityContext: capabilities: add: @@ -73,4 +78,4 @@ spec: - ALL runAsUser: 33 # www-data restartPolicy: Always - terminationGracePeriodSeconds: 60 + terminationGracePeriodSeconds: 300 From f82266ac8c436bc451690179673f092c2a351a48 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Mon, 30 Sep 2019 22:04:35 -0700 Subject: [PATCH 230/523] Add stricter types for GCP modules * Review variables available in google-cloud kubernetes and workers modules and in documentation --- CHANGES.md | 1 + docs/advanced/worker-pools.md | 5 +- docs/cl/google-cloud.md | 7 ++- .../container-linux/kubernetes/variables.tf | 53 ++++++++++--------- .../kubernetes/workers/variables.tf | 32 ++++++----- 5 files changed, 47 insertions(+), 51 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 6263277bb..b1b9cf9d0 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -26,6 +26,7 @@ Notable changes between versions. #### Google Cloud +* Add Terraform v0.12 variables types ([#556](https://github.com/poseidon/typhoon/pull/556)) * Add `node_labels` variable to internal `workers` module ([#550](https://github.com/poseidon/typhoon/pull/550)) #### Addons diff --git a/docs/advanced/worker-pools.md b/docs/advanced/worker-pools.md index a689011e5..6512b6910 100644 --- a/docs/advanced/worker-pools.md +++ b/docs/advanced/worker-pools.md @@ -69,7 +69,6 @@ The AWS internal `workers` module supports a number of [variables](https://githu | spot_price | Spot price in USD for worker instances or 0 to use on-demand instances | 0 | 0.10 | | clc_snippets | Container Linux Config snippets | [] | [example](/advanced/customization/#usage) | | service_cidr | Must match `service_cidr` of cluster | "10.3.0.0/16" | "10.3.0.0/24" | -| cluster_domain_suffix | Must match `cluster_domain_suffix` of cluster | "cluster.local" | "k8s.example.com" | | node_labels | List of initial node labels | [] | ["worker-pool=foo"] | Check the list of valid [instance types](https://aws.amazon.com/ec2/instance-types/) or per-region and per-type [spot prices](https://aws.amazon.com/ec2/spot/pricing/). @@ -136,7 +135,6 @@ The Azure internal `workers` module supports a number of [variables](https://git | priority | Set priority to Low to use reduced cost surplus capacity, with the tradeoff that instances can be deallocated at any time | Regular | Low | | clc_snippets | Container Linux Config snippets | [] | [example](/advanced/customization/#usage) | | service_cidr | CIDR IPv4 range to assign to Kubernetes services | "10.3.0.0/16" | "10.3.0.0/24" | -| cluster_domain_suffix | FQDN suffix for Kubernetes services answered by coredns. | "cluster.local" | "k8s.example.com" | | node_labels | List of initial node labels | [] | ["worker-pool=foo"] | Check the list of valid [machine types](https://azure.microsoft.com/en-us/pricing/details/virtual-machines/linux/) and their [specs](https://docs.microsoft.com/en-us/azure/virtual-machines/linux/sizes-general). Use `az vm list-skus` to get the identifier. @@ -194,9 +192,9 @@ The Google Cloud internal `workers` module supports a number of [variables](http | Name | Description | Example | |:-----|:------------|:--------| | name | Unique name (distinct from cluster name) | "yavin-16x" | +| cluster_name | Must be set to `cluster_name` of cluster | "yavin" | | region | Region for the worker pool instances. May differ from the cluster's region | "europe-west2" | | network | Must be set to `network_name` output by cluster | module.cluster.network_name | -| cluster_name | Must be set to `cluster_name` of cluster | "yavin" | | kubeconfig | Must be set to `kubeconfig` output by cluster | module.cluster.kubeconfig | | ssh_authorized_key | SSH public key for user 'core' | "ssh-rsa AAAAB3NZ..." | @@ -213,7 +211,6 @@ Check the list of regions [docs](https://cloud.google.com/compute/docs/regions-z | preemptible | If true, Compute Engine will terminate instances randomly within 24 hours | false | true | | clc_snippets | Container Linux Config snippets | [] | [example](/advanced/customization/#usage) | | service_cidr | Must match `service_cidr` of cluster | "10.3.0.0/16" | "10.3.0.0/24" | -| cluster_domain_suffix | Must match `cluster_domain_suffix` of cluster | "cluster.local" | "k8s.example.com" | | node_labels | List of initial node labels | [] | ["worker-pool=foo"] | Check the list of valid [machine types](https://cloud.google.com/compute/docs/machine-types). diff --git a/docs/cl/google-cloud.md b/docs/cl/google-cloud.md index a33b4cd71..4c9e3e699 100644 --- a/docs/cl/google-cloud.md +++ b/docs/cl/google-cloud.md @@ -52,7 +52,7 @@ provider "google" { version = "2.15.0" project = "project-id" region = "us-central1" - credentials = "${file("~/.config/google-cloud/terraform.json")}" + credentials = file("~/.config/google-cloud/terraform.json") } provider "ct" { @@ -180,7 +180,7 @@ Check the [variables.tf](https://github.com/poseidon/typhoon/blob/master/google- | dns_zone | Google Cloud DNS zone | "google-cloud.example.com" | | dns_zone_name | Google Cloud DNS zone name | "example-zone" | | ssh_authorized_key | SSH public key for user 'core' | "ssh-rsa AAAAB3NZ..." | -| asset_dir | Path to a directory where generated assets should be placed (contains secrets) | "/home/user/.secrets/clusters/yavin" | +| asset_dir | Absolute path to a directory where generated assets should be placed (contains secrets) | "/home/user/.secrets/clusters/yavin" | Check the list of valid [regions](https://cloud.google.com/compute/docs/regions-zones/regions-zones) and list Container Linux [images](https://cloud.google.com/compute/docs/images) with `gcloud compute images list | grep coreos`. @@ -211,14 +211,13 @@ resource "google_dns_managed_zone" "zone-for-clusters" { | worker_type | Machine type for workers | "n1-standard-1" | See below | | os_image | Container Linux image for compute instances | "coreos-stable" | "coreos-stable-1632-3-0-v20180215" | | disk_size | Size of the disk in GB | 40 | 100 | -| worker_node_labels | List of initial worker node labels | [] | ["worker-pool=default"] | | worker_preemptible | If enabled, Compute Engine will terminate workers randomly within 24 hours | false | true | | controller_clc_snippets | Controller Container Linux Config snippets | [] | [example](/advanced/customization/) | | worker_clc_snippets | Worker Container Linux Config snippets | [] | [example](/advanced/customization/) | | networking | Choice of networking provider | "calico" | "calico" or "flannel" | | pod_cidr | CIDR IPv4 range to assign to Kubernetes pods | "10.2.0.0/16" | "10.22.0.0/16" | | service_cidr | CIDR IPv4 range to assign to Kubernetes services | "10.3.0.0/16" | "10.3.0.0/24" | -| cluster_domain_suffix | FQDN suffix for Kubernetes services answered by coredns. | "cluster.local" | "k8s.example.com" | +| worker_node_labels | List of initial worker node labels | [] | ["worker-pool=default"] | Check the list of valid [machine types](https://cloud.google.com/compute/docs/machine-types). diff --git a/google-cloud/container-linux/kubernetes/variables.tf b/google-cloud/container-linux/kubernetes/variables.tf index e767a8fdf..2425e1f88 100644 --- a/google-cloud/container-linux/kubernetes/variables.tf +++ b/google-cloud/container-linux/kubernetes/variables.tf @@ -23,45 +23,45 @@ variable "dns_zone_name" { # instances variable "controller_count" { - type = string - default = "1" + type = number description = "Number of controllers (i.e. masters)" + default = 1 } variable "worker_count" { - type = string - default = "1" + type = number description = "Number of workers" + default = 1 } variable "controller_type" { type = string - default = "n1-standard-1" description = "Machine type for controllers (see `gcloud compute machine-types list`)" + default = "n1-standard-1" } variable "worker_type" { type = string - default = "n1-standard-1" description = "Machine type for controllers (see `gcloud compute machine-types list`)" + default = "n1-standard-1" } variable "os_image" { type = string - default = "coreos-stable" description = "Container Linux image for compute instances (e.g. coreos-stable)" + default = "coreos-stable" } variable "disk_size" { - type = string - default = "40" + type = number description = "Size of the disk in GB" + default = 40 } variable "worker_preemptible" { - type = string - default = "false" + type = bool description = "If enabled, Compute Engine will terminate workers randomly within 24 hours" + default = false } variable "controller_clc_snippets" { @@ -84,49 +84,42 @@ variable "ssh_authorized_key" { } variable "asset_dir" { - description = "Path to a directory where generated assets should be placed (contains secrets)" type = string + description = "Absolute path to a directory where generated assets should be placed (contains secrets)" } variable "networking" { - description = "Choice of networking provider (flannel or calico)" type = string + description = "Choice of networking provider (flannel or calico)" default = "calico" } variable "pod_cidr" { - description = "CIDR IPv4 range to assign Kubernetes pods" type = string + description = "CIDR IPv4 range to assign Kubernetes pods" default = "10.2.0.0/16" } variable "service_cidr" { + type = string description = < Date: Mon, 30 Sep 2019 22:18:15 -0700 Subject: [PATCH 231/523] Add stricter types to Azure modules * Review variables available in Azure kubernetes and workers modules and sync with documentation * Fix internal workers module default type to Standard_DS1_v2 --- CHANGES.md | 4 +- azure/container-linux/kubernetes/variables.tf | 52 +++++++++---------- .../kubernetes/workers/variables.tf | 27 +++++----- docs/advanced/worker-pools.md | 8 +-- docs/cl/azure.md | 14 ++--- 5 files changed, 54 insertions(+), 51 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index b1b9cf9d0..c4a2c1e64 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -14,6 +14,8 @@ Notable changes between versions. #### Azure +* Add Terraform v0.12 variables types ([#557](https://github.com/poseidon/typhoon/pull/557)) +* Change `workers` module default `vm_type` to `Standard_DS1_v2` (followup to [#539](https://github.com/poseidon/typhoon/pull/539)) * Add `node_labels` variable to internal `workers` module ([#550](https://github.com/poseidon/typhoon/pull/550)) #### Bare-Metal @@ -31,7 +33,7 @@ Notable changes between versions. #### Addons -* Update nginx-ingress from v0.25.1 to [v0.26.0](https://github.com/kubernetes/ingress-nginx/releases/tag/nginx-0.26.0) ([#555](https://github.com/poseidon/typhoon/pull/555)) +* Update nginx-ingress from v0.25.1 to [v0.26.1](https://github.com/kubernetes/ingress-nginx/releases/tag/nginx-0.26.1) ([#555](https://github.com/poseidon/typhoon/pull/555)) * Add lifecycle hook to allow draining for up to 5 minutes * Update Grafana from v6.3.5 to [v6.3.6](https://github.com/grafana/grafana/releases/tag/v6.3.6) diff --git a/azure/container-linux/kubernetes/variables.tf b/azure/container-linux/kubernetes/variables.tf index 31328e39c..e80b91b24 100644 --- a/azure/container-linux/kubernetes/variables.tf +++ b/azure/container-linux/kubernetes/variables.tf @@ -23,27 +23,27 @@ variable "dns_zone_group" { # instances variable "controller_count" { - type = string - default = "1" + type = number description = "Number of controllers (i.e. masters)" + default = 1 } variable "worker_count" { - type = string - default = "1" + type = number description = "Number of workers" + default = 1 } variable "controller_type" { type = string - default = "Standard_B2s" description = "Machine type for controllers (see `az vm list-skus --location centralus`)" + default = "Standard_B2s" } variable "worker_type" { type = string - default = "Standard_DS1_v2" description = "Machine type for workers (see `az vm list-skus --location centralus`)" + default = "Standard_DS1_v2" } variable "os_image" { @@ -53,15 +53,15 @@ variable "os_image" { } variable "disk_size" { - type = string - default = "40" + type = number description = "Size of the disk in GB" + default = 40 } variable "worker_priority" { type = string - default = "Regular" description = "Set worker priority to Low to use reduced cost surplus capacity, with the tradeoff that instances can be deallocated at any time." + default = "Regular" } variable "controller_clc_snippets" { @@ -84,55 +84,47 @@ variable "ssh_authorized_key" { } variable "asset_dir" { - description = "Path to a directory where generated assets should be placed (contains secrets)" type = string + description = "Absolute path to a directory where generated assets should be placed (contains secrets)" } variable "networking" { - description = "Choice of networking provider (flannel or calico)" type = string + description = "Choice of networking provider (flannel or calico)" default = "flannel" } variable "host_cidr" { - description = "CIDR IPv4 range to assign to instances" type = string + description = "CIDR IPv4 range to assign to instances" default = "10.0.0.0/16" } variable "pod_cidr" { - description = "CIDR IPv4 range to assign Kubernetes pods" type = string + description = "CIDR IPv4 range to assign Kubernetes pods" default = "10.2.0.0/16" } variable "service_cidr" { + type = string description = < Date: Tue, 1 Oct 2019 20:50:33 -0700 Subject: [PATCH 232/523] Update kube-state-metrics from v1.7.2 to v1.8.0 * https://github.com/kubernetes/kube-state-metrics/releases/tag/v1.8.0 --- CHANGES.md | 1 + addons/prometheus/exporters/kube-state-metrics/deployment.yaml | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index c4a2c1e64..81ad77fdb 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -33,6 +33,7 @@ Notable changes between versions. #### Addons +* Update kube-state-metrics from v1.7.2 to v1.8.0 * Update nginx-ingress from v0.25.1 to [v0.26.1](https://github.com/kubernetes/ingress-nginx/releases/tag/nginx-0.26.1) ([#555](https://github.com/poseidon/typhoon/pull/555)) * Add lifecycle hook to allow draining for up to 5 minutes * Update Grafana from v6.3.5 to [v6.3.6](https://github.com/grafana/grafana/releases/tag/v6.3.6) diff --git a/addons/prometheus/exporters/kube-state-metrics/deployment.yaml b/addons/prometheus/exporters/kube-state-metrics/deployment.yaml index 74639bec2..c9694eab3 100644 --- a/addons/prometheus/exporters/kube-state-metrics/deployment.yaml +++ b/addons/prometheus/exporters/kube-state-metrics/deployment.yaml @@ -24,7 +24,7 @@ spec: serviceAccountName: kube-state-metrics containers: - name: kube-state-metrics - image: quay.io/coreos/kube-state-metrics:v1.7.2 + image: quay.io/coreos/kube-state-metrics:v1.8.0 ports: - name: metrics containerPort: 8080 From ca7d62720ece8a06ac1c3381d49f455b9ecdc030 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Tue, 1 Oct 2019 20:51:58 -0700 Subject: [PATCH 233/523] Update Grafana from v6.3.6 to v6.4.1 * https://github.com/grafana/grafana/releases/tag/v6.4.1 --- CHANGES.md | 2 +- addons/grafana/deployment.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 81ad77fdb..4aad1cf2f 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -36,7 +36,7 @@ Notable changes between versions. * Update kube-state-metrics from v1.7.2 to v1.8.0 * Update nginx-ingress from v0.25.1 to [v0.26.1](https://github.com/kubernetes/ingress-nginx/releases/tag/nginx-0.26.1) ([#555](https://github.com/poseidon/typhoon/pull/555)) * Add lifecycle hook to allow draining for up to 5 minutes -* Update Grafana from v6.3.5 to [v6.3.6](https://github.com/grafana/grafana/releases/tag/v6.3.6) +* Update Grafana from v6.3.5 to [v6.4.1](https://github.com/grafana/grafana/releases/tag/v6.4.1) ## v1.16.0 diff --git a/addons/grafana/deployment.yaml b/addons/grafana/deployment.yaml index 6bef7bdb6..2f4b95dc1 100644 --- a/addons/grafana/deployment.yaml +++ b/addons/grafana/deployment.yaml @@ -23,7 +23,7 @@ spec: spec: containers: - name: grafana - image: docker.io/grafana/grafana:6.3.6 + image: docker.io/grafana/grafana:6.4.1 env: - name: GF_PATHS_CONFIG value: "/etc/grafana/custom.ini" From 1c5ed84fc2f108c1d01020efb5fd09a69a2ef2d4 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Wed, 2 Oct 2019 21:31:55 -0700 Subject: [PATCH 234/523] Update Kubernetes from v1.16.0 to v1.16.1 * https://github.com/kubernetes/kubernetes/blob/master/CHANGELOG-1.16.md#v1161 --- CHANGES.md | 1 + README.md | 10 +++++----- aws/container-linux/kubernetes/README.md | 2 +- aws/container-linux/kubernetes/bootstrap.tf | 2 +- .../kubernetes/cl/controller.yaml.tmpl | 4 ++-- .../kubernetes/workers/cl/worker.yaml.tmpl | 4 ++-- aws/fedora-coreos/kubernetes/README.md | 2 +- aws/fedora-coreos/kubernetes/bootstrap.tf | 2 +- aws/fedora-coreos/kubernetes/fcc/controller.yaml | 4 ++-- .../kubernetes/workers/fcc/worker.yaml | 2 +- azure/container-linux/kubernetes/README.md | 2 +- azure/container-linux/kubernetes/bootstrap.tf | 2 +- .../kubernetes/cl/controller.yaml.tmpl | 4 ++-- .../kubernetes/workers/cl/worker.yaml.tmpl | 4 ++-- bare-metal/container-linux/kubernetes/README.md | 2 +- bare-metal/container-linux/kubernetes/bootstrap.tf | 2 +- .../kubernetes/cl/controller.yaml.tmpl | 4 ++-- .../container-linux/kubernetes/cl/worker.yaml.tmpl | 2 +- bare-metal/fedora-coreos/kubernetes/README.md | 2 +- bare-metal/fedora-coreos/kubernetes/bootstrap.tf | 2 +- .../fedora-coreos/kubernetes/fcc/controller.yaml | 4 ++-- .../fedora-coreos/kubernetes/fcc/worker.yaml | 2 +- digital-ocean/container-linux/kubernetes/README.md | 2 +- .../container-linux/kubernetes/bootstrap.tf | 2 +- .../kubernetes/cl/controller.yaml.tmpl | 4 ++-- .../container-linux/kubernetes/cl/worker.yaml.tmpl | 4 ++-- docs/advanced/worker-pools.md | 14 +++++++------- docs/cl/aws.md | 10 +++++----- docs/cl/azure.md | 10 +++++----- docs/cl/bare-metal.md | 14 +++++++------- docs/cl/digital-ocean.md | 10 +++++----- docs/cl/google-cloud.md | 10 +++++----- docs/fedora-coreos/aws.md | 8 ++++---- docs/fedora-coreos/bare-metal.md | 8 ++++---- docs/index.md | 10 +++++----- docs/topics/maintenance.md | 10 +++++----- google-cloud/container-linux/kubernetes/README.md | 2 +- .../container-linux/kubernetes/bootstrap.tf | 2 +- .../kubernetes/cl/controller.yaml.tmpl | 4 ++-- .../kubernetes/workers/cl/worker.yaml.tmpl | 4 ++-- 40 files changed, 97 insertions(+), 96 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 4aad1cf2f..1e1974b7e 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -4,6 +4,7 @@ Notable changes between versions. ## Latest +* Kubernetes [v1.16.1](https://github.com/kubernetes/kubernetes/blob/master/CHANGELOG-1.16.md#v1161) * Update etcd from v3.3.15 to [v3.4.1](https://github.com/etcd-io/etcd/releases/tag/v3.4.1) * Update Calico from v3.8.2 to [v3.9.1](https://docs.projectcalico.org/v3.9/release-notes/) diff --git a/README.md b/README.md index 66ccb339a..69b0c4944 100644 --- a/README.md +++ b/README.md @@ -11,7 +11,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features -* Kubernetes v1.16.0 (upstream) +* Kubernetes v1.16.1 (upstream) * Single or multi-master, [Calico](https://www.projectcalico.org/) or [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/) * Advanced features like [worker pools](https://typhoon.psdn.io/advanced/worker-pools/), [preemptible](https://typhoon.psdn.io/cl/google-cloud/#preemption) workers, and [snippets](https://typhoon.psdn.io/advanced/customization/#container-linux) customization @@ -48,7 +48,7 @@ Define a Kubernetes cluster by using the Terraform module for your chosen platfo ```tf module "google-cloud-yavin" { - source = "git::https://github.com/poseidon/typhoon//google-cloud/container-linux/kubernetes?ref=v1.16.0" + source = "git::https://github.com/poseidon/typhoon//google-cloud/container-linux/kubernetes?ref=v1.16.1" # Google Cloud cluster_name = "yavin" @@ -82,9 +82,9 @@ In 4-8 minutes (varies by platform), the cluster will be ready. This Google Clou $ export KUBECONFIG=/home/user/.secrets/clusters/yavin/auth/kubeconfig $ kubectl get nodes NAME ROLES STATUS AGE VERSION -yavin-controller-0.c.example-com.internal Ready 6m v1.16.0 -yavin-worker-jrbf.c.example-com.internal Ready 5m v1.16.0 -yavin-worker-mzdm.c.example-com.internal Ready 5m v1.16.0 +yavin-controller-0.c.example-com.internal Ready 6m v1.16.1 +yavin-worker-jrbf.c.example-com.internal Ready 5m v1.16.1 +yavin-worker-mzdm.c.example-com.internal Ready 5m v1.16.1 ``` List the pods. diff --git a/aws/container-linux/kubernetes/README.md b/aws/container-linux/kubernetes/README.md index bb495cb39..2637d61fd 100644 --- a/aws/container-linux/kubernetes/README.md +++ b/aws/container-linux/kubernetes/README.md @@ -11,7 +11,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features -* Kubernetes v1.16.0 (upstream) +* Kubernetes v1.16.1 (upstream) * Single or multi-master, [Calico](https://www.projectcalico.org/) or [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/) * Advanced features like [worker pools](https://typhoon.psdn.io/advanced/worker-pools/), [spot](https://typhoon.psdn.io/cl/aws/#spot) workers, and [snippets](https://typhoon.psdn.io/advanced/customization/#container-linux) customization diff --git a/aws/container-linux/kubernetes/bootstrap.tf b/aws/container-linux/kubernetes/bootstrap.tf index 70cbe67aa..be3ffa101 100644 --- a/aws/container-linux/kubernetes/bootstrap.tf +++ b/aws/container-linux/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=18b7a74d309da7290474e35701bc6668f0dd035e" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=586d6e36f67c56fb2283f317a7552638368c5779" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/aws/container-linux/kubernetes/cl/controller.yaml.tmpl b/aws/container-linux/kubernetes/cl/controller.yaml.tmpl index 086ced6ba..78188f8a6 100644 --- a/aws/container-linux/kubernetes/cl/controller.yaml.tmpl +++ b/aws/container-linux/kubernetes/cl/controller.yaml.tmpl @@ -113,7 +113,7 @@ systemd: --volume script,kind=host,source=/opt/bootstrap/apply \ --mount volume=script,target=/apply \ --insecure-options=image \ - docker://k8s.gcr.io/hyperkube:v1.16.0 \ + docker://k8s.gcr.io/hyperkube:v1.16.1 \ --net=host \ --dns=host \ --exec=/apply @@ -134,7 +134,7 @@ storage: contents: inline: | KUBELET_IMAGE_URL=docker://k8s.gcr.io/hyperkube - KUBELET_IMAGE_TAG=v1.16.0 + KUBELET_IMAGE_TAG=v1.16.1 - path: /opt/bootstrap/apply filesystem: root mode: 0544 diff --git a/aws/container-linux/kubernetes/workers/cl/worker.yaml.tmpl b/aws/container-linux/kubernetes/workers/cl/worker.yaml.tmpl index 7d18b1631..c44d68ae9 100644 --- a/aws/container-linux/kubernetes/workers/cl/worker.yaml.tmpl +++ b/aws/container-linux/kubernetes/workers/cl/worker.yaml.tmpl @@ -98,7 +98,7 @@ storage: contents: inline: | KUBELET_IMAGE_URL=docker://k8s.gcr.io/hyperkube - KUBELET_IMAGE_TAG=v1.16.0 + KUBELET_IMAGE_TAG=v1.16.1 - path: /etc/sysctl.d/max-user-watches.conf filesystem: root contents: @@ -116,7 +116,7 @@ storage: --volume config,kind=host,source=/etc/kubernetes \ --mount volume=config,target=/etc/kubernetes \ --insecure-options=image \ - docker://k8s.gcr.io/hyperkube:v1.16.0 \ + docker://k8s.gcr.io/hyperkube:v1.16.1 \ --net=host \ --dns=host \ --exec=/kubectl -- --kubeconfig=/etc/kubernetes/kubeconfig delete node $(hostname) diff --git a/aws/fedora-coreos/kubernetes/README.md b/aws/fedora-coreos/kubernetes/README.md index 11361bfb2..e05ce48b2 100644 --- a/aws/fedora-coreos/kubernetes/README.md +++ b/aws/fedora-coreos/kubernetes/README.md @@ -11,7 +11,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features -* Kubernetes v1.16.0 (upstream) +* Kubernetes v1.16.1 (upstream) * Single or multi-master, [Calico](https://www.projectcalico.org/) or [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/) * Advanced features like [worker pools](https://typhoon.psdn.io/advanced/worker-pools/), [spot](https://typhoon.psdn.io/cl/aws/#spot) workers, and [snippets](https://typhoon.psdn.io/advanced/customization/#container-linux) customization diff --git a/aws/fedora-coreos/kubernetes/bootstrap.tf b/aws/fedora-coreos/kubernetes/bootstrap.tf index 54797f2f4..94bb63da3 100644 --- a/aws/fedora-coreos/kubernetes/bootstrap.tf +++ b/aws/fedora-coreos/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=18b7a74d309da7290474e35701bc6668f0dd035e" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=586d6e36f67c56fb2283f317a7552638368c5779" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/aws/fedora-coreos/kubernetes/fcc/controller.yaml b/aws/fedora-coreos/kubernetes/fcc/controller.yaml index 5d3887264..6b79cad9f 100644 --- a/aws/fedora-coreos/kubernetes/fcc/controller.yaml +++ b/aws/fedora-coreos/kubernetes/fcc/controller.yaml @@ -80,7 +80,7 @@ systemd: --volume /var/run:/var/run \ --volume /var/run/lock:/var/run/lock:z \ --volume /opt/cni/bin:/opt/cni/bin:z \ - k8s.gcr.io/hyperkube:v1.16.0 /hyperkube kubelet \ + k8s.gcr.io/hyperkube:v1.16.1 /hyperkube kubelet \ --anonymous-auth=false \ --authentication-token-webhook \ --authorization-mode=Webhook \ @@ -121,7 +121,7 @@ systemd: --network host \ --volume /opt/bootstrap/assets:/assets:ro,Z \ --volume /opt/bootstrap/apply:/apply:ro,Z \ - k8s.gcr.io/hyperkube:v1.16.0 \ + k8s.gcr.io/hyperkube:v1.16.1 \ /apply ExecStartPost=/bin/touch /opt/bootstrap/bootstrap.done ExecStartPost=-/usr/bin/podman stop bootstrap diff --git a/aws/fedora-coreos/kubernetes/workers/fcc/worker.yaml b/aws/fedora-coreos/kubernetes/workers/fcc/worker.yaml index 549749410..19a6c3cd6 100644 --- a/aws/fedora-coreos/kubernetes/workers/fcc/worker.yaml +++ b/aws/fedora-coreos/kubernetes/workers/fcc/worker.yaml @@ -50,7 +50,7 @@ systemd: --volume /var/run:/var/run \ --volume /var/run/lock:/var/run/lock:z \ --volume /opt/cni/bin:/opt/cni/bin:z \ - k8s.gcr.io/hyperkube:v1.16.0 /hyperkube kubelet \ + k8s.gcr.io/hyperkube:v1.16.1 /hyperkube kubelet \ --anonymous-auth=false \ --authentication-token-webhook \ --authorization-mode=Webhook \ diff --git a/azure/container-linux/kubernetes/README.md b/azure/container-linux/kubernetes/README.md index 873615197..1ff25137b 100644 --- a/azure/container-linux/kubernetes/README.md +++ b/azure/container-linux/kubernetes/README.md @@ -11,7 +11,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features -* Kubernetes v1.16.0 (upstream) +* Kubernetes v1.16.1 (upstream) * Single or multi-master, [Calico](https://www.projectcalico.org/) or [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/) * Advanced features like [worker pools](https://typhoon.psdn.io/advanced/worker-pools/), [low-priority](https://typhoon.psdn.io/cl/azure/#low-priority) workers, and [snippets](https://typhoon.psdn.io/advanced/customization/#container-linux) customization diff --git a/azure/container-linux/kubernetes/bootstrap.tf b/azure/container-linux/kubernetes/bootstrap.tf index 09e436ef7..11270a83b 100644 --- a/azure/container-linux/kubernetes/bootstrap.tf +++ b/azure/container-linux/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=18b7a74d309da7290474e35701bc6668f0dd035e" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=586d6e36f67c56fb2283f317a7552638368c5779" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/azure/container-linux/kubernetes/cl/controller.yaml.tmpl b/azure/container-linux/kubernetes/cl/controller.yaml.tmpl index 2007857ea..80c78652a 100644 --- a/azure/container-linux/kubernetes/cl/controller.yaml.tmpl +++ b/azure/container-linux/kubernetes/cl/controller.yaml.tmpl @@ -111,7 +111,7 @@ systemd: --volume script,kind=host,source=/opt/bootstrap/apply \ --mount volume=script,target=/apply \ --insecure-options=image \ - docker://k8s.gcr.io/hyperkube:v1.16.0 \ + docker://k8s.gcr.io/hyperkube:v1.16.1 \ --net=host \ --dns=host \ --exec=/apply @@ -132,7 +132,7 @@ storage: contents: inline: | KUBELET_IMAGE_URL=docker://k8s.gcr.io/hyperkube - KUBELET_IMAGE_TAG=v1.16.0 + KUBELET_IMAGE_TAG=v1.16.1 - path: /opt/bootstrap/apply filesystem: root mode: 0544 diff --git a/azure/container-linux/kubernetes/workers/cl/worker.yaml.tmpl b/azure/container-linux/kubernetes/workers/cl/worker.yaml.tmpl index f12be17ea..3736e8083 100644 --- a/azure/container-linux/kubernetes/workers/cl/worker.yaml.tmpl +++ b/azure/container-linux/kubernetes/workers/cl/worker.yaml.tmpl @@ -96,7 +96,7 @@ storage: contents: inline: | KUBELET_IMAGE_URL=docker://k8s.gcr.io/hyperkube - KUBELET_IMAGE_TAG=v1.16.0 + KUBELET_IMAGE_TAG=v1.16.1 - path: /etc/sysctl.d/max-user-watches.conf filesystem: root contents: @@ -114,7 +114,7 @@ storage: --volume config,kind=host,source=/etc/kubernetes \ --mount volume=config,target=/etc/kubernetes \ --insecure-options=image \ - docker://k8s.gcr.io/hyperkube:v1.16.0 \ + docker://k8s.gcr.io/hyperkube:v1.16.1 \ --net=host \ --dns=host \ --exec=/kubectl -- --kubeconfig=/etc/kubernetes/kubeconfig delete node $(hostname | tr '[:upper:]' '[:lower:]') diff --git a/bare-metal/container-linux/kubernetes/README.md b/bare-metal/container-linux/kubernetes/README.md index 571bd9af7..d87e02512 100644 --- a/bare-metal/container-linux/kubernetes/README.md +++ b/bare-metal/container-linux/kubernetes/README.md @@ -11,7 +11,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features -* Kubernetes v1.16.0 (upstream) +* Kubernetes v1.16.1 (upstream) * Single or multi-master, [Calico](https://www.projectcalico.org/) or [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/) * Advanced features like [snippets](https://typhoon.psdn.io/advanced/customization/#container-linux) customization diff --git a/bare-metal/container-linux/kubernetes/bootstrap.tf b/bare-metal/container-linux/kubernetes/bootstrap.tf index 7cd04c643..00b1b352c 100644 --- a/bare-metal/container-linux/kubernetes/bootstrap.tf +++ b/bare-metal/container-linux/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=18b7a74d309da7290474e35701bc6668f0dd035e" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=586d6e36f67c56fb2283f317a7552638368c5779" cluster_name = var.cluster_name api_servers = [var.k8s_domain_name] diff --git a/bare-metal/container-linux/kubernetes/cl/controller.yaml.tmpl b/bare-metal/container-linux/kubernetes/cl/controller.yaml.tmpl index e63ba3783..e3926a3b1 100644 --- a/bare-metal/container-linux/kubernetes/cl/controller.yaml.tmpl +++ b/bare-metal/container-linux/kubernetes/cl/controller.yaml.tmpl @@ -126,7 +126,7 @@ systemd: --volume script,kind=host,source=/opt/bootstrap/apply \ --mount volume=script,target=/apply \ --insecure-options=image \ - docker://k8s.gcr.io/hyperkube:v1.16.0 \ + docker://k8s.gcr.io/hyperkube:v1.16.1 \ --net=host \ --dns=host \ --exec=/apply @@ -141,7 +141,7 @@ storage: contents: inline: | KUBELET_IMAGE_URL=docker://k8s.gcr.io/hyperkube - KUBELET_IMAGE_TAG=v1.16.0 + KUBELET_IMAGE_TAG=v1.16.1 - path: /etc/hostname filesystem: root mode: 0644 diff --git a/bare-metal/container-linux/kubernetes/cl/worker.yaml.tmpl b/bare-metal/container-linux/kubernetes/cl/worker.yaml.tmpl index 4397dd2f0..48017d7e1 100644 --- a/bare-metal/container-linux/kubernetes/cl/worker.yaml.tmpl +++ b/bare-metal/container-linux/kubernetes/cl/worker.yaml.tmpl @@ -91,7 +91,7 @@ storage: contents: inline: | KUBELET_IMAGE_URL=docker://k8s.gcr.io/hyperkube - KUBELET_IMAGE_TAG=v1.16.0 + KUBELET_IMAGE_TAG=v1.16.1 - path: /etc/hostname filesystem: root mode: 0644 diff --git a/bare-metal/fedora-coreos/kubernetes/README.md b/bare-metal/fedora-coreos/kubernetes/README.md index f28a9898d..3efa25fd5 100644 --- a/bare-metal/fedora-coreos/kubernetes/README.md +++ b/bare-metal/fedora-coreos/kubernetes/README.md @@ -11,7 +11,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features -* Kubernetes v1.16.0 (upstream) +* Kubernetes v1.16.1 (upstream) * Single or multi-master, [Calico](https://www.projectcalico.org/) or [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/) * Advanced features like [snippets](https://typhoon.psdn.io/advanced/customization/#container-linux) customization diff --git a/bare-metal/fedora-coreos/kubernetes/bootstrap.tf b/bare-metal/fedora-coreos/kubernetes/bootstrap.tf index 8ee902b4d..317a48eba 100644 --- a/bare-metal/fedora-coreos/kubernetes/bootstrap.tf +++ b/bare-metal/fedora-coreos/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=18b7a74d309da7290474e35701bc6668f0dd035e" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=586d6e36f67c56fb2283f317a7552638368c5779" cluster_name = var.cluster_name api_servers = [var.k8s_domain_name] diff --git a/bare-metal/fedora-coreos/kubernetes/fcc/controller.yaml b/bare-metal/fedora-coreos/kubernetes/fcc/controller.yaml index f9f4fa546..4cc3fb96a 100644 --- a/bare-metal/fedora-coreos/kubernetes/fcc/controller.yaml +++ b/bare-metal/fedora-coreos/kubernetes/fcc/controller.yaml @@ -81,7 +81,7 @@ systemd: --volume /opt/cni/bin:/opt/cni/bin:z \ --volume /etc/iscsi:/etc/iscsi \ --volume /sbin/iscsiadm:/sbin/iscsiadm \ - k8s.gcr.io/hyperkube:v1.16.0 /hyperkube kubelet \ + k8s.gcr.io/hyperkube:v1.16.1 /hyperkube kubelet \ --anonymous-auth=false \ --authentication-token-webhook \ --authorization-mode=Webhook \ @@ -132,7 +132,7 @@ systemd: --network host \ --volume /opt/bootstrap/assets:/assets:ro,Z \ --volume /opt/bootstrap/apply:/apply:ro,Z \ - k8s.gcr.io/hyperkube:v1.16.0 \ + k8s.gcr.io/hyperkube:v1.16.1 \ /apply ExecStartPost=/bin/touch /opt/bootstrap/bootstrap.done ExecStartPost=-/usr/bin/podman stop bootstrap diff --git a/bare-metal/fedora-coreos/kubernetes/fcc/worker.yaml b/bare-metal/fedora-coreos/kubernetes/fcc/worker.yaml index 1e3736a2b..823b9cb19 100644 --- a/bare-metal/fedora-coreos/kubernetes/fcc/worker.yaml +++ b/bare-metal/fedora-coreos/kubernetes/fcc/worker.yaml @@ -51,7 +51,7 @@ systemd: --volume /opt/cni/bin:/opt/cni/bin:z \ --volume /etc/iscsi:/etc/iscsi \ --volume /sbin/iscsiadm:/sbin/iscsiadm \ - k8s.gcr.io/hyperkube:v1.16.0 /hyperkube kubelet \ + k8s.gcr.io/hyperkube:v1.16.1 /hyperkube kubelet \ --anonymous-auth=false \ --authentication-token-webhook \ --authorization-mode=Webhook \ diff --git a/digital-ocean/container-linux/kubernetes/README.md b/digital-ocean/container-linux/kubernetes/README.md index ad096cf95..1a8067fd6 100644 --- a/digital-ocean/container-linux/kubernetes/README.md +++ b/digital-ocean/container-linux/kubernetes/README.md @@ -11,7 +11,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features -* Kubernetes v1.16.0 (upstream) +* Kubernetes v1.16.1 (upstream) * Single or multi-master, [Calico](https://www.projectcalico.org/) or [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/) * Advanced features like [snippets](https://typhoon.psdn.io/advanced/customization/#container-linux) customization diff --git a/digital-ocean/container-linux/kubernetes/bootstrap.tf b/digital-ocean/container-linux/kubernetes/bootstrap.tf index ec1c8588d..a0418399d 100644 --- a/digital-ocean/container-linux/kubernetes/bootstrap.tf +++ b/digital-ocean/container-linux/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=18b7a74d309da7290474e35701bc6668f0dd035e" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=586d6e36f67c56fb2283f317a7552638368c5779" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/digital-ocean/container-linux/kubernetes/cl/controller.yaml.tmpl b/digital-ocean/container-linux/kubernetes/cl/controller.yaml.tmpl index 28e4c7a8c..284116176 100644 --- a/digital-ocean/container-linux/kubernetes/cl/controller.yaml.tmpl +++ b/digital-ocean/container-linux/kubernetes/cl/controller.yaml.tmpl @@ -123,7 +123,7 @@ systemd: --volume script,kind=host,source=/opt/bootstrap/apply \ --mount volume=script,target=/apply \ --insecure-options=image \ - docker://k8s.gcr.io/hyperkube:v1.16.0 \ + docker://k8s.gcr.io/hyperkube:v1.16.1 \ --net=host \ --dns=host \ --exec=/apply @@ -138,7 +138,7 @@ storage: contents: inline: | KUBELET_IMAGE_URL=docker://k8s.gcr.io/hyperkube - KUBELET_IMAGE_TAG=v1.16.0 + KUBELET_IMAGE_TAG=v1.16.1 - path: /opt/bootstrap/apply filesystem: root mode: 0544 diff --git a/digital-ocean/container-linux/kubernetes/cl/worker.yaml.tmpl b/digital-ocean/container-linux/kubernetes/cl/worker.yaml.tmpl index da19b6aa1..cd5286bf0 100644 --- a/digital-ocean/container-linux/kubernetes/cl/worker.yaml.tmpl +++ b/digital-ocean/container-linux/kubernetes/cl/worker.yaml.tmpl @@ -99,7 +99,7 @@ storage: contents: inline: | KUBELET_IMAGE_URL=docker://k8s.gcr.io/hyperkube - KUBELET_IMAGE_TAG=v1.16.0 + KUBELET_IMAGE_TAG=v1.16.1 - path: /etc/sysctl.d/max-user-watches.conf filesystem: root contents: @@ -117,7 +117,7 @@ storage: --volume config,kind=host,source=/etc/kubernetes \ --mount volume=config,target=/etc/kubernetes \ --insecure-options=image \ - docker://k8s.gcr.io/hyperkube:v1.16.0 \ + docker://k8s.gcr.io/hyperkube:v1.16.1 \ --net=host \ --dns=host \ --exec=/kubectl -- --kubeconfig=/etc/kubernetes/kubeconfig delete node $(hostname) diff --git a/docs/advanced/worker-pools.md b/docs/advanced/worker-pools.md index d0fffa8e2..a9dac3b74 100644 --- a/docs/advanced/worker-pools.md +++ b/docs/advanced/worker-pools.md @@ -79,7 +79,7 @@ Create a cluster following the Azure [tutorial](../cl/azure.md#cluster). Define ```tf module "ramius-worker-pool" { - source = "git::https://github.com/poseidon/typhoon//azure/container-linux/kubernetes/workers?ref=v1.16.0" + source = "git::https://github.com/poseidon/typhoon//azure/container-linux/kubernetes/workers?ref=v1.16.1" # Azure region = module.azure-ramius.region @@ -145,7 +145,7 @@ Create a cluster following the Google Cloud [tutorial](../cl/google-cloud.md#clu ```tf module "yavin-worker-pool" { - source = "git::https://github.com/poseidon/typhoon//google-cloud/container-linux/kubernetes/workers?ref=v1.16.0" + source = "git::https://github.com/poseidon/typhoon//google-cloud/container-linux/kubernetes/workers?ref=v1.16.1" # Google Cloud region = "europe-west2" @@ -176,11 +176,11 @@ Verify a managed instance group of workers joins the cluster within a few minute ``` $ kubectl get nodes NAME STATUS AGE VERSION -yavin-controller-0.c.example-com.internal Ready 6m v1.16.0 -yavin-worker-jrbf.c.example-com.internal Ready 5m v1.16.0 -yavin-worker-mzdm.c.example-com.internal Ready 5m v1.16.0 -yavin-16x-worker-jrbf.c.example-com.internal Ready 3m v1.16.0 -yavin-16x-worker-mzdm.c.example-com.internal Ready 3m v1.16.0 +yavin-controller-0.c.example-com.internal Ready 6m v1.16.1 +yavin-worker-jrbf.c.example-com.internal Ready 5m v1.16.1 +yavin-worker-mzdm.c.example-com.internal Ready 5m v1.16.1 +yavin-16x-worker-jrbf.c.example-com.internal Ready 3m v1.16.1 +yavin-16x-worker-mzdm.c.example-com.internal Ready 3m v1.16.1 ``` ### Variables diff --git a/docs/cl/aws.md b/docs/cl/aws.md index dc96fac7b..ec63098f5 100644 --- a/docs/cl/aws.md +++ b/docs/cl/aws.md @@ -1,6 +1,6 @@ # AWS -In this tutorial, we'll create a Kubernetes v1.16.0 cluster on AWS with Container Linux. +In this tutorial, we'll create a Kubernetes v1.16.1 cluster on AWS with Container Linux. We'll declare a Kubernetes cluster using the Typhoon Terraform module. Then apply the changes to create a VPC, gateway, subnets, security groups, controller instances, worker auto-scaling group, network load balancer, and TLS assets. @@ -70,7 +70,7 @@ Define a Kubernetes cluster using the module `aws/container-linux/kubernetes`. ```tf module "tempest" { - source = "git::https://github.com/poseidon/typhoon//aws/container-linux/kubernetes?ref=v1.16.0" + source = "git::https://github.com/poseidon/typhoon//aws/container-linux/kubernetes?ref=v1.16.1" # AWS cluster_name = "tempest" @@ -135,9 +135,9 @@ In 4-8 minutes, the Kubernetes cluster will be ready. $ export KUBECONFIG=/home/user/.secrets/clusters/tempest/auth/kubeconfig $ kubectl get nodes NAME STATUS ROLES AGE VERSION -ip-10-0-3-155 Ready 10m v1.16.0 -ip-10-0-26-65 Ready 10m v1.16.0 -ip-10-0-41-21 Ready 10m v1.16.0 +ip-10-0-3-155 Ready 10m v1.16.1 +ip-10-0-26-65 Ready 10m v1.16.1 +ip-10-0-41-21 Ready 10m v1.16.1 ``` List the pods. diff --git a/docs/cl/azure.md b/docs/cl/azure.md index c145e32a1..f4dc8cadb 100644 --- a/docs/cl/azure.md +++ b/docs/cl/azure.md @@ -3,7 +3,7 @@ !!! danger Typhoon for Azure is alpha. For production, use AWS, Google Cloud, or bare-metal. As Azure matures, check [errata](https://github.com/poseidon/typhoon/wiki/Errata) for known shortcomings. -In this tutorial, we'll create a Kubernetes v1.16.0 cluster on Azure with Container Linux. +In this tutorial, we'll create a Kubernetes v1.16.1 cluster on Azure with Container Linux. We'll declare a Kubernetes cluster using the Typhoon Terraform module. Then apply the changes to create a resource group, virtual network, subnets, security groups, controller availability set, worker scale set, load balancer, and TLS assets. @@ -66,7 +66,7 @@ Define a Kubernetes cluster using the module `azure/container-linux/kubernetes`. ```tf module "ramius" { - source = "git::https://github.com/poseidon/typhoon//azure/container-linux/kubernetes?ref=v1.16.0" + source = "git::https://github.com/poseidon/typhoon//azure/container-linux/kubernetes?ref=v1.16.1" # Azure cluster_name = "ramius" @@ -132,9 +132,9 @@ In 4-8 minutes, the Kubernetes cluster will be ready. $ export KUBECONFIG=/home/user/.secrets/clusters/ramius/auth/kubeconfig $ kubectl get nodes NAME STATUS ROLES AGE VERSION -ramius-controller-0 Ready 24m v1.16.0 -ramius-worker-000001 Ready 25m v1.16.0 -ramius-worker-000002 Ready 24m v1.16.0 +ramius-controller-0 Ready 24m v1.16.1 +ramius-worker-000001 Ready 25m v1.16.1 +ramius-worker-000002 Ready 24m v1.16.1 ``` List the pods. diff --git a/docs/cl/bare-metal.md b/docs/cl/bare-metal.md index ba8012140..2d5f0b0ac 100644 --- a/docs/cl/bare-metal.md +++ b/docs/cl/bare-metal.md @@ -1,6 +1,6 @@ # Bare-Metal -In this tutorial, we'll network boot and provision a Kubernetes v1.16.0 cluster on bare-metal with Container Linux. +In this tutorial, we'll network boot and provision a Kubernetes v1.16.1 cluster on bare-metal with Container Linux. First, we'll deploy a [Matchbox](https://github.com/poseidon/matchbox) service and setup a network boot environment. Then, we'll declare a Kubernetes cluster using the Typhoon Terraform module and power on machines. On PXE boot, machines will install Container Linux to disk, reboot into the disk install, and provision themselves as Kubernetes controllers or workers via Ignition. @@ -160,7 +160,7 @@ Define a Kubernetes cluster using the module `bare-metal/container-linux/kuberne ```tf module "bare-metal-mercury" { - source = "git::https://github.com/poseidon/typhoon//bare-metal/container-linux/kubernetes?ref=v1.16.0" + source = "git::https://github.com/poseidon/typhoon//bare-metal/container-linux/kubernetes?ref=v1.16.1" # bare-metal cluster_name = "mercury" @@ -263,9 +263,9 @@ Apply complete! Resources: 55 added, 0 changed, 0 destroyed. To watch the install to disk (until machines reboot from disk), SSH to port 2222. ``` -# before v1.16.0 +# before v1.16.1 $ ssh debug@node1.example.com -# after v1.16.0 +# after v1.16.1 $ ssh -p 2222 core@node1.example.com ``` @@ -289,9 +289,9 @@ systemd[1]: Started Kubernetes control plane. $ export KUBECONFIG=/home/user/.secrets/clusters/mercury/auth/kubeconfig $ kubectl get nodes NAME STATUS ROLES AGE VERSION -node1.example.com Ready 10m v1.16.0 -node2.example.com Ready 10m v1.16.0 -node3.example.com Ready 10m v1.16.0 +node1.example.com Ready 10m v1.16.1 +node2.example.com Ready 10m v1.16.1 +node3.example.com Ready 10m v1.16.1 ``` List the pods. diff --git a/docs/cl/digital-ocean.md b/docs/cl/digital-ocean.md index fbbe30cb2..7fa673236 100644 --- a/docs/cl/digital-ocean.md +++ b/docs/cl/digital-ocean.md @@ -1,6 +1,6 @@ # Digital Ocean -In this tutorial, we'll create a Kubernetes v1.16.0 cluster on DigitalOcean with Container Linux. +In this tutorial, we'll create a Kubernetes v1.16.1 cluster on DigitalOcean with Container Linux. We'll declare a Kubernetes cluster using the Typhoon Terraform module. Then apply the changes to create controller droplets, worker droplets, DNS records, tags, and TLS assets. @@ -65,7 +65,7 @@ Define a Kubernetes cluster using the module `digital-ocean/container-linux/kube ```tf module "digital-ocean-nemo" { - source = "git::https://github.com/poseidon/typhoon//digital-ocean/container-linux/kubernetes?ref=v1.16.0" + source = "git::https://github.com/poseidon/typhoon//digital-ocean/container-linux/kubernetes?ref=v1.16.1" # Digital Ocean cluster_name = "nemo" @@ -130,9 +130,9 @@ In 3-6 minutes, the Kubernetes cluster will be ready. $ export KUBECONFIG=/home/user/.secrets/clusters/nemo/auth/kubeconfig $ kubectl get nodes NAME STATUS ROLES AGE VERSION -10.132.110.130 Ready 10m v1.16.0 -10.132.115.81 Ready 10m v1.16.0 -10.132.124.107 Ready 10m v1.16.0 +10.132.110.130 Ready 10m v1.16.1 +10.132.115.81 Ready 10m v1.16.1 +10.132.124.107 Ready 10m v1.16.1 ``` List the pods. diff --git a/docs/cl/google-cloud.md b/docs/cl/google-cloud.md index 4c9e3e699..fa1e10012 100644 --- a/docs/cl/google-cloud.md +++ b/docs/cl/google-cloud.md @@ -1,6 +1,6 @@ # Google Cloud -In this tutorial, we'll create a Kubernetes v1.16.0 cluster on Google Compute Engine with Container Linux. +In this tutorial, we'll create a Kubernetes v1.16.1 cluster on Google Compute Engine with Container Linux. We'll declare a Kubernetes cluster using the Typhoon Terraform module. Then apply the changes to create a network, firewall rules, health checks, controller instances, worker managed instance group, load balancers, and TLS assets. @@ -71,7 +71,7 @@ Define a Kubernetes cluster using the module `google-cloud/container-linux/kuber ```tf module "google-cloud-yavin" { - source = "git::https://github.com/poseidon/typhoon//google-cloud/container-linux/kubernetes?ref=v1.16.0" + source = "git::https://github.com/poseidon/typhoon//google-cloud/container-linux/kubernetes?ref=v1.16.1" # Google Cloud cluster_name = "yavin" @@ -137,9 +137,9 @@ In 4-8 minutes, the Kubernetes cluster will be ready. $ export KUBECONFIG=/home/user/.secrets/clusters/yavin/auth/kubeconfig $ kubectl get nodes NAME ROLES STATUS AGE VERSION -yavin-controller-0.c.example-com.internal Ready 6m v1.16.0 -yavin-worker-jrbf.c.example-com.internal Ready 5m v1.16.0 -yavin-worker-mzdm.c.example-com.internal Ready 5m v1.16.0 +yavin-controller-0.c.example-com.internal Ready 6m v1.16.1 +yavin-worker-jrbf.c.example-com.internal Ready 5m v1.16.1 +yavin-worker-mzdm.c.example-com.internal Ready 5m v1.16.1 ``` List the pods. diff --git a/docs/fedora-coreos/aws.md b/docs/fedora-coreos/aws.md index bfb36c2f1..b3d1d8c6a 100644 --- a/docs/fedora-coreos/aws.md +++ b/docs/fedora-coreos/aws.md @@ -3,7 +3,7 @@ !!! danger Typhoon for Fedora CoreOS is an early preview! Fedora CoreOS itself is a preview! Expect bugs and design shifts. Please help both projects solve problems. Report Fedora CoreOS bugs to [Fedora](https://github.com/coreos/fedora-coreos-tracker/issues). Report Typhoon issues to Typhoon. -In this tutorial, we'll create a Kubernetes v1.16.0 cluster on AWS with Fedora CoreOS. +In this tutorial, we'll create a Kubernetes v1.16.1 cluster on AWS with Fedora CoreOS. We'll declare a Kubernetes cluster using the Typhoon Terraform module. Then apply the changes to create a VPC, gateway, subnets, security groups, controller instances, worker auto-scaling group, network load balancer, and TLS assets. @@ -138,9 +138,9 @@ In 4-8 minutes, the Kubernetes cluster will be ready. $ export KUBECONFIG=/home/user/.secrets/clusters/tempest/auth/kubeconfig $ kubectl get nodes NAME STATUS ROLES AGE VERSION -ip-10-0-3-155 Ready 10m v1.16.0 -ip-10-0-26-65 Ready 10m v1.16.0 -ip-10-0-41-21 Ready 10m v1.16.0 +ip-10-0-3-155 Ready 10m v1.16.1 +ip-10-0-26-65 Ready 10m v1.16.1 +ip-10-0-41-21 Ready 10m v1.16.1 ``` List the pods. diff --git a/docs/fedora-coreos/bare-metal.md b/docs/fedora-coreos/bare-metal.md index ef4238513..570d4135e 100644 --- a/docs/fedora-coreos/bare-metal.md +++ b/docs/fedora-coreos/bare-metal.md @@ -3,7 +3,7 @@ !!! danger Typhoon for Fedora CoreOS is an early preview! Fedora CoreOS itself is a preview! Expect bugs and design shifts. Please help both projects solve problems. Report Fedora CoreOS bugs to [Fedora](https://github.com/coreos/fedora-coreos-tracker/issues). Report Typhoon issues to Typhoon. -In this tutorial, we'll network boot and provision a Kubernetes v1.16.0 cluster on bare-metal with Fedora CoreOS. +In this tutorial, we'll network boot and provision a Kubernetes v1.16.1 cluster on bare-metal with Fedora CoreOS. First, we'll deploy a [Matchbox](https://github.com/poseidon/matchbox) service and setup a network boot environment. Then, we'll declare a Kubernetes cluster using the Typhoon Terraform module and power on machines. On PXE boot, machines will install Fedora CoreOS to disk, reboot into the disk install, and provision themselves as Kubernetes controllers or workers via Ignition. @@ -283,9 +283,9 @@ systemd[1]: Started Kubernetes control plane. $ export KUBECONFIG=/home/user/.secrets/clusters/mercury/auth/kubeconfig $ kubectl get nodes NAME STATUS ROLES AGE VERSION -node1.example.com Ready 10m v1.16.0 -node2.example.com Ready 10m v1.16.0 -node3.example.com Ready 10m v1.16.0 +node1.example.com Ready 10m v1.16.1 +node2.example.com Ready 10m v1.16.1 +node3.example.com Ready 10m v1.16.1 ``` List the pods. diff --git a/docs/index.md b/docs/index.md index beffd63cb..928d5383b 100644 --- a/docs/index.md +++ b/docs/index.md @@ -11,7 +11,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features -* Kubernetes v1.16.0 (upstream) +* Kubernetes v1.16.1 (upstream) * Single or multi-master, [Calico](https://www.projectcalico.org/) or [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/) * Advanced features like [worker pools](advanced/worker-pools/), [preemptible](cl/google-cloud/#preemption) workers, and [snippets](advanced/customization/#container-linux) customization @@ -47,7 +47,7 @@ Define a Kubernetes cluster by using the Terraform module for your chosen platfo ```tf module "google-cloud-yavin" { - source = "git::https://github.com/poseidon/typhoon//google-cloud/container-linux/kubernetes?ref=v1.16.0" + source = "git::https://github.com/poseidon/typhoon//google-cloud/container-linux/kubernetes?ref=v1.16.1" # Google Cloud cluster_name = "yavin" @@ -80,9 +80,9 @@ In 4-8 minutes (varies by platform), the cluster will be ready. This Google Clou $ export KUBECONFIG=/home/user/.secrets/clusters/yavin/auth/kubeconfig $ kubectl get nodes NAME ROLES STATUS AGE VERSION -yavin-controller-0.c.example-com.internal Ready 6m v1.16.0 -yavin-worker-jrbf.c.example-com.internal Ready 5m v1.16.0 -yavin-worker-mzdm.c.example-com.internal Ready 5m v1.16.0 +yavin-controller-0.c.example-com.internal Ready 6m v1.16.1 +yavin-worker-jrbf.c.example-com.internal Ready 5m v1.16.1 +yavin-worker-mzdm.c.example-com.internal Ready 5m v1.16.1 ``` List the pods. diff --git a/docs/topics/maintenance.md b/docs/topics/maintenance.md index a8d2a608c..6e3645d52 100644 --- a/docs/topics/maintenance.md +++ b/docs/topics/maintenance.md @@ -18,7 +18,7 @@ module "google-cloud-yavin" { } module "bare-metal-mercury" { - source = "git::https://github.com/poseidon/typhoon//bare-metal/container-linux/kubernetes?ref=v1.16.0" + source = "git::https://github.com/poseidon/typhoon//bare-metal/container-linux/kubernetes?ref=v1.16.1" ... } ``` @@ -279,15 +279,15 @@ Typhoon modules have been adapted for Terraform v0.12. Provider plugins requirem | Typhoon Release | Terraform version | |-------------------|---------------------| -| v1.16.0 - ? | v0.12.x | -| v1.10.3 - v1.16.0 | v0.11.x | +| v1.16.1 - ? | v0.12.x | +| v1.10.3 - v1.16.1 | v0.11.x | | v1.9.2 - v1.10.2 | v0.10.4+ or v0.11.x | | v1.7.3 - v1.9.1 | v0.10.x | | v1.6.4 - v1.7.2 | v0.9.x | ### New users -New users can start with Terraform v0.12.x and follow the docs for Typhoon v1.16.0+ without issue. +New users can start with Terraform v0.12.x and follow the docs for Typhoon v1.16.1+ without issue. ### Existing users @@ -404,7 +404,7 @@ tree . └── infraB <- new Terraform v0.12.x configs ``` -Define Typhoon clusters in the new config directory using Terraform v0.12 syntax. Follow the Typhoon v1.16.0+ docs (e.g. use `terraform12` in the `infraB` dir). See [AWS](/cl/aws), [Azure](/cl/azure), [Bare-Metal](/cl/bare-metal), [Digital Ocean](/cl/digital-ocean), or [Google-Cloud](/cl/google-cloud)) to create new clusters. Follow the usual [upgrade](/topics/maintenance/#upgrades) process to apply workloads and shift traffic. Later, switch back to the old config directory and deprovision clusters with Terraform v0.11. +Define Typhoon clusters in the new config directory using Terraform v0.12 syntax. Follow the Typhoon v1.16.1+ docs (e.g. use `terraform12` in the `infraB` dir). See [AWS](/cl/aws), [Azure](/cl/azure), [Bare-Metal](/cl/bare-metal), [Digital Ocean](/cl/digital-ocean), or [Google-Cloud](/cl/google-cloud)) to create new clusters. Follow the usual [upgrade](/topics/maintenance/#upgrades) process to apply workloads and shift traffic. Later, switch back to the old config directory and deprovision clusters with Terraform v0.11. ```shell terraform12 init diff --git a/google-cloud/container-linux/kubernetes/README.md b/google-cloud/container-linux/kubernetes/README.md index 6cbdfce1c..825b13677 100644 --- a/google-cloud/container-linux/kubernetes/README.md +++ b/google-cloud/container-linux/kubernetes/README.md @@ -11,7 +11,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features -* Kubernetes v1.16.0 (upstream) +* Kubernetes v1.16.1 (upstream) * Single or multi-master, [Calico](https://www.projectcalico.org/) or [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/) * Advanced features like [worker pools](https://typhoon.psdn.io/advanced/worker-pools/), [preemptible](https://typhoon.psdn.io/cl/google-cloud/#preemption) workers, and [snippets](https://typhoon.psdn.io/advanced/customization/#container-linux) customization diff --git a/google-cloud/container-linux/kubernetes/bootstrap.tf b/google-cloud/container-linux/kubernetes/bootstrap.tf index 94abfd0b7..5738776e7 100644 --- a/google-cloud/container-linux/kubernetes/bootstrap.tf +++ b/google-cloud/container-linux/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=18b7a74d309da7290474e35701bc6668f0dd035e" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=586d6e36f67c56fb2283f317a7552638368c5779" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/google-cloud/container-linux/kubernetes/cl/controller.yaml.tmpl b/google-cloud/container-linux/kubernetes/cl/controller.yaml.tmpl index f864c22ab..5da50b724 100644 --- a/google-cloud/container-linux/kubernetes/cl/controller.yaml.tmpl +++ b/google-cloud/container-linux/kubernetes/cl/controller.yaml.tmpl @@ -112,7 +112,7 @@ systemd: --volume script,kind=host,source=/opt/bootstrap/apply \ --mount volume=script,target=/apply \ --insecure-options=image \ - docker://k8s.gcr.io/hyperkube:v1.16.0 \ + docker://k8s.gcr.io/hyperkube:v1.16.1 \ --net=host \ --dns=host \ --exec=/apply @@ -133,7 +133,7 @@ storage: contents: inline: | KUBELET_IMAGE_URL=docker://k8s.gcr.io/hyperkube - KUBELET_IMAGE_TAG=v1.16.0 + KUBELET_IMAGE_TAG=v1.16.1 - path: /opt/bootstrap/apply filesystem: root mode: 0544 diff --git a/google-cloud/container-linux/kubernetes/workers/cl/worker.yaml.tmpl b/google-cloud/container-linux/kubernetes/workers/cl/worker.yaml.tmpl index 29e6cd9a2..e975fc68e 100644 --- a/google-cloud/container-linux/kubernetes/workers/cl/worker.yaml.tmpl +++ b/google-cloud/container-linux/kubernetes/workers/cl/worker.yaml.tmpl @@ -97,7 +97,7 @@ storage: contents: inline: | KUBELET_IMAGE_URL=docker://k8s.gcr.io/hyperkube - KUBELET_IMAGE_TAG=v1.16.0 + KUBELET_IMAGE_TAG=v1.16.1 - path: /etc/sysctl.d/max-user-watches.conf filesystem: root contents: @@ -115,7 +115,7 @@ storage: --volume config,kind=host,source=/etc/kubernetes \ --mount volume=config,target=/etc/kubernetes \ --insecure-options=image \ - docker://k8s.gcr.io/hyperkube:v1.16.0 \ + docker://k8s.gcr.io/hyperkube:v1.16.1 \ --net=host \ --dns=host \ --exec=/kubectl -- --kubeconfig=/etc/kubernetes/kubeconfig delete node $(hostname) From 995824fa6d5c63f62c1d43b64d3e329a7d0d8245 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Wed, 2 Oct 2019 21:48:24 -0700 Subject: [PATCH 235/523] Add stricter types for DigitalOcean module * Review variables available in DigitalOcean kubernetes module and sync with documentation * Promote Calico for DigitalOcean and Azure beyond experimental (its the primary mode I've used since it was introduced) --- CHANGES.md | 1 + .../container-linux/kubernetes/variables.tf | 44 +++++++++---------- docs/cl/azure.md | 2 +- docs/cl/digital-ocean.md | 15 +++---- 4 files changed, 31 insertions(+), 31 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 1e1974b7e..99eaebc63 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -25,6 +25,7 @@ Notable changes between versions. #### DigitalOcean +* Add Terraform v0.12 variables types ([#560](https://github.com/poseidon/typhoon/pull/560)) * Fix Terraform missing comma error ([#549](https://github.com/poseidon/typhoon/pull/549)) #### Google Cloud diff --git a/digital-ocean/container-linux/kubernetes/variables.tf b/digital-ocean/container-linux/kubernetes/variables.tf index fbd6df519..cf8a1ab18 100644 --- a/digital-ocean/container-linux/kubernetes/variables.tf +++ b/digital-ocean/container-linux/kubernetes/variables.tf @@ -18,33 +18,33 @@ variable "dns_zone" { # instances variable "controller_count" { - type = string - default = "1" + type = number description = "Number of controllers (i.e. masters)" + default = 1 } variable "worker_count" { - type = string - default = "1" + type = number description = "Number of workers" + default = 1 } variable "controller_type" { type = string - default = "s-2vcpu-2gb" description = "Droplet type for controllers (e.g. s-2vcpu-2gb, s-2vcpu-4gb, s-4vcpu-8gb)." + default = "s-2vcpu-2gb" } variable "worker_type" { type = string - default = "s-1vcpu-2gb" description = "Droplet type for workers (e.g. s-1vcpu-2gb, s-2vcpu-2gb)" + default = "s-1vcpu-2gb" } variable "image" { type = string - default = "coreos-stable" description = "Container Linux image for instances (e.g. coreos-stable)" + default = "coreos-stable" } variable "controller_clc_snippets" { @@ -67,48 +67,48 @@ variable "ssh_fingerprints" { } variable "asset_dir" { - description = "Path to a directory where generated assets should be placed (contains secrets)" type = string + description = "Absolute path to a directory where generated assets should be placed (contains secrets)" } variable "networking" { - description = "Choice of networking provider (flannel or calico)" type = string + description = "Choice of networking provider (flannel or calico)" default = "flannel" } variable "pod_cidr" { - description = "CIDR IPv4 range to assign Kubernetes pods" type = string + description = "CIDR IPv4 range to assign Kubernetes pods" default = "10.2.0.0/16" } variable "service_cidr" { + type = string description = < Date: Thu, 3 Oct 2019 18:56:51 -0700 Subject: [PATCH 236/523] Fix Prometheus etcd metrics scraping * Prometheus was configured to use kubernetes discovery of etcd targets based on nodes matching the node label node-role.kubernetes.io/controller=true * Kubernetes v1.16 stopped permitting node role labels node-role.kubernetes.io/* so Typhoon renamed these labels (no longer any association with roles) to node.kubermetes.io/controller=true * As a result, Prometheus didn't discover etcd targets, etcd metrics were missing, etcd alerts were ineffective, and the etcd Grafana dashboard was empty * Introduced: https://github.com/poseidon/typhoon/pull/543 --- CHANGES.md | 2 ++ addons/prometheus/config.yaml | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index 99eaebc63..de1bf06eb 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -35,6 +35,8 @@ Notable changes between versions. #### Addons +* Fix Prometheus etcd target discovery and scraping ([#561](https://github.com/poseidon/typhoon/pull/561)) + * Fix node label matcher for etcd target discovery (regressed in v1.16.0) * Update kube-state-metrics from v1.7.2 to v1.8.0 * Update nginx-ingress from v0.25.1 to [v0.26.1](https://github.com/kubernetes/ingress-nginx/releases/tag/nginx-0.26.1) ([#555](https://github.com/poseidon/typhoon/pull/555)) * Add lifecycle hook to allow draining for up to 5 minutes diff --git a/addons/prometheus/config.yaml b/addons/prometheus/config.yaml index 3ea5b65dd..298c4edc0 100644 --- a/addons/prometheus/config.yaml +++ b/addons/prometheus/config.yaml @@ -115,7 +115,7 @@ data: - role: node scheme: http relabel_configs: - - source_labels: [__meta_kubernetes_node_label_node_role_kubernetes_io_controller] + - source_labels: [__meta_kubernetes_node_label_node_kubernetes_io_controller] action: keep regex: 'true' - action: labelmap From 36ed53924fefd39f75421132a941c542dafd09a5 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Sun, 6 Oct 2019 11:53:49 -0700 Subject: [PATCH 237/523] Add stricter types for bare-metal modules * Review variables available in bare-metal kubernetes modules for Container Linux and Fedora CoreOS * Deprecate cluster_domain_suffix variable * Remove deprecated container_linux_oem variable --- CHANGES.md | 11 ++-- .../kubernetes/cl/install.yaml.tmpl | 1 - .../container-linux/kubernetes/groups.tf | 2 +- .../container-linux/kubernetes/profiles.tf | 2 - .../container-linux/kubernetes/variables.tf | 52 ++++++++----------- .../fedora-coreos/kubernetes/profiles.tf | 6 +-- .../fedora-coreos/kubernetes/variables.tf | 46 ++++++++-------- docs/cl/bare-metal.md | 19 ++++--- docs/fedora-coreos/bare-metal.md | 23 ++++---- 9 files changed, 76 insertions(+), 86 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index de1bf06eb..5ac814b63 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -7,30 +7,31 @@ Notable changes between versions. * Kubernetes [v1.16.1](https://github.com/kubernetes/kubernetes/blob/master/CHANGELOG-1.16.md#v1161) * Update etcd from v3.3.15 to [v3.4.1](https://github.com/etcd-io/etcd/releases/tag/v3.4.1) * Update Calico from v3.8.2 to [v3.9.1](https://docs.projectcalico.org/v3.9/release-notes/) +* Add Terraform v0.12 variables types ([#553](https://github.com/poseidon/typhoon/pull/553), [#557](https://github.com/poseidon/typhoon/pull/557), [#560](https://github.com/poseidon/typhoon/pull/560), [#556](https://github.com/poseidon/typhoon/pull/556), [#562](https://github.com/poseidon/typhoon/pull/562)) + * Deprecate `cluster_domain_suffix` variable #### AWS -* Add Terraform v0.12 variables types ([#553](https://github.com/poseidon/typhoon/pull/553)) * Add `node_labels` variable to internal `workers` module ([#550](https://github.com/poseidon/typhoon/pull/550)) #### Azure -* Add Terraform v0.12 variables types ([#557](https://github.com/poseidon/typhoon/pull/557)) -* Change `workers` module default `vm_type` to `Standard_DS1_v2` (followup to [#539](https://github.com/poseidon/typhoon/pull/539)) +* Promote `networking` provider Calico VXLAN out of experimental (set `networking = "calico"`) * Add `node_labels` variable to internal `workers` module ([#550](https://github.com/poseidon/typhoon/pull/550)) +* Change `workers` module default `vm_type` to `Standard_DS1_v2` (followup to [#539](https://github.com/poseidon/typhoon/pull/539)) #### Bare-Metal * Fix Terraform missing comma error ([#549](https://github.com/poseidon/typhoon/pull/549)) +* Remove deprecated `container_linux_oem` variable ([#562](https://github.com/poseidon/typhoon/pull/562)) #### DigitalOcean -* Add Terraform v0.12 variables types ([#560](https://github.com/poseidon/typhoon/pull/560)) +* Promote `networking` provider Calico VXLAN out of experimental (set `networking = "calico"`) * Fix Terraform missing comma error ([#549](https://github.com/poseidon/typhoon/pull/549)) #### Google Cloud -* Add Terraform v0.12 variables types ([#556](https://github.com/poseidon/typhoon/pull/556)) * Add `node_labels` variable to internal `workers` module ([#550](https://github.com/poseidon/typhoon/pull/550)) #### Addons diff --git a/bare-metal/container-linux/kubernetes/cl/install.yaml.tmpl b/bare-metal/container-linux/kubernetes/cl/install.yaml.tmpl index 0acca9abf..e8562c932 100644 --- a/bare-metal/container-linux/kubernetes/cl/install.yaml.tmpl +++ b/bare-metal/container-linux/kubernetes/cl/install.yaml.tmpl @@ -35,7 +35,6 @@ storage: -d ${install_disk} \ -C ${os_channel} \ -V ${os_version} \ - -o "${container_linux_oem}" \ ${baseurl_flag} \ -i ignition.json udevadm settle diff --git a/bare-metal/container-linux/kubernetes/groups.tf b/bare-metal/container-linux/kubernetes/groups.tf index 2cc410d90..d7ee3c4e4 100644 --- a/bare-metal/container-linux/kubernetes/groups.tf +++ b/bare-metal/container-linux/kubernetes/groups.tf @@ -4,7 +4,7 @@ resource "matchbox_group" "install" { name = format("install-%s", element(concat(var.controller_names, var.worker_names), count.index)) # pick one of 4 Matchbox profiles (Container Linux or Flatcar, cached or non-cached) - profile = local.flavor == "flatcar" ? var.cached_install == "true" ? element(matchbox_profile.cached-flatcar-linux-install.*.name, count.index) : element(matchbox_profile.flatcar-install.*.name, count.index) : var.cached_install == "true" ? element(matchbox_profile.cached-container-linux-install.*.name, count.index) : element(matchbox_profile.container-linux-install.*.name, count.index) + profile = local.flavor == "flatcar" ? var.cached_install ? element(matchbox_profile.cached-flatcar-linux-install.*.name, count.index) : element(matchbox_profile.flatcar-install.*.name, count.index) : var.cached_install ? element(matchbox_profile.cached-container-linux-install.*.name, count.index) : element(matchbox_profile.container-linux-install.*.name, count.index) selector = { mac = element(concat(var.controller_macs, var.worker_macs), count.index) diff --git a/bare-metal/container-linux/kubernetes/profiles.tf b/bare-metal/container-linux/kubernetes/profiles.tf index 0a6ba49e3..78f958f59 100644 --- a/bare-metal/container-linux/kubernetes/profiles.tf +++ b/bare-metal/container-linux/kubernetes/profiles.tf @@ -40,7 +40,6 @@ data "template_file" "container-linux-install-configs" { os_version = var.os_version ignition_endpoint = format("%s/ignition", var.matchbox_http_endpoint) install_disk = var.install_disk - container_linux_oem = var.container_linux_oem ssh_authorized_key = var.ssh_authorized_key # only cached-container-linux profile adds -b baseurl baseurl_flag = "" @@ -82,7 +81,6 @@ data "template_file" "cached-container-linux-install-configs" { os_version = var.os_version ignition_endpoint = format("%s/ignition", var.matchbox_http_endpoint) install_disk = var.install_disk - container_linux_oem = var.container_linux_oem ssh_authorized_key = var.ssh_authorized_key # profile uses -b baseurl to install from matchbox cache baseurl_flag = "-b ${var.matchbox_http_endpoint}/assets/${local.flavor}" diff --git a/bare-metal/container-linux/kubernetes/variables.tf b/bare-metal/container-linux/kubernetes/variables.tf index a95a808b3..5f6d5809f 100644 --- a/bare-metal/container-linux/kubernetes/variables.tf +++ b/bare-metal/container-linux/kubernetes/variables.tf @@ -62,8 +62,8 @@ variable "clc_snippets" { # configuration variable "k8s_domain_name" { - description = "Controller DNS name which resolves to a controller instance. Workers and kubeconfig's will communicate with this endpoint (e.g. cluster.example.com)" type = string + description = "Controller DNS name which resolves to a controller instance. Workers and kubeconfig's will communicate with this endpoint (e.g. cluster.example.com)" } variable "ssh_authorized_key" { @@ -72,63 +72,55 @@ variable "ssh_authorized_key" { } variable "asset_dir" { - description = "Path to a directory where generated assets should be placed (contains secrets)" type = string + description = "Absolute path to a directory where generated assets should be placed (contains secrets)" } variable "networking" { - description = "Choice of networking provider (flannel or calico)" type = string + description = "Choice of networking provider (flannel or calico)" default = "calico" } variable "network_mtu" { + type = number description = "CNI interface MTU (applies to calico only)" - type = string - default = "1480" + default = 1480 } variable "network_ip_autodetection_method" { - description = "Method to autodetect the host IPv4 address (applies to calico only)" type = string + description = "Method to autodetect the host IPv4 address (applies to calico only)" default = "first-found" } variable "pod_cidr" { - description = "CIDR IPv4 range to assign Kubernetes pods" type = string + description = "CIDR IPv4 range to assign Kubernetes pods" default = "10.2.0.0/16" } variable "service_cidr" { + type = string description = < Date: Sun, 6 Oct 2019 17:23:41 -0700 Subject: [PATCH 238/523] Use new Fedora CoreOS kernel/initrd/raw asset names * Fedora CoreOS changed the kernel, initramfs, and raw image asset download paths and names in 30.20191002.0 --- CHANGES.md | 1 + bare-metal/fedora-coreos/kubernetes/profiles.tf | 12 ++++++------ docs/fedora-coreos/bare-metal.md | 2 +- 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 5ac814b63..30badf63d 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -24,6 +24,7 @@ Notable changes between versions. * Fix Terraform missing comma error ([#549](https://github.com/poseidon/typhoon/pull/549)) * Remove deprecated `container_linux_oem` variable ([#562](https://github.com/poseidon/typhoon/pull/562)) +* Use new kernel, initrd, and raw paths for Fedora CoreOS ([#563](https://github.com/poseidon/typhoon/pull/563)) #### DigitalOcean diff --git a/bare-metal/fedora-coreos/kubernetes/profiles.tf b/bare-metal/fedora-coreos/kubernetes/profiles.tf index 018d69b2b..d86a4f91c 100644 --- a/bare-metal/fedora-coreos/kubernetes/profiles.tf +++ b/bare-metal/fedora-coreos/kubernetes/profiles.tf @@ -1,22 +1,22 @@ locals { - remote_kernel = "https://builds.coreos.fedoraproject.org/prod/streams/${var.os_stream}/builds/${var.os_version}/x86_64/fedora-coreos-${var.os_version}-installer-kernel" - remote_initrd = "https://builds.coreos.fedoraproject.org/prod/streams/${var.os_stream}/builds/${var.os_version}/x86_64/fedora-coreos-${var.os_version}-installer-initramfs.img" + remote_kernel = "https://builds.coreos.fedoraproject.org/prod/streams/${var.os_stream}/builds/${var.os_version}/x86_64/fedora-coreos-${var.os_version}-installer-kernel-x86_64" + remote_initrd = "https://builds.coreos.fedoraproject.org/prod/streams/${var.os_stream}/builds/${var.os_version}/x86_64/fedora-coreos-${var.os_version}-installer-initramfs.x86_64.img" remote_args = [ "ip=dhcp", "rd.neednet=1", "coreos.inst=yes", - "coreos.inst.image_url=https://builds.coreos.fedoraproject.org/prod/streams/${var.os_stream}/builds/${var.os_version}/x86_64/fedora-coreos-${var.os_version}-metal.raw.xz", + "coreos.inst.image_url=https://builds.coreos.fedoraproject.org/prod/streams/${var.os_stream}/builds/${var.os_version}/x86_64/fedora-coreos-${var.os_version}-metal.x86_64.raw.xz", "coreos.inst.ignition_url=${var.matchbox_http_endpoint}/ignition?uuid=$${uuid}&mac=$${mac:hexhyp}", "coreos.inst.install_dev=${var.install_disk}" ] - cached_kernel = "/assets/fedora-coreos/fedora-coreos-${var.os_version}-installer-kernel" - cached_initrd = "/assets/fedora-coreos/fedora-coreos-${var.os_version}-installer-initramfs.img" + cached_kernel = "/assets/fedora-coreos/fedora-coreos-${var.os_version}-installer-kernel-x86_64" + cached_initrd = "/assets/fedora-coreos/fedora-coreos-${var.os_version}-installer-initramfs.x86_64.img" cached_args = [ "ip=dhcp", "rd.neednet=1", "coreos.inst=yes", - "coreos.inst.image_url=${var.matchbox_http_endpoint}/assets/fedora-coreos/fedora-coreos-${var.os_version}-metal.raw.xz", + "coreos.inst.image_url=${var.matchbox_http_endpoint}/assets/fedora-coreos/fedora-coreos-${var.os_version}-metal.x86_64.raw.xz", "coreos.inst.ignition_url=${var.matchbox_http_endpoint}/ignition?uuid=$${uuid}&mac=$${mac:hexhyp}", "coreos.inst.install_dev=${var.install_disk}" ] diff --git a/docs/fedora-coreos/bare-metal.md b/docs/fedora-coreos/bare-metal.md index 49be5032f..8ec664104 100644 --- a/docs/fedora-coreos/bare-metal.md +++ b/docs/fedora-coreos/bare-metal.md @@ -169,7 +169,7 @@ module "bare-metal-mercury" { cluster_name = "mercury" matchbox_http_endpoint = "http://matchbox.example.com" os_stream = "testing" - os_version = "30.20190801.0" + os_version = "30.20191002.0" cached_install = true # configuration From 5ef4155e08c45c6377254f74abbb9b1fcf5d7a74 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Sun, 6 Oct 2019 18:05:47 -0700 Subject: [PATCH 239/523] Detect most recent Fedora CoreOS AMI in region * Detect the most recent Fedora CoreOS AMI to allow usage of Fedora CoreOS in supported regions (previously just us-east-1) * Unpin the Fedora CoreOS AMI image which was pinned to images that had been checked. This does mean if Fedora publishes a broken image, it will be selected * Filter out "dev" images which have similar naming --- CHANGES.md | 3 ++- aws/fedora-coreos/kubernetes/ami.tf | 6 ++++-- aws/fedora-coreos/kubernetes/workers/ami.tf | 6 ++++-- docs/fedora-coreos/aws.md | 2 +- 4 files changed, 11 insertions(+), 6 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 30badf63d..af2bb8f1f 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -13,6 +13,7 @@ Notable changes between versions. #### AWS * Add `node_labels` variable to internal `workers` module ([#550](https://github.com/poseidon/typhoon/pull/550)) +* For Fedora CoreOS, detect most recent AMI in the region #### Azure @@ -24,7 +25,7 @@ Notable changes between versions. * Fix Terraform missing comma error ([#549](https://github.com/poseidon/typhoon/pull/549)) * Remove deprecated `container_linux_oem` variable ([#562](https://github.com/poseidon/typhoon/pull/562)) -* Use new kernel, initrd, and raw paths for Fedora CoreOS ([#563](https://github.com/poseidon/typhoon/pull/563)) +* For Fedora CoreOS, use new kernel, initrd, and raw paths ([#563](https://github.com/poseidon/typhoon/pull/563)) #### DigitalOcean diff --git a/aws/fedora-coreos/kubernetes/ami.tf b/aws/fedora-coreos/kubernetes/ami.tf index 626b1f834..7e0990509 100644 --- a/aws/fedora-coreos/kubernetes/ami.tf +++ b/aws/fedora-coreos/kubernetes/ami.tf @@ -13,9 +13,11 @@ data "aws_ami" "fedora-coreos" { values = ["hvm"] } - // pin on known ok versions as preview matures filter { name = "name" - values = ["fedora-coreos-30.20190905.0-hvm"] + values = ["fedora-coreos-30.*.*-hvm"] } + + # try to filter out dev images (AWS filters can't) + name_regex = "^fedora-coreos-30.[0-9]*.[0-9]*-hvm*" } diff --git a/aws/fedora-coreos/kubernetes/workers/ami.tf b/aws/fedora-coreos/kubernetes/workers/ami.tf index 626b1f834..2924c2b69 100644 --- a/aws/fedora-coreos/kubernetes/workers/ami.tf +++ b/aws/fedora-coreos/kubernetes/workers/ami.tf @@ -13,9 +13,11 @@ data "aws_ami" "fedora-coreos" { values = ["hvm"] } - // pin on known ok versions as preview matures filter { name = "name" - values = ["fedora-coreos-30.20190905.0-hvm"] + values = ["fedora-coreos-30.*.*-hvm"] } + + # try to filter out dev images (AWS filters can't) + name_regex = "^fedora-coreos-30.[0-9]*.[0-9]*-hvm*" } diff --git a/docs/fedora-coreos/aws.md b/docs/fedora-coreos/aws.md index b3d1d8c6a..b516862bc 100644 --- a/docs/fedora-coreos/aws.md +++ b/docs/fedora-coreos/aws.md @@ -53,7 +53,7 @@ Configure the AWS provider to use your access key credentials in a `providers.tf ```tf provider "aws" { version = "2.29.0" - region = "us-east-1" # MUST be us-east-1 right now! + region = "eu-central-1" shared_credentials_file = "/home/user/.config/aws/credentials" } From ab72f1ab2df17cafc8ab4b3503301cbcfb81ca02 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Sun, 6 Oct 2019 18:22:20 -0700 Subject: [PATCH 240/523] Update Prometheus from v2.12.0 to v2.13.0 * https://github.com/prometheus/prometheus/releases/tag/v2.13.0 --- CHANGES.md | 4 ++-- addons/prometheus/deployment.yaml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index af2bb8f1f..9ac4e2c1f 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -38,8 +38,8 @@ Notable changes between versions. #### Addons -* Fix Prometheus etcd target discovery and scraping ([#561](https://github.com/poseidon/typhoon/pull/561)) - * Fix node label matcher for etcd target discovery (regressed in v1.16.0) +* Update Prometheus from v2.12.0 to [v2.13.0](https://github.com/prometheus/prometheus/releases/tag/v2.13.0) + * Fix Prometheus etcd target discovery and scraping ([#561](https://github.com/poseidon/typhoon/pull/561), regressed with Kubernetes v1.16.0) * Update kube-state-metrics from v1.7.2 to v1.8.0 * Update nginx-ingress from v0.25.1 to [v0.26.1](https://github.com/kubernetes/ingress-nginx/releases/tag/nginx-0.26.1) ([#555](https://github.com/poseidon/typhoon/pull/555)) * Add lifecycle hook to allow draining for up to 5 minutes diff --git a/addons/prometheus/deployment.yaml b/addons/prometheus/deployment.yaml index 0b21c6114..9c36e126b 100644 --- a/addons/prometheus/deployment.yaml +++ b/addons/prometheus/deployment.yaml @@ -20,7 +20,7 @@ spec: serviceAccountName: prometheus containers: - name: prometheus - image: quay.io/prometheus/prometheus:v2.12.0 + image: quay.io/prometheus/prometheus:v2.13.0 args: - --web.listen-address=0.0.0.0:9090 - --config.file=/etc/prometheus/prometheus.yaml From 5196709fe0c88d4ef64ba24b255be296e4138229 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Sun, 6 Oct 2019 18:30:07 -0700 Subject: [PATCH 241/523] Update docs, CHANGES, and mkdocs-material * Update mkdocs-material from v4.4.2 to v4.4.3 * Update recommended Terraform provider versions * Cleanup the changelog before release --- CHANGES.md | 11 +++++++---- docs/cl/aws.md | 4 ++-- docs/cl/azure.md | 9 ++++----- docs/cl/bare-metal.md | 4 ++-- docs/cl/digital-ocean.md | 4 ++-- docs/cl/google-cloud.md | 4 ++-- docs/fedora-coreos/aws.md | 6 +++--- docs/fedora-coreos/bare-metal.md | 2 +- requirements.txt | 2 +- 9 files changed, 24 insertions(+), 22 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 9ac4e2c1f..1758958cc 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -2,7 +2,7 @@ Notable changes between versions. -## Latest +## v1.16.1 * Kubernetes [v1.16.1](https://github.com/kubernetes/kubernetes/blob/master/CHANGELOG-1.16.md#v1161) * Update etcd from v3.3.15 to [v3.4.1](https://github.com/etcd-io/etcd/releases/tag/v3.4.1) @@ -12,20 +12,22 @@ Notable changes between versions. #### AWS -* Add `node_labels` variable to internal `workers` module ([#550](https://github.com/poseidon/typhoon/pull/550)) +* Add `worker_node_labels` variable to set initial worker node labels ([#550](https://github.com/poseidon/typhoon/pull/550)) +* Add `node_labels` variable to internal `workers` pool module ([#550](https://github.com/poseidon/typhoon/pull/550)) * For Fedora CoreOS, detect most recent AMI in the region #### Azure * Promote `networking` provider Calico VXLAN out of experimental (set `networking = "calico"`) -* Add `node_labels` variable to internal `workers` module ([#550](https://github.com/poseidon/typhoon/pull/550)) +* Add `worker_node_labels` variable to set initial worker node labels ([#550](https://github.com/poseidon/typhoon/pull/550)) +* Add `node_labels` variable to internal `workers` pool module ([#550](https://github.com/poseidon/typhoon/pull/550)) * Change `workers` module default `vm_type` to `Standard_DS1_v2` (followup to [#539](https://github.com/poseidon/typhoon/pull/539)) #### Bare-Metal +* For Fedora CoreOS, use new kernel, initrd, and raw paths ([#563](https://github.com/poseidon/typhoon/pull/563)) * Fix Terraform missing comma error ([#549](https://github.com/poseidon/typhoon/pull/549)) * Remove deprecated `container_linux_oem` variable ([#562](https://github.com/poseidon/typhoon/pull/562)) -* For Fedora CoreOS, use new kernel, initrd, and raw paths ([#563](https://github.com/poseidon/typhoon/pull/563)) #### DigitalOcean @@ -34,6 +36,7 @@ Notable changes between versions. #### Google Cloud +* Add `worker_node_labels` variable to set initial worker node labels ([#550](https://github.com/poseidon/typhoon/pull/550)) * Add `node_labels` variable to internal `workers` module ([#550](https://github.com/poseidon/typhoon/pull/550)) #### Addons diff --git a/docs/cl/aws.md b/docs/cl/aws.md index ec63098f5..b2a96c1bc 100644 --- a/docs/cl/aws.md +++ b/docs/cl/aws.md @@ -18,7 +18,7 @@ Install [Terraform](https://www.terraform.io/downloads.html) v0.12.x on your sys ```sh $ terraform version -Terraform v0.12.7 +Terraform v0.12.9 ``` Add the [terraform-provider-ct](https://github.com/poseidon/terraform-provider-ct) plugin binary for your system to `~/.terraform.d/plugins/`, noting the final name. @@ -49,7 +49,7 @@ Configure the AWS provider to use your access key credentials in a `providers.tf ```tf provider "aws" { - version = "2.29.0" + version = "2.31.0" region = "eu-central-1" shared_credentials_file = "/home/user/.config/aws/credentials" } diff --git a/docs/cl/azure.md b/docs/cl/azure.md index f3c654604..2ac012b74 100644 --- a/docs/cl/azure.md +++ b/docs/cl/azure.md @@ -21,7 +21,7 @@ Install [Terraform](https://www.terraform.io/downloads.html) v0.12.x on your sys ```sh $ terraform version -Terraform v0.12.7 +Terraform v0.12.9 ``` Add the [terraform-provider-ct](https://github.com/poseidon/terraform-provider-ct) plugin binary for your system to `~/.terraform.d/plugins/`, noting the final name. @@ -50,7 +50,7 @@ Configure the Azure provider in a `providers.tf` file. ```tf provider "azurerm" { - version = "1.34.0" + version = "1.35.0" } provider "ct" { @@ -217,7 +217,6 @@ Reference the DNS zone with `azurerm_dns_zone.clusters.name` and its resource gr | worker_type | Machine type for workers | "Standard_DS1_v2" | See below | | os_image | Channel for a Container Linux derivative | "coreos-stable" | coreos-stable, coreos-beta, coreos-alpha | | disk_size | Size of the disk in GB | 40 | 100 | -| worker_node_labels | List of initial worker node labels | [] | ["worker-pool=default"] | | worker_priority | Set priority to Low to use reduced cost surplus capacity, with the tradeoff that instances can be deallocated at any time | Regular | Low | | controller_clc_snippets | Controller Container Linux Config snippets | [] | [example](/advanced/customization/#usage) | | worker_clc_snippets | Worker Container Linux Config snippets | [] | [example](/advanced/customization/#usage) | @@ -225,7 +224,7 @@ Reference the DNS zone with `azurerm_dns_zone.clusters.name` and its resource gr | host_cidr | CIDR IPv4 range to assign to instances | "10.0.0.0/16" | "10.0.0.0/20" | | pod_cidr | CIDR IPv4 range to assign to Kubernetes pods | "10.2.0.0/16" | "10.22.0.0/16" | | service_cidr | CIDR IPv4 range to assign to Kubernetes services | "10.3.0.0/16" | "10.3.0.0/24" | -| cluster_domain_suffix | FQDN suffix for Kubernetes services answered by coredns. | "cluster.local" | "k8s.example.com" | +| worker_node_labels | List of initial worker node labels | [] | ["worker-pool=default"] | Check the list of valid [machine types](https://azure.microsoft.com/en-us/pricing/details/virtual-machines/linux/) and their [specs](https://docs.microsoft.com/en-us/azure/virtual-machines/linux/sizes-general). Use `az vm list-skus` to get the identifier. @@ -233,7 +232,7 @@ Check the list of valid [machine types](https://azure.microsoft.com/en-us/pricin Unlike AWS and GCP, Azure requires its *virtual* networks to have non-overlapping IPv4 CIDRs (yeah, go figure). Instead of each cluster just using `10.0.0.0/16` for instances, each Azure cluster's `host_cidr` must be non-overlapping (e.g. 10.0.0.0/20 for the 1st cluster, 10.0.16.0/20 for the 2nd cluster, etc). !!! warning - Do not choose a `controller_type` smaller than `Standard_DS1_v2`. Smaller instances are not sufficient for running a controller. + Do not choose a `controller_type` smaller than `Standard_B2s`. Smaller instances are not sufficient for running a controller. #### Low Priority diff --git a/docs/cl/bare-metal.md b/docs/cl/bare-metal.md index fb739f1e4..a62e215c1 100644 --- a/docs/cl/bare-metal.md +++ b/docs/cl/bare-metal.md @@ -111,7 +111,7 @@ Install [Terraform](https://www.terraform.io/downloads.html) v0.12.x on your sys ```sh $ terraform version -Terraform v0.12.7 +Terraform v0.12.9 ``` Add the [terraform-provider-matchbox](https://github.com/poseidon/terraform-provider-matchbox) plugin binary for your system to `~/.terraform.d/plugins/`, noting the final name. @@ -166,7 +166,7 @@ module "bare-metal-mercury" { cluster_name = "mercury" matchbox_http_endpoint = "http://matchbox.example.com" os_channel = "coreos-stable" - os_version = "1632.3.0" + os_version = "2191.5.0" # configuration k8s_domain_name = "node1.example.com" diff --git a/docs/cl/digital-ocean.md b/docs/cl/digital-ocean.md index 21b6bc836..20bad53ed 100644 --- a/docs/cl/digital-ocean.md +++ b/docs/cl/digital-ocean.md @@ -18,7 +18,7 @@ Install [Terraform](https://www.terraform.io/downloads.html) v0.12.x on your sys ```sh $ terraform version -Terraform v0.12.7 +Terraform v0.12.9 ``` Add the [terraform-provider-ct](https://github.com/poseidon/terraform-provider-ct) plugin binary for your system to `~/.terraform.d/plugins/`, noting the final name. @@ -50,7 +50,7 @@ Configure the DigitalOcean provider to use your token in a `providers.tf` file. ```tf provider "digitalocean" { - version = "1.7.0" + version = "1.8.0" token = "${chomp(file("~/.config/digital-ocean/token"))}" } diff --git a/docs/cl/google-cloud.md b/docs/cl/google-cloud.md index fa1e10012..1600cb681 100644 --- a/docs/cl/google-cloud.md +++ b/docs/cl/google-cloud.md @@ -18,7 +18,7 @@ Install [Terraform](https://www.terraform.io/downloads.html) v0.12.x on your sys ```sh $ terraform version -Terraform v0.12.7 +Terraform v0.12.9 ``` Add the [terraform-provider-ct](https://github.com/poseidon/terraform-provider-ct) plugin binary for your system to `~/.terraform.d/plugins/`, noting the final name. @@ -49,7 +49,7 @@ Configure the Google Cloud provider to use your service account key, project-id, ```tf provider "google" { - version = "2.15.0" + version = "2.16.0" project = "project-id" region = "us-central1" credentials = file("~/.config/google-cloud/terraform.json") diff --git a/docs/fedora-coreos/aws.md b/docs/fedora-coreos/aws.md index b516862bc..def06dadf 100644 --- a/docs/fedora-coreos/aws.md +++ b/docs/fedora-coreos/aws.md @@ -21,7 +21,7 @@ Install [Terraform](https://www.terraform.io/downloads.html) v0.12.x on your sys ```sh $ terraform version -Terraform v0.12.7 +Terraform v0.12.9 ``` Add the [terraform-provider-ct](https://github.com/poseidon/terraform-provider-ct) plugin binary for your system to `~/.terraform.d/plugins/`, noting the final name. @@ -52,7 +52,7 @@ Configure the AWS provider to use your access key credentials in a `providers.tf ```tf provider "aws" { - version = "2.29.0" + version = "2.31.0" region = "eu-central-1" shared_credentials_file = "/home/user/.config/aws/credentials" } @@ -73,7 +73,7 @@ Define a Kubernetes cluster using the module `aws/fedora-coreos/kubernetes`. ```tf module "aws-tempest" { - source = "git::https://github.com/poseidon/typhoon//aws/fedora-coreos/kubernetes?ref=DEVELOPMENT_SHA" + source = "git::https://github.com/poseidon/typhoon//aws/fedora-coreos/kubernetes?ref=v1.16.1" # AWS cluster_name = "tempest" diff --git a/docs/fedora-coreos/bare-metal.md b/docs/fedora-coreos/bare-metal.md index 8ec664104..cdba05e53 100644 --- a/docs/fedora-coreos/bare-metal.md +++ b/docs/fedora-coreos/bare-metal.md @@ -114,7 +114,7 @@ Install [Terraform](https://www.terraform.io/downloads.html) v0.12.x on your sys ```sh $ terraform version -Terraform v0.12.7 +Terraform v0.12.9 ``` Add the [terraform-provider-matchbox](https://github.com/poseidon/terraform-provider-matchbox) plugin binary for your system to `~/.terraform.d/plugins/`, noting the final name. diff --git a/requirements.txt b/requirements.txt index 0fe02e3b2..892e7256f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ mkdocs==1.0.4 -mkdocs-material==4.4.2 +mkdocs-material==4.4.3 pygments==2.2.0 pymdown-extensions==5.0.0 From 5b9dab66592597ee6f2b7c51062806c50bb85412 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Sun, 6 Oct 2019 12:57:15 -0700 Subject: [PATCH 242/523] Introduce list of detail objects for bare-metal machines * Define bare-metal `controllers` and `workers` as a complex type list(object{name=string, mac=string, domain=string}) to allow clusters with many machines to be defined more cleanly * Remove `controller_names` list variable * Remove `controller_macs` list variable * Remove `controller_domains` list variable * Remove `worker_names` list variable * Remove `worker_macs` list variable * Remove `worker_domains` list variable --- CHANGES.md | 13 ++++ .../container-linux/kubernetes/bootstrap.tf | 2 +- .../container-linux/kubernetes/groups.tf | 24 +++--- .../container-linux/kubernetes/profiles.tf | 73 +++++++++---------- bare-metal/container-linux/kubernetes/ssh.tf | 10 +-- .../container-linux/kubernetes/variables.tf | 50 ++++++------- .../fedora-coreos/kubernetes/bootstrap.tf | 2 +- bare-metal/fedora-coreos/kubernetes/groups.tf | 12 +-- .../fedora-coreos/kubernetes/profiles.tf | 24 +++--- bare-metal/fedora-coreos/kubernetes/ssh.tf | 10 +-- .../fedora-coreos/kubernetes/variables.tf | 50 ++++++------- docs/cl/bare-metal.md | 38 +++++----- docs/fedora-coreos/bare-metal.md | 38 +++++----- 13 files changed, 173 insertions(+), 173 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 1758958cc..684d8d75d 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -2,6 +2,19 @@ Notable changes between versions. +## Latest + +#### Bare-Metal + +* Add `controllers` and `workers` as typed lists of machine detail objects ([#566](https://github.com/poseidon/typhoon/pull/566)) + * Define clusters' machines cleanly and with Terraform v0.12 type constraints (**action required**, see PR example) + * Remove `controller_names` list variable + * Remove `controller_macs` list variable + * Remove `controller_domains` list variable + * Remove `worker_names` list variable + * Remove `worker_macs` list variable + * Remove `worker_domains` list variable + ## v1.16.1 * Kubernetes [v1.16.1](https://github.com/kubernetes/kubernetes/blob/master/CHANGELOG-1.16.md#v1161) diff --git a/bare-metal/container-linux/kubernetes/bootstrap.tf b/bare-metal/container-linux/kubernetes/bootstrap.tf index 00b1b352c..d42200e97 100644 --- a/bare-metal/container-linux/kubernetes/bootstrap.tf +++ b/bare-metal/container-linux/kubernetes/bootstrap.tf @@ -4,7 +4,7 @@ module "bootstrap" { cluster_name = var.cluster_name api_servers = [var.k8s_domain_name] - etcd_servers = var.controller_domains + etcd_servers = var.controllers.*.domain asset_dir = var.asset_dir networking = var.networking network_mtu = var.network_mtu diff --git a/bare-metal/container-linux/kubernetes/groups.tf b/bare-metal/container-linux/kubernetes/groups.tf index d7ee3c4e4..42b4d694e 100644 --- a/bare-metal/container-linux/kubernetes/groups.tf +++ b/bare-metal/container-linux/kubernetes/groups.tf @@ -1,34 +1,34 @@ resource "matchbox_group" "install" { - count = length(var.controller_names) + length(var.worker_names) + count = length(var.controllers) + length(var.workers) - name = format("install-%s", element(concat(var.controller_names, var.worker_names), count.index)) + name = format("install-%s", concat(var.controllers.*.name, var.workers.*.name)[count.index]) # pick one of 4 Matchbox profiles (Container Linux or Flatcar, cached or non-cached) - profile = local.flavor == "flatcar" ? var.cached_install ? element(matchbox_profile.cached-flatcar-linux-install.*.name, count.index) : element(matchbox_profile.flatcar-install.*.name, count.index) : var.cached_install ? element(matchbox_profile.cached-container-linux-install.*.name, count.index) : element(matchbox_profile.container-linux-install.*.name, count.index) + profile = local.flavor == "flatcar" ? var.cached_install ? matchbox_profile.cached-flatcar-linux-install.*.name[count.index] : matchbox_profile.flatcar-install.*.name[count.index] : var.cached_install ? matchbox_profile.cached-container-linux-install.*.name[count.index] : matchbox_profile.container-linux-install.*.name[count.index] selector = { - mac = element(concat(var.controller_macs, var.worker_macs), count.index) + mac = concat(var.controllers.*.mac, var.workers.*.mac)[count.index] } } resource "matchbox_group" "controller" { - count = length(var.controller_names) - name = format("%s-%s", var.cluster_name, element(var.controller_names, count.index)) - profile = element(matchbox_profile.controllers.*.name, count.index) + count = length(var.controllers) + name = format("%s-%s", var.cluster_name, var.controllers[count.index].name) + profile = matchbox_profile.controllers.*.name[count.index] selector = { - mac = element(var.controller_macs, count.index) + mac = var.controllers[count.index].mac os = "installed" } } resource "matchbox_group" "worker" { - count = length(var.worker_names) - name = format("%s-%s", var.cluster_name, element(var.worker_names, count.index)) - profile = element(matchbox_profile.workers.*.name, count.index) + count = length(var.workers) + name = format("%s-%s", var.cluster_name, var.workers[count.index].name) + profile = matchbox_profile.workers.*.name[count.index] selector = { - mac = element(var.worker_macs, count.index) + mac = var.workers[count.index].mac os = "installed" } } diff --git a/bare-metal/container-linux/kubernetes/profiles.tf b/bare-metal/container-linux/kubernetes/profiles.tf index 78f958f59..93cf29fc0 100644 --- a/bare-metal/container-linux/kubernetes/profiles.tf +++ b/bare-metal/container-linux/kubernetes/profiles.tf @@ -1,15 +1,14 @@ locals { # coreos-stable -> coreos flavor, stable channel # flatcar-stable -> flatcar flavor, stable channel - flavor = element(split("-", var.os_channel), 0) - - channel = element(split("-", var.os_channel), 1) + flavor = split("-", var.os_channel)[0] + channel = split("-", var.os_channel)[1] } // Container Linux Install profile (from release.core-os.net) resource "matchbox_profile" "container-linux-install" { - count = length(var.controller_names) + length(var.worker_names) - name = format("%s-container-linux-install-%s", var.cluster_name, element(concat(var.controller_names, var.worker_names), count.index)) + count = length(var.controllers) + length(var.workers) + name = format("%s-container-linux-install-%s", var.cluster_name, concat(var.controllers.*.name, var.workers.*.name)[count.index]) kernel = "${var.download_protocol}://${local.channel}.release.core-os.net/amd64-usr/${var.os_version}/coreos_production_pxe.vmlinuz" @@ -26,11 +25,11 @@ resource "matchbox_profile" "container-linux-install" { var.kernel_args, ]) - container_linux_config = element(data.template_file.container-linux-install-configs.*.rendered, count.index) + container_linux_config = data.template_file.container-linux-install-configs.*.rendered[count.index] } data "template_file" "container-linux-install-configs" { - count = length(var.controller_names) + length(var.worker_names) + count = length(var.controllers) + length(var.workers) template = file("${path.module}/cl/install.yaml.tmpl") @@ -49,8 +48,8 @@ data "template_file" "container-linux-install-configs" { // Container Linux Install profile (from matchbox /assets cache) // Note: Admin must have downloaded os_version into matchbox assets/coreos. resource "matchbox_profile" "cached-container-linux-install" { - count = length(var.controller_names) + length(var.worker_names) - name = format("%s-cached-container-linux-install-%s", var.cluster_name, element(concat(var.controller_names, var.worker_names), count.index)) + count = length(var.controllers) + length(var.workers) + name = format("%s-cached-container-linux-install-%s", var.cluster_name, concat(var.controllers.*.name, var.workers.*.name)[count.index]) kernel = "/assets/coreos/${var.os_version}/coreos_production_pxe.vmlinuz" @@ -67,11 +66,11 @@ resource "matchbox_profile" "cached-container-linux-install" { var.kernel_args, ]) - container_linux_config = element(data.template_file.cached-container-linux-install-configs.*.rendered, count.index) + container_linux_config = data.template_file.cached-container-linux-install-configs.*.rendered[count.index] } data "template_file" "cached-container-linux-install-configs" { - count = length(var.controller_names) + length(var.worker_names) + count = length(var.controllers) + length(var.workers) template = file("${path.module}/cl/install.yaml.tmpl") @@ -89,8 +88,8 @@ data "template_file" "cached-container-linux-install-configs" { // Flatcar Linux install profile (from release.flatcar-linux.net) resource "matchbox_profile" "flatcar-install" { - count = length(var.controller_names) + length(var.worker_names) - name = format("%s-flatcar-install-%s", var.cluster_name, element(concat(var.controller_names, var.worker_names), count.index)) + count = length(var.controllers) + length(var.workers) + name = format("%s-flatcar-install-%s", var.cluster_name, concat(var.controllers.*.name, var.workers.*.name)[count.index]) kernel = "${var.download_protocol}://${local.channel}.release.flatcar-linux.net/amd64-usr/${var.os_version}/flatcar_production_pxe.vmlinuz" @@ -107,14 +106,14 @@ resource "matchbox_profile" "flatcar-install" { var.kernel_args, ]) - container_linux_config = element(data.template_file.container-linux-install-configs.*.rendered, count.index) + container_linux_config = data.template_file.container-linux-install-configs.*.rendered[count.index] } // Flatcar Linux Install profile (from matchbox /assets cache) // Note: Admin must have downloaded os_version into matchbox assets/flatcar. resource "matchbox_profile" "cached-flatcar-linux-install" { - count = length(var.controller_names) + length(var.worker_names) - name = format("%s-cached-flatcar-linux-install-%s", var.cluster_name, element(concat(var.controller_names, var.worker_names), count.index)) + count = length(var.controllers) + length(var.workers) + name = format("%s-cached-flatcar-linux-install-%s", var.cluster_name, concat(var.controllers.*.name, var.workers.*.name)[count.index]) kernel = "/assets/flatcar/${var.os_version}/flatcar_production_pxe.vmlinuz" @@ -131,32 +130,32 @@ resource "matchbox_profile" "cached-flatcar-linux-install" { var.kernel_args, ]) - container_linux_config = element(data.template_file.cached-container-linux-install-configs.*.rendered, count.index) + container_linux_config = data.template_file.cached-container-linux-install-configs.*.rendered[count.index] } // Kubernetes Controller profiles resource "matchbox_profile" "controllers" { - count = length(var.controller_names) - name = format("%s-controller-%s", var.cluster_name, element(var.controller_names, count.index)) - raw_ignition = element(data.ct_config.controller-ignitions.*.rendered, count.index) + count = length(var.controllers) + name = format("%s-controller-%s", var.cluster_name, var.controllers.*.name[count.index]) + raw_ignition = data.ct_config.controller-ignitions.*.rendered[count.index] } data "ct_config" "controller-ignitions" { - count = length(var.controller_names) - content = element(data.template_file.controller-configs.*.rendered, count.index) + count = length(var.controllers) + content = data.template_file.controller-configs.*.rendered[count.index] pretty_print = false - snippets = local.clc_map[element(var.controller_names, count.index)] + snippets = local.clc_map[var.controllers.*.name[count.index]] } data "template_file" "controller-configs" { - count = length(var.controller_names) + count = length(var.controllers) template = file("${path.module}/cl/controller.yaml.tmpl") vars = { - domain_name = element(var.controller_domains, count.index) - etcd_name = element(var.controller_names, count.index) - etcd_initial_cluster = join(",", formatlist("%s=https://%s:2380", var.controller_names, var.controller_domains)) + domain_name = var.controllers.*.domain[count.index] + etcd_name = var.controllers.*.name[count.index] + etcd_initial_cluster = join(",", formatlist("%s=https://%s:2380", var.controllers.*.name, var.controllers.*.domain)) cgroup_driver = var.os_channel == "flatcar-edge" ? "systemd" : "cgroupfs" cluster_dns_service_ip = module.bootstrap.cluster_dns_service_ip cluster_domain_suffix = var.cluster_domain_suffix @@ -166,25 +165,25 @@ data "template_file" "controller-configs" { // Kubernetes Worker profiles resource "matchbox_profile" "workers" { - count = length(var.worker_names) - name = format("%s-worker-%s", var.cluster_name, element(var.worker_names, count.index)) - raw_ignition = element(data.ct_config.worker-ignitions.*.rendered, count.index) + count = length(var.workers) + name = format("%s-worker-%s", var.cluster_name, var.workers.*.name[count.index]) + raw_ignition = data.ct_config.worker-ignitions.*.rendered[count.index] } data "ct_config" "worker-ignitions" { - count = length(var.worker_names) - content = element(data.template_file.worker-configs.*.rendered, count.index) + count = length(var.workers) + content = data.template_file.worker-configs.*.rendered[count.index] pretty_print = false - snippets = local.clc_map[element(var.worker_names, count.index)] + snippets = local.clc_map[var.workers.*.name[count.index]] } data "template_file" "worker-configs" { - count = length(var.worker_names) + count = length(var.workers) template = file("${path.module}/cl/worker.yaml.tmpl") vars = { - domain_name = element(var.worker_domains, count.index) + domain_name = var.workers.*.domain[count.index] cgroup_driver = var.os_channel == "flatcar-edge" ? "systemd" : "cgroupfs" cluster_dns_service_ip = module.bootstrap.cluster_dns_service_ip cluster_domain_suffix = var.cluster_domain_suffix @@ -198,7 +197,7 @@ locals { # Default Container Linux config snippets map every node names to list("\n") so # all lookups succeed clc_defaults = zipmap( - concat(var.controller_names, var.worker_names), + concat(var.controllers.*.name, var.workers.*.name), chunklist(data.template_file.clc-default-snippets.*.rendered, 1), ) @@ -208,7 +207,7 @@ locals { // Horrible hack to generate a Terraform list of node count length data "template_file" "clc-default-snippets" { - count = length(var.controller_names) + length(var.worker_names) + count = length(var.controllers) + length(var.workers) template = "\n" } diff --git a/bare-metal/container-linux/kubernetes/ssh.tf b/bare-metal/container-linux/kubernetes/ssh.tf index 899f76474..54e59fcad 100644 --- a/bare-metal/container-linux/kubernetes/ssh.tf +++ b/bare-metal/container-linux/kubernetes/ssh.tf @@ -1,6 +1,6 @@ # Secure copy assets to controllers. Activates kubelet.service resource "null_resource" "copy-controller-secrets" { - count = length(var.controller_names) + count = length(var.controllers) # Without depends_on, remote-exec could start and wait for machines before # matchbox groups are written, causing a deadlock. @@ -13,7 +13,7 @@ resource "null_resource" "copy-controller-secrets" { connection { type = "ssh" - host = var.controller_domains[count.index] + host = var.controllers.*.domain[count.index] user = "core" timeout = "60m" } @@ -88,7 +88,7 @@ resource "null_resource" "copy-controller-secrets" { # Secure copy kubeconfig to all workers. Activates kubelet.service resource "null_resource" "copy-worker-secrets" { - count = length(var.worker_names) + count = length(var.workers) # Without depends_on, remote-exec could start and wait for machines before # matchbox groups are written, causing a deadlock. @@ -100,7 +100,7 @@ resource "null_resource" "copy-worker-secrets" { connection { type = "ssh" - host = var.worker_domains[count.index] + host = var.workers.*.domain[count.index] user = "core" timeout = "60m" } @@ -129,7 +129,7 @@ resource "null_resource" "bootstrap" { connection { type = "ssh" - host = var.controller_domains[0] + host = var.controllers[0].domain user = "core" timeout = "15m" } diff --git a/bare-metal/container-linux/kubernetes/variables.tf b/bare-metal/container-linux/kubernetes/variables.tf index 5f6d5809f..525658571 100644 --- a/bare-metal/container-linux/kubernetes/variables.tf +++ b/bare-metal/container-linux/kubernetes/variables.tf @@ -21,36 +21,32 @@ variable "os_version" { } # machines -# Terraform's crude "type system" does not properly support lists of maps so we do this. -variable "controller_names" { - type = list(string) - description = "Ordered list of controller names (e.g. [node1])" -} - -variable "controller_macs" { - type = list(string) - description = "Ordered list of controller identifying MAC addresses (e.g. [52:54:00:a1:9c:ae])" -} - -variable "controller_domains" { - type = list(string) - description = "Ordered list of controller FQDNs (e.g. [node1.example.com])" -} - -variable "worker_names" { - type = list(string) - description = "Ordered list of worker names (e.g. [node2, node3])" -} - -variable "worker_macs" { - type = list(string) - description = "Ordered list of worker identifying MAC addresses (e.g. [52:54:00:b2:2f:86, 52:54:00:c3:61:77])" +variable "controllers" { + type = list(object({ + name = string + mac = string + domain = string + })) + description = < Date: Sun, 6 Oct 2019 20:43:26 -0700 Subject: [PATCH 243/523] Update bootstrap module control plane manifests and type constraints * Remove unneeded control plane flags that correspond to defaults * Adopt Terraform v0.12 type constraints in bootstrap module --- aws/container-linux/kubernetes/bootstrap.tf | 2 +- aws/fedora-coreos/kubernetes/bootstrap.tf | 2 +- azure/container-linux/kubernetes/bootstrap.tf | 2 +- bare-metal/container-linux/kubernetes/bootstrap.tf | 2 +- bare-metal/fedora-coreos/kubernetes/bootstrap.tf | 2 +- digital-ocean/container-linux/kubernetes/bootstrap.tf | 2 +- google-cloud/container-linux/kubernetes/bootstrap.tf | 2 +- 7 files changed, 7 insertions(+), 7 deletions(-) diff --git a/aws/container-linux/kubernetes/bootstrap.tf b/aws/container-linux/kubernetes/bootstrap.tf index be3ffa101..162050752 100644 --- a/aws/container-linux/kubernetes/bootstrap.tf +++ b/aws/container-linux/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=586d6e36f67c56fb2283f317a7552638368c5779" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=10d9cec5c256f4622712bf01448df1a2befc37c8" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/aws/fedora-coreos/kubernetes/bootstrap.tf b/aws/fedora-coreos/kubernetes/bootstrap.tf index 94bb63da3..819246b21 100644 --- a/aws/fedora-coreos/kubernetes/bootstrap.tf +++ b/aws/fedora-coreos/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=586d6e36f67c56fb2283f317a7552638368c5779" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=10d9cec5c256f4622712bf01448df1a2befc37c8" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/azure/container-linux/kubernetes/bootstrap.tf b/azure/container-linux/kubernetes/bootstrap.tf index 11270a83b..90e52993e 100644 --- a/azure/container-linux/kubernetes/bootstrap.tf +++ b/azure/container-linux/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=586d6e36f67c56fb2283f317a7552638368c5779" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=10d9cec5c256f4622712bf01448df1a2befc37c8" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/bare-metal/container-linux/kubernetes/bootstrap.tf b/bare-metal/container-linux/kubernetes/bootstrap.tf index d42200e97..5b8e70bc3 100644 --- a/bare-metal/container-linux/kubernetes/bootstrap.tf +++ b/bare-metal/container-linux/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=586d6e36f67c56fb2283f317a7552638368c5779" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=10d9cec5c256f4622712bf01448df1a2befc37c8" cluster_name = var.cluster_name api_servers = [var.k8s_domain_name] diff --git a/bare-metal/fedora-coreos/kubernetes/bootstrap.tf b/bare-metal/fedora-coreos/kubernetes/bootstrap.tf index 202029c2d..99d46dfec 100644 --- a/bare-metal/fedora-coreos/kubernetes/bootstrap.tf +++ b/bare-metal/fedora-coreos/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=586d6e36f67c56fb2283f317a7552638368c5779" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=10d9cec5c256f4622712bf01448df1a2befc37c8" cluster_name = var.cluster_name api_servers = [var.k8s_domain_name] diff --git a/digital-ocean/container-linux/kubernetes/bootstrap.tf b/digital-ocean/container-linux/kubernetes/bootstrap.tf index a0418399d..687a82025 100644 --- a/digital-ocean/container-linux/kubernetes/bootstrap.tf +++ b/digital-ocean/container-linux/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=586d6e36f67c56fb2283f317a7552638368c5779" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=10d9cec5c256f4622712bf01448df1a2befc37c8" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/google-cloud/container-linux/kubernetes/bootstrap.tf b/google-cloud/container-linux/kubernetes/bootstrap.tf index 5738776e7..8473e1255 100644 --- a/google-cloud/container-linux/kubernetes/bootstrap.tf +++ b/google-cloud/container-linux/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=586d6e36f67c56fb2283f317a7552638368c5779" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=10d9cec5c256f4622712bf01448df1a2befc37c8" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] From 69188af5658d6bb8ba22897bf05c18978a60a710 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Sun, 13 Oct 2019 23:53:38 -0700 Subject: [PATCH 244/523] Rename CLUO label from "app" to "name" * Match the labeling pattern in other addons --- addons/cluo/update-agent.yaml | 4 ++-- addons/cluo/update-operator.yaml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/addons/cluo/update-agent.yaml b/addons/cluo/update-agent.yaml index 880615f3e..1d0bef0b6 100644 --- a/addons/cluo/update-agent.yaml +++ b/addons/cluo/update-agent.yaml @@ -10,11 +10,11 @@ spec: maxUnavailable: 1 selector: matchLabels: - app: container-linux-update-agent + name: container-linux-update-agent template: metadata: labels: - app: container-linux-update-agent + name: container-linux-update-agent annotations: seccomp.security.alpha.kubernetes.io/pod: 'docker/default' spec: diff --git a/addons/cluo/update-operator.yaml b/addons/cluo/update-operator.yaml index bb922ffab..d814626b7 100644 --- a/addons/cluo/update-operator.yaml +++ b/addons/cluo/update-operator.yaml @@ -7,11 +7,11 @@ spec: replicas: 1 selector: matchLabels: - app: container-linux-update-operator + name: container-linux-update-operator template: metadata: labels: - app: container-linux-update-operator + name: container-linux-update-operator annotations: seccomp.security.alpha.kubernetes.io/pod: 'docker/default' spec: From a6702573a2155a954cb4a69ef948cd1bbb33c0f6 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Tue, 15 Oct 2019 00:06:15 -0700 Subject: [PATCH 245/523] Update etcd from v3.4.1 to v3.4.2 * https://github.com/etcd-io/etcd/releases/tag/v3.4.2 --- CHANGES.md | 4 +++- aws/container-linux/kubernetes/cl/controller.yaml.tmpl | 2 +- aws/fedora-coreos/kubernetes/fcc/controller.yaml | 2 +- azure/container-linux/kubernetes/cl/controller.yaml.tmpl | 2 +- bare-metal/container-linux/kubernetes/cl/controller.yaml.tmpl | 2 +- bare-metal/fedora-coreos/kubernetes/fcc/controller.yaml | 2 +- .../container-linux/kubernetes/cl/controller.yaml.tmpl | 2 +- .../container-linux/kubernetes/cl/controller.yaml.tmpl | 2 +- 8 files changed, 10 insertions(+), 8 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 684d8d75d..dcbf6a5c9 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -4,6 +4,8 @@ Notable changes between versions. ## Latest +* Update etcd from v3.4.1 to v3.4.2 + #### Bare-Metal * Add `controllers` and `workers` as typed lists of machine detail objects ([#566](https://github.com/poseidon/typhoon/pull/566)) @@ -18,7 +20,7 @@ Notable changes between versions. ## v1.16.1 * Kubernetes [v1.16.1](https://github.com/kubernetes/kubernetes/blob/master/CHANGELOG-1.16.md#v1161) -* Update etcd from v3.3.15 to [v3.4.1](https://github.com/etcd-io/etcd/releases/tag/v3.4.1) +* Update etcd from v3.4.0 to [v3.4.1](https://github.com/etcd-io/etcd/releases/tag/v3.4.1) * Update Calico from v3.8.2 to [v3.9.1](https://docs.projectcalico.org/v3.9/release-notes/) * Add Terraform v0.12 variables types ([#553](https://github.com/poseidon/typhoon/pull/553), [#557](https://github.com/poseidon/typhoon/pull/557), [#560](https://github.com/poseidon/typhoon/pull/560), [#556](https://github.com/poseidon/typhoon/pull/556), [#562](https://github.com/poseidon/typhoon/pull/562)) * Deprecate `cluster_domain_suffix` variable diff --git a/aws/container-linux/kubernetes/cl/controller.yaml.tmpl b/aws/container-linux/kubernetes/cl/controller.yaml.tmpl index 78188f8a6..335480ea8 100644 --- a/aws/container-linux/kubernetes/cl/controller.yaml.tmpl +++ b/aws/container-linux/kubernetes/cl/controller.yaml.tmpl @@ -7,7 +7,7 @@ systemd: - name: 40-etcd-cluster.conf contents: | [Service] - Environment="ETCD_IMAGE_TAG=v3.4.1" + Environment="ETCD_IMAGE_TAG=v3.4.2" Environment="ETCD_NAME=${etcd_name}" Environment="ETCD_ADVERTISE_CLIENT_URLS=https://${etcd_domain}:2379" Environment="ETCD_INITIAL_ADVERTISE_PEER_URLS=https://${etcd_domain}:2380" diff --git a/aws/fedora-coreos/kubernetes/fcc/controller.yaml b/aws/fedora-coreos/kubernetes/fcc/controller.yaml index 6b79cad9f..4fea328f3 100644 --- a/aws/fedora-coreos/kubernetes/fcc/controller.yaml +++ b/aws/fedora-coreos/kubernetes/fcc/controller.yaml @@ -28,7 +28,7 @@ systemd: --network host \ --volume /var/lib/etcd:/var/lib/etcd:rw,Z \ --volume /etc/ssl/etcd:/etc/ssl/certs:ro,Z \ - quay.io/coreos/etcd:v3.4.1 + quay.io/coreos/etcd:v3.4.2 ExecStop=/usr/bin/podman stop etcd [Install] WantedBy=multi-user.target diff --git a/azure/container-linux/kubernetes/cl/controller.yaml.tmpl b/azure/container-linux/kubernetes/cl/controller.yaml.tmpl index 80c78652a..2e7c8dcc0 100644 --- a/azure/container-linux/kubernetes/cl/controller.yaml.tmpl +++ b/azure/container-linux/kubernetes/cl/controller.yaml.tmpl @@ -7,7 +7,7 @@ systemd: - name: 40-etcd-cluster.conf contents: | [Service] - Environment="ETCD_IMAGE_TAG=v3.4.1" + Environment="ETCD_IMAGE_TAG=v3.4.2" Environment="ETCD_NAME=${etcd_name}" Environment="ETCD_ADVERTISE_CLIENT_URLS=https://${etcd_domain}:2379" Environment="ETCD_INITIAL_ADVERTISE_PEER_URLS=https://${etcd_domain}:2380" diff --git a/bare-metal/container-linux/kubernetes/cl/controller.yaml.tmpl b/bare-metal/container-linux/kubernetes/cl/controller.yaml.tmpl index e3926a3b1..fcc7783c4 100644 --- a/bare-metal/container-linux/kubernetes/cl/controller.yaml.tmpl +++ b/bare-metal/container-linux/kubernetes/cl/controller.yaml.tmpl @@ -7,7 +7,7 @@ systemd: - name: 40-etcd-cluster.conf contents: | [Service] - Environment="ETCD_IMAGE_TAG=v3.4.1" + Environment="ETCD_IMAGE_TAG=v3.4.2" Environment="ETCD_NAME=${etcd_name}" Environment="ETCD_ADVERTISE_CLIENT_URLS=https://${domain_name}:2379" Environment="ETCD_INITIAL_ADVERTISE_PEER_URLS=https://${domain_name}:2380" diff --git a/bare-metal/fedora-coreos/kubernetes/fcc/controller.yaml b/bare-metal/fedora-coreos/kubernetes/fcc/controller.yaml index 4cc3fb96a..60c28363a 100644 --- a/bare-metal/fedora-coreos/kubernetes/fcc/controller.yaml +++ b/bare-metal/fedora-coreos/kubernetes/fcc/controller.yaml @@ -28,7 +28,7 @@ systemd: --network host \ --volume /var/lib/etcd:/var/lib/etcd:rw,Z \ --volume /etc/ssl/etcd:/etc/ssl/certs:ro,Z \ - quay.io/coreos/etcd:v3.4.1 + quay.io/coreos/etcd:v3.4.2 ExecStop=/usr/bin/podman stop etcd [Install] WantedBy=multi-user.target diff --git a/digital-ocean/container-linux/kubernetes/cl/controller.yaml.tmpl b/digital-ocean/container-linux/kubernetes/cl/controller.yaml.tmpl index 284116176..7342fa9a0 100644 --- a/digital-ocean/container-linux/kubernetes/cl/controller.yaml.tmpl +++ b/digital-ocean/container-linux/kubernetes/cl/controller.yaml.tmpl @@ -7,7 +7,7 @@ systemd: - name: 40-etcd-cluster.conf contents: | [Service] - Environment="ETCD_IMAGE_TAG=v3.4.1" + Environment="ETCD_IMAGE_TAG=v3.4.2" Environment="ETCD_NAME=${etcd_name}" Environment="ETCD_ADVERTISE_CLIENT_URLS=https://${etcd_domain}:2379" Environment="ETCD_INITIAL_ADVERTISE_PEER_URLS=https://${etcd_domain}:2380" diff --git a/google-cloud/container-linux/kubernetes/cl/controller.yaml.tmpl b/google-cloud/container-linux/kubernetes/cl/controller.yaml.tmpl index 5da50b724..49eaf2cd0 100644 --- a/google-cloud/container-linux/kubernetes/cl/controller.yaml.tmpl +++ b/google-cloud/container-linux/kubernetes/cl/controller.yaml.tmpl @@ -7,7 +7,7 @@ systemd: - name: 40-etcd-cluster.conf contents: | [Service] - Environment="ETCD_IMAGE_TAG=v3.4.1" + Environment="ETCD_IMAGE_TAG=v3.4.2" Environment="ETCD_NAME=${etcd_name}" Environment="ETCD_ADVERTISE_CLIENT_URLS=https://${etcd_domain}:2379" Environment="ETCD_INITIAL_ADVERTISE_PEER_URLS=https://${etcd_domain}:2380" From 24fc440d837afceb189986c5375b68d8ce84ce50 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Tue, 15 Oct 2019 22:42:52 -0700 Subject: [PATCH 246/523] Update Kubernetes from v1.16.1 to v1.16.2 * Update Calico from v3.9.1 to v3.9.2 --- CHANGES.md | 5 ++++- README.md | 10 +++++----- aws/container-linux/kubernetes/README.md | 2 +- aws/container-linux/kubernetes/bootstrap.tf | 2 +- .../kubernetes/cl/controller.yaml.tmpl | 4 ++-- .../kubernetes/workers/cl/worker.yaml.tmpl | 4 ++-- aws/fedora-coreos/kubernetes/README.md | 2 +- aws/fedora-coreos/kubernetes/bootstrap.tf | 2 +- aws/fedora-coreos/kubernetes/fcc/controller.yaml | 4 ++-- .../kubernetes/workers/fcc/worker.yaml | 2 +- azure/container-linux/kubernetes/README.md | 2 +- azure/container-linux/kubernetes/bootstrap.tf | 2 +- .../kubernetes/cl/controller.yaml.tmpl | 4 ++-- .../kubernetes/workers/cl/worker.yaml.tmpl | 4 ++-- bare-metal/container-linux/kubernetes/README.md | 2 +- bare-metal/container-linux/kubernetes/bootstrap.tf | 2 +- .../kubernetes/cl/controller.yaml.tmpl | 4 ++-- .../container-linux/kubernetes/cl/worker.yaml.tmpl | 2 +- bare-metal/fedora-coreos/kubernetes/README.md | 2 +- bare-metal/fedora-coreos/kubernetes/bootstrap.tf | 2 +- .../fedora-coreos/kubernetes/fcc/controller.yaml | 4 ++-- .../fedora-coreos/kubernetes/fcc/worker.yaml | 2 +- digital-ocean/container-linux/kubernetes/README.md | 2 +- .../container-linux/kubernetes/bootstrap.tf | 2 +- .../kubernetes/cl/controller.yaml.tmpl | 4 ++-- .../container-linux/kubernetes/cl/worker.yaml.tmpl | 4 ++-- docs/advanced/worker-pools.md | 14 +++++++------- docs/cl/aws.md | 10 +++++----- docs/cl/azure.md | 10 +++++----- docs/cl/bare-metal.md | 14 +++++++------- docs/cl/digital-ocean.md | 10 +++++----- docs/cl/google-cloud.md | 10 +++++----- docs/fedora-coreos/aws.md | 10 +++++----- docs/fedora-coreos/bare-metal.md | 10 +++++----- docs/index.md | 10 +++++----- docs/topics/maintenance.md | 10 +++++----- google-cloud/container-linux/kubernetes/README.md | 2 +- .../container-linux/kubernetes/bootstrap.tf | 2 +- .../kubernetes/cl/controller.yaml.tmpl | 4 ++-- .../kubernetes/workers/cl/worker.yaml.tmpl | 4 ++-- 40 files changed, 102 insertions(+), 99 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index dcbf6a5c9..f6d9b8603 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -4,7 +4,10 @@ Notable changes between versions. ## Latest -* Update etcd from v3.4.1 to v3.4.2 + +* Kubernetes [v1.16.2](https://github.com/kubernetes/kubernetes/blob/master/CHANGELOG-1.16.md#v1162) +* Update etcd from v3.4.1 to v3.4.2 ([#570](https://github.com/poseidon/typhoon/pull/570)) +* Update Calico from v3.9.1 to [v3.9.2](https://docs.projectcalico.org/v3.9/release-notes/) #### Bare-Metal diff --git a/README.md b/README.md index 69b0c4944..96b4f88cc 100644 --- a/README.md +++ b/README.md @@ -11,7 +11,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features -* Kubernetes v1.16.1 (upstream) +* Kubernetes v1.16.2 (upstream) * Single or multi-master, [Calico](https://www.projectcalico.org/) or [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/) * Advanced features like [worker pools](https://typhoon.psdn.io/advanced/worker-pools/), [preemptible](https://typhoon.psdn.io/cl/google-cloud/#preemption) workers, and [snippets](https://typhoon.psdn.io/advanced/customization/#container-linux) customization @@ -48,7 +48,7 @@ Define a Kubernetes cluster by using the Terraform module for your chosen platfo ```tf module "google-cloud-yavin" { - source = "git::https://github.com/poseidon/typhoon//google-cloud/container-linux/kubernetes?ref=v1.16.1" + source = "git::https://github.com/poseidon/typhoon//google-cloud/container-linux/kubernetes?ref=v1.16.2" # Google Cloud cluster_name = "yavin" @@ -82,9 +82,9 @@ In 4-8 minutes (varies by platform), the cluster will be ready. This Google Clou $ export KUBECONFIG=/home/user/.secrets/clusters/yavin/auth/kubeconfig $ kubectl get nodes NAME ROLES STATUS AGE VERSION -yavin-controller-0.c.example-com.internal Ready 6m v1.16.1 -yavin-worker-jrbf.c.example-com.internal Ready 5m v1.16.1 -yavin-worker-mzdm.c.example-com.internal Ready 5m v1.16.1 +yavin-controller-0.c.example-com.internal Ready 6m v1.16.2 +yavin-worker-jrbf.c.example-com.internal Ready 5m v1.16.2 +yavin-worker-mzdm.c.example-com.internal Ready 5m v1.16.2 ``` List the pods. diff --git a/aws/container-linux/kubernetes/README.md b/aws/container-linux/kubernetes/README.md index 2637d61fd..ac24929d6 100644 --- a/aws/container-linux/kubernetes/README.md +++ b/aws/container-linux/kubernetes/README.md @@ -11,7 +11,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features -* Kubernetes v1.16.1 (upstream) +* Kubernetes v1.16.2 (upstream) * Single or multi-master, [Calico](https://www.projectcalico.org/) or [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/) * Advanced features like [worker pools](https://typhoon.psdn.io/advanced/worker-pools/), [spot](https://typhoon.psdn.io/cl/aws/#spot) workers, and [snippets](https://typhoon.psdn.io/advanced/customization/#container-linux) customization diff --git a/aws/container-linux/kubernetes/bootstrap.tf b/aws/container-linux/kubernetes/bootstrap.tf index 162050752..85921b477 100644 --- a/aws/container-linux/kubernetes/bootstrap.tf +++ b/aws/container-linux/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=10d9cec5c256f4622712bf01448df1a2befc37c8" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=0fcc067476fa1463d057fd43760df222b7262b27" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/aws/container-linux/kubernetes/cl/controller.yaml.tmpl b/aws/container-linux/kubernetes/cl/controller.yaml.tmpl index 335480ea8..dac690ef3 100644 --- a/aws/container-linux/kubernetes/cl/controller.yaml.tmpl +++ b/aws/container-linux/kubernetes/cl/controller.yaml.tmpl @@ -113,7 +113,7 @@ systemd: --volume script,kind=host,source=/opt/bootstrap/apply \ --mount volume=script,target=/apply \ --insecure-options=image \ - docker://k8s.gcr.io/hyperkube:v1.16.1 \ + docker://k8s.gcr.io/hyperkube:v1.16.2 \ --net=host \ --dns=host \ --exec=/apply @@ -134,7 +134,7 @@ storage: contents: inline: | KUBELET_IMAGE_URL=docker://k8s.gcr.io/hyperkube - KUBELET_IMAGE_TAG=v1.16.1 + KUBELET_IMAGE_TAG=v1.16.2 - path: /opt/bootstrap/apply filesystem: root mode: 0544 diff --git a/aws/container-linux/kubernetes/workers/cl/worker.yaml.tmpl b/aws/container-linux/kubernetes/workers/cl/worker.yaml.tmpl index c44d68ae9..2e11d6f5c 100644 --- a/aws/container-linux/kubernetes/workers/cl/worker.yaml.tmpl +++ b/aws/container-linux/kubernetes/workers/cl/worker.yaml.tmpl @@ -98,7 +98,7 @@ storage: contents: inline: | KUBELET_IMAGE_URL=docker://k8s.gcr.io/hyperkube - KUBELET_IMAGE_TAG=v1.16.1 + KUBELET_IMAGE_TAG=v1.16.2 - path: /etc/sysctl.d/max-user-watches.conf filesystem: root contents: @@ -116,7 +116,7 @@ storage: --volume config,kind=host,source=/etc/kubernetes \ --mount volume=config,target=/etc/kubernetes \ --insecure-options=image \ - docker://k8s.gcr.io/hyperkube:v1.16.1 \ + docker://k8s.gcr.io/hyperkube:v1.16.2 \ --net=host \ --dns=host \ --exec=/kubectl -- --kubeconfig=/etc/kubernetes/kubeconfig delete node $(hostname) diff --git a/aws/fedora-coreos/kubernetes/README.md b/aws/fedora-coreos/kubernetes/README.md index e05ce48b2..a9b51d8bc 100644 --- a/aws/fedora-coreos/kubernetes/README.md +++ b/aws/fedora-coreos/kubernetes/README.md @@ -11,7 +11,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features -* Kubernetes v1.16.1 (upstream) +* Kubernetes v1.16.2 (upstream) * Single or multi-master, [Calico](https://www.projectcalico.org/) or [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/) * Advanced features like [worker pools](https://typhoon.psdn.io/advanced/worker-pools/), [spot](https://typhoon.psdn.io/cl/aws/#spot) workers, and [snippets](https://typhoon.psdn.io/advanced/customization/#container-linux) customization diff --git a/aws/fedora-coreos/kubernetes/bootstrap.tf b/aws/fedora-coreos/kubernetes/bootstrap.tf index 819246b21..c0d6937e9 100644 --- a/aws/fedora-coreos/kubernetes/bootstrap.tf +++ b/aws/fedora-coreos/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=10d9cec5c256f4622712bf01448df1a2befc37c8" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=0fcc067476fa1463d057fd43760df222b7262b27" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/aws/fedora-coreos/kubernetes/fcc/controller.yaml b/aws/fedora-coreos/kubernetes/fcc/controller.yaml index 4fea328f3..1da9ab71a 100644 --- a/aws/fedora-coreos/kubernetes/fcc/controller.yaml +++ b/aws/fedora-coreos/kubernetes/fcc/controller.yaml @@ -80,7 +80,7 @@ systemd: --volume /var/run:/var/run \ --volume /var/run/lock:/var/run/lock:z \ --volume /opt/cni/bin:/opt/cni/bin:z \ - k8s.gcr.io/hyperkube:v1.16.1 /hyperkube kubelet \ + k8s.gcr.io/hyperkube:v1.16.2 /hyperkube kubelet \ --anonymous-auth=false \ --authentication-token-webhook \ --authorization-mode=Webhook \ @@ -121,7 +121,7 @@ systemd: --network host \ --volume /opt/bootstrap/assets:/assets:ro,Z \ --volume /opt/bootstrap/apply:/apply:ro,Z \ - k8s.gcr.io/hyperkube:v1.16.1 \ + k8s.gcr.io/hyperkube:v1.16.2 \ /apply ExecStartPost=/bin/touch /opt/bootstrap/bootstrap.done ExecStartPost=-/usr/bin/podman stop bootstrap diff --git a/aws/fedora-coreos/kubernetes/workers/fcc/worker.yaml b/aws/fedora-coreos/kubernetes/workers/fcc/worker.yaml index 19a6c3cd6..2232bf2bd 100644 --- a/aws/fedora-coreos/kubernetes/workers/fcc/worker.yaml +++ b/aws/fedora-coreos/kubernetes/workers/fcc/worker.yaml @@ -50,7 +50,7 @@ systemd: --volume /var/run:/var/run \ --volume /var/run/lock:/var/run/lock:z \ --volume /opt/cni/bin:/opt/cni/bin:z \ - k8s.gcr.io/hyperkube:v1.16.1 /hyperkube kubelet \ + k8s.gcr.io/hyperkube:v1.16.2 /hyperkube kubelet \ --anonymous-auth=false \ --authentication-token-webhook \ --authorization-mode=Webhook \ diff --git a/azure/container-linux/kubernetes/README.md b/azure/container-linux/kubernetes/README.md index 1ff25137b..fed809060 100644 --- a/azure/container-linux/kubernetes/README.md +++ b/azure/container-linux/kubernetes/README.md @@ -11,7 +11,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features -* Kubernetes v1.16.1 (upstream) +* Kubernetes v1.16.2 (upstream) * Single or multi-master, [Calico](https://www.projectcalico.org/) or [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/) * Advanced features like [worker pools](https://typhoon.psdn.io/advanced/worker-pools/), [low-priority](https://typhoon.psdn.io/cl/azure/#low-priority) workers, and [snippets](https://typhoon.psdn.io/advanced/customization/#container-linux) customization diff --git a/azure/container-linux/kubernetes/bootstrap.tf b/azure/container-linux/kubernetes/bootstrap.tf index 90e52993e..87858c034 100644 --- a/azure/container-linux/kubernetes/bootstrap.tf +++ b/azure/container-linux/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=10d9cec5c256f4622712bf01448df1a2befc37c8" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=0fcc067476fa1463d057fd43760df222b7262b27" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/azure/container-linux/kubernetes/cl/controller.yaml.tmpl b/azure/container-linux/kubernetes/cl/controller.yaml.tmpl index 2e7c8dcc0..6313e4896 100644 --- a/azure/container-linux/kubernetes/cl/controller.yaml.tmpl +++ b/azure/container-linux/kubernetes/cl/controller.yaml.tmpl @@ -111,7 +111,7 @@ systemd: --volume script,kind=host,source=/opt/bootstrap/apply \ --mount volume=script,target=/apply \ --insecure-options=image \ - docker://k8s.gcr.io/hyperkube:v1.16.1 \ + docker://k8s.gcr.io/hyperkube:v1.16.2 \ --net=host \ --dns=host \ --exec=/apply @@ -132,7 +132,7 @@ storage: contents: inline: | KUBELET_IMAGE_URL=docker://k8s.gcr.io/hyperkube - KUBELET_IMAGE_TAG=v1.16.1 + KUBELET_IMAGE_TAG=v1.16.2 - path: /opt/bootstrap/apply filesystem: root mode: 0544 diff --git a/azure/container-linux/kubernetes/workers/cl/worker.yaml.tmpl b/azure/container-linux/kubernetes/workers/cl/worker.yaml.tmpl index 3736e8083..a861d3bc9 100644 --- a/azure/container-linux/kubernetes/workers/cl/worker.yaml.tmpl +++ b/azure/container-linux/kubernetes/workers/cl/worker.yaml.tmpl @@ -96,7 +96,7 @@ storage: contents: inline: | KUBELET_IMAGE_URL=docker://k8s.gcr.io/hyperkube - KUBELET_IMAGE_TAG=v1.16.1 + KUBELET_IMAGE_TAG=v1.16.2 - path: /etc/sysctl.d/max-user-watches.conf filesystem: root contents: @@ -114,7 +114,7 @@ storage: --volume config,kind=host,source=/etc/kubernetes \ --mount volume=config,target=/etc/kubernetes \ --insecure-options=image \ - docker://k8s.gcr.io/hyperkube:v1.16.1 \ + docker://k8s.gcr.io/hyperkube:v1.16.2 \ --net=host \ --dns=host \ --exec=/kubectl -- --kubeconfig=/etc/kubernetes/kubeconfig delete node $(hostname | tr '[:upper:]' '[:lower:]') diff --git a/bare-metal/container-linux/kubernetes/README.md b/bare-metal/container-linux/kubernetes/README.md index d87e02512..b15988f9b 100644 --- a/bare-metal/container-linux/kubernetes/README.md +++ b/bare-metal/container-linux/kubernetes/README.md @@ -11,7 +11,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features -* Kubernetes v1.16.1 (upstream) +* Kubernetes v1.16.2 (upstream) * Single or multi-master, [Calico](https://www.projectcalico.org/) or [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/) * Advanced features like [snippets](https://typhoon.psdn.io/advanced/customization/#container-linux) customization diff --git a/bare-metal/container-linux/kubernetes/bootstrap.tf b/bare-metal/container-linux/kubernetes/bootstrap.tf index 5b8e70bc3..18958b808 100644 --- a/bare-metal/container-linux/kubernetes/bootstrap.tf +++ b/bare-metal/container-linux/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=10d9cec5c256f4622712bf01448df1a2befc37c8" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=0fcc067476fa1463d057fd43760df222b7262b27" cluster_name = var.cluster_name api_servers = [var.k8s_domain_name] diff --git a/bare-metal/container-linux/kubernetes/cl/controller.yaml.tmpl b/bare-metal/container-linux/kubernetes/cl/controller.yaml.tmpl index fcc7783c4..39068a52d 100644 --- a/bare-metal/container-linux/kubernetes/cl/controller.yaml.tmpl +++ b/bare-metal/container-linux/kubernetes/cl/controller.yaml.tmpl @@ -126,7 +126,7 @@ systemd: --volume script,kind=host,source=/opt/bootstrap/apply \ --mount volume=script,target=/apply \ --insecure-options=image \ - docker://k8s.gcr.io/hyperkube:v1.16.1 \ + docker://k8s.gcr.io/hyperkube:v1.16.2 \ --net=host \ --dns=host \ --exec=/apply @@ -141,7 +141,7 @@ storage: contents: inline: | KUBELET_IMAGE_URL=docker://k8s.gcr.io/hyperkube - KUBELET_IMAGE_TAG=v1.16.1 + KUBELET_IMAGE_TAG=v1.16.2 - path: /etc/hostname filesystem: root mode: 0644 diff --git a/bare-metal/container-linux/kubernetes/cl/worker.yaml.tmpl b/bare-metal/container-linux/kubernetes/cl/worker.yaml.tmpl index 48017d7e1..52776c704 100644 --- a/bare-metal/container-linux/kubernetes/cl/worker.yaml.tmpl +++ b/bare-metal/container-linux/kubernetes/cl/worker.yaml.tmpl @@ -91,7 +91,7 @@ storage: contents: inline: | KUBELET_IMAGE_URL=docker://k8s.gcr.io/hyperkube - KUBELET_IMAGE_TAG=v1.16.1 + KUBELET_IMAGE_TAG=v1.16.2 - path: /etc/hostname filesystem: root mode: 0644 diff --git a/bare-metal/fedora-coreos/kubernetes/README.md b/bare-metal/fedora-coreos/kubernetes/README.md index 3efa25fd5..8d29f6f4e 100644 --- a/bare-metal/fedora-coreos/kubernetes/README.md +++ b/bare-metal/fedora-coreos/kubernetes/README.md @@ -11,7 +11,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features -* Kubernetes v1.16.1 (upstream) +* Kubernetes v1.16.2 (upstream) * Single or multi-master, [Calico](https://www.projectcalico.org/) or [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/) * Advanced features like [snippets](https://typhoon.psdn.io/advanced/customization/#container-linux) customization diff --git a/bare-metal/fedora-coreos/kubernetes/bootstrap.tf b/bare-metal/fedora-coreos/kubernetes/bootstrap.tf index 99d46dfec..6d64d37f1 100644 --- a/bare-metal/fedora-coreos/kubernetes/bootstrap.tf +++ b/bare-metal/fedora-coreos/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=10d9cec5c256f4622712bf01448df1a2befc37c8" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=0fcc067476fa1463d057fd43760df222b7262b27" cluster_name = var.cluster_name api_servers = [var.k8s_domain_name] diff --git a/bare-metal/fedora-coreos/kubernetes/fcc/controller.yaml b/bare-metal/fedora-coreos/kubernetes/fcc/controller.yaml index 60c28363a..e1951cacd 100644 --- a/bare-metal/fedora-coreos/kubernetes/fcc/controller.yaml +++ b/bare-metal/fedora-coreos/kubernetes/fcc/controller.yaml @@ -81,7 +81,7 @@ systemd: --volume /opt/cni/bin:/opt/cni/bin:z \ --volume /etc/iscsi:/etc/iscsi \ --volume /sbin/iscsiadm:/sbin/iscsiadm \ - k8s.gcr.io/hyperkube:v1.16.1 /hyperkube kubelet \ + k8s.gcr.io/hyperkube:v1.16.2 /hyperkube kubelet \ --anonymous-auth=false \ --authentication-token-webhook \ --authorization-mode=Webhook \ @@ -132,7 +132,7 @@ systemd: --network host \ --volume /opt/bootstrap/assets:/assets:ro,Z \ --volume /opt/bootstrap/apply:/apply:ro,Z \ - k8s.gcr.io/hyperkube:v1.16.1 \ + k8s.gcr.io/hyperkube:v1.16.2 \ /apply ExecStartPost=/bin/touch /opt/bootstrap/bootstrap.done ExecStartPost=-/usr/bin/podman stop bootstrap diff --git a/bare-metal/fedora-coreos/kubernetes/fcc/worker.yaml b/bare-metal/fedora-coreos/kubernetes/fcc/worker.yaml index 823b9cb19..a02a60998 100644 --- a/bare-metal/fedora-coreos/kubernetes/fcc/worker.yaml +++ b/bare-metal/fedora-coreos/kubernetes/fcc/worker.yaml @@ -51,7 +51,7 @@ systemd: --volume /opt/cni/bin:/opt/cni/bin:z \ --volume /etc/iscsi:/etc/iscsi \ --volume /sbin/iscsiadm:/sbin/iscsiadm \ - k8s.gcr.io/hyperkube:v1.16.1 /hyperkube kubelet \ + k8s.gcr.io/hyperkube:v1.16.2 /hyperkube kubelet \ --anonymous-auth=false \ --authentication-token-webhook \ --authorization-mode=Webhook \ diff --git a/digital-ocean/container-linux/kubernetes/README.md b/digital-ocean/container-linux/kubernetes/README.md index 1a8067fd6..348cc4384 100644 --- a/digital-ocean/container-linux/kubernetes/README.md +++ b/digital-ocean/container-linux/kubernetes/README.md @@ -11,7 +11,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features -* Kubernetes v1.16.1 (upstream) +* Kubernetes v1.16.2 (upstream) * Single or multi-master, [Calico](https://www.projectcalico.org/) or [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/) * Advanced features like [snippets](https://typhoon.psdn.io/advanced/customization/#container-linux) customization diff --git a/digital-ocean/container-linux/kubernetes/bootstrap.tf b/digital-ocean/container-linux/kubernetes/bootstrap.tf index 687a82025..a2030561a 100644 --- a/digital-ocean/container-linux/kubernetes/bootstrap.tf +++ b/digital-ocean/container-linux/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=10d9cec5c256f4622712bf01448df1a2befc37c8" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=0fcc067476fa1463d057fd43760df222b7262b27" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/digital-ocean/container-linux/kubernetes/cl/controller.yaml.tmpl b/digital-ocean/container-linux/kubernetes/cl/controller.yaml.tmpl index 7342fa9a0..c9a905549 100644 --- a/digital-ocean/container-linux/kubernetes/cl/controller.yaml.tmpl +++ b/digital-ocean/container-linux/kubernetes/cl/controller.yaml.tmpl @@ -123,7 +123,7 @@ systemd: --volume script,kind=host,source=/opt/bootstrap/apply \ --mount volume=script,target=/apply \ --insecure-options=image \ - docker://k8s.gcr.io/hyperkube:v1.16.1 \ + docker://k8s.gcr.io/hyperkube:v1.16.2 \ --net=host \ --dns=host \ --exec=/apply @@ -138,7 +138,7 @@ storage: contents: inline: | KUBELET_IMAGE_URL=docker://k8s.gcr.io/hyperkube - KUBELET_IMAGE_TAG=v1.16.1 + KUBELET_IMAGE_TAG=v1.16.2 - path: /opt/bootstrap/apply filesystem: root mode: 0544 diff --git a/digital-ocean/container-linux/kubernetes/cl/worker.yaml.tmpl b/digital-ocean/container-linux/kubernetes/cl/worker.yaml.tmpl index cd5286bf0..d7146a5ec 100644 --- a/digital-ocean/container-linux/kubernetes/cl/worker.yaml.tmpl +++ b/digital-ocean/container-linux/kubernetes/cl/worker.yaml.tmpl @@ -99,7 +99,7 @@ storage: contents: inline: | KUBELET_IMAGE_URL=docker://k8s.gcr.io/hyperkube - KUBELET_IMAGE_TAG=v1.16.1 + KUBELET_IMAGE_TAG=v1.16.2 - path: /etc/sysctl.d/max-user-watches.conf filesystem: root contents: @@ -117,7 +117,7 @@ storage: --volume config,kind=host,source=/etc/kubernetes \ --mount volume=config,target=/etc/kubernetes \ --insecure-options=image \ - docker://k8s.gcr.io/hyperkube:v1.16.1 \ + docker://k8s.gcr.io/hyperkube:v1.16.2 \ --net=host \ --dns=host \ --exec=/kubectl -- --kubeconfig=/etc/kubernetes/kubeconfig delete node $(hostname) diff --git a/docs/advanced/worker-pools.md b/docs/advanced/worker-pools.md index a9dac3b74..e37eb0a5b 100644 --- a/docs/advanced/worker-pools.md +++ b/docs/advanced/worker-pools.md @@ -79,7 +79,7 @@ Create a cluster following the Azure [tutorial](../cl/azure.md#cluster). Define ```tf module "ramius-worker-pool" { - source = "git::https://github.com/poseidon/typhoon//azure/container-linux/kubernetes/workers?ref=v1.16.1" + source = "git::https://github.com/poseidon/typhoon//azure/container-linux/kubernetes/workers?ref=v1.16.2" # Azure region = module.azure-ramius.region @@ -145,7 +145,7 @@ Create a cluster following the Google Cloud [tutorial](../cl/google-cloud.md#clu ```tf module "yavin-worker-pool" { - source = "git::https://github.com/poseidon/typhoon//google-cloud/container-linux/kubernetes/workers?ref=v1.16.1" + source = "git::https://github.com/poseidon/typhoon//google-cloud/container-linux/kubernetes/workers?ref=v1.16.2" # Google Cloud region = "europe-west2" @@ -176,11 +176,11 @@ Verify a managed instance group of workers joins the cluster within a few minute ``` $ kubectl get nodes NAME STATUS AGE VERSION -yavin-controller-0.c.example-com.internal Ready 6m v1.16.1 -yavin-worker-jrbf.c.example-com.internal Ready 5m v1.16.1 -yavin-worker-mzdm.c.example-com.internal Ready 5m v1.16.1 -yavin-16x-worker-jrbf.c.example-com.internal Ready 3m v1.16.1 -yavin-16x-worker-mzdm.c.example-com.internal Ready 3m v1.16.1 +yavin-controller-0.c.example-com.internal Ready 6m v1.16.2 +yavin-worker-jrbf.c.example-com.internal Ready 5m v1.16.2 +yavin-worker-mzdm.c.example-com.internal Ready 5m v1.16.2 +yavin-16x-worker-jrbf.c.example-com.internal Ready 3m v1.16.2 +yavin-16x-worker-mzdm.c.example-com.internal Ready 3m v1.16.2 ``` ### Variables diff --git a/docs/cl/aws.md b/docs/cl/aws.md index b2a96c1bc..229af4dda 100644 --- a/docs/cl/aws.md +++ b/docs/cl/aws.md @@ -1,6 +1,6 @@ # AWS -In this tutorial, we'll create a Kubernetes v1.16.1 cluster on AWS with Container Linux. +In this tutorial, we'll create a Kubernetes v1.16.2 cluster on AWS with Container Linux. We'll declare a Kubernetes cluster using the Typhoon Terraform module. Then apply the changes to create a VPC, gateway, subnets, security groups, controller instances, worker auto-scaling group, network load balancer, and TLS assets. @@ -70,7 +70,7 @@ Define a Kubernetes cluster using the module `aws/container-linux/kubernetes`. ```tf module "tempest" { - source = "git::https://github.com/poseidon/typhoon//aws/container-linux/kubernetes?ref=v1.16.1" + source = "git::https://github.com/poseidon/typhoon//aws/container-linux/kubernetes?ref=v1.16.2" # AWS cluster_name = "tempest" @@ -135,9 +135,9 @@ In 4-8 minutes, the Kubernetes cluster will be ready. $ export KUBECONFIG=/home/user/.secrets/clusters/tempest/auth/kubeconfig $ kubectl get nodes NAME STATUS ROLES AGE VERSION -ip-10-0-3-155 Ready 10m v1.16.1 -ip-10-0-26-65 Ready 10m v1.16.1 -ip-10-0-41-21 Ready 10m v1.16.1 +ip-10-0-3-155 Ready 10m v1.16.2 +ip-10-0-26-65 Ready 10m v1.16.2 +ip-10-0-41-21 Ready 10m v1.16.2 ``` List the pods. diff --git a/docs/cl/azure.md b/docs/cl/azure.md index 2ac012b74..f4b52364a 100644 --- a/docs/cl/azure.md +++ b/docs/cl/azure.md @@ -3,7 +3,7 @@ !!! danger Typhoon for Azure is alpha. For production, use AWS, Google Cloud, or bare-metal. As Azure matures, check [errata](https://github.com/poseidon/typhoon/wiki/Errata) for known shortcomings. -In this tutorial, we'll create a Kubernetes v1.16.1 cluster on Azure with Container Linux. +In this tutorial, we'll create a Kubernetes v1.16.2 cluster on Azure with Container Linux. We'll declare a Kubernetes cluster using the Typhoon Terraform module. Then apply the changes to create a resource group, virtual network, subnets, security groups, controller availability set, worker scale set, load balancer, and TLS assets. @@ -66,7 +66,7 @@ Define a Kubernetes cluster using the module `azure/container-linux/kubernetes`. ```tf module "ramius" { - source = "git::https://github.com/poseidon/typhoon//azure/container-linux/kubernetes?ref=v1.16.1" + source = "git::https://github.com/poseidon/typhoon//azure/container-linux/kubernetes?ref=v1.16.2" # Azure cluster_name = "ramius" @@ -132,9 +132,9 @@ In 4-8 minutes, the Kubernetes cluster will be ready. $ export KUBECONFIG=/home/user/.secrets/clusters/ramius/auth/kubeconfig $ kubectl get nodes NAME STATUS ROLES AGE VERSION -ramius-controller-0 Ready 24m v1.16.1 -ramius-worker-000001 Ready 25m v1.16.1 -ramius-worker-000002 Ready 24m v1.16.1 +ramius-controller-0 Ready 24m v1.16.2 +ramius-worker-000001 Ready 25m v1.16.2 +ramius-worker-000002 Ready 24m v1.16.2 ``` List the pods. diff --git a/docs/cl/bare-metal.md b/docs/cl/bare-metal.md index 69fcc3be9..dcdcc6ab5 100644 --- a/docs/cl/bare-metal.md +++ b/docs/cl/bare-metal.md @@ -1,6 +1,6 @@ # Bare-Metal -In this tutorial, we'll network boot and provision a Kubernetes v1.16.1 cluster on bare-metal with Container Linux. +In this tutorial, we'll network boot and provision a Kubernetes v1.16.2 cluster on bare-metal with Container Linux. First, we'll deploy a [Matchbox](https://github.com/poseidon/matchbox) service and setup a network boot environment. Then, we'll declare a Kubernetes cluster using the Typhoon Terraform module and power on machines. On PXE boot, machines will install Container Linux to disk, reboot into the disk install, and provision themselves as Kubernetes controllers or workers via Ignition. @@ -160,7 +160,7 @@ Define a Kubernetes cluster using the module `bare-metal/container-linux/kuberne ```tf module "bare-metal-mercury" { - source = "git::https://github.com/poseidon/typhoon//bare-metal/container-linux/kubernetes?ref=v1.16.1" + source = "git::https://github.com/poseidon/typhoon//bare-metal/container-linux/kubernetes?ref=v1.16.2" # bare-metal cluster_name = "mercury" @@ -265,9 +265,9 @@ Apply complete! Resources: 55 added, 0 changed, 0 destroyed. To watch the install to disk (until machines reboot from disk), SSH to port 2222. ``` -# before v1.16.1 +# before v1.16.2 $ ssh debug@node1.example.com -# after v1.16.1 +# after v1.16.2 $ ssh -p 2222 core@node1.example.com ``` @@ -291,9 +291,9 @@ systemd[1]: Started Kubernetes control plane. $ export KUBECONFIG=/home/user/.secrets/clusters/mercury/auth/kubeconfig $ kubectl get nodes NAME STATUS ROLES AGE VERSION -node1.example.com Ready 10m v1.16.1 -node2.example.com Ready 10m v1.16.1 -node3.example.com Ready 10m v1.16.1 +node1.example.com Ready 10m v1.16.2 +node2.example.com Ready 10m v1.16.2 +node3.example.com Ready 10m v1.16.2 ``` List the pods. diff --git a/docs/cl/digital-ocean.md b/docs/cl/digital-ocean.md index 20bad53ed..1e1597782 100644 --- a/docs/cl/digital-ocean.md +++ b/docs/cl/digital-ocean.md @@ -1,6 +1,6 @@ # Digital Ocean -In this tutorial, we'll create a Kubernetes v1.16.1 cluster on DigitalOcean with Container Linux. +In this tutorial, we'll create a Kubernetes v1.16.2 cluster on DigitalOcean with Container Linux. We'll declare a Kubernetes cluster using the Typhoon Terraform module. Then apply the changes to create controller droplets, worker droplets, DNS records, tags, and TLS assets. @@ -65,7 +65,7 @@ Define a Kubernetes cluster using the module `digital-ocean/container-linux/kube ```tf module "digital-ocean-nemo" { - source = "git::https://github.com/poseidon/typhoon//digital-ocean/container-linux/kubernetes?ref=v1.16.1" + source = "git::https://github.com/poseidon/typhoon//digital-ocean/container-linux/kubernetes?ref=v1.16.2" # Digital Ocean cluster_name = "nemo" @@ -130,9 +130,9 @@ In 3-6 minutes, the Kubernetes cluster will be ready. $ export KUBECONFIG=/home/user/.secrets/clusters/nemo/auth/kubeconfig $ kubectl get nodes NAME STATUS ROLES AGE VERSION -10.132.110.130 Ready 10m v1.16.1 -10.132.115.81 Ready 10m v1.16.1 -10.132.124.107 Ready 10m v1.16.1 +10.132.110.130 Ready 10m v1.16.2 +10.132.115.81 Ready 10m v1.16.2 +10.132.124.107 Ready 10m v1.16.2 ``` List the pods. diff --git a/docs/cl/google-cloud.md b/docs/cl/google-cloud.md index 1600cb681..6beed9382 100644 --- a/docs/cl/google-cloud.md +++ b/docs/cl/google-cloud.md @@ -1,6 +1,6 @@ # Google Cloud -In this tutorial, we'll create a Kubernetes v1.16.1 cluster on Google Compute Engine with Container Linux. +In this tutorial, we'll create a Kubernetes v1.16.2 cluster on Google Compute Engine with Container Linux. We'll declare a Kubernetes cluster using the Typhoon Terraform module. Then apply the changes to create a network, firewall rules, health checks, controller instances, worker managed instance group, load balancers, and TLS assets. @@ -71,7 +71,7 @@ Define a Kubernetes cluster using the module `google-cloud/container-linux/kuber ```tf module "google-cloud-yavin" { - source = "git::https://github.com/poseidon/typhoon//google-cloud/container-linux/kubernetes?ref=v1.16.1" + source = "git::https://github.com/poseidon/typhoon//google-cloud/container-linux/kubernetes?ref=v1.16.2" # Google Cloud cluster_name = "yavin" @@ -137,9 +137,9 @@ In 4-8 minutes, the Kubernetes cluster will be ready. $ export KUBECONFIG=/home/user/.secrets/clusters/yavin/auth/kubeconfig $ kubectl get nodes NAME ROLES STATUS AGE VERSION -yavin-controller-0.c.example-com.internal Ready 6m v1.16.1 -yavin-worker-jrbf.c.example-com.internal Ready 5m v1.16.1 -yavin-worker-mzdm.c.example-com.internal Ready 5m v1.16.1 +yavin-controller-0.c.example-com.internal Ready 6m v1.16.2 +yavin-worker-jrbf.c.example-com.internal Ready 5m v1.16.2 +yavin-worker-mzdm.c.example-com.internal Ready 5m v1.16.2 ``` List the pods. diff --git a/docs/fedora-coreos/aws.md b/docs/fedora-coreos/aws.md index def06dadf..8c8d9eb12 100644 --- a/docs/fedora-coreos/aws.md +++ b/docs/fedora-coreos/aws.md @@ -3,7 +3,7 @@ !!! danger Typhoon for Fedora CoreOS is an early preview! Fedora CoreOS itself is a preview! Expect bugs and design shifts. Please help both projects solve problems. Report Fedora CoreOS bugs to [Fedora](https://github.com/coreos/fedora-coreos-tracker/issues). Report Typhoon issues to Typhoon. -In this tutorial, we'll create a Kubernetes v1.16.1 cluster on AWS with Fedora CoreOS. +In this tutorial, we'll create a Kubernetes v1.16.2 cluster on AWS with Fedora CoreOS. We'll declare a Kubernetes cluster using the Typhoon Terraform module. Then apply the changes to create a VPC, gateway, subnets, security groups, controller instances, worker auto-scaling group, network load balancer, and TLS assets. @@ -73,7 +73,7 @@ Define a Kubernetes cluster using the module `aws/fedora-coreos/kubernetes`. ```tf module "aws-tempest" { - source = "git::https://github.com/poseidon/typhoon//aws/fedora-coreos/kubernetes?ref=v1.16.1" + source = "git::https://github.com/poseidon/typhoon//aws/fedora-coreos/kubernetes?ref=v1.16.2" # AWS cluster_name = "tempest" @@ -138,9 +138,9 @@ In 4-8 minutes, the Kubernetes cluster will be ready. $ export KUBECONFIG=/home/user/.secrets/clusters/tempest/auth/kubeconfig $ kubectl get nodes NAME STATUS ROLES AGE VERSION -ip-10-0-3-155 Ready 10m v1.16.1 -ip-10-0-26-65 Ready 10m v1.16.1 -ip-10-0-41-21 Ready 10m v1.16.1 +ip-10-0-3-155 Ready 10m v1.16.2 +ip-10-0-26-65 Ready 10m v1.16.2 +ip-10-0-41-21 Ready 10m v1.16.2 ``` List the pods. diff --git a/docs/fedora-coreos/bare-metal.md b/docs/fedora-coreos/bare-metal.md index b6f6b3495..28d5ffadc 100644 --- a/docs/fedora-coreos/bare-metal.md +++ b/docs/fedora-coreos/bare-metal.md @@ -3,7 +3,7 @@ !!! danger Typhoon for Fedora CoreOS is an early preview! Fedora CoreOS itself is a preview! Expect bugs and design shifts. Please help both projects solve problems. Report Fedora CoreOS bugs to [Fedora](https://github.com/coreos/fedora-coreos-tracker/issues). Report Typhoon issues to Typhoon. -In this tutorial, we'll network boot and provision a Kubernetes v1.16.1 cluster on bare-metal with Fedora CoreOS. +In this tutorial, we'll network boot and provision a Kubernetes v1.16.2 cluster on bare-metal with Fedora CoreOS. First, we'll deploy a [Matchbox](https://github.com/poseidon/matchbox) service and setup a network boot environment. Then, we'll declare a Kubernetes cluster using the Typhoon Terraform module and power on machines. On PXE boot, machines will install Fedora CoreOS to disk, reboot into the disk install, and provision themselves as Kubernetes controllers or workers via Ignition. @@ -163,7 +163,7 @@ Define a Kubernetes cluster using the module `bare-metal/fedora-coreos/kubernete ```tf module "bare-metal-mercury" { - source = "git::https://github.com/poseidon/typhoon//bare-metal/fedora-coreos/kubernetes?ref=v1.16.1" + source = "git::https://github.com/poseidon/typhoon//bare-metal/fedora-coreos/kubernetes?ref=v1.16.2" # bare-metal cluster_name = "mercury" @@ -285,9 +285,9 @@ systemd[1]: Started Kubernetes control plane. $ export KUBECONFIG=/home/user/.secrets/clusters/mercury/auth/kubeconfig $ kubectl get nodes NAME STATUS ROLES AGE VERSION -node1.example.com Ready 10m v1.16.1 -node2.example.com Ready 10m v1.16.1 -node3.example.com Ready 10m v1.16.1 +node1.example.com Ready 10m v1.16.2 +node2.example.com Ready 10m v1.16.2 +node3.example.com Ready 10m v1.16.2 ``` List the pods. diff --git a/docs/index.md b/docs/index.md index 928d5383b..333cf3207 100644 --- a/docs/index.md +++ b/docs/index.md @@ -11,7 +11,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features -* Kubernetes v1.16.1 (upstream) +* Kubernetes v1.16.2 (upstream) * Single or multi-master, [Calico](https://www.projectcalico.org/) or [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/) * Advanced features like [worker pools](advanced/worker-pools/), [preemptible](cl/google-cloud/#preemption) workers, and [snippets](advanced/customization/#container-linux) customization @@ -47,7 +47,7 @@ Define a Kubernetes cluster by using the Terraform module for your chosen platfo ```tf module "google-cloud-yavin" { - source = "git::https://github.com/poseidon/typhoon//google-cloud/container-linux/kubernetes?ref=v1.16.1" + source = "git::https://github.com/poseidon/typhoon//google-cloud/container-linux/kubernetes?ref=v1.16.2" # Google Cloud cluster_name = "yavin" @@ -80,9 +80,9 @@ In 4-8 minutes (varies by platform), the cluster will be ready. This Google Clou $ export KUBECONFIG=/home/user/.secrets/clusters/yavin/auth/kubeconfig $ kubectl get nodes NAME ROLES STATUS AGE VERSION -yavin-controller-0.c.example-com.internal Ready 6m v1.16.1 -yavin-worker-jrbf.c.example-com.internal Ready 5m v1.16.1 -yavin-worker-mzdm.c.example-com.internal Ready 5m v1.16.1 +yavin-controller-0.c.example-com.internal Ready 6m v1.16.2 +yavin-worker-jrbf.c.example-com.internal Ready 5m v1.16.2 +yavin-worker-mzdm.c.example-com.internal Ready 5m v1.16.2 ``` List the pods. diff --git a/docs/topics/maintenance.md b/docs/topics/maintenance.md index 6e3645d52..42b543dfa 100644 --- a/docs/topics/maintenance.md +++ b/docs/topics/maintenance.md @@ -18,7 +18,7 @@ module "google-cloud-yavin" { } module "bare-metal-mercury" { - source = "git::https://github.com/poseidon/typhoon//bare-metal/container-linux/kubernetes?ref=v1.16.1" + source = "git::https://github.com/poseidon/typhoon//bare-metal/container-linux/kubernetes?ref=v1.16.2" ... } ``` @@ -279,15 +279,15 @@ Typhoon modules have been adapted for Terraform v0.12. Provider plugins requirem | Typhoon Release | Terraform version | |-------------------|---------------------| -| v1.16.1 - ? | v0.12.x | -| v1.10.3 - v1.16.1 | v0.11.x | +| v1.16.2 - ? | v0.12.x | +| v1.10.3 - v1.16.2 | v0.11.x | | v1.9.2 - v1.10.2 | v0.10.4+ or v0.11.x | | v1.7.3 - v1.9.1 | v0.10.x | | v1.6.4 - v1.7.2 | v0.9.x | ### New users -New users can start with Terraform v0.12.x and follow the docs for Typhoon v1.16.1+ without issue. +New users can start with Terraform v0.12.x and follow the docs for Typhoon v1.16.2+ without issue. ### Existing users @@ -404,7 +404,7 @@ tree . └── infraB <- new Terraform v0.12.x configs ``` -Define Typhoon clusters in the new config directory using Terraform v0.12 syntax. Follow the Typhoon v1.16.1+ docs (e.g. use `terraform12` in the `infraB` dir). See [AWS](/cl/aws), [Azure](/cl/azure), [Bare-Metal](/cl/bare-metal), [Digital Ocean](/cl/digital-ocean), or [Google-Cloud](/cl/google-cloud)) to create new clusters. Follow the usual [upgrade](/topics/maintenance/#upgrades) process to apply workloads and shift traffic. Later, switch back to the old config directory and deprovision clusters with Terraform v0.11. +Define Typhoon clusters in the new config directory using Terraform v0.12 syntax. Follow the Typhoon v1.16.2+ docs (e.g. use `terraform12` in the `infraB` dir). See [AWS](/cl/aws), [Azure](/cl/azure), [Bare-Metal](/cl/bare-metal), [Digital Ocean](/cl/digital-ocean), or [Google-Cloud](/cl/google-cloud)) to create new clusters. Follow the usual [upgrade](/topics/maintenance/#upgrades) process to apply workloads and shift traffic. Later, switch back to the old config directory and deprovision clusters with Terraform v0.11. ```shell terraform12 init diff --git a/google-cloud/container-linux/kubernetes/README.md b/google-cloud/container-linux/kubernetes/README.md index 825b13677..f80c41948 100644 --- a/google-cloud/container-linux/kubernetes/README.md +++ b/google-cloud/container-linux/kubernetes/README.md @@ -11,7 +11,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features -* Kubernetes v1.16.1 (upstream) +* Kubernetes v1.16.2 (upstream) * Single or multi-master, [Calico](https://www.projectcalico.org/) or [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/) * Advanced features like [worker pools](https://typhoon.psdn.io/advanced/worker-pools/), [preemptible](https://typhoon.psdn.io/cl/google-cloud/#preemption) workers, and [snippets](https://typhoon.psdn.io/advanced/customization/#container-linux) customization diff --git a/google-cloud/container-linux/kubernetes/bootstrap.tf b/google-cloud/container-linux/kubernetes/bootstrap.tf index 8473e1255..de83be726 100644 --- a/google-cloud/container-linux/kubernetes/bootstrap.tf +++ b/google-cloud/container-linux/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=10d9cec5c256f4622712bf01448df1a2befc37c8" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=0fcc067476fa1463d057fd43760df222b7262b27" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/google-cloud/container-linux/kubernetes/cl/controller.yaml.tmpl b/google-cloud/container-linux/kubernetes/cl/controller.yaml.tmpl index 49eaf2cd0..0b53119f7 100644 --- a/google-cloud/container-linux/kubernetes/cl/controller.yaml.tmpl +++ b/google-cloud/container-linux/kubernetes/cl/controller.yaml.tmpl @@ -112,7 +112,7 @@ systemd: --volume script,kind=host,source=/opt/bootstrap/apply \ --mount volume=script,target=/apply \ --insecure-options=image \ - docker://k8s.gcr.io/hyperkube:v1.16.1 \ + docker://k8s.gcr.io/hyperkube:v1.16.2 \ --net=host \ --dns=host \ --exec=/apply @@ -133,7 +133,7 @@ storage: contents: inline: | KUBELET_IMAGE_URL=docker://k8s.gcr.io/hyperkube - KUBELET_IMAGE_TAG=v1.16.1 + KUBELET_IMAGE_TAG=v1.16.2 - path: /opt/bootstrap/apply filesystem: root mode: 0544 diff --git a/google-cloud/container-linux/kubernetes/workers/cl/worker.yaml.tmpl b/google-cloud/container-linux/kubernetes/workers/cl/worker.yaml.tmpl index e975fc68e..f5e3dea63 100644 --- a/google-cloud/container-linux/kubernetes/workers/cl/worker.yaml.tmpl +++ b/google-cloud/container-linux/kubernetes/workers/cl/worker.yaml.tmpl @@ -97,7 +97,7 @@ storage: contents: inline: | KUBELET_IMAGE_URL=docker://k8s.gcr.io/hyperkube - KUBELET_IMAGE_TAG=v1.16.1 + KUBELET_IMAGE_TAG=v1.16.2 - path: /etc/sysctl.d/max-user-watches.conf filesystem: root contents: @@ -115,7 +115,7 @@ storage: --volume config,kind=host,source=/etc/kubernetes \ --mount volume=config,target=/etc/kubernetes \ --insecure-options=image \ - docker://k8s.gcr.io/hyperkube:v1.16.1 \ + docker://k8s.gcr.io/hyperkube:v1.16.2 \ --net=host \ --dns=host \ --exec=/kubectl -- --kubeconfig=/etc/kubernetes/kubeconfig delete node $(hostname) From e4ac1027c895699e2ddd649d63b254ce62fa9f95 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Tue, 15 Oct 2019 22:58:43 -0700 Subject: [PATCH 247/523] Update Grafana from v6.4.1 to v6.4.2 * https://github.com/grafana/grafana/releases/tag/v6.4.2 --- CHANGES.md | 5 +++++ addons/grafana/deployment.yaml | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index f6d9b8603..dcfab3f66 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -20,6 +20,11 @@ Notable changes between versions. * Remove `worker_macs` list variable * Remove `worker_domains` list variable +#### Addons + +* Update Grafana from v6.4.1 to [v6.4.2](https://github.com/grafana/grafana/releases/tag/v6.4.2) + + ## v1.16.1 * Kubernetes [v1.16.1](https://github.com/kubernetes/kubernetes/blob/master/CHANGELOG-1.16.md#v1161) diff --git a/addons/grafana/deployment.yaml b/addons/grafana/deployment.yaml index 2f4b95dc1..9d273c71f 100644 --- a/addons/grafana/deployment.yaml +++ b/addons/grafana/deployment.yaml @@ -23,7 +23,7 @@ spec: spec: containers: - name: grafana - image: docker.io/grafana/grafana:6.4.1 + image: docker.io/grafana/grafana:6.4.2 env: - name: GF_PATHS_CONFIG value: "/etc/grafana/custom.ini" From e6bc5143aa2f605f14c1164bce08e96bff35ccfe Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Tue, 15 Oct 2019 23:09:41 -0700 Subject: [PATCH 248/523] Default to Calico as the CNI provider on Azure/DigitalOcean * Change `networking` default from flannel to calico on Azure and DigitalOcean * AWS, bare-metal, and Google Cloud continue to default to Calico (as they have since v1.7.5) * Typhoon now defaults to using Calico and supporting NetworkPolicy on all platforms --- CHANGES.md | 11 +++++++++-- azure/container-linux/kubernetes/variables.tf | 2 +- digital-ocean/container-linux/kubernetes/variables.tf | 2 +- docs/cl/azure.md | 8 ++++---- docs/cl/digital-ocean.md | 8 ++++---- docs/topics/security.md | 2 +- 6 files changed, 20 insertions(+), 13 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index dcfab3f66..06f2fbb1d 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -4,10 +4,14 @@ Notable changes between versions. ## Latest - * Kubernetes [v1.16.2](https://github.com/kubernetes/kubernetes/blob/master/CHANGELOG-1.16.md#v1162) * Update etcd from v3.4.1 to v3.4.2 ([#570](https://github.com/poseidon/typhoon/pull/570)) * Update Calico from v3.9.1 to [v3.9.2](https://docs.projectcalico.org/v3.9/release-notes/) + * Default to using Calico and supporting NetworkPolicy on all platforms + +#### Azure + +* Change default networking provider from "flannel" to "calico" ([#573](https://github.com/poseidon/typhoon/pull/573)) #### Bare-Metal @@ -20,11 +24,14 @@ Notable changes between versions. * Remove `worker_macs` list variable * Remove `worker_domains` list variable +#### DigitalOcean + +* Change default networking provider from "flannel" to "calico" ([#573](https://github.com/poseidon/typhoon/pull/573)) + #### Addons * Update Grafana from v6.4.1 to [v6.4.2](https://github.com/grafana/grafana/releases/tag/v6.4.2) - ## v1.16.1 * Kubernetes [v1.16.1](https://github.com/kubernetes/kubernetes/blob/master/CHANGELOG-1.16.md#v1161) diff --git a/azure/container-linux/kubernetes/variables.tf b/azure/container-linux/kubernetes/variables.tf index e80b91b24..52748e636 100644 --- a/azure/container-linux/kubernetes/variables.tf +++ b/azure/container-linux/kubernetes/variables.tf @@ -91,7 +91,7 @@ variable "asset_dir" { variable "networking" { type = string description = "Choice of networking provider (flannel or calico)" - default = "flannel" + default = "calico" } variable "host_cidr" { diff --git a/digital-ocean/container-linux/kubernetes/variables.tf b/digital-ocean/container-linux/kubernetes/variables.tf index cf8a1ab18..7d1f0af9f 100644 --- a/digital-ocean/container-linux/kubernetes/variables.tf +++ b/digital-ocean/container-linux/kubernetes/variables.tf @@ -74,7 +74,7 @@ variable "asset_dir" { variable "networking" { type = string description = "Choice of networking provider (flannel or calico)" - default = "flannel" + default = "calico" } variable "pod_cidr" { diff --git a/docs/cl/azure.md b/docs/cl/azure.md index f4b52364a..909e1f106 100644 --- a/docs/cl/azure.md +++ b/docs/cl/azure.md @@ -144,9 +144,9 @@ $ kubectl get pods --all-namespaces NAMESPACE NAME READY STATUS RESTARTS AGE kube-system coredns-7c6fbb4f4b-b6qzx 1/1 Running 0 26m kube-system coredns-7c6fbb4f4b-j2k3d 1/1 Running 0 26m -kube-system flannel-bwf24 2/2 Running 0 26m -kube-system flannel-ks5qb 2/2 Running 0 26m -kube-system flannel-tq2wg 2/2 Running 0 26m +kube-system calico-node-1m5bf 2/2 Running 0 26m +kube-system calico-node-7jmr1 2/2 Running 0 26m +kube-system calico-node-bknc8 2/2 Running 0 26m kube-system kube-apiserver-ramius-controller-0 1/1 Running 0 26m kube-system kube-controller-manager-ramius-controller-0 1/1 Running 0 26m kube-system kube-proxy-j4vpq 1/1 Running 0 26m @@ -220,7 +220,7 @@ Reference the DNS zone with `azurerm_dns_zone.clusters.name` and its resource gr | worker_priority | Set priority to Low to use reduced cost surplus capacity, with the tradeoff that instances can be deallocated at any time | Regular | Low | | controller_clc_snippets | Controller Container Linux Config snippets | [] | [example](/advanced/customization/#usage) | | worker_clc_snippets | Worker Container Linux Config snippets | [] | [example](/advanced/customization/#usage) | -| networking | Choice of networking provider | "flannel" | "flannel" or "calico" | +| networking | Choice of networking provider | "calico" | "flannel" or "calico" | | host_cidr | CIDR IPv4 range to assign to instances | "10.0.0.0/16" | "10.0.0.0/20" | | pod_cidr | CIDR IPv4 range to assign to Kubernetes pods | "10.2.0.0/16" | "10.22.0.0/16" | | service_cidr | CIDR IPv4 range to assign to Kubernetes services | "10.3.0.0/16" | "10.3.0.0/24" | diff --git a/docs/cl/digital-ocean.md b/docs/cl/digital-ocean.md index 1e1597782..9c0083c3d 100644 --- a/docs/cl/digital-ocean.md +++ b/docs/cl/digital-ocean.md @@ -141,9 +141,9 @@ List the pods. NAMESPACE NAME READY STATUS RESTARTS AGE kube-system coredns-1187388186-ld1j7 1/1 Running 0 11m kube-system coredns-1187388186-rdhf7 1/1 Running 0 11m -kube-system flannel-1cq1v 2/2 Running 0 11m -kube-system flannel-hq9t0 2/2 Running 0 11m -kube-system flannel-v0g9w 2/2 Running 0 11m +kube-system calico-node-1m5bf 2/2 Running 0 11m +kube-system calico-node-7jmr1 2/2 Running 0 11m +kube-system calico-node-bknc8 2/2 Running 0 11m kube-system kube-apiserver-ip-10.132.115.81 1/1 Running 0 11m kube-system kube-controller-manager-ip-10.132.115.81 1/1 Running 0 11m kube-system kube-proxy-6kxjf 1/1 Running 0 11m @@ -219,7 +219,7 @@ Digital Ocean requires the SSH public key be uploaded to your account, so you ma | image | Container Linux image for instances | "coreos-stable" | coreos-stable, coreos-beta, coreos-alpha | | controller_clc_snippets | Controller Container Linux Config snippets | [] | [example](/advanced/customization/) | | worker_clc_snippets | Worker Container Linux Config snippets | [] | [example](/advanced/customization/) | -| networking | Choice of networking provider | "flannel" | "flannel" or "calico" | +| networking | Choice of networking provider | "calico" | "flannel" or "calico" | | pod_cidr | CIDR IPv4 range to assign to Kubernetes pods | "10.2.0.0/16" | "10.22.0.0/16" | | service_cidr | CIDR IPv4 range to assign to Kubernetes services | "10.3.0.0/16" | "10.3.0.0/24" | diff --git a/docs/topics/security.md b/docs/topics/security.md index 7eb0cdec4..60934e4c2 100644 --- a/docs/topics/security.md +++ b/docs/topics/security.md @@ -12,7 +12,7 @@ Typhoon aims to be minimal and secure. We're running it ourselves after all. * Workloads run on worker nodes only, unless they tolerate the master taint * Kubernetes [Network Policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/) and Calico [NetworkPolicy](https://docs.projectcalico.org/latest/reference/calicoctl/resources/networkpolicy) support [^1] -[^1]: Requires `networking = "calico"`. Calico is the default on AWS, bare-metal, and Google Cloud. Azure and Digital Ocean are limited to `networking = "flannel"`. +[^1]: Requires `networking = "calico"`. Calico is the default on all platforms (AWS, Azure, bare-metal, DigitalOcean, and Google Cloud). **Hosts** From 0595915a195809505a3d393d35e96b1616eab17a Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Tue, 15 Oct 2019 23:25:45 -0700 Subject: [PATCH 249/523] Cleanup CHANGES notes --- CHANGES.md | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 06f2fbb1d..40910b51c 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -17,12 +17,8 @@ Notable changes between versions. * Add `controllers` and `workers` as typed lists of machine detail objects ([#566](https://github.com/poseidon/typhoon/pull/566)) * Define clusters' machines cleanly and with Terraform v0.12 type constraints (**action required**, see PR example) - * Remove `controller_names` list variable - * Remove `controller_macs` list variable - * Remove `controller_domains` list variable - * Remove `worker_names` list variable - * Remove `worker_macs` list variable - * Remove `worker_domains` list variable + * Remove `controller_names`, `controller_macs`, and `controller_domains` variables + * Remove `worker_names`, `worker_macs`, and `worker_domains` variables #### DigitalOcean @@ -31,6 +27,7 @@ Notable changes between versions. #### Addons * Update Grafana from v6.4.1 to [v6.4.2](https://github.com/grafana/grafana/releases/tag/v6.4.2) +* Change CLUO label from "app" to "name" ## v1.16.1 From 271d2f6b52fda8d86f53bd0a9c8b093459ada990 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Fri, 18 Oct 2019 00:08:39 -0700 Subject: [PATCH 250/523] Update Grafana from v6.4.2 to v6.4.3 * https://github.com/grafana/grafana/releases/tag/v6.4.3 --- CHANGES.md | 6 ++++++ addons/grafana/deployment.yaml | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index 40910b51c..c156dcbb3 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -4,6 +4,12 @@ Notable changes between versions. ## Latest +#### Addons + +* Update Grafana from v6.4.2 to v6.4.3 + +## v1.16.2 + * Kubernetes [v1.16.2](https://github.com/kubernetes/kubernetes/blob/master/CHANGELOG-1.16.md#v1162) * Update etcd from v3.4.1 to v3.4.2 ([#570](https://github.com/poseidon/typhoon/pull/570)) * Update Calico from v3.9.1 to [v3.9.2](https://docs.projectcalico.org/v3.9/release-notes/) diff --git a/addons/grafana/deployment.yaml b/addons/grafana/deployment.yaml index 9d273c71f..fbe51f503 100644 --- a/addons/grafana/deployment.yaml +++ b/addons/grafana/deployment.yaml @@ -23,7 +23,7 @@ spec: spec: containers: - name: grafana - image: docker.io/grafana/grafana:6.4.2 + image: docker.io/grafana/grafana:6.4.3 env: - name: GF_PATHS_CONFIG value: "/etc/grafana/custom.ini" From 130c97f8eb95490c2f34c6f65338bb662b5d52c2 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Fri, 18 Oct 2019 00:10:25 -0700 Subject: [PATCH 251/523] Update Prometheus from v2.13.0 to v2.13.1 * https://github.com/prometheus/prometheus/releases/tag/v2.13.1 --- CHANGES.md | 1 + addons/prometheus/deployment.yaml | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index c156dcbb3..ec11de26f 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -6,6 +6,7 @@ Notable changes between versions. #### Addons +* Update Prometheus from v2.13.0 to v2.13.1 * Update Grafana from v6.4.2 to v6.4.3 ## v1.16.2 diff --git a/addons/prometheus/deployment.yaml b/addons/prometheus/deployment.yaml index 9c36e126b..01d87d117 100644 --- a/addons/prometheus/deployment.yaml +++ b/addons/prometheus/deployment.yaml @@ -20,7 +20,7 @@ spec: serviceAccountName: prometheus containers: - name: prometheus - image: quay.io/prometheus/prometheus:v2.13.0 + image: quay.io/prometheus/prometheus:v2.13.1 args: - --web.listen-address=0.0.0.0:9090 - --config.file=/etc/prometheus/prometheus.yaml From 68da420adc230fbbd0628cbc0480efaefc2dc005 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Sat, 19 Oct 2019 17:43:47 -0700 Subject: [PATCH 252/523] Refresh Prometheus rules/alerts and Grafana dashboards * Update Prometheus rules/alerts and Grafana dashboards * Remove dashboards that were moved to node-exporter, they may be added back later if valuable * Remove kube-prometheus based rules/alerts (ClockSkew alert) --- CHANGES.md | 1 + addons/grafana/dashboards-coredns.yaml | 8 +- addons/grafana/dashboards-etcd.yaml | 8 +- addons/grafana/dashboards-k8s-nodes.yaml | 1705 ++--------- addons/grafana/dashboards-k8s-resources.yaml | 2666 ++++++------------ addons/grafana/dashboards-k8s.yaml | 14 +- addons/grafana/dashboards-nginx-ingress.yaml | 8 +- addons/grafana/dashboards-prom.yaml | 8 +- addons/prometheus/rules.yaml | 462 +-- 9 files changed, 1174 insertions(+), 3706 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index ec11de26f..7e3f4afe2 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -7,6 +7,7 @@ Notable changes between versions. #### Addons * Update Prometheus from v2.13.0 to v2.13.1 + * Refresh rules, alerts, and dashboards from upstreams * Update Grafana from v6.4.2 to v6.4.3 ## v1.16.2 diff --git a/addons/grafana/dashboards-coredns.yaml b/addons/grafana/dashboards-coredns.yaml index 44520b7d4..f6e15fe01 100644 --- a/addons/grafana/dashboards-coredns.yaml +++ b/addons/grafana/dashboards-coredns.yaml @@ -1,8 +1,4 @@ apiVersion: v1 -kind: ConfigMap -metadata: - name: grafana-dashboards-coredns - namespace: monitoring data: coredns.json: |- { @@ -1034,3 +1030,7 @@ data: "uid": "2f3f749259235f58698ea949170d3bd5", "version": 0 } +kind: ConfigMap +metadata: + name: grafana-dashboards-coredns + namespace: monitoring diff --git a/addons/grafana/dashboards-etcd.yaml b/addons/grafana/dashboards-etcd.yaml index 21339a16b..bf219e716 100644 --- a/addons/grafana/dashboards-etcd.yaml +++ b/addons/grafana/dashboards-etcd.yaml @@ -1,8 +1,4 @@ apiVersion: v1 -kind: ConfigMap -metadata: - name: grafana-dashboards-etcd - namespace: monitoring data: etcd.json: |- { @@ -1229,3 +1225,7 @@ data: "uid": "c2f4e12cdf69feb95caa41a5a1b423d9", "version": 215 } +kind: ConfigMap +metadata: + name: grafana-dashboards-etcd + namespace: monitoring diff --git a/addons/grafana/dashboards-k8s-nodes.yaml b/addons/grafana/dashboards-k8s-nodes.yaml index a0d7963cd..9702c0f62 100644 --- a/addons/grafana/dashboards-k8s-nodes.yaml +++ b/addons/grafana/dashboards-k8s-nodes.yaml @@ -1,8 +1,4 @@ apiVersion: v1 -kind: ConfigMap -metadata: - name: grafana-dashboards-k8s-nodes - namespace: monitoring data: kubelet.json: |- { @@ -2453,7 +2449,7 @@ data: "uid": "3138fa155d5915769fbded898ac09fd9", "version": 0 } - nodes.json: |- + proxy.json: |- { "__inputs": [ @@ -2480,6 +2476,90 @@ data: "collapse": false, "collapsed": false, "panels": [ + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "$datasource", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + + }, + "id": 2, + "interval": null, + "links": [ + + ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "span": 2, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum(up{job=\"kube-proxy\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": "", + "title": "Up", + "tooltip": { + "shared": false + }, + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "min" + }, { "aliasColors": { @@ -2492,7 +2572,7 @@ data: "gridPos": { }, - "id": 2, + "id": 3, "legend": { "alignAsTable": false, "avg": false, @@ -2519,37 +2599,16 @@ data: ], "spaceLength": 10, - "span": 6, + "span": 5, "stack": false, "steppedLine": false, "targets": [ { - "expr": "max(node_load1{cluster=\"$cluster\", job=\"node-exporter\", instance=\"$instance\"})", + "expr": "sum(rate(kubeproxy_sync_proxy_rules_duration_seconds_count{job=\"kube-proxy\", instance=~\"$instance\"}[5m]))", "format": "time_series", "intervalFactor": 2, - "legendFormat": "load 1m", + "legendFormat": "rate", "refId": "A" - }, - { - "expr": "max(node_load5{cluster=\"$cluster\", job=\"node-exporter\", instance=\"$instance\"})", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "load 5m", - "refId": "B" - }, - { - "expr": "max(node_load15{cluster=\"$cluster\", job=\"node-exporter\", instance=\"$instance\"})", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "load 15m", - "refId": "C" - }, - { - "expr": "count(node_cpu_seconds_total{cluster=\"$cluster\", job=\"node-exporter\", instance=\"$instance\", mode=\"user\"})", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "logical cores", - "refId": "D" } ], "thresholds": [ @@ -2557,7 +2616,7 @@ data: ], "timeFrom": null, "timeShift": null, - "title": "System load", + "title": "Rules Sync Rate", "tooltip": { "shared": false, "sort": 0, @@ -2575,19 +2634,19 @@ data: }, "yaxes": [ { - "format": "short", + "format": "ops", "label": null, "logBase": 1, "max": null, - "min": null, + "min": 0, "show": true }, { - "format": "short", + "format": "ops", "label": null, "logBase": 1, "max": null, - "min": null, + "min": 0, "show": true } ] @@ -2604,17 +2663,17 @@ data: "gridPos": { }, - "id": 3, + "id": 4, "legend": { - "alignAsTable": false, + "alignAsTable": "true", "avg": false, - "current": false, + "current": "true", "max": false, "min": false, - "rightSide": false, - "show": true, + "rightSide": "true", + "show": "true", "total": false, - "values": false + "values": "true" }, "lines": true, "linewidth": 1, @@ -2631,15 +2690,15 @@ data: ], "spaceLength": 10, - "span": 6, + "span": 5, "stack": false, "steppedLine": false, "targets": [ { - "expr": "sum by (cpu) (irate(node_cpu_seconds_total{cluster=\"$cluster\", job=\"node-exporter\", mode!=\"idle\", instance=\"$instance\"}[5m]))", + "expr": "histogram_quantile(0.99,rate(kubeproxy_sync_proxy_rules_duration_seconds_bucket{job=\"kube-proxy\", instance=~\"$instance\"}[5m]))", "format": "time_series", "intervalFactor": 2, - "legendFormat": "{{cpu}}", + "legendFormat": "{{instance}}", "refId": "A" } ], @@ -2648,7 +2707,7 @@ data: ], "timeFrom": null, "timeShift": null, - "title": "Usage Per Core", + "title": "Rule Sync Latency 99th Quantile", "tooltip": { "shared": false, "sort": 0, @@ -2666,19 +2725,19 @@ data: }, "yaxes": [ { - "format": "percentunit", + "format": "s", "label": null, "logBase": 1, "max": null, - "min": null, + "min": 0, "show": true }, { - "format": "percentunit", + "format": "s", "label": null, "logBase": 1, "max": null, - "min": null, + "min": 0, "show": true } ] @@ -2708,17 +2767,17 @@ data: "gridPos": { }, - "id": 4, + "id": 5, "legend": { - "alignAsTable": "true", - "avg": "true", - "current": "true", - "max": "false", - "min": "false", - "rightSide": "true", - "show": "true", - "total": "false", - "values": "true" + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": false }, "lines": true, "linewidth": 1, @@ -2735,15 +2794,15 @@ data: ], "spaceLength": 10, - "span": 9, + "span": 6, "stack": false, "steppedLine": false, "targets": [ { - "expr": "max (sum by (cpu) (irate(node_cpu_seconds_total{cluster=\"$cluster\", job=\"node-exporter\", mode!=\"idle\", instance=\"$instance\"}[2m])) ) * 100\n", + "expr": "sum(rate(kubeproxy_network_programming_duration_seconds_count{job=\"kube-proxy\", instance=~\"$instance\"}[5m]))", "format": "time_series", - "intervalFactor": 10, - "legendFormat": "{{ cpu }}", + "intervalFactor": 2, + "legendFormat": "rate", "refId": "A" } ], @@ -2752,7 +2811,7 @@ data: ], "timeFrom": null, "timeShift": null, - "title": "CPU Utilization", + "title": "Network Programming Rate", "tooltip": { "shared": false, "sort": 0, @@ -2770,1504 +2829,80 @@ data: }, "yaxes": [ { - "format": "percent", + "format": "ops", "label": null, "logBase": 1, - "max": 100, + "max": null, "min": 0, "show": true }, { - "format": "percent", + "format": "ops", "label": null, "logBase": 1, - "max": 100, + "max": null, "min": 0, "show": true } ] }, { - "cacheTimeout": null, - "colorBackground": false, - "colorValue": false, - "colors": [ - "rgba(50, 172, 45, 0.97)", - "rgba(237, 129, 40, 0.89)", - "rgba(245, 54, 54, 0.9)" - ], - "datasource": "$datasource", - "format": "percent", - "gauge": { - "maxValue": 100, - "minValue": 0, - "show": true, - "thresholdLabels": false, - "thresholdMarkers": true + "aliasColors": { + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, "gridPos": { }, - "id": 5, - "interval": null, + "id": 6, + "legend": { + "alignAsTable": "true", + "avg": false, + "current": "true", + "max": false, + "min": false, + "rightSide": "true", + "show": "true", + "total": false, + "values": "true" + }, + "lines": true, + "linewidth": 1, "links": [ ], - "mappingType": 1, - "mappingTypes": [ - { - "name": "value to text", - "value": 1 - }, - { - "name": "range to text", - "value": 2 - } - ], - "maxDataPoints": 100, - "nullPointMode": "connected", - "nullText": null, - "postfix": "", - "postfixFontSize": "50%", - "prefix": "", - "prefixFontSize": "50%", - "rangeMaps": [ - { - "from": "null", - "text": "N/A", - "to": "null" - } + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + ], - "span": 3, - "sparkline": { - "fillColor": "rgba(31, 118, 189, 0.18)", - "full": false, - "lineColor": "rgb(31, 120, 193)", - "show": false - }, - "tableColumn": "", + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, "targets": [ { - "expr": "avg(sum by (cpu) (irate(node_cpu_seconds_total{cluster=\"$cluster\", job=\"node-exporter\", mode!=\"idle\", instance=\"$instance\"}[2m]))) * 100\n", + "expr": "histogram_quantile(0.99, sum(rate(kubeproxy_network_programming_duration_seconds_bucket{job=\"kube-proxy\", instance=~\"$instance\"}[5m])) by (instance, le))", "format": "time_series", "intervalFactor": 2, - "legendFormat": "", + "legendFormat": "{{instance}}", "refId": "A" } ], - "thresholds": "80, 90", - "title": "CPU Usage", - "tooltip": { - "shared": false - }, - "type": "singlestat", - "valueFontSize": "80%", - "valueMaps": [ - { - "op": "=", - "text": "N/A", - "value": "null" - } - ], - "valueName": "current" - } - ], - "repeat": null, - "repeatIteration": null, - "repeatRowId": null, - "showTitle": false, - "title": "Dashboard Row", - "titleSize": "h6", - "type": "row" - }, - { - "collapse": false, - "collapsed": false, - "panels": [ - { - "aliasColors": { - - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 1, - "gridPos": { - - }, - "id": 6, - "legend": { - "alignAsTable": false, - "avg": false, - "current": false, - "max": false, - "min": false, - "rightSide": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [ - - ], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "repeat": null, - "seriesOverrides": [ - - ], - "spaceLength": 10, - "span": 9, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "max(\n node_memory_MemTotal_bytes{cluster=\"$cluster\", job=\"node-exporter\", instance=\"$instance\"}\n - node_memory_MemFree_bytes{cluster=\"$cluster\", job=\"node-exporter\", instance=\"$instance\"}\n - node_memory_Buffers_bytes{cluster=\"$cluster\", job=\"node-exporter\", instance=\"$instance\"}\n - node_memory_Cached_bytes{cluster=\"$cluster\", job=\"node-exporter\", instance=\"$instance\"}\n)\n", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "memory used", - "refId": "A" - }, - { - "expr": "max(node_memory_Buffers_bytes{cluster=\"$cluster\", job=\"node-exporter\", instance=\"$instance\"})", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "memory buffers", - "refId": "B" - }, - { - "expr": "max(node_memory_Cached_bytes{cluster=\"$cluster\", job=\"node-exporter\", instance=\"$instance\"})", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "memory cached", - "refId": "C" - }, - { - "expr": "max(node_memory_MemFree_bytes{cluster=\"$cluster\", job=\"node-exporter\", instance=\"$instance\"})", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "memory free", - "refId": "D" - } - ], - "thresholds": [ - - ], - "timeFrom": null, - "timeShift": null, - "title": "Memory Usage", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ - - ] - }, - "yaxes": [ - { - "format": "bytes", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "bytes", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] - }, - { - "cacheTimeout": null, - "colorBackground": false, - "colorValue": false, - "colors": [ - "rgba(50, 172, 45, 0.97)", - "rgba(237, 129, 40, 0.89)", - "rgba(245, 54, 54, 0.9)" - ], - "datasource": "$datasource", - "format": "percent", - "gauge": { - "maxValue": 100, - "minValue": 0, - "show": true, - "thresholdLabels": false, - "thresholdMarkers": true - }, - "gridPos": { - - }, - "id": 7, - "interval": null, - "links": [ - - ], - "mappingType": 1, - "mappingTypes": [ - { - "name": "value to text", - "value": 1 - }, - { - "name": "range to text", - "value": 2 - } - ], - "maxDataPoints": 100, - "nullPointMode": "connected", - "nullText": null, - "postfix": "", - "postfixFontSize": "50%", - "prefix": "", - "prefixFontSize": "50%", - "rangeMaps": [ - { - "from": "null", - "text": "N/A", - "to": "null" - } - ], - "span": 3, - "sparkline": { - "fillColor": "rgba(31, 118, 189, 0.18)", - "full": false, - "lineColor": "rgb(31, 120, 193)", - "show": false - }, - "tableColumn": "", - "targets": [ - { - "expr": "max(\n (\n (\n node_memory_MemTotal_bytes{cluster=\"$cluster\", job=\"node-exporter\", instance=\"$instance\"}\n - node_memory_MemFree_bytes{cluster=\"$cluster\", job=\"node-exporter\", instance=\"$instance\"}\n - node_memory_Buffers_bytes{cluster=\"$cluster\", job=\"node-exporter\", instance=\"$instance\"}\n - node_memory_Cached_bytes{cluster=\"$cluster\", job=\"node-exporter\", instance=\"$instance\"}\n )\n / node_memory_MemTotal_bytes{cluster=\"$cluster\", job=\"node-exporter\", instance=\"$instance\"}\n ) * 100)\n", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "", - "refId": "A" - } - ], - "thresholds": "80, 90", - "title": "Memory Usage", - "tooltip": { - "shared": false - }, - "type": "singlestat", - "valueFontSize": "80%", - "valueMaps": [ - { - "op": "=", - "text": "N/A", - "value": "null" - } - ], - "valueName": "current" - } - ], - "repeat": null, - "repeatIteration": null, - "repeatRowId": null, - "showTitle": false, - "title": "Dashboard Row", - "titleSize": "h6", - "type": "row" - }, - { - "collapse": false, - "collapsed": false, - "panels": [ - { - "aliasColors": { - - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 1, - "gridPos": { - - }, - "id": 8, - "legend": { - "alignAsTable": false, - "avg": false, - "current": false, - "max": false, - "min": false, - "rightSide": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [ - - ], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "repeat": null, - "seriesOverrides": [ - { - "alias": "read", - "yaxis": 1 - }, - { - "alias": "io time", - "yaxis": 2 - } - ], - "spaceLength": 10, - "span": 6, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "max(rate(node_disk_read_bytes_total{cluster=\"$cluster\", job=\"node-exporter\", instance=\"$instance\"}[2m]))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "read", - "refId": "A" - }, - { - "expr": "max(rate(node_disk_written_bytes_total{cluster=\"$cluster\", job=\"node-exporter\", instance=\"$instance\"}[2m]))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "written", - "refId": "B" - }, - { - "expr": "max(rate(node_disk_io_time_seconds_total{cluster=\"$cluster\", job=\"node-exporter\", instance=\"$instance\"}[2m]))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "io time", - "refId": "C" - } - ], - "thresholds": [ - - ], - "timeFrom": null, - "timeShift": null, - "title": "Disk I/O", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ - - ] - }, - "yaxes": [ - { - "format": "bytes", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "ms", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] - }, - { - "aliasColors": { - - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 1, - "gridPos": { - - }, - "id": 9, - "legend": { - "alignAsTable": false, - "avg": false, - "current": false, - "max": false, - "min": false, - "rightSide": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [ - - ], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "repeat": null, - "seriesOverrides": [ - - ], - "spaceLength": 10, - "span": 6, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "node:node_filesystem_usage:{cluster=\"$cluster\", instance=\"$instance\"}", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{device}} disk used", - "refId": "A" - }, - { - "expr": "node:node_filesystem_usage:{cluster=\"$cluster\", instance=\"$instance\"}", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{device}} disk free", - "refId": "B" - } - ], - "thresholds": [ - - ], - "timeFrom": null, - "timeShift": null, - "title": "Disk Space Usage", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ - - ] - }, - "yaxes": [ - { - "format": "percentunit", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "percentunit", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] - } - ], - "repeat": null, - "repeatIteration": null, - "repeatRowId": null, - "showTitle": false, - "title": "Dashboard Row", - "titleSize": "h6", - "type": "row" - }, - { - "collapse": false, - "collapsed": false, - "panels": [ - { - "aliasColors": { - - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 1, - "gridPos": { - - }, - "id": 10, - "legend": { - "alignAsTable": false, - "avg": false, - "current": false, - "max": false, - "min": false, - "rightSide": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [ - - ], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "repeat": null, - "seriesOverrides": [ - - ], - "spaceLength": 10, - "span": 6, - "stack": true, - "steppedLine": false, - "targets": [ - { - "expr": "rate(node_network_receive_bytes_total{cluster=\"$cluster\", job=\"node-exporter\", instance=\"$instance\", device!~\"lo\"}[5m])", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{device}}", - "refId": "A" - } - ], - "thresholds": [ - - ], - "timeFrom": null, - "timeShift": null, - "title": "Network Received", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ - - ] - }, - "yaxes": [ - { - "format": "bytes", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "bytes", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] - }, - { - "aliasColors": { - - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 1, - "gridPos": { - - }, - "id": 11, - "legend": { - "alignAsTable": false, - "avg": false, - "current": false, - "max": false, - "min": false, - "rightSide": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [ - - ], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "repeat": null, - "seriesOverrides": [ - - ], - "spaceLength": 10, - "span": 6, - "stack": true, - "steppedLine": false, - "targets": [ - { - "expr": "rate(node_network_transmit_bytes_total{cluster=\"$cluster\", job=\"node-exporter\", instance=\"$instance\", device!~\"lo\"}[5m])", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{device}}", - "refId": "A" - } - ], - "thresholds": [ - - ], - "timeFrom": null, - "timeShift": null, - "title": "Network Transmitted", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ - - ] - }, - "yaxes": [ - { - "format": "bytes", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "bytes", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] - } - ], - "repeat": null, - "repeatIteration": null, - "repeatRowId": null, - "showTitle": false, - "title": "Dashboard Row", - "titleSize": "h6", - "type": "row" - }, - { - "collapse": false, - "collapsed": false, - "panels": [ - { - "aliasColors": { - - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 1, - "gridPos": { - - }, - "id": 12, - "legend": { - "alignAsTable": false, - "avg": false, - "current": false, - "max": false, - "min": false, - "rightSide": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [ - - ], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "repeat": null, - "seriesOverrides": [ - - ], - "spaceLength": 10, - "span": 9, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "max(\n node_filesystem_files{cluster=\"$cluster\", job=\"node-exporter\", instance=\"$instance\"}\n - node_filesystem_files_free{cluster=\"$cluster\", job=\"node-exporter\", instance=\"$instance\"}\n)\n", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "inodes used", - "refId": "A" - }, - { - "expr": "max(node_filesystem_files_free{cluster=\"$cluster\", job=\"node-exporter\", instance=\"$instance\"})", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "inodes free", - "refId": "B" - } - ], - "thresholds": [ - - ], - "timeFrom": null, - "timeShift": null, - "title": "Inodes Usage", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ - - ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] - }, - { - "cacheTimeout": null, - "colorBackground": false, - "colorValue": false, - "colors": [ - "rgba(50, 172, 45, 0.97)", - "rgba(237, 129, 40, 0.89)", - "rgba(245, 54, 54, 0.9)" - ], - "datasource": "$datasource", - "format": "percent", - "gauge": { - "maxValue": 100, - "minValue": 0, - "show": true, - "thresholdLabels": false, - "thresholdMarkers": true - }, - "gridPos": { - - }, - "id": 13, - "interval": null, - "links": [ - - ], - "mappingType": 1, - "mappingTypes": [ - { - "name": "value to text", - "value": 1 - }, - { - "name": "range to text", - "value": 2 - } - ], - "maxDataPoints": 100, - "nullPointMode": "connected", - "nullText": null, - "postfix": "", - "postfixFontSize": "50%", - "prefix": "", - "prefixFontSize": "50%", - "rangeMaps": [ - { - "from": "null", - "text": "N/A", - "to": "null" - } - ], - "span": 3, - "sparkline": { - "fillColor": "rgba(31, 118, 189, 0.18)", - "full": false, - "lineColor": "rgb(31, 120, 193)", - "show": false - }, - "tableColumn": "", - "targets": [ - { - "expr": "max(\n (\n (\n node_filesystem_files{cluster=\"$cluster\", job=\"node-exporter\", instance=\"$instance\"}\n - node_filesystem_files_free{cluster=\"$cluster\", job=\"node-exporter\", instance=\"$instance\"}\n )\n / node_filesystem_files{cluster=\"$cluster\", job=\"node-exporter\", instance=\"$instance\"}\n ) * 100)\n", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "", - "refId": "A" - } - ], - "thresholds": "80, 90", - "title": "Inodes Usage", - "tooltip": { - "shared": false - }, - "type": "singlestat", - "valueFontSize": "80%", - "valueMaps": [ - { - "op": "=", - "text": "N/A", - "value": "null" - } - ], - "valueName": "current" - } - ], - "repeat": null, - "repeatIteration": null, - "repeatRowId": null, - "showTitle": false, - "title": "Dashboard Row", - "titleSize": "h6", - "type": "row" - } - ], - "schemaVersion": 14, - "style": "dark", - "tags": [ - "kubernetes-mixin" - ], - "templating": { - "list": [ - { - "current": { - "text": "Prometheus", - "value": "Prometheus" - }, - "hide": 0, - "label": null, - "name": "datasource", - "options": [ - - ], - "query": "prometheus", - "refresh": 1, - "regex": "", - "type": "datasource" - }, - { - "allValue": null, - "current": { - - }, - "datasource": "$datasource", - "hide": 2, - "includeAll": false, - "label": "cluster", - "multi": false, - "name": "cluster", - "options": [ - - ], - "query": "label_values(kube_pod_info, cluster)", - "refresh": 2, - "regex": "", - "sort": 0, - "tagValuesQuery": "", - "tags": [ - - ], - "tagsQuery": "", - "type": "query", - "useTags": false - }, - { - "allValue": null, - "current": { - - }, - "datasource": "$datasource", - "hide": 0, - "includeAll": false, - "label": null, - "multi": false, - "name": "instance", - "options": [ - - ], - "query": "label_values(node_boot_time_seconds{cluster=\"$cluster\", job=\"node-exporter\"}, instance)", - "refresh": 2, - "regex": "", - "sort": 0, - "tagValuesQuery": "", - "tags": [ - - ], - "tagsQuery": "", - "type": "query", - "useTags": false - } - ] - }, - "time": { - "from": "now-1h", - "to": "now" - }, - "timepicker": { - "refresh_intervals": [ - "5s", - "10s", - "30s", - "1m", - "5m", - "15m", - "30m", - "1h", - "2h", - "1d" - ], - "time_options": [ - "5m", - "15m", - "1h", - "6h", - "12h", - "24h", - "2d", - "7d", - "30d" - ] - }, - "timezone": "", - "title": "Kubernetes / Nodes", - "uid": "fa49a4706d07a042595b664c87fb33ea", - "version": 0 - } - proxy.json: |- - { - "__inputs": [ - - ], - "__requires": [ - - ], - "annotations": { - "list": [ - - ] - }, - "editable": false, - "gnetId": null, - "graphTooltip": 0, - "hideControls": false, - "id": null, - "links": [ - - ], - "refresh": "", - "rows": [ - { - "collapse": false, - "collapsed": false, - "panels": [ - { - "cacheTimeout": null, - "colorBackground": false, - "colorValue": false, - "colors": [ - "#299c46", - "rgba(237, 129, 40, 0.89)", - "#d44a3a" - ], - "datasource": "$datasource", - "format": "none", - "gauge": { - "maxValue": 100, - "minValue": 0, - "show": false, - "thresholdLabels": false, - "thresholdMarkers": true - }, - "gridPos": { - - }, - "id": 2, - "interval": null, - "links": [ - - ], - "mappingType": 1, - "mappingTypes": [ - { - "name": "value to text", - "value": 1 - }, - { - "name": "range to text", - "value": 2 - } - ], - "maxDataPoints": 100, - "nullPointMode": "connected", - "nullText": null, - "postfix": "", - "postfixFontSize": "50%", - "prefix": "", - "prefixFontSize": "50%", - "rangeMaps": [ - { - "from": "null", - "text": "N/A", - "to": "null" - } - ], - "span": 2, - "sparkline": { - "fillColor": "rgba(31, 118, 189, 0.18)", - "full": false, - "lineColor": "rgb(31, 120, 193)", - "show": false - }, - "tableColumn": "", - "targets": [ - { - "expr": "sum(up{job=\"kube-proxy\"})", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "", - "refId": "A" - } - ], - "thresholds": "", - "title": "Up", - "tooltip": { - "shared": false - }, - "type": "singlestat", - "valueFontSize": "80%", - "valueMaps": [ - { - "op": "=", - "text": "N/A", - "value": "null" - } - ], - "valueName": "min" - }, - { - "aliasColors": { - - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 1, - "gridPos": { - - }, - "id": 3, - "legend": { - "alignAsTable": false, - "avg": false, - "current": false, - "max": false, - "min": false, - "rightSide": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [ - - ], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "repeat": null, - "seriesOverrides": [ - - ], - "spaceLength": 10, - "span": 5, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum(rate(kubeproxy_sync_proxy_rules_duration_seconds_count{job=\"kube-proxy\", instance=~\"$instance\"}[5m]))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "rate", - "refId": "A" - } - ], - "thresholds": [ - - ], - "timeFrom": null, - "timeShift": null, - "title": "Rules Sync Rate", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ - - ] - }, - "yaxes": [ - { - "format": "ops", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "ops", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - } - ] - }, - { - "aliasColors": { - - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 1, - "gridPos": { - - }, - "id": 4, - "legend": { - "alignAsTable": "true", - "avg": false, - "current": "true", - "max": false, - "min": false, - "rightSide": "true", - "show": "true", - "total": false, - "values": "true" - }, - "lines": true, - "linewidth": 1, - "links": [ - - ], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "repeat": null, - "seriesOverrides": [ - - ], - "spaceLength": 10, - "span": 5, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "histogram_quantile(0.99,rate(kubeproxy_sync_proxy_rules_duration_seconds_bucket{job=\"kube-proxy\", instance=~\"$instance\"}[5m]))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{instance}}", - "refId": "A" - } - ], - "thresholds": [ - - ], - "timeFrom": null, - "timeShift": null, - "title": "Rule Sync Latency 99th Quantile", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ - - ] - }, - "yaxes": [ - { - "format": "s", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "s", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - } - ] - } - ], - "repeat": null, - "repeatIteration": null, - "repeatRowId": null, - "showTitle": false, - "title": "Dashboard Row", - "titleSize": "h6", - "type": "row" - }, - { - "collapse": false, - "collapsed": false, - "panels": [ - { - "aliasColors": { - - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 1, - "gridPos": { - - }, - "id": 5, - "legend": { - "alignAsTable": false, - "avg": false, - "current": false, - "max": false, - "min": false, - "rightSide": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [ - - ], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "repeat": null, - "seriesOverrides": [ - - ], - "spaceLength": 10, - "span": 6, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum(rate(kubeproxy_network_programming_duration_seconds_count{job=\"kube-proxy\", instance=~\"$instance\"}[5m]))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "rate", - "refId": "A" - } - ], - "thresholds": [ - - ], - "timeFrom": null, - "timeShift": null, - "title": "Network Programming Rate", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ - - ] - }, - "yaxes": [ - { - "format": "ops", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "ops", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - } - ] - }, - { - "aliasColors": { - - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 1, - "gridPos": { - - }, - "id": 6, - "legend": { - "alignAsTable": "true", - "avg": false, - "current": "true", - "max": false, - "min": false, - "rightSide": "true", - "show": "true", - "total": false, - "values": "true" - }, - "lines": true, - "linewidth": 1, - "links": [ - - ], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "repeat": null, - "seriesOverrides": [ - - ], - "spaceLength": 10, - "span": 6, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "histogram_quantile(0.99, sum(rate(kubeproxy_network_programming_duration_seconds_bucket{job=\"kube-proxy\", instance=~\"$instance\"}[5m])) by (instance, le))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{instance}}", - "refId": "A" - } - ], - "thresholds": [ - - ], - "timeFrom": null, - "timeShift": null, - "title": "Network Programming Latency 99th Quantile", + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Network Programming Latency 99th Quantile", "tooltip": { "shared": false, "sort": 0, @@ -5003,3 +3638,7 @@ data: "uid": "632e265de029684c40b21cb76bca4f94", "version": 0 } +kind: ConfigMap +metadata: + name: grafana-dashboards-k8s-nodes + namespace: monitoring diff --git a/addons/grafana/dashboards-k8s-resources.yaml b/addons/grafana/dashboards-k8s-resources.yaml index 98a17712c..c8c936c2c 100644 --- a/addons/grafana/dashboards-k8s-resources.yaml +++ b/addons/grafana/dashboards-k8s-resources.yaml @@ -1,10 +1,6 @@ apiVersion: v1 -kind: ConfigMap -metadata: - name: grafana-dashboards-k8s-resources - namespace: monitoring data: - k8s-cluster-rsrc-use.json: |- + k8s-resources-cluster.json: |- { "annotations": { "list": [ @@ -22,7 +18,7 @@ data: "rows": [ { "collapse": false, - "height": "250px", + "height": "100px", "panels": [ { "aliasColors": { @@ -32,7 +28,8 @@ data: "dashLength": 10, "dashes": false, "datasource": "$datasource", - "fill": 10, + "fill": 1, + "format": "percentunit", "id": 1, "legend": { "avg": false, @@ -44,7 +41,7 @@ data: "values": false }, "lines": true, - "linewidth": 0, + "linewidth": 1, "links": [ ], @@ -57,22 +54,19 @@ data: ], "spaceLength": 10, - "span": 6, - "stack": true, + "span": 2, + "stack": false, "steppedLine": false, "targets": [ { - "expr": "node:cluster_cpu_utilisation:ratio{cluster=\"$cluster\"}", + "expr": "1 - avg(rate(node_cpu_seconds_total{mode=\"idle\", cluster=\"$cluster\"}[1m]))", "format": "time_series", + "instant": true, "intervalFactor": 2, - "legendFormat": "{{node}}", - "legendLink": "/d/4ac4f123aae0ff6dbaf4f4f66120033b/k8s-node-rsrc-use", - "step": 10 + "refId": "A" } ], - "thresholds": [ - - ], + "thresholds": "70,80", "timeFrom": null, "timeShift": null, "title": "CPU Utilisation", @@ -81,7 +75,7 @@ data: "sort": 0, "value_type": "individual" }, - "type": "graph", + "type": "singlestat", "xaxis": { "buckets": null, "mode": "time", @@ -93,10 +87,10 @@ data: }, "yaxes": [ { - "format": "percentunit", + "format": "short", "label": null, "logBase": 1, - "max": 1, + "max": null, "min": 0, "show": true }, @@ -118,7 +112,8 @@ data: "dashLength": 10, "dashes": false, "datasource": "$datasource", - "fill": 10, + "fill": 1, + "format": "percentunit", "id": 2, "legend": { "avg": false, @@ -130,7 +125,7 @@ data: "values": false }, "lines": true, - "linewidth": 0, + "linewidth": 1, "links": [ ], @@ -143,31 +138,28 @@ data: ], "spaceLength": 10, - "span": 6, - "stack": true, + "span": 2, + "stack": false, "steppedLine": false, "targets": [ { - "expr": "node:node_cpu_saturation_load1:{cluster=\"$cluster\"} / scalar(sum(min(kube_pod_info{cluster=\"$cluster\"}) by (node)))", + "expr": "sum(kube_pod_container_resource_requests_cpu_cores{cluster=\"$cluster\"}) / sum(kube_node_status_allocatable_cpu_cores{cluster=\"$cluster\"})", "format": "time_series", + "instant": true, "intervalFactor": 2, - "legendFormat": "{{node}}", - "legendLink": "/d/4ac4f123aae0ff6dbaf4f4f66120033b/k8s-node-rsrc-use", - "step": 10 + "refId": "A" } ], - "thresholds": [ - - ], + "thresholds": "70,80", "timeFrom": null, "timeShift": null, - "title": "CPU Saturation (Load1)", + "title": "CPU Requests Commitment", "tooltip": { "shared": false, "sort": 0, "value_type": "individual" }, - "type": "graph", + "type": "singlestat", "xaxis": { "buckets": null, "mode": "time", @@ -179,10 +171,10 @@ data: }, "yaxes": [ { - "format": "percentunit", + "format": "short", "label": null, "logBase": 1, - "max": 1, + "max": null, "min": 0, "show": true }, @@ -195,19 +187,7 @@ data: "show": false } ] - } - ], - "repeat": null, - "repeatIteration": null, - "repeatRowId": null, - "showTitle": true, - "title": "CPU", - "titleSize": "h6" - }, - { - "collapse": false, - "height": "250px", - "panels": [ + }, { "aliasColors": { @@ -216,7 +196,8 @@ data: "dashLength": 10, "dashes": false, "datasource": "$datasource", - "fill": 10, + "fill": 1, + "format": "percentunit", "id": 3, "legend": { "avg": false, @@ -228,7 +209,7 @@ data: "values": false }, "lines": true, - "linewidth": 0, + "linewidth": 1, "links": [ ], @@ -241,31 +222,28 @@ data: ], "spaceLength": 10, - "span": 6, - "stack": true, + "span": 2, + "stack": false, "steppedLine": false, "targets": [ { - "expr": "node:cluster_memory_utilisation:ratio{cluster=\"$cluster\"}", + "expr": "sum(kube_pod_container_resource_limits_cpu_cores{cluster=\"$cluster\"}) / sum(kube_node_status_allocatable_cpu_cores{cluster=\"$cluster\"})", "format": "time_series", + "instant": true, "intervalFactor": 2, - "legendFormat": "{{node}}", - "legendLink": "/d/4ac4f123aae0ff6dbaf4f4f66120033b/k8s-node-rsrc-use", - "step": 10 + "refId": "A" } ], - "thresholds": [ - - ], + "thresholds": "70,80", "timeFrom": null, "timeShift": null, - "title": "Memory Utilisation", + "title": "CPU Limits Commitment", "tooltip": { "shared": false, "sort": 0, "value_type": "individual" }, - "type": "graph", + "type": "singlestat", "xaxis": { "buckets": null, "mode": "time", @@ -277,10 +255,10 @@ data: }, "yaxes": [ { - "format": "percentunit", + "format": "short", "label": null, "logBase": 1, - "max": 1, + "max": null, "min": 0, "show": true }, @@ -302,7 +280,8 @@ data: "dashLength": 10, "dashes": false, "datasource": "$datasource", - "fill": 10, + "fill": 1, + "format": "percentunit", "id": 4, "legend": { "avg": false, @@ -314,7 +293,7 @@ data: "values": false }, "lines": true, - "linewidth": 0, + "linewidth": 1, "links": [ ], @@ -327,31 +306,28 @@ data: ], "spaceLength": 10, - "span": 6, - "stack": true, + "span": 2, + "stack": false, "steppedLine": false, "targets": [ { - "expr": "node:node_memory_swap_io_bytes:sum_rate{cluster=\"$cluster\"}", + "expr": "1 - sum(:node_memory_MemFreeCachedBuffers_bytes:sum{cluster=\"$cluster\"}) / sum(kube_node_status_allocatable_memory_bytes{cluster=\"$cluster\"})", "format": "time_series", + "instant": true, "intervalFactor": 2, - "legendFormat": "{{node}}", - "legendLink": "/d/4ac4f123aae0ff6dbaf4f4f66120033b/k8s-node-rsrc-use", - "step": 10 + "refId": "A" } ], - "thresholds": [ - - ], + "thresholds": "70,80", "timeFrom": null, "timeShift": null, - "title": "Memory Saturation (Swap I/O)", + "title": "Memory Utilisation", "tooltip": { "shared": false, "sort": 0, "value_type": "individual" }, - "type": "graph", + "type": "singlestat", "xaxis": { "buckets": null, "mode": "time", @@ -363,7 +339,7 @@ data: }, "yaxes": [ { - "format": "Bps", + "format": "short", "label": null, "logBase": 1, "max": null, @@ -379,19 +355,7 @@ data: "show": false } ] - } - ], - "repeat": null, - "repeatIteration": null, - "repeatRowId": null, - "showTitle": true, - "title": "Memory", - "titleSize": "h6" - }, - { - "collapse": false, - "height": "250px", - "panels": [ + }, { "aliasColors": { @@ -400,7 +364,8 @@ data: "dashLength": 10, "dashes": false, "datasource": "$datasource", - "fill": 10, + "fill": 1, + "format": "percentunit", "id": 5, "legend": { "avg": false, @@ -412,7 +377,7 @@ data: "values": false }, "lines": true, - "linewidth": 0, + "linewidth": 1, "links": [ ], @@ -425,31 +390,28 @@ data: ], "spaceLength": 10, - "span": 6, - "stack": true, + "span": 2, + "stack": false, "steppedLine": false, "targets": [ { - "expr": "node:node_disk_utilisation:avg_irate{cluster=\"$cluster\"} / scalar(:kube_pod_info_node_count:{cluster=\"$cluster\"})", + "expr": "sum(kube_pod_container_resource_requests_memory_bytes{cluster=\"$cluster\"}) / sum(kube_node_status_allocatable_memory_bytes{cluster=\"$cluster\"})", "format": "time_series", + "instant": true, "intervalFactor": 2, - "legendFormat": "{{node}}", - "legendLink": "/d/4ac4f123aae0ff6dbaf4f4f66120033b/k8s-node-rsrc-use", - "step": 10 + "refId": "A" } ], - "thresholds": [ - - ], + "thresholds": "70,80", "timeFrom": null, "timeShift": null, - "title": "Disk IO Utilisation", + "title": "Memory Requests Commitment", "tooltip": { "shared": false, "sort": 0, "value_type": "individual" }, - "type": "graph", + "type": "singlestat", "xaxis": { "buckets": null, "mode": "time", @@ -461,10 +423,10 @@ data: }, "yaxes": [ { - "format": "percentunit", + "format": "short", "label": null, "logBase": 1, - "max": 1, + "max": null, "min": 0, "show": true }, @@ -486,7 +448,8 @@ data: "dashLength": 10, "dashes": false, "datasource": "$datasource", - "fill": 10, + "fill": 1, + "format": "percentunit", "id": 6, "legend": { "avg": false, @@ -498,7 +461,7 @@ data: "values": false }, "lines": true, - "linewidth": 0, + "linewidth": 1, "links": [ ], @@ -511,31 +474,28 @@ data: ], "spaceLength": 10, - "span": 6, - "stack": true, + "span": 2, + "stack": false, "steppedLine": false, "targets": [ { - "expr": "node:node_disk_saturation:avg_irate{cluster=\"$cluster\"} / scalar(:kube_pod_info_node_count:{cluster=\"$cluster\"})", + "expr": "sum(kube_pod_container_resource_limits_memory_bytes{cluster=\"$cluster\"}) / sum(kube_node_status_allocatable_memory_bytes{cluster=\"$cluster\"})", "format": "time_series", + "instant": true, "intervalFactor": 2, - "legendFormat": "{{node}}", - "legendLink": "/d/4ac4f123aae0ff6dbaf4f4f66120033b/k8s-node-rsrc-use", - "step": 10 + "refId": "A" } ], - "thresholds": [ - - ], + "thresholds": "70,80", "timeFrom": null, "timeShift": null, - "title": "Disk IO Saturation", + "title": "Memory Limits Commitment", "tooltip": { "shared": false, "sort": 0, "value_type": "individual" }, - "type": "graph", + "type": "singlestat", "xaxis": { "buckets": null, "mode": "time", @@ -547,10 +507,10 @@ data: }, "yaxes": [ { - "format": "percentunit", + "format": "short", "label": null, "logBase": 1, - "max": 1, + "max": null, "min": 0, "show": true }, @@ -568,8 +528,8 @@ data: "repeat": null, "repeatIteration": null, "repeatRowId": null, - "showTitle": true, - "title": "Disk", + "showTitle": false, + "title": "Headlines", "titleSize": "h6" }, { @@ -609,16 +569,16 @@ data: ], "spaceLength": 10, - "span": 6, + "span": 12, "stack": true, "steppedLine": false, "targets": [ { - "expr": "node:node_net_utilisation:sum_irate{cluster=\"$cluster\"}", + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\"}) by (namespace)", "format": "time_series", "intervalFactor": 2, - "legendFormat": "{{node}}", - "legendLink": "/d/4ac4f123aae0ff6dbaf4f4f66120033b/k8s-node-rsrc-use", + "legendFormat": "{{namespace}}", + "legendLink": null, "step": 10 } ], @@ -627,7 +587,7 @@ data: ], "timeFrom": null, "timeShift": null, - "title": "Net Utilisation (Transmitted)", + "title": "CPU Usage", "tooltip": { "shared": false, "sort": 0, @@ -644,97 +604,11 @@ data: ] }, "yaxes": [ - { - "format": "Bps", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, { "format": "short", "label": null, "logBase": 1, "max": null, - "min": null, - "show": false - } - ] - }, - { - "aliasColors": { - - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 10, - "id": 8, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 0, - "links": [ - - ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - - ], - "spaceLength": 10, - "span": 6, - "stack": true, - "steppedLine": false, - "targets": [ - { - "expr": "node:node_net_saturation:sum_irate{cluster=\"$cluster\"}", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{node}}", - "legendLink": "/d/4ac4f123aae0ff6dbaf4f4f66120033b/k8s-node-rsrc-use", - "step": 10 - } - ], - "thresholds": [ - - ], - "timeFrom": null, - "timeShift": null, - "title": "Net Saturation (Dropped)", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ - - ] - }, - "yaxes": [ - { - "format": "Bps", - "label": null, - "logBase": 1, - "max": null, "min": 0, "show": true }, @@ -753,7 +627,7 @@ data: "repeatIteration": null, "repeatRowId": null, "showTitle": true, - "title": "Network", + "title": "CPU", "titleSize": "h6" }, { @@ -768,8 +642,8 @@ data: "dashLength": 10, "dashes": false, "datasource": "$datasource", - "fill": 10, - "id": 9, + "fill": 1, + "id": 8, "legend": { "avg": false, "current": false, @@ -780,7 +654,7 @@ data: "values": false }, "lines": true, - "linewidth": 0, + "linewidth": 1, "links": [ ], @@ -794,1361 +668,253 @@ data: ], "spaceLength": 10, "span": 12, - "stack": true, + "stack": false, "steppedLine": false, - "targets": [ - { - "expr": "sum(max(node_filesystem_size_bytes{fstype=~\"ext[234]|btrfs|xfs|zfs\", cluster=\"$cluster\"} - node_filesystem_avail_bytes{fstype=~\"ext[234]|btrfs|xfs|zfs\", cluster=\"$cluster\"}) by (device,pod,namespace)) by (pod,namespace)\n/ scalar(sum(max(node_filesystem_size_bytes{fstype=~\"ext[234]|btrfs|xfs|zfs\", cluster=\"$cluster\"}) by (device,pod,namespace)))\n* on (namespace, pod) group_left (node) node_namespace_pod:kube_pod_info:{cluster=\"$cluster\"}\n", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{node}}", - "legendLink": "/d/4ac4f123aae0ff6dbaf4f4f66120033b/k8s-node-rsrc-use", - "step": 10 - } - ], - "thresholds": [ - - ], - "timeFrom": null, - "timeShift": null, - "title": "Disk Capacity", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ - - ] - }, - "yaxes": [ - { - "format": "percentunit", - "label": null, - "logBase": 1, - "max": 1, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] - } - ], - "repeat": null, - "repeatIteration": null, - "repeatRowId": null, - "showTitle": true, - "title": "Storage", - "titleSize": "h6" - } - ], - "schemaVersion": 14, - "style": "dark", - "tags": [ - "kubernetes-mixin" - ], - "templating": { - "list": [ - { - "current": { - "text": "Prometheus", - "value": "Prometheus" - }, - "hide": 0, - "label": null, - "name": "datasource", - "options": [ - - ], - "query": "prometheus", - "refresh": 1, - "regex": "", - "type": "datasource" - }, - { - "allValue": null, - "current": { - "text": "prod", - "value": "prod" - }, - "datasource": "$datasource", - "hide": 2, - "includeAll": false, - "label": "cluster", - "multi": false, - "name": "cluster", - "options": [ - - ], - "query": "label_values(kube_node_info, cluster)", - "refresh": 1, - "regex": "", - "sort": 2, - "tagValuesQuery": "", - "tags": [ - - ], - "tagsQuery": "", - "type": "query", - "useTags": false - } - ] - }, - "time": { - "from": "now-1h", - "to": "now" - }, - "timepicker": { - "refresh_intervals": [ - "5s", - "10s", - "30s", - "1m", - "5m", - "15m", - "30m", - "1h", - "2h", - "1d" - ], - "time_options": [ - "5m", - "15m", - "1h", - "6h", - "12h", - "24h", - "2d", - "7d", - "30d" - ] - }, - "timezone": "", - "title": "Kubernetes / USE Method / Cluster", - "uid": "a6e7d1362e1ddbb79db21d5bb40d7137", - "version": 0 - } - k8s-node-rsrc-use.json: |- - { - "annotations": { - "list": [ - - ] - }, - "editable": true, - "gnetId": null, - "graphTooltip": 0, - "hideControls": false, - "links": [ - - ], - "refresh": "10s", - "rows": [ - { - "collapse": false, - "height": "250px", - "panels": [ - { - "aliasColors": { - - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 1, - "id": 1, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [ - - ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - - ], - "spaceLength": 10, - "span": 6, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "node:node_cpu_utilisation:avg1m{cluster=\"$cluster\", node=\"$node\"}", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "Utilisation", - "legendLink": null, - "step": 10 - } - ], - "thresholds": [ - - ], - "timeFrom": null, - "timeShift": null, - "title": "CPU Utilisation", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ - - ] - }, - "yaxes": [ - { - "format": "percentunit", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] - }, - { - "aliasColors": { - - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 1, - "id": 2, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [ - - ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - - ], - "spaceLength": 10, - "span": 6, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "node:node_cpu_saturation_load1:{cluster=\"$cluster\", node=\"$node\"}", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "Saturation", - "legendLink": null, - "step": 10 - } - ], - "thresholds": [ - - ], - "timeFrom": null, - "timeShift": null, - "title": "CPU Saturation (Load1)", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ - - ] - }, - "yaxes": [ - { - "format": "percentunit", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] - } - ], - "repeat": null, - "repeatIteration": null, - "repeatRowId": null, - "showTitle": true, - "title": "CPU", - "titleSize": "h6" - }, - { - "collapse": false, - "height": "250px", - "panels": [ - { - "aliasColors": { - - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 1, - "id": 3, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [ - - ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - - ], - "spaceLength": 10, - "span": 6, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "node:node_memory_utilisation:{cluster=\"$cluster\", node=\"$node\"}", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "Memory", - "legendLink": null, - "step": 10 - } - ], - "thresholds": [ - - ], - "timeFrom": null, - "timeShift": null, - "title": "Memory Utilisation", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ - - ] - }, - "yaxes": [ - { - "format": "percentunit", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] - }, - { - "aliasColors": { - - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 1, - "id": 4, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [ - - ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - - ], - "spaceLength": 10, - "span": 6, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "node:node_memory_swap_io_bytes:sum_rate{cluster=\"$cluster\", node=\"$node\"}", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "Swap IO", - "legendLink": null, - "step": 10 - } - ], - "thresholds": [ - - ], - "timeFrom": null, - "timeShift": null, - "title": "Memory Saturation (Swap I/O)", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ - - ] - }, - "yaxes": [ - { - "format": "Bps", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] - } - ], - "repeat": null, - "repeatIteration": null, - "repeatRowId": null, - "showTitle": true, - "title": "Memory", - "titleSize": "h6" - }, - { - "collapse": false, - "height": "250px", - "panels": [ - { - "aliasColors": { - - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 1, - "id": 5, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [ - - ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - - ], - "spaceLength": 10, - "span": 6, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "node:node_disk_utilisation:avg_irate{cluster=\"$cluster\", node=\"$node\"}", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "Utilisation", - "legendLink": null, - "step": 10 - } - ], - "thresholds": [ - - ], - "timeFrom": null, - "timeShift": null, - "title": "Disk IO Utilisation", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ - - ] - }, - "yaxes": [ - { - "format": "percentunit", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] - }, - { - "aliasColors": { - - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 1, - "id": 6, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [ - - ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - - ], - "spaceLength": 10, - "span": 6, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "node:node_disk_saturation:avg_irate{cluster=\"$cluster\", node=\"$node\"}", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "Saturation", - "legendLink": null, - "step": 10 - } - ], - "thresholds": [ - - ], - "timeFrom": null, - "timeShift": null, - "title": "Disk IO Saturation", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ - - ] - }, - "yaxes": [ - { - "format": "percentunit", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] - } - ], - "repeat": null, - "repeatIteration": null, - "repeatRowId": null, - "showTitle": true, - "title": "Disk", - "titleSize": "h6" - }, - { - "collapse": false, - "height": "250px", - "panels": [ - { - "aliasColors": { - - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 1, - "id": 7, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [ - - ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - - ], - "spaceLength": 10, - "span": 6, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "node:node_net_utilisation:sum_irate{cluster=\"$cluster\", node=\"$node\"}", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "Utilisation", - "legendLink": null, - "step": 10 - } - ], - "thresholds": [ - - ], - "timeFrom": null, - "timeShift": null, - "title": "Net Utilisation (Transmitted)", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ - - ] - }, - "yaxes": [ - { - "format": "Bps", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] - }, - { - "aliasColors": { - - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 1, - "id": 8, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [ - - ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - - ], - "spaceLength": 10, - "span": 6, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "node:node_net_saturation:sum_irate{cluster=\"$cluster\", node=\"$node\"}", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "Saturation", - "legendLink": null, - "step": 10 - } - ], - "thresholds": [ - - ], - "timeFrom": null, - "timeShift": null, - "title": "Net Saturation (Dropped)", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ - - ] - }, - "yaxes": [ - { - "format": "Bps", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] - } - ], - "repeat": null, - "repeatIteration": null, - "repeatRowId": null, - "showTitle": true, - "title": "Net", - "titleSize": "h6" - }, - { - "collapse": false, - "height": "250px", - "panels": [ - { - "aliasColors": { - - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 1, - "id": 9, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [ - - ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - - ], - "spaceLength": 10, - "span": 12, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "node:node_filesystem_usage:{cluster=\"$cluster\"}\n* on (namespace, pod) group_left (node) node_namespace_pod:kube_pod_info:{cluster=\"$cluster\", node=\"$node\"}\n", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{device}}", - "legendLink": null, - "step": 10 - } - ], - "thresholds": [ - - ], - "timeFrom": null, - "timeShift": null, - "title": "Disk Utilisation", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ - - ] - }, - "yaxes": [ + "styles": [ { - "format": "percentunit", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" }, { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] - } - ], - "repeat": null, - "repeatIteration": null, - "repeatRowId": null, - "showTitle": true, - "title": "Disk", - "titleSize": "h6" - } - ], - "schemaVersion": 14, - "style": "dark", - "tags": [ - "kubernetes-mixin" - ], - "templating": { - "list": [ - { - "current": { - "text": "Prometheus", - "value": "Prometheus" - }, - "hide": 0, - "label": null, - "name": "datasource", - "options": [ - - ], - "query": "prometheus", - "refresh": 1, - "regex": "", - "type": "datasource" - }, - { - "allValue": null, - "current": { - "text": "prod", - "value": "prod" - }, - "datasource": "$datasource", - "hide": 2, - "includeAll": false, - "label": "cluster", - "multi": false, - "name": "cluster", - "options": [ + "alias": "Pods", + "colorMode": null, + "colors": [ - ], - "query": "label_values(kube_node_info, cluster)", - "refresh": 1, - "regex": "", - "sort": 2, - "tagValuesQuery": "", - "tags": [ + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 0, + "link": true, + "linkTooltip": "Drill down to pods", + "linkUrl": "./d/85a562078cdf77779eaa1add43ccec1e/k8s-resources-namespace?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$__cell_1", + "pattern": "Value #A", + "thresholds": [ - ], - "tagsQuery": "", - "type": "query", - "useTags": false - }, - { - "allValue": null, - "current": { - "text": "prod", - "value": "prod" - }, - "datasource": "$datasource", - "hide": 0, - "includeAll": false, - "label": "node", - "multi": false, - "name": "node", - "options": [ + ], + "type": "number", + "unit": "short" + }, + { + "alias": "Workloads", + "colorMode": null, + "colors": [ - ], - "query": "label_values(kube_node_info{cluster=\"$cluster\"}, node)", - "refresh": 1, - "regex": "", - "sort": 2, - "tagValuesQuery": "", - "tags": [ + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 0, + "link": true, + "linkTooltip": "Drill down to workloads", + "linkUrl": "./d/a87fb0d919ec0ea5f6543124e16c42a5/k8s-resources-workloads-namespace?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$__cell_1", + "pattern": "Value #B", + "thresholds": [ - ], - "tagsQuery": "", - "type": "query", - "useTags": false - } - ] - }, - "time": { - "from": "now-1h", - "to": "now" - }, - "timepicker": { - "refresh_intervals": [ - "5s", - "10s", - "30s", - "1m", - "5m", - "15m", - "30m", - "1h", - "2h", - "1d" - ], - "time_options": [ - "5m", - "15m", - "1h", - "6h", - "12h", - "24h", - "2d", - "7d", - "30d" - ] - }, - "timezone": "", - "title": "Kubernetes / USE Method / Node", - "uid": "4ac4f123aae0ff6dbaf4f4f66120033b", - "version": 0 - } - k8s-resources-cluster.json: |- - { - "annotations": { - "list": [ + ], + "type": "number", + "unit": "short" + }, + { + "alias": "CPU Usage", + "colorMode": null, + "colors": [ - ] - }, - "editable": true, - "gnetId": null, - "graphTooltip": 0, - "hideControls": false, - "links": [ + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #C", + "thresholds": [ - ], - "refresh": "10s", - "rows": [ - { - "collapse": false, - "height": "100px", - "panels": [ - { - "aliasColors": { + ], + "type": "number", + "unit": "short" + }, + { + "alias": "CPU Requests", + "colorMode": null, + "colors": [ - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 1, - "format": "percentunit", - "id": 1, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [ + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #D", + "thresholds": [ - ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ + ], + "type": "number", + "unit": "short" + }, + { + "alias": "CPU Requests %", + "colorMode": null, + "colors": [ - ], - "spaceLength": 10, - "span": 2, - "stack": false, - "steppedLine": false, - "targets": [ + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #E", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, { - "expr": "1 - avg(rate(node_cpu_seconds_total{mode=\"idle\", cluster=\"$cluster\"}[1m]))", - "format": "time_series", - "instant": true, - "intervalFactor": 2, - "refId": "A" - } - ], - "thresholds": "70,80", - "timeFrom": null, - "timeShift": null, - "title": "CPU Utilisation", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "singlestat", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ + "alias": "CPU Limits", + "colorMode": null, + "colors": [ - ] - }, - "yaxes": [ + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #F", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true + "alias": "CPU Limits %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #G", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" }, { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] - }, - { - "aliasColors": { + "alias": "Namespace", + "colorMode": null, + "colors": [ - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 1, - "format": "percentunit", - "id": 2, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [ + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": true, + "linkTooltip": "Drill down to pods", + "linkUrl": "./d/85a562078cdf77779eaa1add43ccec1e/k8s-resources-namespace?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$__cell", + "pattern": "namespace", + "thresholds": [ - ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ + ], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "/.*/", + "thresholds": [ + ], + "type": "string", + "unit": "short" + } ], - "spaceLength": 10, - "span": 2, - "stack": false, - "steppedLine": false, "targets": [ { - "expr": "sum(kube_pod_container_resource_requests_cpu_cores{cluster=\"$cluster\"}) / sum(kube_node_status_allocatable_cpu_cores{cluster=\"$cluster\"})", - "format": "time_series", + "expr": "count(mixin_pod_workload{cluster=\"$cluster\"}) by (namespace)", + "format": "table", "instant": true, "intervalFactor": 2, - "refId": "A" - } - ], - "thresholds": "70,80", - "timeFrom": null, - "timeShift": null, - "title": "CPU Requests Commitment", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "singlestat", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ - - ] - }, - "yaxes": [ + "legendFormat": "", + "refId": "A", + "step": 10 + }, { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true + "expr": "count(avg(mixin_pod_workload{cluster=\"$cluster\"}) by (workload, namespace)) by (namespace)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 }, { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] - }, - { - "aliasColors": { - - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 1, - "format": "percentunit", - "id": 3, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [ - - ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - - ], - "spaceLength": 10, - "span": 2, - "stack": false, - "steppedLine": false, - "targets": [ + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\"}) by (namespace)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "C", + "step": 10 + }, + { + "expr": "sum(kube_pod_container_resource_requests_cpu_cores{cluster=\"$cluster\"}) by (namespace)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "D", + "step": 10 + }, { - "expr": "sum(kube_pod_container_resource_limits_cpu_cores{cluster=\"$cluster\"}) / sum(kube_node_status_allocatable_cpu_cores{cluster=\"$cluster\"})", - "format": "time_series", + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\"}) by (namespace) / sum(kube_pod_container_resource_requests_cpu_cores{cluster=\"$cluster\"}) by (namespace)", + "format": "table", "instant": true, "intervalFactor": 2, - "refId": "A" + "legendFormat": "", + "refId": "E", + "step": 10 + }, + { + "expr": "sum(kube_pod_container_resource_limits_cpu_cores{cluster=\"$cluster\"}) by (namespace)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "F", + "step": 10 + }, + { + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\"}) by (namespace) / sum(kube_pod_container_resource_limits_cpu_cores{cluster=\"$cluster\"}) by (namespace)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "G", + "step": 10 } ], - "thresholds": "70,80", + "thresholds": [ + + ], "timeFrom": null, "timeShift": null, - "title": "CPU Limits Commitment", + "title": "CPU Quota", "tooltip": { "shared": false, "sort": 0, "value_type": "individual" }, - "type": "singlestat", + "transform": "table", + "type": "table", "xaxis": { "buckets": null, "mode": "time", @@ -2176,7 +942,19 @@ data: "show": false } ] - }, + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "CPU Quota", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ { "aliasColors": { @@ -2185,9 +963,8 @@ data: "dashLength": 10, "dashes": false, "datasource": "$datasource", - "fill": 1, - "format": "percentunit", - "id": 4, + "fill": 10, + "id": 9, "legend": { "avg": false, "current": false, @@ -2198,7 +975,7 @@ data: "values": false }, "lines": true, - "linewidth": 1, + "linewidth": 0, "links": [ ], @@ -2211,28 +988,31 @@ data: ], "spaceLength": 10, - "span": 2, - "stack": false, + "span": 12, + "stack": true, "steppedLine": false, "targets": [ { - "expr": "1 - sum(:node_memory_MemFreeCachedBuffers_bytes:sum{cluster=\"$cluster\"}) / sum(kube_node_status_allocatable_memory_bytes{cluster=\"$cluster\"})", + "expr": "sum(container_memory_rss{cluster=\"$cluster\", container!=\"\"}) by (namespace)", "format": "time_series", - "instant": true, "intervalFactor": 2, - "refId": "A" + "legendFormat": "{{namespace}}", + "legendLink": null, + "step": 10 } ], - "thresholds": "70,80", + "thresholds": [ + + ], "timeFrom": null, "timeShift": null, - "title": "Memory Utilisation", + "title": "Memory Usage (w/o cache)", "tooltip": { "shared": false, "sort": 0, "value_type": "individual" }, - "type": "singlestat", + "type": "graph", "xaxis": { "buckets": null, "mode": "time", @@ -2244,7 +1024,7 @@ data: }, "yaxes": [ { - "format": "short", + "format": "bytes", "label": null, "logBase": 1, "max": null, @@ -2260,7 +1040,19 @@ data: "show": false } ] - }, + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Memory", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ { "aliasColors": { @@ -2270,8 +1062,7 @@ data: "dashes": false, "datasource": "$datasource", "fill": 1, - "format": "percentunit", - "id": 5, + "id": 10, "legend": { "avg": false, "current": false, @@ -2285,122 +1076,264 @@ data: "linewidth": 1, "links": [ - ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "Pods", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 0, + "link": true, + "linkTooltip": "Drill down to pods", + "linkUrl": "./d/85a562078cdf77779eaa1add43ccec1e/k8s-resources-namespace?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$__cell_1", + "pattern": "Value #A", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "Workloads", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 0, + "link": true, + "linkTooltip": "Drill down to workloads", + "linkUrl": "./d/a87fb0d919ec0ea5f6543124e16c42a5/k8s-resources-workloads-namespace?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$__cell_1", + "pattern": "Value #B", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "Memory Usage", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #C", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Requests", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #D", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Requests %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #E", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "Memory Limits", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #F", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Limits %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #G", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "Namespace", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": true, + "linkTooltip": "Drill down to pods", + "linkUrl": "./d/85a562078cdf77779eaa1add43ccec1e/k8s-resources-namespace?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$__cell", + "pattern": "namespace", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "/.*/", + "thresholds": [ + + ], + "type": "string", + "unit": "short" + } ], - "spaceLength": 10, - "span": 2, - "stack": false, - "steppedLine": false, "targets": [ { - "expr": "sum(kube_pod_container_resource_requests_memory_bytes{cluster=\"$cluster\"}) / sum(kube_node_status_allocatable_memory_bytes{cluster=\"$cluster\"})", - "format": "time_series", + "expr": "count(mixin_pod_workload{cluster=\"$cluster\"}) by (namespace)", + "format": "table", "instant": true, "intervalFactor": 2, - "refId": "A" - } - ], - "thresholds": "70,80", - "timeFrom": null, - "timeShift": null, - "title": "Memory Requests Commitment", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "singlestat", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ - - ] - }, - "yaxes": [ + "legendFormat": "", + "refId": "A", + "step": 10 + }, { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true + "expr": "count(avg(mixin_pod_workload{cluster=\"$cluster\"}) by (workload, namespace)) by (namespace)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 }, { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] - }, - { - "aliasColors": { - - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 1, - "format": "percentunit", - "id": 6, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [ - - ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - - ], - "spaceLength": 10, - "span": 2, - "stack": false, - "steppedLine": false, - "targets": [ + "expr": "sum(container_memory_rss{cluster=\"$cluster\", container!=\"\"}) by (namespace)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "C", + "step": 10 + }, { - "expr": "sum(kube_pod_container_resource_limits_memory_bytes{cluster=\"$cluster\"}) / sum(kube_node_status_allocatable_memory_bytes{cluster=\"$cluster\"})", - "format": "time_series", + "expr": "sum(kube_pod_container_resource_requests_memory_bytes{cluster=\"$cluster\"}) by (namespace)", + "format": "table", "instant": true, "intervalFactor": 2, - "refId": "A" + "legendFormat": "", + "refId": "D", + "step": 10 + }, + { + "expr": "sum(container_memory_rss{cluster=\"$cluster\", container!=\"\"}) by (namespace) / sum(kube_pod_container_resource_requests_memory_bytes{cluster=\"$cluster\"}) by (namespace)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "E", + "step": 10 + }, + { + "expr": "sum(kube_pod_container_resource_limits_memory_bytes{cluster=\"$cluster\"}) by (namespace)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "F", + "step": 10 + }, + { + "expr": "sum(container_memory_rss{cluster=\"$cluster\", container!=\"\"}) by (namespace) / sum(kube_pod_container_resource_limits_memory_bytes{cluster=\"$cluster\"}) by (namespace)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "G", + "step": 10 } ], - "thresholds": "70,80", + "thresholds": [ + + ], "timeFrom": null, "timeShift": null, - "title": "Memory Limits Commitment", + "title": "Requests by Namespace", "tooltip": { "shared": false, "sort": 0, "value_type": "individual" }, - "type": "singlestat", + "transform": "table", + "type": "table", "xaxis": { "buckets": null, "mode": "time", @@ -2433,10 +1366,113 @@ data: "repeat": null, "repeatIteration": null, "repeatRowId": null, - "showTitle": false, - "title": "Headlines", + "showTitle": true, + "title": "Memory Requests", "titleSize": "h6" - }, + } + ], + "schemaVersion": 14, + "style": "dark", + "tags": [ + "kubernetes-mixin" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": null, + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + "text": "prod", + "value": "prod" + }, + "datasource": "$datasource", + "hide": 2, + "includeAll": false, + "label": "cluster", + "multi": false, + "name": "cluster", + "options": [ + + ], + "query": "label_values(node_cpu_seconds_total, cluster)", + "refresh": 1, + "regex": "", + "sort": 2, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "", + "title": "Kubernetes / Compute Resources / Cluster", + "uid": "efa86fd1d0c121a26444b636a3f509a8", + "version": 0 + } + k8s-resources-namespace.json: |- + { + "annotations": { + "list": [ + + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "links": [ + + ], + "refresh": "10s", + "rows": [ { "collapse": false, "height": "250px", @@ -2450,7 +1486,7 @@ data: "dashes": false, "datasource": "$datasource", "fill": 10, - "id": 7, + "id": 1, "legend": { "avg": false, "current": false, @@ -2479,10 +1515,10 @@ data: "steppedLine": false, "targets": [ { - "expr": "sum(namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\"}) by (namespace)", + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)", "format": "time_series", "intervalFactor": 2, - "legendFormat": "{{namespace}}", + "legendFormat": "{{pod}}", "legendLink": null, "step": 10 } @@ -2532,7 +1568,7 @@ data: "repeatIteration": null, "repeatRowId": null, "showTitle": true, - "title": "CPU", + "title": "CPU Usage", "titleSize": "h6" }, { @@ -2548,7 +1584,7 @@ data: "dashes": false, "datasource": "$datasource", "fill": 1, - "id": 8, + "id": 2, "legend": { "avg": false, "current": false, @@ -2582,42 +1618,6 @@ data: "pattern": "Time", "type": "hidden" }, - { - "alias": "Pods", - "colorMode": null, - "colors": [ - - ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 0, - "link": true, - "linkTooltip": "Drill down to pods", - "linkUrl": "/d/85a562078cdf77779eaa1add43ccec1e/k8s-resources-namespace?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$__cell_1", - "pattern": "Value #A", - "thresholds": [ - - ], - "type": "number", - "unit": "short" - }, - { - "alias": "Workloads", - "colorMode": null, - "colors": [ - - ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 0, - "link": true, - "linkTooltip": "Drill down to workloads", - "linkUrl": "/d/a87fb0d919ec0ea5f6543124e16c42a5/k8s-resources-workloads-namespace?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$__cell_1", - "pattern": "Value #B", - "thresholds": [ - - ], - "type": "number", - "unit": "short" - }, { "alias": "CPU Usage", "colorMode": null, @@ -2629,7 +1629,7 @@ data: "link": false, "linkTooltip": "Drill down", "linkUrl": "", - "pattern": "Value #C", + "pattern": "Value #A", "thresholds": [ ], @@ -2647,7 +1647,7 @@ data: "link": false, "linkTooltip": "Drill down", "linkUrl": "", - "pattern": "Value #D", + "pattern": "Value #B", "thresholds": [ ], @@ -2665,7 +1665,7 @@ data: "link": false, "linkTooltip": "Drill down", "linkUrl": "", - "pattern": "Value #E", + "pattern": "Value #C", "thresholds": [ ], @@ -2683,7 +1683,7 @@ data: "link": false, "linkTooltip": "Drill down", "linkUrl": "", - "pattern": "Value #F", + "pattern": "Value #D", "thresholds": [ ], @@ -2701,7 +1701,7 @@ data: "link": false, "linkTooltip": "Drill down", "linkUrl": "", - "pattern": "Value #G", + "pattern": "Value #E", "thresholds": [ ], @@ -2709,7 +1709,7 @@ data: "unit": "percentunit" }, { - "alias": "Namespace", + "alias": "Pod", "colorMode": null, "colors": [ @@ -2717,9 +1717,9 @@ data: "dateFormat": "YYYY-MM-DD HH:mm:ss", "decimals": 2, "link": true, - "linkTooltip": "Drill down to pods", - "linkUrl": "/d/85a562078cdf77779eaa1add43ccec1e/k8s-resources-namespace?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$__cell", - "pattern": "namespace", + "linkTooltip": "Drill down", + "linkUrl": "./d/6581e46e4e5c7ba40a07646395ef7b23/k8s-resources-pod?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$namespace&var-pod=$__cell", + "pattern": "pod", "thresholds": [ ], @@ -2744,7 +1744,7 @@ data: ], "targets": [ { - "expr": "count(mixin_pod_workload{cluster=\"$cluster\"}) by (namespace)", + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)", "format": "table", "instant": true, "intervalFactor": 2, @@ -2753,7 +1753,7 @@ data: "step": 10 }, { - "expr": "count(avg(mixin_pod_workload{cluster=\"$cluster\"}) by (workload, namespace)) by (namespace)", + "expr": "sum(kube_pod_container_resource_requests_cpu_cores{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)", "format": "table", "instant": true, "intervalFactor": 2, @@ -2762,7 +1762,7 @@ data: "step": 10 }, { - "expr": "sum(namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\"}) by (namespace)", + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod) / sum(kube_pod_container_resource_requests_cpu_cores{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)", "format": "table", "instant": true, "intervalFactor": 2, @@ -2771,7 +1771,7 @@ data: "step": 10 }, { - "expr": "sum(kube_pod_container_resource_requests_cpu_cores{cluster=\"$cluster\"}) by (namespace)", + "expr": "sum(kube_pod_container_resource_limits_cpu_cores{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)", "format": "table", "instant": true, "intervalFactor": 2, @@ -2780,31 +1780,13 @@ data: "step": 10 }, { - "expr": "sum(namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\"}) by (namespace) / sum(kube_pod_container_resource_requests_cpu_cores{cluster=\"$cluster\"}) by (namespace)", + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod) / sum(kube_pod_container_resource_limits_cpu_cores{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)", "format": "table", "instant": true, "intervalFactor": 2, "legendFormat": "", "refId": "E", "step": 10 - }, - { - "expr": "sum(kube_pod_container_resource_limits_cpu_cores{cluster=\"$cluster\"}) by (namespace)", - "format": "table", - "instant": true, - "intervalFactor": 2, - "legendFormat": "", - "refId": "F", - "step": 10 - }, - { - "expr": "sum(namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\"}) by (namespace) / sum(kube_pod_container_resource_limits_cpu_cores{cluster=\"$cluster\"}) by (namespace)", - "format": "table", - "instant": true, - "intervalFactor": 2, - "legendFormat": "", - "refId": "G", - "step": 10 } ], "thresholds": [ @@ -2869,7 +1851,7 @@ data: "dashes": false, "datasource": "$datasource", "fill": 10, - "id": 9, + "id": 3, "legend": { "avg": false, "current": false, @@ -2898,10 +1880,10 @@ data: "steppedLine": false, "targets": [ { - "expr": "sum(container_memory_rss{cluster=\"$cluster\", container!=\"\"}) by (namespace)", + "expr": "sum(container_memory_working_set_bytes{cluster=\"$cluster\", namespace=\"$namespace\", container!=\"\"}) by (pod)", "format": "time_series", "intervalFactor": 2, - "legendFormat": "{{namespace}}", + "legendFormat": "{{pod}}", "legendLink": null, "step": 10 } @@ -2951,7 +1933,7 @@ data: "repeatIteration": null, "repeatRowId": null, "showTitle": true, - "title": "Memory", + "title": "Memory Usage", "titleSize": "h6" }, { @@ -2967,7 +1949,7 @@ data: "dashes": false, "datasource": "$datasource", "fill": 1, - "id": 10, + "id": 4, "legend": { "avg": false, "current": false, @@ -3002,43 +1984,43 @@ data: "type": "hidden" }, { - "alias": "Pods", + "alias": "Memory Usage", "colorMode": null, "colors": [ ], "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 0, - "link": true, - "linkTooltip": "Drill down to pods", - "linkUrl": "/d/85a562078cdf77779eaa1add43ccec1e/k8s-resources-namespace?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$__cell_1", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", "pattern": "Value #A", "thresholds": [ ], "type": "number", - "unit": "short" + "unit": "bytes" }, { - "alias": "Workloads", + "alias": "Memory Requests", "colorMode": null, "colors": [ ], "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 0, - "link": true, - "linkTooltip": "Drill down to workloads", - "linkUrl": "/d/a87fb0d919ec0ea5f6543124e16c42a5/k8s-resources-workloads-namespace?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$__cell_1", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", "pattern": "Value #B", "thresholds": [ ], "type": "number", - "unit": "short" + "unit": "bytes" }, { - "alias": "Memory Usage", + "alias": "Memory Requests %", "colorMode": null, "colors": [ @@ -3053,10 +2035,10 @@ data: ], "type": "number", - "unit": "bytes" + "unit": "percentunit" }, { - "alias": "Memory Requests", + "alias": "Memory Limits", "colorMode": null, "colors": [ @@ -3074,7 +2056,7 @@ data: "unit": "bytes" }, { - "alias": "Memory Requests %", + "alias": "Memory Limits %", "colorMode": null, "colors": [ @@ -3092,7 +2074,7 @@ data: "unit": "percentunit" }, { - "alias": "Memory Limits", + "alias": "Memory Usage (RSS)", "colorMode": null, "colors": [ @@ -3110,7 +2092,7 @@ data: "unit": "bytes" }, { - "alias": "Memory Limits %", + "alias": "Memory Usage (Cache)", "colorMode": null, "colors": [ @@ -3125,10 +2107,28 @@ data: ], "type": "number", - "unit": "percentunit" + "unit": "bytes" }, { - "alias": "Namespace", + "alias": "Memory Usage (Swap)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #H", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Pod", "colorMode": null, "colors": [ @@ -3136,9 +2136,9 @@ data: "dateFormat": "YYYY-MM-DD HH:mm:ss", "decimals": 2, "link": true, - "linkTooltip": "Drill down to pods", - "linkUrl": "/d/85a562078cdf77779eaa1add43ccec1e/k8s-resources-namespace?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$__cell", - "pattern": "namespace", + "linkTooltip": "Drill down", + "linkUrl": "./d/6581e46e4e5c7ba40a07646395ef7b23/k8s-resources-pod?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$namespace&var-pod=$__cell", + "pattern": "pod", "thresholds": [ ], @@ -3163,7 +2163,7 @@ data: ], "targets": [ { - "expr": "count(mixin_pod_workload{cluster=\"$cluster\"}) by (namespace)", + "expr": "sum(container_memory_working_set_bytes{cluster=\"$cluster\", namespace=\"$namespace\",container!=\"\"}) by (pod)", "format": "table", "instant": true, "intervalFactor": 2, @@ -3172,7 +2172,7 @@ data: "step": 10 }, { - "expr": "count(avg(mixin_pod_workload{cluster=\"$cluster\"}) by (workload, namespace)) by (namespace)", + "expr": "sum(kube_pod_container_resource_requests_memory_bytes{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)", "format": "table", "instant": true, "intervalFactor": 2, @@ -3181,7 +2181,7 @@ data: "step": 10 }, { - "expr": "sum(container_memory_rss{cluster=\"$cluster\", container!=\"\"}) by (namespace)", + "expr": "sum(container_memory_working_set_bytes{cluster=\"$cluster\", namespace=\"$namespace\",container!=\"\"}) by (pod) / sum(kube_pod_container_resource_requests_memory_bytes{namespace=\"$namespace\"}) by (pod)", "format": "table", "instant": true, "intervalFactor": 2, @@ -3190,7 +2190,7 @@ data: "step": 10 }, { - "expr": "sum(kube_pod_container_resource_requests_memory_bytes{cluster=\"$cluster\"}) by (namespace)", + "expr": "sum(kube_pod_container_resource_limits_memory_bytes{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)", "format": "table", "instant": true, "intervalFactor": 2, @@ -3199,7 +2199,7 @@ data: "step": 10 }, { - "expr": "sum(container_memory_rss{cluster=\"$cluster\", container!=\"\"}) by (namespace) / sum(kube_pod_container_resource_requests_memory_bytes{cluster=\"$cluster\"}) by (namespace)", + "expr": "sum(container_memory_working_set_bytes{cluster=\"$cluster\", namespace=\"$namespace\",container!=\"\"}) by (pod) / sum(kube_pod_container_resource_limits_memory_bytes{namespace=\"$namespace\"}) by (pod)", "format": "table", "instant": true, "intervalFactor": 2, @@ -3208,7 +2208,7 @@ data: "step": 10 }, { - "expr": "sum(kube_pod_container_resource_limits_memory_bytes{cluster=\"$cluster\"}) by (namespace)", + "expr": "sum(container_memory_rss{cluster=\"$cluster\", namespace=\"$namespace\",container!=\"\"}) by (pod)", "format": "table", "instant": true, "intervalFactor": 2, @@ -3217,13 +2217,22 @@ data: "step": 10 }, { - "expr": "sum(container_memory_rss{cluster=\"$cluster\", container!=\"\"}) by (namespace) / sum(kube_pod_container_resource_limits_memory_bytes{cluster=\"$cluster\"}) by (namespace)", + "expr": "sum(container_memory_cache{cluster=\"$cluster\", namespace=\"$namespace\",container!=\"\"}) by (pod)", "format": "table", "instant": true, "intervalFactor": 2, "legendFormat": "", "refId": "G", "step": 10 + }, + { + "expr": "sum(container_memory_swap{cluster=\"$cluster\", namespace=\"$namespace\",container!=\"\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "H", + "step": 10 } ], "thresholds": [ @@ -3231,7 +2240,7 @@ data: ], "timeFrom": null, "timeShift": null, - "title": "Requests by Namespace", + "title": "Memory Quota", "tooltip": { "shared": false, "sort": 0, @@ -3272,7 +2281,7 @@ data: "repeatIteration": null, "repeatRowId": null, "showTitle": true, - "title": "Memory Requests", + "title": "Memory Quota", "titleSize": "h6" } ], @@ -3314,7 +2323,34 @@ data: "options": [ ], - "query": "label_values(node_cpu_seconds_total, cluster)", + "query": "label_values(kube_pod_info, cluster)", + "refresh": 1, + "regex": "", + "sort": 2, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + "text": "prod", + "value": "prod" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": "namespace", + "multi": false, + "name": "namespace", + "options": [ + + ], + "query": "label_values(kube_pod_info{cluster=\"$cluster\"}, namespace)", "refresh": 1, "regex": "", "sort": 2, @@ -3358,11 +2394,11 @@ data: ] }, "timezone": "", - "title": "Kubernetes / Compute Resources / Cluster", - "uid": "efa86fd1d0c121a26444b636a3f509a8", + "title": "Kubernetes / Compute Resources / Namespace (Pods)", + "uid": "85a562078cdf77779eaa1add43ccec1e", "version": 0 } - k8s-resources-namespace.json: |- + k8s-resources-node.json: |- { "annotations": { "list": [ @@ -3420,7 +2456,7 @@ data: "steppedLine": false, "targets": [ { - "expr": "sum(namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)", + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", node=\"$node\"}) by (pod)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{pod}}", @@ -3621,9 +2657,9 @@ data: ], "dateFormat": "YYYY-MM-DD HH:mm:ss", "decimals": 2, - "link": true, + "link": false, "linkTooltip": "Drill down", - "linkUrl": "/d/6581e46e4e5c7ba40a07646395ef7b23/k8s-resources-pod?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$namespace&var-pod=$__cell", + "linkUrl": "", "pattern": "pod", "thresholds": [ @@ -3649,7 +2685,7 @@ data: ], "targets": [ { - "expr": "sum(namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)", + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", node=\"$node\"}) by (pod)", "format": "table", "instant": true, "intervalFactor": 2, @@ -3658,7 +2694,7 @@ data: "step": 10 }, { - "expr": "sum(kube_pod_container_resource_requests_cpu_cores{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)", + "expr": "sum(kube_pod_container_resource_requests_cpu_cores{cluster=\"$cluster\", node=\"$node\"}) by (pod)", "format": "table", "instant": true, "intervalFactor": 2, @@ -3667,7 +2703,7 @@ data: "step": 10 }, { - "expr": "sum(namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod) / sum(kube_pod_container_resource_requests_cpu_cores{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)", + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", node=\"$node\"}) by (pod) / sum(kube_pod_container_resource_requests_cpu_cores{cluster=\"$cluster\", node=\"$node\"}) by (pod)", "format": "table", "instant": true, "intervalFactor": 2, @@ -3676,7 +2712,7 @@ data: "step": 10 }, { - "expr": "sum(kube_pod_container_resource_limits_cpu_cores{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)", + "expr": "sum(kube_pod_container_resource_limits_cpu_cores{cluster=\"$cluster\", node=\"$node\"}) by (pod)", "format": "table", "instant": true, "intervalFactor": 2, @@ -3685,7 +2721,7 @@ data: "step": 10 }, { - "expr": "sum(namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod) / sum(kube_pod_container_resource_limits_cpu_cores{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)", + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", node=\"$node\"}) by (pod) / sum(kube_pod_container_resource_limits_cpu_cores{cluster=\"$cluster\", node=\"$node\"}) by (pod)", "format": "table", "instant": true, "intervalFactor": 2, @@ -3785,7 +2821,7 @@ data: "steppedLine": false, "targets": [ { - "expr": "sum(container_memory_usage_bytes{cluster=\"$cluster\", namespace=\"$namespace\", container!=\"\"}) by (pod)", + "expr": "sum(container_memory_working_set_bytes{cluster=\"$cluster\", node=\"$node\", container!=\"\"}) by (pod)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{pod}}", @@ -4015,7 +3051,7 @@ data: "unit": "bytes" }, { - "alias": "Memory Usage (Swap", + "alias": "Memory Usage (Swap)", "colorMode": null, "colors": [ @@ -4040,9 +3076,9 @@ data: ], "dateFormat": "YYYY-MM-DD HH:mm:ss", "decimals": 2, - "link": true, + "link": false, "linkTooltip": "Drill down", - "linkUrl": "/d/6581e46e4e5c7ba40a07646395ef7b23/k8s-resources-pod?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$namespace&var-pod=$__cell", + "linkUrl": "", "pattern": "pod", "thresholds": [ @@ -4068,7 +3104,7 @@ data: ], "targets": [ { - "expr": "sum(container_memory_usage_bytes{cluster=\"$cluster\", namespace=\"$namespace\",container!=\"\"}) by (pod)", + "expr": "sum(container_memory_working_set_bytes{cluster=\"$cluster\", node=\"$node\",container!=\"\"}) by (pod)", "format": "table", "instant": true, "intervalFactor": 2, @@ -4077,7 +3113,7 @@ data: "step": 10 }, { - "expr": "sum(kube_pod_container_resource_requests_memory_bytes{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)", + "expr": "sum(kube_pod_container_resource_requests_memory_bytes{cluster=\"$cluster\", node=\"$node\"}) by (pod)", "format": "table", "instant": true, "intervalFactor": 2, @@ -4086,7 +3122,7 @@ data: "step": 10 }, { - "expr": "sum(container_memory_usage_bytes{cluster=\"$cluster\", namespace=\"$namespace\",container!=\"\"}) by (pod) / sum(kube_pod_container_resource_requests_memory_bytes{namespace=\"$namespace\"}) by (pod)", + "expr": "sum(container_memory_working_set_bytes{cluster=\"$cluster\", node=\"$node\",container!=\"\"}) by (pod) / sum(kube_pod_container_resource_requests_memory_bytes{node=\"$node\"}) by (pod)", "format": "table", "instant": true, "intervalFactor": 2, @@ -4095,7 +3131,7 @@ data: "step": 10 }, { - "expr": "sum(kube_pod_container_resource_limits_memory_bytes{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)", + "expr": "sum(kube_pod_container_resource_limits_memory_bytes{cluster=\"$cluster\", node=\"$node\"}) by (pod)", "format": "table", "instant": true, "intervalFactor": 2, @@ -4104,7 +3140,7 @@ data: "step": 10 }, { - "expr": "sum(container_memory_usage_bytes{cluster=\"$cluster\", namespace=\"$namespace\",container!=\"\"}) by (pod) / sum(kube_pod_container_resource_limits_memory_bytes{namespace=\"$namespace\"}) by (pod)", + "expr": "sum(container_memory_working_set_bytes{cluster=\"$cluster\", node=\"$node\",container!=\"\"}) by (pod) / sum(kube_pod_container_resource_limits_memory_bytes{node=\"$node\"}) by (pod)", "format": "table", "instant": true, "intervalFactor": 2, @@ -4113,7 +3149,7 @@ data: "step": 10 }, { - "expr": "sum(container_memory_rss{cluster=\"$cluster\", namespace=\"$namespace\",container!=\"\"}) by (pod)", + "expr": "sum(container_memory_rss{cluster=\"$cluster\", node=\"$node\",container!=\"\"}) by (pod)", "format": "table", "instant": true, "intervalFactor": 2, @@ -4122,7 +3158,7 @@ data: "step": 10 }, { - "expr": "sum(container_memory_cache{cluster=\"$cluster\", namespace=\"$namespace\",container!=\"\"}) by (pod)", + "expr": "sum(container_memory_cache{cluster=\"$cluster\", node=\"$node\",container!=\"\"}) by (pod)", "format": "table", "instant": true, "intervalFactor": 2, @@ -4131,7 +3167,7 @@ data: "step": 10 }, { - "expr": "sum(container_memory_swap{cluster=\"$cluster\", namespace=\"$namespace\",container!=\"\"}) by (pod)", + "expr": "sum(container_memory_swap{cluster=\"$cluster\", node=\"$node\",container!=\"\"}) by (pod)", "format": "table", "instant": true, "intervalFactor": 2, @@ -4249,13 +3285,13 @@ data: "datasource": "$datasource", "hide": 0, "includeAll": false, - "label": "namespace", + "label": "node", "multi": false, - "name": "namespace", + "name": "node", "options": [ ], - "query": "label_values(kube_pod_info{cluster=\"$cluster\"}, namespace)", + "query": "label_values(kube_pod_info{cluster=\"$cluster\"}, node)", "refresh": 1, "regex": "", "sort": 2, @@ -4299,8 +3335,8 @@ data: ] }, "timezone": "", - "title": "Kubernetes / Compute Resources / Namespace (Pods)", - "uid": "85a562078cdf77779eaa1add43ccec1e", + "title": "Kubernetes / Compute Resources / Node (Pods)", + "uid": "200ac8fdbfbb74b39aff88118e4d1c2c", "version": 0 } k8s-resources-pod.json: |- @@ -4361,7 +3397,7 @@ data: "steppedLine": false, "targets": [ { - "expr": "sum(namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{namespace=\"$namespace\", pod=\"$pod\", container!=\"POD\", cluster=\"$cluster\"}) by (container)", + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{namespace=\"$namespace\", pod=\"$pod\", container!=\"POD\", cluster=\"$cluster\"}) by (container)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{container}}", @@ -4590,7 +3626,7 @@ data: ], "targets": [ { - "expr": "sum(namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\", container!=\"POD\"}) by (container)", + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\", container!=\"POD\"}) by (container)", "format": "table", "instant": true, "intervalFactor": 2, @@ -4608,7 +3644,7 @@ data: "step": 10 }, { - "expr": "sum(namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container) / sum(kube_pod_container_resource_requests_cpu_cores{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container)", + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container) / sum(kube_pod_container_resource_requests_cpu_cores{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container)", "format": "table", "instant": true, "intervalFactor": 2, @@ -4626,7 +3662,7 @@ data: "step": 10 }, { - "expr": "sum(namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container) / sum(kube_pod_container_resource_limits_cpu_cores{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container)", + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container) / sum(kube_pod_container_resource_limits_cpu_cores{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container)", "format": "table", "instant": true, "intervalFactor": 2, @@ -4972,7 +4008,7 @@ data: "unit": "bytes" }, { - "alias": "Memory Usage (Swap", + "alias": "Memory Usage (Swap)", "colorMode": null, "colors": [ @@ -5025,7 +4061,7 @@ data: ], "targets": [ { - "expr": "sum(container_memory_usage_bytes{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\", container!=\"POD\", container!=\"\"}) by (container)", + "expr": "sum(container_memory_working_set_bytes{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\", container!=\"POD\", container!=\"\"}) by (container)", "format": "table", "instant": true, "intervalFactor": 2, @@ -5043,7 +4079,7 @@ data: "step": 10 }, { - "expr": "sum(container_memory_usage_bytes{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container) / sum(kube_pod_container_resource_requests_memory_bytes{namespace=\"$namespace\", pod=\"$pod\"}) by (container)", + "expr": "sum(container_memory_working_set_bytes{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container) / sum(kube_pod_container_resource_requests_memory_bytes{namespace=\"$namespace\", pod=\"$pod\"}) by (container)", "format": "table", "instant": true, "intervalFactor": 2, @@ -5061,7 +4097,7 @@ data: "step": 10 }, { - "expr": "sum(container_memory_usage_bytes{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\", container!=\"\"}) by (container) / sum(kube_pod_container_resource_limits_memory_bytes{namespace=\"$namespace\", pod=\"$pod\"}) by (container)", + "expr": "sum(container_memory_working_set_bytes{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\", container!=\"\"}) by (container) / sum(kube_pod_container_resource_limits_memory_bytes{namespace=\"$namespace\", pod=\"$pod\"}) by (container)", "format": "table", "instant": true, "intervalFactor": 2, @@ -5345,7 +4381,7 @@ data: "steppedLine": false, "targets": [ { - "expr": "sum(\n namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\"}\n * on(namespace,pod)\n group_left(workload, workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", + "expr": "sum(\n node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\"}\n * on(namespace,pod)\n group_left(workload, workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{pod}}", @@ -5548,7 +4584,7 @@ data: "decimals": 2, "link": true, "linkTooltip": "Drill down", - "linkUrl": "/d/6581e46e4e5c7ba40a07646395ef7b23/k8s-resources-pod?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$namespace&var-pod=$__cell", + "linkUrl": "./d/6581e46e4e5c7ba40a07646395ef7b23/k8s-resources-pod?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$namespace&var-pod=$__cell", "pattern": "pod", "thresholds": [ @@ -5574,7 +4610,7 @@ data: ], "targets": [ { - "expr": "sum(\n namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\"}\n * on(namespace,pod)\n group_left(workload, workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", + "expr": "sum(\n node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\"}\n * on(namespace,pod)\n group_left(workload, workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", "format": "table", "instant": true, "intervalFactor": 2, @@ -5592,7 +4628,7 @@ data: "step": 10 }, { - "expr": "sum(\n namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\"}\n * on(namespace,pod)\n group_left(workload, workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n/sum(\n kube_pod_container_resource_requests_cpu_cores{cluster=\"$cluster\", namespace=\"$namespace\"}\n * on(namespace,pod)\n group_left(workload, workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", + "expr": "sum(\n node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\"}\n * on(namespace,pod)\n group_left(workload, workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n/sum(\n kube_pod_container_resource_requests_cpu_cores{cluster=\"$cluster\", namespace=\"$namespace\"}\n * on(namespace,pod)\n group_left(workload, workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", "format": "table", "instant": true, "intervalFactor": 2, @@ -5610,7 +4646,7 @@ data: "step": 10 }, { - "expr": "sum(\n namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\"}\n * on(namespace,pod)\n group_left(workload, workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n/sum(\n kube_pod_container_resource_limits_cpu_cores{cluster=\"$cluster\", namespace=\"$namespace\"}\n * on(namespace,pod)\n group_left(workload, workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", + "expr": "sum(\n node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\"}\n * on(namespace,pod)\n group_left(workload, workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n/sum(\n kube_pod_container_resource_limits_cpu_cores{cluster=\"$cluster\", namespace=\"$namespace\"}\n * on(namespace,pod)\n group_left(workload, workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", "format": "table", "instant": true, "intervalFactor": 2, @@ -5710,7 +4746,7 @@ data: "steppedLine": false, "targets": [ { - "expr": "sum(\n container_memory_usage_bytes{cluster=\"$cluster\", namespace=\"$namespace\", container!=\"\"}\n * on(namespace,pod)\n group_left(workload, workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", + "expr": "sum(\n container_memory_working_set_bytes{cluster=\"$cluster\", namespace=\"$namespace\", container!=\"\"}\n * on(namespace,pod)\n group_left(workload, workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{pod}}", @@ -5913,7 +4949,7 @@ data: "decimals": 2, "link": true, "linkTooltip": "Drill down", - "linkUrl": "/d/6581e46e4e5c7ba40a07646395ef7b23/k8s-resources-pod?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$namespace&var-pod=$__cell", + "linkUrl": "./d/6581e46e4e5c7ba40a07646395ef7b23/k8s-resources-pod?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$namespace&var-pod=$__cell", "pattern": "pod", "thresholds": [ @@ -5939,7 +4975,7 @@ data: ], "targets": [ { - "expr": "sum(\n container_memory_usage_bytes{cluster=\"$cluster\", namespace=\"$namespace\", container!=\"\"}\n * on(namespace,pod)\n group_left(workload, workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", + "expr": "sum(\n container_memory_working_set_bytes{cluster=\"$cluster\", namespace=\"$namespace\", container!=\"\"}\n * on(namespace,pod)\n group_left(workload, workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", "format": "table", "instant": true, "intervalFactor": 2, @@ -5957,7 +4993,7 @@ data: "step": 10 }, { - "expr": "sum(\n container_memory_usage_bytes{cluster=\"$cluster\", namespace=\"$namespace\", container!=\"\"}\n * on(namespace,pod)\n group_left(workload, workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n/sum(\n kube_pod_container_resource_requests_memory_bytes{cluster=\"$cluster\", namespace=\"$namespace\"}\n * on(namespace,pod)\n group_left(workload, workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", + "expr": "sum(\n container_memory_working_set_bytes{cluster=\"$cluster\", namespace=\"$namespace\", container!=\"\"}\n * on(namespace,pod)\n group_left(workload, workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n/sum(\n kube_pod_container_resource_requests_memory_bytes{cluster=\"$cluster\", namespace=\"$namespace\"}\n * on(namespace,pod)\n group_left(workload, workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", "format": "table", "instant": true, "intervalFactor": 2, @@ -5975,7 +5011,7 @@ data: "step": 10 }, { - "expr": "sum(\n container_memory_usage_bytes{cluster=\"$cluster\", namespace=\"$namespace\", container!=\"\"}\n * on(namespace,pod)\n group_left(workload, workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n/sum(\n kube_pod_container_resource_limits_memory_bytes{cluster=\"$cluster\", namespace=\"$namespace\"}\n * on(namespace,pod)\n group_left(workload, workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", + "expr": "sum(\n container_memory_working_set_bytes{cluster=\"$cluster\", namespace=\"$namespace\", container!=\"\"}\n * on(namespace,pod)\n group_left(workload, workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n/sum(\n kube_pod_container_resource_limits_memory_bytes{cluster=\"$cluster\", namespace=\"$namespace\"}\n * on(namespace,pod)\n group_left(workload, workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", "format": "table", "instant": true, "intervalFactor": 2, @@ -6259,7 +5295,7 @@ data: "steppedLine": false, "targets": [ { - "expr": "sum(\n namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\"}\n* on(namespace,pod)\n group_left(workload, workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\"}\n) by (workload, workload_type)\n", + "expr": "sum(\n node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\"}\n* on(namespace,pod)\n group_left(workload, workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\"}\n) by (workload, workload_type)\n", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{workload}} - {{workload_type}}", @@ -6480,7 +5516,7 @@ data: "decimals": 2, "link": true, "linkTooltip": "Drill down", - "linkUrl": "/d/a164a7f0339f99e89cea5cb47e9be617/k8s-resources-workload?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$namespace&var-workload=$__cell&var-type=$__cell_2", + "linkUrl": "./d/a164a7f0339f99e89cea5cb47e9be617/k8s-resources-workload?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$namespace&var-workload=$__cell&var-type=$__cell_2", "pattern": "workload", "thresholds": [ @@ -6533,7 +5569,7 @@ data: "step": 10 }, { - "expr": "sum(\n namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\"}\n* on(namespace,pod)\n group_left(workload, workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\"}\n) by (workload, workload_type)\n", + "expr": "sum(\n node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\"}\n* on(namespace,pod)\n group_left(workload, workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\"}\n) by (workload, workload_type)\n", "format": "table", "instant": true, "intervalFactor": 2, @@ -6551,7 +5587,7 @@ data: "step": 10 }, { - "expr": "sum(\n namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\"}\n* on(namespace,pod)\n group_left(workload, workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\"}\n) by (workload, workload_type)\n/sum(\n kube_pod_container_resource_requests_cpu_cores{cluster=\"$cluster\", namespace=\"$namespace\"}\n* on(namespace,pod)\n group_left(workload, workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\"}\n) by (workload, workload_type)\n", + "expr": "sum(\n node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\"}\n* on(namespace,pod)\n group_left(workload, workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\"}\n) by (workload, workload_type)\n/sum(\n kube_pod_container_resource_requests_cpu_cores{cluster=\"$cluster\", namespace=\"$namespace\"}\n* on(namespace,pod)\n group_left(workload, workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\"}\n) by (workload, workload_type)\n", "format": "table", "instant": true, "intervalFactor": 2, @@ -6569,7 +5605,7 @@ data: "step": 10 }, { - "expr": "sum(\n namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\"}\n* on(namespace,pod)\n group_left(workload, workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\"}\n) by (workload, workload_type)\n/sum(\n kube_pod_container_resource_limits_cpu_cores{cluster=\"$cluster\", namespace=\"$namespace\"}\n* on(namespace,pod)\n group_left(workload, workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\"}\n) by (workload, workload_type)\n", + "expr": "sum(\n node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\"}\n* on(namespace,pod)\n group_left(workload, workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\"}\n) by (workload, workload_type)\n/sum(\n kube_pod_container_resource_limits_cpu_cores{cluster=\"$cluster\", namespace=\"$namespace\"}\n* on(namespace,pod)\n group_left(workload, workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\"}\n) by (workload, workload_type)\n", "format": "table", "instant": true, "intervalFactor": 2, @@ -6669,7 +5705,7 @@ data: "steppedLine": false, "targets": [ { - "expr": "sum(\n container_memory_usage_bytes{cluster=\"$cluster\", namespace=\"$namespace\", container!=\"\"}\n * on(namespace,pod)\n group_left(workload, workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\"}\n) by (workload, workload_type)\n", + "expr": "sum(\n container_memory_working_set_bytes{cluster=\"$cluster\", namespace=\"$namespace\", container!=\"\"}\n * on(namespace,pod)\n group_left(workload, workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\"}\n) by (workload, workload_type)\n", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{workload}} - {{workload_type}}", @@ -6890,7 +5926,7 @@ data: "decimals": 2, "link": true, "linkTooltip": "Drill down", - "linkUrl": "/d/a164a7f0339f99e89cea5cb47e9be617/k8s-resources-workload?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$namespace&var-workload=$__cell&var-type=$__cell_2", + "linkUrl": "./d/a164a7f0339f99e89cea5cb47e9be617/k8s-resources-workload?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$namespace&var-workload=$__cell&var-type=$__cell_2", "pattern": "workload", "thresholds": [ @@ -6943,7 +5979,7 @@ data: "step": 10 }, { - "expr": "sum(\n container_memory_usage_bytes{cluster=\"$cluster\", namespace=\"$namespace\", container!=\"\"}\n * on(namespace,pod)\n group_left(workload, workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\"}\n) by (workload, workload_type)\n", + "expr": "sum(\n container_memory_working_set_bytes{cluster=\"$cluster\", namespace=\"$namespace\", container!=\"\"}\n * on(namespace,pod)\n group_left(workload, workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\"}\n) by (workload, workload_type)\n", "format": "table", "instant": true, "intervalFactor": 2, @@ -6961,7 +5997,7 @@ data: "step": 10 }, { - "expr": "sum(\n container_memory_usage_bytes{cluster=\"$cluster\", namespace=\"$namespace\", container!=\"\"}\n * on(namespace,pod)\n group_left(workload, workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\"}\n) by (workload, workload_type)\n/sum(\n kube_pod_container_resource_requests_memory_bytes{cluster=\"$cluster\", namespace=\"$namespace\"}\n* on(namespace,pod)\n group_left(workload, workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\"}\n) by (workload, workload_type)\n", + "expr": "sum(\n container_memory_working_set_bytes{cluster=\"$cluster\", namespace=\"$namespace\", container!=\"\"}\n * on(namespace,pod)\n group_left(workload, workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\"}\n) by (workload, workload_type)\n/sum(\n kube_pod_container_resource_requests_memory_bytes{cluster=\"$cluster\", namespace=\"$namespace\"}\n* on(namespace,pod)\n group_left(workload, workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\"}\n) by (workload, workload_type)\n", "format": "table", "instant": true, "intervalFactor": 2, @@ -6979,7 +6015,7 @@ data: "step": 10 }, { - "expr": "sum(\n container_memory_usage_bytes{cluster=\"$cluster\", namespace=\"$namespace\", container!=\"\"}\n * on(namespace,pod)\n group_left(workload, workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\"}\n) by (workload, workload_type)\n/sum(\n kube_pod_container_resource_limits_memory_bytes{cluster=\"$cluster\", namespace=\"$namespace\"}\n* on(namespace,pod)\n group_left(workload, workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\"}\n) by (workload, workload_type)\n", + "expr": "sum(\n container_memory_working_set_bytes{cluster=\"$cluster\", namespace=\"$namespace\", container!=\"\"}\n * on(namespace,pod)\n group_left(workload, workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\"}\n) by (workload, workload_type)\n/sum(\n kube_pod_container_resource_limits_memory_bytes{cluster=\"$cluster\", namespace=\"$namespace\"}\n* on(namespace,pod)\n group_left(workload, workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\"}\n) by (workload, workload_type)\n", "format": "table", "instant": true, "intervalFactor": 2, @@ -7151,3 +6187,7 @@ data: "uid": "a87fb0d919ec0ea5f6543124e16c42a5", "version": 0 } +kind: ConfigMap +metadata: + name: grafana-dashboards-k8s-resources + namespace: monitoring diff --git a/addons/grafana/dashboards-k8s.yaml b/addons/grafana/dashboards-k8s.yaml index 0f5a8c2eb..299b919f0 100644 --- a/addons/grafana/dashboards-k8s.yaml +++ b/addons/grafana/dashboards-k8s.yaml @@ -1,8 +1,4 @@ apiVersion: v1 -kind: ConfigMap -metadata: - name: grafana-dashboards-k8s - namespace: monitoring data: apiserver.json: |- { @@ -3052,7 +3048,7 @@ data: "refId": "C" }, { - "expr": "sum by(container) (container_memory_cache{job=\"kubernetes-cadvisor\", namespace=\"$namespace\", pod=~\"$pod\", container=~\"$container\", container!=\"POD\"})", + "expr": "sum by(container) (container_memory_cache{job=\"kubernetes-cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"$pod\", container=~\"$container\", container!=\"POD\"})", "format": "time_series", "intervalFactor": 2, "legendFormat": "Cache: {{ container }}", @@ -5471,7 +5467,7 @@ data: "options": [ ], - "query": "label_values(kube_statefulset_metadata_generation{job=\"kube-state-metrics\"}, namespace)", + "query": "label_values(kube_statefulset_metadata_generation{job=\"kube-state-metrics\", cluster=\"$cluster\"}, namespace)", "refresh": 2, "regex": "", "sort": 0, @@ -5497,7 +5493,7 @@ data: "options": [ ], - "query": "label_values(kube_statefulset_metadata_generation{job=\"kube-state-metrics\", namespace=\"$namespace\"}, statefulset)", + "query": "label_values(kube_statefulset_metadata_generation{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\"}, statefulset)", "refresh": 2, "regex": "", "sort": 0, @@ -5545,3 +5541,7 @@ data: "uid": "a31c1f46e6f727cb37c0d731a7245005", "version": 0 } +kind: ConfigMap +metadata: + name: grafana-dashboards-k8s + namespace: monitoring diff --git a/addons/grafana/dashboards-nginx-ingress.yaml b/addons/grafana/dashboards-nginx-ingress.yaml index 0f7595206..c3a8fe2a6 100644 --- a/addons/grafana/dashboards-nginx-ingress.yaml +++ b/addons/grafana/dashboards-nginx-ingress.yaml @@ -1,8 +1,4 @@ apiVersion: v1 -kind: ConfigMap -metadata: - name: grafana-dashboards-nginx-ingress - namespace: monitoring data: nginx.json: |- { @@ -1057,3 +1053,7 @@ data: "uid": "f4af03eca476c08ecf2b5cf15fd60168", "version": 0 } +kind: ConfigMap +metadata: + name: grafana-dashboards-nginx-ingress + namespace: monitoring diff --git a/addons/grafana/dashboards-prom.yaml b/addons/grafana/dashboards-prom.yaml index b1f4c06c5..0c97844cb 100644 --- a/addons/grafana/dashboards-prom.yaml +++ b/addons/grafana/dashboards-prom.yaml @@ -1,8 +1,4 @@ apiVersion: v1 -kind: ConfigMap -metadata: - name: grafana-dashboards-prom - namespace: monitoring data: prometheus-remote-write.json: |- { @@ -2158,3 +2154,7 @@ data: "uid": "", "version": 0 } +kind: ConfigMap +metadata: + name: grafana-dashboards-prom + namespace: monitoring diff --git a/addons/prometheus/rules.yaml b/addons/prometheus/rules.yaml index 22e54a912..51f5aa9f8 100644 --- a/addons/prometheus/rules.yaml +++ b/addons/prometheus/rules.yaml @@ -1,8 +1,4 @@ apiVersion: v1 -kind: ConfigMap -metadata: - name: prometheus-rules - namespace: monitoring data: etcd.yaml: |- { @@ -10,6 +6,17 @@ data: { "name": "etcd", "rules": [ + { + "alert": "etcdMembersDown", + "annotations": { + "message": "etcd cluster \"{{ $labels.job }}\": members are down ({{ $value }})." + }, + "expr": "max by (job) (\n sum by (job) (up{job=~\".*etcd.*\"} == bool 0)\nor\n count by (job,endpoint) (\n sum by (job,endpoint,To) (rate(etcd_network_peer_sent_failures_total{job=~\".*etcd.*\"}[3m])) > 0.01\n )\n)\n> 0\n", + "for": "3m", + "labels": { + "severity": "critical" + } + }, { "alert": "etcdInsufficientMembers", "annotations": { @@ -135,27 +142,6 @@ data: } ] } - extra.yaml: |- - { - "groups": [ - { - "name": "extra.rules", - "rules": [ - { - "alert": "InactiveRAIDDisk", - "annotations": { - "message": "{{ $value }} RAID disk(s) on node {{ $labels.instance }} are inactive." - }, - "expr": "node_md_disks - node_md_disks_active > 0", - "for": "10m", - "labels": { - "severity": "warning" - } - } - ] - } - ] - } kube.yaml: |- { "groups": [ @@ -167,21 +153,13 @@ data: "record": "namespace:container_cpu_usage_seconds_total:sum_rate" }, { - "expr": "sum by (namespace, pod, container) (\n rate(container_cpu_usage_seconds_total{job=\"kubernetes-cadvisor\", image!=\"\", container!=\"POD\"}[5m])\n)\n", - "record": "namespace_pod_container:container_cpu_usage_seconds_total:sum_rate" + "expr": "sum by (namespace, pod, container) (\n rate(container_cpu_usage_seconds_total{job=\"kubernetes-cadvisor\", image!=\"\", container!=\"POD\"}[5m])\n) * on (namespace, pod) group_left(node) max by(namespace, pod, node) (kube_pod_info)\n", + "record": "node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate" }, { "expr": "sum(container_memory_usage_bytes{job=\"kubernetes-cadvisor\", image!=\"\", container!=\"POD\"}) by (namespace)\n", "record": "namespace:container_memory_usage_bytes:sum" }, - { - "expr": "sum by (namespace, label_name) (\n sum(rate(container_cpu_usage_seconds_total{job=\"kubernetes-cadvisor\", image!=\"\", container!=\"POD\"}[5m])) by (namespace, pod)\n * on (namespace, pod)\n group_left(label_name) kube_pod_labels{job=\"kube-state-metrics\"}\n)\n", - "record": "namespace:container_cpu_usage_seconds_total:sum_rate" - }, - { - "expr": "sum by (namespace, label_name) (\n sum(container_memory_usage_bytes{job=\"kubernetes-cadvisor\",image!=\"\", container!=\"POD\"}) by (pod, namespace)\n * on (namespace, pod)\n group_left(label_name) kube_pod_labels{job=\"kube-state-metrics\"}\n)\n", - "record": "namespace:container_memory_usage_bytes:sum" - }, { "expr": "sum by (namespace, label_name) (\n sum(kube_pod_container_resource_requests_memory_bytes{job=\"kube-state-metrics\"} * on (endpoint, instance, job, namespace, pod, service) group_left(phase) (kube_pod_status_phase{phase=~\"^(Pending|Running)$\"} == 1)) by (namespace, pod)\n * on (namespace, pod)\n group_left(label_name) kube_pod_labels{job=\"kube-state-metrics\"}\n)\n", "record": "namespace:kube_pod_container_resource_requests_memory_bytes:sum" @@ -322,117 +300,9 @@ data: "expr": "count by (node) (sum by (node, cpu) (\n node_cpu_seconds_total{job=\"node-exporter\"}\n* on (namespace, pod) group_left(node)\n node_namespace_pod:kube_pod_info:\n))\n", "record": "node:node_num_cpu:sum" }, - { - "expr": "1 - avg(rate(node_cpu_seconds_total{job=\"node-exporter\",mode=\"idle\"}[1m]))\n", - "record": ":node_cpu_utilisation:avg1m" - }, - { - "expr": "1 - avg by (node) (\n rate(node_cpu_seconds_total{job=\"node-exporter\",mode=\"idle\"}[1m])\n* on (namespace, pod) group_left(node)\n node_namespace_pod:kube_pod_info:)\n", - "record": "node:node_cpu_utilisation:avg1m" - }, - { - "expr": "node:node_cpu_utilisation:avg1m\n *\nnode:node_num_cpu:sum\n /\nscalar(sum(node:node_num_cpu:sum))\n", - "record": "node:cluster_cpu_utilisation:ratio" - }, - { - "expr": "sum(node_load1{job=\"node-exporter\"})\n/\nsum(node:node_num_cpu:sum)\n", - "record": ":node_cpu_saturation_load1:" - }, - { - "expr": "sum by (node) (\n node_load1{job=\"node-exporter\"}\n* on (namespace, pod) group_left(node)\n node_namespace_pod:kube_pod_info:\n)\n/\nnode:node_num_cpu:sum\n", - "record": "node:node_cpu_saturation_load1:" - }, - { - "expr": "1 -\nsum(node_memory_MemFree_bytes{job=\"node-exporter\"} + node_memory_Cached_bytes{job=\"node-exporter\"} + node_memory_Buffers_bytes{job=\"node-exporter\"})\n/\nsum(node_memory_MemTotal_bytes{job=\"node-exporter\"})\n", - "record": ":node_memory_utilisation:" - }, { "expr": "sum(node_memory_MemFree_bytes{job=\"node-exporter\"} + node_memory_Cached_bytes{job=\"node-exporter\"} + node_memory_Buffers_bytes{job=\"node-exporter\"})\n", "record": ":node_memory_MemFreeCachedBuffers_bytes:sum" - }, - { - "expr": "sum(node_memory_MemTotal_bytes{job=\"node-exporter\"})\n", - "record": ":node_memory_MemTotal_bytes:sum" - }, - { - "expr": "sum by (node) (\n (node_memory_MemFree_bytes{job=\"node-exporter\"} + node_memory_Cached_bytes{job=\"node-exporter\"} + node_memory_Buffers_bytes{job=\"node-exporter\"})\n * on (namespace, pod) group_left(node)\n node_namespace_pod:kube_pod_info:\n)\n", - "record": "node:node_memory_bytes_available:sum" - }, - { - "expr": "sum by (node) (\n node_memory_MemTotal_bytes{job=\"node-exporter\"}\n * on (namespace, pod) group_left(node)\n node_namespace_pod:kube_pod_info:\n)\n", - "record": "node:node_memory_bytes_total:sum" - }, - { - "expr": "(node:node_memory_bytes_total:sum - node:node_memory_bytes_available:sum)\n/\nnode:node_memory_bytes_total:sum\n", - "record": "node:node_memory_utilisation:ratio" - }, - { - "expr": "(node:node_memory_bytes_total:sum - node:node_memory_bytes_available:sum)\n/\nscalar(sum(node:node_memory_bytes_total:sum))\n", - "record": "node:cluster_memory_utilisation:ratio" - }, - { - "expr": "1e3 * sum(\n (rate(node_vmstat_pgpgin{job=\"node-exporter\"}[1m])\n + rate(node_vmstat_pgpgout{job=\"node-exporter\"}[1m]))\n)\n", - "record": ":node_memory_swap_io_bytes:sum_rate" - }, - { - "expr": "1 -\nsum by (node) (\n (node_memory_MemFree_bytes{job=\"node-exporter\"} + node_memory_Cached_bytes{job=\"node-exporter\"} + node_memory_Buffers_bytes{job=\"node-exporter\"})\n* on (namespace, pod) group_left(node)\n node_namespace_pod:kube_pod_info:\n)\n/\nsum by (node) (\n node_memory_MemTotal_bytes{job=\"node-exporter\"}\n* on (namespace, pod) group_left(node)\n node_namespace_pod:kube_pod_info:\n)\n", - "record": "node:node_memory_utilisation:" - }, - { - "expr": "1 - (node:node_memory_bytes_available:sum / node:node_memory_bytes_total:sum)\n", - "record": "node:node_memory_utilisation_2:" - }, - { - "expr": "1e3 * sum by (node) (\n (rate(node_vmstat_pgpgin{job=\"node-exporter\"}[1m])\n + rate(node_vmstat_pgpgout{job=\"node-exporter\"}[1m]))\n * on (namespace, pod) group_left(node)\n node_namespace_pod:kube_pod_info:\n)\n", - "record": "node:node_memory_swap_io_bytes:sum_rate" - }, - { - "expr": "avg(irate(node_disk_io_time_seconds_total{job=\"node-exporter\",device=~\"nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+\"}[1m]))\n", - "record": ":node_disk_utilisation:avg_irate" - }, - { - "expr": "avg by (node) (\n irate(node_disk_io_time_seconds_total{job=\"node-exporter\",device=~\"nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+\"}[1m])\n* on (namespace, pod) group_left(node)\n node_namespace_pod:kube_pod_info:\n)\n", - "record": "node:node_disk_utilisation:avg_irate" - }, - { - "expr": "avg(irate(node_disk_io_time_weighted_seconds_total{job=\"node-exporter\",device=~\"nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+\"}[1m]))\n", - "record": ":node_disk_saturation:avg_irate" - }, - { - "expr": "avg by (node) (\n irate(node_disk_io_time_weighted_seconds_total{job=\"node-exporter\",device=~\"nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+\"}[1m])\n* on (namespace, pod) group_left(node)\n node_namespace_pod:kube_pod_info:\n)\n", - "record": "node:node_disk_saturation:avg_irate" - }, - { - "expr": "max by (instance, namespace, pod, device) ((node_filesystem_size_bytes{fstype=~\"ext[234]|btrfs|xfs|zfs\"}\n- node_filesystem_avail_bytes{fstype=~\"ext[234]|btrfs|xfs|zfs\"})\n/ node_filesystem_size_bytes{fstype=~\"ext[234]|btrfs|xfs|zfs\"})\n", - "record": "node:node_filesystem_usage:" - }, - { - "expr": "max by (instance, namespace, pod, device) (node_filesystem_avail_bytes{fstype=~\"ext[234]|btrfs|xfs|zfs\"} / node_filesystem_size_bytes{fstype=~\"ext[234]|btrfs|xfs|zfs\"})\n", - "record": "node:node_filesystem_avail:" - }, - { - "expr": "sum(irate(node_network_receive_bytes_total{job=\"node-exporter\",device!~\"veth.+\"}[1m])) +\nsum(irate(node_network_transmit_bytes_total{job=\"node-exporter\",device!~\"veth.+\"}[1m]))\n", - "record": ":node_net_utilisation:sum_irate" - }, - { - "expr": "sum by (node) (\n (irate(node_network_receive_bytes_total{job=\"node-exporter\",device!~\"veth.+\"}[1m]) +\n irate(node_network_transmit_bytes_total{job=\"node-exporter\",device!~\"veth.+\"}[1m]))\n* on (namespace, pod) group_left(node)\n node_namespace_pod:kube_pod_info:\n)\n", - "record": "node:node_net_utilisation:sum_irate" - }, - { - "expr": "sum(irate(node_network_receive_drop_total{job=\"node-exporter\",device!~\"veth.+\"}[1m])) +\nsum(irate(node_network_transmit_drop_total{job=\"node-exporter\",device!~\"veth.+\"}[1m]))\n", - "record": ":node_net_saturation:sum_irate" - }, - { - "expr": "sum by (node) (\n (irate(node_network_receive_drop_total{job=\"node-exporter\",device!~\"veth.+\"}[1m]) +\n irate(node_network_transmit_drop_total{job=\"node-exporter\",device!~\"veth.+\"}[1m]))\n* on (namespace, pod) group_left(node)\n node_namespace_pod:kube_pod_info:\n)\n", - "record": "node:node_net_saturation:sum_irate" - }, - { - "expr": "max(\n max(\n kube_pod_info{job=\"kube-state-metrics\", host_ip!=\"\"}\n ) by (node, host_ip)\n * on (host_ip) group_right (node)\n label_replace(\n (max(node_filesystem_files{job=\"node-exporter\", mountpoint=\"/\"}) by (instance)), \"host_ip\", \"$1\", \"instance\", \"(.*):.*\"\n )\n) by (node)\n", - "record": "node:node_inodes_total:" - }, - { - "expr": "max(\n max(\n kube_pod_info{job=\"kube-state-metrics\", host_ip!=\"\"}\n ) by (node, host_ip)\n * on (host_ip) group_right (node)\n label_replace(\n (max(node_filesystem_files_free{job=\"node-exporter\", mountpoint=\"/\"}) by (instance)), \"host_ip\", \"$1\", \"instance\", \"(.*):.*\"\n )\n) by (node)\n", - "record": "node:node_inodes_free:" } ] }, @@ -499,7 +369,7 @@ data: "runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubepodcrashlooping" }, "expr": "rate(kube_pod_container_status_restarts_total{job=\"kube-state-metrics\"}[15m]) * 60 * 5 > 0\n", - "for": "1h", + "for": "15m", "labels": { "severity": "critical" } @@ -507,11 +377,11 @@ data: { "alert": "KubePodNotReady", "annotations": { - "message": "Pod {{ $labels.namespace }}/{{ $labels.pod }} has been in a non-ready state for longer than an hour.", + "message": "Pod {{ $labels.namespace }}/{{ $labels.pod }} has been in a non-ready state for longer than 15 minutes.", "runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubepodnotready" }, - "expr": "sum by (namespace, pod) (kube_pod_status_phase{job=\"kube-state-metrics\", phase=~\"Failed|Pending|Unknown\"}) > 0\n", - "for": "1h", + "expr": "sum by (namespace, pod) (kube_pod_status_phase{job=\"kube-state-metrics\", phase=~\"Failed|Pending|Unknown\"} * on(namespace, pod) group_left(owner_kind) kube_pod_owner{owner_kind!=\"Job\"}) > 0\n", + "for": "15m", "labels": { "severity": "critical" } @@ -531,11 +401,11 @@ data: { "alert": "KubeDeploymentReplicasMismatch", "annotations": { - "message": "Deployment {{ $labels.namespace }}/{{ $labels.deployment }} has not matched the expected number of replicas for longer than an hour.", + "message": "Deployment {{ $labels.namespace }}/{{ $labels.deployment }} has not matched the expected number of replicas for longer than 15 minutes.", "runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubedeploymentreplicasmismatch" }, "expr": "kube_deployment_spec_replicas{job=\"kube-state-metrics\"}\n !=\nkube_deployment_status_replicas_available{job=\"kube-state-metrics\"}\n", - "for": "1h", + "for": "15m", "labels": { "severity": "critical" } @@ -579,10 +449,10 @@ data: { "alert": "KubeDaemonSetRolloutStuck", "annotations": { - "message": "Only {{ $value }}% of the desired Pods of DaemonSet {{ $labels.namespace }}/{{ $labels.daemonset }} are scheduled and ready.", + "message": "Only {{ $value | humanizePercentage }} of the desired Pods of DaemonSet {{ $labels.namespace }}/{{ $labels.daemonset }} are scheduled and ready.", "runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubedaemonsetrolloutstuck" }, - "expr": "kube_daemonset_status_number_ready{job=\"kube-state-metrics\"}\n /\nkube_daemonset_status_desired_number_scheduled{job=\"kube-state-metrics\"} * 100 < 100\n", + "expr": "kube_daemonset_status_number_ready{job=\"kube-state-metrics\"}\n /\nkube_daemonset_status_desired_number_scheduled{job=\"kube-state-metrics\"} < 1.00\n", "for": "15m", "labels": { "severity": "critical" @@ -642,8 +512,32 @@ data: "message": "Job {{ $labels.namespace }}/{{ $labels.job_name }} failed to complete.", "runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubejobfailed" }, - "expr": "kube_job_status_failed{job=\"kube-state-metrics\"} > 0\n", - "for": "1h", + "expr": "kube_job_failed{job=\"kube-state-metrics\"} > 0\n", + "for": "15m", + "labels": { + "severity": "warning" + } + }, + { + "alert": "KubeHpaReplicasMismatch", + "annotations": { + "message": "HPA {{ $labels.namespace }}/{{ $labels.hpa }} has not matched the desired number of replicas for longer than 15 minutes.", + "runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubehpareplicasmismatch" + }, + "expr": "(kube_hpa_status_desired_replicas{job=\"kube-state-metrics\"}\n !=\nkube_hpa_status_current_replicas{job=\"kube-state-metrics\"})\n and\nchanges(kube_hpa_status_current_replicas[15m]) == 0\n", + "for": "15m", + "labels": { + "severity": "warning" + } + }, + { + "alert": "KubeHpaMaxedOut", + "annotations": { + "message": "HPA {{ $labels.namespace }}/{{ $labels.hpa }} has been running at max replicas for longer than 15 minutes.", + "runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubehpamaxedout" + }, + "expr": "kube_hpa_status_current_replicas{job=\"kube-state-metrics\"}\n ==\nkube_hpa_spec_max_replicas{job=\"kube-state-metrics\"}\n", + "for": "15m", "labels": { "severity": "warning" } @@ -704,10 +598,10 @@ data: { "alert": "KubeQuotaExceeded", "annotations": { - "message": "Namespace {{ $labels.namespace }} is using {{ printf \"%0.0f\" $value }}% of its {{ $labels.resource }} quota.", + "message": "Namespace {{ $labels.namespace }} is using {{ $value | humanizePercentage }} of its {{ $labels.resource }} quota.", "runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubequotaexceeded" }, - "expr": "100 * kube_resourcequota{job=\"kube-state-metrics\", type=\"used\"}\n / ignoring(instance, job, type)\n(kube_resourcequota{job=\"kube-state-metrics\", type=\"hard\"} > 0)\n > 90\n", + "expr": "kube_resourcequota{job=\"kube-state-metrics\", type=\"used\"}\n / ignoring(instance, job, type)\n(kube_resourcequota{job=\"kube-state-metrics\", type=\"hard\"} > 0)\n > 0.90\n", "for": "15m", "labels": { "severity": "warning" @@ -716,10 +610,10 @@ data: { "alert": "CPUThrottlingHigh", "annotations": { - "message": "{{ printf \"%0.0f\" $value }}% throttling of CPU in namespace {{ $labels.namespace }} for container {{ $labels.container }} in pod {{ $labels.pod }}.", + "message": "{{ $value | humanizePercentage }} throttling of CPU in namespace {{ $labels.namespace }} for container {{ $labels.container }} in pod {{ $labels.pod }}.", "runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-cputhrottlinghigh" }, - "expr": "100 * sum(increase(container_cpu_cfs_throttled_periods_total{container!=\"\", }[5m])) by (container, pod, namespace)\n /\nsum(increase(container_cpu_cfs_periods_total{}[5m])) by (container, pod, namespace)\n > 100 \n", + "expr": "sum(increase(container_cpu_cfs_throttled_periods_total{container!=\"\", }[5m])) by (container, pod, namespace)\n /\nsum(increase(container_cpu_cfs_periods_total{}[5m])) by (container, pod, namespace)\n > ( 100 / 100 )\n", "for": "15m", "labels": { "severity": "warning" @@ -733,10 +627,10 @@ data: { "alert": "KubePersistentVolumeUsageCritical", "annotations": { - "message": "The PersistentVolume claimed by {{ $labels.persistentvolumeclaim }} in Namespace {{ $labels.namespace }} is only {{ printf \"%0.2f\" $value }}% free.", + "message": "The PersistentVolume claimed by {{ $labels.persistentvolumeclaim }} in Namespace {{ $labels.namespace }} is only {{ $value | humanizePercentage }} free.", "runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubepersistentvolumeusagecritical" }, - "expr": "100 * kubelet_volume_stats_available_bytes{job=\"kubelet\"}\n /\nkubelet_volume_stats_capacity_bytes{job=\"kubelet\"}\n < 3\n", + "expr": "kubelet_volume_stats_available_bytes{job=\"kubelet\"}\n /\nkubelet_volume_stats_capacity_bytes{job=\"kubelet\"}\n < 0.03\n", "for": "1m", "labels": { "severity": "critical" @@ -745,10 +639,10 @@ data: { "alert": "KubePersistentVolumeFullInFourDays", "annotations": { - "message": "Based on recent sampling, the PersistentVolume claimed by {{ $labels.persistentvolumeclaim }} in Namespace {{ $labels.namespace }} is expected to fill up within four days. Currently {{ printf \"%0.2f\" $value }}% is available.", + "message": "Based on recent sampling, the PersistentVolume claimed by {{ $labels.persistentvolumeclaim }} in Namespace {{ $labels.namespace }} is expected to fill up within four days. Currently {{ $value | humanizePercentage }} is available.", "runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubepersistentvolumefullinfourdays" }, - "expr": "100 * (\n kubelet_volume_stats_available_bytes{job=\"kubelet\"}\n /\n kubelet_volume_stats_capacity_bytes{job=\"kubelet\"}\n) < 15\nand\npredict_linear(kubelet_volume_stats_available_bytes{job=\"kubelet\"}[6h], 4 * 24 * 3600) < 0\n", + "expr": "(\n kubelet_volume_stats_available_bytes{job=\"kubelet\"}\n /\n kubelet_volume_stats_capacity_bytes{job=\"kubelet\"}\n) < 0.15\nand\npredict_linear(kubelet_volume_stats_available_bytes{job=\"kubelet\"}[6h], 4 * 24 * 3600) < 0\n", "for": "5m", "labels": { "severity": "critical" @@ -774,11 +668,11 @@ data: { "alert": "KubeNodeNotReady", "annotations": { - "message": "{{ $labels.node }} has been unready for more than an hour.", + "message": "{{ $labels.node }} has been unready for more than 15 minutes.", "runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubenodenotready" }, "expr": "kube_node_status_condition{job=\"kube-state-metrics\",condition=\"Ready\",status=\"true\"} == 0\n", - "for": "1h", + "for": "15m", "labels": { "severity": "warning" } @@ -789,19 +683,7 @@ data: "message": "There are {{ $value }} different semantic versions of Kubernetes components running.", "runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubeversionmismatch" }, - "expr": "count(count by (gitVersion) (label_replace(kubernetes_build_info{job!=\"coredns\"},\"gitVersion\",\"$1\",\"gitVersion\",\"(v[0-9]*.[0-9]*.[0-9]*).*\"))) > 1\n", - "for": "1h", - "labels": { - "severity": "warning" - } - }, - { - "alert": "KubeClientErrors", - "annotations": { - "message": "Kubernetes API server client '{{ $labels.job }}/{{ $labels.instance }}' is experiencing {{ printf \"%0.0f\" $value }}% errors.'", - "runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubeclienterrors" - }, - "expr": "(sum(rate(rest_client_requests_total{code=~\"5..\"}[5m])) by (instance, job)\n /\nsum(rate(rest_client_requests_total[5m])) by (instance, job))\n* 100 > 1\n", + "expr": "count(count by (gitVersion) (label_replace(kubernetes_build_info{job!~\"kube-dns|coredns\"},\"gitVersion\",\"$1\",\"gitVersion\",\"(v[0-9]*.[0-9]*.[0-9]*).*\"))) > 1\n", "for": "15m", "labels": { "severity": "warning" @@ -810,10 +692,10 @@ data: { "alert": "KubeClientErrors", "annotations": { - "message": "Kubernetes API server client '{{ $labels.job }}/{{ $labels.instance }}' is experiencing {{ printf \"%0.0f\" $value }} errors / second.", + "message": "Kubernetes API server client '{{ $labels.job }}/{{ $labels.instance }}' is experiencing {{ $value | humanizePercentage }} errors.'", "runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubeclienterrors" }, - "expr": "sum(rate(ksm_scrape_error_total{job=\"kube-state-metrics\"}[5m])) by (instance, job) > 0.1\n", + "expr": "(sum(rate(rest_client_requests_total{code=~\"5..\"}[5m])) by (instance, job)\n /\nsum(rate(rest_client_requests_total[5m])) by (instance, job))\n> 0.01\n", "for": "15m", "labels": { "severity": "warning" @@ -822,10 +704,10 @@ data: { "alert": "KubeletTooManyPods", "annotations": { - "message": "Kubelet {{ $labels.instance }} is running {{ $value }} Pods, close to the limit of 110.", + "message": "Kubelet '{{ $labels.node }}' is running at {{ $value | humanizePercentage }} of its Pod capacity.", "runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubelettoomanypods" }, - "expr": "kubelet_running_pod_count{job=\"kubelet\"} > 110 * 0.9\n", + "expr": "max(max(kubelet_running_pod_count{job=\"kubelet\"}) by(instance) * on(instance) group_left(node) kubelet_node_name{job=\"kubelet\"}) by(node) / max(kube_node_status_capacity_pods{job=\"kube-state-metrics\"}) by(node) > 0.95\n", "for": "15m", "labels": { "severity": "warning" @@ -858,10 +740,10 @@ data: { "alert": "KubeAPIErrorsHigh", "annotations": { - "message": "API server is returning errors for {{ $value }}% of requests.", + "message": "API server is returning errors for {{ $value | humanizePercentage }} of requests.", "runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubeapierrorshigh" }, - "expr": "sum(rate(apiserver_request_total{job=\"apiserver\",code=~\"^(?:5..)$\"}[5m]))\n /\nsum(rate(apiserver_request_total{job=\"apiserver\"}[5m])) * 100 > 3\n", + "expr": "sum(rate(apiserver_request_total{job=\"apiserver\",code=~\"^(?:5..)$\"}[5m]))\n /\nsum(rate(apiserver_request_total{job=\"apiserver\"}[5m])) > 0.03\n", "for": "10m", "labels": { "severity": "critical" @@ -870,10 +752,10 @@ data: { "alert": "KubeAPIErrorsHigh", "annotations": { - "message": "API server is returning errors for {{ $value }}% of requests.", + "message": "API server is returning errors for {{ $value | humanizePercentage }} of requests.", "runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubeapierrorshigh" }, - "expr": "sum(rate(apiserver_request_total{job=\"apiserver\",code=~\"^(?:5..)$\"}[5m]))\n /\nsum(rate(apiserver_request_total{job=\"apiserver\"}[5m])) * 100 > 1\n", + "expr": "sum(rate(apiserver_request_total{job=\"apiserver\",code=~\"^(?:5..)$\"}[5m]))\n /\nsum(rate(apiserver_request_total{job=\"apiserver\"}[5m])) > 0.01\n", "for": "10m", "labels": { "severity": "warning" @@ -882,10 +764,10 @@ data: { "alert": "KubeAPIErrorsHigh", "annotations": { - "message": "API server is returning errors for {{ $value }}% of requests for {{ $labels.verb }} {{ $labels.resource }} {{ $labels.subresource }}.", + "message": "API server is returning errors for {{ $value | humanizePercentage }} of requests for {{ $labels.verb }} {{ $labels.resource }} {{ $labels.subresource }}.", "runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubeapierrorshigh" }, - "expr": "sum(rate(apiserver_request_total{job=\"apiserver\",code=~\"^(?:5..)$\"}[5m])) by (resource,subresource,verb)\n /\nsum(rate(apiserver_request_total{job=\"apiserver\"}[5m])) by (resource,subresource,verb) * 100 > 10\n", + "expr": "sum(rate(apiserver_request_total{job=\"apiserver\",code=~\"^(?:5..)$\"}[5m])) by (resource,subresource,verb)\n /\nsum(rate(apiserver_request_total{job=\"apiserver\"}[5m])) by (resource,subresource,verb) > 0.10\n", "for": "10m", "labels": { "severity": "critical" @@ -894,10 +776,10 @@ data: { "alert": "KubeAPIErrorsHigh", "annotations": { - "message": "API server is returning errors for {{ $value }}% of requests for {{ $labels.verb }} {{ $labels.resource }} {{ $labels.subresource }}.", + "message": "API server is returning errors for {{ $value | humanizePercentage }} of requests for {{ $labels.verb }} {{ $labels.resource }} {{ $labels.subresource }}.", "runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubeapierrorshigh" }, - "expr": "sum(rate(apiserver_request_total{job=\"apiserver\",code=~\"^(?:5..)$\"}[5m])) by (resource,subresource,verb)\n /\nsum(rate(apiserver_request_total{job=\"apiserver\"}[5m])) by (resource,subresource,verb) * 100 > 5\n", + "expr": "sum(rate(apiserver_request_total{job=\"apiserver\",code=~\"^(?:5..)$\"}[5m])) by (resource,subresource,verb)\n /\nsum(rate(apiserver_request_total{job=\"apiserver\"}[5m])) by (resource,subresource,verb) > 0.05\n", "for": "10m", "labels": { "severity": "warning" @@ -929,141 +811,6 @@ data: } ] } - kubeprom.yaml: |- - { - "groups": [ - { - "name": "kube-prometheus-node-recording.rules", - "rules": [ - { - "expr": "sum(rate(node_cpu_seconds_total{mode!=\"idle\",mode!=\"iowait\"}[3m])) BY (instance)", - "record": "instance:node_cpu:rate:sum" - }, - { - "expr": "sum((node_filesystem_size_bytes{mountpoint=\"/\"} - node_filesystem_free_bytes{mountpoint=\"/\"})) BY (instance)", - "record": "instance:node_filesystem_usage:sum" - }, - { - "expr": "sum(rate(node_network_receive_bytes_total[3m])) BY (instance)", - "record": "instance:node_network_receive_bytes:rate:sum" - }, - { - "expr": "sum(rate(node_network_transmit_bytes_total[3m])) BY (instance)", - "record": "instance:node_network_transmit_bytes:rate:sum" - }, - { - "expr": "sum(rate(node_cpu_seconds_total{mode!=\"idle\",mode!=\"iowait\"}[5m])) WITHOUT (cpu, mode) / ON(instance) GROUP_LEFT() count(sum(node_cpu_seconds_total) BY (instance, cpu)) BY (instance)", - "record": "instance:node_cpu:ratio" - }, - { - "expr": "sum(rate(node_cpu_seconds_total{mode!=\"idle\",mode!=\"iowait\"}[5m]))", - "record": "cluster:node_cpu:sum_rate5m" - }, - { - "expr": "cluster:node_cpu_seconds_total:rate5m / count(sum(node_cpu_seconds_total) BY (instance, cpu))", - "record": "cluster:node_cpu:ratio" - } - ] - }, - { - "name": "kube-prometheus-node-alerting.rules", - "rules": [ - { - "alert": "NodeDiskRunningFull", - "annotations": { - "message": "Device {{ $labels.device }} of node-exporter {{ $labels.namespace }}/{{ $labels.pod }} will be full within the next 24 hours." - }, - "expr": "(node:node_filesystem_usage: > 0.85) and (predict_linear(node:node_filesystem_avail:[6h], 3600 * 24) < 0)\n", - "for": "30m", - "labels": { - "severity": "warning" - } - }, - { - "alert": "NodeDiskRunningFull", - "annotations": { - "message": "Device {{ $labels.device }} of node-exporter {{ $labels.namespace }}/{{ $labels.pod }} will be full within the next 2 hours." - }, - "expr": "(node:node_filesystem_usage: > 0.85) and (predict_linear(node:node_filesystem_avail:[30m], 3600 * 2) < 0)\n", - "for": "10m", - "labels": { - "severity": "critical" - } - } - ] - }, - { - "name": "node-time", - "rules": [ - { - "alert": "ClockSkewDetected", - "annotations": { - "message": "Clock skew detected on node-exporter {{ $labels.namespace }}/{{ $labels.pod }}. Ensure NTP is configured correctly on this host." - }, - "expr": "abs(node_timex_offset_seconds{job=\"node-exporter\"}) > 0.03\n", - "for": "2m", - "labels": { - "severity": "warning" - } - } - ] - }, - { - "name": "node-network", - "rules": [ - { - "alert": "NetworkReceiveErrors", - "annotations": { - "message": "Network interface \"{{ $labels.device }}\" showing receive errors on node-exporter {{ $labels.namespace }}/{{ $labels.pod }}\"" - }, - "expr": "rate(node_network_receive_errs_total{job=\"node-exporter\",device!~\"veth.+|tunl.+\"}[2m]) > 0\n", - "for": "2m", - "labels": { - "severity": "warning" - } - }, - { - "alert": "NetworkTransmitErrors", - "annotations": { - "message": "Network interface \"{{ $labels.device }}\" showing transmit errors on node-exporter {{ $labels.namespace }}/{{ $labels.pod }}\"" - }, - "expr": "rate(node_network_transmit_errs_total{job=\"node-exporter\",device!~\"veth.+|tunl.+\"}[2m]) > 0\n", - "for": "2m", - "labels": { - "severity": "warning" - } - }, - { - "alert": "NodeNetworkInterfaceFlapping", - "annotations": { - "message": "Network interface \"{{ $labels.device }}\" changing it's up status often on node-exporter {{ $labels.namespace }}/{{ $labels.pod }}\"" - }, - "expr": "changes(node_network_up{job=\"node-exporter\",device!~\"veth.+|tunl.+\"}[2m]) > 2\n", - "for": "2m", - "labels": { - "severity": "warning" - } - } - ] - }, - { - "name": "general.rules", - "rules": [ - { - "alert": "TargetDown", - "annotations": { - "message": "{{ $value }}% of the {{ $labels.job }} targets are down." - }, - "expr": "100 * (count(up == 0) BY (job) / count(up) BY (job)) > 10", - "for": "10m", - "labels": { - "severity": "warning" - } - } - ] - } - ] - } prom.yaml: |- { "groups": [ @@ -1154,18 +901,6 @@ data: "severity": "warning" } }, - { - "alert": "PrometheusTSDBWALCorruptions", - "annotations": { - "description": "Prometheus {{$labels.instance}} has detected {{$value | humanize}} corruptions of the write-ahead log (WAL) over the last 3h.", - "summary": "Prometheus is detecting WAL corruptions." - }, - "expr": "increase(tsdb_wal_corruptions_total{job=\"prometheus\"}[3h]) > 0\n", - "for": "4h", - "labels": { - "severity": "warning" - } - }, { "alert": "PrometheusNotIngestingSamples", "annotations": { @@ -1181,7 +916,7 @@ data: { "alert": "PrometheusDuplicateTimestamps", "annotations": { - "description": "Prometheus {{$labels.instance}} is dropping {{$value | humanize}} samples/s with different values but duplicated timestamp.", + "description": "Prometheus {{$labels.instance}} is dropping {{ printf \"%.4g\" $value }} samples/s with different values but duplicated timestamp.", "summary": "Prometheus is dropping samples with duplicate timestamps." }, "expr": "rate(prometheus_target_scrapes_sample_duplicate_timestamp_total{job=\"prometheus\"}[5m]) > 0\n", @@ -1193,7 +928,7 @@ data: { "alert": "PrometheusOutOfOrderTimestamps", "annotations": { - "description": "Prometheus {{$labels.instance}} is dropping {{$value | humanize}} samples/s with timestamps arriving out of order.", + "description": "Prometheus {{$labels.instance}} is dropping {{ printf \"%.4g\" $value }} samples/s with timestamps arriving out of order.", "summary": "Prometheus drops samples with out-of-order timestamps." }, "expr": "rate(prometheus_target_scrapes_sample_out_of_order_total{job=\"prometheus\"}[5m]) > 0\n", @@ -1226,6 +961,18 @@ data: "severity": "critical" } }, + { + "alert": "PrometheusRemoteWriteDesiredShards", + "annotations": { + "description": "Prometheus {{$labels.instance}} remote write desired shards calculation wants to run {{ printf $value }} shards, which is more than the max of {{ printf `prometheus_remote_storage_shards_max{instance=\"%s\",job=\"prometheus\"}` $labels.instance | query | first | value }}.", + "summary": "Prometheus remote write desired shards calculation wants to run more than configured max shards." + }, + "expr": "# Without max_over_time, failed scrapes could create false negatives, see\n# https://www.robustperception.io/alerting-on-gauges-in-prometheus-2-0 for details.\n(\n max_over_time(prometheus_remote_storage_shards_desired{job=\"prometheus\"}[5m])\n> on(job, instance) group_right\n max_over_time(prometheus_remote_storage_shards_max{job=\"prometheus\"}[5m])\n)\n", + "for": "15m", + "labels": { + "severity": "warning" + } + }, { "alert": "PrometheusRuleFailures", "annotations": { @@ -1254,3 +1001,44 @@ data: } ] } + typhoon.yaml: |- + { + "groups": [ + { + "name": "general.rules", + "rules": [ + { + "alert": "TargetDown", + "annotations": { + "message": "{{ printf \"%.4g\" $value }}% of the {{ $labels.job }} targets are down." + }, + "expr": "100 * (count(up == 0) BY (job, namespace, service) / count(up) BY (job, namespace, service)) > 10", + "for": "10m", + "labels": { + "severity": "warning" + } + } + ] + }, + { + "name": "extra.rules", + "rules": [ + { + "alert": "InactiveRAIDDisk", + "annotations": { + "message": "{{ $value }} RAID disk(s) on node {{ $labels.instance }} are inactive." + }, + "expr": "node_md_disks - node_md_disks_active > 0", + "for": "10m", + "labels": { + "severity": "warning" + } + } + ] + } + ] + } +kind: ConfigMap +metadata: + name: prometheus-rules + namespace: monitoring From de90cb9246e410ae2cb0653da1f45a17ee1dc135 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Sat, 19 Oct 2019 18:23:26 -0700 Subject: [PATCH 253/523] Remove kube-state-metrics addon-resizer * addon-resizer is outdated and has been dropped from kube-state-metrics examples. Those using it should look to the cluster-proportional-vertical-autoscaler. * Eliminate addon-resizer log spew * Remove associated Role and RoleBinding * Also fix kube-state-metrics readinessProbe port --- CHANGES.md | 1 + .../kube-state-metrics/deployment.yaml | 29 +---------------- .../resizer-role-binding.yaml | 13 -------- .../kube-state-metrics/resizer-role.yaml | 31 ------------------- 4 files changed, 2 insertions(+), 72 deletions(-) delete mode 100644 addons/prometheus/exporters/kube-state-metrics/resizer-role-binding.yaml delete mode 100644 addons/prometheus/exporters/kube-state-metrics/resizer-role.yaml diff --git a/CHANGES.md b/CHANGES.md index 7e3f4afe2..604b258c7 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -8,6 +8,7 @@ Notable changes between versions. * Update Prometheus from v2.13.0 to v2.13.1 * Refresh rules, alerts, and dashboards from upstreams +* Remove addon-resizer from kube-state-metrics ([#575](https://github.com/poseidon/typhoon/pull/575)) * Update Grafana from v6.4.2 to v6.4.3 ## v1.16.2 diff --git a/addons/prometheus/exporters/kube-state-metrics/deployment.yaml b/addons/prometheus/exporters/kube-state-metrics/deployment.yaml index c9694eab3..b4f0a47fb 100644 --- a/addons/prometheus/exporters/kube-state-metrics/deployment.yaml +++ b/addons/prometheus/exporters/kube-state-metrics/deployment.yaml @@ -37,33 +37,6 @@ spec: readinessProbe: httpGet: path: / - port: 8080 + port: 8081 initialDelaySeconds: 5 timeoutSeconds: 5 - - name: addon-resizer - image: k8s.gcr.io/addon-resizer:1.8.5 - resources: - limits: - cpu: 100m - memory: 30Mi - requests: - cpu: 100m - memory: 30Mi - env: - - name: MY_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: MY_POD_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - command: - - /pod_nanny - - --container=kube-state-metrics - - --cpu=100m - - --extra-cpu=1m - - --memory=100Mi - - --extra-memory=2Mi - - --threshold=5 - - --deployment=kube-state-metrics diff --git a/addons/prometheus/exporters/kube-state-metrics/resizer-role-binding.yaml b/addons/prometheus/exporters/kube-state-metrics/resizer-role-binding.yaml deleted file mode 100644 index f10a2bccb..000000000 --- a/addons/prometheus/exporters/kube-state-metrics/resizer-role-binding.yaml +++ /dev/null @@ -1,13 +0,0 @@ -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - name: kube-state-metrics - namespace: monitoring -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: Role - name: kube-state-metrics -subjects: -- kind: ServiceAccount - name: kube-state-metrics - namespace: monitoring diff --git a/addons/prometheus/exporters/kube-state-metrics/resizer-role.yaml b/addons/prometheus/exporters/kube-state-metrics/resizer-role.yaml deleted file mode 100644 index 6d12b57c3..000000000 --- a/addons/prometheus/exporters/kube-state-metrics/resizer-role.yaml +++ /dev/null @@ -1,31 +0,0 @@ -apiVersion: rbac.authorization.k8s.io/v1 -kind: Role -metadata: - name: kube-state-metrics - namespace: monitoring -rules: -- apiGroups: - - "" - resources: - - pods - verbs: - - get -- apiGroups: - - extensions - resources: - - deployments - resourceNames: - - kube-state-metrics - verbs: - - get - - update -- apiGroups: - - apps - resources: - - deployments - resourceNames: - - kube-state-metrics - verbs: - - get - - update - From 33d4c2fd68e770c8492a225606742ff76b1e7d97 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Sun, 20 Oct 2019 16:05:09 -0700 Subject: [PATCH 254/523] Add explicit annotation for Prometheus port to scrape * Without the prometheus.io/port annotation, Prometheus service discovery can scrape other Prometheus ports that may be available. * For example, Prometheus sidecars (not included) may be scraped and that may be unintended --- addons/prometheus/service.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/addons/prometheus/service.yaml b/addons/prometheus/service.yaml index 50014e257..c1b9eecf9 100644 --- a/addons/prometheus/service.yaml +++ b/addons/prometheus/service.yaml @@ -5,6 +5,7 @@ metadata: namespace: monitoring annotations: prometheus.io/scrape: 'true' + prometheus.io/port: '9090' spec: type: ClusterIP selector: From eb7b6d39f23cdfeacbec46b63d2f246d2ef8e760 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Sun, 20 Oct 2019 23:16:55 -0700 Subject: [PATCH 255/523] Improve minor aspects of CoreDNS and nginx-ingress dashboards * Add default 10s refresh rate to custom dashboards to match those from Kubernetes * Show labels for "instance" as "pod" for clarity * Add cluster filter for internal use --- addons/grafana/dashboards-coredns.yaml | 32 ++++++++++- addons/grafana/dashboards-nginx-ingress.yaml | 60 ++++++++++++++------ 2 files changed, 72 insertions(+), 20 deletions(-) diff --git a/addons/grafana/dashboards-coredns.yaml b/addons/grafana/dashboards-coredns.yaml index f6e15fe01..eb4395b79 100644 --- a/addons/grafana/dashboards-coredns.yaml +++ b/addons/grafana/dashboards-coredns.yaml @@ -22,7 +22,7 @@ data: "links": [ ], - "refresh": "", + "refresh": "10s", "rows": [ { "collapse": false, @@ -972,17 +972,43 @@ data: "allValue": null, "current": { + }, + "datasource": "$datasource", + "hide": 2, + "includeAll": false, + "label": "cluster", + "multi": false, + "name": "cluster", + "options": [ + + ], + "query": "label_values(kube_pod_info, cluster)", + "refresh": 2, + "regex": "", + "sort": 0, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + }, "datasource": "$datasource", "hide": 0, "includeAll": true, - "label": null, + "label": "pod", "multi": false, "name": "instance", "options": [ ], - "query": "label_values(coredns_build_info{job=\"coredns\"}, instance)", + "query": "label_values(coredns_build_info{cluster=\"$cluster\", job=\"coredns\"}, instance)", "refresh": 2, "regex": "", "sort": 0, diff --git a/addons/grafana/dashboards-nginx-ingress.yaml b/addons/grafana/dashboards-nginx-ingress.yaml index c3a8fe2a6..7af93fa8a 100644 --- a/addons/grafana/dashboards-nginx-ingress.yaml +++ b/addons/grafana/dashboards-nginx-ingress.yaml @@ -22,7 +22,7 @@ data: "links": [ ], - "refresh": "", + "refresh": "10s", "rows": [ { "collapse": false, @@ -90,7 +90,7 @@ data: "tableColumn": "", "targets": [ { - "expr": "round(sum(irate(nginx_ingress_controller_requests{controller_pod=~\"$controller\",controller_class=~\"$controller_class\",namespace=~\"$namespace\"}[2m])), 0.01)", + "expr": "round(sum(irate(nginx_ingress_controller_requests{cluster=~\"$cluster\", controller_pod=~\"$controller\",controller_class=~\"$controller_class\", controller_namespace=~\"$namespace\"}[2m])), 0.01)", "format": "time_series", "intervalFactor": 2, "legendFormat": "", @@ -172,7 +172,7 @@ data: "tableColumn": "", "targets": [ { - "expr": "sum(avg_over_time(nginx_ingress_controller_nginx_process_connections{controller_pod=~\"$controller\",controller_class=~\"$controller_class\",controller_namespace=~\"$namespace\"}[2m]))", + "expr": "sum(avg_over_time(nginx_ingress_controller_nginx_process_connections{cluster=~\"$cluster\", controller_pod=~\"$controller\",controller_class=~\"$controller_class\",controller_namespace=~\"$namespace\"}[2m]))", "format": "time_series", "intervalFactor": 2, "legendFormat": "", @@ -254,7 +254,7 @@ data: "tableColumn": "", "targets": [ { - "expr": "sum(rate(nginx_ingress_controller_requests{controller_pod=~\"$controller\",controller_class=~\"$controller_class\",namespace=~\"$namespace\",status!~\"[4-5].*\"}[2m])) / sum(rate(nginx_ingress_controller_requests{controller_pod=~\"$controller\",controller_class=~\"$controller_class\",namespace=~\"$namespace\"}[2m]))", + "expr": "sum(rate(nginx_ingress_controller_requests{cluster=~\"$cluster\", controller_pod=~\"$controller\",controller_class=~\"$controller_class\",controller_namespace=~\"$namespace\",status!~\"[4-5].*\"}[2m])) / sum(rate(nginx_ingress_controller_requests{cluster=~\"$cluster\", controller_pod=~\"$controller\",controller_class=~\"$controller_class\",controller_namespace=~\"$namespace\"}[2m]))", "format": "time_series", "intervalFactor": 2, "legendFormat": "", @@ -331,7 +331,7 @@ data: "steppedLine": false, "targets": [ { - "expr": "round(sum(irate(nginx_ingress_controller_requests{controller_pod=~\"$controller\",controller_class=~\"$controller_class\",controller_namespace=~\"$namespace\",ingress=~\"$ingress\"}[2m])) by (ingress), 0.01)", + "expr": "round(sum(irate(nginx_ingress_controller_requests{cluster=~\"$cluster\", controller_pod=~\"$controller\",controller_class=~\"$controller_class\",controller_namespace=~\"$namespace\",ingress=~\"$ingress\"}[2m])) by (ingress), 0.01)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{ingress}}", @@ -422,7 +422,7 @@ data: "steppedLine": false, "targets": [ { - "expr": "sum(rate(nginx_ingress_controller_requests{controller_pod=~\"$controller\",controller_class=~\"$controller_class\",namespace=~\"$namespace\",ingress=~\"$ingress\",status!~\"[4-5].*\"}[2m])) by (ingress) / sum(rate(nginx_ingress_controller_requests{controller_pod=~\"$controller\",controller_class=~\"$controller_class\",namespace=~\"$namespace\",ingress=~\"$ingress\"}[2m])) by (ingress)", + "expr": "sum(rate(nginx_ingress_controller_requests{cluster=~\"$cluster\", controller_pod=~\"$controller\",controller_class=~\"$controller_class\",controller_namespace=~\"$namespace\",ingress=~\"$ingress\",status!~\"[4-5].*\"}[2m])) by (ingress) / sum(rate(nginx_ingress_controller_requests{cluster=~\"$cluster\", controller_pod=~\"$controller\",controller_class=~\"$controller_class\",controller_namespace=~\"$namespace\",ingress=~\"$ingress\"}[2m])) by (ingress)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{ingress}}", @@ -526,21 +526,21 @@ data: "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.99, sum(rate(nginx_ingress_controller_request_duration_seconds_bucket{ingress!=\"\",controller_pod=~\"$controller\",controller_class=~\"$controller_class\",controller_namespace=~\"$namespace\",ingress=~\"$ingress\"}[2m])) by (le, ingress))", + "expr": "histogram_quantile(0.99, sum(rate(nginx_ingress_controller_request_duration_seconds_bucket{cluster=~\"$cluster\", ingress!=\"\",controller_pod=~\"$controller\",controller_class=~\"$controller_class\",controller_namespace=~\"$namespace\",ingress=~\"$ingress\"}[2m])) by (le, ingress))", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{ingress}} 99%", "refId": "A" }, { - "expr": "histogram_quantile(0.90, sum(rate(nginx_ingress_controller_request_duration_seconds_bucket{ingress!=\"\",controller_pod=~\"$controller\",controller_class=~\"$controller_class\",controller_namespace=~\"$namespace\",ingress=~\"$ingress\"}[2m])) by (le, ingress))", + "expr": "histogram_quantile(0.90, sum(rate(nginx_ingress_controller_request_duration_seconds_bucket{cluster=~\"$cluster\", ingress!=\"\",controller_pod=~\"$controller\",controller_class=~\"$controller_class\",controller_namespace=~\"$namespace\",ingress=~\"$ingress\"}[2m])) by (le, ingress))", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{ingress}} 90%", "refId": "B" }, { - "expr": "histogram_quantile(0.50, sum(rate(nginx_ingress_controller_request_duration_seconds_bucket{ingress!=\"\",controller_pod=~\"$controller\",controller_class=~\"$controller_class\",controller_namespace=~\"$namespace\",ingress=~\"$ingress\"}[2m])) by (le, ingress))", + "expr": "histogram_quantile(0.50, sum(rate(nginx_ingress_controller_request_duration_seconds_bucket{cluster=~\"$cluster\", ingress!=\"\",controller_pod=~\"$controller\",controller_class=~\"$controller_class\",controller_namespace=~\"$namespace\",ingress=~\"$ingress\"}[2m])) by (le, ingress))", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{ingress}} 50%", @@ -644,14 +644,14 @@ data: "steppedLine": false, "targets": [ { - "expr": "sum (irate (nginx_ingress_controller_request_size_sum{controller_pod=~\"$controller\",controller_class=~\"$controller_class\",controller_namespace=~\"$namespace\"}[2m]))", + "expr": "sum (irate (nginx_ingress_controller_request_size_sum{cluster=~\"$cluster\", controller_pod=~\"$controller\",controller_class=~\"$controller_class\",controller_namespace=~\"$namespace\"}[2m]))", "format": "time_series", "intervalFactor": 2, "legendFormat": "received", "refId": "A" }, { - "expr": "sum (irate (nginx_ingress_controller_response_size_sum{controller_pod=~\"$controller\",controller_class=~\"$controller_class\",controller_namespace=~\"$namespace\"}[2m]))", + "expr": "sum (irate (nginx_ingress_controller_response_size_sum{cluster=~\"$cluster\", controller_pod=~\"$controller\",controller_class=~\"$controller_class\",controller_namespace=~\"$namespace\"}[2m]))", "format": "time_series", "intervalFactor": 2, "legendFormat": "sent", @@ -742,7 +742,7 @@ data: "steppedLine": false, "targets": [ { - "expr": "avg(nginx_ingress_controller_nginx_process_resident_memory_bytes{controller_pod=~\"$controller\",controller_class=~\"$controller_class\",controller_namespace=~\"$namespace\"}) by (controller_pod)", + "expr": "avg(nginx_ingress_controller_nginx_process_resident_memory_bytes{cluster=~\"$cluster\", controller_pod=~\"$controller\",controller_class=~\"$controller_class\",controller_namespace=~\"$namespace\"}) by (controller_pod)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{controller_pod}}", @@ -833,7 +833,7 @@ data: "steppedLine": false, "targets": [ { - "expr": "sum(rate(nginx_ingress_controller_nginx_process_cpu_seconds_total{controller_pod=~\"$controller\",controller_class=~\"$controller_class\",controller_namespace=~\"$namespace\"}[2m])) by (controller_pod)", + "expr": "sum(rate(nginx_ingress_controller_nginx_process_cpu_seconds_total{cluster=~\"$cluster\", controller_pod=~\"$controller\",controller_class=~\"$controller_class\",controller_namespace=~\"$namespace\"}[2m])) by (controller_pod)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{controller_pod}}", @@ -917,6 +917,32 @@ data: "allValue": ".*", "current": { + }, + "datasource": "$datasource", + "hide": 2, + "includeAll": true, + "label": "cluster", + "multi": false, + "name": "cluster", + "options": [ + + ], + "query": "label_values(kube_pod_info, cluster)", + "refresh": 2, + "regex": "", + "sort": 0, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": ".*", + "current": { + }, "datasource": "$datasource", "hide": 0, @@ -927,7 +953,7 @@ data: "options": [ ], - "query": "label_values(nginx_ingress_controller_config_hash, controller_namespace)", + "query": "label_values(nginx_ingress_controller_config_hash{cluster=~\"$cluster\"}, controller_namespace)", "refresh": 2, "regex": "", "sort": 0, @@ -953,7 +979,7 @@ data: "options": [ ], - "query": "label_values(nginx_ingress_controller_config_hash{namespace=~\"$namespace\"}, controller_class)", + "query": "label_values(nginx_ingress_controller_config_hash{cluster=~\"$cluster\", namespace=~\"$namespace\"}, controller_class)", "refresh": 2, "regex": "", "sort": 0, @@ -979,7 +1005,7 @@ data: "options": [ ], - "query": "label_values(nginx_ingress_controller_config_hash{namespace=~\"$namespace\",controller_class=~\"$controller_class\"}, controller_pod)", + "query": "label_values(nginx_ingress_controller_config_hash{cluster=~\"$cluster\", namespace=~\"$namespace\", controller_class=~\"$controller_class\"}, controller_pod)", "refresh": 2, "regex": "", "sort": 0, @@ -1005,7 +1031,7 @@ data: "options": [ ], - "query": "label_values(nginx_ingress_controller_requests{namespace=~\"$namespace\",controller_class=~\"$controller_class\",controller=~\"$controller\"}, ingress)", + "query": "label_values(nginx_ingress_controller_requests{cluster=~\"$cluster\", namespace=~\"$namespace\", controller_class=~\"$controller_class\", controller=~\"$controller\"}, ingress)", "refresh": 2, "regex": "", "sort": 0, From d418045929531f9482aa5848141c1943bb585ebc Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Tue, 15 Oct 2019 23:31:57 -0700 Subject: [PATCH 256/523] Switch kube-proxy from iptables mode to ipvs mode * Kubernetes v1.11 considered kube-proxy IPVS mode GA * Many problems were found #321 * Since then, major blockers seem to have been addressed --- CHANGES.md | 2 ++ aws/container-linux/kubernetes/bootstrap.tf | 2 +- aws/fedora-coreos/kubernetes/bootstrap.tf | 2 +- azure/container-linux/kubernetes/bootstrap.tf | 2 +- bare-metal/container-linux/kubernetes/bootstrap.tf | 2 +- bare-metal/fedora-coreos/kubernetes/bootstrap.tf | 2 +- digital-ocean/container-linux/kubernetes/bootstrap.tf | 2 +- google-cloud/container-linux/kubernetes/bootstrap.tf | 2 +- 8 files changed, 9 insertions(+), 7 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 604b258c7..b3501c58f 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -4,6 +4,8 @@ Notable changes between versions. ## Latest +* Switch `kube-proxy` from iptables to ipvs mode ([#574](https://github.com/poseidon/typhoon/pull/574)) + #### Addons * Update Prometheus from v2.13.0 to v2.13.1 diff --git a/aws/container-linux/kubernetes/bootstrap.tf b/aws/container-linux/kubernetes/bootstrap.tf index 85921b477..51dadec19 100644 --- a/aws/container-linux/kubernetes/bootstrap.tf +++ b/aws/container-linux/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=0fcc067476fa1463d057fd43760df222b7262b27" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=e09d6bef33693455ee77b9e6b6882dc7d35c523c" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/aws/fedora-coreos/kubernetes/bootstrap.tf b/aws/fedora-coreos/kubernetes/bootstrap.tf index c0d6937e9..1bc9d791a 100644 --- a/aws/fedora-coreos/kubernetes/bootstrap.tf +++ b/aws/fedora-coreos/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=0fcc067476fa1463d057fd43760df222b7262b27" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=e09d6bef33693455ee77b9e6b6882dc7d35c523c" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/azure/container-linux/kubernetes/bootstrap.tf b/azure/container-linux/kubernetes/bootstrap.tf index 87858c034..8ca2a2959 100644 --- a/azure/container-linux/kubernetes/bootstrap.tf +++ b/azure/container-linux/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=0fcc067476fa1463d057fd43760df222b7262b27" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=e09d6bef33693455ee77b9e6b6882dc7d35c523c" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/bare-metal/container-linux/kubernetes/bootstrap.tf b/bare-metal/container-linux/kubernetes/bootstrap.tf index 18958b808..fa3a628d2 100644 --- a/bare-metal/container-linux/kubernetes/bootstrap.tf +++ b/bare-metal/container-linux/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=0fcc067476fa1463d057fd43760df222b7262b27" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=e09d6bef33693455ee77b9e6b6882dc7d35c523c" cluster_name = var.cluster_name api_servers = [var.k8s_domain_name] diff --git a/bare-metal/fedora-coreos/kubernetes/bootstrap.tf b/bare-metal/fedora-coreos/kubernetes/bootstrap.tf index 6d64d37f1..f51f9bf44 100644 --- a/bare-metal/fedora-coreos/kubernetes/bootstrap.tf +++ b/bare-metal/fedora-coreos/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=0fcc067476fa1463d057fd43760df222b7262b27" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=e09d6bef33693455ee77b9e6b6882dc7d35c523c" cluster_name = var.cluster_name api_servers = [var.k8s_domain_name] diff --git a/digital-ocean/container-linux/kubernetes/bootstrap.tf b/digital-ocean/container-linux/kubernetes/bootstrap.tf index a2030561a..0764a4eb6 100644 --- a/digital-ocean/container-linux/kubernetes/bootstrap.tf +++ b/digital-ocean/container-linux/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=0fcc067476fa1463d057fd43760df222b7262b27" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=e09d6bef33693455ee77b9e6b6882dc7d35c523c" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/google-cloud/container-linux/kubernetes/bootstrap.tf b/google-cloud/container-linux/kubernetes/bootstrap.tf index de83be726..9e3fa00ef 100644 --- a/google-cloud/container-linux/kubernetes/bootstrap.tf +++ b/google-cloud/container-linux/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=0fcc067476fa1463d057fd43760df222b7262b27" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=e09d6bef33693455ee77b9e6b6882dc7d35c523c" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] From 4775e9d0f72a7edd7358163d8aa114e1242cc20c Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Sun, 27 Oct 2019 00:49:46 -0700 Subject: [PATCH 257/523] Upgrade Calico v3.9.2 to v3.10.0 * Allow advertising Kubernetes service ClusterIPs to BGPPeer routers via a BGPConfiguration * Improve EdgeRouter docs about routes and BGP * https://docs.projectcalico.org/v3.10/release-notes/ * https://docs.projectcalico.org/v3.10/networking/advertise-service-ips --- CHANGES.md | 2 + aws/container-linux/kubernetes/bootstrap.tf | 2 +- aws/fedora-coreos/kubernetes/bootstrap.tf | 2 +- azure/container-linux/kubernetes/bootstrap.tf | 2 +- .../container-linux/kubernetes/bootstrap.tf | 2 +- .../fedora-coreos/kubernetes/bootstrap.tf | 2 +- .../container-linux/kubernetes/bootstrap.tf | 2 +- docs/topics/hardware.md | 146 +++++++++++------- .../container-linux/kubernetes/bootstrap.tf | 2 +- 9 files changed, 99 insertions(+), 63 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index b3501c58f..5756c6608 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -4,6 +4,8 @@ Notable changes between versions. ## Latest +* Upgrade Calico from v3.9.2 to [v3.10.0](https://docs.projectcalico.org/v3.10/release-notes/) + * Allow advertising service ClusterIPs to peer routers via a [BGPConfiguration](https://docs.projectcalico.org/v3.10/networking/advertise-service-ips) * Switch `kube-proxy` from iptables to ipvs mode ([#574](https://github.com/poseidon/typhoon/pull/574)) #### Addons diff --git a/aws/container-linux/kubernetes/bootstrap.tf b/aws/container-linux/kubernetes/bootstrap.tf index 51dadec19..a68b40a14 100644 --- a/aws/container-linux/kubernetes/bootstrap.tf +++ b/aws/container-linux/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=e09d6bef33693455ee77b9e6b6882dc7d35c523c" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=3c7334ab55b4ebef4109072da99452d59ee6179a" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/aws/fedora-coreos/kubernetes/bootstrap.tf b/aws/fedora-coreos/kubernetes/bootstrap.tf index 1bc9d791a..d87f50730 100644 --- a/aws/fedora-coreos/kubernetes/bootstrap.tf +++ b/aws/fedora-coreos/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=e09d6bef33693455ee77b9e6b6882dc7d35c523c" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=3c7334ab55b4ebef4109072da99452d59ee6179a" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/azure/container-linux/kubernetes/bootstrap.tf b/azure/container-linux/kubernetes/bootstrap.tf index 8ca2a2959..b80fbca69 100644 --- a/azure/container-linux/kubernetes/bootstrap.tf +++ b/azure/container-linux/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=e09d6bef33693455ee77b9e6b6882dc7d35c523c" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=3c7334ab55b4ebef4109072da99452d59ee6179a" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/bare-metal/container-linux/kubernetes/bootstrap.tf b/bare-metal/container-linux/kubernetes/bootstrap.tf index fa3a628d2..b806a3575 100644 --- a/bare-metal/container-linux/kubernetes/bootstrap.tf +++ b/bare-metal/container-linux/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=e09d6bef33693455ee77b9e6b6882dc7d35c523c" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=3c7334ab55b4ebef4109072da99452d59ee6179a" cluster_name = var.cluster_name api_servers = [var.k8s_domain_name] diff --git a/bare-metal/fedora-coreos/kubernetes/bootstrap.tf b/bare-metal/fedora-coreos/kubernetes/bootstrap.tf index f51f9bf44..5ebfee5f1 100644 --- a/bare-metal/fedora-coreos/kubernetes/bootstrap.tf +++ b/bare-metal/fedora-coreos/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=e09d6bef33693455ee77b9e6b6882dc7d35c523c" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=3c7334ab55b4ebef4109072da99452d59ee6179a" cluster_name = var.cluster_name api_servers = [var.k8s_domain_name] diff --git a/digital-ocean/container-linux/kubernetes/bootstrap.tf b/digital-ocean/container-linux/kubernetes/bootstrap.tf index 0764a4eb6..c4a348b64 100644 --- a/digital-ocean/container-linux/kubernetes/bootstrap.tf +++ b/digital-ocean/container-linux/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=e09d6bef33693455ee77b9e6b6882dc7d35c523c" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=3c7334ab55b4ebef4109072da99452d59ee6179a" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/docs/topics/hardware.md b/docs/topics/hardware.md index 5751b18bf..d95ad2275 100644 --- a/docs/topics/hardware.md +++ b/docs/topics/hardware.md @@ -6,6 +6,39 @@ Typhoon ensures certain networking hardware integrates well with bare-metal Kube Ubiquiti EdgeRouters and EdgeOS work well with bare-metal Kubernetes clusters. Familiarity with EdgeRouter setup and CLI usage is required. +### DHCP + +Assign static IPs to clients with known MAC addresses. This is called a static mapping by EdgeOS. Configure the router with the commands based on region inventory. + +``` +configure +show service dhcp-server shared-network +set service dhcp-server shared-network-name LAN subnet SUBNET static-mapping NAME mac-address MACADDR +set service dhcp-server shared-network-name LAN subnet SUBNET static-mapping NAME ip-address 10.0.0.20 +``` + +### DNS + +Add DNS A records to static IPs as `dnsmasq` host-records. + +``` +configure +set service dns forwarding options host-record=node.example.com,10.0.0.20 +``` + +Forward `*.svc.cluster.local` queries to the CoreDNS Kubernetes service IP to allow clients to resolve Kubernetes services. + +``` +set service dns forwarding options server=/svc.cluster.local/10.3.0.10 +commit-confirm +``` + +Restart `dnsmasq`. + +``` +sudo /etc/init.d/dnsmasq restart +``` + ### PXE Ubiquiti EdgeRouters can provide a PXE-enabled network boot environment for client machines. @@ -65,55 +98,88 @@ set service dns forwarding options tftp-root=/config/tftpboot commit-confirm ``` -### DHCP +### Routing -Assign static IPs to clients with known MAC addresses. This is called a static mapping by EdgeOS. Configure the router with the commands based on region inventory. +#### Static Routes + +Add static route(s) to Kubernetes node(s) that can route to Kubernetes service IPs (default: 10.3.0.0/16). Kubernetes service IPs will become routeable on the LAN. ``` configure -show service dhcp-server shared-network -set service dhcp-server shared-network-name LAN subnet SUBNET static-mapping NAME mac-address MACADDR -set service dhcp-server shared-network-name LAN subnet SUBNET static-mapping NAME ip-address 10.0.0.20 +show protocols static route +set protocols static route 10.3.0.0/16 next-hop NODE_IP +commit-confirm ``` -### DNS +!!! note + Adding multiple next-hop nodes provides equal-cost multi-path (ECMP) routing. EdgeOS v2.0+ is required. The kernel in prior versions used flow-hash to balanced packets, whereas with v2.0, round-robin sessions are used. -Assign DNS A records to nodes as options to `dnsmasq`. +#### BGP + +EdgeRouter can exchange routes with other autonomous systems, including a cluster's Calico AS. Peers will exchange `podCIDR` routes to make individual pods routeable on the LAN. + +Define the EdgeRouter AS (if undefined). ``` configure -set service dns forwarding options host-record=node.example.com,10.0.0.20 +show protocols bgp 1 +set protocols bgp 1 parameters router-id ROUTER_IP ``` -Restart `dnsmasq`. +Peer with node(s) in another AS (eg. Calico default 64512) ``` -sudo /etc/init.d/dnsmasq restart +set protocols bgp 1 neighbor NODE1_IP remote-as 64512 +set protocols bgp 1 neighbor NODE2_IP remote-as 64512 +set protocols bgp 1 neighbor NODE3_IP remote-as 64512 +commit-confirm ``` -Configure queries for `*.svc.cluster.local` to be forwarded to the Kubernetes `coredns` service IP to allow hosts to resolve cluster-local Kubernetes names. +Configure Calico node(s) as to peer with the EdgeRouter. ``` -configure -show service dns forwarding -set service dns forwarding options server=/svc.cluster.local/10.3.0.10 -commit-confirm +apiVersion: crd.projectcalico.org/v1 +kind: BGPPeer +metadata: + name: NODE_NAME-to-edgerouter +spec: + peerIP: ROUTER_IP + asNumber: 1 + node: NODE_NAME ``` -### Kubernetes Services +Or, if every node is to be peered (i.e. full mesh), define a global BGPPeer. -Add static routes for the Kubernetes IPv4 service range to Kubernetes node(s) so hosts can route to Kubernetes services (default: 10.3.0.0/16). +``` +apiVersion: crd.projectcalico.org/v1 +kind: BGPPeer +metadata: + name: global +spec: + peerIP: ROUTER_IP + asNumber: 1 +``` + +If Calico nodes should advertise Kubernetes Service IPs (i.e. ClusterIPs) as well, add a `BGPConfiguration`. ``` -configure -show protocols static route -set protocols static route 10.3.0.0/16 next-hop NODE_IP -... -commit-confirm +apiVersion: crd.projectcalico.org/v1 +kind: BGPConfiguration +metadata: + name: default +spec: + logSeverityScreen: Info + nodeToNodeMeshEnabled: true + serviceClusterIPs: + - cidr: 10.3.0.0/16 ``` -!!! note - Adding multiple next-hop nodes provides equal-cost multi-path (ECMP) routing. EdgeOS v2.0+ is required. The kernel in prior versions used flow-hash to balanced packets, whereas with v2.0, round-robin sessions are used. +Show a summary of peers and exchanged routes. + +``` +show ip bgp summary +show ip route bgp +``` ### Port Forwarding @@ -150,35 +216,3 @@ set service gui https-port 4443 commit-confirm ``` -### BGP - -Add the EdgeRouter as a global BGP peer for nodes in a Kubernetes cluster (requires Calico). Neighbors will exchange `podCIDR` routes and individual pods will become routable on the LAN. - -Configure node(s) as BGP neighbors. - -``` -show protocols bgp 1 -set protocols bgp 1 parameters router-id LAN_IP -set protocols bgp 1 neighbor NODE1_IP remote-as 64512 -set protocols bgp 1 neighbor NODE2_IP remote-as 64512 -set protocols bgp 1 neighbor NODE3_IP remote-as 64512 -``` - -View the neighbors and exchanged routes. - -``` -show ip bgp neighbors -show ip route bgp -``` - -Be sure to register the peer by creating a Calico `BGPPeer` CRD with `kubectl apply`. - -``` -apiVersion: crd.projectcalico.org/v1 -kind: BGPPeer -metadata: - name: NAME -spec: - peerIP: LAN_IP - asNumber: 64512 -``` diff --git a/google-cloud/container-linux/kubernetes/bootstrap.tf b/google-cloud/container-linux/kubernetes/bootstrap.tf index 9e3fa00ef..878bfa4a9 100644 --- a/google-cloud/container-linux/kubernetes/bootstrap.tf +++ b/google-cloud/container-linux/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=e09d6bef33693455ee77b9e6b6882dc7d35c523c" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=3c7334ab55b4ebef4109072da99452d59ee6179a" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] From d4573092b54e08a9d8686b967c727cbeaa7ad624 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Mon, 28 Oct 2019 02:22:15 -0700 Subject: [PATCH 258/523] Improve Kubelet and Compute Resource dashboards * Add cluster filter to Kubelet dashboard * Add network details in resource dashboards * https://github.com/kubernetes-monitoring/kubernetes-mixin/pull/275 * https://github.com/kubernetes-monitoring/kubernetes-mixin/pull/284 * https://github.com/kubernetes-monitoring/kubernetes-mixin/pull/285 --- addons/grafana/dashboards-k8s-nodes.yaml | 90 +- .../grafana/dashboards-k8s-resources-1.yaml | 5375 +++++++++++++++ .../grafana/dashboards-k8s-resources-2.yaml | 5753 +++++++++++++++++ addons/grafana/dashboards-k8s.yaml | 26 +- addons/prometheus/rules.yaml | 282 +- 5 files changed, 11367 insertions(+), 159 deletions(-) create mode 100644 addons/grafana/dashboards-k8s-resources-1.yaml create mode 100644 addons/grafana/dashboards-k8s-resources-2.yaml diff --git a/addons/grafana/dashboards-k8s-nodes.yaml b/addons/grafana/dashboards-k8s-nodes.yaml index 9702c0f62..1b42cd429 100644 --- a/addons/grafana/dashboards-k8s-nodes.yaml +++ b/addons/grafana/dashboards-k8s-nodes.yaml @@ -88,7 +88,7 @@ data: "tableColumn": "", "targets": [ { - "expr": "sum(up{job=\"kubelet\"})", + "expr": "sum(up{cluster=\"$cluster\", job=\"kubelet\"})", "format": "time_series", "intervalFactor": 2, "legendFormat": "", @@ -172,7 +172,7 @@ data: "tableColumn": "", "targets": [ { - "expr": "sum(kubelet_running_pod_count{job=\"kubelet\", instance=~\"$instance\"})", + "expr": "sum(kubelet_running_pod_count{cluster=\"$cluster\", job=\"kubelet\", instance=~\"$instance\"})", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{instance}}", @@ -256,7 +256,7 @@ data: "tableColumn": "", "targets": [ { - "expr": "sum(kubelet_running_container_count{job=\"kubelet\", instance=~\"$instance\"})", + "expr": "sum(kubelet_running_container_count{cluster=\"$cluster\", job=\"kubelet\", instance=~\"$instance\"})", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{instance}}", @@ -340,7 +340,7 @@ data: "tableColumn": "", "targets": [ { - "expr": "sum(volume_manager_total_volumes{job=\"kubelet\", instance=~\"$instance\", state=\"actual_state_of_world\"})", + "expr": "sum(volume_manager_total_volumes{cluster=\"$cluster\", job=\"kubelet\", instance=~\"$instance\", state=\"actual_state_of_world\"})", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{instance}}", @@ -424,7 +424,7 @@ data: "tableColumn": "", "targets": [ { - "expr": "sum(volume_manager_total_volumes{job=\"kubelet\", instance=~\"$instance\",state=\"desired_state_of_world\"})", + "expr": "sum(volume_manager_total_volumes{cluster=\"$cluster\", job=\"kubelet\", instance=~\"$instance\",state=\"desired_state_of_world\"})", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{instance}}", @@ -508,7 +508,7 @@ data: "tableColumn": "", "targets": [ { - "expr": "sum(rate(kubelet_node_config_error{job=\"kubelet\", instance=~\"$instance\"}[5m]))", + "expr": "sum(rate(kubelet_node_config_error{cluster=\"$cluster\", job=\"kubelet\", instance=~\"$instance\"}[5m]))", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{instance}}", @@ -588,7 +588,7 @@ data: "steppedLine": false, "targets": [ { - "expr": "sum(rate(kubelet_runtime_operations_total{job=\"kubelet\",instance=~\"$instance\"}[5m])) by (operation_type, instance)", + "expr": "sum(rate(kubelet_runtime_operations_total{cluster=\"$cluster\",job=\"kubelet\",instance=~\"$instance\"}[5m])) by (operation_type, instance)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{instance}} {{operation_type}}", @@ -679,7 +679,7 @@ data: "steppedLine": false, "targets": [ { - "expr": "sum(rate(kubelet_runtime_operations_errors_total{job=\"kubelet\",instance=~\"$instance\"}[5m])) by (instance, operation_type)", + "expr": "sum(rate(kubelet_runtime_operations_errors_total{cluster=\"$cluster\",job=\"kubelet\",instance=~\"$instance\"}[5m])) by (instance, operation_type)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{instance}} {{operation_type}}", @@ -783,7 +783,7 @@ data: "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.99, sum(rate(kubelet_runtime_operations_duration_seconds_bucket{job=\"kubelet\",instance=~\"$instance\"}[5m])) by (instance, operation_type, le))", + "expr": "histogram_quantile(0.99, sum(rate(kubelet_runtime_operations_duration_seconds_bucket{cluster=\"$cluster\",job=\"kubelet\",instance=~\"$instance\"}[5m])) by (instance, operation_type, le))", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{instance}} {{operation_type}}", @@ -887,14 +887,14 @@ data: "steppedLine": false, "targets": [ { - "expr": "sum(rate(kubelet_pod_start_duration_seconds_count{job=\"kubelet\",instance=~\"$instance\"}[5m])) by (instance)", + "expr": "sum(rate(kubelet_pod_start_duration_seconds_count{cluster=\"$cluster\",job=\"kubelet\",instance=~\"$instance\"}[5m])) by (instance)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{instance}} pod", "refId": "A" }, { - "expr": "sum(rate(kubelet_pod_worker_duration_seconds_count{job=\"kubelet\",instance=~\"$instance\"}[5m])) by (instance)", + "expr": "sum(rate(kubelet_pod_worker_duration_seconds_count{cluster=\"$cluster\",job=\"kubelet\",instance=~\"$instance\"}[5m])) by (instance)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{instance}} worker", @@ -985,14 +985,14 @@ data: "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.99, sum(rate(kubelet_pod_start_duration_seconds_count{job=\"kubelet\",instance=~\"$instance\"}[5m])) by (instance, le))", + "expr": "histogram_quantile(0.99, sum(rate(kubelet_pod_start_duration_seconds_count{cluster=\"$cluster\",job=\"kubelet\",instance=~\"$instance\"}[5m])) by (instance, le))", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{instance}} pod", "refId": "A" }, { - "expr": "histogram_quantile(0.99, sum(rate(kubelet_pod_worker_duration_seconds_bucket{job=\"kubelet\",instance=~\"$instance\"}[5m])) by (instance, le))", + "expr": "histogram_quantile(0.99, sum(rate(kubelet_pod_worker_duration_seconds_bucket{cluster=\"$cluster\",job=\"kubelet\",instance=~\"$instance\"}[5m])) by (instance, le))", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{instance}} worker", @@ -1098,7 +1098,7 @@ data: "steppedLine": false, "targets": [ { - "expr": "sum(rate(storage_operation_duration_seconds_count{job=\"kubelet\",instance=~\"$instance\"}[5m])) by (instance, operation_name, volume_plugin)", + "expr": "sum(rate(storage_operation_duration_seconds_count{cluster=\"$cluster\",job=\"kubelet\",instance=~\"$instance\"}[5m])) by (instance, operation_name, volume_plugin)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{instance}} {{operation_name}} {{volume_plugin}}", @@ -1191,7 +1191,7 @@ data: "steppedLine": false, "targets": [ { - "expr": "sum(rate(storage_operation_errors_total{job=\"kubelet\",instance=~\"$instance\"}[5m])) by (instance, operation_name, volume_plugin)", + "expr": "sum(rate(storage_operation_errors_total{cluster=\"$cluster\",job=\"kubelet\",instance=~\"$instance\"}[5m])) by (instance, operation_name, volume_plugin)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{instance}} {{operation_name}} {{volume_plugin}}", @@ -1297,7 +1297,7 @@ data: "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.99, sum(rate(storage_operation_duration_seconds_bucket{job=\"kubelet\", instance=~\"$instance\"}[5m])) by (instance, operation_name, volume_plugin, le))", + "expr": "histogram_quantile(0.99, sum(rate(storage_operation_duration_seconds_bucket{cluster=\"$cluster\", job=\"kubelet\", instance=~\"$instance\"}[5m])) by (instance, operation_name, volume_plugin, le))", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{instance}} {{operation_name}} {{volume_plugin}}", @@ -1401,7 +1401,7 @@ data: "steppedLine": false, "targets": [ { - "expr": "sum(rate(kubelet_cgroup_manager_duration_seconds_count{job=\"kubelet\", instance=~\"$instance\"}[5m])) by (instance, operation_type)", + "expr": "sum(rate(kubelet_cgroup_manager_duration_seconds_count{cluster=\"$cluster\", job=\"kubelet\", instance=~\"$instance\"}[5m])) by (instance, operation_type)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{operation_type}}", @@ -1492,7 +1492,7 @@ data: "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.99, sum(rate(kubelet_cgroup_manager_duration_seconds_bucket{job=\"kubelet\", instance=~\"$instance\"}[5m])) by (instance, operation_type, le))", + "expr": "histogram_quantile(0.99, sum(rate(kubelet_cgroup_manager_duration_seconds_bucket{cluster=\"$cluster\", job=\"kubelet\", instance=~\"$instance\"}[5m])) by (instance, operation_type, le))", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{instance}} {{operation_type}}", @@ -1597,7 +1597,7 @@ data: "steppedLine": false, "targets": [ { - "expr": "sum(rate(kubelet_pleg_relist_duration_seconds_count{job=\"kubelet\", instance=~\"$instance\"}[5m])) by (instance)", + "expr": "sum(rate(kubelet_pleg_relist_duration_seconds_count{cluster=\"$cluster\", job=\"kubelet\", instance=~\"$instance\"}[5m])) by (instance)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{instance}}", @@ -1688,7 +1688,7 @@ data: "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.99, sum(rate(kubelet_pleg_relist_interval_seconds_bucket{job=\"kubelet\",instance=~\"$instance\"}[5m])) by (instance, le))", + "expr": "histogram_quantile(0.99, sum(rate(kubelet_pleg_relist_interval_seconds_bucket{cluster=\"$cluster\",job=\"kubelet\",instance=~\"$instance\"}[5m])) by (instance, le))", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{instance}}", @@ -1792,7 +1792,7 @@ data: "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.99, sum(rate(kubelet_pleg_relist_duration_seconds_bucket{job=\"kubelet\",instance=~\"$instance\"}[5m])) by (instance, le))", + "expr": "histogram_quantile(0.99, sum(rate(kubelet_pleg_relist_duration_seconds_bucket{cluster=\"$cluster\",job=\"kubelet\",instance=~\"$instance\"}[5m])) by (instance, le))", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{instance}}", @@ -1896,28 +1896,28 @@ data: "steppedLine": false, "targets": [ { - "expr": "sum(rate(rest_client_requests_total{job=\"kubelet\", instance=~\"$instance\",code=~\"2..\"}[5m]))", + "expr": "sum(rate(rest_client_requests_total{cluster=\"$cluster\",job=\"kubelet\", instance=~\"$instance\",code=~\"2..\"}[5m]))", "format": "time_series", "intervalFactor": 2, "legendFormat": "2xx", "refId": "A" }, { - "expr": "sum(rate(rest_client_requests_total{job=\"kubelet\", instance=~\"$instance\",code=~\"3..\"}[5m]))", + "expr": "sum(rate(rest_client_requests_total{cluster=\"$cluster\",job=\"kubelet\", instance=~\"$instance\",code=~\"3..\"}[5m]))", "format": "time_series", "intervalFactor": 2, "legendFormat": "3xx", "refId": "B" }, { - "expr": "sum(rate(rest_client_requests_total{job=\"kubelet\", instance=~\"$instance\",code=~\"4..\"}[5m]))", + "expr": "sum(rate(rest_client_requests_total{cluster=\"$cluster\",job=\"kubelet\", instance=~\"$instance\",code=~\"4..\"}[5m]))", "format": "time_series", "intervalFactor": 2, "legendFormat": "4xx", "refId": "C" }, { - "expr": "sum(rate(rest_client_requests_total{job=\"kubelet\", instance=~\"$instance\",code=~\"5..\"}[5m]))", + "expr": "sum(rate(rest_client_requests_total{cluster=\"$cluster\",job=\"kubelet\", instance=~\"$instance\",code=~\"5..\"}[5m]))", "format": "time_series", "intervalFactor": 2, "legendFormat": "5xx", @@ -2021,7 +2021,7 @@ data: "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.99, sum(rate(rest_client_request_latency_seconds_bucket{job=\"kubelet\", instance=~\"$instance\"}[5m])) by (instance, verb, url, le))", + "expr": "histogram_quantile(0.99, sum(rate(rest_client_request_latency_seconds_bucket{cluster=\"$cluster\",job=\"kubelet\", instance=~\"$instance\"}[5m])) by (instance, verb, url, le))", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{instance}} {{verb}} {{url}}", @@ -2125,7 +2125,7 @@ data: "steppedLine": false, "targets": [ { - "expr": "process_resident_memory_bytes{job=\"kubelet\",instance=~\"$instance\"}", + "expr": "process_resident_memory_bytes{cluster=\"$cluster\",job=\"kubelet\",instance=~\"$instance\"}", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{instance}}", @@ -2216,7 +2216,7 @@ data: "steppedLine": false, "targets": [ { - "expr": "rate(process_cpu_seconds_total{job=\"kubelet\",instance=~\"$instance\"}[5m])", + "expr": "rate(process_cpu_seconds_total{cluster=\"$cluster\",job=\"kubelet\",instance=~\"$instance\"}[5m])", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{instance}}", @@ -2307,7 +2307,7 @@ data: "steppedLine": false, "targets": [ { - "expr": "go_goroutines{job=\"kubelet\",instance=~\"$instance\"}", + "expr": "go_goroutines{cluster=\"$cluster\",job=\"kubelet\",instance=~\"$instance\"}", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{instance}}", @@ -2391,6 +2391,32 @@ data: "allValue": null, "current": { + }, + "datasource": "$datasource", + "hide": 2, + "includeAll": false, + "label": "cluster", + "multi": false, + "name": "cluster", + "options": [ + + ], + "query": "label_values(kube_pod_info, cluster)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + }, "datasource": "$datasource", "hide": 0, @@ -2401,10 +2427,10 @@ data: "options": [ ], - "query": "label_values(kubelet_runtime_operations{job=\"kubelet\"}, instance)", + "query": "label_values(kubelet_runtime_operations{cluster=\"$cluster\", job=\"kubelet\"}, instance)", "refresh": 2, "regex": "", - "sort": 0, + "sort": 1, "tagValuesQuery": "", "tags": [ @@ -3593,7 +3619,7 @@ data: "query": "label_values(kubeproxy_network_programming_duration_seconds_bucket{job=\"kube-proxy\"}, instance)", "refresh": 2, "regex": "", - "sort": 0, + "sort": 1, "tagValuesQuery": "", "tags": [ diff --git a/addons/grafana/dashboards-k8s-resources-1.yaml b/addons/grafana/dashboards-k8s-resources-1.yaml new file mode 100644 index 000000000..110f2f338 --- /dev/null +++ b/addons/grafana/dashboards-k8s-resources-1.yaml @@ -0,0 +1,5375 @@ +apiVersion: v1 +data: + k8s-resources-cluster.json: |- + { + "annotations": { + "list": [ + + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "links": [ + + ], + "refresh": "10s", + "rows": [ + { + "collapse": false, + "height": "100px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "format": "percentunit", + "id": 1, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "1 - avg(rate(node_cpu_seconds_total{mode=\"idle\", cluster=\"$cluster\"}[1m]))", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "refId": "A" + } + ], + "thresholds": "70,80", + "timeFrom": null, + "timeShift": null, + "title": "CPU Utilisation", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "singlestat", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "format": "percentunit", + "id": 2, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(kube_pod_container_resource_requests_cpu_cores{cluster=\"$cluster\"}) / sum(kube_node_status_allocatable_cpu_cores{cluster=\"$cluster\"})", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "refId": "A" + } + ], + "thresholds": "70,80", + "timeFrom": null, + "timeShift": null, + "title": "CPU Requests Commitment", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "singlestat", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "format": "percentunit", + "id": 3, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(kube_pod_container_resource_limits_cpu_cores{cluster=\"$cluster\"}) / sum(kube_node_status_allocatable_cpu_cores{cluster=\"$cluster\"})", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "refId": "A" + } + ], + "thresholds": "70,80", + "timeFrom": null, + "timeShift": null, + "title": "CPU Limits Commitment", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "singlestat", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "format": "percentunit", + "id": 4, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "1 - sum(:node_memory_MemAvailable_bytes:sum{cluster=\"$cluster\"}) / sum(kube_node_status_allocatable_memory_bytes{cluster=\"$cluster\"})", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "refId": "A" + } + ], + "thresholds": "70,80", + "timeFrom": null, + "timeShift": null, + "title": "Memory Utilisation", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "singlestat", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "format": "percentunit", + "id": 5, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(kube_pod_container_resource_requests_memory_bytes{cluster=\"$cluster\"}) / sum(kube_node_status_allocatable_memory_bytes{cluster=\"$cluster\"})", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "refId": "A" + } + ], + "thresholds": "70,80", + "timeFrom": null, + "timeShift": null, + "title": "Memory Requests Commitment", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "singlestat", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "format": "percentunit", + "id": 6, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(kube_pod_container_resource_limits_memory_bytes{cluster=\"$cluster\"}) / sum(kube_node_status_allocatable_memory_bytes{cluster=\"$cluster\"})", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "refId": "A" + } + ], + "thresholds": "70,80", + "timeFrom": null, + "timeShift": null, + "title": "Memory Limits Commitment", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "singlestat", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Headlines", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 7, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\"}) by (namespace)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{namespace}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU Usage", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "CPU", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 8, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "Pods", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 0, + "link": true, + "linkTooltip": "Drill down to pods", + "linkUrl": "./d/85a562078cdf77779eaa1add43ccec1e/k8s-resources-namespace?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$__cell_1", + "pattern": "Value #A", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "Workloads", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 0, + "link": true, + "linkTooltip": "Drill down to workloads", + "linkUrl": "./d/a87fb0d919ec0ea5f6543124e16c42a5/k8s-resources-workloads-namespace?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$__cell_1", + "pattern": "Value #B", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "CPU Usage", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #C", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "CPU Requests", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #D", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "CPU Requests %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #E", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "CPU Limits", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #F", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "CPU Limits %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #G", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "Namespace", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": true, + "linkTooltip": "Drill down to pods", + "linkUrl": "./d/85a562078cdf77779eaa1add43ccec1e/k8s-resources-namespace?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$__cell", + "pattern": "namespace", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "/.*/", + "thresholds": [ + + ], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "expr": "count(mixin_pod_workload{cluster=\"$cluster\"}) by (namespace)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10 + }, + { + "expr": "count(avg(mixin_pod_workload{cluster=\"$cluster\"}) by (workload, namespace)) by (namespace)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + }, + { + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\"}) by (namespace)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "C", + "step": 10 + }, + { + "expr": "sum(kube_pod_container_resource_requests_cpu_cores{cluster=\"$cluster\"}) by (namespace)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "D", + "step": 10 + }, + { + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\"}) by (namespace) / sum(kube_pod_container_resource_requests_cpu_cores{cluster=\"$cluster\"}) by (namespace)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "E", + "step": 10 + }, + { + "expr": "sum(kube_pod_container_resource_limits_cpu_cores{cluster=\"$cluster\"}) by (namespace)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "F", + "step": 10 + }, + { + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\"}) by (namespace) / sum(kube_pod_container_resource_limits_cpu_cores{cluster=\"$cluster\"}) by (namespace)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "G", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU Quota", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "transform": "table", + "type": "table", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "CPU Quota", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 9, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(container_memory_rss{cluster=\"$cluster\", container!=\"\"}) by (namespace)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{namespace}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Memory Usage (w/o cache)", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Memory", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 10, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "Pods", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 0, + "link": true, + "linkTooltip": "Drill down to pods", + "linkUrl": "./d/85a562078cdf77779eaa1add43ccec1e/k8s-resources-namespace?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$__cell_1", + "pattern": "Value #A", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "Workloads", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 0, + "link": true, + "linkTooltip": "Drill down to workloads", + "linkUrl": "./d/a87fb0d919ec0ea5f6543124e16c42a5/k8s-resources-workloads-namespace?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$__cell_1", + "pattern": "Value #B", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "Memory Usage", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #C", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Requests", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #D", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Requests %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #E", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "Memory Limits", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #F", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Limits %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #G", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "Namespace", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": true, + "linkTooltip": "Drill down to pods", + "linkUrl": "./d/85a562078cdf77779eaa1add43ccec1e/k8s-resources-namespace?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$__cell", + "pattern": "namespace", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "/.*/", + "thresholds": [ + + ], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "expr": "count(mixin_pod_workload{cluster=\"$cluster\"}) by (namespace)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10 + }, + { + "expr": "count(avg(mixin_pod_workload{cluster=\"$cluster\"}) by (workload, namespace)) by (namespace)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + }, + { + "expr": "sum(container_memory_rss{cluster=\"$cluster\", container!=\"\"}) by (namespace)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "C", + "step": 10 + }, + { + "expr": "sum(kube_pod_container_resource_requests_memory_bytes{cluster=\"$cluster\"}) by (namespace)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "D", + "step": 10 + }, + { + "expr": "sum(container_memory_rss{cluster=\"$cluster\", container!=\"\"}) by (namespace) / sum(kube_pod_container_resource_requests_memory_bytes{cluster=\"$cluster\"}) by (namespace)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "E", + "step": 10 + }, + { + "expr": "sum(kube_pod_container_resource_limits_memory_bytes{cluster=\"$cluster\"}) by (namespace)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "F", + "step": 10 + }, + { + "expr": "sum(container_memory_rss{cluster=\"$cluster\", container!=\"\"}) by (namespace) / sum(kube_pod_container_resource_limits_memory_bytes{cluster=\"$cluster\"}) by (namespace)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "G", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Requests by Namespace", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "transform": "table", + "type": "table", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Memory Requests", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 11, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "Current Receive Bandwidth", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #A", + "thresholds": [ + + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Current Transmit Bandwidth", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #B", + "thresholds": [ + + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Rate of Received Packets", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #C", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Rate of Transmitted Packets", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #D", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Rate of Received Packets Dropped", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #E", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Rate of Transmitted Packets Dropped", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #F", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Namespace", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": true, + "linkTooltip": "Drill down to pods", + "linkUrl": "./d/85a562078cdf77779eaa1add43ccec1e/k8s-resources-namespace?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$__cell", + "pattern": "namespace", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "/.*/", + "thresholds": [ + + ], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "expr": "sum(irate(container_network_receive_bytes_total{cluster=\"$cluster\", namespace=~\".+\"}[$interval])) by (namespace)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10 + }, + { + "expr": "sum(irate(container_network_transmit_bytes_total{cluster=\"$cluster\", namespace=~\".+\"}[$interval])) by (namespace)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + }, + { + "expr": "sum(irate(container_network_receive_packets_total{cluster=\"$cluster\", namespace=~\".+\"}[$interval])) by (namespace)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "C", + "step": 10 + }, + { + "expr": "sum(irate(container_network_transmit_packets_total{cluster=\"$cluster\", namespace=~\".+\"}[$interval])) by (namespace)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "D", + "step": 10 + }, + { + "expr": "sum(irate(container_network_receive_packets_dropped_total{cluster=\"$cluster\", namespace=~\".+\"}[$interval])) by (namespace)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "E", + "step": 10 + }, + { + "expr": "sum(irate(container_network_transmit_packets_dropped_total{cluster=\"$cluster\", namespace=~\".+\"}[$interval])) by (namespace)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "F", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Current Network Usage", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "transform": "table", + "type": "table", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Network", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 12, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_receive_bytes_total{cluster=\"$cluster\", namespace=~\".+\"}[$interval])) by (namespace)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{namespace}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Receive Bandwidth", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Network", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 13, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_transmit_bytes_total{cluster=\"$cluster\", namespace=~\".+\"}[$interval])) by (namespace)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{namespace}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Transmit Bandwidth", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Network", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 14, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "avg(irate(container_network_receive_bytes_total{cluster=\"$cluster\", namespace=~\".+\"}[$interval])) by (namespace)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{namespace}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Average Container Bandwidth by Namespace: Received", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Network", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 15, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "avg(irate(container_network_transmit_bytes_total{cluster=\"$cluster\", namespace=~\".+\"}[$interval])) by (namespace)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{namespace}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Average Container Bandwidth by Namespace: Transmitted", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Network", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 16, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_receive_packets_total{cluster=\"$cluster\", namespace=~\".+\"}[$interval])) by (namespace)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{namespace}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Received Packets", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Network", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 17, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_receive_packets_total{cluster=\"$cluster\", namespace=~\".+\"}[$interval])) by (namespace)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{namespace}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Transmitted Packets", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Network", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 18, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_receive_packets_dropped_total{cluster=\"$cluster\", namespace=~\".+\"}[$interval])) by (namespace)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{namespace}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Received Packets Dropped", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Network", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 19, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_transmit_packets_dropped_total{cluster=\"$cluster\", namespace=~\".+\"}[$interval])) by (namespace)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{namespace}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Transmitted Packets Dropped", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Network", + "titleSize": "h6" + } + ], + "schemaVersion": 14, + "style": "dark", + "tags": [ + "kubernetes-mixin" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": null, + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + "text": "prod", + "value": "prod" + }, + "datasource": "$datasource", + "hide": 2, + "includeAll": false, + "label": "cluster", + "multi": false, + "name": "cluster", + "options": [ + + ], + "query": "label_values(node_cpu_seconds_total, cluster)", + "refresh": 1, + "regex": "", + "sort": 2, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "auto": false, + "auto_count": 30, + "auto_min": "10s", + "current": { + "text": "5m", + "value": "5m" + }, + "datasource": "prometheus", + "hide": 2, + "includeAll": false, + "label": null, + "multi": false, + "name": "interval", + "options": [ + { + "selected": true, + "text": "4h", + "value": "4h" + } + ], + "query": "4h", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "interval", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "", + "title": "Kubernetes / Compute Resources / Cluster", + "uid": "efa86fd1d0c121a26444b636a3f509a8", + "version": 0 + } + k8s-resources-namespace.json: |- + { + "annotations": { + "list": [ + + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "links": [ + + ], + "refresh": "10s", + "rows": [ + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 1, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{pod}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU Usage", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "CPU Usage", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 2, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "CPU Usage", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #A", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "CPU Requests", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #B", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "CPU Requests %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #C", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "CPU Limits", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #D", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "CPU Limits %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #E", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "Pod", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": true, + "linkTooltip": "Drill down", + "linkUrl": "./d/6581e46e4e5c7ba40a07646395ef7b23/k8s-resources-pod?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$namespace&var-pod=$__cell", + "pattern": "pod", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "/.*/", + "thresholds": [ + + ], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10 + }, + { + "expr": "sum(kube_pod_container_resource_requests_cpu_cores{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + }, + { + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod) / sum(kube_pod_container_resource_requests_cpu_cores{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "C", + "step": 10 + }, + { + "expr": "sum(kube_pod_container_resource_limits_cpu_cores{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "D", + "step": 10 + }, + { + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod) / sum(kube_pod_container_resource_limits_cpu_cores{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "E", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU Quota", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "transform": "table", + "type": "table", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "CPU Quota", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 3, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(container_memory_working_set_bytes{cluster=\"$cluster\", namespace=\"$namespace\", container!=\"\"}) by (pod)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{pod}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Memory Usage (w/o cache)", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Memory Usage", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 4, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "Memory Usage", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #A", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Requests", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #B", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Requests %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #C", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "Memory Limits", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #D", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Limits %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #E", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "Memory Usage (RSS)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #F", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Usage (Cache)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #G", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Usage (Swap)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #H", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Pod", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": true, + "linkTooltip": "Drill down", + "linkUrl": "./d/6581e46e4e5c7ba40a07646395ef7b23/k8s-resources-pod?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$namespace&var-pod=$__cell", + "pattern": "pod", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "/.*/", + "thresholds": [ + + ], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "expr": "sum(container_memory_working_set_bytes{cluster=\"$cluster\", namespace=\"$namespace\",container!=\"\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10 + }, + { + "expr": "sum(kube_pod_container_resource_requests_memory_bytes{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + }, + { + "expr": "sum(container_memory_working_set_bytes{cluster=\"$cluster\", namespace=\"$namespace\",container!=\"\"}) by (pod) / sum(kube_pod_container_resource_requests_memory_bytes{namespace=\"$namespace\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "C", + "step": 10 + }, + { + "expr": "sum(kube_pod_container_resource_limits_memory_bytes{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "D", + "step": 10 + }, + { + "expr": "sum(container_memory_working_set_bytes{cluster=\"$cluster\", namespace=\"$namespace\",container!=\"\"}) by (pod) / sum(kube_pod_container_resource_limits_memory_bytes{namespace=\"$namespace\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "E", + "step": 10 + }, + { + "expr": "sum(container_memory_rss{cluster=\"$cluster\", namespace=\"$namespace\",container!=\"\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "F", + "step": 10 + }, + { + "expr": "sum(container_memory_cache{cluster=\"$cluster\", namespace=\"$namespace\",container!=\"\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "G", + "step": 10 + }, + { + "expr": "sum(container_memory_swap{cluster=\"$cluster\", namespace=\"$namespace\",container!=\"\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "H", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Memory Quota", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "transform": "table", + "type": "table", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Memory Quota", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 5, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "Current Receive Bandwidth", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #A", + "thresholds": [ + + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Current Transmit Bandwidth", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #B", + "thresholds": [ + + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Rate of Received Packets", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #C", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Rate of Transmitted Packets", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #D", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Rate of Received Packets Dropped", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #E", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Rate of Transmitted Packets Dropped", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #F", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Pod", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": true, + "linkTooltip": "Drill down to pods", + "linkUrl": "./d/6581e46e4e5c7ba40a07646395ef7b23/k8s-resources-pod?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$namespace&var-pod=$__cell", + "pattern": "pod", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "/.*/", + "thresholds": [ + + ], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "expr": "sum(irate(container_network_receive_bytes_total{cluster=\"$cluster\", namespace=~\"$namespace\"}[$interval])) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10 + }, + { + "expr": "sum(irate(container_network_transmit_bytes_total{cluster=\"$cluster\", namespace=~\"$namespace\"}[$interval])) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + }, + { + "expr": "sum(irate(container_network_receive_packets_total{cluster=\"$cluster\", namespace=~\"$namespace\"}[$interval])) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "C", + "step": 10 + }, + { + "expr": "sum(irate(container_network_transmit_packets_total{cluster=\"$cluster\", namespace=~\"$namespace\"}[$interval])) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "D", + "step": 10 + }, + { + "expr": "sum(irate(container_network_receive_packets_dropped_total{cluster=\"$cluster\", namespace=~\"$namespace\"}[$interval])) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "E", + "step": 10 + }, + { + "expr": "sum(irate(container_network_transmit_packets_dropped_total{cluster=\"$cluster\", namespace=~\"$namespace\"}[$interval])) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "F", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Current Network Usage", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "transform": "table", + "type": "table", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Network", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 6, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_receive_bytes_total{cluster=\"$cluster\", namespace=~\"$namespace\"}[$interval])) by (pod)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{pod}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Receive Bandwidth", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Network", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 7, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_transmit_bytes_total{cluster=\"$cluster\", namespace=~\"$namespace\"}[$interval])) by (pod)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{pod}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Transmit Bandwidth", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Network", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 8, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_receive_packets_total{cluster=\"$cluster\", namespace=~\"$namespace\"}[$interval])) by (pod)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{pod}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Received Packets", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Network", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 9, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_receive_packets_total{cluster=\"$cluster\", namespace=~\"$namespace\"}[$interval])) by (pod)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{pod}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Transmitted Packets", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Network", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 10, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_receive_packets_dropped_total{cluster=\"$cluster\", namespace=~\"$namespace\"}[$interval])) by (pod)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{pod}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Received Packets Dropped", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Network", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 11, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_transmit_packets_dropped_total{cluster=\"$cluster\", namespace=~\"$namespace\"}[$interval])) by (pod)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{pod}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Transmitted Packets Dropped", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Network", + "titleSize": "h6" + } + ], + "schemaVersion": 14, + "style": "dark", + "tags": [ + "kubernetes-mixin" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": null, + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + "text": "prod", + "value": "prod" + }, + "datasource": "$datasource", + "hide": 2, + "includeAll": false, + "label": "cluster", + "multi": false, + "name": "cluster", + "options": [ + + ], + "query": "label_values(kube_pod_info, cluster)", + "refresh": 1, + "regex": "", + "sort": 2, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + "text": "prod", + "value": "prod" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": "namespace", + "multi": false, + "name": "namespace", + "options": [ + + ], + "query": "label_values(kube_pod_info{cluster=\"$cluster\"}, namespace)", + "refresh": 1, + "regex": "", + "sort": 2, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "auto": false, + "auto_count": 30, + "auto_min": "10s", + "current": { + "text": "5m", + "value": "5m" + }, + "datasource": "prometheus", + "hide": 2, + "includeAll": false, + "label": null, + "multi": false, + "name": "interval", + "options": [ + { + "selected": true, + "text": "4h", + "value": "4h" + } + ], + "query": "4h", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "interval", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "", + "title": "Kubernetes / Compute Resources / Namespace (Pods)", + "uid": "85a562078cdf77779eaa1add43ccec1e", + "version": 0 + } + k8s-resources-node.json: |- + { + "annotations": { + "list": [ + + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "links": [ + + ], + "refresh": "10s", + "rows": [ + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 1, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", node=\"$node\"}) by (pod)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{pod}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU Usage", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "CPU Usage", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 2, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "CPU Usage", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #A", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "CPU Requests", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #B", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "CPU Requests %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #C", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "CPU Limits", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #D", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "CPU Limits %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #E", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "Pod", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "pod", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "/.*/", + "thresholds": [ + + ], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", node=\"$node\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10 + }, + { + "expr": "sum(kube_pod_container_resource_requests_cpu_cores{cluster=\"$cluster\", node=\"$node\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + }, + { + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", node=\"$node\"}) by (pod) / sum(kube_pod_container_resource_requests_cpu_cores{cluster=\"$cluster\", node=\"$node\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "C", + "step": 10 + }, + { + "expr": "sum(kube_pod_container_resource_limits_cpu_cores{cluster=\"$cluster\", node=\"$node\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "D", + "step": 10 + }, + { + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", node=\"$node\"}) by (pod) / sum(kube_pod_container_resource_limits_cpu_cores{cluster=\"$cluster\", node=\"$node\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "E", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU Quota", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "transform": "table", + "type": "table", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "CPU Quota", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 3, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(node_namespace_pod_container:container_memory_working_set_bytes{cluster=\"$cluster\", node=\"$node\", container!=\"\"}) by (pod)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{pod}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Memory Usage (w/o cache)", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Memory Usage", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 4, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "Memory Usage", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #A", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Requests", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #B", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Requests %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #C", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "Memory Limits", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #D", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Limits %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #E", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "Memory Usage (RSS)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #F", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Usage (Cache)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #G", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Usage (Swap)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #H", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Pod", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "pod", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "/.*/", + "thresholds": [ + + ], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "expr": "sum(node_namespace_pod_container:container_memory_working_set_bytes{cluster=\"$cluster\", node=\"$node\",container!=\"\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10 + }, + { + "expr": "sum(kube_pod_container_resource_requests_memory_bytes{cluster=\"$cluster\", node=\"$node\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + }, + { + "expr": "sum(node_namespace_pod_container:container_memory_working_set_bytes{cluster=\"$cluster\", node=\"$node\",container!=\"\"}) by (pod) / sum(kube_pod_container_resource_requests_memory_bytes{node=\"$node\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "C", + "step": 10 + }, + { + "expr": "sum(kube_pod_container_resource_limits_memory_bytes{cluster=\"$cluster\", node=\"$node\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "D", + "step": 10 + }, + { + "expr": "sum(node_namespace_pod_container:container_memory_working_set_bytes{cluster=\"$cluster\", node=\"$node\",container!=\"\"}) by (pod) / sum(kube_pod_container_resource_limits_memory_bytes{node=\"$node\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "E", + "step": 10 + }, + { + "expr": "sum(node_namespace_pod_container:container_memory_rss{cluster=\"$cluster\", node=\"$node\",container!=\"\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "F", + "step": 10 + }, + { + "expr": "sum(node_namespace_pod_container:container_memory_cache{cluster=\"$cluster\", node=\"$node\",container!=\"\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "G", + "step": 10 + }, + { + "expr": "sum(node_namespace_pod_container:container_memory_swap{cluster=\"$cluster\", node=\"$node\",container!=\"\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "H", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Memory Quota", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "transform": "table", + "type": "table", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Memory Quota", + "titleSize": "h6" + } + ], + "schemaVersion": 14, + "style": "dark", + "tags": [ + "kubernetes-mixin" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": null, + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + "text": "prod", + "value": "prod" + }, + "datasource": "$datasource", + "hide": 2, + "includeAll": false, + "label": "cluster", + "multi": false, + "name": "cluster", + "options": [ + + ], + "query": "label_values(kube_pod_info, cluster)", + "refresh": 1, + "regex": "", + "sort": 2, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + "text": "prod", + "value": "prod" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": "node", + "multi": false, + "name": "node", + "options": [ + + ], + "query": "label_values(kube_pod_info{cluster=\"$cluster\"}, node)", + "refresh": 1, + "regex": "", + "sort": 2, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "", + "title": "Kubernetes / Compute Resources / Node (Pods)", + "uid": "200ac8fdbfbb74b39aff88118e4d1c2c", + "version": 0 + } +kind: ConfigMap +metadata: + name: grafana-dashboards-k8s-resources-1 + namespace: monitoring diff --git a/addons/grafana/dashboards-k8s-resources-2.yaml b/addons/grafana/dashboards-k8s-resources-2.yaml new file mode 100644 index 000000000..60dcfec49 --- /dev/null +++ b/addons/grafana/dashboards-k8s-resources-2.yaml @@ -0,0 +1,5753 @@ +apiVersion: v1 +data: + k8s-resources-pod.json: |- + { + "annotations": { + "list": [ + + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "links": [ + + ], + "refresh": "10s", + "rows": [ + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 1, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{namespace=\"$namespace\", pod=\"$pod\", container!=\"POD\", cluster=\"$cluster\"}) by (container)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{container}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU Usage", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "CPU Usage", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 2, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "CPU Usage", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #A", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "CPU Requests", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #B", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "CPU Requests %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #C", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "CPU Limits", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #D", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "CPU Limits %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #E", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "Container", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "container", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "/.*/", + "thresholds": [ + + ], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\", container!=\"POD\"}) by (container)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10 + }, + { + "expr": "sum(kube_pod_container_resource_requests_cpu_cores{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + }, + { + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container) / sum(kube_pod_container_resource_requests_cpu_cores{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "C", + "step": 10 + }, + { + "expr": "sum(kube_pod_container_resource_limits_cpu_cores{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "D", + "step": 10 + }, + { + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container) / sum(kube_pod_container_resource_limits_cpu_cores{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "E", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU Quota", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "transform": "table", + "type": "table", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "CPU Quota", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 3, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(container_memory_rss{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\", container!=\"POD\", container!=\"\"}) by (container)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{container}} (RSS)", + "legendLink": null, + "step": 10 + }, + { + "expr": "sum(container_memory_cache{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\", container!=\"POD\", container!=\"\"}) by (container)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{container}} (Cache)", + "legendLink": null, + "step": 10 + }, + { + "expr": "sum(container_memory_swap{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\", container!=\"POD\", container!=\"\"}) by (container)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{container}} (Swap)", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Memory Usage", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Memory Usage", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 4, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "Memory Usage", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #A", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Requests", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #B", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Requests %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #C", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "Memory Limits", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #D", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Limits %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #E", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "Memory Usage (RSS)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #F", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Usage (Cache)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #G", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Usage (Swap)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #H", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Container", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "container", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "/.*/", + "thresholds": [ + + ], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "expr": "sum(container_memory_working_set_bytes{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\", container!=\"POD\", container!=\"\"}) by (container)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10 + }, + { + "expr": "sum(kube_pod_container_resource_requests_memory_bytes{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + }, + { + "expr": "sum(container_memory_working_set_bytes{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container) / sum(kube_pod_container_resource_requests_memory_bytes{namespace=\"$namespace\", pod=\"$pod\"}) by (container)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "C", + "step": 10 + }, + { + "expr": "sum(kube_pod_container_resource_limits_memory_bytes{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\", container!=\"\"}) by (container)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "D", + "step": 10 + }, + { + "expr": "sum(container_memory_working_set_bytes{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\", container!=\"\"}) by (container) / sum(kube_pod_container_resource_limits_memory_bytes{namespace=\"$namespace\", pod=\"$pod\"}) by (container)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "E", + "step": 10 + }, + { + "expr": "sum(container_memory_rss{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\", container != \"\", container != \"POD\"}) by (container)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "F", + "step": 10 + }, + { + "expr": "sum(container_memory_cache{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\", container != \"\", container != \"POD\"}) by (container)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "G", + "step": 10 + }, + { + "expr": "sum(container_memory_swap{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\", container != \"\", container != \"POD\"}) by (container)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "H", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Memory Quota", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "transform": "table", + "type": "table", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Memory Quota", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 5, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_receive_bytes_total{namespace=~\"$namespace\", pod=~\"$pod\"}[$interval])) by (pod)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{pod}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Receive Bandwidth", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Network", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 6, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_transmit_bytes_total{namespace=~\"$namespace\", pod=~\"$pod\"}[$interval])) by (pod)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{pod}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Transmit Bandwidth", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Network", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 7, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_receive_packets_total{namespace=~\"$namespace\", pod=~\"$pod\"}[$interval])) by (pod)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{pod}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Received Packets", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Network", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 8, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_transmit_packets_total{namespace=~\"$namespace\", pod=~\"$pod\"}[$interval])) by (pod)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{pod}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Transmitted Packets", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Network", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 9, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_receive_packets_dropped_total{namespace=~\"$namespace\", pod=~\"$pod\"}[$interval])) by (pod)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{pod}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Received Packets Dropped", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Network", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 10, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_transmit_packets_dropped_total{namespace=~\"$namespace\", pod=~\"$pod\"}[$interval])) by (pod)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{pod}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Transmitted Packets Dropped", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Network", + "titleSize": "h6" + } + ], + "schemaVersion": 14, + "style": "dark", + "tags": [ + "kubernetes-mixin" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": null, + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + "text": "prod", + "value": "prod" + }, + "datasource": "$datasource", + "hide": 2, + "includeAll": false, + "label": "cluster", + "multi": false, + "name": "cluster", + "options": [ + + ], + "query": "label_values(kube_pod_info, cluster)", + "refresh": 1, + "regex": "", + "sort": 2, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + "text": "prod", + "value": "prod" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": "namespace", + "multi": false, + "name": "namespace", + "options": [ + + ], + "query": "label_values(kube_pod_info{cluster=\"$cluster\"}, namespace)", + "refresh": 1, + "regex": "", + "sort": 2, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + "text": "prod", + "value": "prod" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": "pod", + "multi": false, + "name": "pod", + "options": [ + + ], + "query": "label_values(kube_pod_info{cluster=\"$cluster\", namespace=\"$namespace\"}, pod)", + "refresh": 1, + "regex": "", + "sort": 2, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "auto": false, + "auto_count": 30, + "auto_min": "10s", + "current": { + "text": "5m", + "value": "5m" + }, + "datasource": "prometheus", + "hide": 2, + "includeAll": false, + "label": null, + "multi": false, + "name": "interval", + "options": [ + { + "selected": true, + "text": "4h", + "value": "4h" + } + ], + "query": "4h", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "interval", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "", + "title": "Kubernetes / Compute Resources / Pod", + "uid": "6581e46e4e5c7ba40a07646395ef7b23", + "version": 0 + } + k8s-resources-workload.json: |- + { + "annotations": { + "list": [ + + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "links": [ + + ], + "refresh": "10s", + "rows": [ + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 1, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(\n node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\"}\n * on(namespace,pod)\n group_left(workload, workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{pod}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU Usage", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "CPU Usage", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 2, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "CPU Usage", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #A", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "CPU Requests", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #B", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "CPU Requests %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #C", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "CPU Limits", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #D", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "CPU Limits %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #E", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "Pod", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": true, + "linkTooltip": "Drill down", + "linkUrl": "./d/6581e46e4e5c7ba40a07646395ef7b23/k8s-resources-pod?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$namespace&var-pod=$__cell", + "pattern": "pod", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "/.*/", + "thresholds": [ + + ], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "expr": "sum(\n node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\"}\n * on(namespace,pod)\n group_left(workload, workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10 + }, + { + "expr": "sum(\n kube_pod_container_resource_requests_cpu_cores{cluster=\"$cluster\", namespace=\"$namespace\"}\n * on(namespace,pod)\n group_left(workload, workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + }, + { + "expr": "sum(\n node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\"}\n * on(namespace,pod)\n group_left(workload, workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n/sum(\n kube_pod_container_resource_requests_cpu_cores{cluster=\"$cluster\", namespace=\"$namespace\"}\n * on(namespace,pod)\n group_left(workload, workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "C", + "step": 10 + }, + { + "expr": "sum(\n kube_pod_container_resource_limits_cpu_cores{cluster=\"$cluster\", namespace=\"$namespace\"}\n * on(namespace,pod)\n group_left(workload, workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "D", + "step": 10 + }, + { + "expr": "sum(\n node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\"}\n * on(namespace,pod)\n group_left(workload, workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n/sum(\n kube_pod_container_resource_limits_cpu_cores{cluster=\"$cluster\", namespace=\"$namespace\"}\n * on(namespace,pod)\n group_left(workload, workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "E", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU Quota", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "transform": "table", + "type": "table", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "CPU Quota", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 3, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(\n container_memory_working_set_bytes{cluster=\"$cluster\", namespace=\"$namespace\", container!=\"\"}\n * on(namespace,pod)\n group_left(workload, workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{pod}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Memory Usage", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Memory Usage", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 4, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "Memory Usage", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #A", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Requests", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #B", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Requests %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #C", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "Memory Limits", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #D", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Limits %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #E", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "Pod", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": true, + "linkTooltip": "Drill down", + "linkUrl": "./d/6581e46e4e5c7ba40a07646395ef7b23/k8s-resources-pod?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$namespace&var-pod=$__cell", + "pattern": "pod", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "/.*/", + "thresholds": [ + + ], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "expr": "sum(\n container_memory_working_set_bytes{cluster=\"$cluster\", namespace=\"$namespace\", container!=\"\"}\n * on(namespace,pod)\n group_left(workload, workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10 + }, + { + "expr": "sum(\n kube_pod_container_resource_requests_memory_bytes{cluster=\"$cluster\", namespace=\"$namespace\"}\n * on(namespace,pod)\n group_left(workload, workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + }, + { + "expr": "sum(\n container_memory_working_set_bytes{cluster=\"$cluster\", namespace=\"$namespace\", container!=\"\"}\n * on(namespace,pod)\n group_left(workload, workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n/sum(\n kube_pod_container_resource_requests_memory_bytes{cluster=\"$cluster\", namespace=\"$namespace\"}\n * on(namespace,pod)\n group_left(workload, workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "C", + "step": 10 + }, + { + "expr": "sum(\n kube_pod_container_resource_limits_memory_bytes{cluster=\"$cluster\", namespace=\"$namespace\"}\n * on(namespace,pod)\n group_left(workload, workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "D", + "step": 10 + }, + { + "expr": "sum(\n container_memory_working_set_bytes{cluster=\"$cluster\", namespace=\"$namespace\", container!=\"\"}\n * on(namespace,pod)\n group_left(workload, workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n/sum(\n kube_pod_container_resource_limits_memory_bytes{cluster=\"$cluster\", namespace=\"$namespace\"}\n * on(namespace,pod)\n group_left(workload, workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "E", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Memory Quota", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "transform": "table", + "type": "table", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Memory Quota", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 5, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "Current Receive Bandwidth", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #A", + "thresholds": [ + + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Current Transmit Bandwidth", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #B", + "thresholds": [ + + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Rate of Received Packets", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #C", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Rate of Transmitted Packets", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #D", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Rate of Received Packets Dropped", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #E", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Rate of Transmitted Packets Dropped", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #F", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Pod", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": true, + "linkTooltip": "Drill down", + "linkUrl": "./d/6581e46e4e5c7ba40a07646395ef7b23/k8s-resources-pod?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$namespace&var-pod=$__cell", + "pattern": "pod", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "/.*/", + "thresholds": [ + + ], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "expr": "(sum(irate(container_network_receive_bytes_total{cluster=\"$cluster\", namespace=~\"$namespace\"}[$interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=~\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10 + }, + { + "expr": "(sum(irate(container_network_transmit_bytes_total{cluster=\"$cluster\", namespace=~\"$namespace\"}[$interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=~\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + }, + { + "expr": "(sum(irate(container_network_receive_packets_total{cluster=\"$cluster\", namespace=~\"$namespace\"}[$interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=~\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "C", + "step": 10 + }, + { + "expr": "(sum(irate(container_network_transmit_packets_total{cluster=\"$cluster\", namespace=~\"$namespace\"}[$interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=~\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "D", + "step": 10 + }, + { + "expr": "(sum(irate(container_network_receive_packets_dropped_total{cluster=\"$cluster\", namespace=~\"$namespace\"}[$interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=~\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "E", + "step": 10 + }, + { + "expr": "(sum(irate(container_network_transmit_packets_dropped_total{cluster=\"$cluster\", namespace=~\"$namespace\"}[$interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=~\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "F", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Current Network Usage", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "transform": "table", + "type": "table", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Network", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 6, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "(sum(irate(container_network_receive_bytes_total{cluster=\"$cluster\", namespace=~\"$namespace\"}[$interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=~\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{pod}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Receive Bandwidth", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Network", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 7, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "(sum(irate(container_network_transmit_bytes_total{cluster=\"$cluster\", namespace=~\"$namespace\"}[$interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=~\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{pod}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Transmit Bandwidth", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Network", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 8, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "(avg(irate(container_network_receive_bytes_total{cluster=\"$cluster\", namespace=~\"$namespace\"}[$interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=~\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{pod}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Average Container Bandwidth by Pod: Received", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Network", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 9, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "(avg(irate(container_network_transmit_bytes_total{cluster=\"$cluster\", namespace=~\"$namespace\"}[$interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=~\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{pod}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Average Container Bandwidth by Pod: Transmitted", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Network", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 10, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "(sum(irate(container_network_receive_packets_total{cluster=\"$cluster\", namespace=~\"$namespace\"}[$interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=~\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{pod}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Received Packets", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Network", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 11, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "(sum(irate(container_network_transmit_packets_total{cluster=\"$cluster\", namespace=~\"$namespace\"}[$interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=~\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{pod}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Transmitted Packets", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Network", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 12, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "(sum(irate(container_network_receive_packets_dropped_total{cluster=\"$cluster\", namespace=~\"$namespace\"}[$interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=~\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{pod}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Received Packets Dropped", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Network", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 13, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "(sum(irate(container_network_transmit_packets_dropped_total{cluster=\"$cluster\", namespace=~\"$namespace\"}[$interval])\n* on (namespace,pod) \ngroup_left(workload,workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=~\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{pod}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Transmitted Packets Dropped", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Network", + "titleSize": "h6" + } + ], + "schemaVersion": 14, + "style": "dark", + "tags": [ + "kubernetes-mixin" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": null, + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + "text": "prod", + "value": "prod" + }, + "datasource": "$datasource", + "hide": 2, + "includeAll": false, + "label": "cluster", + "multi": false, + "name": "cluster", + "options": [ + + ], + "query": "label_values(kube_pod_info, cluster)", + "refresh": 1, + "regex": "", + "sort": 2, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + "text": "prod", + "value": "prod" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": "namespace", + "multi": false, + "name": "namespace", + "options": [ + + ], + "query": "label_values(kube_pod_info{cluster=\"$cluster\"}, namespace)", + "refresh": 1, + "regex": "", + "sort": 2, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + "text": "prod", + "value": "prod" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": "workload", + "multi": false, + "name": "workload", + "options": [ + + ], + "query": "label_values(mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\"}, workload)", + "refresh": 1, + "regex": "", + "sort": 2, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + "text": "prod", + "value": "prod" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": "type", + "multi": false, + "name": "type", + "options": [ + + ], + "query": "label_values(mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\"}, workload_type)", + "refresh": 1, + "regex": "", + "sort": 2, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "auto": false, + "auto_count": 30, + "auto_min": "10s", + "current": { + "text": "5m", + "value": "5m" + }, + "datasource": "prometheus", + "hide": 2, + "includeAll": false, + "label": null, + "multi": false, + "name": "interval", + "options": [ + { + "selected": true, + "text": "4h", + "value": "4h" + } + ], + "query": "4h", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "interval", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "", + "title": "Kubernetes / Compute Resources / Workload", + "uid": "a164a7f0339f99e89cea5cb47e9be617", + "version": 0 + } + k8s-resources-workloads-namespace.json: |- + { + "annotations": { + "list": [ + + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "links": [ + + ], + "refresh": "10s", + "rows": [ + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 1, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(\n node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\"}\n* on(namespace,pod)\n group_left(workload, workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{workload}} - {{workload_type}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU Usage", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "CPU Usage", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 2, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "Running Pods", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 0, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #A", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "CPU Usage", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #B", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "CPU Requests", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #C", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "CPU Requests %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #D", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "CPU Limits", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #E", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "CPU Limits %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #F", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "Workload", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": true, + "linkTooltip": "Drill down", + "linkUrl": "./d/a164a7f0339f99e89cea5cb47e9be617/k8s-resources-workload?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$namespace&var-workload=$__cell&var-type=$__cell_2", + "pattern": "workload", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "Workload Type", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "workload_type", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "/.*/", + "thresholds": [ + + ], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "expr": "count(mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}) by (workload, workload_type)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10 + }, + { + "expr": "sum(\n node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\"}\n* on(namespace,pod)\n group_left(workload, workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + }, + { + "expr": "sum(\n kube_pod_container_resource_requests_cpu_cores{cluster=\"$cluster\", namespace=\"$namespace\"}\n* on(namespace,pod)\n group_left(workload, workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "C", + "step": 10 + }, + { + "expr": "sum(\n node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\"}\n* on(namespace,pod)\n group_left(workload, workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n/sum(\n kube_pod_container_resource_requests_cpu_cores{cluster=\"$cluster\", namespace=\"$namespace\"}\n* on(namespace,pod)\n group_left(workload, workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "D", + "step": 10 + }, + { + "expr": "sum(\n kube_pod_container_resource_limits_cpu_cores{cluster=\"$cluster\", namespace=\"$namespace\"}\n* on(namespace,pod)\n group_left(workload, workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "E", + "step": 10 + }, + { + "expr": "sum(\n node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\"}\n* on(namespace,pod)\n group_left(workload, workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n/sum(\n kube_pod_container_resource_limits_cpu_cores{cluster=\"$cluster\", namespace=\"$namespace\"}\n* on(namespace,pod)\n group_left(workload, workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "F", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU Quota", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "transform": "table", + "type": "table", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "CPU Quota", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 3, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(\n container_memory_working_set_bytes{cluster=\"$cluster\", namespace=\"$namespace\", container!=\"\"}\n * on(namespace,pod)\n group_left(workload, workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{workload}} - {{workload_type}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Memory Usage", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Memory Usage", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 4, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "Running Pods", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 0, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #A", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "Memory Usage", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #B", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Requests", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #C", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Requests %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #D", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "Memory Limits", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #E", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Limits %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #F", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "Workload", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": true, + "linkTooltip": "Drill down", + "linkUrl": "./d/a164a7f0339f99e89cea5cb47e9be617/k8s-resources-workload?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$namespace&var-workload=$__cell&var-type=$__cell_2", + "pattern": "workload", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "Workload Type", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "workload_type", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "/.*/", + "thresholds": [ + + ], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "expr": "count(mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}) by (workload, workload_type)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10 + }, + { + "expr": "sum(\n container_memory_working_set_bytes{cluster=\"$cluster\", namespace=\"$namespace\", container!=\"\"}\n * on(namespace,pod)\n group_left(workload, workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + }, + { + "expr": "sum(\n kube_pod_container_resource_requests_memory_bytes{cluster=\"$cluster\", namespace=\"$namespace\"}\n* on(namespace,pod)\n group_left(workload, workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "C", + "step": 10 + }, + { + "expr": "sum(\n container_memory_working_set_bytes{cluster=\"$cluster\", namespace=\"$namespace\", container!=\"\"}\n * on(namespace,pod)\n group_left(workload, workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n/sum(\n kube_pod_container_resource_requests_memory_bytes{cluster=\"$cluster\", namespace=\"$namespace\"}\n* on(namespace,pod)\n group_left(workload, workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "D", + "step": 10 + }, + { + "expr": "sum(\n kube_pod_container_resource_limits_memory_bytes{cluster=\"$cluster\", namespace=\"$namespace\"}\n* on(namespace,pod)\n group_left(workload, workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "E", + "step": 10 + }, + { + "expr": "sum(\n container_memory_working_set_bytes{cluster=\"$cluster\", namespace=\"$namespace\", container!=\"\"}\n * on(namespace,pod)\n group_left(workload, workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n/sum(\n kube_pod_container_resource_limits_memory_bytes{cluster=\"$cluster\", namespace=\"$namespace\"}\n* on(namespace,pod)\n group_left(workload, workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "F", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Memory Quota", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "transform": "table", + "type": "table", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Memory Quota", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 5, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "Current Receive Bandwidth", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #A", + "thresholds": [ + + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Current Transmit Bandwidth", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #B", + "thresholds": [ + + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Rate of Received Packets", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #C", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Rate of Transmitted Packets", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #D", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Rate of Received Packets Dropped", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #E", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Rate of Transmitted Packets Dropped", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #F", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Workload", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": true, + "linkTooltip": "Drill down to pods", + "linkUrl": "./d/a164a7f0339f99e89cea5cb47e9be617/k8s-resources-workload?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$namespace&var-workload=$__cell&var-type=$type", + "pattern": "workload", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "Workload Type", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "workload_type", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "/.*/", + "thresholds": [ + + ], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "expr": "(sum(irate(container_network_receive_bytes_total{cluster=\"$cluster\", namespace=~\"$namespace\"}[$interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=~\"$namespace\", workload_type=\"$type\"}) by (workload))\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10 + }, + { + "expr": "(sum(irate(container_network_transmit_bytes_total{cluster=\"$cluster\", namespace=~\"$namespace\"}[$interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=~\"$namespace\", workload_type=\"$type\"}) by (workload))\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + }, + { + "expr": "(sum(irate(container_network_receive_packets_total{cluster=\"$cluster\", namespace=~\"$namespace\"}[$interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=~\"$namespace\", workload_type=\"$type\"}) by (workload))\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "C", + "step": 10 + }, + { + "expr": "(sum(irate(container_network_transmit_packets_total{cluster=\"$cluster\", namespace=~\"$namespace\"}[$interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=~\"$namespace\", workload_type=\"$type\"}) by (workload))\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "D", + "step": 10 + }, + { + "expr": "(sum(irate(container_network_receive_packets_dropped_total{cluster=\"$cluster\", namespace=~\"$namespace\"}[$interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=~\"$namespace\", workload_type=\"$type\"}) by (workload))\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "E", + "step": 10 + }, + { + "expr": "(sum(irate(container_network_transmit_packets_dropped_total{cluster=\"$cluster\", namespace=~\"$namespace\"}[$interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=~\"$namespace\", workload_type=\"$type\"}) by (workload))\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "F", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Current Network Usage", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "transform": "table", + "type": "table", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Network", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 6, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "(sum(irate(container_network_receive_bytes_total{cluster=\"$cluster\", namespace=~\"$namespace\"}[$interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=~\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{workload}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Receive Bandwidth", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Network", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 7, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "(sum(irate(container_network_transmit_bytes_total{cluster=\"$cluster\", namespace=~\"$namespace\"}[$interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=~\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{workload}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Transmit Bandwidth", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Network", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 8, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "(avg(irate(container_network_receive_bytes_total{cluster=\"$cluster\", namespace=~\"$namespace\"}[$interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=~\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{workload}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Average Container Bandwidth by Workload: Received", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Network", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 9, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "(avg(irate(container_network_transmit_bytes_total{cluster=\"$cluster\", namespace=~\"$namespace\"}[$interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=~\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{workload}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Average Container Bandwidth by Workload: Transmitted", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Network", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 10, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "(sum(irate(container_network_receive_packets_total{cluster=\"$cluster\", namespace=~\"$namespace\"}[$interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=~\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{workload}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Received Packets", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Network", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 11, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "(sum(irate(container_network_transmit_packets_total{cluster=\"$cluster\", namespace=~\"$namespace\"}[$interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=~\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{workload}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Transmitted Packets", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Network", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 12, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "(sum(irate(container_network_receive_packets_dropped_total{cluster=\"$cluster\", namespace=~\"$namespace\"}[$interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=~\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{workload}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Received Packets Dropped", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Network", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 13, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "(sum(irate(container_network_transmit_packets_dropped_total{cluster=\"$cluster\", namespace=~\"$namespace\"}[$interval])\n* on (namespace,pod) \ngroup_left(workload,workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=~\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{workload}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Transmitted Packets Dropped", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Network", + "titleSize": "h6" + } + ], + "schemaVersion": 14, + "style": "dark", + "tags": [ + "kubernetes-mixin" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": null, + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + "text": "prod", + "value": "prod" + }, + "datasource": "$datasource", + "hide": 2, + "includeAll": false, + "label": "cluster", + "multi": false, + "name": "cluster", + "options": [ + + ], + "query": "label_values(kube_pod_info, cluster)", + "refresh": 1, + "regex": "", + "sort": 2, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + "text": "prod", + "value": "prod" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": "namespace", + "multi": false, + "name": "namespace", + "options": [ + + ], + "query": "label_values(kube_pod_info{cluster=\"$cluster\"}, namespace)", + "refresh": 1, + "regex": "", + "sort": 2, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "auto": false, + "auto_count": 30, + "auto_min": "10s", + "current": { + "text": "5m", + "value": "5m" + }, + "datasource": "prometheus", + "hide": 2, + "includeAll": false, + "label": null, + "multi": false, + "name": "interval", + "options": [ + { + "selected": true, + "text": "4h", + "value": "4h" + } + ], + "query": "4h", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "interval", + "useTags": false + }, + { + "allValue": null, + "auto": false, + "auto_count": 30, + "auto_min": "10s", + "current": { + "text": "deployment", + "value": "deployment" + }, + "datasource": "$datasource", + "definition": "label_values(mixin_pod_workload{namespace=~\"$namespace\", workload=~\".+\"}, workload_type)", + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "type", + "options": [ + + ], + "query": "label_values(mixin_pod_workload{namespace=~\"$namespace\", workload=~\".+\"}, workload_type)", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "", + "title": "Kubernetes / Compute Resources / Namespace (Workloads)", + "uid": "a87fb0d919ec0ea5f6543124e16c42a5", + "version": 0 + } +kind: ConfigMap +metadata: + name: grafana-dashboards-k8s-resources-2 + namespace: monitoring diff --git a/addons/grafana/dashboards-k8s.yaml b/addons/grafana/dashboards-k8s.yaml index 299b919f0..56a2e5bca 100644 --- a/addons/grafana/dashboards-k8s.yaml +++ b/addons/grafana/dashboards-k8s.yaml @@ -1236,7 +1236,7 @@ data: "query": "label_values(apiserver_request_total{job=\"apiserver\"}, instance)", "refresh": 2, "regex": "", - "sort": 0, + "sort": 1, "tagValuesQuery": "", "tags": [ @@ -2347,7 +2347,7 @@ data: "query": "label_values(process_cpu_seconds_total{job=\"kube-controller-manager\"}, instance)", "refresh": 2, "regex": "", - "sort": 0, + "sort": 1, "tagValuesQuery": "", "tags": [ @@ -2846,7 +2846,7 @@ data: "query": "label_values(kubelet_volume_stats_capacity_bytes, cluster)", "refresh": 2, "regex": "", - "sort": 0, + "sort": 1, "tagValuesQuery": "", "tags": [ @@ -2872,7 +2872,7 @@ data: "query": "label_values(kubelet_volume_stats_capacity_bytes{cluster=\"$cluster\", job=\"kubelet\"}, namespace)", "refresh": 2, "regex": "", - "sort": 0, + "sort": 1, "tagValuesQuery": "", "tags": [ @@ -2898,7 +2898,7 @@ data: "query": "label_values(kubelet_volume_stats_capacity_bytes{cluster=\"$cluster\", job=\"kubelet\", namespace=\"$namespace\"}, persistentvolumeclaim)", "refresh": 2, "regex": "", - "sort": 0, + "sort": 1, "tagValuesQuery": "", "tags": [ @@ -3478,7 +3478,7 @@ data: "query": "label_values(kube_pod_info, cluster)", "refresh": 2, "regex": "", - "sort": 0, + "sort": 1, "tagValuesQuery": "", "tags": [ @@ -3504,7 +3504,7 @@ data: "query": "label_values(kube_pod_info{cluster=\"$cluster\"}, namespace)", "refresh": 2, "regex": "", - "sort": 0, + "sort": 1, "tagValuesQuery": "", "tags": [ @@ -3530,7 +3530,7 @@ data: "query": "label_values(kube_pod_info{cluster=\"$cluster\", namespace=~\"$namespace\"}, pod)", "refresh": 2, "regex": "", - "sort": 0, + "sort": 1, "tagValuesQuery": "", "tags": [ @@ -3556,7 +3556,7 @@ data: "query": "label_values(kube_pod_container_info{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}, container)", "refresh": 2, "regex": "", - "sort": 0, + "sort": 1, "tagValuesQuery": "", "tags": [ @@ -4592,7 +4592,7 @@ data: "query": "label_values(process_cpu_seconds_total{job=\"kube-scheduler\"}, instance)", "refresh": 2, "regex": "", - "sort": 0, + "sort": 1, "tagValuesQuery": "", "tags": [ @@ -5444,7 +5444,7 @@ data: "query": "label_values(kube_statefulset_metadata_generation, cluster)", "refresh": 2, "regex": "", - "sort": 0, + "sort": 1, "tagValuesQuery": "", "tags": [ @@ -5470,7 +5470,7 @@ data: "query": "label_values(kube_statefulset_metadata_generation{job=\"kube-state-metrics\", cluster=\"$cluster\"}, namespace)", "refresh": 2, "regex": "", - "sort": 0, + "sort": 1, "tagValuesQuery": "", "tags": [ @@ -5496,7 +5496,7 @@ data: "query": "label_values(kube_statefulset_metadata_generation{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\"}, statefulset)", "refresh": 2, "regex": "", - "sort": 0, + "sort": 1, "tagValuesQuery": "", "tags": [ diff --git a/addons/prometheus/rules.yaml b/addons/prometheus/rules.yaml index 51f5aa9f8..d24d73d55 100644 --- a/addons/prometheus/rules.yaml +++ b/addons/prometheus/rules.yaml @@ -145,6 +145,32 @@ data: kube.yaml: |- { "groups": [ + { + "name": "kube-apiserver.rules", + "rules": [ + { + "expr": "histogram_quantile(0.99, sum(rate(apiserver_request_duration_seconds_bucket{job=\"apiserver\"}[5m])) without(instance, pod))\n", + "labels": { + "quantile": "0.99" + }, + "record": "cluster_quantile:apiserver_request_duration_seconds:histogram_quantile" + }, + { + "expr": "histogram_quantile(0.9, sum(rate(apiserver_request_duration_seconds_bucket{job=\"apiserver\"}[5m])) without(instance, pod))\n", + "labels": { + "quantile": "0.9" + }, + "record": "cluster_quantile:apiserver_request_duration_seconds:histogram_quantile" + }, + { + "expr": "histogram_quantile(0.5, sum(rate(apiserver_request_duration_seconds_bucket{job=\"apiserver\"}[5m])) without(instance, pod))\n", + "labels": { + "quantile": "0.5" + }, + "record": "cluster_quantile:apiserver_request_duration_seconds:histogram_quantile" + } + ] + }, { "name": "k8s.rules", "rules": [ @@ -156,16 +182,32 @@ data: "expr": "sum by (namespace, pod, container) (\n rate(container_cpu_usage_seconds_total{job=\"kubernetes-cadvisor\", image!=\"\", container!=\"POD\"}[5m])\n) * on (namespace, pod) group_left(node) max by(namespace, pod, node) (kube_pod_info)\n", "record": "node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate" }, + { + "expr": "container_memory_working_set_bytes{job=\"kubernetes-cadvisor\", image!=\"\"}\n* on (namespace, pod) group_left(node) max by(namespace, pod, node) (kube_pod_info)\n", + "record": "node_namespace_pod_container:container_memory_working_set_bytes" + }, + { + "expr": "container_memory_rss{job=\"kubernetes-cadvisor\", image!=\"\"}\n* on (namespace, pod) group_left(node) max by(namespace, pod, node) (kube_pod_info)\n", + "record": "node_namespace_pod_container:container_memory_rss" + }, + { + "expr": "container_memory_cache{job=\"kubernetes-cadvisor\", image!=\"\"}\n* on (namespace, pod) group_left(node) max by(namespace, pod, node) (kube_pod_info)\n", + "record": "node_namespace_pod_container:container_memory_cache" + }, + { + "expr": "container_memory_swap{job=\"kubernetes-cadvisor\", image!=\"\"}\n* on (namespace, pod) group_left(node) max by(namespace, pod, node) (kube_pod_info)\n", + "record": "node_namespace_pod_container:container_memory_swap" + }, { "expr": "sum(container_memory_usage_bytes{job=\"kubernetes-cadvisor\", image!=\"\", container!=\"POD\"}) by (namespace)\n", "record": "namespace:container_memory_usage_bytes:sum" }, { - "expr": "sum by (namespace, label_name) (\n sum(kube_pod_container_resource_requests_memory_bytes{job=\"kube-state-metrics\"} * on (endpoint, instance, job, namespace, pod, service) group_left(phase) (kube_pod_status_phase{phase=~\"^(Pending|Running)$\"} == 1)) by (namespace, pod)\n * on (namespace, pod)\n group_left(label_name) kube_pod_labels{job=\"kube-state-metrics\"}\n)\n", + "expr": "sum by (namespace, label_name) (\n sum(kube_pod_container_resource_requests_memory_bytes{job=\"kube-state-metrics\"} * on (endpoint, instance, job, namespace, pod, service) group_left(phase) (kube_pod_status_phase{phase=~\"Pending|Running\"} == 1)) by (namespace, pod)\n * on (namespace, pod)\n group_left(label_name) kube_pod_labels{job=\"kube-state-metrics\"}\n)\n", "record": "namespace:kube_pod_container_resource_requests_memory_bytes:sum" }, { - "expr": "sum by (namespace, label_name) (\n sum(kube_pod_container_resource_requests_cpu_cores{job=\"kube-state-metrics\"} * on (endpoint, instance, job, namespace, pod, service) group_left(phase) (kube_pod_status_phase{phase=~\"^(Pending|Running)$\"} == 1)) by (namespace, pod)\n * on (namespace, pod)\n group_left(label_name) kube_pod_labels{job=\"kube-state-metrics\"}\n)\n", + "expr": "sum by (namespace, label_name) (\n sum(kube_pod_container_resource_requests_cpu_cores{job=\"kube-state-metrics\"} * on (endpoint, instance, job, namespace, pod, service) group_left(phase) (kube_pod_status_phase{phase=~\"Pending|Running\"} == 1)) by (namespace, pod)\n * on (namespace, pod)\n group_left(label_name) kube_pod_labels{job=\"kube-state-metrics\"}\n)\n", "record": "namespace:kube_pod_container_resource_requests_cpu_cores:sum" }, { @@ -259,32 +301,6 @@ data: } ] }, - { - "name": "kube-apiserver.rules", - "rules": [ - { - "expr": "histogram_quantile(0.99, sum(rate(apiserver_request_duration_seconds_bucket{job=\"apiserver\"}[5m])) without(instance, pod))\n", - "labels": { - "quantile": "0.99" - }, - "record": "cluster_quantile:apiserver_request_duration_seconds:histogram_quantile" - }, - { - "expr": "histogram_quantile(0.9, sum(rate(apiserver_request_duration_seconds_bucket{job=\"apiserver\"}[5m])) without(instance, pod))\n", - "labels": { - "quantile": "0.9" - }, - "record": "cluster_quantile:apiserver_request_duration_seconds:histogram_quantile" - }, - { - "expr": "histogram_quantile(0.5, sum(rate(apiserver_request_duration_seconds_bucket{job=\"apiserver\"}[5m])) without(instance, pod))\n", - "labels": { - "quantile": "0.5" - }, - "record": "cluster_quantile:apiserver_request_duration_seconds:histogram_quantile" - } - ] - }, { "name": "node.rules", "rules": [ @@ -301,61 +317,8 @@ data: "record": "node:node_num_cpu:sum" }, { - "expr": "sum(node_memory_MemFree_bytes{job=\"node-exporter\"} + node_memory_Cached_bytes{job=\"node-exporter\"} + node_memory_Buffers_bytes{job=\"node-exporter\"})\n", - "record": ":node_memory_MemFreeCachedBuffers_bytes:sum" - } - ] - }, - { - "name": "kubernetes-absent", - "rules": [ - { - "alert": "KubeAPIDown", - "annotations": { - "message": "KubeAPI has disappeared from Prometheus target discovery.", - "runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubeapidown" - }, - "expr": "absent(up{job=\"apiserver\"} == 1)\n", - "for": "15m", - "labels": { - "severity": "critical" - } - }, - { - "alert": "KubeControllerManagerDown", - "annotations": { - "message": "KubeControllerManager has disappeared from Prometheus target discovery.", - "runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubecontrollermanagerdown" - }, - "expr": "absent(up{job=\"kube-controller-manager\"} == 1)\n", - "for": "15m", - "labels": { - "severity": "critical" - } - }, - { - "alert": "KubeSchedulerDown", - "annotations": { - "message": "KubeScheduler has disappeared from Prometheus target discovery.", - "runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubeschedulerdown" - }, - "expr": "absent(up{job=\"kube-scheduler\"} == 1)\n", - "for": "15m", - "labels": { - "severity": "critical" - } - }, - { - "alert": "KubeletDown", - "annotations": { - "message": "Kubelet has disappeared from Prometheus target discovery.", - "runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubeletdown" - }, - "expr": "absent(up{job=\"kubelet\"} == 1)\n", - "for": "15m", - "labels": { - "severity": "critical" - } + "expr": "sum(\n node_memory_MemAvailable_bytes{job=\"node-exporter\"} or\n (\n node_memory_Buffers_bytes{job=\"node-exporter\"} +\n node_memory_Cached_bytes{job=\"node-exporter\"} +\n node_memory_MemFree_bytes{job=\"node-exporter\"} +\n node_memory_Slab_bytes{job=\"node-exporter\"}\n )\n)\n", + "record": ":node_memory_MemAvailable_bytes:sum" } ] }, @@ -458,6 +421,18 @@ data: "severity": "critical" } }, + { + "alert": "KubeContainerWaiting", + "annotations": { + "message": "Pod {{ $labels.namespace }}/{{ $labels.pod }} container {{ $labels.container}} has been in waiting state for longer than 1 hour.", + "runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubecontainerwaiting" + }, + "expr": "sum by (namespace, pod, container) (kube_pod_container_status_waiting_reason{job=\"kube-state-metrics\"}) > 0\n", + "for": "1h", + "labels": { + "severity": "warning" + } + }, { "alert": "KubeDaemonSetNotScheduled", "annotations": { @@ -665,18 +640,6 @@ data: { "name": "kubernetes-system", "rules": [ - { - "alert": "KubeNodeNotReady", - "annotations": { - "message": "{{ $labels.node }} has been unready for more than 15 minutes.", - "runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubenodenotready" - }, - "expr": "kube_node_status_condition{job=\"kube-state-metrics\",condition=\"Ready\",status=\"true\"} == 0\n", - "for": "15m", - "labels": { - "severity": "warning" - } - }, { "alert": "KubeVersionMismatch", "annotations": { @@ -700,26 +663,19 @@ data: "labels": { "severity": "warning" } - }, - { - "alert": "KubeletTooManyPods", - "annotations": { - "message": "Kubelet '{{ $labels.node }}' is running at {{ $value | humanizePercentage }} of its Pod capacity.", - "runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubelettoomanypods" - }, - "expr": "max(max(kubelet_running_pod_count{job=\"kubelet\"}) by(instance) * on(instance) group_left(node) kubelet_node_name{job=\"kubelet\"}) by(node) / max(kube_node_status_capacity_pods{job=\"kube-state-metrics\"}) by(node) > 0.95\n", - "for": "15m", - "labels": { - "severity": "warning" - } - }, + } + ] + }, + { + "name": "kubernetes-system-apiserver", + "rules": [ { "alert": "KubeAPILatencyHigh", "annotations": { "message": "The API server has a 99th percentile latency of {{ $value }} seconds for {{ $labels.verb }} {{ $labels.resource }}.", "runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubeapilatencyhigh" }, - "expr": "cluster_quantile:apiserver_request_duration_seconds:histogram_quantile{job=\"apiserver\",quantile=\"0.99\",subresource!=\"log\",verb!~\"^(?:LIST|WATCH|WATCHLIST|PROXY|CONNECT)$\"} > 1\n", + "expr": "cluster_quantile:apiserver_request_duration_seconds:histogram_quantile{job=\"apiserver\",quantile=\"0.99\",subresource!=\"log\",verb!~\"LIST|WATCH|WATCHLIST|PROXY|CONNECT\"} > 1\n", "for": "10m", "labels": { "severity": "warning" @@ -731,7 +687,7 @@ data: "message": "The API server has a 99th percentile latency of {{ $value }} seconds for {{ $labels.verb }} {{ $labels.resource }}.", "runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubeapilatencyhigh" }, - "expr": "cluster_quantile:apiserver_request_duration_seconds:histogram_quantile{job=\"apiserver\",quantile=\"0.99\",subresource!=\"log\",verb!~\"^(?:LIST|WATCH|WATCHLIST|PROXY|CONNECT)$\"} > 4\n", + "expr": "cluster_quantile:apiserver_request_duration_seconds:histogram_quantile{job=\"apiserver\",quantile=\"0.99\",subresource!=\"log\",verb!~\"LIST|WATCH|WATCHLIST|PROXY|CONNECT\"} > 4\n", "for": "10m", "labels": { "severity": "critical" @@ -743,7 +699,7 @@ data: "message": "API server is returning errors for {{ $value | humanizePercentage }} of requests.", "runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubeapierrorshigh" }, - "expr": "sum(rate(apiserver_request_total{job=\"apiserver\",code=~\"^(?:5..)$\"}[5m]))\n /\nsum(rate(apiserver_request_total{job=\"apiserver\"}[5m])) > 0.03\n", + "expr": "sum(rate(apiserver_request_total{job=\"apiserver\",code=~\"5..\"}[5m]))\n /\nsum(rate(apiserver_request_total{job=\"apiserver\"}[5m])) > 0.03\n", "for": "10m", "labels": { "severity": "critical" @@ -755,7 +711,7 @@ data: "message": "API server is returning errors for {{ $value | humanizePercentage }} of requests.", "runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubeapierrorshigh" }, - "expr": "sum(rate(apiserver_request_total{job=\"apiserver\",code=~\"^(?:5..)$\"}[5m]))\n /\nsum(rate(apiserver_request_total{job=\"apiserver\"}[5m])) > 0.01\n", + "expr": "sum(rate(apiserver_request_total{job=\"apiserver\",code=~\"5..\"}[5m]))\n /\nsum(rate(apiserver_request_total{job=\"apiserver\"}[5m])) > 0.01\n", "for": "10m", "labels": { "severity": "warning" @@ -767,7 +723,7 @@ data: "message": "API server is returning errors for {{ $value | humanizePercentage }} of requests for {{ $labels.verb }} {{ $labels.resource }} {{ $labels.subresource }}.", "runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubeapierrorshigh" }, - "expr": "sum(rate(apiserver_request_total{job=\"apiserver\",code=~\"^(?:5..)$\"}[5m])) by (resource,subresource,verb)\n /\nsum(rate(apiserver_request_total{job=\"apiserver\"}[5m])) by (resource,subresource,verb) > 0.10\n", + "expr": "sum(rate(apiserver_request_total{job=\"apiserver\",code=~\"5..\"}[5m])) by (resource,subresource,verb)\n /\nsum(rate(apiserver_request_total{job=\"apiserver\"}[5m])) by (resource,subresource,verb) > 0.10\n", "for": "10m", "labels": { "severity": "critical" @@ -779,7 +735,7 @@ data: "message": "API server is returning errors for {{ $value | humanizePercentage }} of requests for {{ $labels.verb }} {{ $labels.resource }} {{ $labels.subresource }}.", "runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubeapierrorshigh" }, - "expr": "sum(rate(apiserver_request_total{job=\"apiserver\",code=~\"^(?:5..)$\"}[5m])) by (resource,subresource,verb)\n /\nsum(rate(apiserver_request_total{job=\"apiserver\"}[5m])) by (resource,subresource,verb) > 0.05\n", + "expr": "sum(rate(apiserver_request_total{job=\"apiserver\",code=~\"5..\"}[5m])) by (resource,subresource,verb)\n /\nsum(rate(apiserver_request_total{job=\"apiserver\"}[5m])) by (resource,subresource,verb) > 0.05\n", "for": "10m", "labels": { "severity": "warning" @@ -806,6 +762,104 @@ data: "labels": { "severity": "critical" } + }, + { + "alert": "KubeAPIDown", + "annotations": { + "message": "KubeAPI has disappeared from Prometheus target discovery.", + "runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubeapidown" + }, + "expr": "absent(up{job=\"apiserver\"} == 1)\n", + "for": "15m", + "labels": { + "severity": "critical" + } + } + ] + }, + { + "name": "kubernetes-system-kubelet", + "rules": [ + { + "alert": "KubeNodeNotReady", + "annotations": { + "message": "{{ $labels.node }} has been unready for more than 15 minutes.", + "runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubenodenotready" + }, + "expr": "kube_node_status_condition{job=\"kube-state-metrics\",condition=\"Ready\",status=\"true\"} == 0\n", + "for": "15m", + "labels": { + "severity": "warning" + } + }, + { + "alert": "KubeNodeUnreachable", + "annotations": { + "message": "{{ $labels.node }} is unreachable and some workloads may be rescheduled.", + "runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubenodeunreachable" + }, + "expr": "kube_node_spec_taint{job=\"kube-state-metrics\",key=\"node.kubernetes.io/unreachable\",effect=\"NoSchedule\"} == 1\n", + "labels": { + "severity": "warning" + } + }, + { + "alert": "KubeletTooManyPods", + "annotations": { + "message": "Kubelet '{{ $labels.node }}' is running at {{ $value | humanizePercentage }} of its Pod capacity.", + "runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubelettoomanypods" + }, + "expr": "max(max(kubelet_running_pod_count{job=\"kubelet\"}) by(instance) * on(instance) group_left(node) kubelet_node_name{job=\"kubelet\"}) by(node) / max(kube_node_status_capacity_pods{job=\"kube-state-metrics\"}) by(node) > 0.95\n", + "for": "15m", + "labels": { + "severity": "warning" + } + }, + { + "alert": "KubeletDown", + "annotations": { + "message": "Kubelet has disappeared from Prometheus target discovery.", + "runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubeletdown" + }, + "expr": "absent(up{job=\"kubelet\"} == 1)\n", + "for": "15m", + "labels": { + "severity": "critical" + } + } + ] + }, + { + "name": "kubernetes-system-scheduler", + "rules": [ + { + "alert": "KubeSchedulerDown", + "annotations": { + "message": "KubeScheduler has disappeared from Prometheus target discovery.", + "runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubeschedulerdown" + }, + "expr": "absent(up{job=\"kube-scheduler\"} == 1)\n", + "for": "15m", + "labels": { + "severity": "critical" + } + } + ] + }, + { + "name": "kubernetes-system-controller-manager", + "rules": [ + { + "alert": "KubeControllerManagerDown", + "annotations": { + "message": "KubeControllerManager has disappeared from Prometheus target discovery.", + "runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubecontrollermanagerdown" + }, + "expr": "absent(up{job=\"kube-controller-manager\"} == 1)\n", + "for": "15m", + "labels": { + "severity": "critical" + } } ] } From 38957163cb7fcab7449b3e94bef842b8da21a67b Mon Sep 17 00:00:00 2001 From: Konstantinos Koukopoulos Date: Thu, 31 Oct 2019 10:05:04 +0200 Subject: [PATCH 259/523] Output resource_group_id in Azure (#577) * Add an output variable `resource_group_id` to the azure module --- azure/container-linux/kubernetes/outputs.tf | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/azure/container-linux/kubernetes/outputs.tf b/azure/container-linux/kubernetes/outputs.tf index 39169d8df..fe5eaffb0 100644 --- a/azure/container-linux/kubernetes/outputs.tf +++ b/azure/container-linux/kubernetes/outputs.tf @@ -19,6 +19,10 @@ output "resource_group_name" { value = azurerm_resource_group.cluster.name } +output "resource_group_id" { + value = azurerm_resource_group.cluster.id +} + output "subnet_id" { value = azurerm_subnet.worker.id } @@ -53,4 +57,3 @@ output "backend_address_pool_id" { description = "ID of the worker backend address pool" value = azurerm_lb_backend_address_pool.worker.id } - From 0034a15711f893d954853f23d7dc59f8b76e114e Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Thu, 7 Nov 2019 11:09:59 -0800 Subject: [PATCH 260/523] Update Calico from v3.10.0 to v3.10.1 * https://docs.projectcalico.org/v3.10/release-notes/ --- CHANGES.md | 2 +- aws/container-linux/kubernetes/bootstrap.tf | 2 +- aws/fedora-coreos/kubernetes/bootstrap.tf | 2 +- azure/container-linux/kubernetes/bootstrap.tf | 2 +- bare-metal/container-linux/kubernetes/bootstrap.tf | 2 +- bare-metal/fedora-coreos/kubernetes/bootstrap.tf | 2 +- digital-ocean/container-linux/kubernetes/bootstrap.tf | 2 +- google-cloud/container-linux/kubernetes/bootstrap.tf | 2 +- 8 files changed, 8 insertions(+), 8 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 5756c6608..40985f10d 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -4,7 +4,7 @@ Notable changes between versions. ## Latest -* Upgrade Calico from v3.9.2 to [v3.10.0](https://docs.projectcalico.org/v3.10/release-notes/) +* Upgrade Calico from v3.9.2 to [v3.10.1](https://docs.projectcalico.org/v3.10/release-notes/) * Allow advertising service ClusterIPs to peer routers via a [BGPConfiguration](https://docs.projectcalico.org/v3.10/networking/advertise-service-ips) * Switch `kube-proxy` from iptables to ipvs mode ([#574](https://github.com/poseidon/typhoon/pull/574)) diff --git a/aws/container-linux/kubernetes/bootstrap.tf b/aws/container-linux/kubernetes/bootstrap.tf index a68b40a14..a8524d56e 100644 --- a/aws/container-linux/kubernetes/bootstrap.tf +++ b/aws/container-linux/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=3c7334ab55b4ebef4109072da99452d59ee6179a" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=a2b1dbe2c0c728c58fbdd34da12d19fa8bfdfcc2" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/aws/fedora-coreos/kubernetes/bootstrap.tf b/aws/fedora-coreos/kubernetes/bootstrap.tf index d87f50730..0192a9e2a 100644 --- a/aws/fedora-coreos/kubernetes/bootstrap.tf +++ b/aws/fedora-coreos/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=3c7334ab55b4ebef4109072da99452d59ee6179a" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=a2b1dbe2c0c728c58fbdd34da12d19fa8bfdfcc2" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/azure/container-linux/kubernetes/bootstrap.tf b/azure/container-linux/kubernetes/bootstrap.tf index b80fbca69..d5f08c1c2 100644 --- a/azure/container-linux/kubernetes/bootstrap.tf +++ b/azure/container-linux/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=3c7334ab55b4ebef4109072da99452d59ee6179a" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=a2b1dbe2c0c728c58fbdd34da12d19fa8bfdfcc2" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/bare-metal/container-linux/kubernetes/bootstrap.tf b/bare-metal/container-linux/kubernetes/bootstrap.tf index b806a3575..a1c0d807a 100644 --- a/bare-metal/container-linux/kubernetes/bootstrap.tf +++ b/bare-metal/container-linux/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=3c7334ab55b4ebef4109072da99452d59ee6179a" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=a2b1dbe2c0c728c58fbdd34da12d19fa8bfdfcc2" cluster_name = var.cluster_name api_servers = [var.k8s_domain_name] diff --git a/bare-metal/fedora-coreos/kubernetes/bootstrap.tf b/bare-metal/fedora-coreos/kubernetes/bootstrap.tf index 5ebfee5f1..c1c127ade 100644 --- a/bare-metal/fedora-coreos/kubernetes/bootstrap.tf +++ b/bare-metal/fedora-coreos/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=3c7334ab55b4ebef4109072da99452d59ee6179a" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=a2b1dbe2c0c728c58fbdd34da12d19fa8bfdfcc2" cluster_name = var.cluster_name api_servers = [var.k8s_domain_name] diff --git a/digital-ocean/container-linux/kubernetes/bootstrap.tf b/digital-ocean/container-linux/kubernetes/bootstrap.tf index c4a348b64..1d215ee50 100644 --- a/digital-ocean/container-linux/kubernetes/bootstrap.tf +++ b/digital-ocean/container-linux/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=3c7334ab55b4ebef4109072da99452d59ee6179a" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=a2b1dbe2c0c728c58fbdd34da12d19fa8bfdfcc2" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/google-cloud/container-linux/kubernetes/bootstrap.tf b/google-cloud/container-linux/kubernetes/bootstrap.tf index 878bfa4a9..b0467f676 100644 --- a/google-cloud/container-linux/kubernetes/bootstrap.tf +++ b/google-cloud/container-linux/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=3c7334ab55b4ebef4109072da99452d59ee6179a" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=a2b1dbe2c0c728c58fbdd34da12d19fa8bfdfcc2" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] From 2c163503f1da46a954e3a716b4914d04adec34f9 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Thu, 7 Nov 2019 11:12:44 -0800 Subject: [PATCH 261/523] Update etcd from v3.4.2 to v3.4.3 * etcd v3.4.3 builds with Go v1.12.12 instead of v1.12.9 and adds a few minor metrics fixes * https://github.com/etcd-io/etcd/compare/v3.4.2...v3.4.3 --- CHANGES.md | 1 + aws/container-linux/kubernetes/cl/controller.yaml.tmpl | 2 +- aws/fedora-coreos/kubernetes/fcc/controller.yaml | 2 +- azure/container-linux/kubernetes/cl/controller.yaml.tmpl | 2 +- bare-metal/container-linux/kubernetes/cl/controller.yaml.tmpl | 2 +- bare-metal/fedora-coreos/kubernetes/fcc/controller.yaml | 2 +- .../container-linux/kubernetes/cl/controller.yaml.tmpl | 2 +- google-cloud/container-linux/kubernetes/cl/controller.yaml.tmpl | 2 +- 8 files changed, 8 insertions(+), 7 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 40985f10d..0d3713b52 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -4,6 +4,7 @@ Notable changes between versions. ## Latest +* Update etcd from v3.4.2 to v3.4.3 ([#582](https://github.com/poseidon/typhoon/pull/582)) * Upgrade Calico from v3.9.2 to [v3.10.1](https://docs.projectcalico.org/v3.10/release-notes/) * Allow advertising service ClusterIPs to peer routers via a [BGPConfiguration](https://docs.projectcalico.org/v3.10/networking/advertise-service-ips) * Switch `kube-proxy` from iptables to ipvs mode ([#574](https://github.com/poseidon/typhoon/pull/574)) diff --git a/aws/container-linux/kubernetes/cl/controller.yaml.tmpl b/aws/container-linux/kubernetes/cl/controller.yaml.tmpl index dac690ef3..24e1202df 100644 --- a/aws/container-linux/kubernetes/cl/controller.yaml.tmpl +++ b/aws/container-linux/kubernetes/cl/controller.yaml.tmpl @@ -7,7 +7,7 @@ systemd: - name: 40-etcd-cluster.conf contents: | [Service] - Environment="ETCD_IMAGE_TAG=v3.4.2" + Environment="ETCD_IMAGE_TAG=v3.4.3" Environment="ETCD_NAME=${etcd_name}" Environment="ETCD_ADVERTISE_CLIENT_URLS=https://${etcd_domain}:2379" Environment="ETCD_INITIAL_ADVERTISE_PEER_URLS=https://${etcd_domain}:2380" diff --git a/aws/fedora-coreos/kubernetes/fcc/controller.yaml b/aws/fedora-coreos/kubernetes/fcc/controller.yaml index 1da9ab71a..d22f4c756 100644 --- a/aws/fedora-coreos/kubernetes/fcc/controller.yaml +++ b/aws/fedora-coreos/kubernetes/fcc/controller.yaml @@ -28,7 +28,7 @@ systemd: --network host \ --volume /var/lib/etcd:/var/lib/etcd:rw,Z \ --volume /etc/ssl/etcd:/etc/ssl/certs:ro,Z \ - quay.io/coreos/etcd:v3.4.2 + quay.io/coreos/etcd:v3.4.3 ExecStop=/usr/bin/podman stop etcd [Install] WantedBy=multi-user.target diff --git a/azure/container-linux/kubernetes/cl/controller.yaml.tmpl b/azure/container-linux/kubernetes/cl/controller.yaml.tmpl index 6313e4896..16aa71cf1 100644 --- a/azure/container-linux/kubernetes/cl/controller.yaml.tmpl +++ b/azure/container-linux/kubernetes/cl/controller.yaml.tmpl @@ -7,7 +7,7 @@ systemd: - name: 40-etcd-cluster.conf contents: | [Service] - Environment="ETCD_IMAGE_TAG=v3.4.2" + Environment="ETCD_IMAGE_TAG=v3.4.3" Environment="ETCD_NAME=${etcd_name}" Environment="ETCD_ADVERTISE_CLIENT_URLS=https://${etcd_domain}:2379" Environment="ETCD_INITIAL_ADVERTISE_PEER_URLS=https://${etcd_domain}:2380" diff --git a/bare-metal/container-linux/kubernetes/cl/controller.yaml.tmpl b/bare-metal/container-linux/kubernetes/cl/controller.yaml.tmpl index 39068a52d..61201a6eb 100644 --- a/bare-metal/container-linux/kubernetes/cl/controller.yaml.tmpl +++ b/bare-metal/container-linux/kubernetes/cl/controller.yaml.tmpl @@ -7,7 +7,7 @@ systemd: - name: 40-etcd-cluster.conf contents: | [Service] - Environment="ETCD_IMAGE_TAG=v3.4.2" + Environment="ETCD_IMAGE_TAG=v3.4.3" Environment="ETCD_NAME=${etcd_name}" Environment="ETCD_ADVERTISE_CLIENT_URLS=https://${domain_name}:2379" Environment="ETCD_INITIAL_ADVERTISE_PEER_URLS=https://${domain_name}:2380" diff --git a/bare-metal/fedora-coreos/kubernetes/fcc/controller.yaml b/bare-metal/fedora-coreos/kubernetes/fcc/controller.yaml index e1951cacd..d0aaf749e 100644 --- a/bare-metal/fedora-coreos/kubernetes/fcc/controller.yaml +++ b/bare-metal/fedora-coreos/kubernetes/fcc/controller.yaml @@ -28,7 +28,7 @@ systemd: --network host \ --volume /var/lib/etcd:/var/lib/etcd:rw,Z \ --volume /etc/ssl/etcd:/etc/ssl/certs:ro,Z \ - quay.io/coreos/etcd:v3.4.2 + quay.io/coreos/etcd:v3.4.3 ExecStop=/usr/bin/podman stop etcd [Install] WantedBy=multi-user.target diff --git a/digital-ocean/container-linux/kubernetes/cl/controller.yaml.tmpl b/digital-ocean/container-linux/kubernetes/cl/controller.yaml.tmpl index c9a905549..55dad7e32 100644 --- a/digital-ocean/container-linux/kubernetes/cl/controller.yaml.tmpl +++ b/digital-ocean/container-linux/kubernetes/cl/controller.yaml.tmpl @@ -7,7 +7,7 @@ systemd: - name: 40-etcd-cluster.conf contents: | [Service] - Environment="ETCD_IMAGE_TAG=v3.4.2" + Environment="ETCD_IMAGE_TAG=v3.4.3" Environment="ETCD_NAME=${etcd_name}" Environment="ETCD_ADVERTISE_CLIENT_URLS=https://${etcd_domain}:2379" Environment="ETCD_INITIAL_ADVERTISE_PEER_URLS=https://${etcd_domain}:2380" diff --git a/google-cloud/container-linux/kubernetes/cl/controller.yaml.tmpl b/google-cloud/container-linux/kubernetes/cl/controller.yaml.tmpl index 0b53119f7..c69121bb1 100644 --- a/google-cloud/container-linux/kubernetes/cl/controller.yaml.tmpl +++ b/google-cloud/container-linux/kubernetes/cl/controller.yaml.tmpl @@ -7,7 +7,7 @@ systemd: - name: 40-etcd-cluster.conf contents: | [Service] - Environment="ETCD_IMAGE_TAG=v3.4.2" + Environment="ETCD_IMAGE_TAG=v3.4.3" Environment="ETCD_NAME=${etcd_name}" Environment="ETCD_ADVERTISE_CLIENT_URLS=https://${etcd_domain}:2379" Environment="ETCD_INITIAL_ADVERTISE_PEER_URLS=https://${etcd_domain}:2380" From a3807086d45ca7576498115eba641be7fb633943 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Thu, 7 Nov 2019 11:46:50 -0800 Subject: [PATCH 262/523] Update Prometheus from v2.13.1 to v2.14.0-rc.0 * Happy PromCon 2019! * https://github.com/prometheus/prometheus/releases/tag/v2.14.0-rc.0 --- CHANGES.md | 1 + addons/prometheus/deployment.yaml | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index 0d3713b52..1813c7333 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -11,6 +11,7 @@ Notable changes between versions. #### Addons +* Update Prometheus from v2.13.1 to [v2.14.0-rc.0](https://github.com/prometheus/prometheus/releases/tag/v2.14.0-rc.0) * Update Prometheus from v2.13.0 to v2.13.1 * Refresh rules, alerts, and dashboards from upstreams * Remove addon-resizer from kube-state-metrics ([#575](https://github.com/poseidon/typhoon/pull/575)) diff --git a/addons/prometheus/deployment.yaml b/addons/prometheus/deployment.yaml index 01d87d117..fe9c499ad 100644 --- a/addons/prometheus/deployment.yaml +++ b/addons/prometheus/deployment.yaml @@ -20,7 +20,7 @@ spec: serviceAccountName: prometheus containers: - name: prometheus - image: quay.io/prometheus/prometheus:v2.13.1 + image: quay.io/prometheus/prometheus:v2.14.0-rc.0 args: - --web.listen-address=0.0.0.0:9090 - --config.file=/etc/prometheus/prometheus.yaml From a8b77923388f380e6924e9a817b4a4c658333dd5 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Thu, 7 Nov 2019 12:00:25 -0800 Subject: [PATCH 263/523] Update Grafana from v6.4.3 to v6.4.4 * https://github.com/grafana/grafana/releases/tag/v6.4.4 --- CHANGES.md | 2 +- addons/grafana/deployment.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 1813c7333..f6ca84d1c 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -15,7 +15,7 @@ Notable changes between versions. * Update Prometheus from v2.13.0 to v2.13.1 * Refresh rules, alerts, and dashboards from upstreams * Remove addon-resizer from kube-state-metrics ([#575](https://github.com/poseidon/typhoon/pull/575)) -* Update Grafana from v6.4.2 to v6.4.3 +* Update Grafana from v6.4.2 to v6.4.4 ## v1.16.2 diff --git a/addons/grafana/deployment.yaml b/addons/grafana/deployment.yaml index fbe51f503..9ea495758 100644 --- a/addons/grafana/deployment.yaml +++ b/addons/grafana/deployment.yaml @@ -23,7 +23,7 @@ spec: spec: containers: - name: grafana - image: docker.io/grafana/grafana:6.4.3 + image: docker.io/grafana/grafana:6.4.4 env: - name: GF_PATHS_CONFIG value: "/etc/grafana/custom.ini" From d7061020ba79b50316bf2ee08df711273a031ee7 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Wed, 13 Nov 2019 13:05:15 -0800 Subject: [PATCH 264/523] Update Kubernetes from v1.16.2 to v1.16.3 * https://github.com/kubernetes/kubernetes/blob/master/CHANGELOG-1.16.md#v1163 --- CHANGES.md | 3 +++ README.md | 10 +++++----- aws/container-linux/kubernetes/README.md | 2 +- aws/container-linux/kubernetes/bootstrap.tf | 2 +- .../kubernetes/cl/controller.yaml.tmpl | 4 ++-- .../kubernetes/workers/cl/worker.yaml.tmpl | 4 ++-- aws/fedora-coreos/kubernetes/README.md | 2 +- aws/fedora-coreos/kubernetes/bootstrap.tf | 2 +- aws/fedora-coreos/kubernetes/fcc/controller.yaml | 4 ++-- .../kubernetes/workers/fcc/worker.yaml | 2 +- azure/container-linux/kubernetes/README.md | 2 +- azure/container-linux/kubernetes/bootstrap.tf | 2 +- .../kubernetes/cl/controller.yaml.tmpl | 4 ++-- .../kubernetes/workers/cl/worker.yaml.tmpl | 4 ++-- bare-metal/container-linux/kubernetes/README.md | 2 +- bare-metal/container-linux/kubernetes/bootstrap.tf | 2 +- .../kubernetes/cl/controller.yaml.tmpl | 4 ++-- .../container-linux/kubernetes/cl/worker.yaml.tmpl | 2 +- bare-metal/fedora-coreos/kubernetes/README.md | 2 +- bare-metal/fedora-coreos/kubernetes/bootstrap.tf | 2 +- .../fedora-coreos/kubernetes/fcc/controller.yaml | 4 ++-- .../fedora-coreos/kubernetes/fcc/worker.yaml | 2 +- digital-ocean/container-linux/kubernetes/README.md | 2 +- .../container-linux/kubernetes/bootstrap.tf | 2 +- .../kubernetes/cl/controller.yaml.tmpl | 4 ++-- .../container-linux/kubernetes/cl/worker.yaml.tmpl | 4 ++-- docs/advanced/worker-pools.md | 14 +++++++------- docs/cl/aws.md | 10 +++++----- docs/cl/azure.md | 10 +++++----- docs/cl/bare-metal.md | 14 +++++++------- docs/cl/digital-ocean.md | 10 +++++----- docs/cl/google-cloud.md | 10 +++++----- docs/fedora-coreos/aws.md | 10 +++++----- docs/fedora-coreos/bare-metal.md | 10 +++++----- docs/index.md | 10 +++++----- docs/topics/maintenance.md | 10 +++++----- google-cloud/container-linux/kubernetes/README.md | 2 +- .../container-linux/kubernetes/bootstrap.tf | 2 +- .../kubernetes/cl/controller.yaml.tmpl | 4 ++-- .../kubernetes/workers/cl/worker.yaml.tmpl | 4 ++-- 40 files changed, 101 insertions(+), 98 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index f6ca84d1c..f56c36308 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -4,6 +4,9 @@ Notable changes between versions. ## Latest +## v1.16.3 + +* Kubernetes [v1.16.3](https://github.com/kubernetes/kubernetes/blob/master/CHANGELOG-1.16.md#v1163) * Update etcd from v3.4.2 to v3.4.3 ([#582](https://github.com/poseidon/typhoon/pull/582)) * Upgrade Calico from v3.9.2 to [v3.10.1](https://docs.projectcalico.org/v3.10/release-notes/) * Allow advertising service ClusterIPs to peer routers via a [BGPConfiguration](https://docs.projectcalico.org/v3.10/networking/advertise-service-ips) diff --git a/README.md b/README.md index 96b4f88cc..92f310e90 100644 --- a/README.md +++ b/README.md @@ -11,7 +11,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features -* Kubernetes v1.16.2 (upstream) +* Kubernetes v1.16.3 (upstream) * Single or multi-master, [Calico](https://www.projectcalico.org/) or [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/) * Advanced features like [worker pools](https://typhoon.psdn.io/advanced/worker-pools/), [preemptible](https://typhoon.psdn.io/cl/google-cloud/#preemption) workers, and [snippets](https://typhoon.psdn.io/advanced/customization/#container-linux) customization @@ -48,7 +48,7 @@ Define a Kubernetes cluster by using the Terraform module for your chosen platfo ```tf module "google-cloud-yavin" { - source = "git::https://github.com/poseidon/typhoon//google-cloud/container-linux/kubernetes?ref=v1.16.2" + source = "git::https://github.com/poseidon/typhoon//google-cloud/container-linux/kubernetes?ref=v1.16.3" # Google Cloud cluster_name = "yavin" @@ -82,9 +82,9 @@ In 4-8 minutes (varies by platform), the cluster will be ready. This Google Clou $ export KUBECONFIG=/home/user/.secrets/clusters/yavin/auth/kubeconfig $ kubectl get nodes NAME ROLES STATUS AGE VERSION -yavin-controller-0.c.example-com.internal Ready 6m v1.16.2 -yavin-worker-jrbf.c.example-com.internal Ready 5m v1.16.2 -yavin-worker-mzdm.c.example-com.internal Ready 5m v1.16.2 +yavin-controller-0.c.example-com.internal Ready 6m v1.16.3 +yavin-worker-jrbf.c.example-com.internal Ready 5m v1.16.3 +yavin-worker-mzdm.c.example-com.internal Ready 5m v1.16.3 ``` List the pods. diff --git a/aws/container-linux/kubernetes/README.md b/aws/container-linux/kubernetes/README.md index ac24929d6..746726f86 100644 --- a/aws/container-linux/kubernetes/README.md +++ b/aws/container-linux/kubernetes/README.md @@ -11,7 +11,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features -* Kubernetes v1.16.2 (upstream) +* Kubernetes v1.16.3 (upstream) * Single or multi-master, [Calico](https://www.projectcalico.org/) or [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/) * Advanced features like [worker pools](https://typhoon.psdn.io/advanced/worker-pools/), [spot](https://typhoon.psdn.io/cl/aws/#spot) workers, and [snippets](https://typhoon.psdn.io/advanced/customization/#container-linux) customization diff --git a/aws/container-linux/kubernetes/bootstrap.tf b/aws/container-linux/kubernetes/bootstrap.tf index a8524d56e..7da8b732e 100644 --- a/aws/container-linux/kubernetes/bootstrap.tf +++ b/aws/container-linux/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=a2b1dbe2c0c728c58fbdd34da12d19fa8bfdfcc2" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=0daa1276c633fea28e41b2c2c18831e2584deb24" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/aws/container-linux/kubernetes/cl/controller.yaml.tmpl b/aws/container-linux/kubernetes/cl/controller.yaml.tmpl index 24e1202df..c4cb7b2f0 100644 --- a/aws/container-linux/kubernetes/cl/controller.yaml.tmpl +++ b/aws/container-linux/kubernetes/cl/controller.yaml.tmpl @@ -113,7 +113,7 @@ systemd: --volume script,kind=host,source=/opt/bootstrap/apply \ --mount volume=script,target=/apply \ --insecure-options=image \ - docker://k8s.gcr.io/hyperkube:v1.16.2 \ + docker://k8s.gcr.io/hyperkube:v1.16.3 \ --net=host \ --dns=host \ --exec=/apply @@ -134,7 +134,7 @@ storage: contents: inline: | KUBELET_IMAGE_URL=docker://k8s.gcr.io/hyperkube - KUBELET_IMAGE_TAG=v1.16.2 + KUBELET_IMAGE_TAG=v1.16.3 - path: /opt/bootstrap/apply filesystem: root mode: 0544 diff --git a/aws/container-linux/kubernetes/workers/cl/worker.yaml.tmpl b/aws/container-linux/kubernetes/workers/cl/worker.yaml.tmpl index 2e11d6f5c..8c495585c 100644 --- a/aws/container-linux/kubernetes/workers/cl/worker.yaml.tmpl +++ b/aws/container-linux/kubernetes/workers/cl/worker.yaml.tmpl @@ -98,7 +98,7 @@ storage: contents: inline: | KUBELET_IMAGE_URL=docker://k8s.gcr.io/hyperkube - KUBELET_IMAGE_TAG=v1.16.2 + KUBELET_IMAGE_TAG=v1.16.3 - path: /etc/sysctl.d/max-user-watches.conf filesystem: root contents: @@ -116,7 +116,7 @@ storage: --volume config,kind=host,source=/etc/kubernetes \ --mount volume=config,target=/etc/kubernetes \ --insecure-options=image \ - docker://k8s.gcr.io/hyperkube:v1.16.2 \ + docker://k8s.gcr.io/hyperkube:v1.16.3 \ --net=host \ --dns=host \ --exec=/kubectl -- --kubeconfig=/etc/kubernetes/kubeconfig delete node $(hostname) diff --git a/aws/fedora-coreos/kubernetes/README.md b/aws/fedora-coreos/kubernetes/README.md index a9b51d8bc..3adc79621 100644 --- a/aws/fedora-coreos/kubernetes/README.md +++ b/aws/fedora-coreos/kubernetes/README.md @@ -11,7 +11,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features -* Kubernetes v1.16.2 (upstream) +* Kubernetes v1.16.3 (upstream) * Single or multi-master, [Calico](https://www.projectcalico.org/) or [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/) * Advanced features like [worker pools](https://typhoon.psdn.io/advanced/worker-pools/), [spot](https://typhoon.psdn.io/cl/aws/#spot) workers, and [snippets](https://typhoon.psdn.io/advanced/customization/#container-linux) customization diff --git a/aws/fedora-coreos/kubernetes/bootstrap.tf b/aws/fedora-coreos/kubernetes/bootstrap.tf index 0192a9e2a..908ed4f70 100644 --- a/aws/fedora-coreos/kubernetes/bootstrap.tf +++ b/aws/fedora-coreos/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=a2b1dbe2c0c728c58fbdd34da12d19fa8bfdfcc2" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=0daa1276c633fea28e41b2c2c18831e2584deb24" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/aws/fedora-coreos/kubernetes/fcc/controller.yaml b/aws/fedora-coreos/kubernetes/fcc/controller.yaml index d22f4c756..dc2edb21e 100644 --- a/aws/fedora-coreos/kubernetes/fcc/controller.yaml +++ b/aws/fedora-coreos/kubernetes/fcc/controller.yaml @@ -80,7 +80,7 @@ systemd: --volume /var/run:/var/run \ --volume /var/run/lock:/var/run/lock:z \ --volume /opt/cni/bin:/opt/cni/bin:z \ - k8s.gcr.io/hyperkube:v1.16.2 /hyperkube kubelet \ + k8s.gcr.io/hyperkube:v1.16.3 /hyperkube kubelet \ --anonymous-auth=false \ --authentication-token-webhook \ --authorization-mode=Webhook \ @@ -121,7 +121,7 @@ systemd: --network host \ --volume /opt/bootstrap/assets:/assets:ro,Z \ --volume /opt/bootstrap/apply:/apply:ro,Z \ - k8s.gcr.io/hyperkube:v1.16.2 \ + k8s.gcr.io/hyperkube:v1.16.3 \ /apply ExecStartPost=/bin/touch /opt/bootstrap/bootstrap.done ExecStartPost=-/usr/bin/podman stop bootstrap diff --git a/aws/fedora-coreos/kubernetes/workers/fcc/worker.yaml b/aws/fedora-coreos/kubernetes/workers/fcc/worker.yaml index 2232bf2bd..b31ac0ab4 100644 --- a/aws/fedora-coreos/kubernetes/workers/fcc/worker.yaml +++ b/aws/fedora-coreos/kubernetes/workers/fcc/worker.yaml @@ -50,7 +50,7 @@ systemd: --volume /var/run:/var/run \ --volume /var/run/lock:/var/run/lock:z \ --volume /opt/cni/bin:/opt/cni/bin:z \ - k8s.gcr.io/hyperkube:v1.16.2 /hyperkube kubelet \ + k8s.gcr.io/hyperkube:v1.16.3 /hyperkube kubelet \ --anonymous-auth=false \ --authentication-token-webhook \ --authorization-mode=Webhook \ diff --git a/azure/container-linux/kubernetes/README.md b/azure/container-linux/kubernetes/README.md index fed809060..9c1635dda 100644 --- a/azure/container-linux/kubernetes/README.md +++ b/azure/container-linux/kubernetes/README.md @@ -11,7 +11,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features -* Kubernetes v1.16.2 (upstream) +* Kubernetes v1.16.3 (upstream) * Single or multi-master, [Calico](https://www.projectcalico.org/) or [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/) * Advanced features like [worker pools](https://typhoon.psdn.io/advanced/worker-pools/), [low-priority](https://typhoon.psdn.io/cl/azure/#low-priority) workers, and [snippets](https://typhoon.psdn.io/advanced/customization/#container-linux) customization diff --git a/azure/container-linux/kubernetes/bootstrap.tf b/azure/container-linux/kubernetes/bootstrap.tf index d5f08c1c2..e82cce158 100644 --- a/azure/container-linux/kubernetes/bootstrap.tf +++ b/azure/container-linux/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=a2b1dbe2c0c728c58fbdd34da12d19fa8bfdfcc2" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=0daa1276c633fea28e41b2c2c18831e2584deb24" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/azure/container-linux/kubernetes/cl/controller.yaml.tmpl b/azure/container-linux/kubernetes/cl/controller.yaml.tmpl index 16aa71cf1..491350685 100644 --- a/azure/container-linux/kubernetes/cl/controller.yaml.tmpl +++ b/azure/container-linux/kubernetes/cl/controller.yaml.tmpl @@ -111,7 +111,7 @@ systemd: --volume script,kind=host,source=/opt/bootstrap/apply \ --mount volume=script,target=/apply \ --insecure-options=image \ - docker://k8s.gcr.io/hyperkube:v1.16.2 \ + docker://k8s.gcr.io/hyperkube:v1.16.3 \ --net=host \ --dns=host \ --exec=/apply @@ -132,7 +132,7 @@ storage: contents: inline: | KUBELET_IMAGE_URL=docker://k8s.gcr.io/hyperkube - KUBELET_IMAGE_TAG=v1.16.2 + KUBELET_IMAGE_TAG=v1.16.3 - path: /opt/bootstrap/apply filesystem: root mode: 0544 diff --git a/azure/container-linux/kubernetes/workers/cl/worker.yaml.tmpl b/azure/container-linux/kubernetes/workers/cl/worker.yaml.tmpl index a861d3bc9..6c23c66b0 100644 --- a/azure/container-linux/kubernetes/workers/cl/worker.yaml.tmpl +++ b/azure/container-linux/kubernetes/workers/cl/worker.yaml.tmpl @@ -96,7 +96,7 @@ storage: contents: inline: | KUBELET_IMAGE_URL=docker://k8s.gcr.io/hyperkube - KUBELET_IMAGE_TAG=v1.16.2 + KUBELET_IMAGE_TAG=v1.16.3 - path: /etc/sysctl.d/max-user-watches.conf filesystem: root contents: @@ -114,7 +114,7 @@ storage: --volume config,kind=host,source=/etc/kubernetes \ --mount volume=config,target=/etc/kubernetes \ --insecure-options=image \ - docker://k8s.gcr.io/hyperkube:v1.16.2 \ + docker://k8s.gcr.io/hyperkube:v1.16.3 \ --net=host \ --dns=host \ --exec=/kubectl -- --kubeconfig=/etc/kubernetes/kubeconfig delete node $(hostname | tr '[:upper:]' '[:lower:]') diff --git a/bare-metal/container-linux/kubernetes/README.md b/bare-metal/container-linux/kubernetes/README.md index b15988f9b..2fd5ec5f8 100644 --- a/bare-metal/container-linux/kubernetes/README.md +++ b/bare-metal/container-linux/kubernetes/README.md @@ -11,7 +11,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features -* Kubernetes v1.16.2 (upstream) +* Kubernetes v1.16.3 (upstream) * Single or multi-master, [Calico](https://www.projectcalico.org/) or [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/) * Advanced features like [snippets](https://typhoon.psdn.io/advanced/customization/#container-linux) customization diff --git a/bare-metal/container-linux/kubernetes/bootstrap.tf b/bare-metal/container-linux/kubernetes/bootstrap.tf index a1c0d807a..0d06dcc7b 100644 --- a/bare-metal/container-linux/kubernetes/bootstrap.tf +++ b/bare-metal/container-linux/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=a2b1dbe2c0c728c58fbdd34da12d19fa8bfdfcc2" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=0daa1276c633fea28e41b2c2c18831e2584deb24" cluster_name = var.cluster_name api_servers = [var.k8s_domain_name] diff --git a/bare-metal/container-linux/kubernetes/cl/controller.yaml.tmpl b/bare-metal/container-linux/kubernetes/cl/controller.yaml.tmpl index 61201a6eb..a6c1beced 100644 --- a/bare-metal/container-linux/kubernetes/cl/controller.yaml.tmpl +++ b/bare-metal/container-linux/kubernetes/cl/controller.yaml.tmpl @@ -126,7 +126,7 @@ systemd: --volume script,kind=host,source=/opt/bootstrap/apply \ --mount volume=script,target=/apply \ --insecure-options=image \ - docker://k8s.gcr.io/hyperkube:v1.16.2 \ + docker://k8s.gcr.io/hyperkube:v1.16.3 \ --net=host \ --dns=host \ --exec=/apply @@ -141,7 +141,7 @@ storage: contents: inline: | KUBELET_IMAGE_URL=docker://k8s.gcr.io/hyperkube - KUBELET_IMAGE_TAG=v1.16.2 + KUBELET_IMAGE_TAG=v1.16.3 - path: /etc/hostname filesystem: root mode: 0644 diff --git a/bare-metal/container-linux/kubernetes/cl/worker.yaml.tmpl b/bare-metal/container-linux/kubernetes/cl/worker.yaml.tmpl index 52776c704..dab0a6be8 100644 --- a/bare-metal/container-linux/kubernetes/cl/worker.yaml.tmpl +++ b/bare-metal/container-linux/kubernetes/cl/worker.yaml.tmpl @@ -91,7 +91,7 @@ storage: contents: inline: | KUBELET_IMAGE_URL=docker://k8s.gcr.io/hyperkube - KUBELET_IMAGE_TAG=v1.16.2 + KUBELET_IMAGE_TAG=v1.16.3 - path: /etc/hostname filesystem: root mode: 0644 diff --git a/bare-metal/fedora-coreos/kubernetes/README.md b/bare-metal/fedora-coreos/kubernetes/README.md index 8d29f6f4e..1f389a7ca 100644 --- a/bare-metal/fedora-coreos/kubernetes/README.md +++ b/bare-metal/fedora-coreos/kubernetes/README.md @@ -11,7 +11,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features -* Kubernetes v1.16.2 (upstream) +* Kubernetes v1.16.3 (upstream) * Single or multi-master, [Calico](https://www.projectcalico.org/) or [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/) * Advanced features like [snippets](https://typhoon.psdn.io/advanced/customization/#container-linux) customization diff --git a/bare-metal/fedora-coreos/kubernetes/bootstrap.tf b/bare-metal/fedora-coreos/kubernetes/bootstrap.tf index c1c127ade..39c8be609 100644 --- a/bare-metal/fedora-coreos/kubernetes/bootstrap.tf +++ b/bare-metal/fedora-coreos/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=a2b1dbe2c0c728c58fbdd34da12d19fa8bfdfcc2" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=0daa1276c633fea28e41b2c2c18831e2584deb24" cluster_name = var.cluster_name api_servers = [var.k8s_domain_name] diff --git a/bare-metal/fedora-coreos/kubernetes/fcc/controller.yaml b/bare-metal/fedora-coreos/kubernetes/fcc/controller.yaml index d0aaf749e..a88013b8d 100644 --- a/bare-metal/fedora-coreos/kubernetes/fcc/controller.yaml +++ b/bare-metal/fedora-coreos/kubernetes/fcc/controller.yaml @@ -81,7 +81,7 @@ systemd: --volume /opt/cni/bin:/opt/cni/bin:z \ --volume /etc/iscsi:/etc/iscsi \ --volume /sbin/iscsiadm:/sbin/iscsiadm \ - k8s.gcr.io/hyperkube:v1.16.2 /hyperkube kubelet \ + k8s.gcr.io/hyperkube:v1.16.3 /hyperkube kubelet \ --anonymous-auth=false \ --authentication-token-webhook \ --authorization-mode=Webhook \ @@ -132,7 +132,7 @@ systemd: --network host \ --volume /opt/bootstrap/assets:/assets:ro,Z \ --volume /opt/bootstrap/apply:/apply:ro,Z \ - k8s.gcr.io/hyperkube:v1.16.2 \ + k8s.gcr.io/hyperkube:v1.16.3 \ /apply ExecStartPost=/bin/touch /opt/bootstrap/bootstrap.done ExecStartPost=-/usr/bin/podman stop bootstrap diff --git a/bare-metal/fedora-coreos/kubernetes/fcc/worker.yaml b/bare-metal/fedora-coreos/kubernetes/fcc/worker.yaml index a02a60998..f4fc87cdb 100644 --- a/bare-metal/fedora-coreos/kubernetes/fcc/worker.yaml +++ b/bare-metal/fedora-coreos/kubernetes/fcc/worker.yaml @@ -51,7 +51,7 @@ systemd: --volume /opt/cni/bin:/opt/cni/bin:z \ --volume /etc/iscsi:/etc/iscsi \ --volume /sbin/iscsiadm:/sbin/iscsiadm \ - k8s.gcr.io/hyperkube:v1.16.2 /hyperkube kubelet \ + k8s.gcr.io/hyperkube:v1.16.3 /hyperkube kubelet \ --anonymous-auth=false \ --authentication-token-webhook \ --authorization-mode=Webhook \ diff --git a/digital-ocean/container-linux/kubernetes/README.md b/digital-ocean/container-linux/kubernetes/README.md index 348cc4384..f2e070422 100644 --- a/digital-ocean/container-linux/kubernetes/README.md +++ b/digital-ocean/container-linux/kubernetes/README.md @@ -11,7 +11,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features -* Kubernetes v1.16.2 (upstream) +* Kubernetes v1.16.3 (upstream) * Single or multi-master, [Calico](https://www.projectcalico.org/) or [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/) * Advanced features like [snippets](https://typhoon.psdn.io/advanced/customization/#container-linux) customization diff --git a/digital-ocean/container-linux/kubernetes/bootstrap.tf b/digital-ocean/container-linux/kubernetes/bootstrap.tf index 1d215ee50..38d69631d 100644 --- a/digital-ocean/container-linux/kubernetes/bootstrap.tf +++ b/digital-ocean/container-linux/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=a2b1dbe2c0c728c58fbdd34da12d19fa8bfdfcc2" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=0daa1276c633fea28e41b2c2c18831e2584deb24" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/digital-ocean/container-linux/kubernetes/cl/controller.yaml.tmpl b/digital-ocean/container-linux/kubernetes/cl/controller.yaml.tmpl index 55dad7e32..ffc8cf329 100644 --- a/digital-ocean/container-linux/kubernetes/cl/controller.yaml.tmpl +++ b/digital-ocean/container-linux/kubernetes/cl/controller.yaml.tmpl @@ -123,7 +123,7 @@ systemd: --volume script,kind=host,source=/opt/bootstrap/apply \ --mount volume=script,target=/apply \ --insecure-options=image \ - docker://k8s.gcr.io/hyperkube:v1.16.2 \ + docker://k8s.gcr.io/hyperkube:v1.16.3 \ --net=host \ --dns=host \ --exec=/apply @@ -138,7 +138,7 @@ storage: contents: inline: | KUBELET_IMAGE_URL=docker://k8s.gcr.io/hyperkube - KUBELET_IMAGE_TAG=v1.16.2 + KUBELET_IMAGE_TAG=v1.16.3 - path: /opt/bootstrap/apply filesystem: root mode: 0544 diff --git a/digital-ocean/container-linux/kubernetes/cl/worker.yaml.tmpl b/digital-ocean/container-linux/kubernetes/cl/worker.yaml.tmpl index d7146a5ec..3ca5f7912 100644 --- a/digital-ocean/container-linux/kubernetes/cl/worker.yaml.tmpl +++ b/digital-ocean/container-linux/kubernetes/cl/worker.yaml.tmpl @@ -99,7 +99,7 @@ storage: contents: inline: | KUBELET_IMAGE_URL=docker://k8s.gcr.io/hyperkube - KUBELET_IMAGE_TAG=v1.16.2 + KUBELET_IMAGE_TAG=v1.16.3 - path: /etc/sysctl.d/max-user-watches.conf filesystem: root contents: @@ -117,7 +117,7 @@ storage: --volume config,kind=host,source=/etc/kubernetes \ --mount volume=config,target=/etc/kubernetes \ --insecure-options=image \ - docker://k8s.gcr.io/hyperkube:v1.16.2 \ + docker://k8s.gcr.io/hyperkube:v1.16.3 \ --net=host \ --dns=host \ --exec=/kubectl -- --kubeconfig=/etc/kubernetes/kubeconfig delete node $(hostname) diff --git a/docs/advanced/worker-pools.md b/docs/advanced/worker-pools.md index e37eb0a5b..3eeb4edb6 100644 --- a/docs/advanced/worker-pools.md +++ b/docs/advanced/worker-pools.md @@ -79,7 +79,7 @@ Create a cluster following the Azure [tutorial](../cl/azure.md#cluster). Define ```tf module "ramius-worker-pool" { - source = "git::https://github.com/poseidon/typhoon//azure/container-linux/kubernetes/workers?ref=v1.16.2" + source = "git::https://github.com/poseidon/typhoon//azure/container-linux/kubernetes/workers?ref=v1.16.3" # Azure region = module.azure-ramius.region @@ -145,7 +145,7 @@ Create a cluster following the Google Cloud [tutorial](../cl/google-cloud.md#clu ```tf module "yavin-worker-pool" { - source = "git::https://github.com/poseidon/typhoon//google-cloud/container-linux/kubernetes/workers?ref=v1.16.2" + source = "git::https://github.com/poseidon/typhoon//google-cloud/container-linux/kubernetes/workers?ref=v1.16.3" # Google Cloud region = "europe-west2" @@ -176,11 +176,11 @@ Verify a managed instance group of workers joins the cluster within a few minute ``` $ kubectl get nodes NAME STATUS AGE VERSION -yavin-controller-0.c.example-com.internal Ready 6m v1.16.2 -yavin-worker-jrbf.c.example-com.internal Ready 5m v1.16.2 -yavin-worker-mzdm.c.example-com.internal Ready 5m v1.16.2 -yavin-16x-worker-jrbf.c.example-com.internal Ready 3m v1.16.2 -yavin-16x-worker-mzdm.c.example-com.internal Ready 3m v1.16.2 +yavin-controller-0.c.example-com.internal Ready 6m v1.16.3 +yavin-worker-jrbf.c.example-com.internal Ready 5m v1.16.3 +yavin-worker-mzdm.c.example-com.internal Ready 5m v1.16.3 +yavin-16x-worker-jrbf.c.example-com.internal Ready 3m v1.16.3 +yavin-16x-worker-mzdm.c.example-com.internal Ready 3m v1.16.3 ``` ### Variables diff --git a/docs/cl/aws.md b/docs/cl/aws.md index 229af4dda..9613b34f1 100644 --- a/docs/cl/aws.md +++ b/docs/cl/aws.md @@ -1,6 +1,6 @@ # AWS -In this tutorial, we'll create a Kubernetes v1.16.2 cluster on AWS with Container Linux. +In this tutorial, we'll create a Kubernetes v1.16.3 cluster on AWS with Container Linux. We'll declare a Kubernetes cluster using the Typhoon Terraform module. Then apply the changes to create a VPC, gateway, subnets, security groups, controller instances, worker auto-scaling group, network load balancer, and TLS assets. @@ -70,7 +70,7 @@ Define a Kubernetes cluster using the module `aws/container-linux/kubernetes`. ```tf module "tempest" { - source = "git::https://github.com/poseidon/typhoon//aws/container-linux/kubernetes?ref=v1.16.2" + source = "git::https://github.com/poseidon/typhoon//aws/container-linux/kubernetes?ref=v1.16.3" # AWS cluster_name = "tempest" @@ -135,9 +135,9 @@ In 4-8 minutes, the Kubernetes cluster will be ready. $ export KUBECONFIG=/home/user/.secrets/clusters/tempest/auth/kubeconfig $ kubectl get nodes NAME STATUS ROLES AGE VERSION -ip-10-0-3-155 Ready 10m v1.16.2 -ip-10-0-26-65 Ready 10m v1.16.2 -ip-10-0-41-21 Ready 10m v1.16.2 +ip-10-0-3-155 Ready 10m v1.16.3 +ip-10-0-26-65 Ready 10m v1.16.3 +ip-10-0-41-21 Ready 10m v1.16.3 ``` List the pods. diff --git a/docs/cl/azure.md b/docs/cl/azure.md index 909e1f106..cffad6371 100644 --- a/docs/cl/azure.md +++ b/docs/cl/azure.md @@ -3,7 +3,7 @@ !!! danger Typhoon for Azure is alpha. For production, use AWS, Google Cloud, or bare-metal. As Azure matures, check [errata](https://github.com/poseidon/typhoon/wiki/Errata) for known shortcomings. -In this tutorial, we'll create a Kubernetes v1.16.2 cluster on Azure with Container Linux. +In this tutorial, we'll create a Kubernetes v1.16.3 cluster on Azure with Container Linux. We'll declare a Kubernetes cluster using the Typhoon Terraform module. Then apply the changes to create a resource group, virtual network, subnets, security groups, controller availability set, worker scale set, load balancer, and TLS assets. @@ -66,7 +66,7 @@ Define a Kubernetes cluster using the module `azure/container-linux/kubernetes`. ```tf module "ramius" { - source = "git::https://github.com/poseidon/typhoon//azure/container-linux/kubernetes?ref=v1.16.2" + source = "git::https://github.com/poseidon/typhoon//azure/container-linux/kubernetes?ref=v1.16.3" # Azure cluster_name = "ramius" @@ -132,9 +132,9 @@ In 4-8 minutes, the Kubernetes cluster will be ready. $ export KUBECONFIG=/home/user/.secrets/clusters/ramius/auth/kubeconfig $ kubectl get nodes NAME STATUS ROLES AGE VERSION -ramius-controller-0 Ready 24m v1.16.2 -ramius-worker-000001 Ready 25m v1.16.2 -ramius-worker-000002 Ready 24m v1.16.2 +ramius-controller-0 Ready 24m v1.16.3 +ramius-worker-000001 Ready 25m v1.16.3 +ramius-worker-000002 Ready 24m v1.16.3 ``` List the pods. diff --git a/docs/cl/bare-metal.md b/docs/cl/bare-metal.md index dcdcc6ab5..71b72fa91 100644 --- a/docs/cl/bare-metal.md +++ b/docs/cl/bare-metal.md @@ -1,6 +1,6 @@ # Bare-Metal -In this tutorial, we'll network boot and provision a Kubernetes v1.16.2 cluster on bare-metal with Container Linux. +In this tutorial, we'll network boot and provision a Kubernetes v1.16.3 cluster on bare-metal with Container Linux. First, we'll deploy a [Matchbox](https://github.com/poseidon/matchbox) service and setup a network boot environment. Then, we'll declare a Kubernetes cluster using the Typhoon Terraform module and power on machines. On PXE boot, machines will install Container Linux to disk, reboot into the disk install, and provision themselves as Kubernetes controllers or workers via Ignition. @@ -160,7 +160,7 @@ Define a Kubernetes cluster using the module `bare-metal/container-linux/kuberne ```tf module "bare-metal-mercury" { - source = "git::https://github.com/poseidon/typhoon//bare-metal/container-linux/kubernetes?ref=v1.16.2" + source = "git::https://github.com/poseidon/typhoon//bare-metal/container-linux/kubernetes?ref=v1.16.3" # bare-metal cluster_name = "mercury" @@ -265,9 +265,9 @@ Apply complete! Resources: 55 added, 0 changed, 0 destroyed. To watch the install to disk (until machines reboot from disk), SSH to port 2222. ``` -# before v1.16.2 +# before v1.16.3 $ ssh debug@node1.example.com -# after v1.16.2 +# after v1.16.3 $ ssh -p 2222 core@node1.example.com ``` @@ -291,9 +291,9 @@ systemd[1]: Started Kubernetes control plane. $ export KUBECONFIG=/home/user/.secrets/clusters/mercury/auth/kubeconfig $ kubectl get nodes NAME STATUS ROLES AGE VERSION -node1.example.com Ready 10m v1.16.2 -node2.example.com Ready 10m v1.16.2 -node3.example.com Ready 10m v1.16.2 +node1.example.com Ready 10m v1.16.3 +node2.example.com Ready 10m v1.16.3 +node3.example.com Ready 10m v1.16.3 ``` List the pods. diff --git a/docs/cl/digital-ocean.md b/docs/cl/digital-ocean.md index 9c0083c3d..8327759e3 100644 --- a/docs/cl/digital-ocean.md +++ b/docs/cl/digital-ocean.md @@ -1,6 +1,6 @@ # Digital Ocean -In this tutorial, we'll create a Kubernetes v1.16.2 cluster on DigitalOcean with Container Linux. +In this tutorial, we'll create a Kubernetes v1.16.3 cluster on DigitalOcean with Container Linux. We'll declare a Kubernetes cluster using the Typhoon Terraform module. Then apply the changes to create controller droplets, worker droplets, DNS records, tags, and TLS assets. @@ -65,7 +65,7 @@ Define a Kubernetes cluster using the module `digital-ocean/container-linux/kube ```tf module "digital-ocean-nemo" { - source = "git::https://github.com/poseidon/typhoon//digital-ocean/container-linux/kubernetes?ref=v1.16.2" + source = "git::https://github.com/poseidon/typhoon//digital-ocean/container-linux/kubernetes?ref=v1.16.3" # Digital Ocean cluster_name = "nemo" @@ -130,9 +130,9 @@ In 3-6 minutes, the Kubernetes cluster will be ready. $ export KUBECONFIG=/home/user/.secrets/clusters/nemo/auth/kubeconfig $ kubectl get nodes NAME STATUS ROLES AGE VERSION -10.132.110.130 Ready 10m v1.16.2 -10.132.115.81 Ready 10m v1.16.2 -10.132.124.107 Ready 10m v1.16.2 +10.132.110.130 Ready 10m v1.16.3 +10.132.115.81 Ready 10m v1.16.3 +10.132.124.107 Ready 10m v1.16.3 ``` List the pods. diff --git a/docs/cl/google-cloud.md b/docs/cl/google-cloud.md index 6beed9382..2ab5c3b26 100644 --- a/docs/cl/google-cloud.md +++ b/docs/cl/google-cloud.md @@ -1,6 +1,6 @@ # Google Cloud -In this tutorial, we'll create a Kubernetes v1.16.2 cluster on Google Compute Engine with Container Linux. +In this tutorial, we'll create a Kubernetes v1.16.3 cluster on Google Compute Engine with Container Linux. We'll declare a Kubernetes cluster using the Typhoon Terraform module. Then apply the changes to create a network, firewall rules, health checks, controller instances, worker managed instance group, load balancers, and TLS assets. @@ -71,7 +71,7 @@ Define a Kubernetes cluster using the module `google-cloud/container-linux/kuber ```tf module "google-cloud-yavin" { - source = "git::https://github.com/poseidon/typhoon//google-cloud/container-linux/kubernetes?ref=v1.16.2" + source = "git::https://github.com/poseidon/typhoon//google-cloud/container-linux/kubernetes?ref=v1.16.3" # Google Cloud cluster_name = "yavin" @@ -137,9 +137,9 @@ In 4-8 minutes, the Kubernetes cluster will be ready. $ export KUBECONFIG=/home/user/.secrets/clusters/yavin/auth/kubeconfig $ kubectl get nodes NAME ROLES STATUS AGE VERSION -yavin-controller-0.c.example-com.internal Ready 6m v1.16.2 -yavin-worker-jrbf.c.example-com.internal Ready 5m v1.16.2 -yavin-worker-mzdm.c.example-com.internal Ready 5m v1.16.2 +yavin-controller-0.c.example-com.internal Ready 6m v1.16.3 +yavin-worker-jrbf.c.example-com.internal Ready 5m v1.16.3 +yavin-worker-mzdm.c.example-com.internal Ready 5m v1.16.3 ``` List the pods. diff --git a/docs/fedora-coreos/aws.md b/docs/fedora-coreos/aws.md index 8c8d9eb12..850278a2a 100644 --- a/docs/fedora-coreos/aws.md +++ b/docs/fedora-coreos/aws.md @@ -3,7 +3,7 @@ !!! danger Typhoon for Fedora CoreOS is an early preview! Fedora CoreOS itself is a preview! Expect bugs and design shifts. Please help both projects solve problems. Report Fedora CoreOS bugs to [Fedora](https://github.com/coreos/fedora-coreos-tracker/issues). Report Typhoon issues to Typhoon. -In this tutorial, we'll create a Kubernetes v1.16.2 cluster on AWS with Fedora CoreOS. +In this tutorial, we'll create a Kubernetes v1.16.3 cluster on AWS with Fedora CoreOS. We'll declare a Kubernetes cluster using the Typhoon Terraform module. Then apply the changes to create a VPC, gateway, subnets, security groups, controller instances, worker auto-scaling group, network load balancer, and TLS assets. @@ -73,7 +73,7 @@ Define a Kubernetes cluster using the module `aws/fedora-coreos/kubernetes`. ```tf module "aws-tempest" { - source = "git::https://github.com/poseidon/typhoon//aws/fedora-coreos/kubernetes?ref=v1.16.2" + source = "git::https://github.com/poseidon/typhoon//aws/fedora-coreos/kubernetes?ref=v1.16.3" # AWS cluster_name = "tempest" @@ -138,9 +138,9 @@ In 4-8 minutes, the Kubernetes cluster will be ready. $ export KUBECONFIG=/home/user/.secrets/clusters/tempest/auth/kubeconfig $ kubectl get nodes NAME STATUS ROLES AGE VERSION -ip-10-0-3-155 Ready 10m v1.16.2 -ip-10-0-26-65 Ready 10m v1.16.2 -ip-10-0-41-21 Ready 10m v1.16.2 +ip-10-0-3-155 Ready 10m v1.16.3 +ip-10-0-26-65 Ready 10m v1.16.3 +ip-10-0-41-21 Ready 10m v1.16.3 ``` List the pods. diff --git a/docs/fedora-coreos/bare-metal.md b/docs/fedora-coreos/bare-metal.md index 28d5ffadc..7bc5157cb 100644 --- a/docs/fedora-coreos/bare-metal.md +++ b/docs/fedora-coreos/bare-metal.md @@ -3,7 +3,7 @@ !!! danger Typhoon for Fedora CoreOS is an early preview! Fedora CoreOS itself is a preview! Expect bugs and design shifts. Please help both projects solve problems. Report Fedora CoreOS bugs to [Fedora](https://github.com/coreos/fedora-coreos-tracker/issues). Report Typhoon issues to Typhoon. -In this tutorial, we'll network boot and provision a Kubernetes v1.16.2 cluster on bare-metal with Fedora CoreOS. +In this tutorial, we'll network boot and provision a Kubernetes v1.16.3 cluster on bare-metal with Fedora CoreOS. First, we'll deploy a [Matchbox](https://github.com/poseidon/matchbox) service and setup a network boot environment. Then, we'll declare a Kubernetes cluster using the Typhoon Terraform module and power on machines. On PXE boot, machines will install Fedora CoreOS to disk, reboot into the disk install, and provision themselves as Kubernetes controllers or workers via Ignition. @@ -163,7 +163,7 @@ Define a Kubernetes cluster using the module `bare-metal/fedora-coreos/kubernete ```tf module "bare-metal-mercury" { - source = "git::https://github.com/poseidon/typhoon//bare-metal/fedora-coreos/kubernetes?ref=v1.16.2" + source = "git::https://github.com/poseidon/typhoon//bare-metal/fedora-coreos/kubernetes?ref=v1.16.3" # bare-metal cluster_name = "mercury" @@ -285,9 +285,9 @@ systemd[1]: Started Kubernetes control plane. $ export KUBECONFIG=/home/user/.secrets/clusters/mercury/auth/kubeconfig $ kubectl get nodes NAME STATUS ROLES AGE VERSION -node1.example.com Ready 10m v1.16.2 -node2.example.com Ready 10m v1.16.2 -node3.example.com Ready 10m v1.16.2 +node1.example.com Ready 10m v1.16.3 +node2.example.com Ready 10m v1.16.3 +node3.example.com Ready 10m v1.16.3 ``` List the pods. diff --git a/docs/index.md b/docs/index.md index 333cf3207..32c52cad9 100644 --- a/docs/index.md +++ b/docs/index.md @@ -11,7 +11,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features -* Kubernetes v1.16.2 (upstream) +* Kubernetes v1.16.3 (upstream) * Single or multi-master, [Calico](https://www.projectcalico.org/) or [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/) * Advanced features like [worker pools](advanced/worker-pools/), [preemptible](cl/google-cloud/#preemption) workers, and [snippets](advanced/customization/#container-linux) customization @@ -47,7 +47,7 @@ Define a Kubernetes cluster by using the Terraform module for your chosen platfo ```tf module "google-cloud-yavin" { - source = "git::https://github.com/poseidon/typhoon//google-cloud/container-linux/kubernetes?ref=v1.16.2" + source = "git::https://github.com/poseidon/typhoon//google-cloud/container-linux/kubernetes?ref=v1.16.3" # Google Cloud cluster_name = "yavin" @@ -80,9 +80,9 @@ In 4-8 minutes (varies by platform), the cluster will be ready. This Google Clou $ export KUBECONFIG=/home/user/.secrets/clusters/yavin/auth/kubeconfig $ kubectl get nodes NAME ROLES STATUS AGE VERSION -yavin-controller-0.c.example-com.internal Ready 6m v1.16.2 -yavin-worker-jrbf.c.example-com.internal Ready 5m v1.16.2 -yavin-worker-mzdm.c.example-com.internal Ready 5m v1.16.2 +yavin-controller-0.c.example-com.internal Ready 6m v1.16.3 +yavin-worker-jrbf.c.example-com.internal Ready 5m v1.16.3 +yavin-worker-mzdm.c.example-com.internal Ready 5m v1.16.3 ``` List the pods. diff --git a/docs/topics/maintenance.md b/docs/topics/maintenance.md index 42b543dfa..c25544da3 100644 --- a/docs/topics/maintenance.md +++ b/docs/topics/maintenance.md @@ -18,7 +18,7 @@ module "google-cloud-yavin" { } module "bare-metal-mercury" { - source = "git::https://github.com/poseidon/typhoon//bare-metal/container-linux/kubernetes?ref=v1.16.2" + source = "git::https://github.com/poseidon/typhoon//bare-metal/container-linux/kubernetes?ref=v1.16.3" ... } ``` @@ -279,15 +279,15 @@ Typhoon modules have been adapted for Terraform v0.12. Provider plugins requirem | Typhoon Release | Terraform version | |-------------------|---------------------| -| v1.16.2 - ? | v0.12.x | -| v1.10.3 - v1.16.2 | v0.11.x | +| v1.16.3 - ? | v0.12.x | +| v1.10.3 - v1.16.3 | v0.11.x | | v1.9.2 - v1.10.2 | v0.10.4+ or v0.11.x | | v1.7.3 - v1.9.1 | v0.10.x | | v1.6.4 - v1.7.2 | v0.9.x | ### New users -New users can start with Terraform v0.12.x and follow the docs for Typhoon v1.16.2+ without issue. +New users can start with Terraform v0.12.x and follow the docs for Typhoon v1.16.3+ without issue. ### Existing users @@ -404,7 +404,7 @@ tree . └── infraB <- new Terraform v0.12.x configs ``` -Define Typhoon clusters in the new config directory using Terraform v0.12 syntax. Follow the Typhoon v1.16.2+ docs (e.g. use `terraform12` in the `infraB` dir). See [AWS](/cl/aws), [Azure](/cl/azure), [Bare-Metal](/cl/bare-metal), [Digital Ocean](/cl/digital-ocean), or [Google-Cloud](/cl/google-cloud)) to create new clusters. Follow the usual [upgrade](/topics/maintenance/#upgrades) process to apply workloads and shift traffic. Later, switch back to the old config directory and deprovision clusters with Terraform v0.11. +Define Typhoon clusters in the new config directory using Terraform v0.12 syntax. Follow the Typhoon v1.16.3+ docs (e.g. use `terraform12` in the `infraB` dir). See [AWS](/cl/aws), [Azure](/cl/azure), [Bare-Metal](/cl/bare-metal), [Digital Ocean](/cl/digital-ocean), or [Google-Cloud](/cl/google-cloud)) to create new clusters. Follow the usual [upgrade](/topics/maintenance/#upgrades) process to apply workloads and shift traffic. Later, switch back to the old config directory and deprovision clusters with Terraform v0.11. ```shell terraform12 init diff --git a/google-cloud/container-linux/kubernetes/README.md b/google-cloud/container-linux/kubernetes/README.md index f80c41948..16cac3264 100644 --- a/google-cloud/container-linux/kubernetes/README.md +++ b/google-cloud/container-linux/kubernetes/README.md @@ -11,7 +11,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features -* Kubernetes v1.16.2 (upstream) +* Kubernetes v1.16.3 (upstream) * Single or multi-master, [Calico](https://www.projectcalico.org/) or [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/) * Advanced features like [worker pools](https://typhoon.psdn.io/advanced/worker-pools/), [preemptible](https://typhoon.psdn.io/cl/google-cloud/#preemption) workers, and [snippets](https://typhoon.psdn.io/advanced/customization/#container-linux) customization diff --git a/google-cloud/container-linux/kubernetes/bootstrap.tf b/google-cloud/container-linux/kubernetes/bootstrap.tf index b0467f676..f99e9e693 100644 --- a/google-cloud/container-linux/kubernetes/bootstrap.tf +++ b/google-cloud/container-linux/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=a2b1dbe2c0c728c58fbdd34da12d19fa8bfdfcc2" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=0daa1276c633fea28e41b2c2c18831e2584deb24" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/google-cloud/container-linux/kubernetes/cl/controller.yaml.tmpl b/google-cloud/container-linux/kubernetes/cl/controller.yaml.tmpl index c69121bb1..49ed0fc25 100644 --- a/google-cloud/container-linux/kubernetes/cl/controller.yaml.tmpl +++ b/google-cloud/container-linux/kubernetes/cl/controller.yaml.tmpl @@ -112,7 +112,7 @@ systemd: --volume script,kind=host,source=/opt/bootstrap/apply \ --mount volume=script,target=/apply \ --insecure-options=image \ - docker://k8s.gcr.io/hyperkube:v1.16.2 \ + docker://k8s.gcr.io/hyperkube:v1.16.3 \ --net=host \ --dns=host \ --exec=/apply @@ -133,7 +133,7 @@ storage: contents: inline: | KUBELET_IMAGE_URL=docker://k8s.gcr.io/hyperkube - KUBELET_IMAGE_TAG=v1.16.2 + KUBELET_IMAGE_TAG=v1.16.3 - path: /opt/bootstrap/apply filesystem: root mode: 0544 diff --git a/google-cloud/container-linux/kubernetes/workers/cl/worker.yaml.tmpl b/google-cloud/container-linux/kubernetes/workers/cl/worker.yaml.tmpl index f5e3dea63..c7c1bd6d5 100644 --- a/google-cloud/container-linux/kubernetes/workers/cl/worker.yaml.tmpl +++ b/google-cloud/container-linux/kubernetes/workers/cl/worker.yaml.tmpl @@ -97,7 +97,7 @@ storage: contents: inline: | KUBELET_IMAGE_URL=docker://k8s.gcr.io/hyperkube - KUBELET_IMAGE_TAG=v1.16.2 + KUBELET_IMAGE_TAG=v1.16.3 - path: /etc/sysctl.d/max-user-watches.conf filesystem: root contents: @@ -115,7 +115,7 @@ storage: --volume config,kind=host,source=/etc/kubernetes \ --mount volume=config,target=/etc/kubernetes \ --insecure-options=image \ - docker://k8s.gcr.io/hyperkube:v1.16.2 \ + docker://k8s.gcr.io/hyperkube:v1.16.3 \ --net=host \ --dns=host \ --exec=/kubectl -- --kubeconfig=/etc/kubernetes/kubeconfig delete node $(hostname) From 42b6df89c87c0ed0690cbbaa9ad8427dbe834b46 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Wed, 13 Nov 2019 13:41:11 -0800 Subject: [PATCH 265/523] Update Prometheus from v2.14.0-rc.0 to v2.14.0 * https://github.com/prometheus/prometheus/releases/tag/v2.14.0 --- CHANGES.md | 3 +-- addons/prometheus/deployment.yaml | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index f56c36308..cc91dace8 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -14,8 +14,7 @@ Notable changes between versions. #### Addons -* Update Prometheus from v2.13.1 to [v2.14.0-rc.0](https://github.com/prometheus/prometheus/releases/tag/v2.14.0-rc.0) -* Update Prometheus from v2.13.0 to v2.13.1 +* Update Prometheus from v2.13.0 to [v2.14.0](https://github.com/prometheus/prometheus/releases/tag/v2.14.0) * Refresh rules, alerts, and dashboards from upstreams * Remove addon-resizer from kube-state-metrics ([#575](https://github.com/poseidon/typhoon/pull/575)) * Update Grafana from v6.4.2 to v6.4.4 diff --git a/addons/prometheus/deployment.yaml b/addons/prometheus/deployment.yaml index fe9c499ad..886b84b70 100644 --- a/addons/prometheus/deployment.yaml +++ b/addons/prometheus/deployment.yaml @@ -20,7 +20,7 @@ spec: serviceAccountName: prometheus containers: - name: prometheus - image: quay.io/prometheus/prometheus:v2.14.0-rc.0 + image: quay.io/prometheus/prometheus:v2.14.0 args: - --web.listen-address=0.0.0.0:9090 - --config.file=/etc/prometheus/prometheus.yaml From ad117f459294e29a2fe47e5e38c17a085e277f3a Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Wed, 13 Nov 2019 13:53:46 -0800 Subject: [PATCH 266/523] Update recommended Terraform provider versions * Recommend provider plugin version tested against --- docs/cl/aws.md | 4 ++-- docs/cl/azure.md | 4 ++-- docs/cl/bare-metal.md | 2 +- docs/cl/digital-ocean.md | 4 ++-- docs/cl/google-cloud.md | 4 ++-- docs/fedora-coreos/aws.md | 4 ++-- docs/fedora-coreos/bare-metal.md | 2 +- 7 files changed, 12 insertions(+), 12 deletions(-) diff --git a/docs/cl/aws.md b/docs/cl/aws.md index 9613b34f1..3bf2244ef 100644 --- a/docs/cl/aws.md +++ b/docs/cl/aws.md @@ -18,7 +18,7 @@ Install [Terraform](https://www.terraform.io/downloads.html) v0.12.x on your sys ```sh $ terraform version -Terraform v0.12.9 +Terraform v0.12.12 ``` Add the [terraform-provider-ct](https://github.com/poseidon/terraform-provider-ct) plugin binary for your system to `~/.terraform.d/plugins/`, noting the final name. @@ -49,7 +49,7 @@ Configure the AWS provider to use your access key credentials in a `providers.tf ```tf provider "aws" { - version = "2.31.0" + version = "2.35.0" region = "eu-central-1" shared_credentials_file = "/home/user/.config/aws/credentials" } diff --git a/docs/cl/azure.md b/docs/cl/azure.md index cffad6371..dc334fba6 100644 --- a/docs/cl/azure.md +++ b/docs/cl/azure.md @@ -21,7 +21,7 @@ Install [Terraform](https://www.terraform.io/downloads.html) v0.12.x on your sys ```sh $ terraform version -Terraform v0.12.9 +Terraform v0.12.12 ``` Add the [terraform-provider-ct](https://github.com/poseidon/terraform-provider-ct) plugin binary for your system to `~/.terraform.d/plugins/`, noting the final name. @@ -50,7 +50,7 @@ Configure the Azure provider in a `providers.tf` file. ```tf provider "azurerm" { - version = "1.35.0" + version = "1.36.1" } provider "ct" { diff --git a/docs/cl/bare-metal.md b/docs/cl/bare-metal.md index 71b72fa91..8d9e91221 100644 --- a/docs/cl/bare-metal.md +++ b/docs/cl/bare-metal.md @@ -111,7 +111,7 @@ Install [Terraform](https://www.terraform.io/downloads.html) v0.12.x on your sys ```sh $ terraform version -Terraform v0.12.9 +Terraform v0.12.12 ``` Add the [terraform-provider-matchbox](https://github.com/poseidon/terraform-provider-matchbox) plugin binary for your system to `~/.terraform.d/plugins/`, noting the final name. diff --git a/docs/cl/digital-ocean.md b/docs/cl/digital-ocean.md index 8327759e3..4dea64f34 100644 --- a/docs/cl/digital-ocean.md +++ b/docs/cl/digital-ocean.md @@ -18,7 +18,7 @@ Install [Terraform](https://www.terraform.io/downloads.html) v0.12.x on your sys ```sh $ terraform version -Terraform v0.12.9 +Terraform v0.12.12 ``` Add the [terraform-provider-ct](https://github.com/poseidon/terraform-provider-ct) plugin binary for your system to `~/.terraform.d/plugins/`, noting the final name. @@ -50,7 +50,7 @@ Configure the DigitalOcean provider to use your token in a `providers.tf` file. ```tf provider "digitalocean" { - version = "1.8.0" + version = "1.11.0" token = "${chomp(file("~/.config/digital-ocean/token"))}" } diff --git a/docs/cl/google-cloud.md b/docs/cl/google-cloud.md index 2ab5c3b26..bbe91a24b 100644 --- a/docs/cl/google-cloud.md +++ b/docs/cl/google-cloud.md @@ -18,7 +18,7 @@ Install [Terraform](https://www.terraform.io/downloads.html) v0.12.x on your sys ```sh $ terraform version -Terraform v0.12.9 +Terraform v0.12.12 ``` Add the [terraform-provider-ct](https://github.com/poseidon/terraform-provider-ct) plugin binary for your system to `~/.terraform.d/plugins/`, noting the final name. @@ -49,7 +49,7 @@ Configure the Google Cloud provider to use your service account key, project-id, ```tf provider "google" { - version = "2.16.0" + version = "2.19.0" project = "project-id" region = "us-central1" credentials = file("~/.config/google-cloud/terraform.json") diff --git a/docs/fedora-coreos/aws.md b/docs/fedora-coreos/aws.md index 850278a2a..6c0de1f9a 100644 --- a/docs/fedora-coreos/aws.md +++ b/docs/fedora-coreos/aws.md @@ -21,7 +21,7 @@ Install [Terraform](https://www.terraform.io/downloads.html) v0.12.x on your sys ```sh $ terraform version -Terraform v0.12.9 +Terraform v0.12.12 ``` Add the [terraform-provider-ct](https://github.com/poseidon/terraform-provider-ct) plugin binary for your system to `~/.terraform.d/plugins/`, noting the final name. @@ -52,7 +52,7 @@ Configure the AWS provider to use your access key credentials in a `providers.tf ```tf provider "aws" { - version = "2.31.0" + version = "2.35.0" region = "eu-central-1" shared_credentials_file = "/home/user/.config/aws/credentials" } diff --git a/docs/fedora-coreos/bare-metal.md b/docs/fedora-coreos/bare-metal.md index 7bc5157cb..a832fdd01 100644 --- a/docs/fedora-coreos/bare-metal.md +++ b/docs/fedora-coreos/bare-metal.md @@ -114,7 +114,7 @@ Install [Terraform](https://www.terraform.io/downloads.html) v0.12.x on your sys ```sh $ terraform version -Terraform v0.12.9 +Terraform v0.12.12 ``` Add the [terraform-provider-matchbox](https://github.com/poseidon/terraform-provider-matchbox) plugin binary for your system to `~/.terraform.d/plugins/`, noting the final name. From cb0598e275bd0d069e52f9af1004d2ea9d63f354 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Wed, 13 Nov 2019 14:09:34 -0800 Subject: [PATCH 267/523] Adopt Terraform v0.12 templatefile function * Update terraform-render-bootstrap module to adopt the Terrform v0.12 templatefile function feature to replace the use of terraform-provider-template's `template_dir` * Require Terraform v0.12.6+ which adds `for_each` Background: * `template_dir` was added to `terraform-provider-template` to add support for template directory rendering in CoreOS Tectonic Kubernetes distribution (~2017) * Terraform v0.12 introduced a native `templatefile` function and v0.12.6 introduced native `for_each` support (July 2019) that makes it possible to replace `template_dir` usage --- CHANGES.md | 3 +++ aws/container-linux/kubernetes/bootstrap.tf | 2 +- aws/container-linux/kubernetes/versions.tf | 2 +- aws/fedora-coreos/kubernetes/bootstrap.tf | 2 +- aws/fedora-coreos/kubernetes/versions.tf | 2 +- azure/container-linux/kubernetes/bootstrap.tf | 2 +- azure/container-linux/kubernetes/versions.tf | 2 +- bare-metal/container-linux/kubernetes/bootstrap.tf | 2 +- bare-metal/container-linux/kubernetes/versions.tf | 2 +- bare-metal/fedora-coreos/kubernetes/bootstrap.tf | 2 +- bare-metal/fedora-coreos/kubernetes/versions.tf | 2 +- digital-ocean/container-linux/kubernetes/bootstrap.tf | 2 +- digital-ocean/container-linux/kubernetes/versions.tf | 2 +- google-cloud/container-linux/kubernetes/bootstrap.tf | 2 +- google-cloud/container-linux/kubernetes/versions.tf | 2 +- 15 files changed, 17 insertions(+), 14 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index cc91dace8..439a6d487 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -4,6 +4,9 @@ Notable changes between versions. ## Latest +* Require Terraform version v0.12.6+ (action required) + * Replace internal usage of `template_dir` with `templatefile` function + ## v1.16.3 * Kubernetes [v1.16.3](https://github.com/kubernetes/kubernetes/blob/master/CHANGELOG-1.16.md#v1163) diff --git a/aws/container-linux/kubernetes/bootstrap.tf b/aws/container-linux/kubernetes/bootstrap.tf index 7da8b732e..cea530589 100644 --- a/aws/container-linux/kubernetes/bootstrap.tf +++ b/aws/container-linux/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=0daa1276c633fea28e41b2c2c18831e2584deb24" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=1bba891d95b143866614c928f87026e4c79f2eae" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/aws/container-linux/kubernetes/versions.tf b/aws/container-linux/kubernetes/versions.tf index 5f5ac8a23..f7a10ff15 100644 --- a/aws/container-linux/kubernetes/versions.tf +++ b/aws/container-linux/kubernetes/versions.tf @@ -1,7 +1,7 @@ # Terraform version and plugin versions terraform { - required_version = "~> 0.12.0" + required_version = "~> 0.12.6" required_providers { aws = "~> 2.23" ct = "~> 0.3" diff --git a/aws/fedora-coreos/kubernetes/bootstrap.tf b/aws/fedora-coreos/kubernetes/bootstrap.tf index 908ed4f70..b9ee80c4a 100644 --- a/aws/fedora-coreos/kubernetes/bootstrap.tf +++ b/aws/fedora-coreos/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=0daa1276c633fea28e41b2c2c18831e2584deb24" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=1bba891d95b143866614c928f87026e4c79f2eae" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/aws/fedora-coreos/kubernetes/versions.tf b/aws/fedora-coreos/kubernetes/versions.tf index 83532feb4..bd2776f08 100644 --- a/aws/fedora-coreos/kubernetes/versions.tf +++ b/aws/fedora-coreos/kubernetes/versions.tf @@ -1,7 +1,7 @@ # Terraform version and plugin versions terraform { - required_version = "~> 0.12.0" + required_version = "~> 0.12.6" required_providers { aws = "~> 2.23" ct = "~> 0.4" diff --git a/azure/container-linux/kubernetes/bootstrap.tf b/azure/container-linux/kubernetes/bootstrap.tf index e82cce158..41ca287f8 100644 --- a/azure/container-linux/kubernetes/bootstrap.tf +++ b/azure/container-linux/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=0daa1276c633fea28e41b2c2c18831e2584deb24" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=1bba891d95b143866614c928f87026e4c79f2eae" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/azure/container-linux/kubernetes/versions.tf b/azure/container-linux/kubernetes/versions.tf index f42de22a4..098f81942 100644 --- a/azure/container-linux/kubernetes/versions.tf +++ b/azure/container-linux/kubernetes/versions.tf @@ -1,7 +1,7 @@ # Terraform version and plugin versions terraform { - required_version = "~> 0.12.0" + required_version = "~> 0.12.6" required_providers { azurerm = "~> 1.27" ct = "~> 0.3" diff --git a/bare-metal/container-linux/kubernetes/bootstrap.tf b/bare-metal/container-linux/kubernetes/bootstrap.tf index 0d06dcc7b..f46a56fd9 100644 --- a/bare-metal/container-linux/kubernetes/bootstrap.tf +++ b/bare-metal/container-linux/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=0daa1276c633fea28e41b2c2c18831e2584deb24" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=1bba891d95b143866614c928f87026e4c79f2eae" cluster_name = var.cluster_name api_servers = [var.k8s_domain_name] diff --git a/bare-metal/container-linux/kubernetes/versions.tf b/bare-metal/container-linux/kubernetes/versions.tf index 6caf55c73..f7f7aaf69 100644 --- a/bare-metal/container-linux/kubernetes/versions.tf +++ b/bare-metal/container-linux/kubernetes/versions.tf @@ -1,7 +1,7 @@ # Terraform version and plugin versions terraform { - required_version = "~> 0.12.0" + required_version = "~> 0.12.6" required_providers { matchbox = "~> 0.3.0" ct = "~> 0.3" diff --git a/bare-metal/fedora-coreos/kubernetes/bootstrap.tf b/bare-metal/fedora-coreos/kubernetes/bootstrap.tf index 39c8be609..6b81cf94e 100644 --- a/bare-metal/fedora-coreos/kubernetes/bootstrap.tf +++ b/bare-metal/fedora-coreos/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=0daa1276c633fea28e41b2c2c18831e2584deb24" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=1bba891d95b143866614c928f87026e4c79f2eae" cluster_name = var.cluster_name api_servers = [var.k8s_domain_name] diff --git a/bare-metal/fedora-coreos/kubernetes/versions.tf b/bare-metal/fedora-coreos/kubernetes/versions.tf index a78ffe6b5..fd7df2ffa 100644 --- a/bare-metal/fedora-coreos/kubernetes/versions.tf +++ b/bare-metal/fedora-coreos/kubernetes/versions.tf @@ -1,7 +1,7 @@ # Terraform version and plugin versions terraform { - required_version = "~> 0.12.0" + required_version = "~> 0.12.6" required_providers { matchbox = "~> 0.3.0" ct = "~> 0.4" diff --git a/digital-ocean/container-linux/kubernetes/bootstrap.tf b/digital-ocean/container-linux/kubernetes/bootstrap.tf index 38d69631d..dbb539abe 100644 --- a/digital-ocean/container-linux/kubernetes/bootstrap.tf +++ b/digital-ocean/container-linux/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=0daa1276c633fea28e41b2c2c18831e2584deb24" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=1bba891d95b143866614c928f87026e4c79f2eae" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/digital-ocean/container-linux/kubernetes/versions.tf b/digital-ocean/container-linux/kubernetes/versions.tf index 1690ab5d2..c0e31a276 100644 --- a/digital-ocean/container-linux/kubernetes/versions.tf +++ b/digital-ocean/container-linux/kubernetes/versions.tf @@ -1,7 +1,7 @@ # Terraform version and plugin versions terraform { - required_version = "~> 0.12.0" + required_version = "~> 0.12.6" required_providers { digitalocean = "~> 1.3" ct = "~> 0.3" diff --git a/google-cloud/container-linux/kubernetes/bootstrap.tf b/google-cloud/container-linux/kubernetes/bootstrap.tf index f99e9e693..303456763 100644 --- a/google-cloud/container-linux/kubernetes/bootstrap.tf +++ b/google-cloud/container-linux/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=0daa1276c633fea28e41b2c2c18831e2584deb24" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=1bba891d95b143866614c928f87026e4c79f2eae" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/google-cloud/container-linux/kubernetes/versions.tf b/google-cloud/container-linux/kubernetes/versions.tf index 2dc04ad21..1ebc23a34 100644 --- a/google-cloud/container-linux/kubernetes/versions.tf +++ b/google-cloud/container-linux/kubernetes/versions.tf @@ -1,7 +1,7 @@ # Terraform version and plugin versions terraform { - required_version = "~> 0.12.0" + required_version = "~> 0.12.6" required_providers { google = "~> 2.5" ct = "~> 0.3" From a271b9f34001d118320166c8f2bce5e855a9b086 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Wed, 13 Nov 2019 16:42:17 -0800 Subject: [PATCH 268/523] Update CoreDNS from v1.6.2 to v1.6.5 * Add health `lameduck` option 5s. Before CoreDNS shuts down, it will wait and report unhealthy for 5s to allow time for plugins to shutdown cleanly * Minor bug fixes over a few releases * https://coredns.io/2019/08/31/coredns-1.6.3-release/ * https://coredns.io/2019/09/27/coredns-1.6.4-release/ * https://coredns.io/2019/11/05/coredns-1.6.5-release/ --- CHANGES.md | 6 ++++-- aws/container-linux/kubernetes/bootstrap.tf | 2 +- aws/fedora-coreos/kubernetes/bootstrap.tf | 2 +- azure/container-linux/kubernetes/bootstrap.tf | 2 +- bare-metal/container-linux/kubernetes/bootstrap.tf | 2 +- bare-metal/fedora-coreos/kubernetes/bootstrap.tf | 2 +- digital-ocean/container-linux/kubernetes/bootstrap.tf | 2 +- google-cloud/container-linux/kubernetes/bootstrap.tf | 2 +- 8 files changed, 11 insertions(+), 9 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 439a6d487..320eae770 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -4,8 +4,10 @@ Notable changes between versions. ## Latest -* Require Terraform version v0.12.6+ (action required) - * Replace internal usage of `template_dir` with `templatefile` function +* Update CoreDNS from v1.6.2 to v1.6.5 ([#588](https://github.com/poseidon/typhoon/pull/588)) + * Add health `lameduck` option to wait before shutdown +* Replace usage of `template_dir` with `templatefile` function ([#587](https://github.com/poseidon/typhoon/pull/587)) + * Require Terraform version v0.12.6+ (action required) ## v1.16.3 diff --git a/aws/container-linux/kubernetes/bootstrap.tf b/aws/container-linux/kubernetes/bootstrap.tf index cea530589..93e0fcfd3 100644 --- a/aws/container-linux/kubernetes/bootstrap.tf +++ b/aws/container-linux/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=1bba891d95b143866614c928f87026e4c79f2eae" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=43e1230c550b1456f13bd0df199164139dfbfba7" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/aws/fedora-coreos/kubernetes/bootstrap.tf b/aws/fedora-coreos/kubernetes/bootstrap.tf index b9ee80c4a..7ca452918 100644 --- a/aws/fedora-coreos/kubernetes/bootstrap.tf +++ b/aws/fedora-coreos/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=1bba891d95b143866614c928f87026e4c79f2eae" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=43e1230c550b1456f13bd0df199164139dfbfba7" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/azure/container-linux/kubernetes/bootstrap.tf b/azure/container-linux/kubernetes/bootstrap.tf index 41ca287f8..49f5bf433 100644 --- a/azure/container-linux/kubernetes/bootstrap.tf +++ b/azure/container-linux/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=1bba891d95b143866614c928f87026e4c79f2eae" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=43e1230c550b1456f13bd0df199164139dfbfba7" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/bare-metal/container-linux/kubernetes/bootstrap.tf b/bare-metal/container-linux/kubernetes/bootstrap.tf index f46a56fd9..32e123152 100644 --- a/bare-metal/container-linux/kubernetes/bootstrap.tf +++ b/bare-metal/container-linux/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=1bba891d95b143866614c928f87026e4c79f2eae" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=43e1230c550b1456f13bd0df199164139dfbfba7" cluster_name = var.cluster_name api_servers = [var.k8s_domain_name] diff --git a/bare-metal/fedora-coreos/kubernetes/bootstrap.tf b/bare-metal/fedora-coreos/kubernetes/bootstrap.tf index 6b81cf94e..a2491d39e 100644 --- a/bare-metal/fedora-coreos/kubernetes/bootstrap.tf +++ b/bare-metal/fedora-coreos/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=1bba891d95b143866614c928f87026e4c79f2eae" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=43e1230c550b1456f13bd0df199164139dfbfba7" cluster_name = var.cluster_name api_servers = [var.k8s_domain_name] diff --git a/digital-ocean/container-linux/kubernetes/bootstrap.tf b/digital-ocean/container-linux/kubernetes/bootstrap.tf index dbb539abe..f5871b1a5 100644 --- a/digital-ocean/container-linux/kubernetes/bootstrap.tf +++ b/digital-ocean/container-linux/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=1bba891d95b143866614c928f87026e4c79f2eae" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=43e1230c550b1456f13bd0df199164139dfbfba7" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/google-cloud/container-linux/kubernetes/bootstrap.tf b/google-cloud/container-linux/kubernetes/bootstrap.tf index 303456763..dbc9384fc 100644 --- a/google-cloud/container-linux/kubernetes/bootstrap.tf +++ b/google-cloud/container-linux/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=1bba891d95b143866614c928f87026e4c79f2eae" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=43e1230c550b1456f13bd0df199164139dfbfba7" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] From 0e4ee5efc96dbf2c134f57bbc9e93e36fcedf618 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Wed, 13 Nov 2019 17:13:36 -0800 Subject: [PATCH 269/523] Add small CPU resource requests to static pods * Set small CPU requests on static pods kube-apiserver, kube-controller-manager, and kube-scheduler to align with upstream tooling and for edge cases * Effectively, a practical case for these requests hasn't been observed. However, a small static pod CPU request may offer a slight benefit if a controller became overloaded and the below mechanisms were insufficient Existing safeguards: * Control plane nodes are tainted to isolate them from ordinary workloads. Even dense workloads can only compress CPU resources on worker nodes. * Control plane static pods use the highest priority class, so contention favors control plane pods (over say node-exporter) and CPU is compressible too. See: https://github.com/poseidon/terraform-render-bootstrap/pull/161 --- CHANGES.md | 2 ++ aws/container-linux/kubernetes/bootstrap.tf | 2 +- aws/fedora-coreos/kubernetes/bootstrap.tf | 2 +- azure/container-linux/kubernetes/bootstrap.tf | 2 +- bare-metal/container-linux/kubernetes/bootstrap.tf | 2 +- bare-metal/fedora-coreos/kubernetes/bootstrap.tf | 2 +- digital-ocean/container-linux/kubernetes/bootstrap.tf | 2 +- google-cloud/container-linux/kubernetes/bootstrap.tf | 2 +- 8 files changed, 9 insertions(+), 7 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 320eae770..fb099950a 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -6,6 +6,8 @@ Notable changes between versions. * Update CoreDNS from v1.6.2 to v1.6.5 ([#588](https://github.com/poseidon/typhoon/pull/588)) * Add health `lameduck` option to wait before shutdown +* Add CPU requests to control plane static pods ([#589](https://github.com/poseidon/typhoon/pull/589)) + * May provide slight edge case benefits and aligns with upstream * Replace usage of `template_dir` with `templatefile` function ([#587](https://github.com/poseidon/typhoon/pull/587)) * Require Terraform version v0.12.6+ (action required) diff --git a/aws/container-linux/kubernetes/bootstrap.tf b/aws/container-linux/kubernetes/bootstrap.tf index 93e0fcfd3..20663f7e4 100644 --- a/aws/container-linux/kubernetes/bootstrap.tf +++ b/aws/container-linux/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=43e1230c550b1456f13bd0df199164139dfbfba7" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/aws/fedora-coreos/kubernetes/bootstrap.tf b/aws/fedora-coreos/kubernetes/bootstrap.tf index 7ca452918..59639ac92 100644 --- a/aws/fedora-coreos/kubernetes/bootstrap.tf +++ b/aws/fedora-coreos/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=43e1230c550b1456f13bd0df199164139dfbfba7" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/azure/container-linux/kubernetes/bootstrap.tf b/azure/container-linux/kubernetes/bootstrap.tf index 49f5bf433..60da3ca36 100644 --- a/azure/container-linux/kubernetes/bootstrap.tf +++ b/azure/container-linux/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=43e1230c550b1456f13bd0df199164139dfbfba7" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/bare-metal/container-linux/kubernetes/bootstrap.tf b/bare-metal/container-linux/kubernetes/bootstrap.tf index 32e123152..d9898814f 100644 --- a/bare-metal/container-linux/kubernetes/bootstrap.tf +++ b/bare-metal/container-linux/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=43e1230c550b1456f13bd0df199164139dfbfba7" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=" cluster_name = var.cluster_name api_servers = [var.k8s_domain_name] diff --git a/bare-metal/fedora-coreos/kubernetes/bootstrap.tf b/bare-metal/fedora-coreos/kubernetes/bootstrap.tf index a2491d39e..587d155ca 100644 --- a/bare-metal/fedora-coreos/kubernetes/bootstrap.tf +++ b/bare-metal/fedora-coreos/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=43e1230c550b1456f13bd0df199164139dfbfba7" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=" cluster_name = var.cluster_name api_servers = [var.k8s_domain_name] diff --git a/digital-ocean/container-linux/kubernetes/bootstrap.tf b/digital-ocean/container-linux/kubernetes/bootstrap.tf index f5871b1a5..339dde6ca 100644 --- a/digital-ocean/container-linux/kubernetes/bootstrap.tf +++ b/digital-ocean/container-linux/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=43e1230c550b1456f13bd0df199164139dfbfba7" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/google-cloud/container-linux/kubernetes/bootstrap.tf b/google-cloud/container-linux/kubernetes/bootstrap.tf index dbc9384fc..79ec3db70 100644 --- a/google-cloud/container-linux/kubernetes/bootstrap.tf +++ b/google-cloud/container-linux/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=43e1230c550b1456f13bd0df199164139dfbfba7" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] From 19ee57dc04d8b929800fee6a154618b4694bc120 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Wed, 13 Nov 2019 17:31:11 -0800 Subject: [PATCH 270/523] Use GCP region_instance_group_manager version block format * terraform-provider-google v2.19.0 deprecates `instance_template` within `google_compute_region_instance_group_manager` in order to support a scheme with multiple version blocks. Adapt our single version to the new format to resolve deprecation warnings. * Fixes: Warning: "instance_template": [DEPRECATED] This field will be replaced by `version.instance_template` in 3.0.0 * Require terraform-provider-google v2.19.0+ (action required) --- CHANGES.md | 6 ++++++ google-cloud/container-linux/kubernetes/versions.tf | 8 ++++---- .../container-linux/kubernetes/workers/workers.tf | 5 ++++- 3 files changed, 14 insertions(+), 5 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index fb099950a..c6144af6d 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -11,6 +11,12 @@ Notable changes between versions. * Replace usage of `template_dir` with `templatefile` function ([#587](https://github.com/poseidon/typhoon/pull/587)) * Require Terraform version v0.12.6+ (action required) +#### Google + +* Use new `google_compute_region_instance_group_manager` version block format + * Fixes warning that `instance_template` is deprecated + * Require `terraform-provider-google` v2.19.0+ (action required) + ## v1.16.3 * Kubernetes [v1.16.3](https://github.com/kubernetes/kubernetes/blob/master/CHANGELOG-1.16.md#v1163) diff --git a/google-cloud/container-linux/kubernetes/versions.tf b/google-cloud/container-linux/kubernetes/versions.tf index 1ebc23a34..f28854f96 100644 --- a/google-cloud/container-linux/kubernetes/versions.tf +++ b/google-cloud/container-linux/kubernetes/versions.tf @@ -3,9 +3,9 @@ terraform { required_version = "~> 0.12.6" required_providers { - google = "~> 2.5" - ct = "~> 0.3" - template = "~> 2.1" - null = "~> 2.1" + google = "~> 2.19" + ct = "~> 0.3" + template = "~> 2.1" + null = "~> 2.1" } } diff --git a/google-cloud/container-linux/kubernetes/workers/workers.tf b/google-cloud/container-linux/kubernetes/workers/workers.tf index 83eea9484..0c0b8ba24 100644 --- a/google-cloud/container-linux/kubernetes/workers/workers.tf +++ b/google-cloud/container-linux/kubernetes/workers/workers.tf @@ -5,8 +5,11 @@ resource "google_compute_region_instance_group_manager" "workers" { # instance name prefix for instances in the group base_instance_name = "${var.name}-worker" - instance_template = google_compute_instance_template.worker.self_link region = var.region + version { + name = "default" + instance_template = google_compute_instance_template.worker.self_link + } target_size = var.worker_count target_pools = [google_compute_target_pool.workers.self_link] From 8a9e8595aec7ea44942e392b7c00c3529990ff57 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Wed, 13 Nov 2019 23:44:02 -0800 Subject: [PATCH 271/523] Fix terraform fmt formatting --- aws/container-linux/kubernetes/ssh.tf | 4 +- aws/container-linux/kubernetes/variables.tf | 20 +++++----- .../kubernetes/workers/variables.tf | 12 +++--- aws/fedora-coreos/kubernetes/ssh.tf | 2 +- aws/fedora-coreos/kubernetes/variables.tf | 20 +++++----- aws/fedora-coreos/kubernetes/workers/ami.tf | 2 +- .../kubernetes/workers/variables.tf | 12 +++--- .../kubernetes/workers/workers.tf | 6 +-- azure/container-linux/kubernetes/ssh.tf | 2 +- azure/container-linux/kubernetes/variables.tf | 20 +++++----- .../kubernetes/workers/variables.tf | 12 +++--- .../kubernetes/workers/workers.tf | 4 +- .../container-linux/kubernetes/profiles.tf | 30 +++++++------- bare-metal/container-linux/kubernetes/ssh.tf | 2 +- .../container-linux/kubernetes/variables.tf | 40 +++++++++---------- bare-metal/fedora-coreos/kubernetes/ssh.tf | 2 +- .../fedora-coreos/kubernetes/variables.tf | 36 ++++++++--------- .../container-linux/kubernetes/network.tf | 2 +- .../container-linux/kubernetes/ssh.tf | 2 +- .../container-linux/kubernetes/variables.tf | 16 ++++---- .../container-linux/kubernetes/ssh.tf | 4 +- .../container-linux/kubernetes/variables.tf | 20 +++++----- .../kubernetes/workers/variables.tf | 24 +++++------ .../kubernetes/workers/workers.tf | 4 +- 24 files changed, 149 insertions(+), 149 deletions(-) diff --git a/aws/container-linux/kubernetes/ssh.tf b/aws/container-linux/kubernetes/ssh.tf index 1821e195b..1d13aa55d 100644 --- a/aws/container-linux/kubernetes/ssh.tf +++ b/aws/container-linux/kubernetes/ssh.tf @@ -1,7 +1,7 @@ # Secure copy assets to controllers. resource "null_resource" "copy-controller-secrets" { count = var.controller_count - + depends_on = [ module.bootstrap, ] @@ -47,7 +47,7 @@ resource "null_resource" "copy-controller-secrets" { content = module.bootstrap.etcd_peer_key destination = "$HOME/etcd-peer.key" } - + provisioner "file" { source = var.asset_dir destination = "$HOME/assets" diff --git a/aws/container-linux/kubernetes/variables.tf b/aws/container-linux/kubernetes/variables.tf index ba9c172c7..b0189ce2f 100644 --- a/aws/container-linux/kubernetes/variables.tf +++ b/aws/container-linux/kubernetes/variables.tf @@ -126,37 +126,37 @@ variable "pod_cidr" { } variable "service_cidr" { - type = string + type = string description = < coreos flavor, stable channel # flatcar-stable -> flatcar flavor, stable channel - flavor = split("-", var.os_channel)[0] + flavor = split("-", var.os_channel)[0] channel = split("-", var.os_channel)[1] } @@ -34,12 +34,12 @@ data "template_file" "container-linux-install-configs" { template = file("${path.module}/cl/install.yaml.tmpl") vars = { - os_flavor = local.flavor - os_channel = local.channel - os_version = var.os_version - ignition_endpoint = format("%s/ignition", var.matchbox_http_endpoint) - install_disk = var.install_disk - ssh_authorized_key = var.ssh_authorized_key + os_flavor = local.flavor + os_channel = local.channel + os_version = var.os_version + ignition_endpoint = format("%s/ignition", var.matchbox_http_endpoint) + install_disk = var.install_disk + ssh_authorized_key = var.ssh_authorized_key # only cached-container-linux profile adds -b baseurl baseurl_flag = "" } @@ -75,12 +75,12 @@ data "template_file" "cached-container-linux-install-configs" { template = file("${path.module}/cl/install.yaml.tmpl") vars = { - os_flavor = local.flavor - os_channel = local.channel - os_version = var.os_version - ignition_endpoint = format("%s/ignition", var.matchbox_http_endpoint) - install_disk = var.install_disk - ssh_authorized_key = var.ssh_authorized_key + os_flavor = local.flavor + os_channel = local.channel + os_version = var.os_version + ignition_endpoint = format("%s/ignition", var.matchbox_http_endpoint) + install_disk = var.install_disk + ssh_authorized_key = var.ssh_authorized_key # profile uses -b baseurl to install from matchbox cache baseurl_flag = "-b ${var.matchbox_http_endpoint}/assets/${local.flavor}" } @@ -156,7 +156,7 @@ data "template_file" "controller-configs" { domain_name = var.controllers.*.domain[count.index] etcd_name = var.controllers.*.name[count.index] etcd_initial_cluster = join(",", formatlist("%s=https://%s:2380", var.controllers.*.name, var.controllers.*.domain)) - cgroup_driver = var.os_channel == "flatcar-edge" ? "systemd" : "cgroupfs" + cgroup_driver = var.os_channel == "flatcar-edge" ? "systemd" : "cgroupfs" cluster_dns_service_ip = module.bootstrap.cluster_dns_service_ip cluster_domain_suffix = var.cluster_domain_suffix ssh_authorized_key = var.ssh_authorized_key @@ -184,7 +184,7 @@ data "template_file" "worker-configs" { vars = { domain_name = var.workers.*.domain[count.index] - cgroup_driver = var.os_channel == "flatcar-edge" ? "systemd" : "cgroupfs" + cgroup_driver = var.os_channel == "flatcar-edge" ? "systemd" : "cgroupfs" cluster_dns_service_ip = module.bootstrap.cluster_dns_service_ip cluster_domain_suffix = var.cluster_domain_suffix ssh_authorized_key = var.ssh_authorized_key diff --git a/bare-metal/container-linux/kubernetes/ssh.tf b/bare-metal/container-linux/kubernetes/ssh.tf index 54e59fcad..5edea88e4 100644 --- a/bare-metal/container-linux/kubernetes/ssh.tf +++ b/bare-metal/container-linux/kubernetes/ssh.tf @@ -57,7 +57,7 @@ resource "null_resource" "copy-controller-secrets" { content = module.bootstrap.etcd_peer_key destination = "$HOME/etcd-peer.key" } - + provisioner "file" { source = var.asset_dir destination = "$HOME/assets" diff --git a/bare-metal/container-linux/kubernetes/variables.tf b/bare-metal/container-linux/kubernetes/variables.tf index 525658571..4ebdbf064 100644 --- a/bare-metal/container-linux/kubernetes/variables.tf +++ b/bare-metal/container-linux/kubernetes/variables.tf @@ -24,8 +24,8 @@ variable "os_version" { variable "controllers" { type = list(object({ - name = string - mac = string + name = string + mac = string domain = string })) description = < Date: Wed, 13 Nov 2019 23:39:19 -0800 Subject: [PATCH 272/523] Add node-exporter alerts and Grafana dashboard * Add Prometheus alerts from node-exporter * Add Grafana dashboard nodes.json, from node-exporter * Not adding recording rules, since those are only used by some node-exporter USE dashboards not being included --- CHANGES.md | 4 + addons/grafana/dashboards-node-exporter.yaml | 968 +++++++++++++++++++ addons/grafana/deployment.yaml | 5 + addons/prometheus/rules.yaml | 130 +++ 4 files changed, 1107 insertions(+) create mode 100644 addons/grafana/dashboards-node-exporter.yaml diff --git a/CHANGES.md b/CHANGES.md index c6144af6d..5787a1a44 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -17,6 +17,10 @@ Notable changes between versions. * Fixes warning that `instance_template` is deprecated * Require `terraform-provider-google` v2.19.0+ (action required) +#### Addons + +* Add node alerts and Grafana dashboard from node-exporter ([#591](https://github.com/poseidon/typhoon/pull/591)) + ## v1.16.3 * Kubernetes [v1.16.3](https://github.com/kubernetes/kubernetes/blob/master/CHANGELOG-1.16.md#v1163) diff --git a/addons/grafana/dashboards-node-exporter.yaml b/addons/grafana/dashboards-node-exporter.yaml new file mode 100644 index 000000000..70e523996 --- /dev/null +++ b/addons/grafana/dashboards-node-exporter.yaml @@ -0,0 +1,968 @@ +apiVersion: v1 +data: + nodes.json: |- + { + "__inputs": [ + + ], + "__requires": [ + + ], + "annotations": { + "list": [ + + ] + }, + "editable": false, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "id": null, + "links": [ + + ], + "refresh": "", + "rows": [ + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "gridPos": { + + }, + "id": 2, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "(\n (1 - rate(node_cpu_seconds_total{job=\"node-exporter\", mode=\"idle\", instance=\"$instance\"}[$__interval]))\n/ ignoring(cpu) group_left\n count without (cpu)( node_cpu_seconds_total{job=\"node-exporter\", mode=\"idle\", instance=\"$instance\"})\n)\n", + "format": "time_series", + "interval": "1m", + "intervalFactor": 5, + "legendFormat": "{{cpu}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU Usage", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": 1, + "min": 0, + "show": true + }, + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": 1, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 0, + "gridPos": { + + }, + "id": 3, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "node_load1{job=\"node-exporter\", instance=\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "1m load average", + "refId": "A" + }, + { + "expr": "node_load5{job=\"node-exporter\", instance=\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "5m load average", + "refId": "B" + }, + { + "expr": "node_load15{job=\"node-exporter\", instance=\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "15m load average", + "refId": "C" + }, + { + "expr": "count(node_cpu_seconds_total{job=\"node-exporter\", instance=\"$instance\", mode=\"idle\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "logical cores", + "refId": "D" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Load Average", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "gridPos": { + + }, + "id": 4, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 9, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "(\n node_memory_MemTotal_bytes{job=\"node-exporter\", instance=\"$instance\"}\n-\n node_memory_MemFree_bytes{job=\"node-exporter\", instance=\"$instance\"}\n-\n node_memory_Buffers_bytes{job=\"node-exporter\", instance=\"$instance\"}\n-\n node_memory_Cached_bytes{job=\"node-exporter\", instance=\"$instance\"}\n)\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "memory used", + "refId": "A" + }, + { + "expr": "node_memory_Buffers_bytes{job=\"node-exporter\", instance=\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "memory buffers", + "refId": "B" + }, + { + "expr": "node_memory_Cached_bytes{job=\"node-exporter\", instance=\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "memory cached", + "refId": "C" + }, + { + "expr": "node_memory_MemFree_bytes{job=\"node-exporter\", instance=\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "memory free", + "refId": "D" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Memory Usage", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "datasource": "$datasource", + "format": "percent", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": true, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + + }, + "id": 5, + "interval": null, + "links": [ + + ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "span": 3, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "100 -\n(\n node_memory_MemAvailable_bytes{job=\"node-exporter\", instance=\"$instance\"}\n/\n node_memory_MemTotal_bytes{job=\"node-exporter\", instance=\"$instance\"}\n* 100\n)\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": "80, 90", + "title": "Memory Usage", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 0, + "gridPos": { + + }, + "id": 6, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + { + "alias": "/ read| written/", + "yaxis": 1 + }, + { + "alias": "/ io time/", + "yaxis": 2 + } + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(node_disk_read_bytes_total{job=\"node-exporter\", instance=\"$instance\", device!~\"dm.*\"}[$__interval])", + "format": "time_series", + "interval": "1m", + "intervalFactor": 2, + "legendFormat": "{{device}} read", + "refId": "A" + }, + { + "expr": "rate(node_disk_written_bytes_total{job=\"node-exporter\", instance=\"$instance\", device!~\"dm.*\"}[$__interval])", + "format": "time_series", + "interval": "1m", + "intervalFactor": 2, + "legendFormat": "{{device}} written", + "refId": "B" + }, + { + "expr": "rate(node_disk_io_time_seconds_total{job=\"node-exporter\", instance=\"$instance\", device!~\"dm.*\"}[$__interval])", + "format": "time_series", + "interval": "1m", + "intervalFactor": 2, + "legendFormat": "{{device}} io time", + "refId": "C" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Disk I/O", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "gridPos": { + + }, + "id": 7, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + { + "alias": "used", + "color": "#E0B400" + }, + { + "alias": "available", + "color": "#73BF69" + } + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(\n max by (device) (\n node_filesystem_size_bytes{job=\"node-exporter\", instance=\"$instance\", fstype!~\"tmpfs|nsfs|vfat\"}\n -\n node_filesystem_avail_bytes{job=\"node-exporter\", instance=\"$instance\", fstype!~\"tmpfs|nsfs|vfat\"}\n )\n)\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "used", + "refId": "A" + }, + { + "expr": "sum(\n max by (device) (\n node_filesystem_avail_bytes{job=\"node-exporter\", instance=\"$instance\", fstype!~\"tmpfs|nsfs|vfat\"}\n )\n)\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "available", + "refId": "B" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Disk Space Usage", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 0, + "gridPos": { + + }, + "id": 8, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(node_network_receive_bytes_total{job=\"node-exporter\", instance=\"$instance\", device!=\"lo\"}[$__interval])", + "format": "time_series", + "interval": "1m", + "intervalFactor": 2, + "legendFormat": "{{device}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Network Received", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 0, + "gridPos": { + + }, + "id": 9, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(node_network_transmit_bytes_total{job=\"node-exporter\", instance=\"$instance\", device!=\"lo\"}[$__interval])", + "format": "time_series", + "interval": "1m", + "intervalFactor": 2, + "legendFormat": "{{device}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Network Transmitted", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + } + ], + "schemaVersion": 14, + "style": "dark", + "tags": [ + + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": null, + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "instance", + "options": [ + + ], + "query": "label_values(node_exporter_build_info{job=\"node-exporter\"}, instance)", + "refresh": 2, + "regex": "", + "sort": 0, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "", + "title": "Nodes", + "uid": "fa49a4706d07a042595b664c87fb33ea", + "version": 0 + } +kind: ConfigMap +metadata: + name: grafana-dashboards-node-exporter + namespace: monitoring diff --git a/addons/grafana/deployment.yaml b/addons/grafana/deployment.yaml index 9ea495758..8e7e10348 100644 --- a/addons/grafana/deployment.yaml +++ b/addons/grafana/deployment.yaml @@ -56,6 +56,8 @@ spec: mountPath: /etc/grafana/provisioning/dashboards - name: dashboards-etcd mountPath: /etc/grafana/dashboards/etcd + - name: dashboards-node-exporter + mountPath: /etc/grafana/dashboards/node-exporter - name: dashboards-prom mountPath: /etc/grafana/dashboards/prom - name: dashboards-k8s @@ -81,6 +83,9 @@ spec: - name: dashboards-etcd configMap: name: grafana-dashboards-etcd + - name: dashboards-node-exporter + configMap: + name: grafana-dashboards-node-exporter - name: dashboards-prom configMap: name: grafana-dashboards-prom diff --git a/addons/prometheus/rules.yaml b/addons/prometheus/rules.yaml index d24d73d55..78746f236 100644 --- a/addons/prometheus/rules.yaml +++ b/addons/prometheus/rules.yaml @@ -865,6 +865,136 @@ data: } ] } + node-exporter.yaml: |- + { + "groups": [ + { + "name": "node-exporter", + "rules": [ + { + "alert": "NodeFilesystemSpaceFillingUp", + "annotations": { + "description": "Filesystem on {{ $labels.device }} at {{ $labels.instance }} has only {{ printf \"%.2f\" $value }}% available space left and is filling up.", + "summary": "Filesystem is predicted to run out of space within the next 24 hours." + }, + "expr": "(\n node_filesystem_avail_bytes{job=\"node-exporter\",fstype!~\"tmpfs|nsfs|vfat\"} / node_filesystem_size_bytes{job=\"node-exporter\",fstype!~\"tmpfs|nsfs|vfat\"} * 100 < 40\nand\n predict_linear(node_filesystem_avail_bytes{job=\"node-exporter\",fstype!~\"tmpfs|nsfs|vfat\"}[6h], 24*60*60) < 0\nand\n node_filesystem_readonly{job=\"node-exporter\",fstype!~\"tmpfs|nsfs|vfat\"} == 0\n)\n", + "for": "1h", + "labels": { + "severity": "warning" + } + }, + { + "alert": "NodeFilesystemSpaceFillingUp", + "annotations": { + "description": "Filesystem on {{ $labels.device }} at {{ $labels.instance }} has only {{ printf \"%.2f\" $value }}% available space left and is filling up fast.", + "summary": "Filesystem is predicted to run out of space within the next 4 hours." + }, + "expr": "(\n node_filesystem_avail_bytes{job=\"node-exporter\",fstype!~\"tmpfs|nsfs|vfat\"} / node_filesystem_size_bytes{job=\"node-exporter\",fstype!~\"tmpfs|nsfs|vfat\"} * 100 < 20\nand\n predict_linear(node_filesystem_avail_bytes{job=\"node-exporter\",fstype!~\"tmpfs|nsfs|vfat\"}[6h], 4*60*60) < 0\nand\n node_filesystem_readonly{job=\"node-exporter\",fstype!~\"tmpfs|nsfs|vfat\"} == 0\n)\n", + "for": "1h", + "labels": { + "severity": "critical" + } + }, + { + "alert": "NodeFilesystemAlmostOutOfSpace", + "annotations": { + "description": "Filesystem on {{ $labels.device }} at {{ $labels.instance }} has only {{ printf \"%.2f\" $value }}% available space left.", + "summary": "Filesystem has less than 5% space left." + }, + "expr": "(\n node_filesystem_avail_bytes{job=\"node-exporter\",fstype!~\"tmpfs|nsfs|vfat\"} / node_filesystem_size_bytes{job=\"node-exporter\",fstype!~\"tmpfs|nsfs|vfat\"} * 100 < 5\nand\n node_filesystem_readonly{job=\"node-exporter\",fstype!~\"tmpfs|nsfs|vfat\"} == 0\n)\n", + "for": "1h", + "labels": { + "severity": "warning" + } + }, + { + "alert": "NodeFilesystemAlmostOutOfSpace", + "annotations": { + "description": "Filesystem on {{ $labels.device }} at {{ $labels.instance }} has only {{ printf \"%.2f\" $value }}% available space left.", + "summary": "Filesystem has less than 3% space left." + }, + "expr": "(\n node_filesystem_avail_bytes{job=\"node-exporter\",fstype!~\"tmpfs|nsfs|vfat\"} / node_filesystem_size_bytes{job=\"node-exporter\",fstype!~\"tmpfs|nsfs|vfat\"} * 100 < 3\nand\n node_filesystem_readonly{job=\"node-exporter\",fstype!~\"tmpfs|nsfs|vfat\"} == 0\n)\n", + "for": "1h", + "labels": { + "severity": "critical" + } + }, + { + "alert": "NodeFilesystemFilesFillingUp", + "annotations": { + "description": "Filesystem on {{ $labels.device }} at {{ $labels.instance }} has only {{ printf \"%.2f\" $value }}% available inodes left and is filling up.", + "summary": "Filesystem is predicted to run out of inodes within the next 24 hours." + }, + "expr": "(\n node_filesystem_files_free{job=\"node-exporter\",fstype!~\"tmpfs|nsfs|vfat\"} / node_filesystem_files{job=\"node-exporter\",fstype!~\"tmpfs|nsfs|vfat\"} * 100 < 40\nand\n predict_linear(node_filesystem_files_free{job=\"node-exporter\",fstype!~\"tmpfs|nsfs|vfat\"}[6h], 24*60*60) < 0\nand\n node_filesystem_readonly{job=\"node-exporter\",fstype!~\"tmpfs|nsfs|vfat\"} == 0\n)\n", + "for": "1h", + "labels": { + "severity": "warning" + } + }, + { + "alert": "NodeFilesystemFilesFillingUp", + "annotations": { + "description": "Filesystem on {{ $labels.device }} at {{ $labels.instance }} has only {{ printf \"%.2f\" $value }}% available inodes left and is filling up fast.", + "summary": "Filesystem is predicted to run out of inodes within the next 4 hours." + }, + "expr": "(\n node_filesystem_files_free{job=\"node-exporter\",fstype!~\"tmpfs|nsfs|vfat\"} / node_filesystem_files{job=\"node-exporter\",fstype!~\"tmpfs|nsfs|vfat\"} * 100 < 20\nand\n predict_linear(node_filesystem_files_free{job=\"node-exporter\",fstype!~\"tmpfs|nsfs|vfat\"}[6h], 4*60*60) < 0\nand\n node_filesystem_readonly{job=\"node-exporter\",fstype!~\"tmpfs|nsfs|vfat\"} == 0\n)\n", + "for": "1h", + "labels": { + "severity": "critical" + } + }, + { + "alert": "NodeFilesystemAlmostOutOfFiles", + "annotations": { + "description": "Filesystem on {{ $labels.device }} at {{ $labels.instance }} has only {{ printf \"%.2f\" $value }}% available inodes left.", + "summary": "Filesystem has less than 5% inodes left." + }, + "expr": "(\n node_filesystem_files_free{job=\"node-exporter\",fstype!~\"tmpfs|nsfs|vfat\"} / node_filesystem_files{job=\"node-exporter\",fstype!~\"tmpfs|nsfs|vfat\"} * 100 < 5\nand\n node_filesystem_readonly{job=\"node-exporter\",fstype!~\"tmpfs|nsfs|vfat\"} == 0\n)\n", + "for": "1h", + "labels": { + "severity": "warning" + } + }, + { + "alert": "NodeFilesystemAlmostOutOfFiles", + "annotations": { + "description": "Filesystem on {{ $labels.device }} at {{ $labels.instance }} has only {{ printf \"%.2f\" $value }}% available inodes left.", + "summary": "Filesystem has less than 3% inodes left." + }, + "expr": "(\n node_filesystem_files_free{job=\"node-exporter\",fstype!~\"tmpfs|nsfs|vfat\"} / node_filesystem_files{job=\"node-exporter\",fstype!~\"tmpfs|nsfs|vfat\"} * 100 < 3\nand\n node_filesystem_readonly{job=\"node-exporter\",fstype!~\"tmpfs|nsfs|vfat\"} == 0\n)\n", + "for": "1h", + "labels": { + "severity": "critical" + } + }, + { + "alert": "NodeNetworkReceiveErrs", + "annotations": { + "description": "{{ $labels.instance }} interface {{ $labels.device }} has encountered {{ printf \"%.0f\" $value }} receive errors in the last two minutes.", + "summary": "Network interface is reporting many receive errors." + }, + "expr": "increase(node_network_receive_errs_total[2m]) > 10\n", + "for": "1h", + "labels": { + "severity": "warning" + } + }, + { + "alert": "NodeNetworkTransmitErrs", + "annotations": { + "description": "{{ $labels.instance }} interface {{ $labels.device }} has encountered {{ printf \"%.0f\" $value }} transmit errors in the last two minutes.", + "summary": "Network interface is reporting many transmit errors." + }, + "expr": "increase(node_network_transmit_errs_total[2m]) > 10\n", + "for": "1h", + "labels": { + "severity": "warning" + } + } + ] + } + ] + } prom.yaml: |- { "groups": [ From 4704b494f0dab879d9e3e6dd2d7132c38d9ba70c Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Mon, 18 Nov 2019 23:00:34 -0800 Subject: [PATCH 273/523] Update mkdocs-material from v4.4.3 to v4.4.0 * Upgrade dependency packages as well --- requirements.txt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/requirements.txt b/requirements.txt index 892e7256f..252fc3c57 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ mkdocs==1.0.4 -mkdocs-material==4.4.3 -pygments==2.2.0 -pymdown-extensions==5.0.0 +mkdocs-material==4.5.0 +pygments==2.4.2 +pymdown-extensions==6.2.0 From 4b485a9bf2f75313a9467f5691493e89312a7535 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Thu, 21 Nov 2019 22:34:09 -0800 Subject: [PATCH 274/523] Fix recent deletion of bootstrap module pinned SHA * Fix deletion of bootstrap module pinned SHA, which was introduced recently through an automation mistake creating https://github.com/poseidon/typhoon/pull/589 --- aws/container-linux/kubernetes/bootstrap.tf | 2 +- aws/fedora-coreos/kubernetes/bootstrap.tf | 2 +- azure/container-linux/kubernetes/bootstrap.tf | 2 +- bare-metal/container-linux/kubernetes/bootstrap.tf | 2 +- bare-metal/fedora-coreos/kubernetes/bootstrap.tf | 2 +- digital-ocean/container-linux/kubernetes/bootstrap.tf | 2 +- google-cloud/container-linux/kubernetes/bootstrap.tf | 2 +- 7 files changed, 7 insertions(+), 7 deletions(-) diff --git a/aws/container-linux/kubernetes/bootstrap.tf b/aws/container-linux/kubernetes/bootstrap.tf index 20663f7e4..0d7266e7b 100644 --- a/aws/container-linux/kubernetes/bootstrap.tf +++ b/aws/container-linux/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=0f1f16c612a6877d25a3fedcb476b3087a3de999" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/aws/fedora-coreos/kubernetes/bootstrap.tf b/aws/fedora-coreos/kubernetes/bootstrap.tf index 59639ac92..f8651eae5 100644 --- a/aws/fedora-coreos/kubernetes/bootstrap.tf +++ b/aws/fedora-coreos/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=0f1f16c612a6877d25a3fedcb476b3087a3de999" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/azure/container-linux/kubernetes/bootstrap.tf b/azure/container-linux/kubernetes/bootstrap.tf index 60da3ca36..b82a5dd0b 100644 --- a/azure/container-linux/kubernetes/bootstrap.tf +++ b/azure/container-linux/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=0f1f16c612a6877d25a3fedcb476b3087a3de999" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/bare-metal/container-linux/kubernetes/bootstrap.tf b/bare-metal/container-linux/kubernetes/bootstrap.tf index d9898814f..f04bc00e5 100644 --- a/bare-metal/container-linux/kubernetes/bootstrap.tf +++ b/bare-metal/container-linux/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=0f1f16c612a6877d25a3fedcb476b3087a3de999" cluster_name = var.cluster_name api_servers = [var.k8s_domain_name] diff --git a/bare-metal/fedora-coreos/kubernetes/bootstrap.tf b/bare-metal/fedora-coreos/kubernetes/bootstrap.tf index 587d155ca..eeb55a8b3 100644 --- a/bare-metal/fedora-coreos/kubernetes/bootstrap.tf +++ b/bare-metal/fedora-coreos/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=0f1f16c612a6877d25a3fedcb476b3087a3de999" cluster_name = var.cluster_name api_servers = [var.k8s_domain_name] diff --git a/digital-ocean/container-linux/kubernetes/bootstrap.tf b/digital-ocean/container-linux/kubernetes/bootstrap.tf index 339dde6ca..9a762a5c1 100644 --- a/digital-ocean/container-linux/kubernetes/bootstrap.tf +++ b/digital-ocean/container-linux/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=0f1f16c612a6877d25a3fedcb476b3087a3de999" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/google-cloud/container-linux/kubernetes/bootstrap.tf b/google-cloud/container-linux/kubernetes/bootstrap.tf index 79ec3db70..157944f53 100644 --- a/google-cloud/container-linux/kubernetes/bootstrap.tf +++ b/google-cloud/container-linux/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=0f1f16c612a6877d25a3fedcb476b3087a3de999" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] From ddea7dc45252080eefabaf26d3bd18c3931abfb1 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Mon, 25 Nov 2019 22:21:24 -0800 Subject: [PATCH 275/523] Use new resource dashboards in Grafana deployment * kubernetes-mixin pod resource dashboards were split into two ConfigMap parts because they provide richer networking details * New dashboards have been used by the author at the global level, but were missing in the per-cluster Grafana tracked here --- CHANGES.md | 1 + addons/grafana/dashboards-k8s-resources.yaml | 6193 ------------------ addons/grafana/deployment.yaml | 13 +- 3 files changed, 10 insertions(+), 6197 deletions(-) delete mode 100644 addons/grafana/dashboards-k8s-resources.yaml diff --git a/CHANGES.md b/CHANGES.md index 5787a1a44..4b7647fd1 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -19,6 +19,7 @@ Notable changes between versions. #### Addons +* Add pod networking details in dashboards ([#593](https://github.com/poseidon/typhoon/pull/593)) * Add node alerts and Grafana dashboard from node-exporter ([#591](https://github.com/poseidon/typhoon/pull/591)) ## v1.16.3 diff --git a/addons/grafana/dashboards-k8s-resources.yaml b/addons/grafana/dashboards-k8s-resources.yaml deleted file mode 100644 index c8c936c2c..000000000 --- a/addons/grafana/dashboards-k8s-resources.yaml +++ /dev/null @@ -1,6193 +0,0 @@ -apiVersion: v1 -data: - k8s-resources-cluster.json: |- - { - "annotations": { - "list": [ - - ] - }, - "editable": true, - "gnetId": null, - "graphTooltip": 0, - "hideControls": false, - "links": [ - - ], - "refresh": "10s", - "rows": [ - { - "collapse": false, - "height": "100px", - "panels": [ - { - "aliasColors": { - - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 1, - "format": "percentunit", - "id": 1, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [ - - ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - - ], - "spaceLength": 10, - "span": 2, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "1 - avg(rate(node_cpu_seconds_total{mode=\"idle\", cluster=\"$cluster\"}[1m]))", - "format": "time_series", - "instant": true, - "intervalFactor": 2, - "refId": "A" - } - ], - "thresholds": "70,80", - "timeFrom": null, - "timeShift": null, - "title": "CPU Utilisation", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "singlestat", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ - - ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] - }, - { - "aliasColors": { - - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 1, - "format": "percentunit", - "id": 2, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [ - - ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - - ], - "spaceLength": 10, - "span": 2, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum(kube_pod_container_resource_requests_cpu_cores{cluster=\"$cluster\"}) / sum(kube_node_status_allocatable_cpu_cores{cluster=\"$cluster\"})", - "format": "time_series", - "instant": true, - "intervalFactor": 2, - "refId": "A" - } - ], - "thresholds": "70,80", - "timeFrom": null, - "timeShift": null, - "title": "CPU Requests Commitment", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "singlestat", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ - - ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] - }, - { - "aliasColors": { - - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 1, - "format": "percentunit", - "id": 3, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [ - - ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - - ], - "spaceLength": 10, - "span": 2, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum(kube_pod_container_resource_limits_cpu_cores{cluster=\"$cluster\"}) / sum(kube_node_status_allocatable_cpu_cores{cluster=\"$cluster\"})", - "format": "time_series", - "instant": true, - "intervalFactor": 2, - "refId": "A" - } - ], - "thresholds": "70,80", - "timeFrom": null, - "timeShift": null, - "title": "CPU Limits Commitment", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "singlestat", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ - - ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] - }, - { - "aliasColors": { - - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 1, - "format": "percentunit", - "id": 4, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [ - - ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - - ], - "spaceLength": 10, - "span": 2, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "1 - sum(:node_memory_MemFreeCachedBuffers_bytes:sum{cluster=\"$cluster\"}) / sum(kube_node_status_allocatable_memory_bytes{cluster=\"$cluster\"})", - "format": "time_series", - "instant": true, - "intervalFactor": 2, - "refId": "A" - } - ], - "thresholds": "70,80", - "timeFrom": null, - "timeShift": null, - "title": "Memory Utilisation", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "singlestat", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ - - ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] - }, - { - "aliasColors": { - - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 1, - "format": "percentunit", - "id": 5, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [ - - ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - - ], - "spaceLength": 10, - "span": 2, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum(kube_pod_container_resource_requests_memory_bytes{cluster=\"$cluster\"}) / sum(kube_node_status_allocatable_memory_bytes{cluster=\"$cluster\"})", - "format": "time_series", - "instant": true, - "intervalFactor": 2, - "refId": "A" - } - ], - "thresholds": "70,80", - "timeFrom": null, - "timeShift": null, - "title": "Memory Requests Commitment", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "singlestat", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ - - ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] - }, - { - "aliasColors": { - - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 1, - "format": "percentunit", - "id": 6, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [ - - ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - - ], - "spaceLength": 10, - "span": 2, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum(kube_pod_container_resource_limits_memory_bytes{cluster=\"$cluster\"}) / sum(kube_node_status_allocatable_memory_bytes{cluster=\"$cluster\"})", - "format": "time_series", - "instant": true, - "intervalFactor": 2, - "refId": "A" - } - ], - "thresholds": "70,80", - "timeFrom": null, - "timeShift": null, - "title": "Memory Limits Commitment", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "singlestat", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ - - ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] - } - ], - "repeat": null, - "repeatIteration": null, - "repeatRowId": null, - "showTitle": false, - "title": "Headlines", - "titleSize": "h6" - }, - { - "collapse": false, - "height": "250px", - "panels": [ - { - "aliasColors": { - - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 10, - "id": 7, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 0, - "links": [ - - ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - - ], - "spaceLength": 10, - "span": 12, - "stack": true, - "steppedLine": false, - "targets": [ - { - "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\"}) by (namespace)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{namespace}}", - "legendLink": null, - "step": 10 - } - ], - "thresholds": [ - - ], - "timeFrom": null, - "timeShift": null, - "title": "CPU Usage", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ - - ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] - } - ], - "repeat": null, - "repeatIteration": null, - "repeatRowId": null, - "showTitle": true, - "title": "CPU", - "titleSize": "h6" - }, - { - "collapse": false, - "height": "250px", - "panels": [ - { - "aliasColors": { - - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 1, - "id": 8, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [ - - ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - - ], - "spaceLength": 10, - "span": 12, - "stack": false, - "steppedLine": false, - "styles": [ - { - "alias": "Time", - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "pattern": "Time", - "type": "hidden" - }, - { - "alias": "Pods", - "colorMode": null, - "colors": [ - - ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 0, - "link": true, - "linkTooltip": "Drill down to pods", - "linkUrl": "./d/85a562078cdf77779eaa1add43ccec1e/k8s-resources-namespace?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$__cell_1", - "pattern": "Value #A", - "thresholds": [ - - ], - "type": "number", - "unit": "short" - }, - { - "alias": "Workloads", - "colorMode": null, - "colors": [ - - ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 0, - "link": true, - "linkTooltip": "Drill down to workloads", - "linkUrl": "./d/a87fb0d919ec0ea5f6543124e16c42a5/k8s-resources-workloads-namespace?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$__cell_1", - "pattern": "Value #B", - "thresholds": [ - - ], - "type": "number", - "unit": "short" - }, - { - "alias": "CPU Usage", - "colorMode": null, - "colors": [ - - ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "link": false, - "linkTooltip": "Drill down", - "linkUrl": "", - "pattern": "Value #C", - "thresholds": [ - - ], - "type": "number", - "unit": "short" - }, - { - "alias": "CPU Requests", - "colorMode": null, - "colors": [ - - ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "link": false, - "linkTooltip": "Drill down", - "linkUrl": "", - "pattern": "Value #D", - "thresholds": [ - - ], - "type": "number", - "unit": "short" - }, - { - "alias": "CPU Requests %", - "colorMode": null, - "colors": [ - - ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "link": false, - "linkTooltip": "Drill down", - "linkUrl": "", - "pattern": "Value #E", - "thresholds": [ - - ], - "type": "number", - "unit": "percentunit" - }, - { - "alias": "CPU Limits", - "colorMode": null, - "colors": [ - - ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "link": false, - "linkTooltip": "Drill down", - "linkUrl": "", - "pattern": "Value #F", - "thresholds": [ - - ], - "type": "number", - "unit": "short" - }, - { - "alias": "CPU Limits %", - "colorMode": null, - "colors": [ - - ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "link": false, - "linkTooltip": "Drill down", - "linkUrl": "", - "pattern": "Value #G", - "thresholds": [ - - ], - "type": "number", - "unit": "percentunit" - }, - { - "alias": "Namespace", - "colorMode": null, - "colors": [ - - ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "link": true, - "linkTooltip": "Drill down to pods", - "linkUrl": "./d/85a562078cdf77779eaa1add43ccec1e/k8s-resources-namespace?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$__cell", - "pattern": "namespace", - "thresholds": [ - - ], - "type": "number", - "unit": "short" - }, - { - "alias": "", - "colorMode": null, - "colors": [ - - ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "pattern": "/.*/", - "thresholds": [ - - ], - "type": "string", - "unit": "short" - } - ], - "targets": [ - { - "expr": "count(mixin_pod_workload{cluster=\"$cluster\"}) by (namespace)", - "format": "table", - "instant": true, - "intervalFactor": 2, - "legendFormat": "", - "refId": "A", - "step": 10 - }, - { - "expr": "count(avg(mixin_pod_workload{cluster=\"$cluster\"}) by (workload, namespace)) by (namespace)", - "format": "table", - "instant": true, - "intervalFactor": 2, - "legendFormat": "", - "refId": "B", - "step": 10 - }, - { - "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\"}) by (namespace)", - "format": "table", - "instant": true, - "intervalFactor": 2, - "legendFormat": "", - "refId": "C", - "step": 10 - }, - { - "expr": "sum(kube_pod_container_resource_requests_cpu_cores{cluster=\"$cluster\"}) by (namespace)", - "format": "table", - "instant": true, - "intervalFactor": 2, - "legendFormat": "", - "refId": "D", - "step": 10 - }, - { - "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\"}) by (namespace) / sum(kube_pod_container_resource_requests_cpu_cores{cluster=\"$cluster\"}) by (namespace)", - "format": "table", - "instant": true, - "intervalFactor": 2, - "legendFormat": "", - "refId": "E", - "step": 10 - }, - { - "expr": "sum(kube_pod_container_resource_limits_cpu_cores{cluster=\"$cluster\"}) by (namespace)", - "format": "table", - "instant": true, - "intervalFactor": 2, - "legendFormat": "", - "refId": "F", - "step": 10 - }, - { - "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\"}) by (namespace) / sum(kube_pod_container_resource_limits_cpu_cores{cluster=\"$cluster\"}) by (namespace)", - "format": "table", - "instant": true, - "intervalFactor": 2, - "legendFormat": "", - "refId": "G", - "step": 10 - } - ], - "thresholds": [ - - ], - "timeFrom": null, - "timeShift": null, - "title": "CPU Quota", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "transform": "table", - "type": "table", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ - - ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] - } - ], - "repeat": null, - "repeatIteration": null, - "repeatRowId": null, - "showTitle": true, - "title": "CPU Quota", - "titleSize": "h6" - }, - { - "collapse": false, - "height": "250px", - "panels": [ - { - "aliasColors": { - - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 10, - "id": 9, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 0, - "links": [ - - ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - - ], - "spaceLength": 10, - "span": 12, - "stack": true, - "steppedLine": false, - "targets": [ - { - "expr": "sum(container_memory_rss{cluster=\"$cluster\", container!=\"\"}) by (namespace)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{namespace}}", - "legendLink": null, - "step": 10 - } - ], - "thresholds": [ - - ], - "timeFrom": null, - "timeShift": null, - "title": "Memory Usage (w/o cache)", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ - - ] - }, - "yaxes": [ - { - "format": "bytes", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] - } - ], - "repeat": null, - "repeatIteration": null, - "repeatRowId": null, - "showTitle": true, - "title": "Memory", - "titleSize": "h6" - }, - { - "collapse": false, - "height": "250px", - "panels": [ - { - "aliasColors": { - - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 1, - "id": 10, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [ - - ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - - ], - "spaceLength": 10, - "span": 12, - "stack": false, - "steppedLine": false, - "styles": [ - { - "alias": "Time", - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "pattern": "Time", - "type": "hidden" - }, - { - "alias": "Pods", - "colorMode": null, - "colors": [ - - ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 0, - "link": true, - "linkTooltip": "Drill down to pods", - "linkUrl": "./d/85a562078cdf77779eaa1add43ccec1e/k8s-resources-namespace?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$__cell_1", - "pattern": "Value #A", - "thresholds": [ - - ], - "type": "number", - "unit": "short" - }, - { - "alias": "Workloads", - "colorMode": null, - "colors": [ - - ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 0, - "link": true, - "linkTooltip": "Drill down to workloads", - "linkUrl": "./d/a87fb0d919ec0ea5f6543124e16c42a5/k8s-resources-workloads-namespace?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$__cell_1", - "pattern": "Value #B", - "thresholds": [ - - ], - "type": "number", - "unit": "short" - }, - { - "alias": "Memory Usage", - "colorMode": null, - "colors": [ - - ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "link": false, - "linkTooltip": "Drill down", - "linkUrl": "", - "pattern": "Value #C", - "thresholds": [ - - ], - "type": "number", - "unit": "bytes" - }, - { - "alias": "Memory Requests", - "colorMode": null, - "colors": [ - - ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "link": false, - "linkTooltip": "Drill down", - "linkUrl": "", - "pattern": "Value #D", - "thresholds": [ - - ], - "type": "number", - "unit": "bytes" - }, - { - "alias": "Memory Requests %", - "colorMode": null, - "colors": [ - - ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "link": false, - "linkTooltip": "Drill down", - "linkUrl": "", - "pattern": "Value #E", - "thresholds": [ - - ], - "type": "number", - "unit": "percentunit" - }, - { - "alias": "Memory Limits", - "colorMode": null, - "colors": [ - - ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "link": false, - "linkTooltip": "Drill down", - "linkUrl": "", - "pattern": "Value #F", - "thresholds": [ - - ], - "type": "number", - "unit": "bytes" - }, - { - "alias": "Memory Limits %", - "colorMode": null, - "colors": [ - - ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "link": false, - "linkTooltip": "Drill down", - "linkUrl": "", - "pattern": "Value #G", - "thresholds": [ - - ], - "type": "number", - "unit": "percentunit" - }, - { - "alias": "Namespace", - "colorMode": null, - "colors": [ - - ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "link": true, - "linkTooltip": "Drill down to pods", - "linkUrl": "./d/85a562078cdf77779eaa1add43ccec1e/k8s-resources-namespace?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$__cell", - "pattern": "namespace", - "thresholds": [ - - ], - "type": "number", - "unit": "short" - }, - { - "alias": "", - "colorMode": null, - "colors": [ - - ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "pattern": "/.*/", - "thresholds": [ - - ], - "type": "string", - "unit": "short" - } - ], - "targets": [ - { - "expr": "count(mixin_pod_workload{cluster=\"$cluster\"}) by (namespace)", - "format": "table", - "instant": true, - "intervalFactor": 2, - "legendFormat": "", - "refId": "A", - "step": 10 - }, - { - "expr": "count(avg(mixin_pod_workload{cluster=\"$cluster\"}) by (workload, namespace)) by (namespace)", - "format": "table", - "instant": true, - "intervalFactor": 2, - "legendFormat": "", - "refId": "B", - "step": 10 - }, - { - "expr": "sum(container_memory_rss{cluster=\"$cluster\", container!=\"\"}) by (namespace)", - "format": "table", - "instant": true, - "intervalFactor": 2, - "legendFormat": "", - "refId": "C", - "step": 10 - }, - { - "expr": "sum(kube_pod_container_resource_requests_memory_bytes{cluster=\"$cluster\"}) by (namespace)", - "format": "table", - "instant": true, - "intervalFactor": 2, - "legendFormat": "", - "refId": "D", - "step": 10 - }, - { - "expr": "sum(container_memory_rss{cluster=\"$cluster\", container!=\"\"}) by (namespace) / sum(kube_pod_container_resource_requests_memory_bytes{cluster=\"$cluster\"}) by (namespace)", - "format": "table", - "instant": true, - "intervalFactor": 2, - "legendFormat": "", - "refId": "E", - "step": 10 - }, - { - "expr": "sum(kube_pod_container_resource_limits_memory_bytes{cluster=\"$cluster\"}) by (namespace)", - "format": "table", - "instant": true, - "intervalFactor": 2, - "legendFormat": "", - "refId": "F", - "step": 10 - }, - { - "expr": "sum(container_memory_rss{cluster=\"$cluster\", container!=\"\"}) by (namespace) / sum(kube_pod_container_resource_limits_memory_bytes{cluster=\"$cluster\"}) by (namespace)", - "format": "table", - "instant": true, - "intervalFactor": 2, - "legendFormat": "", - "refId": "G", - "step": 10 - } - ], - "thresholds": [ - - ], - "timeFrom": null, - "timeShift": null, - "title": "Requests by Namespace", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "transform": "table", - "type": "table", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ - - ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] - } - ], - "repeat": null, - "repeatIteration": null, - "repeatRowId": null, - "showTitle": true, - "title": "Memory Requests", - "titleSize": "h6" - } - ], - "schemaVersion": 14, - "style": "dark", - "tags": [ - "kubernetes-mixin" - ], - "templating": { - "list": [ - { - "current": { - "text": "Prometheus", - "value": "Prometheus" - }, - "hide": 0, - "label": null, - "name": "datasource", - "options": [ - - ], - "query": "prometheus", - "refresh": 1, - "regex": "", - "type": "datasource" - }, - { - "allValue": null, - "current": { - "text": "prod", - "value": "prod" - }, - "datasource": "$datasource", - "hide": 2, - "includeAll": false, - "label": "cluster", - "multi": false, - "name": "cluster", - "options": [ - - ], - "query": "label_values(node_cpu_seconds_total, cluster)", - "refresh": 1, - "regex": "", - "sort": 2, - "tagValuesQuery": "", - "tags": [ - - ], - "tagsQuery": "", - "type": "query", - "useTags": false - } - ] - }, - "time": { - "from": "now-1h", - "to": "now" - }, - "timepicker": { - "refresh_intervals": [ - "5s", - "10s", - "30s", - "1m", - "5m", - "15m", - "30m", - "1h", - "2h", - "1d" - ], - "time_options": [ - "5m", - "15m", - "1h", - "6h", - "12h", - "24h", - "2d", - "7d", - "30d" - ] - }, - "timezone": "", - "title": "Kubernetes / Compute Resources / Cluster", - "uid": "efa86fd1d0c121a26444b636a3f509a8", - "version": 0 - } - k8s-resources-namespace.json: |- - { - "annotations": { - "list": [ - - ] - }, - "editable": true, - "gnetId": null, - "graphTooltip": 0, - "hideControls": false, - "links": [ - - ], - "refresh": "10s", - "rows": [ - { - "collapse": false, - "height": "250px", - "panels": [ - { - "aliasColors": { - - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 10, - "id": 1, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 0, - "links": [ - - ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - - ], - "spaceLength": 10, - "span": 12, - "stack": true, - "steppedLine": false, - "targets": [ - { - "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{pod}}", - "legendLink": null, - "step": 10 - } - ], - "thresholds": [ - - ], - "timeFrom": null, - "timeShift": null, - "title": "CPU Usage", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ - - ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] - } - ], - "repeat": null, - "repeatIteration": null, - "repeatRowId": null, - "showTitle": true, - "title": "CPU Usage", - "titleSize": "h6" - }, - { - "collapse": false, - "height": "250px", - "panels": [ - { - "aliasColors": { - - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 1, - "id": 2, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [ - - ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - - ], - "spaceLength": 10, - "span": 12, - "stack": false, - "steppedLine": false, - "styles": [ - { - "alias": "Time", - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "pattern": "Time", - "type": "hidden" - }, - { - "alias": "CPU Usage", - "colorMode": null, - "colors": [ - - ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "link": false, - "linkTooltip": "Drill down", - "linkUrl": "", - "pattern": "Value #A", - "thresholds": [ - - ], - "type": "number", - "unit": "short" - }, - { - "alias": "CPU Requests", - "colorMode": null, - "colors": [ - - ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "link": false, - "linkTooltip": "Drill down", - "linkUrl": "", - "pattern": "Value #B", - "thresholds": [ - - ], - "type": "number", - "unit": "short" - }, - { - "alias": "CPU Requests %", - "colorMode": null, - "colors": [ - - ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "link": false, - "linkTooltip": "Drill down", - "linkUrl": "", - "pattern": "Value #C", - "thresholds": [ - - ], - "type": "number", - "unit": "percentunit" - }, - { - "alias": "CPU Limits", - "colorMode": null, - "colors": [ - - ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "link": false, - "linkTooltip": "Drill down", - "linkUrl": "", - "pattern": "Value #D", - "thresholds": [ - - ], - "type": "number", - "unit": "short" - }, - { - "alias": "CPU Limits %", - "colorMode": null, - "colors": [ - - ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "link": false, - "linkTooltip": "Drill down", - "linkUrl": "", - "pattern": "Value #E", - "thresholds": [ - - ], - "type": "number", - "unit": "percentunit" - }, - { - "alias": "Pod", - "colorMode": null, - "colors": [ - - ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "link": true, - "linkTooltip": "Drill down", - "linkUrl": "./d/6581e46e4e5c7ba40a07646395ef7b23/k8s-resources-pod?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$namespace&var-pod=$__cell", - "pattern": "pod", - "thresholds": [ - - ], - "type": "number", - "unit": "short" - }, - { - "alias": "", - "colorMode": null, - "colors": [ - - ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "pattern": "/.*/", - "thresholds": [ - - ], - "type": "string", - "unit": "short" - } - ], - "targets": [ - { - "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)", - "format": "table", - "instant": true, - "intervalFactor": 2, - "legendFormat": "", - "refId": "A", - "step": 10 - }, - { - "expr": "sum(kube_pod_container_resource_requests_cpu_cores{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)", - "format": "table", - "instant": true, - "intervalFactor": 2, - "legendFormat": "", - "refId": "B", - "step": 10 - }, - { - "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod) / sum(kube_pod_container_resource_requests_cpu_cores{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)", - "format": "table", - "instant": true, - "intervalFactor": 2, - "legendFormat": "", - "refId": "C", - "step": 10 - }, - { - "expr": "sum(kube_pod_container_resource_limits_cpu_cores{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)", - "format": "table", - "instant": true, - "intervalFactor": 2, - "legendFormat": "", - "refId": "D", - "step": 10 - }, - { - "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod) / sum(kube_pod_container_resource_limits_cpu_cores{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)", - "format": "table", - "instant": true, - "intervalFactor": 2, - "legendFormat": "", - "refId": "E", - "step": 10 - } - ], - "thresholds": [ - - ], - "timeFrom": null, - "timeShift": null, - "title": "CPU Quota", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "transform": "table", - "type": "table", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ - - ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] - } - ], - "repeat": null, - "repeatIteration": null, - "repeatRowId": null, - "showTitle": true, - "title": "CPU Quota", - "titleSize": "h6" - }, - { - "collapse": false, - "height": "250px", - "panels": [ - { - "aliasColors": { - - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 10, - "id": 3, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 0, - "links": [ - - ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - - ], - "spaceLength": 10, - "span": 12, - "stack": true, - "steppedLine": false, - "targets": [ - { - "expr": "sum(container_memory_working_set_bytes{cluster=\"$cluster\", namespace=\"$namespace\", container!=\"\"}) by (pod)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{pod}}", - "legendLink": null, - "step": 10 - } - ], - "thresholds": [ - - ], - "timeFrom": null, - "timeShift": null, - "title": "Memory Usage (w/o cache)", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ - - ] - }, - "yaxes": [ - { - "format": "bytes", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] - } - ], - "repeat": null, - "repeatIteration": null, - "repeatRowId": null, - "showTitle": true, - "title": "Memory Usage", - "titleSize": "h6" - }, - { - "collapse": false, - "height": "250px", - "panels": [ - { - "aliasColors": { - - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 1, - "id": 4, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [ - - ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - - ], - "spaceLength": 10, - "span": 12, - "stack": false, - "steppedLine": false, - "styles": [ - { - "alias": "Time", - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "pattern": "Time", - "type": "hidden" - }, - { - "alias": "Memory Usage", - "colorMode": null, - "colors": [ - - ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "link": false, - "linkTooltip": "Drill down", - "linkUrl": "", - "pattern": "Value #A", - "thresholds": [ - - ], - "type": "number", - "unit": "bytes" - }, - { - "alias": "Memory Requests", - "colorMode": null, - "colors": [ - - ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "link": false, - "linkTooltip": "Drill down", - "linkUrl": "", - "pattern": "Value #B", - "thresholds": [ - - ], - "type": "number", - "unit": "bytes" - }, - { - "alias": "Memory Requests %", - "colorMode": null, - "colors": [ - - ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "link": false, - "linkTooltip": "Drill down", - "linkUrl": "", - "pattern": "Value #C", - "thresholds": [ - - ], - "type": "number", - "unit": "percentunit" - }, - { - "alias": "Memory Limits", - "colorMode": null, - "colors": [ - - ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "link": false, - "linkTooltip": "Drill down", - "linkUrl": "", - "pattern": "Value #D", - "thresholds": [ - - ], - "type": "number", - "unit": "bytes" - }, - { - "alias": "Memory Limits %", - "colorMode": null, - "colors": [ - - ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "link": false, - "linkTooltip": "Drill down", - "linkUrl": "", - "pattern": "Value #E", - "thresholds": [ - - ], - "type": "number", - "unit": "percentunit" - }, - { - "alias": "Memory Usage (RSS)", - "colorMode": null, - "colors": [ - - ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "link": false, - "linkTooltip": "Drill down", - "linkUrl": "", - "pattern": "Value #F", - "thresholds": [ - - ], - "type": "number", - "unit": "bytes" - }, - { - "alias": "Memory Usage (Cache)", - "colorMode": null, - "colors": [ - - ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "link": false, - "linkTooltip": "Drill down", - "linkUrl": "", - "pattern": "Value #G", - "thresholds": [ - - ], - "type": "number", - "unit": "bytes" - }, - { - "alias": "Memory Usage (Swap)", - "colorMode": null, - "colors": [ - - ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "link": false, - "linkTooltip": "Drill down", - "linkUrl": "", - "pattern": "Value #H", - "thresholds": [ - - ], - "type": "number", - "unit": "bytes" - }, - { - "alias": "Pod", - "colorMode": null, - "colors": [ - - ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "link": true, - "linkTooltip": "Drill down", - "linkUrl": "./d/6581e46e4e5c7ba40a07646395ef7b23/k8s-resources-pod?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$namespace&var-pod=$__cell", - "pattern": "pod", - "thresholds": [ - - ], - "type": "number", - "unit": "short" - }, - { - "alias": "", - "colorMode": null, - "colors": [ - - ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "pattern": "/.*/", - "thresholds": [ - - ], - "type": "string", - "unit": "short" - } - ], - "targets": [ - { - "expr": "sum(container_memory_working_set_bytes{cluster=\"$cluster\", namespace=\"$namespace\",container!=\"\"}) by (pod)", - "format": "table", - "instant": true, - "intervalFactor": 2, - "legendFormat": "", - "refId": "A", - "step": 10 - }, - { - "expr": "sum(kube_pod_container_resource_requests_memory_bytes{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)", - "format": "table", - "instant": true, - "intervalFactor": 2, - "legendFormat": "", - "refId": "B", - "step": 10 - }, - { - "expr": "sum(container_memory_working_set_bytes{cluster=\"$cluster\", namespace=\"$namespace\",container!=\"\"}) by (pod) / sum(kube_pod_container_resource_requests_memory_bytes{namespace=\"$namespace\"}) by (pod)", - "format": "table", - "instant": true, - "intervalFactor": 2, - "legendFormat": "", - "refId": "C", - "step": 10 - }, - { - "expr": "sum(kube_pod_container_resource_limits_memory_bytes{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)", - "format": "table", - "instant": true, - "intervalFactor": 2, - "legendFormat": "", - "refId": "D", - "step": 10 - }, - { - "expr": "sum(container_memory_working_set_bytes{cluster=\"$cluster\", namespace=\"$namespace\",container!=\"\"}) by (pod) / sum(kube_pod_container_resource_limits_memory_bytes{namespace=\"$namespace\"}) by (pod)", - "format": "table", - "instant": true, - "intervalFactor": 2, - "legendFormat": "", - "refId": "E", - "step": 10 - }, - { - "expr": "sum(container_memory_rss{cluster=\"$cluster\", namespace=\"$namespace\",container!=\"\"}) by (pod)", - "format": "table", - "instant": true, - "intervalFactor": 2, - "legendFormat": "", - "refId": "F", - "step": 10 - }, - { - "expr": "sum(container_memory_cache{cluster=\"$cluster\", namespace=\"$namespace\",container!=\"\"}) by (pod)", - "format": "table", - "instant": true, - "intervalFactor": 2, - "legendFormat": "", - "refId": "G", - "step": 10 - }, - { - "expr": "sum(container_memory_swap{cluster=\"$cluster\", namespace=\"$namespace\",container!=\"\"}) by (pod)", - "format": "table", - "instant": true, - "intervalFactor": 2, - "legendFormat": "", - "refId": "H", - "step": 10 - } - ], - "thresholds": [ - - ], - "timeFrom": null, - "timeShift": null, - "title": "Memory Quota", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "transform": "table", - "type": "table", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ - - ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] - } - ], - "repeat": null, - "repeatIteration": null, - "repeatRowId": null, - "showTitle": true, - "title": "Memory Quota", - "titleSize": "h6" - } - ], - "schemaVersion": 14, - "style": "dark", - "tags": [ - "kubernetes-mixin" - ], - "templating": { - "list": [ - { - "current": { - "text": "Prometheus", - "value": "Prometheus" - }, - "hide": 0, - "label": null, - "name": "datasource", - "options": [ - - ], - "query": "prometheus", - "refresh": 1, - "regex": "", - "type": "datasource" - }, - { - "allValue": null, - "current": { - "text": "prod", - "value": "prod" - }, - "datasource": "$datasource", - "hide": 2, - "includeAll": false, - "label": "cluster", - "multi": false, - "name": "cluster", - "options": [ - - ], - "query": "label_values(kube_pod_info, cluster)", - "refresh": 1, - "regex": "", - "sort": 2, - "tagValuesQuery": "", - "tags": [ - - ], - "tagsQuery": "", - "type": "query", - "useTags": false - }, - { - "allValue": null, - "current": { - "text": "prod", - "value": "prod" - }, - "datasource": "$datasource", - "hide": 0, - "includeAll": false, - "label": "namespace", - "multi": false, - "name": "namespace", - "options": [ - - ], - "query": "label_values(kube_pod_info{cluster=\"$cluster\"}, namespace)", - "refresh": 1, - "regex": "", - "sort": 2, - "tagValuesQuery": "", - "tags": [ - - ], - "tagsQuery": "", - "type": "query", - "useTags": false - } - ] - }, - "time": { - "from": "now-1h", - "to": "now" - }, - "timepicker": { - "refresh_intervals": [ - "5s", - "10s", - "30s", - "1m", - "5m", - "15m", - "30m", - "1h", - "2h", - "1d" - ], - "time_options": [ - "5m", - "15m", - "1h", - "6h", - "12h", - "24h", - "2d", - "7d", - "30d" - ] - }, - "timezone": "", - "title": "Kubernetes / Compute Resources / Namespace (Pods)", - "uid": "85a562078cdf77779eaa1add43ccec1e", - "version": 0 - } - k8s-resources-node.json: |- - { - "annotations": { - "list": [ - - ] - }, - "editable": true, - "gnetId": null, - "graphTooltip": 0, - "hideControls": false, - "links": [ - - ], - "refresh": "10s", - "rows": [ - { - "collapse": false, - "height": "250px", - "panels": [ - { - "aliasColors": { - - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 10, - "id": 1, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 0, - "links": [ - - ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - - ], - "spaceLength": 10, - "span": 12, - "stack": true, - "steppedLine": false, - "targets": [ - { - "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", node=\"$node\"}) by (pod)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{pod}}", - "legendLink": null, - "step": 10 - } - ], - "thresholds": [ - - ], - "timeFrom": null, - "timeShift": null, - "title": "CPU Usage", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ - - ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] - } - ], - "repeat": null, - "repeatIteration": null, - "repeatRowId": null, - "showTitle": true, - "title": "CPU Usage", - "titleSize": "h6" - }, - { - "collapse": false, - "height": "250px", - "panels": [ - { - "aliasColors": { - - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 1, - "id": 2, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [ - - ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - - ], - "spaceLength": 10, - "span": 12, - "stack": false, - "steppedLine": false, - "styles": [ - { - "alias": "Time", - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "pattern": "Time", - "type": "hidden" - }, - { - "alias": "CPU Usage", - "colorMode": null, - "colors": [ - - ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "link": false, - "linkTooltip": "Drill down", - "linkUrl": "", - "pattern": "Value #A", - "thresholds": [ - - ], - "type": "number", - "unit": "short" - }, - { - "alias": "CPU Requests", - "colorMode": null, - "colors": [ - - ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "link": false, - "linkTooltip": "Drill down", - "linkUrl": "", - "pattern": "Value #B", - "thresholds": [ - - ], - "type": "number", - "unit": "short" - }, - { - "alias": "CPU Requests %", - "colorMode": null, - "colors": [ - - ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "link": false, - "linkTooltip": "Drill down", - "linkUrl": "", - "pattern": "Value #C", - "thresholds": [ - - ], - "type": "number", - "unit": "percentunit" - }, - { - "alias": "CPU Limits", - "colorMode": null, - "colors": [ - - ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "link": false, - "linkTooltip": "Drill down", - "linkUrl": "", - "pattern": "Value #D", - "thresholds": [ - - ], - "type": "number", - "unit": "short" - }, - { - "alias": "CPU Limits %", - "colorMode": null, - "colors": [ - - ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "link": false, - "linkTooltip": "Drill down", - "linkUrl": "", - "pattern": "Value #E", - "thresholds": [ - - ], - "type": "number", - "unit": "percentunit" - }, - { - "alias": "Pod", - "colorMode": null, - "colors": [ - - ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "link": false, - "linkTooltip": "Drill down", - "linkUrl": "", - "pattern": "pod", - "thresholds": [ - - ], - "type": "number", - "unit": "short" - }, - { - "alias": "", - "colorMode": null, - "colors": [ - - ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "pattern": "/.*/", - "thresholds": [ - - ], - "type": "string", - "unit": "short" - } - ], - "targets": [ - { - "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", node=\"$node\"}) by (pod)", - "format": "table", - "instant": true, - "intervalFactor": 2, - "legendFormat": "", - "refId": "A", - "step": 10 - }, - { - "expr": "sum(kube_pod_container_resource_requests_cpu_cores{cluster=\"$cluster\", node=\"$node\"}) by (pod)", - "format": "table", - "instant": true, - "intervalFactor": 2, - "legendFormat": "", - "refId": "B", - "step": 10 - }, - { - "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", node=\"$node\"}) by (pod) / sum(kube_pod_container_resource_requests_cpu_cores{cluster=\"$cluster\", node=\"$node\"}) by (pod)", - "format": "table", - "instant": true, - "intervalFactor": 2, - "legendFormat": "", - "refId": "C", - "step": 10 - }, - { - "expr": "sum(kube_pod_container_resource_limits_cpu_cores{cluster=\"$cluster\", node=\"$node\"}) by (pod)", - "format": "table", - "instant": true, - "intervalFactor": 2, - "legendFormat": "", - "refId": "D", - "step": 10 - }, - { - "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", node=\"$node\"}) by (pod) / sum(kube_pod_container_resource_limits_cpu_cores{cluster=\"$cluster\", node=\"$node\"}) by (pod)", - "format": "table", - "instant": true, - "intervalFactor": 2, - "legendFormat": "", - "refId": "E", - "step": 10 - } - ], - "thresholds": [ - - ], - "timeFrom": null, - "timeShift": null, - "title": "CPU Quota", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "transform": "table", - "type": "table", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ - - ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] - } - ], - "repeat": null, - "repeatIteration": null, - "repeatRowId": null, - "showTitle": true, - "title": "CPU Quota", - "titleSize": "h6" - }, - { - "collapse": false, - "height": "250px", - "panels": [ - { - "aliasColors": { - - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 10, - "id": 3, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 0, - "links": [ - - ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - - ], - "spaceLength": 10, - "span": 12, - "stack": true, - "steppedLine": false, - "targets": [ - { - "expr": "sum(container_memory_working_set_bytes{cluster=\"$cluster\", node=\"$node\", container!=\"\"}) by (pod)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{pod}}", - "legendLink": null, - "step": 10 - } - ], - "thresholds": [ - - ], - "timeFrom": null, - "timeShift": null, - "title": "Memory Usage (w/o cache)", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ - - ] - }, - "yaxes": [ - { - "format": "bytes", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] - } - ], - "repeat": null, - "repeatIteration": null, - "repeatRowId": null, - "showTitle": true, - "title": "Memory Usage", - "titleSize": "h6" - }, - { - "collapse": false, - "height": "250px", - "panels": [ - { - "aliasColors": { - - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 1, - "id": 4, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [ - - ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - - ], - "spaceLength": 10, - "span": 12, - "stack": false, - "steppedLine": false, - "styles": [ - { - "alias": "Time", - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "pattern": "Time", - "type": "hidden" - }, - { - "alias": "Memory Usage", - "colorMode": null, - "colors": [ - - ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "link": false, - "linkTooltip": "Drill down", - "linkUrl": "", - "pattern": "Value #A", - "thresholds": [ - - ], - "type": "number", - "unit": "bytes" - }, - { - "alias": "Memory Requests", - "colorMode": null, - "colors": [ - - ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "link": false, - "linkTooltip": "Drill down", - "linkUrl": "", - "pattern": "Value #B", - "thresholds": [ - - ], - "type": "number", - "unit": "bytes" - }, - { - "alias": "Memory Requests %", - "colorMode": null, - "colors": [ - - ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "link": false, - "linkTooltip": "Drill down", - "linkUrl": "", - "pattern": "Value #C", - "thresholds": [ - - ], - "type": "number", - "unit": "percentunit" - }, - { - "alias": "Memory Limits", - "colorMode": null, - "colors": [ - - ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "link": false, - "linkTooltip": "Drill down", - "linkUrl": "", - "pattern": "Value #D", - "thresholds": [ - - ], - "type": "number", - "unit": "bytes" - }, - { - "alias": "Memory Limits %", - "colorMode": null, - "colors": [ - - ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "link": false, - "linkTooltip": "Drill down", - "linkUrl": "", - "pattern": "Value #E", - "thresholds": [ - - ], - "type": "number", - "unit": "percentunit" - }, - { - "alias": "Memory Usage (RSS)", - "colorMode": null, - "colors": [ - - ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "link": false, - "linkTooltip": "Drill down", - "linkUrl": "", - "pattern": "Value #F", - "thresholds": [ - - ], - "type": "number", - "unit": "bytes" - }, - { - "alias": "Memory Usage (Cache)", - "colorMode": null, - "colors": [ - - ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "link": false, - "linkTooltip": "Drill down", - "linkUrl": "", - "pattern": "Value #G", - "thresholds": [ - - ], - "type": "number", - "unit": "bytes" - }, - { - "alias": "Memory Usage (Swap)", - "colorMode": null, - "colors": [ - - ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "link": false, - "linkTooltip": "Drill down", - "linkUrl": "", - "pattern": "Value #H", - "thresholds": [ - - ], - "type": "number", - "unit": "bytes" - }, - { - "alias": "Pod", - "colorMode": null, - "colors": [ - - ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "link": false, - "linkTooltip": "Drill down", - "linkUrl": "", - "pattern": "pod", - "thresholds": [ - - ], - "type": "number", - "unit": "short" - }, - { - "alias": "", - "colorMode": null, - "colors": [ - - ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "pattern": "/.*/", - "thresholds": [ - - ], - "type": "string", - "unit": "short" - } - ], - "targets": [ - { - "expr": "sum(container_memory_working_set_bytes{cluster=\"$cluster\", node=\"$node\",container!=\"\"}) by (pod)", - "format": "table", - "instant": true, - "intervalFactor": 2, - "legendFormat": "", - "refId": "A", - "step": 10 - }, - { - "expr": "sum(kube_pod_container_resource_requests_memory_bytes{cluster=\"$cluster\", node=\"$node\"}) by (pod)", - "format": "table", - "instant": true, - "intervalFactor": 2, - "legendFormat": "", - "refId": "B", - "step": 10 - }, - { - "expr": "sum(container_memory_working_set_bytes{cluster=\"$cluster\", node=\"$node\",container!=\"\"}) by (pod) / sum(kube_pod_container_resource_requests_memory_bytes{node=\"$node\"}) by (pod)", - "format": "table", - "instant": true, - "intervalFactor": 2, - "legendFormat": "", - "refId": "C", - "step": 10 - }, - { - "expr": "sum(kube_pod_container_resource_limits_memory_bytes{cluster=\"$cluster\", node=\"$node\"}) by (pod)", - "format": "table", - "instant": true, - "intervalFactor": 2, - "legendFormat": "", - "refId": "D", - "step": 10 - }, - { - "expr": "sum(container_memory_working_set_bytes{cluster=\"$cluster\", node=\"$node\",container!=\"\"}) by (pod) / sum(kube_pod_container_resource_limits_memory_bytes{node=\"$node\"}) by (pod)", - "format": "table", - "instant": true, - "intervalFactor": 2, - "legendFormat": "", - "refId": "E", - "step": 10 - }, - { - "expr": "sum(container_memory_rss{cluster=\"$cluster\", node=\"$node\",container!=\"\"}) by (pod)", - "format": "table", - "instant": true, - "intervalFactor": 2, - "legendFormat": "", - "refId": "F", - "step": 10 - }, - { - "expr": "sum(container_memory_cache{cluster=\"$cluster\", node=\"$node\",container!=\"\"}) by (pod)", - "format": "table", - "instant": true, - "intervalFactor": 2, - "legendFormat": "", - "refId": "G", - "step": 10 - }, - { - "expr": "sum(container_memory_swap{cluster=\"$cluster\", node=\"$node\",container!=\"\"}) by (pod)", - "format": "table", - "instant": true, - "intervalFactor": 2, - "legendFormat": "", - "refId": "H", - "step": 10 - } - ], - "thresholds": [ - - ], - "timeFrom": null, - "timeShift": null, - "title": "Memory Quota", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "transform": "table", - "type": "table", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ - - ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] - } - ], - "repeat": null, - "repeatIteration": null, - "repeatRowId": null, - "showTitle": true, - "title": "Memory Quota", - "titleSize": "h6" - } - ], - "schemaVersion": 14, - "style": "dark", - "tags": [ - "kubernetes-mixin" - ], - "templating": { - "list": [ - { - "current": { - "text": "Prometheus", - "value": "Prometheus" - }, - "hide": 0, - "label": null, - "name": "datasource", - "options": [ - - ], - "query": "prometheus", - "refresh": 1, - "regex": "", - "type": "datasource" - }, - { - "allValue": null, - "current": { - "text": "prod", - "value": "prod" - }, - "datasource": "$datasource", - "hide": 2, - "includeAll": false, - "label": "cluster", - "multi": false, - "name": "cluster", - "options": [ - - ], - "query": "label_values(kube_pod_info, cluster)", - "refresh": 1, - "regex": "", - "sort": 2, - "tagValuesQuery": "", - "tags": [ - - ], - "tagsQuery": "", - "type": "query", - "useTags": false - }, - { - "allValue": null, - "current": { - "text": "prod", - "value": "prod" - }, - "datasource": "$datasource", - "hide": 0, - "includeAll": false, - "label": "node", - "multi": false, - "name": "node", - "options": [ - - ], - "query": "label_values(kube_pod_info{cluster=\"$cluster\"}, node)", - "refresh": 1, - "regex": "", - "sort": 2, - "tagValuesQuery": "", - "tags": [ - - ], - "tagsQuery": "", - "type": "query", - "useTags": false - } - ] - }, - "time": { - "from": "now-1h", - "to": "now" - }, - "timepicker": { - "refresh_intervals": [ - "5s", - "10s", - "30s", - "1m", - "5m", - "15m", - "30m", - "1h", - "2h", - "1d" - ], - "time_options": [ - "5m", - "15m", - "1h", - "6h", - "12h", - "24h", - "2d", - "7d", - "30d" - ] - }, - "timezone": "", - "title": "Kubernetes / Compute Resources / Node (Pods)", - "uid": "200ac8fdbfbb74b39aff88118e4d1c2c", - "version": 0 - } - k8s-resources-pod.json: |- - { - "annotations": { - "list": [ - - ] - }, - "editable": true, - "gnetId": null, - "graphTooltip": 0, - "hideControls": false, - "links": [ - - ], - "refresh": "10s", - "rows": [ - { - "collapse": false, - "height": "250px", - "panels": [ - { - "aliasColors": { - - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 10, - "id": 1, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 0, - "links": [ - - ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - - ], - "spaceLength": 10, - "span": 12, - "stack": true, - "steppedLine": false, - "targets": [ - { - "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{namespace=\"$namespace\", pod=\"$pod\", container!=\"POD\", cluster=\"$cluster\"}) by (container)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{container}}", - "legendLink": null, - "step": 10 - } - ], - "thresholds": [ - - ], - "timeFrom": null, - "timeShift": null, - "title": "CPU Usage", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ - - ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] - } - ], - "repeat": null, - "repeatIteration": null, - "repeatRowId": null, - "showTitle": true, - "title": "CPU Usage", - "titleSize": "h6" - }, - { - "collapse": false, - "height": "250px", - "panels": [ - { - "aliasColors": { - - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 1, - "id": 2, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [ - - ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - - ], - "spaceLength": 10, - "span": 12, - "stack": false, - "steppedLine": false, - "styles": [ - { - "alias": "Time", - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "pattern": "Time", - "type": "hidden" - }, - { - "alias": "CPU Usage", - "colorMode": null, - "colors": [ - - ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "link": false, - "linkTooltip": "Drill down", - "linkUrl": "", - "pattern": "Value #A", - "thresholds": [ - - ], - "type": "number", - "unit": "short" - }, - { - "alias": "CPU Requests", - "colorMode": null, - "colors": [ - - ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "link": false, - "linkTooltip": "Drill down", - "linkUrl": "", - "pattern": "Value #B", - "thresholds": [ - - ], - "type": "number", - "unit": "short" - }, - { - "alias": "CPU Requests %", - "colorMode": null, - "colors": [ - - ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "link": false, - "linkTooltip": "Drill down", - "linkUrl": "", - "pattern": "Value #C", - "thresholds": [ - - ], - "type": "number", - "unit": "percentunit" - }, - { - "alias": "CPU Limits", - "colorMode": null, - "colors": [ - - ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "link": false, - "linkTooltip": "Drill down", - "linkUrl": "", - "pattern": "Value #D", - "thresholds": [ - - ], - "type": "number", - "unit": "short" - }, - { - "alias": "CPU Limits %", - "colorMode": null, - "colors": [ - - ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "link": false, - "linkTooltip": "Drill down", - "linkUrl": "", - "pattern": "Value #E", - "thresholds": [ - - ], - "type": "number", - "unit": "percentunit" - }, - { - "alias": "Container", - "colorMode": null, - "colors": [ - - ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "link": false, - "linkTooltip": "Drill down", - "linkUrl": "", - "pattern": "container", - "thresholds": [ - - ], - "type": "number", - "unit": "short" - }, - { - "alias": "", - "colorMode": null, - "colors": [ - - ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "pattern": "/.*/", - "thresholds": [ - - ], - "type": "string", - "unit": "short" - } - ], - "targets": [ - { - "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\", container!=\"POD\"}) by (container)", - "format": "table", - "instant": true, - "intervalFactor": 2, - "legendFormat": "", - "refId": "A", - "step": 10 - }, - { - "expr": "sum(kube_pod_container_resource_requests_cpu_cores{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container)", - "format": "table", - "instant": true, - "intervalFactor": 2, - "legendFormat": "", - "refId": "B", - "step": 10 - }, - { - "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container) / sum(kube_pod_container_resource_requests_cpu_cores{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container)", - "format": "table", - "instant": true, - "intervalFactor": 2, - "legendFormat": "", - "refId": "C", - "step": 10 - }, - { - "expr": "sum(kube_pod_container_resource_limits_cpu_cores{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container)", - "format": "table", - "instant": true, - "intervalFactor": 2, - "legendFormat": "", - "refId": "D", - "step": 10 - }, - { - "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container) / sum(kube_pod_container_resource_limits_cpu_cores{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container)", - "format": "table", - "instant": true, - "intervalFactor": 2, - "legendFormat": "", - "refId": "E", - "step": 10 - } - ], - "thresholds": [ - - ], - "timeFrom": null, - "timeShift": null, - "title": "CPU Quota", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "transform": "table", - "type": "table", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ - - ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] - } - ], - "repeat": null, - "repeatIteration": null, - "repeatRowId": null, - "showTitle": true, - "title": "CPU Quota", - "titleSize": "h6" - }, - { - "collapse": false, - "height": "250px", - "panels": [ - { - "aliasColors": { - - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 10, - "id": 3, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 0, - "links": [ - - ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - - ], - "spaceLength": 10, - "span": 12, - "stack": true, - "steppedLine": false, - "targets": [ - { - "expr": "sum(container_memory_rss{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\", container!=\"POD\", container!=\"\"}) by (container)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{container}} (RSS)", - "legendLink": null, - "step": 10 - }, - { - "expr": "sum(container_memory_cache{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\", container!=\"POD\", container!=\"\"}) by (container)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{container}} (Cache)", - "legendLink": null, - "step": 10 - }, - { - "expr": "sum(container_memory_swap{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\", container!=\"POD\", container!=\"\"}) by (container)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{container}} (Swap)", - "legendLink": null, - "step": 10 - } - ], - "thresholds": [ - - ], - "timeFrom": null, - "timeShift": null, - "title": "Memory Usage", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ - - ] - }, - "yaxes": [ - { - "format": "bytes", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] - } - ], - "repeat": null, - "repeatIteration": null, - "repeatRowId": null, - "showTitle": true, - "title": "Memory Usage", - "titleSize": "h6" - }, - { - "collapse": false, - "height": "250px", - "panels": [ - { - "aliasColors": { - - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 1, - "id": 4, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [ - - ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - - ], - "spaceLength": 10, - "span": 12, - "stack": false, - "steppedLine": false, - "styles": [ - { - "alias": "Time", - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "pattern": "Time", - "type": "hidden" - }, - { - "alias": "Memory Usage", - "colorMode": null, - "colors": [ - - ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "link": false, - "linkTooltip": "Drill down", - "linkUrl": "", - "pattern": "Value #A", - "thresholds": [ - - ], - "type": "number", - "unit": "bytes" - }, - { - "alias": "Memory Requests", - "colorMode": null, - "colors": [ - - ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "link": false, - "linkTooltip": "Drill down", - "linkUrl": "", - "pattern": "Value #B", - "thresholds": [ - - ], - "type": "number", - "unit": "bytes" - }, - { - "alias": "Memory Requests %", - "colorMode": null, - "colors": [ - - ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "link": false, - "linkTooltip": "Drill down", - "linkUrl": "", - "pattern": "Value #C", - "thresholds": [ - - ], - "type": "number", - "unit": "percentunit" - }, - { - "alias": "Memory Limits", - "colorMode": null, - "colors": [ - - ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "link": false, - "linkTooltip": "Drill down", - "linkUrl": "", - "pattern": "Value #D", - "thresholds": [ - - ], - "type": "number", - "unit": "bytes" - }, - { - "alias": "Memory Limits %", - "colorMode": null, - "colors": [ - - ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "link": false, - "linkTooltip": "Drill down", - "linkUrl": "", - "pattern": "Value #E", - "thresholds": [ - - ], - "type": "number", - "unit": "percentunit" - }, - { - "alias": "Memory Usage (RSS)", - "colorMode": null, - "colors": [ - - ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "link": false, - "linkTooltip": "Drill down", - "linkUrl": "", - "pattern": "Value #F", - "thresholds": [ - - ], - "type": "number", - "unit": "bytes" - }, - { - "alias": "Memory Usage (Cache)", - "colorMode": null, - "colors": [ - - ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "link": false, - "linkTooltip": "Drill down", - "linkUrl": "", - "pattern": "Value #G", - "thresholds": [ - - ], - "type": "number", - "unit": "bytes" - }, - { - "alias": "Memory Usage (Swap)", - "colorMode": null, - "colors": [ - - ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "link": false, - "linkTooltip": "Drill down", - "linkUrl": "", - "pattern": "Value #H", - "thresholds": [ - - ], - "type": "number", - "unit": "bytes" - }, - { - "alias": "Container", - "colorMode": null, - "colors": [ - - ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "link": false, - "linkTooltip": "Drill down", - "linkUrl": "", - "pattern": "container", - "thresholds": [ - - ], - "type": "number", - "unit": "short" - }, - { - "alias": "", - "colorMode": null, - "colors": [ - - ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "pattern": "/.*/", - "thresholds": [ - - ], - "type": "string", - "unit": "short" - } - ], - "targets": [ - { - "expr": "sum(container_memory_working_set_bytes{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\", container!=\"POD\", container!=\"\"}) by (container)", - "format": "table", - "instant": true, - "intervalFactor": 2, - "legendFormat": "", - "refId": "A", - "step": 10 - }, - { - "expr": "sum(kube_pod_container_resource_requests_memory_bytes{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container)", - "format": "table", - "instant": true, - "intervalFactor": 2, - "legendFormat": "", - "refId": "B", - "step": 10 - }, - { - "expr": "sum(container_memory_working_set_bytes{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container) / sum(kube_pod_container_resource_requests_memory_bytes{namespace=\"$namespace\", pod=\"$pod\"}) by (container)", - "format": "table", - "instant": true, - "intervalFactor": 2, - "legendFormat": "", - "refId": "C", - "step": 10 - }, - { - "expr": "sum(kube_pod_container_resource_limits_memory_bytes{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\", container!=\"\"}) by (container)", - "format": "table", - "instant": true, - "intervalFactor": 2, - "legendFormat": "", - "refId": "D", - "step": 10 - }, - { - "expr": "sum(container_memory_working_set_bytes{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\", container!=\"\"}) by (container) / sum(kube_pod_container_resource_limits_memory_bytes{namespace=\"$namespace\", pod=\"$pod\"}) by (container)", - "format": "table", - "instant": true, - "intervalFactor": 2, - "legendFormat": "", - "refId": "E", - "step": 10 - }, - { - "expr": "sum(container_memory_rss{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\", container != \"\", container != \"POD\"}) by (container)", - "format": "table", - "instant": true, - "intervalFactor": 2, - "legendFormat": "", - "refId": "F", - "step": 10 - }, - { - "expr": "sum(container_memory_cache{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\", container != \"\", container != \"POD\"}) by (container)", - "format": "table", - "instant": true, - "intervalFactor": 2, - "legendFormat": "", - "refId": "G", - "step": 10 - }, - { - "expr": "sum(container_memory_swap{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\", container != \"\", container != \"POD\"}) by (container)", - "format": "table", - "instant": true, - "intervalFactor": 2, - "legendFormat": "", - "refId": "H", - "step": 10 - } - ], - "thresholds": [ - - ], - "timeFrom": null, - "timeShift": null, - "title": "Memory Quota", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "transform": "table", - "type": "table", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ - - ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] - } - ], - "repeat": null, - "repeatIteration": null, - "repeatRowId": null, - "showTitle": true, - "title": "Memory Quota", - "titleSize": "h6" - } - ], - "schemaVersion": 14, - "style": "dark", - "tags": [ - "kubernetes-mixin" - ], - "templating": { - "list": [ - { - "current": { - "text": "Prometheus", - "value": "Prometheus" - }, - "hide": 0, - "label": null, - "name": "datasource", - "options": [ - - ], - "query": "prometheus", - "refresh": 1, - "regex": "", - "type": "datasource" - }, - { - "allValue": null, - "current": { - "text": "prod", - "value": "prod" - }, - "datasource": "$datasource", - "hide": 2, - "includeAll": false, - "label": "cluster", - "multi": false, - "name": "cluster", - "options": [ - - ], - "query": "label_values(kube_pod_info, cluster)", - "refresh": 1, - "regex": "", - "sort": 2, - "tagValuesQuery": "", - "tags": [ - - ], - "tagsQuery": "", - "type": "query", - "useTags": false - }, - { - "allValue": null, - "current": { - "text": "prod", - "value": "prod" - }, - "datasource": "$datasource", - "hide": 0, - "includeAll": false, - "label": "namespace", - "multi": false, - "name": "namespace", - "options": [ - - ], - "query": "label_values(kube_pod_info{cluster=\"$cluster\"}, namespace)", - "refresh": 1, - "regex": "", - "sort": 2, - "tagValuesQuery": "", - "tags": [ - - ], - "tagsQuery": "", - "type": "query", - "useTags": false - }, - { - "allValue": null, - "current": { - "text": "prod", - "value": "prod" - }, - "datasource": "$datasource", - "hide": 0, - "includeAll": false, - "label": "pod", - "multi": false, - "name": "pod", - "options": [ - - ], - "query": "label_values(kube_pod_info{cluster=\"$cluster\", namespace=\"$namespace\"}, pod)", - "refresh": 1, - "regex": "", - "sort": 2, - "tagValuesQuery": "", - "tags": [ - - ], - "tagsQuery": "", - "type": "query", - "useTags": false - } - ] - }, - "time": { - "from": "now-1h", - "to": "now" - }, - "timepicker": { - "refresh_intervals": [ - "5s", - "10s", - "30s", - "1m", - "5m", - "15m", - "30m", - "1h", - "2h", - "1d" - ], - "time_options": [ - "5m", - "15m", - "1h", - "6h", - "12h", - "24h", - "2d", - "7d", - "30d" - ] - }, - "timezone": "", - "title": "Kubernetes / Compute Resources / Pod", - "uid": "6581e46e4e5c7ba40a07646395ef7b23", - "version": 0 - } - k8s-resources-workload.json: |- - { - "annotations": { - "list": [ - - ] - }, - "editable": true, - "gnetId": null, - "graphTooltip": 0, - "hideControls": false, - "links": [ - - ], - "refresh": "10s", - "rows": [ - { - "collapse": false, - "height": "250px", - "panels": [ - { - "aliasColors": { - - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 10, - "id": 1, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 0, - "links": [ - - ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - - ], - "spaceLength": 10, - "span": 12, - "stack": true, - "steppedLine": false, - "targets": [ - { - "expr": "sum(\n node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\"}\n * on(namespace,pod)\n group_left(workload, workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{pod}}", - "legendLink": null, - "step": 10 - } - ], - "thresholds": [ - - ], - "timeFrom": null, - "timeShift": null, - "title": "CPU Usage", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ - - ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] - } - ], - "repeat": null, - "repeatIteration": null, - "repeatRowId": null, - "showTitle": true, - "title": "CPU Usage", - "titleSize": "h6" - }, - { - "collapse": false, - "height": "250px", - "panels": [ - { - "aliasColors": { - - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 1, - "id": 2, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [ - - ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - - ], - "spaceLength": 10, - "span": 12, - "stack": false, - "steppedLine": false, - "styles": [ - { - "alias": "Time", - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "pattern": "Time", - "type": "hidden" - }, - { - "alias": "CPU Usage", - "colorMode": null, - "colors": [ - - ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "link": false, - "linkTooltip": "Drill down", - "linkUrl": "", - "pattern": "Value #A", - "thresholds": [ - - ], - "type": "number", - "unit": "short" - }, - { - "alias": "CPU Requests", - "colorMode": null, - "colors": [ - - ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "link": false, - "linkTooltip": "Drill down", - "linkUrl": "", - "pattern": "Value #B", - "thresholds": [ - - ], - "type": "number", - "unit": "short" - }, - { - "alias": "CPU Requests %", - "colorMode": null, - "colors": [ - - ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "link": false, - "linkTooltip": "Drill down", - "linkUrl": "", - "pattern": "Value #C", - "thresholds": [ - - ], - "type": "number", - "unit": "percentunit" - }, - { - "alias": "CPU Limits", - "colorMode": null, - "colors": [ - - ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "link": false, - "linkTooltip": "Drill down", - "linkUrl": "", - "pattern": "Value #D", - "thresholds": [ - - ], - "type": "number", - "unit": "short" - }, - { - "alias": "CPU Limits %", - "colorMode": null, - "colors": [ - - ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "link": false, - "linkTooltip": "Drill down", - "linkUrl": "", - "pattern": "Value #E", - "thresholds": [ - - ], - "type": "number", - "unit": "percentunit" - }, - { - "alias": "Pod", - "colorMode": null, - "colors": [ - - ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "link": true, - "linkTooltip": "Drill down", - "linkUrl": "./d/6581e46e4e5c7ba40a07646395ef7b23/k8s-resources-pod?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$namespace&var-pod=$__cell", - "pattern": "pod", - "thresholds": [ - - ], - "type": "number", - "unit": "short" - }, - { - "alias": "", - "colorMode": null, - "colors": [ - - ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "pattern": "/.*/", - "thresholds": [ - - ], - "type": "string", - "unit": "short" - } - ], - "targets": [ - { - "expr": "sum(\n node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\"}\n * on(namespace,pod)\n group_left(workload, workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", - "format": "table", - "instant": true, - "intervalFactor": 2, - "legendFormat": "", - "refId": "A", - "step": 10 - }, - { - "expr": "sum(\n kube_pod_container_resource_requests_cpu_cores{cluster=\"$cluster\", namespace=\"$namespace\"}\n * on(namespace,pod)\n group_left(workload, workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", - "format": "table", - "instant": true, - "intervalFactor": 2, - "legendFormat": "", - "refId": "B", - "step": 10 - }, - { - "expr": "sum(\n node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\"}\n * on(namespace,pod)\n group_left(workload, workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n/sum(\n kube_pod_container_resource_requests_cpu_cores{cluster=\"$cluster\", namespace=\"$namespace\"}\n * on(namespace,pod)\n group_left(workload, workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", - "format": "table", - "instant": true, - "intervalFactor": 2, - "legendFormat": "", - "refId": "C", - "step": 10 - }, - { - "expr": "sum(\n kube_pod_container_resource_limits_cpu_cores{cluster=\"$cluster\", namespace=\"$namespace\"}\n * on(namespace,pod)\n group_left(workload, workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", - "format": "table", - "instant": true, - "intervalFactor": 2, - "legendFormat": "", - "refId": "D", - "step": 10 - }, - { - "expr": "sum(\n node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\"}\n * on(namespace,pod)\n group_left(workload, workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n/sum(\n kube_pod_container_resource_limits_cpu_cores{cluster=\"$cluster\", namespace=\"$namespace\"}\n * on(namespace,pod)\n group_left(workload, workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", - "format": "table", - "instant": true, - "intervalFactor": 2, - "legendFormat": "", - "refId": "E", - "step": 10 - } - ], - "thresholds": [ - - ], - "timeFrom": null, - "timeShift": null, - "title": "CPU Quota", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "transform": "table", - "type": "table", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ - - ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] - } - ], - "repeat": null, - "repeatIteration": null, - "repeatRowId": null, - "showTitle": true, - "title": "CPU Quota", - "titleSize": "h6" - }, - { - "collapse": false, - "height": "250px", - "panels": [ - { - "aliasColors": { - - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 10, - "id": 3, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 0, - "links": [ - - ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - - ], - "spaceLength": 10, - "span": 12, - "stack": true, - "steppedLine": false, - "targets": [ - { - "expr": "sum(\n container_memory_working_set_bytes{cluster=\"$cluster\", namespace=\"$namespace\", container!=\"\"}\n * on(namespace,pod)\n group_left(workload, workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{pod}}", - "legendLink": null, - "step": 10 - } - ], - "thresholds": [ - - ], - "timeFrom": null, - "timeShift": null, - "title": "Memory Usage", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ - - ] - }, - "yaxes": [ - { - "format": "bytes", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] - } - ], - "repeat": null, - "repeatIteration": null, - "repeatRowId": null, - "showTitle": true, - "title": "Memory Usage", - "titleSize": "h6" - }, - { - "collapse": false, - "height": "250px", - "panels": [ - { - "aliasColors": { - - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 1, - "id": 4, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [ - - ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - - ], - "spaceLength": 10, - "span": 12, - "stack": false, - "steppedLine": false, - "styles": [ - { - "alias": "Time", - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "pattern": "Time", - "type": "hidden" - }, - { - "alias": "Memory Usage", - "colorMode": null, - "colors": [ - - ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "link": false, - "linkTooltip": "Drill down", - "linkUrl": "", - "pattern": "Value #A", - "thresholds": [ - - ], - "type": "number", - "unit": "bytes" - }, - { - "alias": "Memory Requests", - "colorMode": null, - "colors": [ - - ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "link": false, - "linkTooltip": "Drill down", - "linkUrl": "", - "pattern": "Value #B", - "thresholds": [ - - ], - "type": "number", - "unit": "bytes" - }, - { - "alias": "Memory Requests %", - "colorMode": null, - "colors": [ - - ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "link": false, - "linkTooltip": "Drill down", - "linkUrl": "", - "pattern": "Value #C", - "thresholds": [ - - ], - "type": "number", - "unit": "percentunit" - }, - { - "alias": "Memory Limits", - "colorMode": null, - "colors": [ - - ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "link": false, - "linkTooltip": "Drill down", - "linkUrl": "", - "pattern": "Value #D", - "thresholds": [ - - ], - "type": "number", - "unit": "bytes" - }, - { - "alias": "Memory Limits %", - "colorMode": null, - "colors": [ - - ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "link": false, - "linkTooltip": "Drill down", - "linkUrl": "", - "pattern": "Value #E", - "thresholds": [ - - ], - "type": "number", - "unit": "percentunit" - }, - { - "alias": "Pod", - "colorMode": null, - "colors": [ - - ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "link": true, - "linkTooltip": "Drill down", - "linkUrl": "./d/6581e46e4e5c7ba40a07646395ef7b23/k8s-resources-pod?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$namespace&var-pod=$__cell", - "pattern": "pod", - "thresholds": [ - - ], - "type": "number", - "unit": "short" - }, - { - "alias": "", - "colorMode": null, - "colors": [ - - ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "pattern": "/.*/", - "thresholds": [ - - ], - "type": "string", - "unit": "short" - } - ], - "targets": [ - { - "expr": "sum(\n container_memory_working_set_bytes{cluster=\"$cluster\", namespace=\"$namespace\", container!=\"\"}\n * on(namespace,pod)\n group_left(workload, workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", - "format": "table", - "instant": true, - "intervalFactor": 2, - "legendFormat": "", - "refId": "A", - "step": 10 - }, - { - "expr": "sum(\n kube_pod_container_resource_requests_memory_bytes{cluster=\"$cluster\", namespace=\"$namespace\"}\n * on(namespace,pod)\n group_left(workload, workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", - "format": "table", - "instant": true, - "intervalFactor": 2, - "legendFormat": "", - "refId": "B", - "step": 10 - }, - { - "expr": "sum(\n container_memory_working_set_bytes{cluster=\"$cluster\", namespace=\"$namespace\", container!=\"\"}\n * on(namespace,pod)\n group_left(workload, workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n/sum(\n kube_pod_container_resource_requests_memory_bytes{cluster=\"$cluster\", namespace=\"$namespace\"}\n * on(namespace,pod)\n group_left(workload, workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", - "format": "table", - "instant": true, - "intervalFactor": 2, - "legendFormat": "", - "refId": "C", - "step": 10 - }, - { - "expr": "sum(\n kube_pod_container_resource_limits_memory_bytes{cluster=\"$cluster\", namespace=\"$namespace\"}\n * on(namespace,pod)\n group_left(workload, workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", - "format": "table", - "instant": true, - "intervalFactor": 2, - "legendFormat": "", - "refId": "D", - "step": 10 - }, - { - "expr": "sum(\n container_memory_working_set_bytes{cluster=\"$cluster\", namespace=\"$namespace\", container!=\"\"}\n * on(namespace,pod)\n group_left(workload, workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n/sum(\n kube_pod_container_resource_limits_memory_bytes{cluster=\"$cluster\", namespace=\"$namespace\"}\n * on(namespace,pod)\n group_left(workload, workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", - "format": "table", - "instant": true, - "intervalFactor": 2, - "legendFormat": "", - "refId": "E", - "step": 10 - } - ], - "thresholds": [ - - ], - "timeFrom": null, - "timeShift": null, - "title": "Memory Quota", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "transform": "table", - "type": "table", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ - - ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] - } - ], - "repeat": null, - "repeatIteration": null, - "repeatRowId": null, - "showTitle": true, - "title": "Memory Quota", - "titleSize": "h6" - } - ], - "schemaVersion": 14, - "style": "dark", - "tags": [ - "kubernetes-mixin" - ], - "templating": { - "list": [ - { - "current": { - "text": "Prometheus", - "value": "Prometheus" - }, - "hide": 0, - "label": null, - "name": "datasource", - "options": [ - - ], - "query": "prometheus", - "refresh": 1, - "regex": "", - "type": "datasource" - }, - { - "allValue": null, - "current": { - "text": "prod", - "value": "prod" - }, - "datasource": "$datasource", - "hide": 2, - "includeAll": false, - "label": "cluster", - "multi": false, - "name": "cluster", - "options": [ - - ], - "query": "label_values(kube_pod_info, cluster)", - "refresh": 1, - "regex": "", - "sort": 2, - "tagValuesQuery": "", - "tags": [ - - ], - "tagsQuery": "", - "type": "query", - "useTags": false - }, - { - "allValue": null, - "current": { - "text": "prod", - "value": "prod" - }, - "datasource": "$datasource", - "hide": 0, - "includeAll": false, - "label": "namespace", - "multi": false, - "name": "namespace", - "options": [ - - ], - "query": "label_values(kube_pod_info{cluster=\"$cluster\"}, namespace)", - "refresh": 1, - "regex": "", - "sort": 2, - "tagValuesQuery": "", - "tags": [ - - ], - "tagsQuery": "", - "type": "query", - "useTags": false - }, - { - "allValue": null, - "current": { - "text": "prod", - "value": "prod" - }, - "datasource": "$datasource", - "hide": 0, - "includeAll": false, - "label": "workload", - "multi": false, - "name": "workload", - "options": [ - - ], - "query": "label_values(mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\"}, workload)", - "refresh": 1, - "regex": "", - "sort": 2, - "tagValuesQuery": "", - "tags": [ - - ], - "tagsQuery": "", - "type": "query", - "useTags": false - }, - { - "allValue": null, - "current": { - "text": "prod", - "value": "prod" - }, - "datasource": "$datasource", - "hide": 0, - "includeAll": false, - "label": "type", - "multi": false, - "name": "type", - "options": [ - - ], - "query": "label_values(mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\"}, workload_type)", - "refresh": 1, - "regex": "", - "sort": 2, - "tagValuesQuery": "", - "tags": [ - - ], - "tagsQuery": "", - "type": "query", - "useTags": false - } - ] - }, - "time": { - "from": "now-1h", - "to": "now" - }, - "timepicker": { - "refresh_intervals": [ - "5s", - "10s", - "30s", - "1m", - "5m", - "15m", - "30m", - "1h", - "2h", - "1d" - ], - "time_options": [ - "5m", - "15m", - "1h", - "6h", - "12h", - "24h", - "2d", - "7d", - "30d" - ] - }, - "timezone": "", - "title": "Kubernetes / Compute Resources / Workload", - "uid": "a164a7f0339f99e89cea5cb47e9be617", - "version": 0 - } - k8s-resources-workloads-namespace.json: |- - { - "annotations": { - "list": [ - - ] - }, - "editable": true, - "gnetId": null, - "graphTooltip": 0, - "hideControls": false, - "links": [ - - ], - "refresh": "10s", - "rows": [ - { - "collapse": false, - "height": "250px", - "panels": [ - { - "aliasColors": { - - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 10, - "id": 1, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 0, - "links": [ - - ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - - ], - "spaceLength": 10, - "span": 12, - "stack": true, - "steppedLine": false, - "targets": [ - { - "expr": "sum(\n node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\"}\n* on(namespace,pod)\n group_left(workload, workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\"}\n) by (workload, workload_type)\n", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{workload}} - {{workload_type}}", - "legendLink": null, - "step": 10 - } - ], - "thresholds": [ - - ], - "timeFrom": null, - "timeShift": null, - "title": "CPU Usage", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ - - ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] - } - ], - "repeat": null, - "repeatIteration": null, - "repeatRowId": null, - "showTitle": true, - "title": "CPU Usage", - "titleSize": "h6" - }, - { - "collapse": false, - "height": "250px", - "panels": [ - { - "aliasColors": { - - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 1, - "id": 2, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [ - - ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - - ], - "spaceLength": 10, - "span": 12, - "stack": false, - "steppedLine": false, - "styles": [ - { - "alias": "Time", - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "pattern": "Time", - "type": "hidden" - }, - { - "alias": "Running Pods", - "colorMode": null, - "colors": [ - - ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 0, - "link": false, - "linkTooltip": "Drill down", - "linkUrl": "", - "pattern": "Value #A", - "thresholds": [ - - ], - "type": "number", - "unit": "short" - }, - { - "alias": "CPU Usage", - "colorMode": null, - "colors": [ - - ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "link": false, - "linkTooltip": "Drill down", - "linkUrl": "", - "pattern": "Value #B", - "thresholds": [ - - ], - "type": "number", - "unit": "short" - }, - { - "alias": "CPU Requests", - "colorMode": null, - "colors": [ - - ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "link": false, - "linkTooltip": "Drill down", - "linkUrl": "", - "pattern": "Value #C", - "thresholds": [ - - ], - "type": "number", - "unit": "short" - }, - { - "alias": "CPU Requests %", - "colorMode": null, - "colors": [ - - ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "link": false, - "linkTooltip": "Drill down", - "linkUrl": "", - "pattern": "Value #D", - "thresholds": [ - - ], - "type": "number", - "unit": "percentunit" - }, - { - "alias": "CPU Limits", - "colorMode": null, - "colors": [ - - ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "link": false, - "linkTooltip": "Drill down", - "linkUrl": "", - "pattern": "Value #E", - "thresholds": [ - - ], - "type": "number", - "unit": "short" - }, - { - "alias": "CPU Limits %", - "colorMode": null, - "colors": [ - - ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "link": false, - "linkTooltip": "Drill down", - "linkUrl": "", - "pattern": "Value #F", - "thresholds": [ - - ], - "type": "number", - "unit": "percentunit" - }, - { - "alias": "Workload", - "colorMode": null, - "colors": [ - - ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "link": true, - "linkTooltip": "Drill down", - "linkUrl": "./d/a164a7f0339f99e89cea5cb47e9be617/k8s-resources-workload?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$namespace&var-workload=$__cell&var-type=$__cell_2", - "pattern": "workload", - "thresholds": [ - - ], - "type": "number", - "unit": "short" - }, - { - "alias": "Workload Type", - "colorMode": null, - "colors": [ - - ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "link": false, - "linkTooltip": "Drill down", - "linkUrl": "", - "pattern": "workload_type", - "thresholds": [ - - ], - "type": "number", - "unit": "short" - }, - { - "alias": "", - "colorMode": null, - "colors": [ - - ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "pattern": "/.*/", - "thresholds": [ - - ], - "type": "string", - "unit": "short" - } - ], - "targets": [ - { - "expr": "count(mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\"}) by (workload, workload_type)", - "format": "table", - "instant": true, - "intervalFactor": 2, - "legendFormat": "", - "refId": "A", - "step": 10 - }, - { - "expr": "sum(\n node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\"}\n* on(namespace,pod)\n group_left(workload, workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\"}\n) by (workload, workload_type)\n", - "format": "table", - "instant": true, - "intervalFactor": 2, - "legendFormat": "", - "refId": "B", - "step": 10 - }, - { - "expr": "sum(\n kube_pod_container_resource_requests_cpu_cores{cluster=\"$cluster\", namespace=\"$namespace\"}\n* on(namespace,pod)\n group_left(workload, workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\"}\n) by (workload, workload_type)\n", - "format": "table", - "instant": true, - "intervalFactor": 2, - "legendFormat": "", - "refId": "C", - "step": 10 - }, - { - "expr": "sum(\n node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\"}\n* on(namespace,pod)\n group_left(workload, workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\"}\n) by (workload, workload_type)\n/sum(\n kube_pod_container_resource_requests_cpu_cores{cluster=\"$cluster\", namespace=\"$namespace\"}\n* on(namespace,pod)\n group_left(workload, workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\"}\n) by (workload, workload_type)\n", - "format": "table", - "instant": true, - "intervalFactor": 2, - "legendFormat": "", - "refId": "D", - "step": 10 - }, - { - "expr": "sum(\n kube_pod_container_resource_limits_cpu_cores{cluster=\"$cluster\", namespace=\"$namespace\"}\n* on(namespace,pod)\n group_left(workload, workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\"}\n) by (workload, workload_type)\n", - "format": "table", - "instant": true, - "intervalFactor": 2, - "legendFormat": "", - "refId": "E", - "step": 10 - }, - { - "expr": "sum(\n node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\"}\n* on(namespace,pod)\n group_left(workload, workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\"}\n) by (workload, workload_type)\n/sum(\n kube_pod_container_resource_limits_cpu_cores{cluster=\"$cluster\", namespace=\"$namespace\"}\n* on(namespace,pod)\n group_left(workload, workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\"}\n) by (workload, workload_type)\n", - "format": "table", - "instant": true, - "intervalFactor": 2, - "legendFormat": "", - "refId": "F", - "step": 10 - } - ], - "thresholds": [ - - ], - "timeFrom": null, - "timeShift": null, - "title": "CPU Quota", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "transform": "table", - "type": "table", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ - - ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] - } - ], - "repeat": null, - "repeatIteration": null, - "repeatRowId": null, - "showTitle": true, - "title": "CPU Quota", - "titleSize": "h6" - }, - { - "collapse": false, - "height": "250px", - "panels": [ - { - "aliasColors": { - - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 10, - "id": 3, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 0, - "links": [ - - ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - - ], - "spaceLength": 10, - "span": 12, - "stack": true, - "steppedLine": false, - "targets": [ - { - "expr": "sum(\n container_memory_working_set_bytes{cluster=\"$cluster\", namespace=\"$namespace\", container!=\"\"}\n * on(namespace,pod)\n group_left(workload, workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\"}\n) by (workload, workload_type)\n", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{workload}} - {{workload_type}}", - "legendLink": null, - "step": 10 - } - ], - "thresholds": [ - - ], - "timeFrom": null, - "timeShift": null, - "title": "Memory Usage", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ - - ] - }, - "yaxes": [ - { - "format": "bytes", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] - } - ], - "repeat": null, - "repeatIteration": null, - "repeatRowId": null, - "showTitle": true, - "title": "Memory Usage", - "titleSize": "h6" - }, - { - "collapse": false, - "height": "250px", - "panels": [ - { - "aliasColors": { - - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 1, - "id": 4, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [ - - ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - - ], - "spaceLength": 10, - "span": 12, - "stack": false, - "steppedLine": false, - "styles": [ - { - "alias": "Time", - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "pattern": "Time", - "type": "hidden" - }, - { - "alias": "Running Pods", - "colorMode": null, - "colors": [ - - ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 0, - "link": false, - "linkTooltip": "Drill down", - "linkUrl": "", - "pattern": "Value #A", - "thresholds": [ - - ], - "type": "number", - "unit": "short" - }, - { - "alias": "Memory Usage", - "colorMode": null, - "colors": [ - - ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "link": false, - "linkTooltip": "Drill down", - "linkUrl": "", - "pattern": "Value #B", - "thresholds": [ - - ], - "type": "number", - "unit": "bytes" - }, - { - "alias": "Memory Requests", - "colorMode": null, - "colors": [ - - ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "link": false, - "linkTooltip": "Drill down", - "linkUrl": "", - "pattern": "Value #C", - "thresholds": [ - - ], - "type": "number", - "unit": "bytes" - }, - { - "alias": "Memory Requests %", - "colorMode": null, - "colors": [ - - ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "link": false, - "linkTooltip": "Drill down", - "linkUrl": "", - "pattern": "Value #D", - "thresholds": [ - - ], - "type": "number", - "unit": "percentunit" - }, - { - "alias": "Memory Limits", - "colorMode": null, - "colors": [ - - ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "link": false, - "linkTooltip": "Drill down", - "linkUrl": "", - "pattern": "Value #E", - "thresholds": [ - - ], - "type": "number", - "unit": "bytes" - }, - { - "alias": "Memory Limits %", - "colorMode": null, - "colors": [ - - ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "link": false, - "linkTooltip": "Drill down", - "linkUrl": "", - "pattern": "Value #F", - "thresholds": [ - - ], - "type": "number", - "unit": "percentunit" - }, - { - "alias": "Workload", - "colorMode": null, - "colors": [ - - ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "link": true, - "linkTooltip": "Drill down", - "linkUrl": "./d/a164a7f0339f99e89cea5cb47e9be617/k8s-resources-workload?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$namespace&var-workload=$__cell&var-type=$__cell_2", - "pattern": "workload", - "thresholds": [ - - ], - "type": "number", - "unit": "short" - }, - { - "alias": "Workload Type", - "colorMode": null, - "colors": [ - - ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "link": false, - "linkTooltip": "Drill down", - "linkUrl": "", - "pattern": "workload_type", - "thresholds": [ - - ], - "type": "number", - "unit": "short" - }, - { - "alias": "", - "colorMode": null, - "colors": [ - - ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "pattern": "/.*/", - "thresholds": [ - - ], - "type": "string", - "unit": "short" - } - ], - "targets": [ - { - "expr": "count(mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\"}) by (workload, workload_type)", - "format": "table", - "instant": true, - "intervalFactor": 2, - "legendFormat": "", - "refId": "A", - "step": 10 - }, - { - "expr": "sum(\n container_memory_working_set_bytes{cluster=\"$cluster\", namespace=\"$namespace\", container!=\"\"}\n * on(namespace,pod)\n group_left(workload, workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\"}\n) by (workload, workload_type)\n", - "format": "table", - "instant": true, - "intervalFactor": 2, - "legendFormat": "", - "refId": "B", - "step": 10 - }, - { - "expr": "sum(\n kube_pod_container_resource_requests_memory_bytes{cluster=\"$cluster\", namespace=\"$namespace\"}\n* on(namespace,pod)\n group_left(workload, workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\"}\n) by (workload, workload_type)\n", - "format": "table", - "instant": true, - "intervalFactor": 2, - "legendFormat": "", - "refId": "C", - "step": 10 - }, - { - "expr": "sum(\n container_memory_working_set_bytes{cluster=\"$cluster\", namespace=\"$namespace\", container!=\"\"}\n * on(namespace,pod)\n group_left(workload, workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\"}\n) by (workload, workload_type)\n/sum(\n kube_pod_container_resource_requests_memory_bytes{cluster=\"$cluster\", namespace=\"$namespace\"}\n* on(namespace,pod)\n group_left(workload, workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\"}\n) by (workload, workload_type)\n", - "format": "table", - "instant": true, - "intervalFactor": 2, - "legendFormat": "", - "refId": "D", - "step": 10 - }, - { - "expr": "sum(\n kube_pod_container_resource_limits_memory_bytes{cluster=\"$cluster\", namespace=\"$namespace\"}\n* on(namespace,pod)\n group_left(workload, workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\"}\n) by (workload, workload_type)\n", - "format": "table", - "instant": true, - "intervalFactor": 2, - "legendFormat": "", - "refId": "E", - "step": 10 - }, - { - "expr": "sum(\n container_memory_working_set_bytes{cluster=\"$cluster\", namespace=\"$namespace\", container!=\"\"}\n * on(namespace,pod)\n group_left(workload, workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\"}\n) by (workload, workload_type)\n/sum(\n kube_pod_container_resource_limits_memory_bytes{cluster=\"$cluster\", namespace=\"$namespace\"}\n* on(namespace,pod)\n group_left(workload, workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\"}\n) by (workload, workload_type)\n", - "format": "table", - "instant": true, - "intervalFactor": 2, - "legendFormat": "", - "refId": "F", - "step": 10 - } - ], - "thresholds": [ - - ], - "timeFrom": null, - "timeShift": null, - "title": "Memory Quota", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "transform": "table", - "type": "table", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ - - ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] - } - ], - "repeat": null, - "repeatIteration": null, - "repeatRowId": null, - "showTitle": true, - "title": "Memory Quota", - "titleSize": "h6" - } - ], - "schemaVersion": 14, - "style": "dark", - "tags": [ - "kubernetes-mixin" - ], - "templating": { - "list": [ - { - "current": { - "text": "Prometheus", - "value": "Prometheus" - }, - "hide": 0, - "label": null, - "name": "datasource", - "options": [ - - ], - "query": "prometheus", - "refresh": 1, - "regex": "", - "type": "datasource" - }, - { - "allValue": null, - "current": { - "text": "prod", - "value": "prod" - }, - "datasource": "$datasource", - "hide": 2, - "includeAll": false, - "label": "cluster", - "multi": false, - "name": "cluster", - "options": [ - - ], - "query": "label_values(kube_pod_info, cluster)", - "refresh": 1, - "regex": "", - "sort": 2, - "tagValuesQuery": "", - "tags": [ - - ], - "tagsQuery": "", - "type": "query", - "useTags": false - }, - { - "allValue": null, - "current": { - "text": "prod", - "value": "prod" - }, - "datasource": "$datasource", - "hide": 0, - "includeAll": false, - "label": "namespace", - "multi": false, - "name": "namespace", - "options": [ - - ], - "query": "label_values(kube_pod_info{cluster=\"$cluster\"}, namespace)", - "refresh": 1, - "regex": "", - "sort": 2, - "tagValuesQuery": "", - "tags": [ - - ], - "tagsQuery": "", - "type": "query", - "useTags": false - } - ] - }, - "time": { - "from": "now-1h", - "to": "now" - }, - "timepicker": { - "refresh_intervals": [ - "5s", - "10s", - "30s", - "1m", - "5m", - "15m", - "30m", - "1h", - "2h", - "1d" - ], - "time_options": [ - "5m", - "15m", - "1h", - "6h", - "12h", - "24h", - "2d", - "7d", - "30d" - ] - }, - "timezone": "", - "title": "Kubernetes / Compute Resources / Namespace (Workloads)", - "uid": "a87fb0d919ec0ea5f6543124e16c42a5", - "version": 0 - } -kind: ConfigMap -metadata: - name: grafana-dashboards-k8s-resources - namespace: monitoring diff --git a/addons/grafana/deployment.yaml b/addons/grafana/deployment.yaml index 8e7e10348..1bd74acef 100644 --- a/addons/grafana/deployment.yaml +++ b/addons/grafana/deployment.yaml @@ -64,8 +64,10 @@ spec: mountPath: /etc/grafana/dashboards/k8s - name: dashboards-k8s-nodes mountPath: /etc/grafana/dashboards/k8s-nodes - - name: dashboards-k8s-resources - mountPath: /etc/grafana/dashboards/k8s-resources + - name: dashboards-k8s-resources-1 + mountPath: /etc/grafana/dashboards/k8s-resources-1 + - name: dashboards-k8s-resources-2 + mountPath: /etc/grafana/dashboards/k8s-resources-2 - name: dashboards-coredns mountPath: /etc/grafana/dashboards/coredns - name: dashboards-nginx-ingress @@ -95,9 +97,12 @@ spec: - name: dashboards-k8s-nodes configMap: name: grafana-dashboards-k8s-nodes - - name: dashboards-k8s-resources + - name: dashboards-k8s-resources-1 configMap: - name: grafana-dashboards-k8s-resources + name: grafana-dashboards-k8s-resources-1 + - name: dashboards-k8s-resources-2 + configMap: + name: grafana-dashboards-k8s-resources-2 - name: dashboards-coredns configMap: name: grafana-dashboards-coredns From 030a4cec19dee90e22fb9e307178a64bf586bd5a Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Mon, 25 Nov 2019 22:45:58 -0800 Subject: [PATCH 276/523] Update Grafana from v6.4.4 to v6.5.0 * https://grafana.com/docs/guides/whats-new-in-v6-5/ --- CHANGES.md | 1 + addons/grafana/deployment.yaml | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index 4b7647fd1..130ed167d 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -19,6 +19,7 @@ Notable changes between versions. #### Addons +* Update Grafana from v6.4.4 to [v6.5.0](https://grafana.com/docs/guides/whats-new-in-v6-5/) * Add pod networking details in dashboards ([#593](https://github.com/poseidon/typhoon/pull/593)) * Add node alerts and Grafana dashboard from node-exporter ([#591](https://github.com/poseidon/typhoon/pull/591)) diff --git a/addons/grafana/deployment.yaml b/addons/grafana/deployment.yaml index 1bd74acef..0b120c67e 100644 --- a/addons/grafana/deployment.yaml +++ b/addons/grafana/deployment.yaml @@ -23,7 +23,7 @@ spec: spec: containers: - name: grafana - image: docker.io/grafana/grafana:6.4.4 + image: docker.io/grafana/grafana:6.5.0 env: - name: GF_PATHS_CONFIG value: "/etc/grafana/custom.ini" From 26674083b603bc6ff9a9356fce5ee1d492396d17 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Thu, 28 Nov 2019 14:11:25 -0800 Subject: [PATCH 277/523] Update Grafana from v6.5.0 to v6.5.1 * https://github.com/grafana/grafana/releases/tag/v6.5.1 --- CHANGES.md | 2 +- addons/grafana/deployment.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 130ed167d..bb99f125c 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -19,7 +19,7 @@ Notable changes between versions. #### Addons -* Update Grafana from v6.4.4 to [v6.5.0](https://grafana.com/docs/guides/whats-new-in-v6-5/) +* Update Grafana from v6.4.4 to [v6.5.1](https://grafana.com/docs/guides/whats-new-in-v6-5/) * Add pod networking details in dashboards ([#593](https://github.com/poseidon/typhoon/pull/593)) * Add node alerts and Grafana dashboard from node-exporter ([#591](https://github.com/poseidon/typhoon/pull/591)) diff --git a/addons/grafana/deployment.yaml b/addons/grafana/deployment.yaml index 0b120c67e..83fad50d1 100644 --- a/addons/grafana/deployment.yaml +++ b/addons/grafana/deployment.yaml @@ -23,7 +23,7 @@ spec: spec: containers: - name: grafana - image: docker.io/grafana/grafana:6.5.0 + image: docker.io/grafana/grafana:6.5.1 env: - name: GF_PATHS_CONFIG value: "/etc/grafana/custom.ini" From aa275796cb5827a32646adf3fa15290f21d4c6fc Mon Sep 17 00:00:00 2001 From: Arve Knudsen Date: Tue, 3 Dec 2019 06:20:47 +0100 Subject: [PATCH 278/523] Fix DigitalOcean controller and worker ipv4/ipv6 outputs (#594) * Fix controller and worker ipv4/ipv4 outputs to be lists of strings * With Terraform v0.11 syntax, an enclosing list was required to coerce the output to be a list of strings * With Terraform v0.12 syntax, the enclosing list shouldn't be needed --- digital-ocean/container-linux/kubernetes/outputs.tf | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/digital-ocean/container-linux/kubernetes/outputs.tf b/digital-ocean/container-linux/kubernetes/outputs.tf index fe5338efd..50934e37a 100644 --- a/digital-ocean/container-linux/kubernetes/outputs.tf +++ b/digital-ocean/container-linux/kubernetes/outputs.tf @@ -12,19 +12,19 @@ output "workers_dns" { } output "controllers_ipv4" { - value = [digitalocean_droplet.controllers.*.ipv4_address] + value = digitalocean_droplet.controllers.*.ipv4_address } output "controllers_ipv6" { - value = [digitalocean_droplet.controllers.*.ipv6_address] + value = digitalocean_droplet.controllers.*.ipv6_address } output "workers_ipv4" { - value = [digitalocean_droplet.workers.*.ipv4_address] + value = digitalocean_droplet.workers.*.ipv4_address } output "workers_ipv6" { - value = [digitalocean_droplet.workers.*.ipv6_address] + value = digitalocean_droplet.workers.*.ipv6_address } # Outputs for custom firewalls From 5fa002f4f7ac7f456d385fe6a765cafed7f684aa Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Mon, 2 Dec 2019 21:21:16 -0800 Subject: [PATCH 279/523] Update mkdocs-material from v4.5.0 to v4.5.1 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 252fc3c57..122696998 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ mkdocs==1.0.4 -mkdocs-material==4.5.0 +mkdocs-material==4.5.1 pygments==2.4.2 pymdown-extensions==6.2.0 From 283727526547934bfcba7bbb4594ce7963f379e7 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Wed, 4 Dec 2019 22:10:55 -0800 Subject: [PATCH 280/523] Introduce cluster creation without local writes to asset_dir * Allow generated assets (TLS materials, manifests) to be securely distributed to controller node(s) via file provisioner (i.e. ssh-agent) as an assets bundle file, rather than relying on assets being locally rendered to disk in an asset_dir and then securely distributed * Change `asset_dir` from required to optional. Left unset, asset_dir defaults to "" and no assets will be written to files on the machine that runs terraform apply * Enhancement: Managed cluster assets are kept only in Terraform state, which supports different backends (GCS, S3, etcd, etc) and optional encryption. terraform apply accesses state, runs in-memory, and distributes sensitive materials to controllers without making use of local disk (simplifies use in CI systems) * Enhancement: Improve asset unpack and layout process to position etcd certificates and control plane certificates more cleanly, without unneeded secret materials Details: * Terraform file provisioner support for distributing directories of contents (with unknown structure) has been limited to reading from a local directory, meaning local writes to asset_dir were required. https://github.com/poseidon/typhoon/issues/585 discusses the problem and newer or upcoming Terraform features that might help. * Observation: Terraform provisioner support for single files works well, but iteration isn't viable. We're also constrained to Terraform language features on the apply side (no extra plugins, no shelling out) and CoreOS / Fedora tools on the receive side. * Take a map representation of the contents that would have been splayed out in asset_dir and pack/encode them into a single file format devised for easy unpacking. Use an awk one-liner on the receive side to unpack. In pratice, this has worked well and its rather nice that a single assets file is transferred by file provisioner (all or none) Rel: https://github.com/poseidon/terraform-render-bootstrap/pull/162 --- aws/container-linux/kubernetes/bootstrap.tf | 2 +- .../kubernetes/cl/controller.yaml.tmpl | 26 +++++++- aws/container-linux/kubernetes/ssh.tf | 63 ++++-------------- aws/fedora-coreos/kubernetes/bootstrap.tf | 2 +- .../kubernetes/fcc/controller.yaml | 24 ++++++- aws/fedora-coreos/kubernetes/ssh.tf | 65 ++++--------------- azure/container-linux/kubernetes/bootstrap.tf | 2 +- .../kubernetes/cl/controller.yaml.tmpl | 26 +++++++- azure/container-linux/kubernetes/ssh.tf | 65 ++++--------------- .../container-linux/kubernetes/bootstrap.tf | 2 +- .../kubernetes/cl/controller.yaml.tmpl | 26 +++++++- bare-metal/container-linux/kubernetes/ssh.tf | 63 ++++-------------- .../fedora-coreos/kubernetes/bootstrap.tf | 2 +- .../kubernetes/fcc/controller.yaml | 24 ++++++- bare-metal/fedora-coreos/kubernetes/ssh.tf | 61 ++++------------- .../container-linux/kubernetes/bootstrap.tf | 2 +- .../kubernetes/cl/controller.yaml.tmpl | 26 +++++++- .../container-linux/kubernetes/ssh.tf | 63 ++++-------------- .../container-linux/kubernetes/bootstrap.tf | 2 +- .../kubernetes/cl/controller.yaml.tmpl | 26 +++++++- .../container-linux/kubernetes/ssh.tf | 65 ++++--------------- 21 files changed, 258 insertions(+), 379 deletions(-) diff --git a/aws/container-linux/kubernetes/bootstrap.tf b/aws/container-linux/kubernetes/bootstrap.tf index 0d7266e7b..1db669b98 100644 --- a/aws/container-linux/kubernetes/bootstrap.tf +++ b/aws/container-linux/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=0f1f16c612a6877d25a3fedcb476b3087a3de999" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=dce49114a083436c5af6d9174b9c1248786ba3b6" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/aws/container-linux/kubernetes/cl/controller.yaml.tmpl b/aws/container-linux/kubernetes/cl/controller.yaml.tmpl index c4cb7b2f0..246c617ef 100644 --- a/aws/container-linux/kubernetes/cl/controller.yaml.tmpl +++ b/aws/container-linux/kubernetes/cl/controller.yaml.tmpl @@ -108,6 +108,8 @@ systemd: ExecStartPre=-/usr/bin/bash -c 'set -x && [ -n "$(ls /opt/bootstrap/assets/manifests-*/* 2>/dev/null)" ] && mv /opt/bootstrap/assets/manifests-*/* /opt/bootstrap/assets/manifests && rm -rf /opt/bootstrap/assets/manifests-*' ExecStart=/usr/bin/rkt run \ --trust-keys-from-https \ + --volume config,kind=host,source=/etc/kubernetes/bootstrap-secrets \ + --mount volume=config,target=/etc/kubernetes/secrets \ --volume assets,kind=host,source=/opt/bootstrap/assets \ --mount volume=assets,target=/assets \ --volume script,kind=host,source=/opt/bootstrap/apply \ @@ -135,13 +137,35 @@ storage: inline: | KUBELET_IMAGE_URL=docker://k8s.gcr.io/hyperkube KUBELET_IMAGE_TAG=v1.16.3 + - path: /opt/bootstrap/layout + filesystem: root + mode: 0544 + contents: + inline: | + #!/bin/bash -e + mkdir -p -- auth tls/etcd tls/k8s static-manifests manifests/coredns manifests-networking + awk '/#####/ {filename=$2; next} {print > filename}' assets + mkdir -p /etc/ssl/etcd/etcd + mkdir -p /etc/kubernetes/bootstrap-secrets + mv tls/etcd/{peer*,server*} /etc/ssl/etcd/etcd/ + mv tls/etcd/etcd-client* /etc/kubernetes/bootstrap-secrets/ + chown -R etcd:etcd /etc/ssl/etcd + chmod -R 500 /etc/ssl/etcd + mv auth/kubeconfig /etc/kubernetes/bootstrap-secrets/ + mv tls/k8s/* /etc/kubernetes/bootstrap-secrets/ + sudo mkdir -p /etc/kubernetes/manifests + sudo mv static-manifests/* /etc/kubernetes/manifests/ + sudo mkdir -p /opt/bootstrap/assets + sudo mv manifests /opt/bootstrap/assets/manifests + sudo mv manifests-networking /opt/bootstrap/assets/manifests-networking + rm -rf assets auth static-manifests tls - path: /opt/bootstrap/apply filesystem: root mode: 0544 contents: inline: | #!/bin/bash -e - export KUBECONFIG=/assets/auth/kubeconfig + export KUBECONFIG=/etc/kubernetes/secrets/kubeconfig until kubectl version; do echo "Waiting for static pod control plane" sleep 5 diff --git a/aws/container-linux/kubernetes/ssh.tf b/aws/container-linux/kubernetes/ssh.tf index 1d13aa55d..f180f3129 100644 --- a/aws/container-linux/kubernetes/ssh.tf +++ b/aws/container-linux/kubernetes/ssh.tf @@ -1,3 +1,12 @@ +locals { + # format assets for distribution + assets_bundle = [ + # header with the unpack location + for key, value in module.bootstrap.assets_dist: + format("##### %s\n%s", key, value) + ] +} + # Secure copy assets to controllers. resource "null_resource" "copy-controller-secrets" { count = var.controller_count @@ -14,63 +23,13 @@ resource "null_resource" "copy-controller-secrets" { } provisioner "file" { - content = module.bootstrap.etcd_ca_cert - destination = "$HOME/etcd-client-ca.crt" - } - - provisioner "file" { - content = module.bootstrap.etcd_client_cert - destination = "$HOME/etcd-client.crt" - } - - provisioner "file" { - content = module.bootstrap.etcd_client_key - destination = "$HOME/etcd-client.key" - } - - provisioner "file" { - content = module.bootstrap.etcd_server_cert - destination = "$HOME/etcd-server.crt" - } - - provisioner "file" { - content = module.bootstrap.etcd_server_key - destination = "$HOME/etcd-server.key" - } - - provisioner "file" { - content = module.bootstrap.etcd_peer_cert - destination = "$HOME/etcd-peer.crt" - } - - provisioner "file" { - content = module.bootstrap.etcd_peer_key - destination = "$HOME/etcd-peer.key" - } - - provisioner "file" { - source = var.asset_dir + content = join("\n", local.assets_bundle) destination = "$HOME/assets" } provisioner "remote-exec" { inline = [ - "sudo mkdir -p /etc/ssl/etcd/etcd", - "sudo mv etcd-client* /etc/ssl/etcd/", - "sudo cp /etc/ssl/etcd/etcd-client-ca.crt /etc/ssl/etcd/etcd/server-ca.crt", - "sudo mv etcd-server.crt /etc/ssl/etcd/etcd/server.crt", - "sudo mv etcd-server.key /etc/ssl/etcd/etcd/server.key", - "sudo cp /etc/ssl/etcd/etcd-client-ca.crt /etc/ssl/etcd/etcd/peer-ca.crt", - "sudo mv etcd-peer.crt /etc/ssl/etcd/etcd/peer.crt", - "sudo mv etcd-peer.key /etc/ssl/etcd/etcd/peer.key", - "sudo chown -R etcd:etcd /etc/ssl/etcd", - "sudo chmod -R 500 /etc/ssl/etcd", - "sudo mv $HOME/assets /opt/bootstrap/assets", - "sudo mkdir -p /etc/kubernetes/manifests", - "sudo mkdir -p /etc/kubernetes/bootstrap-secrets", - "sudo cp -r /opt/bootstrap/assets/tls/* /etc/kubernetes/bootstrap-secrets/", - "sudo cp /opt/bootstrap/assets/auth/kubeconfig /etc/kubernetes/bootstrap-secrets/", - "sudo cp -r /opt/bootstrap/assets/static-manifests/* /etc/kubernetes/manifests/", + "sudo /opt/bootstrap/layout", ] } } diff --git a/aws/fedora-coreos/kubernetes/bootstrap.tf b/aws/fedora-coreos/kubernetes/bootstrap.tf index f8651eae5..c2716fb8c 100644 --- a/aws/fedora-coreos/kubernetes/bootstrap.tf +++ b/aws/fedora-coreos/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=0f1f16c612a6877d25a3fedcb476b3087a3de999" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=dce49114a083436c5af6d9174b9c1248786ba3b6" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/aws/fedora-coreos/kubernetes/fcc/controller.yaml b/aws/fedora-coreos/kubernetes/fcc/controller.yaml index dc2edb21e..8db338720 100644 --- a/aws/fedora-coreos/kubernetes/fcc/controller.yaml +++ b/aws/fedora-coreos/kubernetes/fcc/controller.yaml @@ -119,6 +119,7 @@ systemd: ExecStartPre=-/usr/bin/bash -c 'set -x && [ -n "$(ls /opt/bootstrap/assets/manifests-*/* 2>/dev/null)" ] && mv /opt/bootstrap/assets/manifests-*/* /opt/bootstrap/assets/manifests && rm -rf /opt/bootstrap/assets/manifests-*' ExecStart=/usr/bin/podman run --name bootstrap \ --network host \ + --volume /etc/kubernetes/bootstrap-secrets:/etc/kubernetes/secrets:ro,Z \ --volume /opt/bootstrap/assets:/assets:ro,Z \ --volume /opt/bootstrap/apply:/apply:ro,Z \ k8s.gcr.io/hyperkube:v1.16.3 \ @@ -135,12 +136,33 @@ storage: contents: inline: | ${kubeconfig} + - path: /opt/bootstrap/layout + mode: 0544 + contents: + inline: | + #!/bin/bash -e + mkdir -p -- auth tls/etcd tls/k8s static-manifests manifests/coredns manifests-networking + awk '/#####/ {filename=$2; next} {print > filename}' assets + mkdir -p /etc/ssl/etcd/etcd + mkdir -p /etc/kubernetes/bootstrap-secrets + mv tls/etcd/{peer*,server*} /etc/ssl/etcd/etcd/ + mv tls/etcd/etcd-client* /etc/kubernetes/bootstrap-secrets/ + chown -R etcd:etcd /etc/ssl/etcd + chmod -R 500 /etc/ssl/etcd + mv auth/kubeconfig /etc/kubernetes/bootstrap-secrets/ + mv tls/k8s/* /etc/kubernetes/bootstrap-secrets/ + sudo mkdir -p /etc/kubernetes/manifests + sudo mv static-manifests/* /etc/kubernetes/manifests/ + sudo mkdir -p /opt/bootstrap/assets + sudo mv manifests /opt/bootstrap/assets/manifests + sudo mv manifests-networking /opt/bootstrap/assets/manifests-networking + rm -rf assets auth static-manifests tls - path: /opt/bootstrap/apply mode: 0544 contents: inline: | #!/bin/bash -e - export KUBECONFIG=/assets/auth/kubeconfig + export KUBECONFIG=/etc/kubernetes/secrets/kubeconfig until kubectl version; do echo "Waiting for static pod control plane" sleep 5 diff --git a/aws/fedora-coreos/kubernetes/ssh.tf b/aws/fedora-coreos/kubernetes/ssh.tf index d4a61eb5b..f11ae0b25 100644 --- a/aws/fedora-coreos/kubernetes/ssh.tf +++ b/aws/fedora-coreos/kubernetes/ssh.tf @@ -1,3 +1,12 @@ +locals { + # format assets for distribution + assets_bundle = [ + # header with the unpack location + for key, value in module.bootstrap.assets_dist: + format("##### %s\n%s", key, value) + ] +} + # Secure copy assets to controllers. resource "null_resource" "copy-controller-secrets" { count = var.controller_count @@ -12,65 +21,15 @@ resource "null_resource" "copy-controller-secrets" { user = "core" timeout = "15m" } - - provisioner "file" { - content = module.bootstrap.etcd_ca_cert - destination = "$HOME/etcd-client-ca.crt" - } - - provisioner "file" { - content = module.bootstrap.etcd_client_cert - destination = "$HOME/etcd-client.crt" - } - - provisioner "file" { - content = module.bootstrap.etcd_client_key - destination = "$HOME/etcd-client.key" - } - - provisioner "file" { - content = module.bootstrap.etcd_server_cert - destination = "$HOME/etcd-server.crt" - } - - provisioner "file" { - content = module.bootstrap.etcd_server_key - destination = "$HOME/etcd-server.key" - } - - provisioner "file" { - content = module.bootstrap.etcd_peer_cert - destination = "$HOME/etcd-peer.crt" - } - - provisioner "file" { - content = module.bootstrap.etcd_peer_key - destination = "$HOME/etcd-peer.key" - } - + provisioner "file" { - source = var.asset_dir + content = join("\n", local.assets_bundle) destination = "$HOME/assets" } provisioner "remote-exec" { inline = [ - "sudo mkdir -p /etc/ssl/etcd/etcd", - "sudo mv etcd-client* /etc/ssl/etcd/", - "sudo cp /etc/ssl/etcd/etcd-client-ca.crt /etc/ssl/etcd/etcd/server-ca.crt", - "sudo mv etcd-server.crt /etc/ssl/etcd/etcd/server.crt", - "sudo mv etcd-server.key /etc/ssl/etcd/etcd/server.key", - "sudo cp /etc/ssl/etcd/etcd-client-ca.crt /etc/ssl/etcd/etcd/peer-ca.crt", - "sudo mv etcd-peer.crt /etc/ssl/etcd/etcd/peer.crt", - "sudo mv etcd-peer.key /etc/ssl/etcd/etcd/peer.key", - "sudo chown -R etcd:etcd /etc/ssl/etcd", - "sudo chmod -R 500 /etc/ssl/etcd", - "sudo mv $HOME/assets /opt/bootstrap/assets", - "sudo mkdir -p /etc/kubernetes/manifests", - "sudo mkdir -p /etc/kubernetes/bootstrap-secrets", - "sudo cp -r /opt/bootstrap/assets/tls/* /etc/kubernetes/bootstrap-secrets/", - "sudo cp /opt/bootstrap/assets/auth/kubeconfig /etc/kubernetes/bootstrap-secrets/", - "sudo cp -r /opt/bootstrap/assets/static-manifests/* /etc/kubernetes/manifests/" + "sudo /opt/bootstrap/layout", ] } } diff --git a/azure/container-linux/kubernetes/bootstrap.tf b/azure/container-linux/kubernetes/bootstrap.tf index b82a5dd0b..e26c8cf21 100644 --- a/azure/container-linux/kubernetes/bootstrap.tf +++ b/azure/container-linux/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=0f1f16c612a6877d25a3fedcb476b3087a3de999" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=dce49114a083436c5af6d9174b9c1248786ba3b6" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/azure/container-linux/kubernetes/cl/controller.yaml.tmpl b/azure/container-linux/kubernetes/cl/controller.yaml.tmpl index 491350685..5e83e996a 100644 --- a/azure/container-linux/kubernetes/cl/controller.yaml.tmpl +++ b/azure/container-linux/kubernetes/cl/controller.yaml.tmpl @@ -106,6 +106,8 @@ systemd: ExecStartPre=-/usr/bin/bash -c 'set -x && [ -n "$(ls /opt/bootstrap/assets/manifests-*/* 2>/dev/null)" ] && mv /opt/bootstrap/assets/manifests-*/* /opt/bootstrap/assets/manifests && rm -rf /opt/bootstrap/assets/manifests-*' ExecStart=/usr/bin/rkt run \ --trust-keys-from-https \ + --volume config,kind=host,source=/etc/kubernetes/bootstrap-secrets \ + --mount volume=config,target=/etc/kubernetes/secrets \ --volume assets,kind=host,source=/opt/bootstrap/assets \ --mount volume=assets,target=/assets \ --volume script,kind=host,source=/opt/bootstrap/apply \ @@ -133,13 +135,35 @@ storage: inline: | KUBELET_IMAGE_URL=docker://k8s.gcr.io/hyperkube KUBELET_IMAGE_TAG=v1.16.3 + - path: /opt/bootstrap/layout + filesystem: root + mode: 0544 + contents: + inline: | + #!/bin/bash -e + mkdir -p -- auth tls/etcd tls/k8s static-manifests manifests/coredns manifests-networking + awk '/#####/ {filename=$2; next} {print > filename}' assets + mkdir -p /etc/ssl/etcd/etcd + mkdir -p /etc/kubernetes/bootstrap-secrets + mv tls/etcd/{peer*,server*} /etc/ssl/etcd/etcd/ + mv tls/etcd/etcd-client* /etc/kubernetes/bootstrap-secrets/ + chown -R etcd:etcd /etc/ssl/etcd + chmod -R 500 /etc/ssl/etcd + mv auth/kubeconfig /etc/kubernetes/bootstrap-secrets/ + mv tls/k8s/* /etc/kubernetes/bootstrap-secrets/ + sudo mkdir -p /etc/kubernetes/manifests + sudo mv static-manifests/* /etc/kubernetes/manifests/ + sudo mkdir -p /opt/bootstrap/assets + sudo mv manifests /opt/bootstrap/assets/manifests + sudo mv manifests-networking /opt/bootstrap/assets/manifests-networking + rm -rf assets auth static-manifests tls - path: /opt/bootstrap/apply filesystem: root mode: 0544 contents: inline: | #!/bin/bash -e - export KUBECONFIG=/assets/auth/kubeconfig + export KUBECONFIG=/etc/kubernetes/secrets/kubeconfig until kubectl version; do echo "Waiting for static pod control plane" sleep 5 diff --git a/azure/container-linux/kubernetes/ssh.tf b/azure/container-linux/kubernetes/ssh.tf index 2f0507f61..f93e096dd 100644 --- a/azure/container-linux/kubernetes/ssh.tf +++ b/azure/container-linux/kubernetes/ssh.tf @@ -1,3 +1,12 @@ +locals { + # format assets for distribution + assets_bundle = [ + # header with the unpack location + for key, value in module.bootstrap.assets_dist: + format("##### %s\n%s", key, value) + ] +} + # Secure copy assets to controllers. resource "null_resource" "copy-controller-secrets" { count = var.controller_count @@ -13,65 +22,15 @@ resource "null_resource" "copy-controller-secrets" { user = "core" timeout = "15m" } - - provisioner "file" { - content = module.bootstrap.etcd_ca_cert - destination = "$HOME/etcd-client-ca.crt" - } - - provisioner "file" { - content = module.bootstrap.etcd_client_cert - destination = "$HOME/etcd-client.crt" - } - - provisioner "file" { - content = module.bootstrap.etcd_client_key - destination = "$HOME/etcd-client.key" - } - - provisioner "file" { - content = module.bootstrap.etcd_server_cert - destination = "$HOME/etcd-server.crt" - } - - provisioner "file" { - content = module.bootstrap.etcd_server_key - destination = "$HOME/etcd-server.key" - } - - provisioner "file" { - content = module.bootstrap.etcd_peer_cert - destination = "$HOME/etcd-peer.crt" - } - - provisioner "file" { - content = module.bootstrap.etcd_peer_key - destination = "$HOME/etcd-peer.key" - } - + provisioner "file" { - source = var.asset_dir + content = join("\n", local.assets_bundle) destination = "$HOME/assets" } provisioner "remote-exec" { inline = [ - "sudo mkdir -p /etc/ssl/etcd/etcd", - "sudo mv etcd-client* /etc/ssl/etcd/", - "sudo cp /etc/ssl/etcd/etcd-client-ca.crt /etc/ssl/etcd/etcd/server-ca.crt", - "sudo mv etcd-server.crt /etc/ssl/etcd/etcd/server.crt", - "sudo mv etcd-server.key /etc/ssl/etcd/etcd/server.key", - "sudo cp /etc/ssl/etcd/etcd-client-ca.crt /etc/ssl/etcd/etcd/peer-ca.crt", - "sudo mv etcd-peer.crt /etc/ssl/etcd/etcd/peer.crt", - "sudo mv etcd-peer.key /etc/ssl/etcd/etcd/peer.key", - "sudo chown -R etcd:etcd /etc/ssl/etcd", - "sudo chmod -R 500 /etc/ssl/etcd", - "sudo mv $HOME/assets /opt/bootstrap/assets", - "sudo mkdir -p /etc/kubernetes/manifests", - "sudo mkdir -p /etc/kubernetes/bootstrap-secrets", - "sudo cp -r /opt/bootstrap/assets/tls/* /etc/kubernetes/bootstrap-secrets/", - "sudo cp /opt/bootstrap/assets/auth/kubeconfig /etc/kubernetes/bootstrap-secrets/", - "sudo cp -r /opt/bootstrap/assets/static-manifests/* /etc/kubernetes/manifests/", + "sudo /opt/bootstrap/layout", ] } } diff --git a/bare-metal/container-linux/kubernetes/bootstrap.tf b/bare-metal/container-linux/kubernetes/bootstrap.tf index f04bc00e5..cb751a163 100644 --- a/bare-metal/container-linux/kubernetes/bootstrap.tf +++ b/bare-metal/container-linux/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=0f1f16c612a6877d25a3fedcb476b3087a3de999" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=dce49114a083436c5af6d9174b9c1248786ba3b6" cluster_name = var.cluster_name api_servers = [var.k8s_domain_name] diff --git a/bare-metal/container-linux/kubernetes/cl/controller.yaml.tmpl b/bare-metal/container-linux/kubernetes/cl/controller.yaml.tmpl index a6c1beced..2a6b32e49 100644 --- a/bare-metal/container-linux/kubernetes/cl/controller.yaml.tmpl +++ b/bare-metal/container-linux/kubernetes/cl/controller.yaml.tmpl @@ -121,6 +121,8 @@ systemd: ExecStartPre=-/usr/bin/bash -c 'set -x && [ -n "$(ls /opt/bootstrap/assets/manifests-*/* 2>/dev/null)" ] && mv /opt/bootstrap/assets/manifests-*/* /opt/bootstrap/assets/manifests && rm -rf /opt/bootstrap/assets/manifests-*' ExecStart=/usr/bin/rkt run \ --trust-keys-from-https \ + --volume config,kind=host,source=/etc/kubernetes/bootstrap-secrets \ + --mount volume=config,target=/etc/kubernetes/secrets \ --volume assets,kind=host,source=/opt/bootstrap/assets \ --mount volume=assets,target=/assets \ --volume script,kind=host,source=/opt/bootstrap/apply \ @@ -148,13 +150,35 @@ storage: contents: inline: ${domain_name} + - path: /opt/bootstrap/layout + filesystem: root + mode: 0544 + contents: + inline: | + #!/bin/bash -e + mkdir -p -- auth tls/etcd tls/k8s static-manifests manifests/coredns manifests-networking + awk '/#####/ {filename=$2; next} {print > filename}' assets + mkdir -p /etc/ssl/etcd/etcd + mkdir -p /etc/kubernetes/bootstrap-secrets + mv tls/etcd/{peer*,server*} /etc/ssl/etcd/etcd/ + mv tls/etcd/etcd-client* /etc/kubernetes/bootstrap-secrets/ + chown -R etcd:etcd /etc/ssl/etcd + chmod -R 500 /etc/ssl/etcd + mv auth/kubeconfig /etc/kubernetes/bootstrap-secrets/ + mv tls/k8s/* /etc/kubernetes/bootstrap-secrets/ + sudo mkdir -p /etc/kubernetes/manifests + sudo mv static-manifests/* /etc/kubernetes/manifests/ + sudo mkdir -p /opt/bootstrap/assets + sudo mv manifests /opt/bootstrap/assets/manifests + sudo mv manifests-networking /opt/bootstrap/assets/manifests-networking + rm -rf assets auth static-manifests tls - path: /opt/bootstrap/apply filesystem: root mode: 0544 contents: inline: | #!/bin/bash -e - export KUBECONFIG=/assets/auth/kubeconfig + export KUBECONFIG=/etc/kubernetes/secrets/kubeconfig until kubectl version; do echo "Waiting for static pod control plane" sleep 5 diff --git a/bare-metal/container-linux/kubernetes/ssh.tf b/bare-metal/container-linux/kubernetes/ssh.tf index 5edea88e4..fcab47358 100644 --- a/bare-metal/container-linux/kubernetes/ssh.tf +++ b/bare-metal/container-linux/kubernetes/ssh.tf @@ -1,3 +1,12 @@ +locals { + # format assets for distribution + assets_bundle = [ + # header with the unpack location + for key, value in module.bootstrap.assets_dist: + format("##### %s\n%s", key, value) + ] +} + # Secure copy assets to controllers. Activates kubelet.service resource "null_resource" "copy-controller-secrets" { count = length(var.controllers) @@ -24,64 +33,14 @@ resource "null_resource" "copy-controller-secrets" { } provisioner "file" { - content = module.bootstrap.etcd_ca_cert - destination = "$HOME/etcd-client-ca.crt" - } - - provisioner "file" { - content = module.bootstrap.etcd_client_cert - destination = "$HOME/etcd-client.crt" - } - - provisioner "file" { - content = module.bootstrap.etcd_client_key - destination = "$HOME/etcd-client.key" - } - - provisioner "file" { - content = module.bootstrap.etcd_server_cert - destination = "$HOME/etcd-server.crt" - } - - provisioner "file" { - content = module.bootstrap.etcd_server_key - destination = "$HOME/etcd-server.key" - } - - provisioner "file" { - content = module.bootstrap.etcd_peer_cert - destination = "$HOME/etcd-peer.crt" - } - - provisioner "file" { - content = module.bootstrap.etcd_peer_key - destination = "$HOME/etcd-peer.key" - } - - provisioner "file" { - source = var.asset_dir + content = join("\n", local.assets_bundle) destination = "$HOME/assets" } provisioner "remote-exec" { inline = [ - "sudo mkdir -p /etc/ssl/etcd/etcd", - "sudo mv etcd-client* /etc/ssl/etcd/", - "sudo cp /etc/ssl/etcd/etcd-client-ca.crt /etc/ssl/etcd/etcd/server-ca.crt", - "sudo mv etcd-server.crt /etc/ssl/etcd/etcd/server.crt", - "sudo mv etcd-server.key /etc/ssl/etcd/etcd/server.key", - "sudo cp /etc/ssl/etcd/etcd-client-ca.crt /etc/ssl/etcd/etcd/peer-ca.crt", - "sudo mv etcd-peer.crt /etc/ssl/etcd/etcd/peer.crt", - "sudo mv etcd-peer.key /etc/ssl/etcd/etcd/peer.key", - "sudo chown -R etcd:etcd /etc/ssl/etcd", - "sudo chmod -R 500 /etc/ssl/etcd", - "sudo mv $HOME/assets /opt/bootstrap/assets", - "sudo mkdir -p /etc/kubernetes/manifests", - "sudo mkdir -p /etc/kubernetes/bootstrap-secrets", "sudo mv $HOME/kubeconfig /etc/kubernetes/kubeconfig", - "sudo cp -r /opt/bootstrap/assets/tls/* /etc/kubernetes/bootstrap-secrets/", - "sudo cp /opt/bootstrap/assets/auth/kubeconfig /etc/kubernetes/bootstrap-secrets/", - "sudo cp -r /opt/bootstrap/assets/static-manifests/* /etc/kubernetes/manifests/", + "sudo /opt/bootstrap/layout", ] } } diff --git a/bare-metal/fedora-coreos/kubernetes/bootstrap.tf b/bare-metal/fedora-coreos/kubernetes/bootstrap.tf index eeb55a8b3..d04849434 100644 --- a/bare-metal/fedora-coreos/kubernetes/bootstrap.tf +++ b/bare-metal/fedora-coreos/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=0f1f16c612a6877d25a3fedcb476b3087a3de999" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=dce49114a083436c5af6d9174b9c1248786ba3b6" cluster_name = var.cluster_name api_servers = [var.k8s_domain_name] diff --git a/bare-metal/fedora-coreos/kubernetes/fcc/controller.yaml b/bare-metal/fedora-coreos/kubernetes/fcc/controller.yaml index a88013b8d..2132bb247 100644 --- a/bare-metal/fedora-coreos/kubernetes/fcc/controller.yaml +++ b/bare-metal/fedora-coreos/kubernetes/fcc/controller.yaml @@ -130,6 +130,7 @@ systemd: ExecStartPre=-/usr/bin/bash -c 'set -x && [ -n "$(ls /opt/bootstrap/assets/manifests-*/* 2>/dev/null)" ] && mv /opt/bootstrap/assets/manifests-*/* /opt/bootstrap/assets/manifests && rm -rf /opt/bootstrap/assets/manifests-*' ExecStart=/usr/bin/podman run --name bootstrap \ --network host \ + --volume /etc/kubernetes/bootstrap-secrets:/etc/kubernetes/secrets:ro,Z \ --volume /opt/bootstrap/assets:/assets:ro,Z \ --volume /opt/bootstrap/apply:/apply:ro,Z \ k8s.gcr.io/hyperkube:v1.16.3 \ @@ -146,12 +147,33 @@ storage: contents: inline: ${domain_name} + - path: /opt/bootstrap/layout + mode: 0544 + contents: + inline: | + #!/bin/bash -e + mkdir -p -- auth tls/etcd tls/k8s static-manifests manifests/coredns manifests-networking + awk '/#####/ {filename=$2; next} {print > filename}' assets + mkdir -p /etc/ssl/etcd/etcd + mkdir -p /etc/kubernetes/bootstrap-secrets + mv tls/etcd/{peer*,server*} /etc/ssl/etcd/etcd/ + mv tls/etcd/etcd-client* /etc/kubernetes/bootstrap-secrets/ + chown -R etcd:etcd /etc/ssl/etcd + chmod -R 500 /etc/ssl/etcd + mv auth/kubeconfig /etc/kubernetes/bootstrap-secrets/ + mv tls/k8s/* /etc/kubernetes/bootstrap-secrets/ + sudo mkdir -p /etc/kubernetes/manifests + sudo mv static-manifests/* /etc/kubernetes/manifests/ + sudo mkdir -p /opt/bootstrap/assets + sudo mv manifests /opt/bootstrap/assets/manifests + sudo mv manifests-networking /opt/bootstrap/assets/manifests-networking + rm -rf assets auth static-manifests tls - path: /opt/bootstrap/apply mode: 0544 contents: inline: | #!/bin/bash -e - export KUBECONFIG=/assets/auth/kubeconfig + export KUBECONFIG=/etc/kubernetes/secrets/kubeconfig until kubectl version; do echo "Waiting for static pod control plane" sleep 5 diff --git a/bare-metal/fedora-coreos/kubernetes/ssh.tf b/bare-metal/fedora-coreos/kubernetes/ssh.tf index 8ad6d0d1e..560d96fe8 100644 --- a/bare-metal/fedora-coreos/kubernetes/ssh.tf +++ b/bare-metal/fedora-coreos/kubernetes/ssh.tf @@ -1,3 +1,12 @@ +locals { + # format assets for distribution + assets_bundle = [ + # header with the unpack location + for key, value in module.bootstrap.assets_dist: + format("##### %s\n%s", key, value) + ] +} + # Secure copy assets to controllers. Activates kubelet.service resource "null_resource" "copy-controller-secrets" { count = length(var.controllers) @@ -23,62 +32,14 @@ resource "null_resource" "copy-controller-secrets" { } provisioner "file" { - content = module.bootstrap.etcd_ca_cert - destination = "$HOME/etcd-client-ca.crt" - } - - provisioner "file" { - content = module.bootstrap.etcd_client_cert - destination = "$HOME/etcd-client.crt" - } - - provisioner "file" { - content = module.bootstrap.etcd_client_key - destination = "$HOME/etcd-client.key" - } - - provisioner "file" { - content = module.bootstrap.etcd_server_cert - destination = "$HOME/etcd-server.crt" - } - - provisioner "file" { - content = module.bootstrap.etcd_server_key - destination = "$HOME/etcd-server.key" - } - - provisioner "file" { - content = module.bootstrap.etcd_peer_cert - destination = "$HOME/etcd-peer.crt" - } - - provisioner "file" { - content = module.bootstrap.etcd_peer_key - destination = "$HOME/etcd-peer.key" - } - - provisioner "file" { - source = var.asset_dir + content = join("\n", local.assets_bundle) destination = "$HOME/assets" } provisioner "remote-exec" { inline = [ - "sudo mkdir -p /etc/ssl/etcd/etcd", - "sudo mv etcd-client* /etc/ssl/etcd/", - "sudo cp /etc/ssl/etcd/etcd-client-ca.crt /etc/ssl/etcd/etcd/server-ca.crt", - "sudo mv etcd-server.crt /etc/ssl/etcd/etcd/server.crt", - "sudo mv etcd-server.key /etc/ssl/etcd/etcd/server.key", - "sudo cp /etc/ssl/etcd/etcd-client-ca.crt /etc/ssl/etcd/etcd/peer-ca.crt", - "sudo mv etcd-peer.crt /etc/ssl/etcd/etcd/peer.crt", - "sudo mv etcd-peer.key /etc/ssl/etcd/etcd/peer.key", - "sudo mv $HOME/assets /opt/bootstrap/assets", - "sudo mkdir -p /etc/kubernetes/manifests", - "sudo mkdir -p /etc/kubernetes/bootstrap-secrets", "sudo mv $HOME/kubeconfig /etc/kubernetes/kubeconfig", - "sudo cp -r /opt/bootstrap/assets/tls/* /etc/kubernetes/bootstrap-secrets/", - "sudo cp /opt/bootstrap/assets/auth/kubeconfig /etc/kubernetes/bootstrap-secrets/", - "sudo cp -r /opt/bootstrap/assets/static-manifests/* /etc/kubernetes/manifests/" + "sudo /opt/bootstrap/layout", ] } } diff --git a/digital-ocean/container-linux/kubernetes/bootstrap.tf b/digital-ocean/container-linux/kubernetes/bootstrap.tf index 9a762a5c1..196ae533d 100644 --- a/digital-ocean/container-linux/kubernetes/bootstrap.tf +++ b/digital-ocean/container-linux/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=0f1f16c612a6877d25a3fedcb476b3087a3de999" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=dce49114a083436c5af6d9174b9c1248786ba3b6" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/digital-ocean/container-linux/kubernetes/cl/controller.yaml.tmpl b/digital-ocean/container-linux/kubernetes/cl/controller.yaml.tmpl index ffc8cf329..8c94dd65a 100644 --- a/digital-ocean/container-linux/kubernetes/cl/controller.yaml.tmpl +++ b/digital-ocean/container-linux/kubernetes/cl/controller.yaml.tmpl @@ -118,6 +118,8 @@ systemd: ExecStartPre=-/usr/bin/bash -c 'set -x && [ -n "$(ls /opt/bootstrap/assets/manifests-*/* 2>/dev/null)" ] && mv /opt/bootstrap/assets/manifests-*/* /opt/bootstrap/assets/manifests && rm -rf /opt/bootstrap/assets/manifests-*' ExecStart=/usr/bin/rkt run \ --trust-keys-from-https \ + --volume config,kind=host,source=/etc/kubernetes/bootstrap-secrets \ + --mount volume=config,target=/etc/kubernetes/secrets \ --volume assets,kind=host,source=/opt/bootstrap/assets \ --mount volume=assets,target=/assets \ --volume script,kind=host,source=/opt/bootstrap/apply \ @@ -139,13 +141,35 @@ storage: inline: | KUBELET_IMAGE_URL=docker://k8s.gcr.io/hyperkube KUBELET_IMAGE_TAG=v1.16.3 + - path: /opt/bootstrap/layout + filesystem: root + mode: 0544 + contents: + inline: | + #!/bin/bash -e + mkdir -p -- auth tls/etcd tls/k8s static-manifests manifests/coredns manifests-networking + awk '/#####/ {filename=$2; next} {print > filename}' assets + mkdir -p /etc/ssl/etcd/etcd + mkdir -p /etc/kubernetes/bootstrap-secrets + mv tls/etcd/{peer*,server*} /etc/ssl/etcd/etcd/ + mv tls/etcd/etcd-client* /etc/kubernetes/bootstrap-secrets/ + chown -R etcd:etcd /etc/ssl/etcd + chmod -R 500 /etc/ssl/etcd + mv auth/kubeconfig /etc/kubernetes/bootstrap-secrets/ + mv tls/k8s/* /etc/kubernetes/bootstrap-secrets/ + sudo mkdir -p /etc/kubernetes/manifests + sudo mv static-manifests/* /etc/kubernetes/manifests/ + sudo mkdir -p /opt/bootstrap/assets + sudo mv manifests /opt/bootstrap/assets/manifests + sudo mv manifests-networking /opt/bootstrap/assets/manifests-networking + rm -rf assets auth static-manifests tls - path: /opt/bootstrap/apply filesystem: root mode: 0544 contents: inline: | #!/bin/bash -e - export KUBECONFIG=/assets/auth/kubeconfig + export KUBECONFIG=/etc/kubernetes/secrets/kubeconfig until kubectl version; do echo "Waiting for static pod control plane" sleep 5 diff --git a/digital-ocean/container-linux/kubernetes/ssh.tf b/digital-ocean/container-linux/kubernetes/ssh.tf index 12f5bcb90..469fc947d 100644 --- a/digital-ocean/container-linux/kubernetes/ssh.tf +++ b/digital-ocean/container-linux/kubernetes/ssh.tf @@ -1,3 +1,12 @@ +locals { + # format assets for distribution + assets_bundle = [ + # header with the unpack location + for key, value in module.bootstrap.assets_dist: + format("##### %s\n%s", key, value) + ] +} + # Secure copy assets to controllers. Activates kubelet.service resource "null_resource" "copy-controller-secrets" { count = var.controller_count @@ -20,64 +29,14 @@ resource "null_resource" "copy-controller-secrets" { } provisioner "file" { - content = module.bootstrap.etcd_ca_cert - destination = "$HOME/etcd-client-ca.crt" - } - - provisioner "file" { - content = module.bootstrap.etcd_client_cert - destination = "$HOME/etcd-client.crt" - } - - provisioner "file" { - content = module.bootstrap.etcd_client_key - destination = "$HOME/etcd-client.key" - } - - provisioner "file" { - content = module.bootstrap.etcd_server_cert - destination = "$HOME/etcd-server.crt" - } - - provisioner "file" { - content = module.bootstrap.etcd_server_key - destination = "$HOME/etcd-server.key" - } - - provisioner "file" { - content = module.bootstrap.etcd_peer_cert - destination = "$HOME/etcd-peer.crt" - } - - provisioner "file" { - content = module.bootstrap.etcd_peer_key - destination = "$HOME/etcd-peer.key" - } - - provisioner "file" { - source = var.asset_dir + content = join("\n", local.assets_bundle) destination = "$HOME/assets" } provisioner "remote-exec" { inline = [ - "sudo mkdir -p /etc/ssl/etcd/etcd", - "sudo mv etcd-client* /etc/ssl/etcd/", - "sudo cp /etc/ssl/etcd/etcd-client-ca.crt /etc/ssl/etcd/etcd/server-ca.crt", - "sudo mv etcd-server.crt /etc/ssl/etcd/etcd/server.crt", - "sudo mv etcd-server.key /etc/ssl/etcd/etcd/server.key", - "sudo cp /etc/ssl/etcd/etcd-client-ca.crt /etc/ssl/etcd/etcd/peer-ca.crt", - "sudo mv etcd-peer.crt /etc/ssl/etcd/etcd/peer.crt", - "sudo mv etcd-peer.key /etc/ssl/etcd/etcd/peer.key", - "sudo chown -R etcd:etcd /etc/ssl/etcd", - "sudo chmod -R 500 /etc/ssl/etcd", - "sudo mv $HOME/assets /opt/bootstrap/assets", - "sudo mkdir -p /etc/kubernetes/manifests", - "sudo mkdir -p /etc/kubernetes/bootstrap-secrets", "sudo mv $HOME/kubeconfig /etc/kubernetes/kubeconfig", - "sudo cp -r /opt/bootstrap/assets/tls/* /etc/kubernetes/bootstrap-secrets/", - "sudo cp /opt/bootstrap/assets/auth/kubeconfig /etc/kubernetes/bootstrap-secrets/", - "sudo cp -r /opt/bootstrap/assets/static-manifests/* /etc/kubernetes/manifests/", + "sudo /opt/bootstrap/layout", ] } } diff --git a/google-cloud/container-linux/kubernetes/bootstrap.tf b/google-cloud/container-linux/kubernetes/bootstrap.tf index 157944f53..36ae40e33 100644 --- a/google-cloud/container-linux/kubernetes/bootstrap.tf +++ b/google-cloud/container-linux/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=0f1f16c612a6877d25a3fedcb476b3087a3de999" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=dce49114a083436c5af6d9174b9c1248786ba3b6" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/google-cloud/container-linux/kubernetes/cl/controller.yaml.tmpl b/google-cloud/container-linux/kubernetes/cl/controller.yaml.tmpl index 49ed0fc25..caf120ee9 100644 --- a/google-cloud/container-linux/kubernetes/cl/controller.yaml.tmpl +++ b/google-cloud/container-linux/kubernetes/cl/controller.yaml.tmpl @@ -107,6 +107,8 @@ systemd: ExecStartPre=-/usr/bin/bash -c 'set -x && [ -n "$(ls /opt/bootstrap/assets/manifests-*/* 2>/dev/null)" ] && mv /opt/bootstrap/assets/manifests-*/* /opt/bootstrap/assets/manifests && rm -rf /opt/bootstrap/assets/manifests-*' ExecStart=/usr/bin/rkt run \ --trust-keys-from-https \ + --volume config,kind=host,source=/etc/kubernetes/bootstrap-secrets \ + --mount volume=config,target=/etc/kubernetes/secrets \ --volume assets,kind=host,source=/opt/bootstrap/assets \ --mount volume=assets,target=/assets \ --volume script,kind=host,source=/opt/bootstrap/apply \ @@ -134,13 +136,35 @@ storage: inline: | KUBELET_IMAGE_URL=docker://k8s.gcr.io/hyperkube KUBELET_IMAGE_TAG=v1.16.3 + - path: /opt/bootstrap/layout + filesystem: root + mode: 0544 + contents: + inline: | + #!/bin/bash -e + mkdir -p -- auth tls/etcd tls/k8s static-manifests manifests/coredns manifests-networking + awk '/#####/ {filename=$2; next} {print > filename}' assets + mkdir -p /etc/ssl/etcd/etcd + mkdir -p /etc/kubernetes/bootstrap-secrets + mv tls/etcd/{peer*,server*} /etc/ssl/etcd/etcd/ + mv tls/etcd/etcd-client* /etc/kubernetes/bootstrap-secrets/ + chown -R etcd:etcd /etc/ssl/etcd + chmod -R 500 /etc/ssl/etcd + mv auth/kubeconfig /etc/kubernetes/bootstrap-secrets/ + mv tls/k8s/* /etc/kubernetes/bootstrap-secrets/ + sudo mkdir -p /etc/kubernetes/manifests + sudo mv static-manifests/* /etc/kubernetes/manifests/ + sudo mkdir -p /opt/bootstrap/assets + sudo mv manifests /opt/bootstrap/assets/manifests + sudo mv manifests-networking /opt/bootstrap/assets/manifests-networking + rm -rf assets auth static-manifests tls - path: /opt/bootstrap/apply filesystem: root mode: 0544 contents: inline: | #!/bin/bash -e - export KUBECONFIG=/assets/auth/kubeconfig + export KUBECONFIG=/etc/kubernetes/secrets/kubeconfig until kubectl version; do echo "Waiting for static pod control plane" sleep 5 diff --git a/google-cloud/container-linux/kubernetes/ssh.tf b/google-cloud/container-linux/kubernetes/ssh.tf index a85e23d1e..f6983447e 100644 --- a/google-cloud/container-linux/kubernetes/ssh.tf +++ b/google-cloud/container-linux/kubernetes/ssh.tf @@ -1,3 +1,12 @@ +locals { + # format assets for distribution + assets_bundle = [ + # header with the unpack location + for key, value in module.bootstrap.assets_dist: + format("##### %s\n%s", key, value) + ] +} + # Secure copy assets to controllers. resource "null_resource" "copy-controller-secrets" { count = var.controller_count @@ -12,65 +21,15 @@ resource "null_resource" "copy-controller-secrets" { user = "core" timeout = "15m" } - - provisioner "file" { - content = module.bootstrap.etcd_ca_cert - destination = "$HOME/etcd-client-ca.crt" - } - - provisioner "file" { - content = module.bootstrap.etcd_client_cert - destination = "$HOME/etcd-client.crt" - } - - provisioner "file" { - content = module.bootstrap.etcd_client_key - destination = "$HOME/etcd-client.key" - } - - provisioner "file" { - content = module.bootstrap.etcd_server_cert - destination = "$HOME/etcd-server.crt" - } - - provisioner "file" { - content = module.bootstrap.etcd_server_key - destination = "$HOME/etcd-server.key" - } - - provisioner "file" { - content = module.bootstrap.etcd_peer_cert - destination = "$HOME/etcd-peer.crt" - } - - provisioner "file" { - content = module.bootstrap.etcd_peer_key - destination = "$HOME/etcd-peer.key" - } - + provisioner "file" { - source = var.asset_dir + content = join("\n", local.assets_bundle) destination = "$HOME/assets" } provisioner "remote-exec" { inline = [ - "sudo mkdir -p /etc/ssl/etcd/etcd", - "sudo mv etcd-client* /etc/ssl/etcd/", - "sudo cp /etc/ssl/etcd/etcd-client-ca.crt /etc/ssl/etcd/etcd/server-ca.crt", - "sudo mv etcd-server.crt /etc/ssl/etcd/etcd/server.crt", - "sudo mv etcd-server.key /etc/ssl/etcd/etcd/server.key", - "sudo cp /etc/ssl/etcd/etcd-client-ca.crt /etc/ssl/etcd/etcd/peer-ca.crt", - "sudo mv etcd-peer.crt /etc/ssl/etcd/etcd/peer.crt", - "sudo mv etcd-peer.key /etc/ssl/etcd/etcd/peer.key", - "sudo chown -R etcd:etcd /etc/ssl/etcd", - "sudo chmod -R 500 /etc/ssl/etcd", - "sudo mv $HOME/assets /opt/bootstrap/assets", - "sudo mkdir -p /etc/kubernetes/manifests", - "sudo mkdir -p /etc/kubernetes/bootstrap-secrets", - "sudo cp -r /opt/bootstrap/assets/tls/* /etc/kubernetes/bootstrap-secrets/", - "sudo cp /opt/bootstrap/assets/auth/kubeconfig /etc/kubernetes/bootstrap-secrets/", - "sudo cp -r /opt/bootstrap/assets/static-manifests/* /etc/kubernetes/manifests/", + "sudo /opt/bootstrap/layout", ] } } From d9c7a9e04908a9bc55b4fdca3f79b056ca9fb809 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Thu, 5 Dec 2019 22:56:42 -0800 Subject: [PATCH 281/523] Add/update docs for asset_dir and kubeconfig usage * Original tutorials favored including the platform (e.g. google-cloud) in modules (e.g. google-cloud-yavin). Prefer naming conventions where each module / cluster has a simple name (e.g. yavin) since the platform is usually redundant * Retain the example cluster naming themes per platform --- CHANGES.md | 8 +++- README.md | 11 +++-- aws/container-linux/kubernetes/variables.tf | 1 + aws/fedora-coreos/kubernetes/variables.tf | 1 + azure/container-linux/kubernetes/variables.tf | 1 + .../container-linux/kubernetes/variables.tf | 1 + .../fedora-coreos/kubernetes/variables.tf | 1 + .../container-linux/kubernetes/variables.tf | 1 + docs/addons/ingress.md | 10 ++--- docs/advanced/customization.md | 6 +-- docs/advanced/worker-pools.md | 24 +++++------ docs/architecture/concepts.md | 2 +- docs/cl/aws.md | 22 ++++++---- docs/cl/azure.md | 22 ++++++---- docs/cl/bare-metal.md | 40 +++++++++++-------- docs/cl/digital-ocean.md | 26 +++++++----- docs/cl/google-cloud.md | 26 +++++++----- docs/fedora-coreos/aws.md | 24 +++++++---- docs/fedora-coreos/bare-metal.md | 38 +++++++++++------- docs/index.md | 11 +++-- docs/topics/maintenance.md | 14 +++---- .../container-linux/kubernetes/variables.tf | 1 + 22 files changed, 185 insertions(+), 106 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index bb99f125c..9ce519d63 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -4,9 +4,15 @@ Notable changes between versions. ## Latest +* Manage clusters without using a local `asset_dir` ([#595](https://github.com/poseidon/typhoon/pull/595)) + * Change `asset_dir` to be optional. Default to "" to skip writing assets locally + * Keep cluster assets (TLS materials, kubeconfig) only in Terraform state, which supports different [remote backends](https://www.terraform.io/docs/backends/types/remote.html) and optional encryption at rest + * Allow `terraform apply` from stateless automation systems + * Improve asset unpacking on controllers to remove unused materials + * Obtain a kubeconfig from Terraform module outputs * Update CoreDNS from v1.6.2 to v1.6.5 ([#588](https://github.com/poseidon/typhoon/pull/588)) * Add health `lameduck` option to wait before shutdown -* Add CPU requests to control plane static pods ([#589](https://github.com/poseidon/typhoon/pull/589)) +* Add CPU requests for control plane static pods ([#589](https://github.com/poseidon/typhoon/pull/589)) * May provide slight edge case benefits and aligns with upstream * Replace usage of `template_dir` with `templatefile` function ([#587](https://github.com/poseidon/typhoon/pull/587)) * Require Terraform version v0.12.6+ (action required) diff --git a/README.md b/README.md index 92f310e90..b26355a40 100644 --- a/README.md +++ b/README.md @@ -47,7 +47,7 @@ A preview of Typhoon for [Fedora CoreOS](https://getfedora.org/coreos/) is avail Define a Kubernetes cluster by using the Terraform module for your chosen platform and operating system. Here's a minimal example: ```tf -module "google-cloud-yavin" { +module "yavin" { source = "git::https://github.com/poseidon/typhoon//google-cloud/container-linux/kubernetes?ref=v1.16.3" # Google Cloud @@ -58,12 +58,17 @@ module "google-cloud-yavin" { # configuration ssh_authorized_key = "ssh-rsa AAAAB3Nz..." - asset_dir = "/home/user/.secrets/clusters/yavin" # optional worker_count = 2 worker_preemptible = true } + +# Obtain cluster kubeconfig +resource "local_file" "kubeconfig-yavin" { + content = module.yavin.kubeconfig-admin + filename = "/home/user/.kube/configs/yavin-config" +} ``` Initialize modules, plan the changes to be made, and apply the changes. @@ -79,7 +84,7 @@ Apply complete! Resources: 64 added, 0 changed, 0 destroyed. In 4-8 minutes (varies by platform), the cluster will be ready. This Google Cloud example creates a `yavin.example.com` DNS record to resolve to a network load balancer across controller nodes. ```sh -$ export KUBECONFIG=/home/user/.secrets/clusters/yavin/auth/kubeconfig +$ export KUBECONFIG=/home/user/.kube/configs/yavin-config $ kubectl get nodes NAME ROLES STATUS AGE VERSION yavin-controller-0.c.example-com.internal Ready 6m v1.16.3 diff --git a/aws/container-linux/kubernetes/variables.tf b/aws/container-linux/kubernetes/variables.tf index b0189ce2f..21c41ff1a 100644 --- a/aws/container-linux/kubernetes/variables.tf +++ b/aws/container-linux/kubernetes/variables.tf @@ -99,6 +99,7 @@ variable "ssh_authorized_key" { variable "asset_dir" { type = string description = "Absolute path to a directory where generated assets should be placed (contains secrets)" + default = "" } variable "networking" { diff --git a/aws/fedora-coreos/kubernetes/variables.tf b/aws/fedora-coreos/kubernetes/variables.tf index 5d48f19e9..890ad2887 100644 --- a/aws/fedora-coreos/kubernetes/variables.tf +++ b/aws/fedora-coreos/kubernetes/variables.tf @@ -99,6 +99,7 @@ variable "ssh_authorized_key" { variable "asset_dir" { type = string description = "Absolute path to a directory where generated assets should be placed (contains secrets)" + default = "" } variable "networking" { diff --git a/azure/container-linux/kubernetes/variables.tf b/azure/container-linux/kubernetes/variables.tf index 2102454fb..c86ab9aa1 100644 --- a/azure/container-linux/kubernetes/variables.tf +++ b/azure/container-linux/kubernetes/variables.tf @@ -86,6 +86,7 @@ variable "ssh_authorized_key" { variable "asset_dir" { type = string description = "Absolute path to a directory where generated assets should be placed (contains secrets)" + default = "" } variable "networking" { diff --git a/bare-metal/container-linux/kubernetes/variables.tf b/bare-metal/container-linux/kubernetes/variables.tf index 4ebdbf064..80dd70bcf 100644 --- a/bare-metal/container-linux/kubernetes/variables.tf +++ b/bare-metal/container-linux/kubernetes/variables.tf @@ -70,6 +70,7 @@ variable "ssh_authorized_key" { variable "asset_dir" { type = string description = "Absolute path to a directory where generated assets should be placed (contains secrets)" + default = "" } variable "networking" { diff --git a/bare-metal/fedora-coreos/kubernetes/variables.tf b/bare-metal/fedora-coreos/kubernetes/variables.tf index 159c751ca..8e956754b 100644 --- a/bare-metal/fedora-coreos/kubernetes/variables.tf +++ b/bare-metal/fedora-coreos/kubernetes/variables.tf @@ -71,6 +71,7 @@ variable "ssh_authorized_key" { variable "asset_dir" { type = string description = "Absolute path to a directory where generated assets should be placed (contains secrets)" + default = "" } variable "networking" { diff --git a/digital-ocean/container-linux/kubernetes/variables.tf b/digital-ocean/container-linux/kubernetes/variables.tf index b347e3cf4..0c25d379d 100644 --- a/digital-ocean/container-linux/kubernetes/variables.tf +++ b/digital-ocean/container-linux/kubernetes/variables.tf @@ -69,6 +69,7 @@ variable "ssh_fingerprints" { variable "asset_dir" { type = string description = "Absolute path to a directory where generated assets should be placed (contains secrets)" + default = "" } variable "networking" { diff --git a/docs/addons/ingress.md b/docs/addons/ingress.md index 4bbbfbe1a..14689c370 100644 --- a/docs/addons/ingress.md +++ b/docs/addons/ingress.md @@ -31,7 +31,7 @@ resource "google_dns_record_set" "some-application" { name = "app.example.com." type = "CNAME" ttl = 300 - rrdatas = ["${module.aws-tempest.ingress_dns_name}."] + rrdatas = ["${module.tempest.ingress_dns_name}."] } ``` @@ -64,7 +64,7 @@ resource "google_dns_record_set" "some-application" { name = "app.example.com." type = "A" ttl = 300 - rrdatas = [module.azure-ramius.ingress_static_ipv4] + rrdatas = [module.ramius.ingress_static_ipv4] } ``` @@ -120,7 +120,7 @@ resource "google_dns_record_set" "some-application" { name = "app.example.com." type = "CNAME" ttl = 300 - rrdatas = ["${module.digital-ocean-nemo.workers_dns}."] + rrdatas = ["${module.nemo.workers_dns}."] } ``` @@ -158,7 +158,7 @@ resource "google_dns_record_set" "app-record-a" { name = "app.example.com." type = "A" ttl = 300 - rrdatas = [module.google-cloud-yavin.ingress_static_ipv4] + rrdatas = [module.yavin.ingress_static_ipv4] } resource "google_dns_record_set" "app-record-aaaa" { @@ -169,6 +169,6 @@ resource "google_dns_record_set" "app-record-aaaa" { name = "app.example.com." type = "AAAA" ttl = 300 - rrdatas = [module.google-cloud-yavin.ingress_static_ipv6] + rrdatas = [module.yavin.ingress_static_ipv6] } ``` diff --git a/docs/advanced/customization.md b/docs/advanced/customization.md index e9ad8de74..2fd3373fb 100644 --- a/docs/advanced/customization.md +++ b/docs/advanced/customization.md @@ -72,7 +72,7 @@ Write Container Linux Configs *snippets* as files in the repository where you ke [AWS](/cl/aws/#cluster), [Azure](/cl/azure/#cluster), [DigitalOcean](/cl/digital-ocean/#cluster), and [Google Cloud](/cl/google-cloud/#cluster) clusters allow populating a list of `controller_clc_snippets` or `worker_clc_snippets`. ``` -module "digital-ocean-nemo" { +module "nemo" { ... controller_count = 1 @@ -92,7 +92,7 @@ module "digital-ocean-nemo" { [Bare-Metal](/cl/bare-metal/#cluster) clusters allow different Container Linux snippets to be used for each node (since hardware may be heterogeneous). Populate the optional `clc_snippets` map variable with any controller or worker name keys and lists of snippets. ``` -module "bare-metal-mercury" { +module "mercury" { ... controller_names = ["node1"] worker_names = [ @@ -141,7 +141,7 @@ Container Linux Configs (and the CoreOS Ignition system) create immutable infras Typhoon chooses variables to expose with purpose. If you must customize clusters in ways that aren't supported by input variables, fork Typhoon and maintain a repository with customizations. Reference the repository by changing the username. ``` -module "digital-ocean-nemo" { +module "nemo" { source = "git::https://github.com/USERNAME/typhoon//digital-ocean/container-linux/kubernetes?ref=myspecialcase" ... } diff --git a/docs/advanced/worker-pools.md b/docs/advanced/worker-pools.md index 3eeb4edb6..645fc3ee6 100644 --- a/docs/advanced/worker-pools.md +++ b/docs/advanced/worker-pools.md @@ -17,13 +17,13 @@ module "tempest-worker-pool" { source = "git::https://github.com/poseidon/typhoon//aws/container-linux/kubernetes/workers?ref=v1.14.3" # AWS - vpc_id = module.aws-tempest.vpc_id - subnet_ids = module.aws-tempest.subnet_ids - security_groups = module.aws-tempest.worker_security_groups + vpc_id = module.tempest.vpc_id + subnet_ids = module.tempest.subnet_ids + security_groups = module.tempest.worker_security_groups # configuration name = "tempest-pool" - kubeconfig = module.aws-tempest.kubeconfig + kubeconfig = module.tempest.kubeconfig ssh_authorized_key = var.ssh_authorized_key # optional @@ -82,15 +82,15 @@ module "ramius-worker-pool" { source = "git::https://github.com/poseidon/typhoon//azure/container-linux/kubernetes/workers?ref=v1.16.3" # Azure - region = module.azure-ramius.region - resource_group_name = module.azure-ramius.resource_group_name - subnet_id = module.azure-ramius.subnet_id - security_group_id = module.azure-ramius.security_group_id - backend_address_pool_id = module.azure-ramius.backend_address_pool_id + region = module.ramius.region + resource_group_name = module.ramius.resource_group_name + subnet_id = module.ramius.subnet_id + security_group_id = module.ramius.security_group_id + backend_address_pool_id = module.ramius.backend_address_pool_id # configuration name = "ramius-low-priority" - kubeconfig = module.azure-ramius.kubeconfig + kubeconfig = module.ramius.kubeconfig ssh_authorized_key = var.ssh_authorized_key # optional @@ -149,12 +149,12 @@ module "yavin-worker-pool" { # Google Cloud region = "europe-west2" - network = module.google-cloud-yavin.network_name + network = module.yavin.network_name cluster_name = "yavin" # configuration name = "yavin-16x" - kubeconfig = module.google-cloud-yavin.kubeconfig + kubeconfig = module.yavin.kubeconfig ssh_authorized_key = var.ssh_authorized_key # optional diff --git a/docs/architecture/concepts.md b/docs/architecture/concepts.md index 255ab02f4..dda1a6004 100644 --- a/docs/architecture/concepts.md +++ b/docs/architecture/concepts.md @@ -47,7 +47,7 @@ Terraform [modules](https://www.terraform.io/docs/modules/usage.html) allow a co Clusters are declared in Terraform by referencing the module. ```tf -module "google-cloud-yavin" { +module "yavin" { source = "git::https://github.com/poseidon/typhoon//google-cloud/container-linux/kubernetes" cluster_name = "yavin" ... diff --git a/docs/cl/aws.md b/docs/cl/aws.md index 3bf2244ef..52c046dcc 100644 --- a/docs/cl/aws.md +++ b/docs/cl/aws.md @@ -79,7 +79,6 @@ module "tempest" { # configuration ssh_authorized_key = "ssh-rsa AAAAB3Nz..." - asset_dir = "/home/user/.secrets/clusters/tempest" # optional worker_count = 2 @@ -118,9 +117,9 @@ Apply the changes to create the cluster. ```sh $ terraform apply ... -module.aws-tempest.null_resource.bootstrap: Still creating... (4m50s elapsed) -module.aws-tempest.null_resource.bootstrap: Still creating... (5m0s elapsed) -module.aws-tempest.null_resource.bootstrap: Creation complete after 11m8s (ID: 3961816482286168143) +module.tempest.null_resource.bootstrap: Still creating... (4m50s elapsed) +module.tempest.null_resource.bootstrap: Still creating... (5m0s elapsed) +module.tempest.null_resource.bootstrap: Creation complete after 11m8s (ID: 3961816482286168143) Apply complete! Resources: 98 added, 0 changed, 0 destroyed. ``` @@ -129,10 +128,19 @@ In 4-8 minutes, the Kubernetes cluster will be ready. ## Verify -[Install kubectl](https://kubernetes.io/docs/tasks/tools/install-kubectl/) on your system. Use the generated `kubeconfig` credentials to access the Kubernetes cluster and list nodes. +[Install kubectl](https://kubernetes.io/docs/tasks/tools/install-kubectl/) on your system. Obtain the generated cluster `kubeconfig` from module outputs (e.g. write to a local file). ``` -$ export KUBECONFIG=/home/user/.secrets/clusters/tempest/auth/kubeconfig +resource "local_file" "kubeconfig-tempest" { + content = module.tempest.kubeconfig-admin + filename = "/home/user/.kube/configs/tempest-config" +} +``` + +List nodes in the cluster. + +``` +$ export KUBECONFIG=/home/user/.kube/configs/tempest-config $ kubectl get nodes NAME STATUS ROLES AGE VERSION ip-10-0-3-155 Ready 10m v1.16.3 @@ -177,7 +185,6 @@ Check the [variables.tf](https://github.com/poseidon/typhoon/blob/master/aws/con | dns_zone | AWS Route53 DNS zone | "aws.example.com" | | dns_zone_id | AWS Route53 DNS zone id | "Z3PAABBCFAKEC0" | | ssh_authorized_key | SSH public key for user 'core' | "ssh-rsa AAAAB3NZ..." | -| asset_dir | Absolute path to a directory where generated assets should be placed (contains secrets) | "/home/user/.secrets/clusters/tempest" | #### DNS Zone @@ -200,6 +207,7 @@ Reference the DNS zone id with `aws_route53_zone.zone-for-clusters.zone_id`. | Name | Description | Default | Example | |:-----|:------------|:--------|:--------| +| asset_dir | Absolute path to a directory where generated assets should be placed (contains secrets) | "" (disabled) | "/home/user/.secrets/clusters/tempest" | | controller_count | Number of controllers (i.e. masters) | 1 | 1 | | worker_count | Number of workers | 1 | 3 | | controller_type | EC2 instance type for controllers | "t3.small" | See below | diff --git a/docs/cl/azure.md b/docs/cl/azure.md index dc334fba6..f2afd8155 100644 --- a/docs/cl/azure.md +++ b/docs/cl/azure.md @@ -76,7 +76,6 @@ module "ramius" { # configuration ssh_authorized_key = "ssh-rsa AAAAB3Nz..." - asset_dir = "/home/user/.secrets/clusters/ramius" # optional worker_count = 2 @@ -115,9 +114,9 @@ Apply the changes to create the cluster. ```sh $ terraform apply ... -module.azure-ramius.null_resource.bootstrap: Still creating... (6m50s elapsed) -module.azure-ramius.null_resource.bootstrap: Still creating... (7m0s elapsed) -module.azure-ramius.null_resource.bootstrap: Creation complete after 7m8s (ID: 3961816482286168143) +module.ramius.null_resource.bootstrap: Still creating... (6m50s elapsed) +module.ramius.null_resource.bootstrap: Still creating... (7m0s elapsed) +module.ramius.null_resource.bootstrap: Creation complete after 7m8s (ID: 3961816482286168143) Apply complete! Resources: 86 added, 0 changed, 0 destroyed. ``` @@ -126,10 +125,19 @@ In 4-8 minutes, the Kubernetes cluster will be ready. ## Verify -[Install kubectl](https://kubernetes.io/docs/tasks/tools/install-kubectl/) on your system. Use the generated `kubeconfig` credentials to access the Kubernetes cluster and list nodes. +[Install kubectl](https://kubernetes.io/docs/tasks/tools/install-kubectl/) on your system. Obtain the generated cluster `kubeconfig` from module outputs (e.g. write to a local file). ``` -$ export KUBECONFIG=/home/user/.secrets/clusters/ramius/auth/kubeconfig +resource "local_file" "kubeconfig-ramius" { + content = module.ramius.kubeconfig-admin + filename = "/home/user/.kube/configs/ramius-config" +} +``` + +List nodes in the cluster. + +``` +$ export KUBECONFIG=/home/user/.kube/configs/ramius-config $ kubectl get nodes NAME STATUS ROLES AGE VERSION ramius-controller-0 Ready 24m v1.16.3 @@ -175,7 +183,6 @@ Check the [variables.tf](https://github.com/poseidon/typhoon/blob/master/azure/c | dns_zone | Azure DNS zone | "azure.example.com" | | dns_zone_group | Resource group where the Azure DNS zone resides | "global" | | ssh_authorized_key | SSH public key for user 'core' | "ssh-rsa AAAAB3NZ..." | -| asset_dir | Absolute path to a directory where generated assets should be placed (contains secrets) | "/home/user/.secrets/clusters/ramius" | !!! tip Regions are shown in [docs](https://azure.microsoft.com/en-us/global-infrastructure/regions/) or with `az account list-locations --output table`. @@ -211,6 +218,7 @@ Reference the DNS zone with `azurerm_dns_zone.clusters.name` and its resource gr | Name | Description | Default | Example | |:-----|:------------|:--------|:--------| +| asset_dir | Absolute path to a directory where generated assets should be placed (contains secrets) | "" (disabled) | "/home/user/.secrets/clusters/ramius" | | controller_count | Number of controllers (i.e. masters) | 1 | 1 | | worker_count | Number of workers | 1 | 3 | | controller_type | Machine type for controllers | "Standard_B2s" | See below | diff --git a/docs/cl/bare-metal.md b/docs/cl/bare-metal.md index 8d9e91221..b67930b4b 100644 --- a/docs/cl/bare-metal.md +++ b/docs/cl/bare-metal.md @@ -159,7 +159,7 @@ provider "ct" { Define a Kubernetes cluster using the module `bare-metal/container-linux/kubernetes`. ```tf -module "bare-metal-mercury" { +module "mercury" { source = "git::https://github.com/poseidon/typhoon//bare-metal/container-linux/kubernetes?ref=v1.16.3" # bare-metal @@ -171,7 +171,6 @@ module "bare-metal-mercury" { # configuration k8s_domain_name = "node1.example.com" ssh_authorized_key = "ssh-rsa AAAAB3Nz..." - asset_dir = "/home/user/.secrets/clusters/mercury" # machines controllers = [{ @@ -223,12 +222,12 @@ $ terraform plan Plan: 55 to add, 0 to change, 0 to destroy. ``` -Apply the changes. Terraform will generate bootstrap assets to `asset_dir` and create Matchbox profiles (e.g. controller, worker) and matching rules via the Matchbox API. +Apply the changes. Terraform will generate bootstrap assets and create Matchbox profiles (e.g. controller, worker) and matching rules via the Matchbox API. ```sh $ terraform apply -module.bare-metal-mercury.null_resource.copy-controller-secrets.0: Still creating... (10s elapsed) -module.bare-metal-mercury.null_resource.copy-worker-secrets.0: Still creating... (10s elapsed) +module.mercury.null_resource.copy-controller-secrets.0: Still creating... (10s elapsed) +module.mercury.null_resource.copy-worker-secrets.0: Still creating... (10s elapsed) ... ``` @@ -253,11 +252,11 @@ Machines will network boot, install Container Linux to disk, reboot into the dis Wait for the `bootstrap` step to finish bootstrapping the Kubernetes control plane. This may take 5-15 minutes depending on your network. ``` -module.bare-metal-mercury.null_resource.bootstrap: Still creating... (6m10s elapsed) -module.bare-metal-mercury.null_resource.bootstrap: Still creating... (6m20s elapsed) -module.bare-metal-mercury.null_resource.bootstrap: Still creating... (6m30s elapsed) -module.bare-metal-mercury.null_resource.bootstrap: Still creating... (6m40s elapsed) -module.bare-metal-mercury.null_resource.bootstrap: Creation complete (ID: 5441741360626669024) +module.mercury.null_resource.bootstrap: Still creating... (6m10s elapsed) +module.mercury.null_resource.bootstrap: Still creating... (6m20s elapsed) +module.mercury.null_resource.bootstrap: Still creating... (6m30s elapsed) +module.mercury.null_resource.bootstrap: Still creating... (6m40s elapsed) +module.mercury.null_resource.bootstrap: Creation complete (ID: 5441741360626669024) Apply complete! Resources: 55 added, 0 changed, 0 destroyed. ``` @@ -276,19 +275,28 @@ To watch the bootstrap process in detail, SSH to the first controller and journa ``` $ ssh core@node1.example.com $ journalctl -f -u bootstrap -podman[1750]: The connection to the server cluster.example.com:6443 was refused - did you specify the right host or port? -podman[1750]: Waiting for static pod control plane +rkt[1750]: The connection to the server cluster.example.com:6443 was refused - did you specify the right host or port? +rkt[1750]: Waiting for static pod control plane ... -podman[1750]: serviceaccount/calico-node unchanged +rkt[1750]: serviceaccount/calico-node unchanged systemd[1]: Started Kubernetes control plane. ``` ## Verify -[Install kubectl](https://kubernetes.io/docs/tasks/tools/install-kubectl/) on your system. Use the generated `kubeconfig` credentials to access the Kubernetes cluster and list nodes. +[Install kubectl](https://kubernetes.io/docs/tasks/tools/install-kubectl/) on your system. Obtain the generated cluster `kubeconfig` from module outputs (e.g. write to a local file). ``` -$ export KUBECONFIG=/home/user/.secrets/clusters/mercury/auth/kubeconfig +resource "local_file" "kubeconfig-mercury" { + content = module.mercury.kubeconfig-admin + filename = "/home/user/.kube/configs/mercury-config" +} +``` + +List nodes in the cluster. + +``` +$ export KUBECONFIG=/home/user/.kube/configs/mercury-config $ kubectl get nodes NAME STATUS ROLES AGE VERSION node1.example.com Ready 10m v1.16.3 @@ -335,7 +343,6 @@ Check the [variables.tf](https://github.com/poseidon/typhoon/blob/master/bare-me | os_version | Version for a Container Linux derivative to PXE and install | "1632.3.0" | | k8s_domain_name | FQDN resolving to the controller(s) nodes. Workers and kubectl will communicate with this endpoint | "myk8s.example.com" | | ssh_authorized_key | SSH public key for user 'core' | "ssh-rsa AAAAB3Nz..." | -| asset_dir | Absolute path to a directory where generated assets should be placed (contains secrets) | "/home/user/.secrets/clusters/mercury" | | controllers | List of controller machine detail objects (unique name, identifying MAC address, FQDN) | `[{name="node1", mac="52:54:00:a1:9c:ae", domain="node1.example.com"}]` | | workers | List of worker machine detail objects (unique name, identifying MAC address, FQDN) | `[{name="node2", mac="52:54:00:b2:2f:86", domain="node2.example.com"}, {name="node3", mac="52:54:00:c3:61:77", domain="node3.example.com"}]` | @@ -343,6 +350,7 @@ Check the [variables.tf](https://github.com/poseidon/typhoon/blob/master/bare-me | Name | Description | Default | Example | |:-----|:------------|:--------|:--------| +| asset_dir | Absolute path to a directory where generated assets should be placed (contains secrets) | "" (disabled) | "/home/user/.secrets/clusters/mercury" | | download_protocol | Protocol iPXE uses to download the kernel and initrd. iPXE must be compiled with [crypto](https://ipxe.org/crypto) support for https. Unused if cached_install is true | "https" | "http" | | cached_install | PXE boot and install from the Matchbox `/assets` cache. Admin MUST have downloaded Container Linux or Flatcar images into the cache | false | true | | install_disk | Disk device where Container Linux should be installed | "/dev/sda" | "/dev/sdb" | diff --git a/docs/cl/digital-ocean.md b/docs/cl/digital-ocean.md index 4dea64f34..973bd5241 100644 --- a/docs/cl/digital-ocean.md +++ b/docs/cl/digital-ocean.md @@ -64,7 +64,7 @@ provider "ct" { Define a Kubernetes cluster using the module `digital-ocean/container-linux/kubernetes`. ```tf -module "digital-ocean-nemo" { +module "nemo" { source = "git::https://github.com/poseidon/typhoon//digital-ocean/container-linux/kubernetes?ref=v1.16.3" # Digital Ocean @@ -74,7 +74,6 @@ module "digital-ocean-nemo" { # configuration ssh_fingerprints = ["d7:9d:79:ae:56:32:73:79:95:88:e3:a2:ab:5d:45:e7"] - asset_dir = "/home/user/.secrets/clusters/nemo" # optional worker_count = 2 @@ -111,11 +110,11 @@ Apply the changes to create the cluster. ```sh $ terraform apply -module.digital-ocean-nemo.null_resource.bootstrap: Still creating... (30s elapsed) -module.digital-ocean-nemo.null_resource.bootstrap: Provisioning with 'remote-exec'... +module.nemo.null_resource.bootstrap: Still creating... (30s elapsed) +module.nemo.null_resource.bootstrap: Provisioning with 'remote-exec'... ... -module.digital-ocean-nemo.null_resource.bootstrap: Still creating... (6m20s elapsed) -module.digital-ocean-nemo.null_resource.bootstrap: Creation complete (ID: 7599298447329218468) +module.nemo.null_resource.bootstrap: Still creating... (6m20s elapsed) +module.nemo.null_resource.bootstrap: Creation complete (ID: 7599298447329218468) Apply complete! Resources: 54 added, 0 changed, 0 destroyed. ``` @@ -124,10 +123,19 @@ In 3-6 minutes, the Kubernetes cluster will be ready. ## Verify -[Install kubectl](https://kubernetes.io/docs/tasks/tools/install-kubectl/) on your system. Use the generated `kubeconfig` credentials to access the Kubernetes cluster and list nodes. +[Install kubectl](https://kubernetes.io/docs/tasks/tools/install-kubectl/) on your system. Obtain the generated cluster `kubeconfig` from module outputs (e.g. write to a local file). ``` -$ export KUBECONFIG=/home/user/.secrets/clusters/nemo/auth/kubeconfig +resource "local_file" "kubeconfig-nemo" { + content = module.nemo.kubeconfig-admin + filename = "/home/user/.kube/configs/nemo-config" +} +``` + +List nodes in the cluster. + +``` +$ export KUBECONFIG=/home/user/.kube/configs/nemo-config $ kubectl get nodes NAME STATUS ROLES AGE VERSION 10.132.110.130 Ready 10m v1.16.3 @@ -171,7 +179,6 @@ Check the [variables.tf](https://github.com/poseidon/typhoon/blob/master/digital | region | Digital Ocean region | "nyc1", "sfo2", "fra1", tor1" | | dns_zone | Digital Ocean domain (i.e. DNS zone) | "do.example.com" | | ssh_fingerprints | SSH public key fingerprints | ["d7:9d..."] | -| asset_dir | Absolute path to a directory where generated assets should be placed (contains secrets) | "/home/user/.secrets/nemo" | #### DNS Zone @@ -212,6 +219,7 @@ Digital Ocean requires the SSH public key be uploaded to your account, so you ma | Name | Description | Default | Example | |:-----|:------------|:--------|:--------| +| asset_dir | Absolute path to a directory where generated assets should be placed (contains secrets) | "" (disabled) | "/home/user/.secrets/nemo" | | controller_count | Number of controllers (i.e. masters) | 1 | 1 | | worker_count | Number of workers | 1 | 3 | | controller_type | Droplet type for controllers | "s-2vcpu-2gb" | s-2vcpu-2gb, s-2vcpu-4gb, s-4vcpu-8gb, ... | diff --git a/docs/cl/google-cloud.md b/docs/cl/google-cloud.md index bbe91a24b..243b437a7 100644 --- a/docs/cl/google-cloud.md +++ b/docs/cl/google-cloud.md @@ -70,7 +70,7 @@ Additional configuration options are described in the `google` provider [docs](h Define a Kubernetes cluster using the module `google-cloud/container-linux/kubernetes`. ```tf -module "google-cloud-yavin" { +module "yavin" { source = "git::https://github.com/poseidon/typhoon//google-cloud/container-linux/kubernetes?ref=v1.16.3" # Google Cloud @@ -81,7 +81,6 @@ module "google-cloud-yavin" { # configuration ssh_authorized_key = "ssh-rsa AAAAB3Nz..." - asset_dir = "/home/user/.secrets/clusters/yavin" # optional worker_count = 2 @@ -118,11 +117,11 @@ Apply the changes to create the cluster. ```sh $ terraform apply -module.google-cloud-yavin.null_resource.bootstrap: Still creating... (10s elapsed) +module.yavin.null_resource.bootstrap: Still creating... (10s elapsed) ... -module.google-cloud-yavin.null_resource.bootstrap: Still creating... (5m30s elapsed) -module.google-cloud-yavin.null_resource.bootstrap: Still creating... (5m40s elapsed) -module.google-cloud-yavin.null_resource.bootstrap: Creation complete (ID: 5768638456220583358) +module.yavin.null_resource.bootstrap: Still creating... (5m30s elapsed) +module.yavin.null_resource.bootstrap: Still creating... (5m40s elapsed) +module.yavin.null_resource.bootstrap: Creation complete (ID: 5768638456220583358) Apply complete! Resources: 64 added, 0 changed, 0 destroyed. ``` @@ -131,10 +130,19 @@ In 4-8 minutes, the Kubernetes cluster will be ready. ## Verify -[Install kubectl](https://kubernetes.io/docs/tasks/tools/install-kubectl/) on your system. Use the generated `kubeconfig` credentials to access the Kubernetes cluster and list nodes. +[Install kubectl](https://kubernetes.io/docs/tasks/tools/install-kubectl/) on your system. Obtain the generated cluster `kubeconfig` from module outputs (e.g. write to a local file). ``` -$ export KUBECONFIG=/home/user/.secrets/clusters/yavin/auth/kubeconfig +resource "local_file" "kubeconfig-yavin" { + content = module.yavin.kubeconfig-admin + filename = "/home/user/.kube/configs/yavin-config" +} +``` + +List nodes in the cluster. + +``` +$ export KUBECONFIG=/home/user/.kube/configs/yavin-config $ kubectl get nodes NAME ROLES STATUS AGE VERSION yavin-controller-0.c.example-com.internal Ready 6m v1.16.3 @@ -180,7 +188,6 @@ Check the [variables.tf](https://github.com/poseidon/typhoon/blob/master/google- | dns_zone | Google Cloud DNS zone | "google-cloud.example.com" | | dns_zone_name | Google Cloud DNS zone name | "example-zone" | | ssh_authorized_key | SSH public key for user 'core' | "ssh-rsa AAAAB3NZ..." | -| asset_dir | Absolute path to a directory where generated assets should be placed (contains secrets) | "/home/user/.secrets/clusters/yavin" | Check the list of valid [regions](https://cloud.google.com/compute/docs/regions-zones/regions-zones) and list Container Linux [images](https://cloud.google.com/compute/docs/images) with `gcloud compute images list | grep coreos`. @@ -205,6 +212,7 @@ resource "google_dns_managed_zone" "zone-for-clusters" { | Name | Description | Default | Example | |:-----|:------------|:--------|:--------| +| asset_dir | Absolute path to a directory where generated assets should be placed (contains secrets) | "" (disabled) | "/home/user/.secrets/clusters/yavin" | | controller_count | Number of controllers (i.e. masters) | 1 | 3 | | worker_count | Number of workers | 1 | 3 | | controller_type | Machine type for controllers | "n1-standard-1" | See below | diff --git a/docs/fedora-coreos/aws.md b/docs/fedora-coreos/aws.md index 6c0de1f9a..b0f35a129 100644 --- a/docs/fedora-coreos/aws.md +++ b/docs/fedora-coreos/aws.md @@ -72,7 +72,7 @@ Additional configuration options are described in the `aws` provider [docs](http Define a Kubernetes cluster using the module `aws/fedora-coreos/kubernetes`. ```tf -module "aws-tempest" { +module "tempest" { source = "git::https://github.com/poseidon/typhoon//aws/fedora-coreos/kubernetes?ref=v1.16.3" # AWS @@ -82,7 +82,6 @@ module "aws-tempest" { # configuration ssh_authorized_key = "ssh-rsa AAAAB3Nz..." - asset_dir = "/home/user/.secrets/clusters/tempest" # optional worker_count = 2 @@ -121,9 +120,9 @@ Apply the changes to create the cluster. ```sh $ terraform apply ... -module.aws-tempest.null_resource.bootstrap: Still creating... (4m50s elapsed) -module.aws-tempest.null_resource.bootstrap: Still creating... (5m0s elapsed) -module.aws-tempest.null_resource.bootstrap: Creation complete after 5m8s (ID: 3961816482286168143) +module.tempest.null_resource.bootstrap: Still creating... (4m50s elapsed) +module.tempest.null_resource.bootstrap: Still creating... (5m0s elapsed) +module.tempest.null_resource.bootstrap: Creation complete after 5m8s (ID: 3961816482286168143) Apply complete! Resources: 98 added, 0 changed, 0 destroyed. ``` @@ -132,10 +131,19 @@ In 4-8 minutes, the Kubernetes cluster will be ready. ## Verify -[Install kubectl](https://kubernetes.io/docs/tasks/tools/install-kubectl/) on your system. Use the generated `kubeconfig` credentials to access the Kubernetes cluster and list nodes. +[Install kubectl](https://kubernetes.io/docs/tasks/tools/install-kubectl/) on your system. Obtain the generated cluster `kubeconfig` from module outputs (e.g. write to a local file). ``` -$ export KUBECONFIG=/home/user/.secrets/clusters/tempest/auth/kubeconfig +resource "local_file" "kubeconfig-tempest" { + content = module.tempest.kubeconfig-admin + filename = "/home/user/.kube/configs/tempest-config" +} +``` + +List nodes in the cluster. + +``` +$ export KUBECONFIG=/home/user/.kube/configs/tempest-config $ kubectl get nodes NAME STATUS ROLES AGE VERSION ip-10-0-3-155 Ready 10m v1.16.3 @@ -177,7 +185,6 @@ Check the [variables.tf](https://github.com/poseidon/typhoon/blob/master/aws/fed | dns_zone | AWS Route53 DNS zone | "aws.example.com" | | dns_zone_id | AWS Route53 DNS zone id | "Z3PAABBCFAKEC0" | | ssh_authorized_key | SSH public key for user 'core' | "ssh-rsa AAAAB3NZ..." | -| asset_dir | Absolute path to a directory where generated assets should be placed (contains secrets) | "/home/user/.secrets/clusters/tempest" | #### DNS Zone @@ -200,6 +207,7 @@ Reference the DNS zone id with `aws_route53_zone.zone-for-clusters.zone_id`. | Name | Description | Default | Example | |:-----|:------------|:--------|:--------| +| asset_dir | Absolute path to a directory where generated assets should be placed (contains secrets) | "" (disabled) | "/home/user/.secrets/clusters/tempest" | | controller_count | Number of controllers (i.e. masters) | 1 | 1 | | worker_count | Number of workers | 1 | 3 | | controller_type | EC2 instance type for controllers | "t3.small" | See below | diff --git a/docs/fedora-coreos/bare-metal.md b/docs/fedora-coreos/bare-metal.md index a832fdd01..056f24b4c 100644 --- a/docs/fedora-coreos/bare-metal.md +++ b/docs/fedora-coreos/bare-metal.md @@ -162,7 +162,7 @@ provider "ct" { Define a Kubernetes cluster using the module `bare-metal/fedora-coreos/kubernetes`. ```tf -module "bare-metal-mercury" { +module "mercury" { source = "git::https://github.com/poseidon/typhoon//bare-metal/fedora-coreos/kubernetes?ref=v1.16.3" # bare-metal @@ -175,7 +175,6 @@ module "bare-metal-mercury" { # configuration k8s_domain_name = "node1.example.com" ssh_authorized_key = "ssh-rsa AAAAB3Nz..." - asset_dir = "/home/user/.secrets/clusters/mercury" # machines controllers = [{ @@ -224,14 +223,14 @@ $ terraform plan Plan: 55 to add, 0 to change, 0 to destroy. ``` -Apply the changes. Terraform will generate bootstrap assets to `asset_dir` and create Matchbox profiles (e.g. controller, worker) and matching rules via the Matchbox API. +Apply the changes. Terraform will generate bootstrap assets and create Matchbox profiles (e.g. controller, worker) and matching rules via the Matchbox API. ```sh $ terraform apply -module.bare-metal-mercury.null_resource.copy-kubeconfig.0: Provisioning with 'file'... -module.bare-metal-mercury.null_resource.copy-etcd-secrets.0: Provisioning with 'file'... -module.bare-metal-mercury.null_resource.copy-kubeconfig.0: Still creating... (10s elapsed) -module.bare-metal-mercury.null_resource.copy-etcd-secrets.0: Still creating... (10s elapsed) +module.mercury.null_resource.copy-kubeconfig.0: Provisioning with 'file'... +module.mercury.null_resource.copy-etcd-secrets.0: Provisioning with 'file'... +module.mercury.null_resource.copy-kubeconfig.0: Still creating... (10s elapsed) +module.mercury.null_resource.copy-etcd-secrets.0: Still creating... (10s elapsed) ... ``` @@ -256,11 +255,11 @@ Machines will network boot, install Fedora CoreOS to disk, reboot into the disk Wait for the `bootstrap` step to finish bootstrapping the Kubernetes control plane. This may take 5-15 minutes depending on your network. ``` -module.bare-metal-mercury.null_resource.bootstrap: Still creating... (6m10s elapsed) -module.bare-metal-mercury.null_resource.bootstrap: Still creating... (6m20s elapsed) -module.bare-metal-mercury.null_resource.bootstrap: Still creating... (6m30s elapsed) -module.bare-metal-mercury.null_resource.bootstrap: Still creating... (6m40s elapsed) -module.bare-metal-mercury.null_resource.bootstrap: Creation complete (ID: 5441741360626669024) +module.mercury.null_resource.bootstrap: Still creating... (6m10s elapsed) +module.mercury.null_resource.bootstrap: Still creating... (6m20s elapsed) +module.mercury.null_resource.bootstrap: Still creating... (6m30s elapsed) +module.mercury.null_resource.bootstrap: Still creating... (6m40s elapsed) +module.mercury.null_resource.bootstrap: Creation complete (ID: 5441741360626669024) Apply complete! Resources: 55 added, 0 changed, 0 destroyed. ``` @@ -279,10 +278,19 @@ systemd[1]: Started Kubernetes control plane. ## Verify -[Install kubectl](https://kubernetes.io/docs/tasks/tools/install-kubectl/) on your system. Use the generated `kubeconfig` credentials to access the Kubernetes cluster and list nodes. +[Install kubectl](https://kubernetes.io/docs/tasks/tools/install-kubectl/) on your system. Obtain the generated cluster `kubeconfig` from module outputs (e.g. write to a local file). ``` -$ export KUBECONFIG=/home/user/.secrets/clusters/mercury/auth/kubeconfig +resource "local_file" "kubeconfig-mercury" { + content = module.mercury.kubeconfig-admin + filename = "/home/user/.kube/configs/mercury-config" +} +``` + +List nodes in the cluster. + +``` +$ export KUBECONFIG=/home/user/.kube/configs/mercury-config $ kubectl get nodes NAME STATUS ROLES AGE VERSION node1.example.com Ready 10m v1.16.3 @@ -326,7 +334,6 @@ Check the [variables.tf](https://github.com/poseidon/typhoon/blob/master/bare-me | os_version | Fedora CoreOS version to PXE and install | "30.20190716.1" | | k8s_domain_name | FQDN resolving to the controller(s) nodes. Workers and kubectl will communicate with this endpoint | "myk8s.example.com" | | ssh_authorized_key | SSH public key for user 'core' | "ssh-rsa AAAAB3Nz..." | -| asset_dir | Absolute path to a directory where generated assets should be placed (contains secrets) | "/home/user/.secrets/clusters/mercury" | | controllers | List of controller machine detail objects (unique name, identifying MAC address, FQDN) | `[{name="node1", mac="52:54:00:a1:9c:ae", domain="node1.example.com"}]` | | workers | List of worker machine detail objects (unique name, identifying MAC address, FQDN) | `[{name="node2", mac="52:54:00:b2:2f:86", domain="node2.example.com"}, {name="node3", mac="52:54:00:c3:61:77", domain="node3.example.com"}]` | @@ -334,6 +341,7 @@ Check the [variables.tf](https://github.com/poseidon/typhoon/blob/master/bare-me | Name | Description | Default | Example | |:-----|:------------|:--------|:--------| +| asset_dir | Absolute path to a directory where generated assets should be placed (contains secrets) | "" (disabled) | "/home/user/.secrets/clusters/mercury" | | cached_install | PXE boot and install from the Matchbox `/assets` cache. Admin MUST have downloaded Fedora CoreOS images into the cache | false | true | | install_disk | Disk device where Fedora CoreOS should be installed | "sda" (not "/dev/sda" like Container Linux) | "sdb" | | networking | Choice of networking provider | "calico" | "calico" or "flannel" | diff --git a/docs/index.md b/docs/index.md index 32c52cad9..45c2a0e67 100644 --- a/docs/index.md +++ b/docs/index.md @@ -46,7 +46,7 @@ A preview of Typhoon for [Fedora CoreOS](https://getfedora.org/coreos/) is avail Define a Kubernetes cluster by using the Terraform module for your chosen platform and operating system. Here's a minimal example. ```tf -module "google-cloud-yavin" { +module "yavin" { source = "git::https://github.com/poseidon/typhoon//google-cloud/container-linux/kubernetes?ref=v1.16.3" # Google Cloud @@ -57,11 +57,16 @@ module "google-cloud-yavin" { # configuration ssh_authorized_key = "ssh-rsa AAAAB3Nz..." - asset_dir = "/home/user/.secrets/clusters/yavin" # optional worker_count = 2 } + +# Obtain cluster kubeconfig +resource "local_file" "kubeconfig-yavin" { + content = module.yavin.kubeconfig-admin + filename = "/home/user/.kube/configs/yavin-config" +} ``` Initialize modules, plan the changes to be made, and apply the changes. @@ -77,7 +82,7 @@ Apply complete! Resources: 64 added, 0 changed, 0 destroyed. In 4-8 minutes (varies by platform), the cluster will be ready. This Google Cloud example creates a `yavin.example.com` DNS record to resolve to a network load balancer across controller nodes. ``` -$ export KUBECONFIG=/home/user/.secrets/clusters/yavin/auth/kubeconfig +$ export KUBECONFIG=/home/user/.kube/configs/yavin-config $ kubectl get nodes NAME ROLES STATUS AGE VERSION yavin-controller-0.c.example-com.internal Ready 6m v1.16.3 diff --git a/docs/topics/maintenance.md b/docs/topics/maintenance.md index c25544da3..9bd6b18f7 100644 --- a/docs/topics/maintenance.md +++ b/docs/topics/maintenance.md @@ -12,12 +12,12 @@ Typhoon provides tagged releases to allow clusters to be versioned using ordinary Terraform configs. ``` -module "google-cloud-yavin" { +module "yavin" { source = "git::https://github.com/poseidon/typhoon//google-cloud/container-linux/kubernetes?ref=v1.8.6" ... } -module "bare-metal-mercury" { +module "mercury" { source = "git::https://github.com/poseidon/typhoon//bare-metal/container-linux/kubernetes?ref=v1.16.3" ... } @@ -65,7 +65,7 @@ ipmitool -H node1.example.com -U USER -P PASS chassis bootdev pxe Delete or comment the Terraform config for the cluster. ``` -- module "bare-metal-mercury" { +- module "mercury" { - source = "git::https://github.com/poseidon/typhoon//bare-metal/container-linux/kubernetes" - ... -} @@ -93,7 +93,7 @@ kubectl apply -f mercury -R Once you're confident in the new cluster, delete the Terraform config for the old cluster. ``` -- module "google-cloud-yavin" { +- module "yavin" { - source = "git::https://github.com/poseidon/typhoon//google-cloud/container-linux/kubernetes" - ... -} @@ -261,7 +261,7 @@ terraform apply # add kubeconfig to new workers terraform state list | grep null_resource -terraform taint -module digital-ocean-nemo null_resource.copy-worker-secrets.N +terraform taint -module digital-ocean-nemo null_resource.copy-worker-secrets[N] terraform apply ``` @@ -307,7 +307,7 @@ sudo ln -sf ~/Downloads/terraform-0.12.0/terraform /usr/local/bin/terraform12 For existing Typhoon v1.14.2 or v1.14.3 clusters, edit the Typhoon `ref` to first SHA that introduced Terraform v0.12 support (`3276bf587850218b8f967978a4bf2b05d5f440a2`). The aim is to minimize the diff and convert to using Terraform v0.12.x. For example: ```tf - module "bare-metal-mercury" { + module "mercury" { - source = "git::https://github.com/poseidon/typhoon//bare-metal/container-linux/kubernetes?ref=v1.14.3" + source = "git::https://github.com/poseidon/typhoon//bare-metal/container-linux/kubernetes?ref=3276bf587850218b8f967978a4bf2b05d5f440a2" ... @@ -316,7 +316,7 @@ For existing Typhoon v1.14.2 or v1.14.3 clusters, edit the Typhoon `ref` to firs With Terraform v0.12, Typhoon clusters no longer require the `providers` block (unless you actually need to pass an [aliased provider](https://www.terraform.io/docs/configuration/providers.html#alias-multiple-provider-instances)). A regression in Terraform v0.11 made it neccessary to explicitly pass aliased providers in order for Typhoon to continue to enforce constraints (see [terraform#16824](https://github.com/hashicorp/terraform/issues/16824)). Terraform v0.12 resolves this issue. ```tf - module "bare-metal-mercury" { + module "mercury" { source = "git::https://github.com/poseidon/typhoon//bare-metal/container-linux/kubernetes?ref=3276bf587850218b8f967978a4bf2b05d5f440a2" - providers = { diff --git a/google-cloud/container-linux/kubernetes/variables.tf b/google-cloud/container-linux/kubernetes/variables.tf index 32a835fae..d8ed8e303 100644 --- a/google-cloud/container-linux/kubernetes/variables.tf +++ b/google-cloud/container-linux/kubernetes/variables.tf @@ -86,6 +86,7 @@ variable "ssh_authorized_key" { variable "asset_dir" { type = string description = "Absolute path to a directory where generated assets should be placed (contains secrets)" + default = "" } variable "networking" { From 178afe4a9bc682f85fc40d72609b2cfc547febca Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Sun, 8 Dec 2019 22:05:15 -0800 Subject: [PATCH 282/523] Reduce apiserver metrics cardinality and extraneous labels * Stop mapping node labels to targets discovered via Kubernetes nodes (e.g. etcd, kubelet, cadvisor). It is rarely useful to store node labels (e.g. kubernetes.io/os=linux) on these metrics * kube-apiserver's apiserver_request_duration_seconds_bucket metric has a high cardinality that includes labels for the API group, verb, scope, resource, and component for each object type, including for each CRD. This one metric has ~10k time series in a typical cluster (btw 10-40% of total) * Removing the apiserver request duration outright would make latency alerts a NoOp and break a Grafana apiserver panel. Instead, drop series that have a "group" label. Effectively, only request durations for core Kubernetes APIs will be kept (e.g. cardinality won't grow with each CRD added). This reduces the metric to ~2k unique series --- CHANGES.md | 1 + addons/prometheus/config.yaml | 25 ++++++++++++++----------- 2 files changed, 15 insertions(+), 11 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 9ce519d63..7e36d0b63 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -28,6 +28,7 @@ Notable changes between versions. * Update Grafana from v6.4.4 to [v6.5.1](https://grafana.com/docs/guides/whats-new-in-v6-5/) * Add pod networking details in dashboards ([#593](https://github.com/poseidon/typhoon/pull/593)) * Add node alerts and Grafana dashboard from node-exporter ([#591](https://github.com/poseidon/typhoon/pull/591)) +* Reduce Prometheus time series of high cardinality metrics ([#596](https://github.com/poseidon/typhoon/pull/596)) ## v1.16.3 diff --git a/addons/prometheus/config.yaml b/addons/prometheus/config.yaml index 298c4edc0..38b7472f1 100644 --- a/addons/prometheus/config.yaml +++ b/addons/prometheus/config.yaml @@ -65,6 +65,9 @@ data: - source_labels: [__name__] action: drop regex: apiserver_admission_step_admission_latencies_seconds_.* + - source_labels: [__name__, group] + regex: apiserver_request_duration_seconds_bucket;.+ + action: drop # Scrape config for node (i.e. kubelet) /metrics (e.g. 'kubelet_'). Explore # metrics from a node by scraping kubelet (127.0.0.1:10250/metrics). @@ -81,7 +84,7 @@ data: relabel_configs: - action: labelmap - regex: __meta_kubernetes_node_label_(.+) + regex: __meta_kubernetes_node_name # Scrape config for Kubelet cAdvisor. Explore metrics from a node by # scraping kubelet (127.0.0.1:10250/metrics/cadvisor). @@ -99,7 +102,7 @@ data: relabel_configs: - action: labelmap - regex: __meta_kubernetes_node_label_(.+) + regex: __meta_kubernetes_node_name metric_relabel_configs: - source_labels: [__name__, image] action: drop @@ -115,15 +118,15 @@ data: - role: node scheme: http relabel_configs: - - source_labels: [__meta_kubernetes_node_label_node_kubernetes_io_controller] - action: keep - regex: 'true' - - action: labelmap - regex: __meta_kubernetes_node_label_(.+) - - source_labels: [__meta_kubernetes_node_address_InternalIP] - action: replace - target_label: __address__ - replacement: '${1}:2381' + - source_labels: [__meta_kubernetes_node_label_node_kubernetes_io_controller] + action: keep + regex: 'true' + - action: labelmap + regex: __meta_kubernetes_node_name + - source_labels: [__meta_kubernetes_node_address_InternalIP] + action: replace + target_label: __address__ + replacement: '${1}:2381' # Scrape config for service endpoints. # From 4fce9485c8fae35b5424c5de89b3a7ccf3bfd990 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Sun, 8 Dec 2019 22:54:31 -0800 Subject: [PATCH 283/523] Reduce kube-controller-manager pod eviction timeout from 5m to 1m * Reduce time to delete pods on unready nodes from 5m to 1m * Present since v1.13.3, but mistakenly removed in v1.16.0 static pod control plane migration Related: * https://github.com/poseidon/terraform-render-bootstrap/pull/148 * https://github.com/poseidon/terraform-render-bootstrap/pull/164 --- CHANGES.md | 6 ++++-- aws/container-linux/kubernetes/bootstrap.tf | 2 +- aws/fedora-coreos/kubernetes/bootstrap.tf | 2 +- azure/container-linux/kubernetes/bootstrap.tf | 2 +- bare-metal/container-linux/kubernetes/bootstrap.tf | 2 +- bare-metal/fedora-coreos/kubernetes/bootstrap.tf | 2 +- digital-ocean/container-linux/kubernetes/bootstrap.tf | 2 +- google-cloud/container-linux/kubernetes/bootstrap.tf | 2 +- 8 files changed, 11 insertions(+), 9 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 7e36d0b63..7666d9c07 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -10,12 +10,14 @@ Notable changes between versions. * Allow `terraform apply` from stateless automation systems * Improve asset unpacking on controllers to remove unused materials * Obtain a kubeconfig from Terraform module outputs +* Replace usage of `template_dir` with `templatefile` function ([#587](https://github.com/poseidon/typhoon/pull/587)) + * Require Terraform version v0.12.6+ (action required) * Update CoreDNS from v1.6.2 to v1.6.5 ([#588](https://github.com/poseidon/typhoon/pull/588)) * Add health `lameduck` option to wait before shutdown +* Reduce pod eviction timeout for deleting pods on unready nodes from 5m to 1m ([#597](https://github.com/poseidon/typhoon/pull/597)) + * Present since [v1.13.3](#v1133), but mistakenly removed in v1.16.0 * Add CPU requests for control plane static pods ([#589](https://github.com/poseidon/typhoon/pull/589)) * May provide slight edge case benefits and aligns with upstream -* Replace usage of `template_dir` with `templatefile` function ([#587](https://github.com/poseidon/typhoon/pull/587)) - * Require Terraform version v0.12.6+ (action required) #### Google diff --git a/aws/container-linux/kubernetes/bootstrap.tf b/aws/container-linux/kubernetes/bootstrap.tf index 1db669b98..c623d2f1b 100644 --- a/aws/container-linux/kubernetes/bootstrap.tf +++ b/aws/container-linux/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=dce49114a083436c5af6d9174b9c1248786ba3b6" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=4369c706e2432151211bd33aa18a8ea7d53aaa11" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/aws/fedora-coreos/kubernetes/bootstrap.tf b/aws/fedora-coreos/kubernetes/bootstrap.tf index c2716fb8c..dca11b5fc 100644 --- a/aws/fedora-coreos/kubernetes/bootstrap.tf +++ b/aws/fedora-coreos/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=dce49114a083436c5af6d9174b9c1248786ba3b6" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=4369c706e2432151211bd33aa18a8ea7d53aaa11" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/azure/container-linux/kubernetes/bootstrap.tf b/azure/container-linux/kubernetes/bootstrap.tf index e26c8cf21..b354e180d 100644 --- a/azure/container-linux/kubernetes/bootstrap.tf +++ b/azure/container-linux/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=dce49114a083436c5af6d9174b9c1248786ba3b6" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=4369c706e2432151211bd33aa18a8ea7d53aaa11" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/bare-metal/container-linux/kubernetes/bootstrap.tf b/bare-metal/container-linux/kubernetes/bootstrap.tf index cb751a163..156e1b21d 100644 --- a/bare-metal/container-linux/kubernetes/bootstrap.tf +++ b/bare-metal/container-linux/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=dce49114a083436c5af6d9174b9c1248786ba3b6" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=4369c706e2432151211bd33aa18a8ea7d53aaa11" cluster_name = var.cluster_name api_servers = [var.k8s_domain_name] diff --git a/bare-metal/fedora-coreos/kubernetes/bootstrap.tf b/bare-metal/fedora-coreos/kubernetes/bootstrap.tf index d04849434..61c636d21 100644 --- a/bare-metal/fedora-coreos/kubernetes/bootstrap.tf +++ b/bare-metal/fedora-coreos/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=dce49114a083436c5af6d9174b9c1248786ba3b6" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=4369c706e2432151211bd33aa18a8ea7d53aaa11" cluster_name = var.cluster_name api_servers = [var.k8s_domain_name] diff --git a/digital-ocean/container-linux/kubernetes/bootstrap.tf b/digital-ocean/container-linux/kubernetes/bootstrap.tf index 196ae533d..97e585c76 100644 --- a/digital-ocean/container-linux/kubernetes/bootstrap.tf +++ b/digital-ocean/container-linux/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=dce49114a083436c5af6d9174b9c1248786ba3b6" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=4369c706e2432151211bd33aa18a8ea7d53aaa11" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/google-cloud/container-linux/kubernetes/bootstrap.tf b/google-cloud/container-linux/kubernetes/bootstrap.tf index 36ae40e33..de9e2e7e0 100644 --- a/google-cloud/container-linux/kubernetes/bootstrap.tf +++ b/google-cloud/container-linux/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=dce49114a083436c5af6d9174b9c1248786ba3b6" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=4369c706e2432151211bd33aa18a8ea7d53aaa11" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] From de36d99afc1aa1bf72be89360a6b87373172724d Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Mon, 9 Dec 2019 18:31:58 -0800 Subject: [PATCH 284/523] Update Kubernetes from v1.16.3 to v1.17.0 * https://github.com/kubernetes/kubernetes/blob/master/CHANGELOG-1.17.md/#v1170 --- CHANGES.md | 1 + README.md | 10 +++++----- aws/container-linux/kubernetes/README.md | 2 +- aws/container-linux/kubernetes/bootstrap.tf | 2 +- .../kubernetes/cl/controller.yaml.tmpl | 4 ++-- .../kubernetes/workers/cl/worker.yaml.tmpl | 4 ++-- aws/fedora-coreos/kubernetes/README.md | 2 +- aws/fedora-coreos/kubernetes/bootstrap.tf | 2 +- aws/fedora-coreos/kubernetes/fcc/controller.yaml | 4 ++-- .../kubernetes/workers/fcc/worker.yaml | 2 +- azure/container-linux/kubernetes/README.md | 2 +- azure/container-linux/kubernetes/bootstrap.tf | 2 +- .../kubernetes/cl/controller.yaml.tmpl | 4 ++-- .../kubernetes/workers/cl/worker.yaml.tmpl | 4 ++-- bare-metal/container-linux/kubernetes/README.md | 2 +- bare-metal/container-linux/kubernetes/bootstrap.tf | 2 +- .../kubernetes/cl/controller.yaml.tmpl | 4 ++-- .../container-linux/kubernetes/cl/worker.yaml.tmpl | 2 +- bare-metal/fedora-coreos/kubernetes/README.md | 2 +- bare-metal/fedora-coreos/kubernetes/bootstrap.tf | 2 +- .../fedora-coreos/kubernetes/fcc/controller.yaml | 4 ++-- .../fedora-coreos/kubernetes/fcc/worker.yaml | 2 +- digital-ocean/container-linux/kubernetes/README.md | 2 +- .../container-linux/kubernetes/bootstrap.tf | 2 +- .../kubernetes/cl/controller.yaml.tmpl | 4 ++-- .../container-linux/kubernetes/cl/worker.yaml.tmpl | 4 ++-- docs/advanced/worker-pools.md | 14 +++++++------- docs/cl/aws.md | 10 +++++----- docs/cl/azure.md | 10 +++++----- docs/cl/bare-metal.md | 14 +++++++------- docs/cl/digital-ocean.md | 10 +++++----- docs/cl/google-cloud.md | 10 +++++----- docs/fedora-coreos/aws.md | 10 +++++----- docs/fedora-coreos/bare-metal.md | 10 +++++----- docs/index.md | 10 +++++----- docs/topics/maintenance.md | 10 +++++----- google-cloud/container-linux/kubernetes/README.md | 2 +- .../container-linux/kubernetes/bootstrap.tf | 2 +- .../kubernetes/cl/controller.yaml.tmpl | 4 ++-- .../kubernetes/workers/cl/worker.yaml.tmpl | 4 ++-- 40 files changed, 99 insertions(+), 98 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 7666d9c07..cbb2aeb20 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -4,6 +4,7 @@ Notable changes between versions. ## Latest +* Kubernetes [v1.17.0](https://github.com/kubernetes/kubernetes/blob/master/CHANGELOG-1.17.md#v1170) * Manage clusters without using a local `asset_dir` ([#595](https://github.com/poseidon/typhoon/pull/595)) * Change `asset_dir` to be optional. Default to "" to skip writing assets locally * Keep cluster assets (TLS materials, kubeconfig) only in Terraform state, which supports different [remote backends](https://www.terraform.io/docs/backends/types/remote.html) and optional encryption at rest diff --git a/README.md b/README.md index b26355a40..26cc641d2 100644 --- a/README.md +++ b/README.md @@ -11,7 +11,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features -* Kubernetes v1.16.3 (upstream) +* Kubernetes v1.17.0 (upstream) * Single or multi-master, [Calico](https://www.projectcalico.org/) or [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/) * Advanced features like [worker pools](https://typhoon.psdn.io/advanced/worker-pools/), [preemptible](https://typhoon.psdn.io/cl/google-cloud/#preemption) workers, and [snippets](https://typhoon.psdn.io/advanced/customization/#container-linux) customization @@ -48,7 +48,7 @@ Define a Kubernetes cluster by using the Terraform module for your chosen platfo ```tf module "yavin" { - source = "git::https://github.com/poseidon/typhoon//google-cloud/container-linux/kubernetes?ref=v1.16.3" + source = "git::https://github.com/poseidon/typhoon//google-cloud/container-linux/kubernetes?ref=v1.17.0" # Google Cloud cluster_name = "yavin" @@ -87,9 +87,9 @@ In 4-8 minutes (varies by platform), the cluster will be ready. This Google Clou $ export KUBECONFIG=/home/user/.kube/configs/yavin-config $ kubectl get nodes NAME ROLES STATUS AGE VERSION -yavin-controller-0.c.example-com.internal Ready 6m v1.16.3 -yavin-worker-jrbf.c.example-com.internal Ready 5m v1.16.3 -yavin-worker-mzdm.c.example-com.internal Ready 5m v1.16.3 +yavin-controller-0.c.example-com.internal Ready 6m v1.17.0 +yavin-worker-jrbf.c.example-com.internal Ready 5m v1.17.0 +yavin-worker-mzdm.c.example-com.internal Ready 5m v1.17.0 ``` List the pods. diff --git a/aws/container-linux/kubernetes/README.md b/aws/container-linux/kubernetes/README.md index 746726f86..c89a8b404 100644 --- a/aws/container-linux/kubernetes/README.md +++ b/aws/container-linux/kubernetes/README.md @@ -11,7 +11,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features -* Kubernetes v1.16.3 (upstream) +* Kubernetes v1.17.0 (upstream) * Single or multi-master, [Calico](https://www.projectcalico.org/) or [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/) * Advanced features like [worker pools](https://typhoon.psdn.io/advanced/worker-pools/), [spot](https://typhoon.psdn.io/cl/aws/#spot) workers, and [snippets](https://typhoon.psdn.io/advanced/customization/#container-linux) customization diff --git a/aws/container-linux/kubernetes/bootstrap.tf b/aws/container-linux/kubernetes/bootstrap.tf index c623d2f1b..018dfa979 100644 --- a/aws/container-linux/kubernetes/bootstrap.tf +++ b/aws/container-linux/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=4369c706e2432151211bd33aa18a8ea7d53aaa11" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=0ddd90fd05764796758991dfc532ae5cbbeb66b5" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/aws/container-linux/kubernetes/cl/controller.yaml.tmpl b/aws/container-linux/kubernetes/cl/controller.yaml.tmpl index 246c617ef..53229485a 100644 --- a/aws/container-linux/kubernetes/cl/controller.yaml.tmpl +++ b/aws/container-linux/kubernetes/cl/controller.yaml.tmpl @@ -115,7 +115,7 @@ systemd: --volume script,kind=host,source=/opt/bootstrap/apply \ --mount volume=script,target=/apply \ --insecure-options=image \ - docker://k8s.gcr.io/hyperkube:v1.16.3 \ + docker://k8s.gcr.io/hyperkube:v1.17.0 \ --net=host \ --dns=host \ --exec=/apply @@ -136,7 +136,7 @@ storage: contents: inline: | KUBELET_IMAGE_URL=docker://k8s.gcr.io/hyperkube - KUBELET_IMAGE_TAG=v1.16.3 + KUBELET_IMAGE_TAG=v1.17.0 - path: /opt/bootstrap/layout filesystem: root mode: 0544 diff --git a/aws/container-linux/kubernetes/workers/cl/worker.yaml.tmpl b/aws/container-linux/kubernetes/workers/cl/worker.yaml.tmpl index 8c495585c..8e0ee0fb1 100644 --- a/aws/container-linux/kubernetes/workers/cl/worker.yaml.tmpl +++ b/aws/container-linux/kubernetes/workers/cl/worker.yaml.tmpl @@ -98,7 +98,7 @@ storage: contents: inline: | KUBELET_IMAGE_URL=docker://k8s.gcr.io/hyperkube - KUBELET_IMAGE_TAG=v1.16.3 + KUBELET_IMAGE_TAG=v1.17.0 - path: /etc/sysctl.d/max-user-watches.conf filesystem: root contents: @@ -116,7 +116,7 @@ storage: --volume config,kind=host,source=/etc/kubernetes \ --mount volume=config,target=/etc/kubernetes \ --insecure-options=image \ - docker://k8s.gcr.io/hyperkube:v1.16.3 \ + docker://k8s.gcr.io/hyperkube:v1.17.0 \ --net=host \ --dns=host \ --exec=/kubectl -- --kubeconfig=/etc/kubernetes/kubeconfig delete node $(hostname) diff --git a/aws/fedora-coreos/kubernetes/README.md b/aws/fedora-coreos/kubernetes/README.md index 3adc79621..b389491c8 100644 --- a/aws/fedora-coreos/kubernetes/README.md +++ b/aws/fedora-coreos/kubernetes/README.md @@ -11,7 +11,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features -* Kubernetes v1.16.3 (upstream) +* Kubernetes v1.17.0 (upstream) * Single or multi-master, [Calico](https://www.projectcalico.org/) or [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/) * Advanced features like [worker pools](https://typhoon.psdn.io/advanced/worker-pools/), [spot](https://typhoon.psdn.io/cl/aws/#spot) workers, and [snippets](https://typhoon.psdn.io/advanced/customization/#container-linux) customization diff --git a/aws/fedora-coreos/kubernetes/bootstrap.tf b/aws/fedora-coreos/kubernetes/bootstrap.tf index dca11b5fc..a4dba5010 100644 --- a/aws/fedora-coreos/kubernetes/bootstrap.tf +++ b/aws/fedora-coreos/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=4369c706e2432151211bd33aa18a8ea7d53aaa11" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=0ddd90fd05764796758991dfc532ae5cbbeb66b5" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/aws/fedora-coreos/kubernetes/fcc/controller.yaml b/aws/fedora-coreos/kubernetes/fcc/controller.yaml index 8db338720..e21f14c1f 100644 --- a/aws/fedora-coreos/kubernetes/fcc/controller.yaml +++ b/aws/fedora-coreos/kubernetes/fcc/controller.yaml @@ -80,7 +80,7 @@ systemd: --volume /var/run:/var/run \ --volume /var/run/lock:/var/run/lock:z \ --volume /opt/cni/bin:/opt/cni/bin:z \ - k8s.gcr.io/hyperkube:v1.16.3 /hyperkube kubelet \ + k8s.gcr.io/hyperkube:v1.17.0 /hyperkube kubelet \ --anonymous-auth=false \ --authentication-token-webhook \ --authorization-mode=Webhook \ @@ -122,7 +122,7 @@ systemd: --volume /etc/kubernetes/bootstrap-secrets:/etc/kubernetes/secrets:ro,Z \ --volume /opt/bootstrap/assets:/assets:ro,Z \ --volume /opt/bootstrap/apply:/apply:ro,Z \ - k8s.gcr.io/hyperkube:v1.16.3 \ + k8s.gcr.io/hyperkube:v1.17.0 \ /apply ExecStartPost=/bin/touch /opt/bootstrap/bootstrap.done ExecStartPost=-/usr/bin/podman stop bootstrap diff --git a/aws/fedora-coreos/kubernetes/workers/fcc/worker.yaml b/aws/fedora-coreos/kubernetes/workers/fcc/worker.yaml index b31ac0ab4..fa7505cd0 100644 --- a/aws/fedora-coreos/kubernetes/workers/fcc/worker.yaml +++ b/aws/fedora-coreos/kubernetes/workers/fcc/worker.yaml @@ -50,7 +50,7 @@ systemd: --volume /var/run:/var/run \ --volume /var/run/lock:/var/run/lock:z \ --volume /opt/cni/bin:/opt/cni/bin:z \ - k8s.gcr.io/hyperkube:v1.16.3 /hyperkube kubelet \ + k8s.gcr.io/hyperkube:v1.17.0 /hyperkube kubelet \ --anonymous-auth=false \ --authentication-token-webhook \ --authorization-mode=Webhook \ diff --git a/azure/container-linux/kubernetes/README.md b/azure/container-linux/kubernetes/README.md index 9c1635dda..3adc3b2a4 100644 --- a/azure/container-linux/kubernetes/README.md +++ b/azure/container-linux/kubernetes/README.md @@ -11,7 +11,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features -* Kubernetes v1.16.3 (upstream) +* Kubernetes v1.17.0 (upstream) * Single or multi-master, [Calico](https://www.projectcalico.org/) or [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/) * Advanced features like [worker pools](https://typhoon.psdn.io/advanced/worker-pools/), [low-priority](https://typhoon.psdn.io/cl/azure/#low-priority) workers, and [snippets](https://typhoon.psdn.io/advanced/customization/#container-linux) customization diff --git a/azure/container-linux/kubernetes/bootstrap.tf b/azure/container-linux/kubernetes/bootstrap.tf index b354e180d..5b33ed43a 100644 --- a/azure/container-linux/kubernetes/bootstrap.tf +++ b/azure/container-linux/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=4369c706e2432151211bd33aa18a8ea7d53aaa11" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=0ddd90fd05764796758991dfc532ae5cbbeb66b5" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/azure/container-linux/kubernetes/cl/controller.yaml.tmpl b/azure/container-linux/kubernetes/cl/controller.yaml.tmpl index 5e83e996a..8c54985d4 100644 --- a/azure/container-linux/kubernetes/cl/controller.yaml.tmpl +++ b/azure/container-linux/kubernetes/cl/controller.yaml.tmpl @@ -113,7 +113,7 @@ systemd: --volume script,kind=host,source=/opt/bootstrap/apply \ --mount volume=script,target=/apply \ --insecure-options=image \ - docker://k8s.gcr.io/hyperkube:v1.16.3 \ + docker://k8s.gcr.io/hyperkube:v1.17.0 \ --net=host \ --dns=host \ --exec=/apply @@ -134,7 +134,7 @@ storage: contents: inline: | KUBELET_IMAGE_URL=docker://k8s.gcr.io/hyperkube - KUBELET_IMAGE_TAG=v1.16.3 + KUBELET_IMAGE_TAG=v1.17.0 - path: /opt/bootstrap/layout filesystem: root mode: 0544 diff --git a/azure/container-linux/kubernetes/workers/cl/worker.yaml.tmpl b/azure/container-linux/kubernetes/workers/cl/worker.yaml.tmpl index 6c23c66b0..a12754500 100644 --- a/azure/container-linux/kubernetes/workers/cl/worker.yaml.tmpl +++ b/azure/container-linux/kubernetes/workers/cl/worker.yaml.tmpl @@ -96,7 +96,7 @@ storage: contents: inline: | KUBELET_IMAGE_URL=docker://k8s.gcr.io/hyperkube - KUBELET_IMAGE_TAG=v1.16.3 + KUBELET_IMAGE_TAG=v1.17.0 - path: /etc/sysctl.d/max-user-watches.conf filesystem: root contents: @@ -114,7 +114,7 @@ storage: --volume config,kind=host,source=/etc/kubernetes \ --mount volume=config,target=/etc/kubernetes \ --insecure-options=image \ - docker://k8s.gcr.io/hyperkube:v1.16.3 \ + docker://k8s.gcr.io/hyperkube:v1.17.0 \ --net=host \ --dns=host \ --exec=/kubectl -- --kubeconfig=/etc/kubernetes/kubeconfig delete node $(hostname | tr '[:upper:]' '[:lower:]') diff --git a/bare-metal/container-linux/kubernetes/README.md b/bare-metal/container-linux/kubernetes/README.md index 2fd5ec5f8..821632bd6 100644 --- a/bare-metal/container-linux/kubernetes/README.md +++ b/bare-metal/container-linux/kubernetes/README.md @@ -11,7 +11,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features -* Kubernetes v1.16.3 (upstream) +* Kubernetes v1.17.0 (upstream) * Single or multi-master, [Calico](https://www.projectcalico.org/) or [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/) * Advanced features like [snippets](https://typhoon.psdn.io/advanced/customization/#container-linux) customization diff --git a/bare-metal/container-linux/kubernetes/bootstrap.tf b/bare-metal/container-linux/kubernetes/bootstrap.tf index 156e1b21d..7f6202d9e 100644 --- a/bare-metal/container-linux/kubernetes/bootstrap.tf +++ b/bare-metal/container-linux/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=4369c706e2432151211bd33aa18a8ea7d53aaa11" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=0ddd90fd05764796758991dfc532ae5cbbeb66b5" cluster_name = var.cluster_name api_servers = [var.k8s_domain_name] diff --git a/bare-metal/container-linux/kubernetes/cl/controller.yaml.tmpl b/bare-metal/container-linux/kubernetes/cl/controller.yaml.tmpl index 2a6b32e49..8869eb237 100644 --- a/bare-metal/container-linux/kubernetes/cl/controller.yaml.tmpl +++ b/bare-metal/container-linux/kubernetes/cl/controller.yaml.tmpl @@ -128,7 +128,7 @@ systemd: --volume script,kind=host,source=/opt/bootstrap/apply \ --mount volume=script,target=/apply \ --insecure-options=image \ - docker://k8s.gcr.io/hyperkube:v1.16.3 \ + docker://k8s.gcr.io/hyperkube:v1.17.0 \ --net=host \ --dns=host \ --exec=/apply @@ -143,7 +143,7 @@ storage: contents: inline: | KUBELET_IMAGE_URL=docker://k8s.gcr.io/hyperkube - KUBELET_IMAGE_TAG=v1.16.3 + KUBELET_IMAGE_TAG=v1.17.0 - path: /etc/hostname filesystem: root mode: 0644 diff --git a/bare-metal/container-linux/kubernetes/cl/worker.yaml.tmpl b/bare-metal/container-linux/kubernetes/cl/worker.yaml.tmpl index dab0a6be8..a7ac5eec8 100644 --- a/bare-metal/container-linux/kubernetes/cl/worker.yaml.tmpl +++ b/bare-metal/container-linux/kubernetes/cl/worker.yaml.tmpl @@ -91,7 +91,7 @@ storage: contents: inline: | KUBELET_IMAGE_URL=docker://k8s.gcr.io/hyperkube - KUBELET_IMAGE_TAG=v1.16.3 + KUBELET_IMAGE_TAG=v1.17.0 - path: /etc/hostname filesystem: root mode: 0644 diff --git a/bare-metal/fedora-coreos/kubernetes/README.md b/bare-metal/fedora-coreos/kubernetes/README.md index 1f389a7ca..5e1fecda3 100644 --- a/bare-metal/fedora-coreos/kubernetes/README.md +++ b/bare-metal/fedora-coreos/kubernetes/README.md @@ -11,7 +11,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features -* Kubernetes v1.16.3 (upstream) +* Kubernetes v1.17.0 (upstream) * Single or multi-master, [Calico](https://www.projectcalico.org/) or [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/) * Advanced features like [snippets](https://typhoon.psdn.io/advanced/customization/#container-linux) customization diff --git a/bare-metal/fedora-coreos/kubernetes/bootstrap.tf b/bare-metal/fedora-coreos/kubernetes/bootstrap.tf index 61c636d21..a17e9f25a 100644 --- a/bare-metal/fedora-coreos/kubernetes/bootstrap.tf +++ b/bare-metal/fedora-coreos/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=4369c706e2432151211bd33aa18a8ea7d53aaa11" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=0ddd90fd05764796758991dfc532ae5cbbeb66b5" cluster_name = var.cluster_name api_servers = [var.k8s_domain_name] diff --git a/bare-metal/fedora-coreos/kubernetes/fcc/controller.yaml b/bare-metal/fedora-coreos/kubernetes/fcc/controller.yaml index 2132bb247..bec262a4c 100644 --- a/bare-metal/fedora-coreos/kubernetes/fcc/controller.yaml +++ b/bare-metal/fedora-coreos/kubernetes/fcc/controller.yaml @@ -81,7 +81,7 @@ systemd: --volume /opt/cni/bin:/opt/cni/bin:z \ --volume /etc/iscsi:/etc/iscsi \ --volume /sbin/iscsiadm:/sbin/iscsiadm \ - k8s.gcr.io/hyperkube:v1.16.3 /hyperkube kubelet \ + k8s.gcr.io/hyperkube:v1.17.0 /hyperkube kubelet \ --anonymous-auth=false \ --authentication-token-webhook \ --authorization-mode=Webhook \ @@ -133,7 +133,7 @@ systemd: --volume /etc/kubernetes/bootstrap-secrets:/etc/kubernetes/secrets:ro,Z \ --volume /opt/bootstrap/assets:/assets:ro,Z \ --volume /opt/bootstrap/apply:/apply:ro,Z \ - k8s.gcr.io/hyperkube:v1.16.3 \ + k8s.gcr.io/hyperkube:v1.17.0 \ /apply ExecStartPost=/bin/touch /opt/bootstrap/bootstrap.done ExecStartPost=-/usr/bin/podman stop bootstrap diff --git a/bare-metal/fedora-coreos/kubernetes/fcc/worker.yaml b/bare-metal/fedora-coreos/kubernetes/fcc/worker.yaml index f4fc87cdb..f10ccae47 100644 --- a/bare-metal/fedora-coreos/kubernetes/fcc/worker.yaml +++ b/bare-metal/fedora-coreos/kubernetes/fcc/worker.yaml @@ -51,7 +51,7 @@ systemd: --volume /opt/cni/bin:/opt/cni/bin:z \ --volume /etc/iscsi:/etc/iscsi \ --volume /sbin/iscsiadm:/sbin/iscsiadm \ - k8s.gcr.io/hyperkube:v1.16.3 /hyperkube kubelet \ + k8s.gcr.io/hyperkube:v1.17.0 /hyperkube kubelet \ --anonymous-auth=false \ --authentication-token-webhook \ --authorization-mode=Webhook \ diff --git a/digital-ocean/container-linux/kubernetes/README.md b/digital-ocean/container-linux/kubernetes/README.md index f2e070422..1c248f766 100644 --- a/digital-ocean/container-linux/kubernetes/README.md +++ b/digital-ocean/container-linux/kubernetes/README.md @@ -11,7 +11,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features -* Kubernetes v1.16.3 (upstream) +* Kubernetes v1.17.0 (upstream) * Single or multi-master, [Calico](https://www.projectcalico.org/) or [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/) * Advanced features like [snippets](https://typhoon.psdn.io/advanced/customization/#container-linux) customization diff --git a/digital-ocean/container-linux/kubernetes/bootstrap.tf b/digital-ocean/container-linux/kubernetes/bootstrap.tf index 97e585c76..ebbc25639 100644 --- a/digital-ocean/container-linux/kubernetes/bootstrap.tf +++ b/digital-ocean/container-linux/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=4369c706e2432151211bd33aa18a8ea7d53aaa11" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=0ddd90fd05764796758991dfc532ae5cbbeb66b5" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/digital-ocean/container-linux/kubernetes/cl/controller.yaml.tmpl b/digital-ocean/container-linux/kubernetes/cl/controller.yaml.tmpl index 8c94dd65a..bd0f97f1b 100644 --- a/digital-ocean/container-linux/kubernetes/cl/controller.yaml.tmpl +++ b/digital-ocean/container-linux/kubernetes/cl/controller.yaml.tmpl @@ -125,7 +125,7 @@ systemd: --volume script,kind=host,source=/opt/bootstrap/apply \ --mount volume=script,target=/apply \ --insecure-options=image \ - docker://k8s.gcr.io/hyperkube:v1.16.3 \ + docker://k8s.gcr.io/hyperkube:v1.17.0 \ --net=host \ --dns=host \ --exec=/apply @@ -140,7 +140,7 @@ storage: contents: inline: | KUBELET_IMAGE_URL=docker://k8s.gcr.io/hyperkube - KUBELET_IMAGE_TAG=v1.16.3 + KUBELET_IMAGE_TAG=v1.17.0 - path: /opt/bootstrap/layout filesystem: root mode: 0544 diff --git a/digital-ocean/container-linux/kubernetes/cl/worker.yaml.tmpl b/digital-ocean/container-linux/kubernetes/cl/worker.yaml.tmpl index 3ca5f7912..5891b719f 100644 --- a/digital-ocean/container-linux/kubernetes/cl/worker.yaml.tmpl +++ b/digital-ocean/container-linux/kubernetes/cl/worker.yaml.tmpl @@ -99,7 +99,7 @@ storage: contents: inline: | KUBELET_IMAGE_URL=docker://k8s.gcr.io/hyperkube - KUBELET_IMAGE_TAG=v1.16.3 + KUBELET_IMAGE_TAG=v1.17.0 - path: /etc/sysctl.d/max-user-watches.conf filesystem: root contents: @@ -117,7 +117,7 @@ storage: --volume config,kind=host,source=/etc/kubernetes \ --mount volume=config,target=/etc/kubernetes \ --insecure-options=image \ - docker://k8s.gcr.io/hyperkube:v1.16.3 \ + docker://k8s.gcr.io/hyperkube:v1.17.0 \ --net=host \ --dns=host \ --exec=/kubectl -- --kubeconfig=/etc/kubernetes/kubeconfig delete node $(hostname) diff --git a/docs/advanced/worker-pools.md b/docs/advanced/worker-pools.md index 645fc3ee6..bcb2d6381 100644 --- a/docs/advanced/worker-pools.md +++ b/docs/advanced/worker-pools.md @@ -79,7 +79,7 @@ Create a cluster following the Azure [tutorial](../cl/azure.md#cluster). Define ```tf module "ramius-worker-pool" { - source = "git::https://github.com/poseidon/typhoon//azure/container-linux/kubernetes/workers?ref=v1.16.3" + source = "git::https://github.com/poseidon/typhoon//azure/container-linux/kubernetes/workers?ref=v1.17.0" # Azure region = module.ramius.region @@ -145,7 +145,7 @@ Create a cluster following the Google Cloud [tutorial](../cl/google-cloud.md#clu ```tf module "yavin-worker-pool" { - source = "git::https://github.com/poseidon/typhoon//google-cloud/container-linux/kubernetes/workers?ref=v1.16.3" + source = "git::https://github.com/poseidon/typhoon//google-cloud/container-linux/kubernetes/workers?ref=v1.17.0" # Google Cloud region = "europe-west2" @@ -176,11 +176,11 @@ Verify a managed instance group of workers joins the cluster within a few minute ``` $ kubectl get nodes NAME STATUS AGE VERSION -yavin-controller-0.c.example-com.internal Ready 6m v1.16.3 -yavin-worker-jrbf.c.example-com.internal Ready 5m v1.16.3 -yavin-worker-mzdm.c.example-com.internal Ready 5m v1.16.3 -yavin-16x-worker-jrbf.c.example-com.internal Ready 3m v1.16.3 -yavin-16x-worker-mzdm.c.example-com.internal Ready 3m v1.16.3 +yavin-controller-0.c.example-com.internal Ready 6m v1.17.0 +yavin-worker-jrbf.c.example-com.internal Ready 5m v1.17.0 +yavin-worker-mzdm.c.example-com.internal Ready 5m v1.17.0 +yavin-16x-worker-jrbf.c.example-com.internal Ready 3m v1.17.0 +yavin-16x-worker-mzdm.c.example-com.internal Ready 3m v1.17.0 ``` ### Variables diff --git a/docs/cl/aws.md b/docs/cl/aws.md index 52c046dcc..1bb04d037 100644 --- a/docs/cl/aws.md +++ b/docs/cl/aws.md @@ -1,6 +1,6 @@ # AWS -In this tutorial, we'll create a Kubernetes v1.16.3 cluster on AWS with Container Linux. +In this tutorial, we'll create a Kubernetes v1.17.0 cluster on AWS with Container Linux. We'll declare a Kubernetes cluster using the Typhoon Terraform module. Then apply the changes to create a VPC, gateway, subnets, security groups, controller instances, worker auto-scaling group, network load balancer, and TLS assets. @@ -70,7 +70,7 @@ Define a Kubernetes cluster using the module `aws/container-linux/kubernetes`. ```tf module "tempest" { - source = "git::https://github.com/poseidon/typhoon//aws/container-linux/kubernetes?ref=v1.16.3" + source = "git::https://github.com/poseidon/typhoon//aws/container-linux/kubernetes?ref=v1.17.0" # AWS cluster_name = "tempest" @@ -143,9 +143,9 @@ List nodes in the cluster. $ export KUBECONFIG=/home/user/.kube/configs/tempest-config $ kubectl get nodes NAME STATUS ROLES AGE VERSION -ip-10-0-3-155 Ready 10m v1.16.3 -ip-10-0-26-65 Ready 10m v1.16.3 -ip-10-0-41-21 Ready 10m v1.16.3 +ip-10-0-3-155 Ready 10m v1.17.0 +ip-10-0-26-65 Ready 10m v1.17.0 +ip-10-0-41-21 Ready 10m v1.17.0 ``` List the pods. diff --git a/docs/cl/azure.md b/docs/cl/azure.md index f2afd8155..3701baa63 100644 --- a/docs/cl/azure.md +++ b/docs/cl/azure.md @@ -3,7 +3,7 @@ !!! danger Typhoon for Azure is alpha. For production, use AWS, Google Cloud, or bare-metal. As Azure matures, check [errata](https://github.com/poseidon/typhoon/wiki/Errata) for known shortcomings. -In this tutorial, we'll create a Kubernetes v1.16.3 cluster on Azure with Container Linux. +In this tutorial, we'll create a Kubernetes v1.17.0 cluster on Azure with Container Linux. We'll declare a Kubernetes cluster using the Typhoon Terraform module. Then apply the changes to create a resource group, virtual network, subnets, security groups, controller availability set, worker scale set, load balancer, and TLS assets. @@ -66,7 +66,7 @@ Define a Kubernetes cluster using the module `azure/container-linux/kubernetes`. ```tf module "ramius" { - source = "git::https://github.com/poseidon/typhoon//azure/container-linux/kubernetes?ref=v1.16.3" + source = "git::https://github.com/poseidon/typhoon//azure/container-linux/kubernetes?ref=v1.17.0" # Azure cluster_name = "ramius" @@ -140,9 +140,9 @@ List nodes in the cluster. $ export KUBECONFIG=/home/user/.kube/configs/ramius-config $ kubectl get nodes NAME STATUS ROLES AGE VERSION -ramius-controller-0 Ready 24m v1.16.3 -ramius-worker-000001 Ready 25m v1.16.3 -ramius-worker-000002 Ready 24m v1.16.3 +ramius-controller-0 Ready 24m v1.17.0 +ramius-worker-000001 Ready 25m v1.17.0 +ramius-worker-000002 Ready 24m v1.17.0 ``` List the pods. diff --git a/docs/cl/bare-metal.md b/docs/cl/bare-metal.md index b67930b4b..3bf419d6c 100644 --- a/docs/cl/bare-metal.md +++ b/docs/cl/bare-metal.md @@ -1,6 +1,6 @@ # Bare-Metal -In this tutorial, we'll network boot and provision a Kubernetes v1.16.3 cluster on bare-metal with Container Linux. +In this tutorial, we'll network boot and provision a Kubernetes v1.17.0 cluster on bare-metal with Container Linux. First, we'll deploy a [Matchbox](https://github.com/poseidon/matchbox) service and setup a network boot environment. Then, we'll declare a Kubernetes cluster using the Typhoon Terraform module and power on machines. On PXE boot, machines will install Container Linux to disk, reboot into the disk install, and provision themselves as Kubernetes controllers or workers via Ignition. @@ -160,7 +160,7 @@ Define a Kubernetes cluster using the module `bare-metal/container-linux/kuberne ```tf module "mercury" { - source = "git::https://github.com/poseidon/typhoon//bare-metal/container-linux/kubernetes?ref=v1.16.3" + source = "git::https://github.com/poseidon/typhoon//bare-metal/container-linux/kubernetes?ref=v1.17.0" # bare-metal cluster_name = "mercury" @@ -264,9 +264,9 @@ Apply complete! Resources: 55 added, 0 changed, 0 destroyed. To watch the install to disk (until machines reboot from disk), SSH to port 2222. ``` -# before v1.16.3 +# before v1.17.0 $ ssh debug@node1.example.com -# after v1.16.3 +# after v1.17.0 $ ssh -p 2222 core@node1.example.com ``` @@ -299,9 +299,9 @@ List nodes in the cluster. $ export KUBECONFIG=/home/user/.kube/configs/mercury-config $ kubectl get nodes NAME STATUS ROLES AGE VERSION -node1.example.com Ready 10m v1.16.3 -node2.example.com Ready 10m v1.16.3 -node3.example.com Ready 10m v1.16.3 +node1.example.com Ready 10m v1.17.0 +node2.example.com Ready 10m v1.17.0 +node3.example.com Ready 10m v1.17.0 ``` List the pods. diff --git a/docs/cl/digital-ocean.md b/docs/cl/digital-ocean.md index 973bd5241..a8ffa2cfe 100644 --- a/docs/cl/digital-ocean.md +++ b/docs/cl/digital-ocean.md @@ -1,6 +1,6 @@ # Digital Ocean -In this tutorial, we'll create a Kubernetes v1.16.3 cluster on DigitalOcean with Container Linux. +In this tutorial, we'll create a Kubernetes v1.17.0 cluster on DigitalOcean with Container Linux. We'll declare a Kubernetes cluster using the Typhoon Terraform module. Then apply the changes to create controller droplets, worker droplets, DNS records, tags, and TLS assets. @@ -65,7 +65,7 @@ Define a Kubernetes cluster using the module `digital-ocean/container-linux/kube ```tf module "nemo" { - source = "git::https://github.com/poseidon/typhoon//digital-ocean/container-linux/kubernetes?ref=v1.16.3" + source = "git::https://github.com/poseidon/typhoon//digital-ocean/container-linux/kubernetes?ref=v1.17.0" # Digital Ocean cluster_name = "nemo" @@ -138,9 +138,9 @@ List nodes in the cluster. $ export KUBECONFIG=/home/user/.kube/configs/nemo-config $ kubectl get nodes NAME STATUS ROLES AGE VERSION -10.132.110.130 Ready 10m v1.16.3 -10.132.115.81 Ready 10m v1.16.3 -10.132.124.107 Ready 10m v1.16.3 +10.132.110.130 Ready 10m v1.17.0 +10.132.115.81 Ready 10m v1.17.0 +10.132.124.107 Ready 10m v1.17.0 ``` List the pods. diff --git a/docs/cl/google-cloud.md b/docs/cl/google-cloud.md index 243b437a7..7173f9fb4 100644 --- a/docs/cl/google-cloud.md +++ b/docs/cl/google-cloud.md @@ -1,6 +1,6 @@ # Google Cloud -In this tutorial, we'll create a Kubernetes v1.16.3 cluster on Google Compute Engine with Container Linux. +In this tutorial, we'll create a Kubernetes v1.17.0 cluster on Google Compute Engine with Container Linux. We'll declare a Kubernetes cluster using the Typhoon Terraform module. Then apply the changes to create a network, firewall rules, health checks, controller instances, worker managed instance group, load balancers, and TLS assets. @@ -71,7 +71,7 @@ Define a Kubernetes cluster using the module `google-cloud/container-linux/kuber ```tf module "yavin" { - source = "git::https://github.com/poseidon/typhoon//google-cloud/container-linux/kubernetes?ref=v1.16.3" + source = "git::https://github.com/poseidon/typhoon//google-cloud/container-linux/kubernetes?ref=v1.17.0" # Google Cloud cluster_name = "yavin" @@ -145,9 +145,9 @@ List nodes in the cluster. $ export KUBECONFIG=/home/user/.kube/configs/yavin-config $ kubectl get nodes NAME ROLES STATUS AGE VERSION -yavin-controller-0.c.example-com.internal Ready 6m v1.16.3 -yavin-worker-jrbf.c.example-com.internal Ready 5m v1.16.3 -yavin-worker-mzdm.c.example-com.internal Ready 5m v1.16.3 +yavin-controller-0.c.example-com.internal Ready 6m v1.17.0 +yavin-worker-jrbf.c.example-com.internal Ready 5m v1.17.0 +yavin-worker-mzdm.c.example-com.internal Ready 5m v1.17.0 ``` List the pods. diff --git a/docs/fedora-coreos/aws.md b/docs/fedora-coreos/aws.md index b0f35a129..ba3c10c1a 100644 --- a/docs/fedora-coreos/aws.md +++ b/docs/fedora-coreos/aws.md @@ -3,7 +3,7 @@ !!! danger Typhoon for Fedora CoreOS is an early preview! Fedora CoreOS itself is a preview! Expect bugs and design shifts. Please help both projects solve problems. Report Fedora CoreOS bugs to [Fedora](https://github.com/coreos/fedora-coreos-tracker/issues). Report Typhoon issues to Typhoon. -In this tutorial, we'll create a Kubernetes v1.16.3 cluster on AWS with Fedora CoreOS. +In this tutorial, we'll create a Kubernetes v1.17.0 cluster on AWS with Fedora CoreOS. We'll declare a Kubernetes cluster using the Typhoon Terraform module. Then apply the changes to create a VPC, gateway, subnets, security groups, controller instances, worker auto-scaling group, network load balancer, and TLS assets. @@ -73,7 +73,7 @@ Define a Kubernetes cluster using the module `aws/fedora-coreos/kubernetes`. ```tf module "tempest" { - source = "git::https://github.com/poseidon/typhoon//aws/fedora-coreos/kubernetes?ref=v1.16.3" + source = "git::https://github.com/poseidon/typhoon//aws/fedora-coreos/kubernetes?ref=v1.17.0" # AWS cluster_name = "tempest" @@ -146,9 +146,9 @@ List nodes in the cluster. $ export KUBECONFIG=/home/user/.kube/configs/tempest-config $ kubectl get nodes NAME STATUS ROLES AGE VERSION -ip-10-0-3-155 Ready 10m v1.16.3 -ip-10-0-26-65 Ready 10m v1.16.3 -ip-10-0-41-21 Ready 10m v1.16.3 +ip-10-0-3-155 Ready 10m v1.17.0 +ip-10-0-26-65 Ready 10m v1.17.0 +ip-10-0-41-21 Ready 10m v1.17.0 ``` List the pods. diff --git a/docs/fedora-coreos/bare-metal.md b/docs/fedora-coreos/bare-metal.md index 056f24b4c..8f10bf6a3 100644 --- a/docs/fedora-coreos/bare-metal.md +++ b/docs/fedora-coreos/bare-metal.md @@ -3,7 +3,7 @@ !!! danger Typhoon for Fedora CoreOS is an early preview! Fedora CoreOS itself is a preview! Expect bugs and design shifts. Please help both projects solve problems. Report Fedora CoreOS bugs to [Fedora](https://github.com/coreos/fedora-coreos-tracker/issues). Report Typhoon issues to Typhoon. -In this tutorial, we'll network boot and provision a Kubernetes v1.16.3 cluster on bare-metal with Fedora CoreOS. +In this tutorial, we'll network boot and provision a Kubernetes v1.17.0 cluster on bare-metal with Fedora CoreOS. First, we'll deploy a [Matchbox](https://github.com/poseidon/matchbox) service and setup a network boot environment. Then, we'll declare a Kubernetes cluster using the Typhoon Terraform module and power on machines. On PXE boot, machines will install Fedora CoreOS to disk, reboot into the disk install, and provision themselves as Kubernetes controllers or workers via Ignition. @@ -163,7 +163,7 @@ Define a Kubernetes cluster using the module `bare-metal/fedora-coreos/kubernete ```tf module "mercury" { - source = "git::https://github.com/poseidon/typhoon//bare-metal/fedora-coreos/kubernetes?ref=v1.16.3" + source = "git::https://github.com/poseidon/typhoon//bare-metal/fedora-coreos/kubernetes?ref=v1.17.0" # bare-metal cluster_name = "mercury" @@ -293,9 +293,9 @@ List nodes in the cluster. $ export KUBECONFIG=/home/user/.kube/configs/mercury-config $ kubectl get nodes NAME STATUS ROLES AGE VERSION -node1.example.com Ready 10m v1.16.3 -node2.example.com Ready 10m v1.16.3 -node3.example.com Ready 10m v1.16.3 +node1.example.com Ready 10m v1.17.0 +node2.example.com Ready 10m v1.17.0 +node3.example.com Ready 10m v1.17.0 ``` List the pods. diff --git a/docs/index.md b/docs/index.md index 45c2a0e67..8090ae06e 100644 --- a/docs/index.md +++ b/docs/index.md @@ -11,7 +11,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features -* Kubernetes v1.16.3 (upstream) +* Kubernetes v1.17.0 (upstream) * Single or multi-master, [Calico](https://www.projectcalico.org/) or [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/) * Advanced features like [worker pools](advanced/worker-pools/), [preemptible](cl/google-cloud/#preemption) workers, and [snippets](advanced/customization/#container-linux) customization @@ -47,7 +47,7 @@ Define a Kubernetes cluster by using the Terraform module for your chosen platfo ```tf module "yavin" { - source = "git::https://github.com/poseidon/typhoon//google-cloud/container-linux/kubernetes?ref=v1.16.3" + source = "git::https://github.com/poseidon/typhoon//google-cloud/container-linux/kubernetes?ref=v1.17.0" # Google Cloud cluster_name = "yavin" @@ -85,9 +85,9 @@ In 4-8 minutes (varies by platform), the cluster will be ready. This Google Clou $ export KUBECONFIG=/home/user/.kube/configs/yavin-config $ kubectl get nodes NAME ROLES STATUS AGE VERSION -yavin-controller-0.c.example-com.internal Ready 6m v1.16.3 -yavin-worker-jrbf.c.example-com.internal Ready 5m v1.16.3 -yavin-worker-mzdm.c.example-com.internal Ready 5m v1.16.3 +yavin-controller-0.c.example-com.internal Ready 6m v1.17.0 +yavin-worker-jrbf.c.example-com.internal Ready 5m v1.17.0 +yavin-worker-mzdm.c.example-com.internal Ready 5m v1.17.0 ``` List the pods. diff --git a/docs/topics/maintenance.md b/docs/topics/maintenance.md index 9bd6b18f7..bd97b62eb 100644 --- a/docs/topics/maintenance.md +++ b/docs/topics/maintenance.md @@ -18,7 +18,7 @@ module "yavin" { } module "mercury" { - source = "git::https://github.com/poseidon/typhoon//bare-metal/container-linux/kubernetes?ref=v1.16.3" + source = "git::https://github.com/poseidon/typhoon//bare-metal/container-linux/kubernetes?ref=v1.17.0" ... } ``` @@ -279,15 +279,15 @@ Typhoon modules have been adapted for Terraform v0.12. Provider plugins requirem | Typhoon Release | Terraform version | |-------------------|---------------------| -| v1.16.3 - ? | v0.12.x | -| v1.10.3 - v1.16.3 | v0.11.x | +| v1.17.0 - ? | v0.12.x | +| v1.10.3 - v1.17.0 | v0.11.x | | v1.9.2 - v1.10.2 | v0.10.4+ or v0.11.x | | v1.7.3 - v1.9.1 | v0.10.x | | v1.6.4 - v1.7.2 | v0.9.x | ### New users -New users can start with Terraform v0.12.x and follow the docs for Typhoon v1.16.3+ without issue. +New users can start with Terraform v0.12.x and follow the docs for Typhoon v1.17.0+ without issue. ### Existing users @@ -404,7 +404,7 @@ tree . └── infraB <- new Terraform v0.12.x configs ``` -Define Typhoon clusters in the new config directory using Terraform v0.12 syntax. Follow the Typhoon v1.16.3+ docs (e.g. use `terraform12` in the `infraB` dir). See [AWS](/cl/aws), [Azure](/cl/azure), [Bare-Metal](/cl/bare-metal), [Digital Ocean](/cl/digital-ocean), or [Google-Cloud](/cl/google-cloud)) to create new clusters. Follow the usual [upgrade](/topics/maintenance/#upgrades) process to apply workloads and shift traffic. Later, switch back to the old config directory and deprovision clusters with Terraform v0.11. +Define Typhoon clusters in the new config directory using Terraform v0.12 syntax. Follow the Typhoon v1.17.0+ docs (e.g. use `terraform12` in the `infraB` dir). See [AWS](/cl/aws), [Azure](/cl/azure), [Bare-Metal](/cl/bare-metal), [Digital Ocean](/cl/digital-ocean), or [Google-Cloud](/cl/google-cloud)) to create new clusters. Follow the usual [upgrade](/topics/maintenance/#upgrades) process to apply workloads and shift traffic. Later, switch back to the old config directory and deprovision clusters with Terraform v0.11. ```shell terraform12 init diff --git a/google-cloud/container-linux/kubernetes/README.md b/google-cloud/container-linux/kubernetes/README.md index 16cac3264..0f96f6eb1 100644 --- a/google-cloud/container-linux/kubernetes/README.md +++ b/google-cloud/container-linux/kubernetes/README.md @@ -11,7 +11,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features -* Kubernetes v1.16.3 (upstream) +* Kubernetes v1.17.0 (upstream) * Single or multi-master, [Calico](https://www.projectcalico.org/) or [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/) * Advanced features like [worker pools](https://typhoon.psdn.io/advanced/worker-pools/), [preemptible](https://typhoon.psdn.io/cl/google-cloud/#preemption) workers, and [snippets](https://typhoon.psdn.io/advanced/customization/#container-linux) customization diff --git a/google-cloud/container-linux/kubernetes/bootstrap.tf b/google-cloud/container-linux/kubernetes/bootstrap.tf index de9e2e7e0..1a45eeecb 100644 --- a/google-cloud/container-linux/kubernetes/bootstrap.tf +++ b/google-cloud/container-linux/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=4369c706e2432151211bd33aa18a8ea7d53aaa11" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=0ddd90fd05764796758991dfc532ae5cbbeb66b5" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/google-cloud/container-linux/kubernetes/cl/controller.yaml.tmpl b/google-cloud/container-linux/kubernetes/cl/controller.yaml.tmpl index caf120ee9..75aa1be6e 100644 --- a/google-cloud/container-linux/kubernetes/cl/controller.yaml.tmpl +++ b/google-cloud/container-linux/kubernetes/cl/controller.yaml.tmpl @@ -114,7 +114,7 @@ systemd: --volume script,kind=host,source=/opt/bootstrap/apply \ --mount volume=script,target=/apply \ --insecure-options=image \ - docker://k8s.gcr.io/hyperkube:v1.16.3 \ + docker://k8s.gcr.io/hyperkube:v1.17.0 \ --net=host \ --dns=host \ --exec=/apply @@ -135,7 +135,7 @@ storage: contents: inline: | KUBELET_IMAGE_URL=docker://k8s.gcr.io/hyperkube - KUBELET_IMAGE_TAG=v1.16.3 + KUBELET_IMAGE_TAG=v1.17.0 - path: /opt/bootstrap/layout filesystem: root mode: 0544 diff --git a/google-cloud/container-linux/kubernetes/workers/cl/worker.yaml.tmpl b/google-cloud/container-linux/kubernetes/workers/cl/worker.yaml.tmpl index c7c1bd6d5..4186ccd5c 100644 --- a/google-cloud/container-linux/kubernetes/workers/cl/worker.yaml.tmpl +++ b/google-cloud/container-linux/kubernetes/workers/cl/worker.yaml.tmpl @@ -97,7 +97,7 @@ storage: contents: inline: | KUBELET_IMAGE_URL=docker://k8s.gcr.io/hyperkube - KUBELET_IMAGE_TAG=v1.16.3 + KUBELET_IMAGE_TAG=v1.17.0 - path: /etc/sysctl.d/max-user-watches.conf filesystem: root contents: @@ -115,7 +115,7 @@ storage: --volume config,kind=host,source=/etc/kubernetes \ --mount volume=config,target=/etc/kubernetes \ --insecure-options=image \ - docker://k8s.gcr.io/hyperkube:v1.16.3 \ + docker://k8s.gcr.io/hyperkube:v1.17.0 \ --net=host \ --dns=host \ --exec=/kubectl -- --kubeconfig=/etc/kubernetes/kubeconfig delete node $(hostname) From ed3550dce173f4e146de1ad64fe147b76d47c6b6 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Thu, 21 Nov 2019 23:00:25 -0800 Subject: [PATCH 285/523] Update systemd services for the v0.17.x hyperkube * Binary asset locations within the upstream hyperkube image changed https://github.com/kubernetes/kubernetes/pull/84662 * Fix Container Linux and Flatcar Linux kubelet.service (rkt-fly with fairly dated CoreOS kubelet-wrapper) * Fix Fedora CoreOS kubelet.service (podman) * Fix Fedora CoreOS bootstrap.service * Fix delete-node kubectl usage for workers where nodes may delete themselves on shutdown (e.g. preemptible instances) --- aws/container-linux/kubernetes/cl/controller.yaml.tmpl | 1 + aws/container-linux/kubernetes/workers/cl/worker.yaml.tmpl | 4 +++- aws/fedora-coreos/kubernetes/fcc/controller.yaml | 6 +++--- aws/fedora-coreos/kubernetes/workers/fcc/worker.yaml | 2 +- azure/container-linux/kubernetes/cl/controller.yaml.tmpl | 1 + .../container-linux/kubernetes/workers/cl/worker.yaml.tmpl | 4 +++- .../container-linux/kubernetes/cl/controller.yaml.tmpl | 1 + bare-metal/container-linux/kubernetes/cl/worker.yaml.tmpl | 1 + bare-metal/fedora-coreos/kubernetes/fcc/controller.yaml | 6 +++--- bare-metal/fedora-coreos/kubernetes/fcc/worker.yaml | 2 +- .../container-linux/kubernetes/cl/controller.yaml.tmpl | 1 + .../container-linux/kubernetes/cl/worker.yaml.tmpl | 4 +++- .../container-linux/kubernetes/cl/controller.yaml.tmpl | 1 + .../container-linux/kubernetes/workers/cl/worker.yaml.tmpl | 4 +++- 14 files changed, 26 insertions(+), 12 deletions(-) diff --git a/aws/container-linux/kubernetes/cl/controller.yaml.tmpl b/aws/container-linux/kubernetes/cl/controller.yaml.tmpl index 53229485a..9240653c8 100644 --- a/aws/container-linux/kubernetes/cl/controller.yaml.tmpl +++ b/aws/container-linux/kubernetes/cl/controller.yaml.tmpl @@ -137,6 +137,7 @@ storage: inline: | KUBELET_IMAGE_URL=docker://k8s.gcr.io/hyperkube KUBELET_IMAGE_TAG=v1.17.0 + KUBELET_IMAGE_ARGS="--exec=/usr/local/bin/kubelet" - path: /opt/bootstrap/layout filesystem: root mode: 0544 diff --git a/aws/container-linux/kubernetes/workers/cl/worker.yaml.tmpl b/aws/container-linux/kubernetes/workers/cl/worker.yaml.tmpl index 8e0ee0fb1..6a36c8ce8 100644 --- a/aws/container-linux/kubernetes/workers/cl/worker.yaml.tmpl +++ b/aws/container-linux/kubernetes/workers/cl/worker.yaml.tmpl @@ -99,6 +99,7 @@ storage: inline: | KUBELET_IMAGE_URL=docker://k8s.gcr.io/hyperkube KUBELET_IMAGE_TAG=v1.17.0 + KUBELET_IMAGE_ARGS="--exec=/usr/local/bin/kubelet" - path: /etc/sysctl.d/max-user-watches.conf filesystem: root contents: @@ -119,7 +120,8 @@ storage: docker://k8s.gcr.io/hyperkube:v1.17.0 \ --net=host \ --dns=host \ - --exec=/kubectl -- --kubeconfig=/etc/kubernetes/kubeconfig delete node $(hostname) + -- \ + kubectl --kubeconfig=/etc/kubernetes/kubeconfig delete node $(hostname) passwd: users: - name: core diff --git a/aws/fedora-coreos/kubernetes/fcc/controller.yaml b/aws/fedora-coreos/kubernetes/fcc/controller.yaml index e21f14c1f..f19ba8ddf 100644 --- a/aws/fedora-coreos/kubernetes/fcc/controller.yaml +++ b/aws/fedora-coreos/kubernetes/fcc/controller.yaml @@ -80,7 +80,7 @@ systemd: --volume /var/run:/var/run \ --volume /var/run/lock:/var/run/lock:z \ --volume /opt/cni/bin:/opt/cni/bin:z \ - k8s.gcr.io/hyperkube:v1.17.0 /hyperkube kubelet \ + k8s.gcr.io/hyperkube:v1.17.0 kubelet \ --anonymous-auth=false \ --authentication-token-webhook \ --authorization-mode=Webhook \ @@ -122,8 +122,8 @@ systemd: --volume /etc/kubernetes/bootstrap-secrets:/etc/kubernetes/secrets:ro,Z \ --volume /opt/bootstrap/assets:/assets:ro,Z \ --volume /opt/bootstrap/apply:/apply:ro,Z \ - k8s.gcr.io/hyperkube:v1.17.0 \ - /apply + --entrypoint=/apply \ + k8s.gcr.io/hyperkube:v1.17.0 ExecStartPost=/bin/touch /opt/bootstrap/bootstrap.done ExecStartPost=-/usr/bin/podman stop bootstrap storage: diff --git a/aws/fedora-coreos/kubernetes/workers/fcc/worker.yaml b/aws/fedora-coreos/kubernetes/workers/fcc/worker.yaml index fa7505cd0..d987c8707 100644 --- a/aws/fedora-coreos/kubernetes/workers/fcc/worker.yaml +++ b/aws/fedora-coreos/kubernetes/workers/fcc/worker.yaml @@ -50,7 +50,7 @@ systemd: --volume /var/run:/var/run \ --volume /var/run/lock:/var/run/lock:z \ --volume /opt/cni/bin:/opt/cni/bin:z \ - k8s.gcr.io/hyperkube:v1.17.0 /hyperkube kubelet \ + k8s.gcr.io/hyperkube:v1.17.0 kubelet \ --anonymous-auth=false \ --authentication-token-webhook \ --authorization-mode=Webhook \ diff --git a/azure/container-linux/kubernetes/cl/controller.yaml.tmpl b/azure/container-linux/kubernetes/cl/controller.yaml.tmpl index 8c54985d4..80f53cd36 100644 --- a/azure/container-linux/kubernetes/cl/controller.yaml.tmpl +++ b/azure/container-linux/kubernetes/cl/controller.yaml.tmpl @@ -135,6 +135,7 @@ storage: inline: | KUBELET_IMAGE_URL=docker://k8s.gcr.io/hyperkube KUBELET_IMAGE_TAG=v1.17.0 + KUBELET_IMAGE_ARGS="--exec=/usr/local/bin/kubelet" - path: /opt/bootstrap/layout filesystem: root mode: 0544 diff --git a/azure/container-linux/kubernetes/workers/cl/worker.yaml.tmpl b/azure/container-linux/kubernetes/workers/cl/worker.yaml.tmpl index a12754500..d29ecb367 100644 --- a/azure/container-linux/kubernetes/workers/cl/worker.yaml.tmpl +++ b/azure/container-linux/kubernetes/workers/cl/worker.yaml.tmpl @@ -97,6 +97,7 @@ storage: inline: | KUBELET_IMAGE_URL=docker://k8s.gcr.io/hyperkube KUBELET_IMAGE_TAG=v1.17.0 + KUBELET_IMAGE_ARGS="--exec=/usr/local/bin/kubelet" - path: /etc/sysctl.d/max-user-watches.conf filesystem: root contents: @@ -117,7 +118,8 @@ storage: docker://k8s.gcr.io/hyperkube:v1.17.0 \ --net=host \ --dns=host \ - --exec=/kubectl -- --kubeconfig=/etc/kubernetes/kubeconfig delete node $(hostname | tr '[:upper:]' '[:lower:]') + -- \ + kubectl -- --kubeconfig=/etc/kubernetes/kubeconfig delete node $(hostname | tr '[:upper:]' '[:lower:]') passwd: users: - name: core diff --git a/bare-metal/container-linux/kubernetes/cl/controller.yaml.tmpl b/bare-metal/container-linux/kubernetes/cl/controller.yaml.tmpl index 8869eb237..ddc943f98 100644 --- a/bare-metal/container-linux/kubernetes/cl/controller.yaml.tmpl +++ b/bare-metal/container-linux/kubernetes/cl/controller.yaml.tmpl @@ -144,6 +144,7 @@ storage: inline: | KUBELET_IMAGE_URL=docker://k8s.gcr.io/hyperkube KUBELET_IMAGE_TAG=v1.17.0 + KUBELET_IMAGE_ARGS="--exec=/usr/local/bin/kubelet" - path: /etc/hostname filesystem: root mode: 0644 diff --git a/bare-metal/container-linux/kubernetes/cl/worker.yaml.tmpl b/bare-metal/container-linux/kubernetes/cl/worker.yaml.tmpl index a7ac5eec8..efe18ddc9 100644 --- a/bare-metal/container-linux/kubernetes/cl/worker.yaml.tmpl +++ b/bare-metal/container-linux/kubernetes/cl/worker.yaml.tmpl @@ -92,6 +92,7 @@ storage: inline: | KUBELET_IMAGE_URL=docker://k8s.gcr.io/hyperkube KUBELET_IMAGE_TAG=v1.17.0 + KUBELET_IMAGE_ARGS="--exec=/usr/local/bin/kubelet" - path: /etc/hostname filesystem: root mode: 0644 diff --git a/bare-metal/fedora-coreos/kubernetes/fcc/controller.yaml b/bare-metal/fedora-coreos/kubernetes/fcc/controller.yaml index bec262a4c..196d364cc 100644 --- a/bare-metal/fedora-coreos/kubernetes/fcc/controller.yaml +++ b/bare-metal/fedora-coreos/kubernetes/fcc/controller.yaml @@ -81,7 +81,7 @@ systemd: --volume /opt/cni/bin:/opt/cni/bin:z \ --volume /etc/iscsi:/etc/iscsi \ --volume /sbin/iscsiadm:/sbin/iscsiadm \ - k8s.gcr.io/hyperkube:v1.17.0 /hyperkube kubelet \ + k8s.gcr.io/hyperkube:v1.17.0 kubelet \ --anonymous-auth=false \ --authentication-token-webhook \ --authorization-mode=Webhook \ @@ -133,8 +133,8 @@ systemd: --volume /etc/kubernetes/bootstrap-secrets:/etc/kubernetes/secrets:ro,Z \ --volume /opt/bootstrap/assets:/assets:ro,Z \ --volume /opt/bootstrap/apply:/apply:ro,Z \ - k8s.gcr.io/hyperkube:v1.17.0 \ - /apply + --entrypoint=/apply \ + k8s.gcr.io/hyperkube:v1.17.0 ExecStartPost=/bin/touch /opt/bootstrap/bootstrap.done ExecStartPost=-/usr/bin/podman stop bootstrap storage: diff --git a/bare-metal/fedora-coreos/kubernetes/fcc/worker.yaml b/bare-metal/fedora-coreos/kubernetes/fcc/worker.yaml index f10ccae47..600be9c4b 100644 --- a/bare-metal/fedora-coreos/kubernetes/fcc/worker.yaml +++ b/bare-metal/fedora-coreos/kubernetes/fcc/worker.yaml @@ -51,7 +51,7 @@ systemd: --volume /opt/cni/bin:/opt/cni/bin:z \ --volume /etc/iscsi:/etc/iscsi \ --volume /sbin/iscsiadm:/sbin/iscsiadm \ - k8s.gcr.io/hyperkube:v1.17.0 /hyperkube kubelet \ + k8s.gcr.io/hyperkube:v1.17.0 kubelet \ --anonymous-auth=false \ --authentication-token-webhook \ --authorization-mode=Webhook \ diff --git a/digital-ocean/container-linux/kubernetes/cl/controller.yaml.tmpl b/digital-ocean/container-linux/kubernetes/cl/controller.yaml.tmpl index bd0f97f1b..d51f24fe8 100644 --- a/digital-ocean/container-linux/kubernetes/cl/controller.yaml.tmpl +++ b/digital-ocean/container-linux/kubernetes/cl/controller.yaml.tmpl @@ -141,6 +141,7 @@ storage: inline: | KUBELET_IMAGE_URL=docker://k8s.gcr.io/hyperkube KUBELET_IMAGE_TAG=v1.17.0 + KUBELET_IMAGE_ARGS="--exec=/usr/local/bin/kubelet" - path: /opt/bootstrap/layout filesystem: root mode: 0544 diff --git a/digital-ocean/container-linux/kubernetes/cl/worker.yaml.tmpl b/digital-ocean/container-linux/kubernetes/cl/worker.yaml.tmpl index 5891b719f..4ae4603e5 100644 --- a/digital-ocean/container-linux/kubernetes/cl/worker.yaml.tmpl +++ b/digital-ocean/container-linux/kubernetes/cl/worker.yaml.tmpl @@ -100,6 +100,7 @@ storage: inline: | KUBELET_IMAGE_URL=docker://k8s.gcr.io/hyperkube KUBELET_IMAGE_TAG=v1.17.0 + KUBELET_IMAGE_ARGS="--exec=/usr/local/bin/kubelet" - path: /etc/sysctl.d/max-user-watches.conf filesystem: root contents: @@ -120,4 +121,5 @@ storage: docker://k8s.gcr.io/hyperkube:v1.17.0 \ --net=host \ --dns=host \ - --exec=/kubectl -- --kubeconfig=/etc/kubernetes/kubeconfig delete node $(hostname) + -- \ + kubectl -- --kubeconfig=/etc/kubernetes/kubeconfig delete node $(hostname) diff --git a/google-cloud/container-linux/kubernetes/cl/controller.yaml.tmpl b/google-cloud/container-linux/kubernetes/cl/controller.yaml.tmpl index 75aa1be6e..b2baaa048 100644 --- a/google-cloud/container-linux/kubernetes/cl/controller.yaml.tmpl +++ b/google-cloud/container-linux/kubernetes/cl/controller.yaml.tmpl @@ -136,6 +136,7 @@ storage: inline: | KUBELET_IMAGE_URL=docker://k8s.gcr.io/hyperkube KUBELET_IMAGE_TAG=v1.17.0 + KUBELET_IMAGE_ARGS="--exec=/usr/local/bin/kubelet" - path: /opt/bootstrap/layout filesystem: root mode: 0544 diff --git a/google-cloud/container-linux/kubernetes/workers/cl/worker.yaml.tmpl b/google-cloud/container-linux/kubernetes/workers/cl/worker.yaml.tmpl index 4186ccd5c..4d4bae02e 100644 --- a/google-cloud/container-linux/kubernetes/workers/cl/worker.yaml.tmpl +++ b/google-cloud/container-linux/kubernetes/workers/cl/worker.yaml.tmpl @@ -98,6 +98,7 @@ storage: inline: | KUBELET_IMAGE_URL=docker://k8s.gcr.io/hyperkube KUBELET_IMAGE_TAG=v1.17.0 + KUBELET_IMAGE_ARGS="--exec=/usr/local/bin/kubelet" - path: /etc/sysctl.d/max-user-watches.conf filesystem: root contents: @@ -118,7 +119,8 @@ storage: docker://k8s.gcr.io/hyperkube:v1.17.0 \ --net=host \ --dns=host \ - --exec=/kubectl -- --kubeconfig=/etc/kubernetes/kubeconfig delete node $(hostname) + -- \ + kubectl -- --kubeconfig=/etc/kubernetes/kubeconfig delete node $(hostname) passwd: users: - name: core From c0ce04e1de212271a465cfb5fc627f750047c809 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Mon, 9 Dec 2019 21:03:00 -0800 Subject: [PATCH 286/523] Update Calico from v3.10.1 to v3.10.2 * https://docs.projectcalico.org/v3.10/release-notes/ --- CHANGES.md | 10 +++++----- aws/container-linux/kubernetes/bootstrap.tf | 2 +- aws/fedora-coreos/kubernetes/bootstrap.tf | 2 +- azure/container-linux/kubernetes/bootstrap.tf | 2 +- bare-metal/container-linux/kubernetes/bootstrap.tf | 2 +- bare-metal/fedora-coreos/kubernetes/bootstrap.tf | 2 +- digital-ocean/container-linux/kubernetes/bootstrap.tf | 2 +- google-cloud/container-linux/kubernetes/bootstrap.tf | 2 +- 8 files changed, 12 insertions(+), 12 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index cbb2aeb20..07924f0da 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -6,15 +6,15 @@ Notable changes between versions. * Kubernetes [v1.17.0](https://github.com/kubernetes/kubernetes/blob/master/CHANGELOG-1.17.md#v1170) * Manage clusters without using a local `asset_dir` ([#595](https://github.com/poseidon/typhoon/pull/595)) - * Change `asset_dir` to be optional. Default to "" to skip writing assets locally - * Keep cluster assets (TLS materials, kubeconfig) only in Terraform state, which supports different [remote backends](https://www.terraform.io/docs/backends/types/remote.html) and optional encryption at rest - * Allow `terraform apply` from stateless automation systems - * Improve asset unpacking on controllers to remove unused materials - * Obtain a kubeconfig from Terraform module outputs + * Change `asset_dir` to be optional. Remove the variable to skip writing assets locally (action recommended) + * Allows cluster assets to be kept in Terraform state (pluggable [remote backends](https://www.terraform.io/docs/backends/types/remote.html), encryption) and `terraform apply` from stateless automation systems + * Improve asset unpacking on controllers + * Obtain kubeconfig from Terraform module outputs * Replace usage of `template_dir` with `templatefile` function ([#587](https://github.com/poseidon/typhoon/pull/587)) * Require Terraform version v0.12.6+ (action required) * Update CoreDNS from v1.6.2 to v1.6.5 ([#588](https://github.com/poseidon/typhoon/pull/588)) * Add health `lameduck` option to wait before shutdown +* Update Calico from v3.10.1 to v3.10.2 * Reduce pod eviction timeout for deleting pods on unready nodes from 5m to 1m ([#597](https://github.com/poseidon/typhoon/pull/597)) * Present since [v1.13.3](#v1133), but mistakenly removed in v1.16.0 * Add CPU requests for control plane static pods ([#589](https://github.com/poseidon/typhoon/pull/589)) diff --git a/aws/container-linux/kubernetes/bootstrap.tf b/aws/container-linux/kubernetes/bootstrap.tf index 018dfa979..e178926fe 100644 --- a/aws/container-linux/kubernetes/bootstrap.tf +++ b/aws/container-linux/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=0ddd90fd05764796758991dfc532ae5cbbeb66b5" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=24e5513ee6da4888005aac02101f73fc9e5e829f" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/aws/fedora-coreos/kubernetes/bootstrap.tf b/aws/fedora-coreos/kubernetes/bootstrap.tf index a4dba5010..24efc816e 100644 --- a/aws/fedora-coreos/kubernetes/bootstrap.tf +++ b/aws/fedora-coreos/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=0ddd90fd05764796758991dfc532ae5cbbeb66b5" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=24e5513ee6da4888005aac02101f73fc9e5e829f" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/azure/container-linux/kubernetes/bootstrap.tf b/azure/container-linux/kubernetes/bootstrap.tf index 5b33ed43a..471320b46 100644 --- a/azure/container-linux/kubernetes/bootstrap.tf +++ b/azure/container-linux/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=0ddd90fd05764796758991dfc532ae5cbbeb66b5" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=24e5513ee6da4888005aac02101f73fc9e5e829f" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/bare-metal/container-linux/kubernetes/bootstrap.tf b/bare-metal/container-linux/kubernetes/bootstrap.tf index 7f6202d9e..d4e0430be 100644 --- a/bare-metal/container-linux/kubernetes/bootstrap.tf +++ b/bare-metal/container-linux/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=0ddd90fd05764796758991dfc532ae5cbbeb66b5" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=24e5513ee6da4888005aac02101f73fc9e5e829f" cluster_name = var.cluster_name api_servers = [var.k8s_domain_name] diff --git a/bare-metal/fedora-coreos/kubernetes/bootstrap.tf b/bare-metal/fedora-coreos/kubernetes/bootstrap.tf index a17e9f25a..80f4a8ec6 100644 --- a/bare-metal/fedora-coreos/kubernetes/bootstrap.tf +++ b/bare-metal/fedora-coreos/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=0ddd90fd05764796758991dfc532ae5cbbeb66b5" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=24e5513ee6da4888005aac02101f73fc9e5e829f" cluster_name = var.cluster_name api_servers = [var.k8s_domain_name] diff --git a/digital-ocean/container-linux/kubernetes/bootstrap.tf b/digital-ocean/container-linux/kubernetes/bootstrap.tf index ebbc25639..45c399611 100644 --- a/digital-ocean/container-linux/kubernetes/bootstrap.tf +++ b/digital-ocean/container-linux/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=0ddd90fd05764796758991dfc532ae5cbbeb66b5" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=24e5513ee6da4888005aac02101f73fc9e5e829f" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/google-cloud/container-linux/kubernetes/bootstrap.tf b/google-cloud/container-linux/kubernetes/bootstrap.tf index 1a45eeecb..209e34c92 100644 --- a/google-cloud/container-linux/kubernetes/bootstrap.tf +++ b/google-cloud/container-linux/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=0ddd90fd05764796758991dfc532ae5cbbeb66b5" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=24e5513ee6da4888005aac02101f73fc9e5e829f" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] From f69dc2ea0fd343c0a640848b71215d24875d69a1 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Tue, 10 Dec 2019 22:56:50 -0800 Subject: [PATCH 287/523] Update CHANGES and tutorial notes for release * Update recommended Terraform and provider plugin versions * Update the rough count of resources created per cluster since its not been refreshed in a while (will vary based on cluster options) --- CHANGES.md | 14 ++++++++------ README.md | 2 +- docs/cl/aws.md | 10 +++++----- docs/cl/azure.md | 10 +++++----- docs/cl/bare-metal.md | 6 +++--- docs/cl/digital-ocean.md | 8 ++++---- docs/cl/google-cloud.md | 10 +++++----- docs/fedora-coreos/aws.md | 10 +++++----- docs/fedora-coreos/bare-metal.md | 6 +++--- docs/index.md | 2 +- 10 files changed, 40 insertions(+), 38 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 07924f0da..a3129841d 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -4,17 +4,19 @@ Notable changes between versions. ## Latest +## v1.17.0 + * Kubernetes [v1.17.0](https://github.com/kubernetes/kubernetes/blob/master/CHANGELOG-1.17.md#v1170) * Manage clusters without using a local `asset_dir` ([#595](https://github.com/poseidon/typhoon/pull/595)) - * Change `asset_dir` to be optional. Remove the variable to skip writing assets locally (action recommended) - * Allows cluster assets to be kept in Terraform state (pluggable [remote backends](https://www.terraform.io/docs/backends/types/remote.html), encryption) and `terraform apply` from stateless automation systems + * Change `asset_dir` to be optional. Remove the variable to skip writing assets locally (**action recommended**) + * Allow keeping cluster assets only in Terraform state ([pluggable](https://www.terraform.io/docs/backends/types/remote.html), encryption) and allow `terraform apply` from stateless automation systems * Improve asset unpacking on controllers * Obtain kubeconfig from Terraform module outputs * Replace usage of `template_dir` with `templatefile` function ([#587](https://github.com/poseidon/typhoon/pull/587)) - * Require Terraform version v0.12.6+ (action required) + * Require Terraform version v0.12.6+ (**action required**) * Update CoreDNS from v1.6.2 to v1.6.5 ([#588](https://github.com/poseidon/typhoon/pull/588)) * Add health `lameduck` option to wait before shutdown -* Update Calico from v3.10.1 to v3.10.2 +* Update Calico from v3.10.1 to v3.10.2 ([#599](https://github.com/poseidon/typhoon/pull/599)) * Reduce pod eviction timeout for deleting pods on unready nodes from 5m to 1m ([#597](https://github.com/poseidon/typhoon/pull/597)) * Present since [v1.13.3](#v1133), but mistakenly removed in v1.16.0 * Add CPU requests for control plane static pods ([#589](https://github.com/poseidon/typhoon/pull/589)) @@ -24,14 +26,14 @@ Notable changes between versions. * Use new `google_compute_region_instance_group_manager` version block format * Fixes warning that `instance_template` is deprecated - * Require `terraform-provider-google` v2.19.0+ (action required) + * Require `terraform-provider-google` v2.19.0+ (**action required**) #### Addons * Update Grafana from v6.4.4 to [v6.5.1](https://grafana.com/docs/guides/whats-new-in-v6-5/) * Add pod networking details in dashboards ([#593](https://github.com/poseidon/typhoon/pull/593)) * Add node alerts and Grafana dashboard from node-exporter ([#591](https://github.com/poseidon/typhoon/pull/591)) -* Reduce Prometheus time series of high cardinality metrics ([#596](https://github.com/poseidon/typhoon/pull/596)) +* Reduce Prometheus high cardinality time series ([#596](https://github.com/poseidon/typhoon/pull/596)) ## v1.16.3 diff --git a/README.md b/README.md index 26cc641d2..514bf09fe 100644 --- a/README.md +++ b/README.md @@ -76,7 +76,7 @@ Initialize modules, plan the changes to be made, and apply the changes. ```sh $ terraform init $ terraform plan -Plan: 64 to add, 0 to change, 0 to destroy. +Plan: 62 to add, 0 to change, 0 to destroy. $ terraform apply Apply complete! Resources: 64 added, 0 changed, 0 destroyed. ``` diff --git a/docs/cl/aws.md b/docs/cl/aws.md index 1bb04d037..58e1b0849 100644 --- a/docs/cl/aws.md +++ b/docs/cl/aws.md @@ -10,15 +10,15 @@ Controller hosts are provisioned to run an `etcd-member` peer and a `kubelet` se * AWS Account and IAM credentials * AWS Route53 DNS Zone (registered Domain Name or delegated subdomain) -* Terraform v0.12.x and [terraform-provider-ct](https://github.com/poseidon/terraform-provider-ct) installed locally +* Terraform v0.12.6+ and [terraform-provider-ct](https://github.com/poseidon/terraform-provider-ct) installed locally ## Terraform Setup -Install [Terraform](https://www.terraform.io/downloads.html) v0.12.x on your system. +Install [Terraform](https://www.terraform.io/downloads.html) v0.12.6+ on your system. ```sh $ terraform version -Terraform v0.12.12 +Terraform v0.12.16 ``` Add the [terraform-provider-ct](https://github.com/poseidon/terraform-provider-ct) plugin binary for your system to `~/.terraform.d/plugins/`, noting the final name. @@ -49,7 +49,7 @@ Configure the AWS provider to use your access key credentials in a `providers.tf ```tf provider "aws" { - version = "2.35.0" + version = "2.41.0" region = "eu-central-1" shared_credentials_file = "/home/user/.config/aws/credentials" } @@ -109,7 +109,7 @@ Plan the resources to be created. ```sh $ terraform plan -Plan: 98 to add, 0 to change, 0 to destroy. +Plan: 80 to add, 0 to change, 0 to destroy. ``` Apply the changes to create the cluster. diff --git a/docs/cl/azure.md b/docs/cl/azure.md index 3701baa63..b3212e5b8 100644 --- a/docs/cl/azure.md +++ b/docs/cl/azure.md @@ -13,15 +13,15 @@ Controller hosts are provisioned to run an `etcd-member` peer and a `kubelet` se * Azure account * Azure DNS Zone (registered Domain Name or delegated subdomain) -* Terraform v0.12.x and [terraform-provider-ct](https://github.com/poseidon/terraform-provider-ct) installed locally +* Terraform v0.12.6+ and [terraform-provider-ct](https://github.com/poseidon/terraform-provider-ct) installed locally ## Terraform Setup -Install [Terraform](https://www.terraform.io/downloads.html) v0.12.x on your system. +Install [Terraform](https://www.terraform.io/downloads.html) v0.12.6+ on your system. ```sh $ terraform version -Terraform v0.12.12 +Terraform v0.12.16 ``` Add the [terraform-provider-ct](https://github.com/poseidon/terraform-provider-ct) plugin binary for your system to `~/.terraform.d/plugins/`, noting the final name. @@ -50,7 +50,7 @@ Configure the Azure provider in a `providers.tf` file. ```tf provider "azurerm" { - version = "1.36.1" + version = "1.38.0" } provider "ct" { @@ -118,7 +118,7 @@ module.ramius.null_resource.bootstrap: Still creating... (6m50s elapsed) module.ramius.null_resource.bootstrap: Still creating... (7m0s elapsed) module.ramius.null_resource.bootstrap: Creation complete after 7m8s (ID: 3961816482286168143) -Apply complete! Resources: 86 added, 0 changed, 0 destroyed. +Apply complete! Resources: 69 added, 0 changed, 0 destroyed. ``` In 4-8 minutes, the Kubernetes cluster will be ready. diff --git a/docs/cl/bare-metal.md b/docs/cl/bare-metal.md index 3bf419d6c..b265cb688 100644 --- a/docs/cl/bare-metal.md +++ b/docs/cl/bare-metal.md @@ -12,7 +12,7 @@ Controller hosts are provisioned to run an `etcd-member` peer and a `kubelet` se * PXE-enabled [network boot](https://coreos.com/matchbox/docs/latest/network-setup.html) environment (with HTTPS support) * Matchbox v0.6+ deployment with API enabled * Matchbox credentials `client.crt`, `client.key`, `ca.crt` -* Terraform v0.12.x, [terraform-provider-matchbox](https://github.com/poseidon/terraform-provider-matchbox), and [terraform-provider-ct](https://github.com/poseidon/terraform-provider-ct) installed locally +* Terraform v0.12.6+, [terraform-provider-matchbox](https://github.com/poseidon/terraform-provider-matchbox), and [terraform-provider-ct](https://github.com/poseidon/terraform-provider-ct) installed locally ## Machines @@ -107,11 +107,11 @@ Read about the [many ways](https://coreos.com/matchbox/docs/latest/network-setup ## Terraform Setup -Install [Terraform](https://www.terraform.io/downloads.html) v0.12.x on your system. +Install [Terraform](https://www.terraform.io/downloads.html) v0.12.6+ on your system. ```sh $ terraform version -Terraform v0.12.12 +Terraform v0.12.16 ``` Add the [terraform-provider-matchbox](https://github.com/poseidon/terraform-provider-matchbox) plugin binary for your system to `~/.terraform.d/plugins/`, noting the final name. diff --git a/docs/cl/digital-ocean.md b/docs/cl/digital-ocean.md index a8ffa2cfe..49cde1b34 100644 --- a/docs/cl/digital-ocean.md +++ b/docs/cl/digital-ocean.md @@ -10,15 +10,15 @@ Controller hosts are provisioned to run an `etcd-member` peer and a `kubelet` se * Digital Ocean Account and Token * Digital Ocean Domain (registered Domain Name or delegated subdomain) -* Terraform v0.12.x and [terraform-provider-ct](https://github.com/poseidon/terraform-provider-ct) installed locally +* Terraform v0.12.6+ and [terraform-provider-ct](https://github.com/poseidon/terraform-provider-ct) installed locally ## Terraform Setup -Install [Terraform](https://www.terraform.io/downloads.html) v0.12.x on your system. +Install [Terraform](https://www.terraform.io/downloads.html) v0.12.6+ on your system. ```sh $ terraform version -Terraform v0.12.12 +Terraform v0.12.16 ``` Add the [terraform-provider-ct](https://github.com/poseidon/terraform-provider-ct) plugin binary for your system to `~/.terraform.d/plugins/`, noting the final name. @@ -116,7 +116,7 @@ module.nemo.null_resource.bootstrap: Provisioning with 'remote-exec'... module.nemo.null_resource.bootstrap: Still creating... (6m20s elapsed) module.nemo.null_resource.bootstrap: Creation complete (ID: 7599298447329218468) -Apply complete! Resources: 54 added, 0 changed, 0 destroyed. +Apply complete! Resources: 42 added, 0 changed, 0 destroyed. ``` In 3-6 minutes, the Kubernetes cluster will be ready. diff --git a/docs/cl/google-cloud.md b/docs/cl/google-cloud.md index 7173f9fb4..8171e35fd 100644 --- a/docs/cl/google-cloud.md +++ b/docs/cl/google-cloud.md @@ -10,15 +10,15 @@ Controller hosts are provisioned to run an `etcd-member` peer and a `kubelet` se * Google Cloud Account and Service Account * Google Cloud DNS Zone (registered Domain Name or delegated subdomain) -* Terraform v0.12.x and [terraform-provider-ct](https://github.com/poseidon/terraform-provider-ct) installed locally +* Terraform v0.12.6+ and [terraform-provider-ct](https://github.com/poseidon/terraform-provider-ct) installed locally ## Terraform Setup -Install [Terraform](https://www.terraform.io/downloads.html) v0.12.x on your system. +Install [Terraform](https://www.terraform.io/downloads.html) v0.12.6+ on your system. ```sh $ terraform version -Terraform v0.12.12 +Terraform v0.12.16 ``` Add the [terraform-provider-ct](https://github.com/poseidon/terraform-provider-ct) plugin binary for your system to `~/.terraform.d/plugins/`, noting the final name. @@ -49,7 +49,7 @@ Configure the Google Cloud provider to use your service account key, project-id, ```tf provider "google" { - version = "2.19.0" + version = "2.20.0" project = "project-id" region = "us-central1" credentials = file("~/.config/google-cloud/terraform.json") @@ -123,7 +123,7 @@ module.yavin.null_resource.bootstrap: Still creating... (5m30s elapsed) module.yavin.null_resource.bootstrap: Still creating... (5m40s elapsed) module.yavin.null_resource.bootstrap: Creation complete (ID: 5768638456220583358) -Apply complete! Resources: 64 added, 0 changed, 0 destroyed. +Apply complete! Resources: 62 added, 0 changed, 0 destroyed. ``` In 4-8 minutes, the Kubernetes cluster will be ready. diff --git a/docs/fedora-coreos/aws.md b/docs/fedora-coreos/aws.md index ba3c10c1a..e44bbf5d4 100644 --- a/docs/fedora-coreos/aws.md +++ b/docs/fedora-coreos/aws.md @@ -13,15 +13,15 @@ Controller hosts are provisioned to run an `etcd-member` peer and a `kubelet` se * AWS Account and IAM credentials * AWS Route53 DNS Zone (registered Domain Name or delegated subdomain) -* Terraform v0.12.x and [terraform-provider-ct](https://github.com/poseidon/terraform-provider-ct) installed locally +* Terraform v0.12.6+ and [terraform-provider-ct](https://github.com/poseidon/terraform-provider-ct) installed locally ## Terraform Setup -Install [Terraform](https://www.terraform.io/downloads.html) v0.12.x on your system. +Install [Terraform](https://www.terraform.io/downloads.html) v0.12.6+ on your system. ```sh $ terraform version -Terraform v0.12.12 +Terraform v0.12.16 ``` Add the [terraform-provider-ct](https://github.com/poseidon/terraform-provider-ct) plugin binary for your system to `~/.terraform.d/plugins/`, noting the final name. @@ -52,7 +52,7 @@ Configure the AWS provider to use your access key credentials in a `providers.tf ```tf provider "aws" { - version = "2.35.0" + version = "2.41.0" region = "eu-central-1" shared_credentials_file = "/home/user/.config/aws/credentials" } @@ -112,7 +112,7 @@ Plan the resources to be created. ```sh $ terraform plan -Plan: 98 to add, 0 to change, 0 to destroy. +Plan: 81 to add, 0 to change, 0 to destroy. ``` Apply the changes to create the cluster. diff --git a/docs/fedora-coreos/bare-metal.md b/docs/fedora-coreos/bare-metal.md index 8f10bf6a3..461bb232c 100644 --- a/docs/fedora-coreos/bare-metal.md +++ b/docs/fedora-coreos/bare-metal.md @@ -15,7 +15,7 @@ Controller hosts are provisioned to run an `etcd-member` peer and a `kubelet` se * PXE-enabled [network boot](https://coreos.com/matchbox/docs/latest/network-setup.html) environment (with HTTPS support) * Matchbox v0.6+ deployment with API enabled * Matchbox credentials `client.crt`, `client.key`, `ca.crt` -* Terraform v0.12.x, [terraform-provider-matchbox](https://github.com/poseidon/terraform-provider-matchbox), and [terraform-provider-ct](https://github.com/poseidon/terraform-provider-ct) installed locally +* Terraform v0.12.6+, [terraform-provider-matchbox](https://github.com/poseidon/terraform-provider-matchbox), and [terraform-provider-ct](https://github.com/poseidon/terraform-provider-ct) installed locally ## Machines @@ -110,11 +110,11 @@ Read about the [many ways](https://coreos.com/matchbox/docs/latest/network-setup ## Terraform Setup -Install [Terraform](https://www.terraform.io/downloads.html) v0.12.x on your system. +Install [Terraform](https://www.terraform.io/downloads.html) v0.12.6+ on your system. ```sh $ terraform version -Terraform v0.12.12 +Terraform v0.12.16 ``` Add the [terraform-provider-matchbox](https://github.com/poseidon/terraform-provider-matchbox) plugin binary for your system to `~/.terraform.d/plugins/`, noting the final name. diff --git a/docs/index.md b/docs/index.md index 8090ae06e..bd729c0a8 100644 --- a/docs/index.md +++ b/docs/index.md @@ -74,7 +74,7 @@ Initialize modules, plan the changes to be made, and apply the changes. ```sh $ terraform init $ terraform plan -Plan: 64 to add, 0 to change, 0 to destroy. +Plan: 62 to add, 0 to change, 0 to destroy. $ terraform apply Apply complete! Resources: 64 added, 0 changed, 0 destroyed. ``` From c3e22f3d137e5c2458e51be463f1b40535b46270 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Tue, 10 Dec 2019 23:14:12 -0800 Subject: [PATCH 288/523] Fix minor example typo in README --- README.md | 2 +- docs/index.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 514bf09fe..c1164f7dd 100644 --- a/README.md +++ b/README.md @@ -78,7 +78,7 @@ $ terraform init $ terraform plan Plan: 62 to add, 0 to change, 0 to destroy. $ terraform apply -Apply complete! Resources: 64 added, 0 changed, 0 destroyed. +Apply complete! Resources: 62 added, 0 changed, 0 destroyed. ``` In 4-8 minutes (varies by platform), the cluster will be ready. This Google Cloud example creates a `yavin.example.com` DNS record to resolve to a network load balancer across controller nodes. diff --git a/docs/index.md b/docs/index.md index bd729c0a8..144c98729 100644 --- a/docs/index.md +++ b/docs/index.md @@ -76,7 +76,7 @@ $ terraform init $ terraform plan Plan: 62 to add, 0 to change, 0 to destroy. $ terraform apply -Apply complete! Resources: 64 added, 0 changed, 0 destroyed. +Apply complete! Resources: 62 added, 0 changed, 0 destroyed. ``` In 4-8 minutes (varies by platform), the cluster will be ready. This Google Cloud example creates a `yavin.example.com` DNS record to resolve to a network load balancer across controller nodes. From 2d8e367664590736da0fad64f8fcc5ed3485627e Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Sat, 14 Dec 2019 15:02:28 -0800 Subject: [PATCH 289/523] Update mkdocs-material from v4.5.1 to v4.6.0 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 122696998..9cd287ae0 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ mkdocs==1.0.4 -mkdocs-material==4.5.1 +mkdocs-material==4.6.0 pygments==2.4.2 pymdown-extensions==6.2.0 From 1b9fa2e6886cf4303e92001efd9bc949662777f1 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Sat, 14 Dec 2019 15:25:48 -0800 Subject: [PATCH 290/523] Update Grafana from v6.5.1 to v6.5.2 * https://github.com/grafana/grafana/releases/tag/v6.5.2 --- CHANGES.md | 4 ++++ addons/grafana/deployment.yaml | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index a3129841d..9792f1197 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -4,6 +4,10 @@ Notable changes between versions. ## Latest +#### Addons + +* Update Grafana from v6.5.1 to v6.5.2 + ## v1.17.0 * Kubernetes [v1.17.0](https://github.com/kubernetes/kubernetes/blob/master/CHANGELOG-1.17.md#v1170) diff --git a/addons/grafana/deployment.yaml b/addons/grafana/deployment.yaml index 83fad50d1..93d187282 100644 --- a/addons/grafana/deployment.yaml +++ b/addons/grafana/deployment.yaml @@ -23,7 +23,7 @@ spec: spec: containers: - name: grafana - image: docker.io/grafana/grafana:6.5.1 + image: docker.io/grafana/grafana:6.5.2 env: - name: GF_PATHS_CONFIG value: "/etc/grafana/custom.ini" From 0ecb995890e2f6cf28d25177a74d7055cf1a7ab1 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Sat, 14 Dec 2019 17:20:49 -0800 Subject: [PATCH 291/523] Update kube-state-metrics from v1.8.0 to v1.9.0-rc.1 * https://github.com/kubernetes/kube-state-metrics/releases/tag/v1.9.0-rc.1 * https://github.com/kubernetes/kube-state-metrics/releases/tag/v1.9.0-rc.0 --- .../kube-state-metrics/cluster-role.yaml | 16 ++++++++++++++++ .../exporters/kube-state-metrics/deployment.yaml | 2 +- 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/addons/prometheus/exporters/kube-state-metrics/cluster-role.yaml b/addons/prometheus/exporters/kube-state-metrics/cluster-role.yaml index c19e6f45f..9346b9a91 100644 --- a/addons/prometheus/exporters/kube-state-metrics/cluster-role.yaml +++ b/addons/prometheus/exporters/kube-state-metrics/cluster-role.yaml @@ -74,6 +74,7 @@ rules: - storage.k8s.io resources: - storageclasses + - volumeattachments verbs: - list - watch @@ -84,4 +85,19 @@ rules: verbs: - list - watch +- apiGroups: + - admissionregistration.k8s.io + resources: + - mutatingwebhookconfigurations + - validatingwebhookconfigurations + verbs: + - list + - watch +- apiGroups: + - networking.k8s.io + resources: + - networkpolicies + verbs: + - list + - watch diff --git a/addons/prometheus/exporters/kube-state-metrics/deployment.yaml b/addons/prometheus/exporters/kube-state-metrics/deployment.yaml index b4f0a47fb..4dca32365 100644 --- a/addons/prometheus/exporters/kube-state-metrics/deployment.yaml +++ b/addons/prometheus/exporters/kube-state-metrics/deployment.yaml @@ -24,7 +24,7 @@ spec: serviceAccountName: kube-state-metrics containers: - name: kube-state-metrics - image: quay.io/coreos/kube-state-metrics:v1.8.0 + image: quay.io/coreos/kube-state-metrics:v1.9.0-rc.1 ports: - name: metrics containerPort: 8080 From 00c431a9d20ad8baac64f124ace740f724f92676 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Wed, 18 Dec 2019 23:20:55 -0800 Subject: [PATCH 292/523] Add Kubelet kubeconfig output for DigitalOcean * Allow the raw kubelet kubeconfig to be consumed via Terraform output --- digital-ocean/container-linux/kubernetes/outputs.tf | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/digital-ocean/container-linux/kubernetes/outputs.tf b/digital-ocean/container-linux/kubernetes/outputs.tf index 50934e37a..429893c58 100644 --- a/digital-ocean/container-linux/kubernetes/outputs.tf +++ b/digital-ocean/container-linux/kubernetes/outputs.tf @@ -27,6 +27,12 @@ output "workers_ipv6" { value = digitalocean_droplet.workers.*.ipv6_address } +# Outputs for worker pools + +output "kubeconfig" { + value = module.bootstrap.kubeconfig-kubelet +} + # Outputs for custom firewalls output "controller_tag" { From 52d11096dca9565fe40485bd7729ca77786c66b9 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Fri, 20 Dec 2019 13:53:37 -0800 Subject: [PATCH 293/523] Update kube-state-metrics from v1.9.0-rc.1 to v1.9.0 * https://github.com/kubernetes/kube-state-metrics/releases/tag/v1.9.0 * https://github.com/kubernetes/kube-state-metrics/releases/tag/v1.9.0-rc.1 * https://github.com/kubernetes/kube-state-metrics/releases/tag/v1.9.0-rc.0 --- CHANGES.md | 1 + addons/prometheus/exporters/kube-state-metrics/deployment.yaml | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index 9792f1197..9bcc7f2d6 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -6,6 +6,7 @@ Notable changes between versions. #### Addons +* Update kube-state-metrics from v1.8.0 to v1.9.0 * Update Grafana from v6.5.1 to v6.5.2 ## v1.17.0 diff --git a/addons/prometheus/exporters/kube-state-metrics/deployment.yaml b/addons/prometheus/exporters/kube-state-metrics/deployment.yaml index 4dca32365..8cbd151d5 100644 --- a/addons/prometheus/exporters/kube-state-metrics/deployment.yaml +++ b/addons/prometheus/exporters/kube-state-metrics/deployment.yaml @@ -24,7 +24,7 @@ spec: serviceAccountName: kube-state-metrics containers: - name: kube-state-metrics - image: quay.io/coreos/kube-state-metrics:v1.9.0-rc.1 + image: quay.io/coreos/kube-state-metrics:v1.9.0 ports: - name: metrics containerPort: 8080 From daa8d9d9ec11be050ca6fe017b126a0b8877f450 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Sun, 22 Dec 2019 10:40:56 -0500 Subject: [PATCH 294/523] Update CoreDNS from v1.6.5 to v1.6.6 * https://coredns.io/2019/12/11/coredns-1.6.6-release/ --- CHANGES.md | 2 ++ aws/container-linux/kubernetes/bootstrap.tf | 2 +- aws/fedora-coreos/kubernetes/bootstrap.tf | 2 +- azure/container-linux/kubernetes/bootstrap.tf | 2 +- bare-metal/container-linux/kubernetes/bootstrap.tf | 2 +- bare-metal/fedora-coreos/kubernetes/bootstrap.tf | 2 +- digital-ocean/container-linux/kubernetes/bootstrap.tf | 2 +- google-cloud/container-linux/kubernetes/bootstrap.tf | 2 +- 8 files changed, 9 insertions(+), 7 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 9bcc7f2d6..021c20d60 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -4,6 +4,8 @@ Notable changes between versions. ## Latest +* Update CoreDNS from v1.6.5 to [v1.6.6](https://coredns.io/2019/12/11/coredns-1.6.6-release/) ([#602](https://github.com/poseidon/typhoon/pull/602)) + #### Addons * Update kube-state-metrics from v1.8.0 to v1.9.0 diff --git a/aws/container-linux/kubernetes/bootstrap.tf b/aws/container-linux/kubernetes/bootstrap.tf index e178926fe..e71c18a9e 100644 --- a/aws/container-linux/kubernetes/bootstrap.tf +++ b/aws/container-linux/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=24e5513ee6da4888005aac02101f73fc9e5e829f" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=f021d9cb349737caaa73e00be8ff8057312c6324" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/aws/fedora-coreos/kubernetes/bootstrap.tf b/aws/fedora-coreos/kubernetes/bootstrap.tf index 24efc816e..b50579993 100644 --- a/aws/fedora-coreos/kubernetes/bootstrap.tf +++ b/aws/fedora-coreos/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=24e5513ee6da4888005aac02101f73fc9e5e829f" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=f021d9cb349737caaa73e00be8ff8057312c6324" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/azure/container-linux/kubernetes/bootstrap.tf b/azure/container-linux/kubernetes/bootstrap.tf index 471320b46..78e60f8c5 100644 --- a/azure/container-linux/kubernetes/bootstrap.tf +++ b/azure/container-linux/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=24e5513ee6da4888005aac02101f73fc9e5e829f" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=f021d9cb349737caaa73e00be8ff8057312c6324" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/bare-metal/container-linux/kubernetes/bootstrap.tf b/bare-metal/container-linux/kubernetes/bootstrap.tf index d4e0430be..5ed52b86f 100644 --- a/bare-metal/container-linux/kubernetes/bootstrap.tf +++ b/bare-metal/container-linux/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=24e5513ee6da4888005aac02101f73fc9e5e829f" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=f021d9cb349737caaa73e00be8ff8057312c6324" cluster_name = var.cluster_name api_servers = [var.k8s_domain_name] diff --git a/bare-metal/fedora-coreos/kubernetes/bootstrap.tf b/bare-metal/fedora-coreos/kubernetes/bootstrap.tf index 80f4a8ec6..9145d4ec7 100644 --- a/bare-metal/fedora-coreos/kubernetes/bootstrap.tf +++ b/bare-metal/fedora-coreos/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=24e5513ee6da4888005aac02101f73fc9e5e829f" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=f021d9cb349737caaa73e00be8ff8057312c6324" cluster_name = var.cluster_name api_servers = [var.k8s_domain_name] diff --git a/digital-ocean/container-linux/kubernetes/bootstrap.tf b/digital-ocean/container-linux/kubernetes/bootstrap.tf index 45c399611..09f005994 100644 --- a/digital-ocean/container-linux/kubernetes/bootstrap.tf +++ b/digital-ocean/container-linux/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=24e5513ee6da4888005aac02101f73fc9e5e829f" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=f021d9cb349737caaa73e00be8ff8057312c6324" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/google-cloud/container-linux/kubernetes/bootstrap.tf b/google-cloud/container-linux/kubernetes/bootstrap.tf index 209e34c92..98e8ace68 100644 --- a/google-cloud/container-linux/kubernetes/bootstrap.tf +++ b/google-cloud/container-linux/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=24e5513ee6da4888005aac02101f73fc9e5e829f" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=f021d9cb349737caaa73e00be8ff8057312c6324" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] From f48e43c0b11a4560d1eb699b320ad2c34cca1d08 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Tue, 24 Dec 2019 10:52:19 -0500 Subject: [PATCH 295/523] Update Prometheus from v2.14.0 to v2.15.0 * https://github.com/prometheus/prometheus/releases/tag/v2.15.0 --- CHANGES.md | 1 + addons/prometheus/deployment.yaml | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index 021c20d60..11c6f309f 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -8,6 +8,7 @@ Notable changes between versions. #### Addons +* Update Prometheus from v2.14.0 to [v2.15.0](https://github.com/prometheus/prometheus/releases/tag/v2.15.0) * Update kube-state-metrics from v1.8.0 to v1.9.0 * Update Grafana from v6.5.1 to v6.5.2 diff --git a/addons/prometheus/deployment.yaml b/addons/prometheus/deployment.yaml index 886b84b70..8725c73b0 100644 --- a/addons/prometheus/deployment.yaml +++ b/addons/prometheus/deployment.yaml @@ -20,7 +20,7 @@ spec: serviceAccountName: prometheus containers: - name: prometheus - image: quay.io/prometheus/prometheus:v2.14.0 + image: quay.io/prometheus/prometheus:v2.15.0 args: - --web.listen-address=0.0.0.0:9090 - --config.file=/etc/prometheus/prometheus.yaml From a4e843693f0cded79ebfe59825f1d16f7ef5fc9e Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Thu, 26 Dec 2019 09:11:58 -0500 Subject: [PATCH 296/523] Update Prometheus from v2.15.0 to v2.15.1 * https://github.com/prometheus/prometheus/releases/tag/v2.15.1 --- CHANGES.md | 2 +- addons/prometheus/deployment.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 11c6f309f..e30633c39 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -8,7 +8,7 @@ Notable changes between versions. #### Addons -* Update Prometheus from v2.14.0 to [v2.15.0](https://github.com/prometheus/prometheus/releases/tag/v2.15.0) +* Update Prometheus from v2.14.0 to [v2.15.1](https://github.com/prometheus/prometheus/releases/tag/v2.15.1) * Update kube-state-metrics from v1.8.0 to v1.9.0 * Update Grafana from v6.5.1 to v6.5.2 diff --git a/addons/prometheus/deployment.yaml b/addons/prometheus/deployment.yaml index 8725c73b0..254fa0c88 100644 --- a/addons/prometheus/deployment.yaml +++ b/addons/prometheus/deployment.yaml @@ -20,7 +20,7 @@ spec: serviceAccountName: prometheus containers: - name: prometheus - image: quay.io/prometheus/prometheus:v2.15.0 + image: quay.io/prometheus/prometheus:v2.15.1 args: - --web.listen-address=0.0.0.0:9090 - --config.file=/etc/prometheus/prometheus.yaml From 11565ffa8ad9d3ff837b597c7ab7307081fe4777 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Sat, 28 Dec 2019 11:02:32 -0800 Subject: [PATCH 297/523] Update Calico from v3.10.2 to v3.11.1 * https://docs.projectcalico.org/v3.11/release-notes/ --- CHANGES.md | 1 + aws/container-linux/kubernetes/bootstrap.tf | 2 +- aws/fedora-coreos/kubernetes/bootstrap.tf | 2 +- azure/container-linux/kubernetes/bootstrap.tf | 2 +- bare-metal/container-linux/kubernetes/bootstrap.tf | 2 +- bare-metal/fedora-coreos/kubernetes/bootstrap.tf | 2 +- digital-ocean/container-linux/kubernetes/bootstrap.tf | 2 +- google-cloud/container-linux/kubernetes/bootstrap.tf | 2 +- 8 files changed, 8 insertions(+), 7 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index e30633c39..93282a36f 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -5,6 +5,7 @@ Notable changes between versions. ## Latest * Update CoreDNS from v1.6.5 to [v1.6.6](https://coredns.io/2019/12/11/coredns-1.6.6-release/) ([#602](https://github.com/poseidon/typhoon/pull/602)) +* Update Calico from v3.10.2 to v3.11.1 ([#604](https://github.com/poseidon/typhoon/pull/604)) #### Addons diff --git a/aws/container-linux/kubernetes/bootstrap.tf b/aws/container-linux/kubernetes/bootstrap.tf index e71c18a9e..ae850c6b2 100644 --- a/aws/container-linux/kubernetes/bootstrap.tf +++ b/aws/container-linux/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=f021d9cb349737caaa73e00be8ff8057312c6324" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=c8c21deb7682c2a83a1b86ff6ed88f3e5a20262d" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/aws/fedora-coreos/kubernetes/bootstrap.tf b/aws/fedora-coreos/kubernetes/bootstrap.tf index b50579993..7d60bb946 100644 --- a/aws/fedora-coreos/kubernetes/bootstrap.tf +++ b/aws/fedora-coreos/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=f021d9cb349737caaa73e00be8ff8057312c6324" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=c8c21deb7682c2a83a1b86ff6ed88f3e5a20262d" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/azure/container-linux/kubernetes/bootstrap.tf b/azure/container-linux/kubernetes/bootstrap.tf index 78e60f8c5..218b050e4 100644 --- a/azure/container-linux/kubernetes/bootstrap.tf +++ b/azure/container-linux/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=f021d9cb349737caaa73e00be8ff8057312c6324" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=c8c21deb7682c2a83a1b86ff6ed88f3e5a20262d" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/bare-metal/container-linux/kubernetes/bootstrap.tf b/bare-metal/container-linux/kubernetes/bootstrap.tf index 5ed52b86f..7def8550c 100644 --- a/bare-metal/container-linux/kubernetes/bootstrap.tf +++ b/bare-metal/container-linux/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=f021d9cb349737caaa73e00be8ff8057312c6324" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=c8c21deb7682c2a83a1b86ff6ed88f3e5a20262d" cluster_name = var.cluster_name api_servers = [var.k8s_domain_name] diff --git a/bare-metal/fedora-coreos/kubernetes/bootstrap.tf b/bare-metal/fedora-coreos/kubernetes/bootstrap.tf index 9145d4ec7..bc2364627 100644 --- a/bare-metal/fedora-coreos/kubernetes/bootstrap.tf +++ b/bare-metal/fedora-coreos/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=f021d9cb349737caaa73e00be8ff8057312c6324" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=c8c21deb7682c2a83a1b86ff6ed88f3e5a20262d" cluster_name = var.cluster_name api_servers = [var.k8s_domain_name] diff --git a/digital-ocean/container-linux/kubernetes/bootstrap.tf b/digital-ocean/container-linux/kubernetes/bootstrap.tf index 09f005994..ebb1f80a1 100644 --- a/digital-ocean/container-linux/kubernetes/bootstrap.tf +++ b/digital-ocean/container-linux/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=f021d9cb349737caaa73e00be8ff8057312c6324" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=c8c21deb7682c2a83a1b86ff6ed88f3e5a20262d" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/google-cloud/container-linux/kubernetes/bootstrap.tf b/google-cloud/container-linux/kubernetes/bootstrap.tf index 98e8ace68..9eeeb9795 100644 --- a/google-cloud/container-linux/kubernetes/bootstrap.tf +++ b/google-cloud/container-linux/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=f021d9cb349737caaa73e00be8ff8057312c6324" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=c8c21deb7682c2a83a1b86ff6ed88f3e5a20262d" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] From 50db3d0231775ddfbfbeec783ff7818dc82a978f Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Sat, 28 Dec 2019 12:07:10 -0800 Subject: [PATCH 298/523] Rename CLC files and favor Terraform list index syntax * Rename Container Linux Config (CLC) files to *.yaml to align with Fedora CoreOS Config (FCC) files and for syntax highlighting * Replace common uses of Terraform `element` (which wraps around) with `list[index]` syntax to surface index errors --- aws/container-linux/kubernetes/ami.tf | 4 ++-- .../{controller.yaml.tmpl => controller.yaml} | 0 aws/container-linux/kubernetes/controllers.tf | 15 +++++-------- aws/container-linux/kubernetes/network.tf | 2 +- aws/container-linux/kubernetes/nlb.tf | 2 +- aws/container-linux/kubernetes/ssh.tf | 2 +- aws/container-linux/kubernetes/workers/ami.tf | 4 ++-- .../cl/{worker.yaml.tmpl => worker.yaml} | 0 .../kubernetes/workers/workers.tf | 2 +- aws/fedora-coreos/kubernetes/ssh.tf | 4 ++-- aws/ignore/.gitkeep | 0 .../{controller.yaml.tmpl => controller.yaml} | 0 .../container-linux/kubernetes/controllers.tf | 22 +++++++------------ azure/container-linux/kubernetes/ssh.tf | 6 ++--- .../cl/{worker.yaml.tmpl => worker.yaml} | 0 .../kubernetes/workers/workers.tf | 4 ++-- .../{controller.yaml.tmpl => controller.yaml} | 0 .../cl/{install.yaml.tmpl => install.yaml} | 0 .../cl/{worker.yaml.tmpl => worker.yaml} | 0 .../container-linux/kubernetes/profiles.tf | 8 +++---- bare-metal/container-linux/kubernetes/ssh.tf | 2 +- bare-metal/fedora-coreos/kubernetes/ssh.tf | 2 +- bare-metal/ignore/.gitkeep | 0 .../{controller.yaml.tmpl => controller.yaml} | 0 .../cl/{worker.yaml.tmpl => worker.yaml} | 0 .../container-linux/kubernetes/controllers.tf | 10 ++++----- .../container-linux/kubernetes/ssh.tf | 2 +- .../container-linux/kubernetes/workers.tf | 6 ++--- .../{controller.yaml.tmpl => controller.yaml} | 0 .../container-linux/kubernetes/controllers.tf | 16 ++++++-------- .../container-linux/kubernetes/ssh.tf | 4 ++-- .../cl/{worker.yaml.tmpl => worker.yaml} | 0 .../kubernetes/workers/workers.tf | 2 +- 33 files changed, 54 insertions(+), 65 deletions(-) rename aws/container-linux/kubernetes/cl/{controller.yaml.tmpl => controller.yaml} (100%) rename aws/container-linux/kubernetes/workers/cl/{worker.yaml.tmpl => worker.yaml} (100%) delete mode 100644 aws/ignore/.gitkeep rename azure/container-linux/kubernetes/cl/{controller.yaml.tmpl => controller.yaml} (100%) rename azure/container-linux/kubernetes/workers/cl/{worker.yaml.tmpl => worker.yaml} (100%) rename bare-metal/container-linux/kubernetes/cl/{controller.yaml.tmpl => controller.yaml} (100%) rename bare-metal/container-linux/kubernetes/cl/{install.yaml.tmpl => install.yaml} (100%) rename bare-metal/container-linux/kubernetes/cl/{worker.yaml.tmpl => worker.yaml} (100%) delete mode 100644 bare-metal/ignore/.gitkeep rename digital-ocean/container-linux/kubernetes/cl/{controller.yaml.tmpl => controller.yaml} (100%) rename digital-ocean/container-linux/kubernetes/cl/{worker.yaml.tmpl => worker.yaml} (100%) rename google-cloud/container-linux/kubernetes/cl/{controller.yaml.tmpl => controller.yaml} (100%) rename google-cloud/container-linux/kubernetes/workers/cl/{worker.yaml.tmpl => worker.yaml} (100%) diff --git a/aws/container-linux/kubernetes/ami.tf b/aws/container-linux/kubernetes/ami.tf index b5124b4be..bea85504d 100644 --- a/aws/container-linux/kubernetes/ami.tf +++ b/aws/container-linux/kubernetes/ami.tf @@ -4,8 +4,8 @@ locals { # flatcar-stable -> Flatcar Linux AMI ami_id = local.flavor == "flatcar" ? data.aws_ami.flatcar.image_id : data.aws_ami.coreos.image_id - flavor = element(split("-", var.os_image), 0) - channel = element(split("-", var.os_image), 1) + flavor = split("-", var.os_image)[0] + channel = split("-", var.os_image)[1] } data "aws_ami" "coreos" { diff --git a/aws/container-linux/kubernetes/cl/controller.yaml.tmpl b/aws/container-linux/kubernetes/cl/controller.yaml similarity index 100% rename from aws/container-linux/kubernetes/cl/controller.yaml.tmpl rename to aws/container-linux/kubernetes/cl/controller.yaml diff --git a/aws/container-linux/kubernetes/controllers.tf b/aws/container-linux/kubernetes/controllers.tf index 282679455..76e22b172 100644 --- a/aws/container-linux/kubernetes/controllers.tf +++ b/aws/container-linux/kubernetes/controllers.tf @@ -10,7 +10,7 @@ resource "aws_route53_record" "etcds" { ttl = 300 # private IPv4 address for etcd - records = [element(aws_instance.controllers.*.private_ip, count.index)] + records = [aws_instance.controllers.*.private_ip[count.index]] } # Controller instances @@ -24,7 +24,7 @@ resource "aws_instance" "controllers" { instance_type = var.controller_type ami = local.ami_id - user_data = element(data.ct_config.controller-ignitions.*.rendered, count.index) + user_data = data.ct_config.controller-ignitions.*.rendered[count.index] # storage root_block_device { @@ -36,7 +36,7 @@ resource "aws_instance" "controllers" { # network associate_public_ip_address = true - subnet_id = element(aws_subnet.public.*.id, count.index) + subnet_id = aws_subnet.public.*.id[count.index] vpc_security_group_ids = [aws_security_group.controller.id] lifecycle { @@ -49,11 +49,8 @@ resource "aws_instance" "controllers" { # Controller Ignition configs data "ct_config" "controller-ignitions" { - count = var.controller_count - content = element( - data.template_file.controller-configs.*.rendered, - count.index, - ) + count = var.controller_count + content = data.template_file.controller-configs.*.rendered[count.index] pretty_print = false snippets = var.controller_clc_snippets } @@ -62,7 +59,7 @@ data "ct_config" "controller-ignitions" { data "template_file" "controller-configs" { count = var.controller_count - template = file("${path.module}/cl/controller.yaml.tmpl") + template = file("${path.module}/cl/controller.yaml") vars = { # Cannot use cyclic dependencies on controllers or their DNS records diff --git a/aws/container-linux/kubernetes/network.tf b/aws/container-linux/kubernetes/network.tf index ae159ba52..a93b3f0cb 100644 --- a/aws/container-linux/kubernetes/network.tf +++ b/aws/container-linux/kubernetes/network.tf @@ -62,6 +62,6 @@ resource "aws_route_table_association" "public" { count = length(data.aws_availability_zones.all.names) route_table_id = aws_route_table.default.id - subnet_id = element(aws_subnet.public.*.id, count.index) + subnet_id = aws_subnet.public.*.id[count.index] } diff --git a/aws/container-linux/kubernetes/nlb.tf b/aws/container-linux/kubernetes/nlb.tf index 08458a97e..2b87366a7 100644 --- a/aws/container-linux/kubernetes/nlb.tf +++ b/aws/container-linux/kubernetes/nlb.tf @@ -88,7 +88,7 @@ resource "aws_lb_target_group_attachment" "controllers" { count = var.controller_count target_group_arn = aws_lb_target_group.controllers.arn - target_id = element(aws_instance.controllers.*.id, count.index) + target_id = aws_instance.controllers.*.id[count.index] port = 6443 } diff --git a/aws/container-linux/kubernetes/ssh.tf b/aws/container-linux/kubernetes/ssh.tf index f180f3129..b6396d028 100644 --- a/aws/container-linux/kubernetes/ssh.tf +++ b/aws/container-linux/kubernetes/ssh.tf @@ -2,7 +2,7 @@ locals { # format assets for distribution assets_bundle = [ # header with the unpack location - for key, value in module.bootstrap.assets_dist: + for key, value in module.bootstrap.assets_dist : format("##### %s\n%s", key, value) ] } diff --git a/aws/container-linux/kubernetes/workers/ami.tf b/aws/container-linux/kubernetes/workers/ami.tf index b5124b4be..bea85504d 100644 --- a/aws/container-linux/kubernetes/workers/ami.tf +++ b/aws/container-linux/kubernetes/workers/ami.tf @@ -4,8 +4,8 @@ locals { # flatcar-stable -> Flatcar Linux AMI ami_id = local.flavor == "flatcar" ? data.aws_ami.flatcar.image_id : data.aws_ami.coreos.image_id - flavor = element(split("-", var.os_image), 0) - channel = element(split("-", var.os_image), 1) + flavor = split("-", var.os_image)[0] + channel = split("-", var.os_image)[1] } data "aws_ami" "coreos" { diff --git a/aws/container-linux/kubernetes/workers/cl/worker.yaml.tmpl b/aws/container-linux/kubernetes/workers/cl/worker.yaml similarity index 100% rename from aws/container-linux/kubernetes/workers/cl/worker.yaml.tmpl rename to aws/container-linux/kubernetes/workers/cl/worker.yaml diff --git a/aws/container-linux/kubernetes/workers/workers.tf b/aws/container-linux/kubernetes/workers/workers.tf index 09375ac7e..de0e07aa5 100644 --- a/aws/container-linux/kubernetes/workers/workers.tf +++ b/aws/container-linux/kubernetes/workers/workers.tf @@ -78,7 +78,7 @@ data "ct_config" "worker-ignition" { # Worker Container Linux config data "template_file" "worker-config" { - template = file("${path.module}/cl/worker.yaml.tmpl") + template = file("${path.module}/cl/worker.yaml") vars = { kubeconfig = indent(10, var.kubeconfig) diff --git a/aws/fedora-coreos/kubernetes/ssh.tf b/aws/fedora-coreos/kubernetes/ssh.tf index f11ae0b25..b6396d028 100644 --- a/aws/fedora-coreos/kubernetes/ssh.tf +++ b/aws/fedora-coreos/kubernetes/ssh.tf @@ -2,7 +2,7 @@ locals { # format assets for distribution assets_bundle = [ # header with the unpack location - for key, value in module.bootstrap.assets_dist: + for key, value in module.bootstrap.assets_dist : format("##### %s\n%s", key, value) ] } @@ -21,7 +21,7 @@ resource "null_resource" "copy-controller-secrets" { user = "core" timeout = "15m" } - + provisioner "file" { content = join("\n", local.assets_bundle) destination = "$HOME/assets" diff --git a/aws/ignore/.gitkeep b/aws/ignore/.gitkeep deleted file mode 100644 index e69de29bb..000000000 diff --git a/azure/container-linux/kubernetes/cl/controller.yaml.tmpl b/azure/container-linux/kubernetes/cl/controller.yaml similarity index 100% rename from azure/container-linux/kubernetes/cl/controller.yaml.tmpl rename to azure/container-linux/kubernetes/cl/controller.yaml diff --git a/azure/container-linux/kubernetes/controllers.tf b/azure/container-linux/kubernetes/controllers.tf index 36518d180..6012ad336 100644 --- a/azure/container-linux/kubernetes/controllers.tf +++ b/azure/container-linux/kubernetes/controllers.tf @@ -11,16 +11,13 @@ resource "azurerm_dns_a_record" "etcds" { ttl = 300 # private IPv4 address for etcd - records = [element( - azurerm_network_interface.controllers.*.private_ip_address, - count.index, - )] + records = [azurerm_network_interface.controllers.*.private_ip_address[count.index]] } locals { # Channel for a Container Linux derivative # coreos-stable -> Container Linux Stable - channel = element(split("-", var.os_image), 1) + channel = split("-", var.os_image)[1] } # Controller availability set to spread controllers @@ -63,12 +60,12 @@ resource "azurerm_virtual_machine" "controllers" { } # network - network_interface_ids = [element(azurerm_network_interface.controllers.*.id, count.index)] + network_interface_ids = [azurerm_network_interface.controllers.*.id[count.index]] os_profile { computer_name = "${var.cluster_name}-controller-${count.index}" admin_username = "core" - custom_data = element(data.ct_config.controller-ignitions.*.rendered, count.index) + custom_data = data.ct_config.controller-ignitions.*.rendered[count.index] } # Azure mandates setting an ssh_key, even though Ignition custom_data handles it too @@ -108,7 +105,7 @@ resource "azurerm_network_interface" "controllers" { private_ip_address_allocation = "dynamic" # public IPv4 - public_ip_address_id = element(azurerm_public_ip.controllers.*.id, count.index) + public_ip_address_id = azurerm_public_ip.controllers.*.id[count.index] } } @@ -134,11 +131,8 @@ resource "azurerm_public_ip" "controllers" { # Controller Ignition configs data "ct_config" "controller-ignitions" { - count = var.controller_count - content = element( - data.template_file.controller-configs.*.rendered, - count.index, - ) + count = var.controller_count + content = data.template_file.controller-configs.*.rendered[count.index] pretty_print = false snippets = var.controller_clc_snippets } @@ -147,7 +141,7 @@ data "ct_config" "controller-ignitions" { data "template_file" "controller-configs" { count = var.controller_count - template = file("${path.module}/cl/controller.yaml.tmpl") + template = file("${path.module}/cl/controller.yaml") vars = { # Cannot use cyclic dependencies on controllers or their DNS records diff --git a/azure/container-linux/kubernetes/ssh.tf b/azure/container-linux/kubernetes/ssh.tf index f93e096dd..5ff47b35f 100644 --- a/azure/container-linux/kubernetes/ssh.tf +++ b/azure/container-linux/kubernetes/ssh.tf @@ -2,7 +2,7 @@ locals { # format assets for distribution assets_bundle = [ # header with the unpack location - for key, value in module.bootstrap.assets_dist: + for key, value in module.bootstrap.assets_dist : format("##### %s\n%s", key, value) ] } @@ -22,7 +22,7 @@ resource "null_resource" "copy-controller-secrets" { user = "core" timeout = "15m" } - + provisioner "file" { content = join("\n", local.assets_bundle) destination = "$HOME/assets" @@ -45,7 +45,7 @@ resource "null_resource" "bootstrap" { connection { type = "ssh" - host = element(azurerm_public_ip.controllers.*.ip_address, 0) + host = azurerm_public_ip.controllers.*.ip_address[0] user = "core" timeout = "15m" } diff --git a/azure/container-linux/kubernetes/workers/cl/worker.yaml.tmpl b/azure/container-linux/kubernetes/workers/cl/worker.yaml similarity index 100% rename from azure/container-linux/kubernetes/workers/cl/worker.yaml.tmpl rename to azure/container-linux/kubernetes/workers/cl/worker.yaml diff --git a/azure/container-linux/kubernetes/workers/workers.tf b/azure/container-linux/kubernetes/workers/workers.tf index 82be439db..7acdf63e9 100644 --- a/azure/container-linux/kubernetes/workers/workers.tf +++ b/azure/container-linux/kubernetes/workers/workers.tf @@ -1,7 +1,7 @@ locals { # Channel for a Container Linux derivative # coreos-stable -> Container Linux Stable - channel = element(split("-", var.os_image), 1) + channel = split("-", var.os_image)[1] } # Workers scale set @@ -104,7 +104,7 @@ data "ct_config" "worker-ignition" { # Worker Container Linux configs data "template_file" "worker-config" { - template = file("${path.module}/cl/worker.yaml.tmpl") + template = file("${path.module}/cl/worker.yaml") vars = { kubeconfig = indent(10, var.kubeconfig) diff --git a/bare-metal/container-linux/kubernetes/cl/controller.yaml.tmpl b/bare-metal/container-linux/kubernetes/cl/controller.yaml similarity index 100% rename from bare-metal/container-linux/kubernetes/cl/controller.yaml.tmpl rename to bare-metal/container-linux/kubernetes/cl/controller.yaml diff --git a/bare-metal/container-linux/kubernetes/cl/install.yaml.tmpl b/bare-metal/container-linux/kubernetes/cl/install.yaml similarity index 100% rename from bare-metal/container-linux/kubernetes/cl/install.yaml.tmpl rename to bare-metal/container-linux/kubernetes/cl/install.yaml diff --git a/bare-metal/container-linux/kubernetes/cl/worker.yaml.tmpl b/bare-metal/container-linux/kubernetes/cl/worker.yaml similarity index 100% rename from bare-metal/container-linux/kubernetes/cl/worker.yaml.tmpl rename to bare-metal/container-linux/kubernetes/cl/worker.yaml diff --git a/bare-metal/container-linux/kubernetes/profiles.tf b/bare-metal/container-linux/kubernetes/profiles.tf index 007fc2c43..6bbaa1d5f 100644 --- a/bare-metal/container-linux/kubernetes/profiles.tf +++ b/bare-metal/container-linux/kubernetes/profiles.tf @@ -31,7 +31,7 @@ resource "matchbox_profile" "container-linux-install" { data "template_file" "container-linux-install-configs" { count = length(var.controllers) + length(var.workers) - template = file("${path.module}/cl/install.yaml.tmpl") + template = file("${path.module}/cl/install.yaml") vars = { os_flavor = local.flavor @@ -72,7 +72,7 @@ resource "matchbox_profile" "cached-container-linux-install" { data "template_file" "cached-container-linux-install-configs" { count = length(var.controllers) + length(var.workers) - template = file("${path.module}/cl/install.yaml.tmpl") + template = file("${path.module}/cl/install.yaml") vars = { os_flavor = local.flavor @@ -150,7 +150,7 @@ data "ct_config" "controller-ignitions" { data "template_file" "controller-configs" { count = length(var.controllers) - template = file("${path.module}/cl/controller.yaml.tmpl") + template = file("${path.module}/cl/controller.yaml") vars = { domain_name = var.controllers.*.domain[count.index] @@ -180,7 +180,7 @@ data "ct_config" "worker-ignitions" { data "template_file" "worker-configs" { count = length(var.workers) - template = file("${path.module}/cl/worker.yaml.tmpl") + template = file("${path.module}/cl/worker.yaml") vars = { domain_name = var.workers.*.domain[count.index] diff --git a/bare-metal/container-linux/kubernetes/ssh.tf b/bare-metal/container-linux/kubernetes/ssh.tf index fcab47358..ab91deafb 100644 --- a/bare-metal/container-linux/kubernetes/ssh.tf +++ b/bare-metal/container-linux/kubernetes/ssh.tf @@ -2,7 +2,7 @@ locals { # format assets for distribution assets_bundle = [ # header with the unpack location - for key, value in module.bootstrap.assets_dist: + for key, value in module.bootstrap.assets_dist : format("##### %s\n%s", key, value) ] } diff --git a/bare-metal/fedora-coreos/kubernetes/ssh.tf b/bare-metal/fedora-coreos/kubernetes/ssh.tf index 560d96fe8..092794fd0 100644 --- a/bare-metal/fedora-coreos/kubernetes/ssh.tf +++ b/bare-metal/fedora-coreos/kubernetes/ssh.tf @@ -2,7 +2,7 @@ locals { # format assets for distribution assets_bundle = [ # header with the unpack location - for key, value in module.bootstrap.assets_dist: + for key, value in module.bootstrap.assets_dist : format("##### %s\n%s", key, value) ] } diff --git a/bare-metal/ignore/.gitkeep b/bare-metal/ignore/.gitkeep deleted file mode 100644 index e69de29bb..000000000 diff --git a/digital-ocean/container-linux/kubernetes/cl/controller.yaml.tmpl b/digital-ocean/container-linux/kubernetes/cl/controller.yaml similarity index 100% rename from digital-ocean/container-linux/kubernetes/cl/controller.yaml.tmpl rename to digital-ocean/container-linux/kubernetes/cl/controller.yaml diff --git a/digital-ocean/container-linux/kubernetes/cl/worker.yaml.tmpl b/digital-ocean/container-linux/kubernetes/cl/worker.yaml similarity index 100% rename from digital-ocean/container-linux/kubernetes/cl/worker.yaml.tmpl rename to digital-ocean/container-linux/kubernetes/cl/worker.yaml diff --git a/digital-ocean/container-linux/kubernetes/controllers.tf b/digital-ocean/container-linux/kubernetes/controllers.tf index 58458b4ba..f020e3595 100644 --- a/digital-ocean/container-linux/kubernetes/controllers.tf +++ b/digital-ocean/container-linux/kubernetes/controllers.tf @@ -11,7 +11,7 @@ resource "digitalocean_record" "controllers" { ttl = 300 # IPv4 addresses of controllers - value = element(digitalocean_droplet.controllers.*.ipv4_address, count.index) + value = digitalocean_droplet.controllers.*.ipv4_address[count.index] } # Discrete DNS records for each controller's private IPv4 for etcd usage @@ -27,7 +27,7 @@ resource "digitalocean_record" "etcds" { ttl = 300 # private IPv4 address for etcd - value = element(digitalocean_droplet.controllers.*.ipv4_address_private, count.index) + value = digitalocean_droplet.controllers.*.ipv4_address_private[count.index] } # Controller droplet instances @@ -44,7 +44,7 @@ resource "digitalocean_droplet" "controllers" { ipv6 = true private_networking = true - user_data = element(data.ct_config.controller-ignitions.*.rendered, count.index) + user_data = data.ct_config.controller-ignitions.*.rendered[count.index] ssh_keys = var.ssh_fingerprints tags = [ @@ -64,7 +64,7 @@ resource "digitalocean_tag" "controllers" { # Controller Ignition configs data "ct_config" "controller-ignitions" { count = var.controller_count - content = element(data.template_file.controller-configs.*.rendered, count.index) + content = data.template_file.controller-configs.*.rendered[count.index] pretty_print = false snippets = var.controller_clc_snippets } @@ -73,7 +73,7 @@ data "ct_config" "controller-ignitions" { data "template_file" "controller-configs" { count = var.controller_count - template = file("${path.module}/cl/controller.yaml.tmpl") + template = file("${path.module}/cl/controller.yaml") vars = { # Cannot use cyclic dependencies on controllers or their DNS records diff --git a/digital-ocean/container-linux/kubernetes/ssh.tf b/digital-ocean/container-linux/kubernetes/ssh.tf index 469fc947d..f4888fe02 100644 --- a/digital-ocean/container-linux/kubernetes/ssh.tf +++ b/digital-ocean/container-linux/kubernetes/ssh.tf @@ -2,7 +2,7 @@ locals { # format assets for distribution assets_bundle = [ # header with the unpack location - for key, value in module.bootstrap.assets_dist: + for key, value in module.bootstrap.assets_dist : format("##### %s\n%s", key, value) ] } diff --git a/digital-ocean/container-linux/kubernetes/workers.tf b/digital-ocean/container-linux/kubernetes/workers.tf index fb2fa02d0..6a1c927f4 100644 --- a/digital-ocean/container-linux/kubernetes/workers.tf +++ b/digital-ocean/container-linux/kubernetes/workers.tf @@ -8,7 +8,7 @@ resource "digitalocean_record" "workers-record-a" { name = "${var.cluster_name}-workers" type = "A" ttl = 300 - value = element(digitalocean_droplet.workers.*.ipv4_address, count.index) + value = digitalocean_droplet.workers.*.ipv4_address[count.index] } resource "digitalocean_record" "workers-record-aaaa" { @@ -20,7 +20,7 @@ resource "digitalocean_record" "workers-record-aaaa" { name = "${var.cluster_name}-workers" type = "AAAA" ttl = 300 - value = element(digitalocean_droplet.workers.*.ipv6_address, count.index) + value = digitalocean_droplet.workers.*.ipv6_address[count.index] } # Worker droplet instances @@ -63,7 +63,7 @@ data "ct_config" "worker-ignition" { # Worker Container Linux config data "template_file" "worker-config" { - template = file("${path.module}/cl/worker.yaml.tmpl") + template = file("${path.module}/cl/worker.yaml") vars = { cluster_dns_service_ip = cidrhost(var.service_cidr, 10) diff --git a/google-cloud/container-linux/kubernetes/cl/controller.yaml.tmpl b/google-cloud/container-linux/kubernetes/cl/controller.yaml similarity index 100% rename from google-cloud/container-linux/kubernetes/cl/controller.yaml.tmpl rename to google-cloud/container-linux/kubernetes/cl/controller.yaml diff --git a/google-cloud/container-linux/kubernetes/controllers.tf b/google-cloud/container-linux/kubernetes/controllers.tf index 893a324b2..3669b1732 100644 --- a/google-cloud/container-linux/kubernetes/controllers.tf +++ b/google-cloud/container-linux/kubernetes/controllers.tf @@ -11,7 +11,7 @@ resource "google_dns_record_set" "etcds" { ttl = 300 # private IPv4 address for etcd - rrdatas = [element(google_compute_instance.controllers.*.network_interface.0.network_ip, count.index)] + rrdatas = [google_compute_instance.controllers.*.network_interface.0.network_ip[count.index]] } # Zones in the region @@ -29,12 +29,13 @@ locals { resource "google_compute_instance" "controllers" { count = var.controller_count - name = "${var.cluster_name}-controller-${count.index}" + name = "${var.cluster_name}-controller-${count.index}" + # use a zone in the region and wrap around (e.g. controllers > zones) zone = element(local.zones, count.index) machine_type = var.controller_type metadata = { - user-data = element(data.ct_config.controller-ignitions.*.rendered, count.index) + user-data = data.ct_config.controller-ignitions.*.rendered[count.index] } boot_disk { @@ -64,11 +65,8 @@ resource "google_compute_instance" "controllers" { # Controller Ignition configs data "ct_config" "controller-ignitions" { - count = var.controller_count - content = element( - data.template_file.controller-configs.*.rendered, - count.index, - ) + count = var.controller_count + content = data.template_file.controller-configs.*.rendered[count.index] pretty_print = false snippets = var.controller_clc_snippets } @@ -77,7 +75,7 @@ data "ct_config" "controller-ignitions" { data "template_file" "controller-configs" { count = var.controller_count - template = file("${path.module}/cl/controller.yaml.tmpl") + template = file("${path.module}/cl/controller.yaml") vars = { # Cannot use cyclic dependencies on controllers or their DNS records diff --git a/google-cloud/container-linux/kubernetes/ssh.tf b/google-cloud/container-linux/kubernetes/ssh.tf index f6983447e..6f80c5589 100644 --- a/google-cloud/container-linux/kubernetes/ssh.tf +++ b/google-cloud/container-linux/kubernetes/ssh.tf @@ -2,7 +2,7 @@ locals { # format assets for distribution assets_bundle = [ # header with the unpack location - for key, value in module.bootstrap.assets_dist: + for key, value in module.bootstrap.assets_dist : format("##### %s\n%s", key, value) ] } @@ -21,7 +21,7 @@ resource "null_resource" "copy-controller-secrets" { user = "core" timeout = "15m" } - + provisioner "file" { content = join("\n", local.assets_bundle) destination = "$HOME/assets" diff --git a/google-cloud/container-linux/kubernetes/workers/cl/worker.yaml.tmpl b/google-cloud/container-linux/kubernetes/workers/cl/worker.yaml similarity index 100% rename from google-cloud/container-linux/kubernetes/workers/cl/worker.yaml.tmpl rename to google-cloud/container-linux/kubernetes/workers/cl/worker.yaml diff --git a/google-cloud/container-linux/kubernetes/workers/workers.tf b/google-cloud/container-linux/kubernetes/workers/workers.tf index 47d2cf174..c9f109484 100644 --- a/google-cloud/container-linux/kubernetes/workers/workers.tf +++ b/google-cloud/container-linux/kubernetes/workers/workers.tf @@ -78,7 +78,7 @@ data "ct_config" "worker-ignition" { # Worker Container Linux config data "template_file" "worker-config" { - template = file("${path.module}/cl/worker.yaml.tmpl") + template = file("${path.module}/cl/worker.yaml") vars = { kubeconfig = indent(10, var.kubeconfig) From f1f4cd6fc02b0530cbcc8951ce1edb6e56bdca69 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Sun, 29 Dec 2019 11:17:26 -0800 Subject: [PATCH 299/523] Inline Container Linux kubelet.service, deprecate kubelet-wrapper * Change kubelet.service on Container Linux nodes to ExecStart Kubelet inline to replace the use of the host OS kubelet-wrapper script * Express rkt run flags and volume mounts in a clear, uniform way to make the Kubelet service easier to audit, manage, and understand * Eliminate reliance on a Container Linux kubelet-wrapper script * Typhoon for Fedora CoreOS developed a kubelet.service that similarly uses an inline ExecStart (except with podman instead of rkt) and a more minimal set of volume mounts. Adopt the volume improvements: * Change Kubelet /etc/kubernetes volume to read-only * Change Kubelet /etc/resolv.conf volume to read-only * Remove unneeded /var/lib/cni volume mount Background: * kubelet-wrapper was added in CoreOS around the time of Kubernetes v1.0 to simplify running a CoreOS-built hyperkube ACI image via rkt-fly. The script defaults are no longer ideal (e.g. rkt's notion of trust dates back to quay.io ACI image serving and signing, which informed the OCI standard images we use today, though they still lack rkt's signing ideas). * Shipping kubelet-wrapper was regretted at CoreOS, but remains in the distro for compatibility. The script is not updated to track hyperkube changes, but it is stable and kubelet.env overrides bridge most gaps * Typhoon Container Linux nodes have used kubelet-wrapper to rkt/rkt-fly run the Kubelet via the official k8s.gcr.io hyperkube image using overrides (new image registry, new image format, restart handling, new mounts, new entrypoint in v1.17). * Observation: Most of what it takes to run a Kubelet container is defined in Typhoon, not in kubelet-wrapper. The wrapper's value is now undermined by having to workaround its dated defaults. Typhoon may be better served defining Kubelet.service explicitly * Typhoon for Fedora CoreOS developed a kubelet.service without the use of a host OS kubelet-wrapper which is both clearer and eliminated some volume mounts --- CHANGES.md | 1 + .../kubernetes/cl/controller.yaml | 56 +++++++++------- .../kubernetes/workers/cl/worker.yaml | 58 ++++++++++------- .../kubernetes/cl/controller.yaml | 58 ++++++++++------- .../kubernetes/workers/cl/worker.yaml | 58 ++++++++++------- .../kubernetes/cl/controller.yaml | 64 +++++++++++-------- .../container-linux/kubernetes/cl/worker.yaml | 64 +++++++++++-------- .../kubernetes/cl/controller.yaml | 56 +++++++++------- .../container-linux/kubernetes/cl/worker.yaml | 56 +++++++++------- .../kubernetes/cl/controller.yaml | 57 ++++++++++------- .../kubernetes/workers/cl/worker.yaml | 57 ++++++++++------- 11 files changed, 342 insertions(+), 243 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 93282a36f..e37a02006 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -6,6 +6,7 @@ Notable changes between versions. * Update CoreDNS from v1.6.5 to [v1.6.6](https://coredns.io/2019/12/11/coredns-1.6.6-release/) ([#602](https://github.com/poseidon/typhoon/pull/602)) * Update Calico from v3.10.2 to v3.11.1 ([#604](https://github.com/poseidon/typhoon/pull/604)) +* Inline Kubelet service on Container Linux nodes ([#606](https://github.com/poseidon/typhoon/pull/606)) #### Addons diff --git a/aws/container-linux/kubernetes/cl/controller.yaml b/aws/container-linux/kubernetes/cl/controller.yaml index 9240653c8..95fb29bd0 100644 --- a/aws/container-linux/kubernetes/cl/controller.yaml +++ b/aws/container-linux/kubernetes/cl/controller.yaml @@ -50,29 +50,47 @@ systemd: Description=Kubelet via Hyperkube Wants=rpc-statd.service [Service] - EnvironmentFile=/etc/kubernetes/kubelet.env - Environment="RKT_RUN_ARGS=--uuid-file-save=/var/cache/kubelet-pod.uuid \ - --volume=resolv,kind=host,source=/etc/resolv.conf \ - --mount volume=resolv,target=/etc/resolv.conf \ - --volume var-lib-cni,kind=host,source=/var/lib/cni \ - --mount volume=var-lib-cni,target=/var/lib/cni \ - --volume var-lib-calico,kind=host,source=/var/lib/calico \ - --mount volume=var-lib-calico,target=/var/lib/calico \ - --volume opt-cni-bin,kind=host,source=/opt/cni/bin \ - --mount volume=opt-cni-bin,target=/opt/cni/bin \ - --volume var-log,kind=host,source=/var/log \ - --mount volume=var-log,target=/var/log \ - --insecure-options=image" Environment=KUBELET_CGROUP_DRIVER=${cgroup_driver} ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests ExecStartPre=/bin/mkdir -p /opt/cni/bin - ExecStartPre=/bin/mkdir -p /var/lib/cni ExecStartPre=/bin/mkdir -p /var/lib/calico ExecStartPre=/bin/mkdir -p /var/lib/kubelet/volumeplugins ExecStartPre=/usr/bin/bash -c "grep 'certificate-authority-data' /etc/kubernetes/kubeconfig | awk '{print $2}' | base64 -d > /etc/kubernetes/ca.crt" ExecStartPre=-/usr/bin/rkt rm --uuid-file=/var/cache/kubelet-pod.uuid - ExecStart=/usr/lib/coreos/kubelet-wrapper \ + ExecStart=/usr/bin/rkt run \ + --uuid-file-save=/var/cache/kubelet-pod.uuid \ + --stage1-from-dir=stage1-fly.aci \ + --hosts-entry host \ + --insecure-options=image \ + --volume etc-kubernetes,kind=host,source=/etc/kubernetes,readOnly=true \ + --mount volume=etc-kubernetes,target=/etc/kubernetes \ + --volume etc-machine-id,kind=host,source=/etc/machine-id,readOnly=true \ + --mount volume=etc-machine-id,target=/etc/machine-id \ + --volume etc-os-release,kind=host,source=/usr/lib/os-release,readOnly=true \ + --mount volume=etc-os-release,target=/etc/os-release \ + --volume=etc-resolv,kind=host,source=/etc/resolv.conf,readOnly=true \ + --mount volume=etc-resolv,target=/etc/resolv.conf \ + --volume etc-ssl-certs,kind=host,source=/etc/ssl/certs,readOnly=true \ + --mount volume=etc-ssl-certs,target=/etc/ssl/certs \ + --volume lib-modules,kind=host,source=/lib/modules,readOnly=true \ + --mount volume=lib-modules,target=/lib/modules \ + --volume run,kind=host,source=/run \ + --mount volume=run,target=/run \ + --volume usr-share-certs,kind=host,source=/usr/share/ca-certificates,readOnly=true \ + --mount volume=usr-share-certs,target=/usr/share/ca-certificates \ + --volume var-lib-calico,kind=host,source=/var/lib/calico \ + --mount volume=var-lib-calico,target=/var/lib/calico \ + --volume var-lib-docker,kind=host,source=/var/lib/docker \ + --mount volume=var-lib-docker,target=/var/lib/docker \ + --volume var-lib-kubelet,kind=host,source=/var/lib/kubelet,recursive=true \ + --mount volume=var-lib-kubelet,target=/var/lib/kubelet \ + --volume var-log,kind=host,source=/var/log \ + --mount volume=var-log,target=/var/log \ + --volume opt-cni-bin,kind=host,source=/opt/cni/bin \ + --mount volume=opt-cni-bin,target=/opt/cni/bin \ + docker://k8s.gcr.io/hyperkube:v1.17.0 \ + --exec=/usr/local/bin/kubelet -- \ --anonymous-auth=false \ --authentication-token-webhook \ --authorization-mode=Webhook \ @@ -130,14 +148,6 @@ storage: contents: inline: | ${kubeconfig} - - path: /etc/kubernetes/kubelet.env - filesystem: root - mode: 0644 - contents: - inline: | - KUBELET_IMAGE_URL=docker://k8s.gcr.io/hyperkube - KUBELET_IMAGE_TAG=v1.17.0 - KUBELET_IMAGE_ARGS="--exec=/usr/local/bin/kubelet" - path: /opt/bootstrap/layout filesystem: root mode: 0544 diff --git a/aws/container-linux/kubernetes/workers/cl/worker.yaml b/aws/container-linux/kubernetes/workers/cl/worker.yaml index 6a36c8ce8..ba27b8293 100644 --- a/aws/container-linux/kubernetes/workers/cl/worker.yaml +++ b/aws/container-linux/kubernetes/workers/cl/worker.yaml @@ -25,29 +25,47 @@ systemd: Description=Kubelet via Hyperkube Wants=rpc-statd.service [Service] - EnvironmentFile=/etc/kubernetes/kubelet.env - Environment="RKT_RUN_ARGS=--uuid-file-save=/var/cache/kubelet-pod.uuid \ - --volume=resolv,kind=host,source=/etc/resolv.conf \ - --mount volume=resolv,target=/etc/resolv.conf \ - --volume var-lib-cni,kind=host,source=/var/lib/cni \ - --mount volume=var-lib-cni,target=/var/lib/cni \ - --volume var-lib-calico,kind=host,source=/var/lib/calico \ - --mount volume=var-lib-calico,target=/var/lib/calico \ - --volume opt-cni-bin,kind=host,source=/opt/cni/bin \ - --mount volume=opt-cni-bin,target=/opt/cni/bin \ - --volume var-log,kind=host,source=/var/log \ - --mount volume=var-log,target=/var/log \ - --insecure-options=image" Environment=KUBELET_CGROUP_DRIVER=${cgroup_driver} - ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d + ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests ExecStartPre=/bin/mkdir -p /opt/cni/bin - ExecStartPre=/bin/mkdir -p /var/lib/cni ExecStartPre=/bin/mkdir -p /var/lib/calico ExecStartPre=/bin/mkdir -p /var/lib/kubelet/volumeplugins ExecStartPre=/usr/bin/bash -c "grep 'certificate-authority-data' /etc/kubernetes/kubeconfig | awk '{print $2}' | base64 -d > /etc/kubernetes/ca.crt" ExecStartPre=-/usr/bin/rkt rm --uuid-file=/var/cache/kubelet-pod.uuid - ExecStart=/usr/lib/coreos/kubelet-wrapper \ + ExecStart=/usr/bin/rkt run \ + --uuid-file-save=/var/cache/kubelet-pod.uuid \ + --stage1-from-dir=stage1-fly.aci \ + --hosts-entry host \ + --insecure-options=image \ + --volume etc-kubernetes,kind=host,source=/etc/kubernetes,readOnly=true \ + --mount volume=etc-kubernetes,target=/etc/kubernetes \ + --volume etc-machine-id,kind=host,source=/etc/machine-id,readOnly=true \ + --mount volume=etc-machine-id,target=/etc/machine-id \ + --volume etc-os-release,kind=host,source=/usr/lib/os-release,readOnly=true \ + --mount volume=etc-os-release,target=/etc/os-release \ + --volume=etc-resolv,kind=host,source=/etc/resolv.conf,readOnly=true \ + --mount volume=etc-resolv,target=/etc/resolv.conf \ + --volume etc-ssl-certs,kind=host,source=/etc/ssl/certs,readOnly=true \ + --mount volume=etc-ssl-certs,target=/etc/ssl/certs \ + --volume lib-modules,kind=host,source=/lib/modules,readOnly=true \ + --mount volume=lib-modules,target=/lib/modules \ + --volume run,kind=host,source=/run \ + --mount volume=run,target=/run \ + --volume usr-share-certs,kind=host,source=/usr/share/ca-certificates,readOnly=true \ + --mount volume=usr-share-certs,target=/usr/share/ca-certificates \ + --volume var-lib-calico,kind=host,source=/var/lib/calico \ + --mount volume=var-lib-calico,target=/var/lib/calico \ + --volume var-lib-docker,kind=host,source=/var/lib/docker \ + --mount volume=var-lib-docker,target=/var/lib/docker \ + --volume var-lib-kubelet,kind=host,source=/var/lib/kubelet,recursive=true \ + --mount volume=var-lib-kubelet,target=/var/lib/kubelet \ + --volume var-log,kind=host,source=/var/log \ + --mount volume=var-log,target=/var/log \ + --volume opt-cni-bin,kind=host,source=/opt/cni/bin \ + --mount volume=opt-cni-bin,target=/opt/cni/bin \ + docker://k8s.gcr.io/hyperkube:v1.17.0 \ + --exec=/usr/local/bin/kubelet -- \ --anonymous-auth=false \ --authentication-token-webhook \ --authorization-mode=Webhook \ @@ -92,14 +110,6 @@ storage: contents: inline: | ${kubeconfig} - - path: /etc/kubernetes/kubelet.env - filesystem: root - mode: 0644 - contents: - inline: | - KUBELET_IMAGE_URL=docker://k8s.gcr.io/hyperkube - KUBELET_IMAGE_TAG=v1.17.0 - KUBELET_IMAGE_ARGS="--exec=/usr/local/bin/kubelet" - path: /etc/sysctl.d/max-user-watches.conf filesystem: root contents: diff --git a/azure/container-linux/kubernetes/cl/controller.yaml b/azure/container-linux/kubernetes/cl/controller.yaml index 80f53cd36..236f65e9f 100644 --- a/azure/container-linux/kubernetes/cl/controller.yaml +++ b/azure/container-linux/kubernetes/cl/controller.yaml @@ -50,28 +50,46 @@ systemd: Description=Kubelet via Hyperkube Wants=rpc-statd.service [Service] - EnvironmentFile=/etc/kubernetes/kubelet.env - Environment="RKT_RUN_ARGS=--uuid-file-save=/var/cache/kubelet-pod.uuid \ - --volume=resolv,kind=host,source=/etc/resolv.conf \ - --mount volume=resolv,target=/etc/resolv.conf \ - --volume var-lib-cni,kind=host,source=/var/lib/cni \ - --mount volume=var-lib-cni,target=/var/lib/cni \ - --volume var-lib-calico,kind=host,source=/var/lib/calico \ - --mount volume=var-lib-calico,target=/var/lib/calico \ - --volume opt-cni-bin,kind=host,source=/opt/cni/bin \ - --mount volume=opt-cni-bin,target=/opt/cni/bin \ - --volume var-log,kind=host,source=/var/log \ - --mount volume=var-log,target=/var/log \ - --insecure-options=image" - ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d + ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests ExecStartPre=/bin/mkdir -p /opt/cni/bin - ExecStartPre=/bin/mkdir -p /var/lib/cni ExecStartPre=/bin/mkdir -p /var/lib/calico ExecStartPre=/bin/mkdir -p /var/lib/kubelet/volumeplugins ExecStartPre=/usr/bin/bash -c "grep 'certificate-authority-data' /etc/kubernetes/kubeconfig | awk '{print $2}' | base64 -d > /etc/kubernetes/ca.crt" ExecStartPre=-/usr/bin/rkt rm --uuid-file=/var/cache/kubelet-pod.uuid - ExecStart=/usr/lib/coreos/kubelet-wrapper \ + ExecStart=/usr/bin/rkt run \ + --uuid-file-save=/var/cache/kubelet-pod.uuid \ + --stage1-from-dir=stage1-fly.aci \ + --hosts-entry host \ + --insecure-options=image \ + --volume etc-kubernetes,kind=host,source=/etc/kubernetes,readOnly=true \ + --mount volume=etc-kubernetes,target=/etc/kubernetes \ + --volume etc-machine-id,kind=host,source=/etc/machine-id,readOnly=true \ + --mount volume=etc-machine-id,target=/etc/machine-id \ + --volume etc-os-release,kind=host,source=/usr/lib/os-release,readOnly=true \ + --mount volume=etc-os-release,target=/etc/os-release \ + --volume=etc-resolv,kind=host,source=/etc/resolv.conf,readOnly=true \ + --mount volume=etc-resolv,target=/etc/resolv.conf \ + --volume etc-ssl-certs,kind=host,source=/etc/ssl/certs,readOnly=true \ + --mount volume=etc-ssl-certs,target=/etc/ssl/certs \ + --volume lib-modules,kind=host,source=/lib/modules,readOnly=true \ + --mount volume=lib-modules,target=/lib/modules \ + --volume run,kind=host,source=/run \ + --mount volume=run,target=/run \ + --volume usr-share-certs,kind=host,source=/usr/share/ca-certificates,readOnly=true \ + --mount volume=usr-share-certs,target=/usr/share/ca-certificates \ + --volume var-lib-calico,kind=host,source=/var/lib/calico \ + --mount volume=var-lib-calico,target=/var/lib/calico \ + --volume var-lib-docker,kind=host,source=/var/lib/docker \ + --mount volume=var-lib-docker,target=/var/lib/docker \ + --volume var-lib-kubelet,kind=host,source=/var/lib/kubelet,recursive=true \ + --mount volume=var-lib-kubelet,target=/var/lib/kubelet \ + --volume var-log,kind=host,source=/var/log \ + --mount volume=var-log,target=/var/log \ + --volume opt-cni-bin,kind=host,source=/opt/cni/bin \ + --mount volume=opt-cni-bin,target=/opt/cni/bin \ + docker://k8s.gcr.io/hyperkube:v1.17.0 \ + --exec=/usr/local/bin/kubelet -- \ --anonymous-auth=false \ --authentication-token-webhook \ --authorization-mode=Webhook \ @@ -128,14 +146,6 @@ storage: contents: inline: | ${kubeconfig} - - path: /etc/kubernetes/kubelet.env - filesystem: root - mode: 0644 - contents: - inline: | - KUBELET_IMAGE_URL=docker://k8s.gcr.io/hyperkube - KUBELET_IMAGE_TAG=v1.17.0 - KUBELET_IMAGE_ARGS="--exec=/usr/local/bin/kubelet" - path: /opt/bootstrap/layout filesystem: root mode: 0544 diff --git a/azure/container-linux/kubernetes/workers/cl/worker.yaml b/azure/container-linux/kubernetes/workers/cl/worker.yaml index d29ecb367..ab0992927 100644 --- a/azure/container-linux/kubernetes/workers/cl/worker.yaml +++ b/azure/container-linux/kubernetes/workers/cl/worker.yaml @@ -25,28 +25,46 @@ systemd: Description=Kubelet via Hyperkube Wants=rpc-statd.service [Service] - EnvironmentFile=/etc/kubernetes/kubelet.env - Environment="RKT_RUN_ARGS=--uuid-file-save=/var/cache/kubelet-pod.uuid \ - --volume=resolv,kind=host,source=/etc/resolv.conf \ - --mount volume=resolv,target=/etc/resolv.conf \ - --volume var-lib-cni,kind=host,source=/var/lib/cni \ - --mount volume=var-lib-cni,target=/var/lib/cni \ - --volume var-lib-calico,kind=host,source=/var/lib/calico \ - --mount volume=var-lib-calico,target=/var/lib/calico \ - --volume opt-cni-bin,kind=host,source=/opt/cni/bin \ - --mount volume=opt-cni-bin,target=/opt/cni/bin \ - --volume var-log,kind=host,source=/var/log \ - --mount volume=var-log,target=/var/log \ - --insecure-options=image" - ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d + ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests ExecStartPre=/bin/mkdir -p /opt/cni/bin - ExecStartPre=/bin/mkdir -p /var/lib/cni ExecStartPre=/bin/mkdir -p /var/lib/calico ExecStartPre=/bin/mkdir -p /var/lib/kubelet/volumeplugins ExecStartPre=/usr/bin/bash -c "grep 'certificate-authority-data' /etc/kubernetes/kubeconfig | awk '{print $2}' | base64 -d > /etc/kubernetes/ca.crt" ExecStartPre=-/usr/bin/rkt rm --uuid-file=/var/cache/kubelet-pod.uuid - ExecStart=/usr/lib/coreos/kubelet-wrapper \ + ExecStart=/usr/bin/rkt run \ + --uuid-file-save=/var/cache/kubelet-pod.uuid \ + --stage1-from-dir=stage1-fly.aci \ + --hosts-entry host \ + --insecure-options=image \ + --volume etc-kubernetes,kind=host,source=/etc/kubernetes,readOnly=true \ + --mount volume=etc-kubernetes,target=/etc/kubernetes \ + --volume etc-machine-id,kind=host,source=/etc/machine-id,readOnly=true \ + --mount volume=etc-machine-id,target=/etc/machine-id \ + --volume etc-os-release,kind=host,source=/usr/lib/os-release,readOnly=true \ + --mount volume=etc-os-release,target=/etc/os-release \ + --volume=etc-resolv,kind=host,source=/etc/resolv.conf,readOnly=true \ + --mount volume=etc-resolv,target=/etc/resolv.conf \ + --volume etc-ssl-certs,kind=host,source=/etc/ssl/certs,readOnly=true \ + --mount volume=etc-ssl-certs,target=/etc/ssl/certs \ + --volume lib-modules,kind=host,source=/lib/modules,readOnly=true \ + --mount volume=lib-modules,target=/lib/modules \ + --volume run,kind=host,source=/run \ + --mount volume=run,target=/run \ + --volume usr-share-certs,kind=host,source=/usr/share/ca-certificates,readOnly=true \ + --mount volume=usr-share-certs,target=/usr/share/ca-certificates \ + --volume var-lib-calico,kind=host,source=/var/lib/calico \ + --mount volume=var-lib-calico,target=/var/lib/calico \ + --volume var-lib-docker,kind=host,source=/var/lib/docker \ + --mount volume=var-lib-docker,target=/var/lib/docker \ + --volume var-lib-kubelet,kind=host,source=/var/lib/kubelet,recursive=true \ + --mount volume=var-lib-kubelet,target=/var/lib/kubelet \ + --volume var-log,kind=host,source=/var/log \ + --mount volume=var-log,target=/var/log \ + --volume opt-cni-bin,kind=host,source=/opt/cni/bin \ + --mount volume=opt-cni-bin,target=/opt/cni/bin \ + docker://k8s.gcr.io/hyperkube:v1.17.0 \ + --exec=/usr/local/bin/kubelet -- \ --anonymous-auth=false \ --authentication-token-webhook \ --authorization-mode=Webhook \ @@ -90,14 +108,6 @@ storage: contents: inline: | ${kubeconfig} - - path: /etc/kubernetes/kubelet.env - filesystem: root - mode: 0644 - contents: - inline: | - KUBELET_IMAGE_URL=docker://k8s.gcr.io/hyperkube - KUBELET_IMAGE_TAG=v1.17.0 - KUBELET_IMAGE_ARGS="--exec=/usr/local/bin/kubelet" - path: /etc/sysctl.d/max-user-watches.conf filesystem: root contents: diff --git a/bare-metal/container-linux/kubernetes/cl/controller.yaml b/bare-metal/container-linux/kubernetes/cl/controller.yaml index ddc943f98..42e7d3965 100644 --- a/bare-metal/container-linux/kubernetes/cl/controller.yaml +++ b/bare-metal/container-linux/kubernetes/cl/controller.yaml @@ -58,33 +58,51 @@ systemd: Description=Kubelet via Hyperkube Wants=rpc-statd.service [Service] - EnvironmentFile=/etc/kubernetes/kubelet.env - Environment="RKT_RUN_ARGS=--uuid-file-save=/var/cache/kubelet-pod.uuid \ - --volume=resolv,kind=host,source=/etc/resolv.conf \ - --mount volume=resolv,target=/etc/resolv.conf \ - --volume var-lib-cni,kind=host,source=/var/lib/cni \ - --mount volume=var-lib-cni,target=/var/lib/cni \ - --volume var-lib-calico,kind=host,source=/var/lib/calico \ - --mount volume=var-lib-calico,target=/var/lib/calico \ - --volume opt-cni-bin,kind=host,source=/opt/cni/bin \ - --mount volume=opt-cni-bin,target=/opt/cni/bin \ - --volume var-log,kind=host,source=/var/log \ - --mount volume=var-log,target=/var/log \ - --volume iscsiconf,kind=host,source=/etc/iscsi/ \ - --mount volume=iscsiconf,target=/etc/iscsi/ \ - --volume iscsiadm,kind=host,source=/usr/sbin/iscsiadm \ - --mount volume=iscsiadm,target=/sbin/iscsiadm \ - --insecure-options=image" Environment=KUBELET_CGROUP_DRIVER=${cgroup_driver} ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests ExecStartPre=/bin/mkdir -p /opt/cni/bin - ExecStartPre=/bin/mkdir -p /var/lib/cni ExecStartPre=/bin/mkdir -p /var/lib/calico ExecStartPre=/bin/mkdir -p /var/lib/kubelet/volumeplugins ExecStartPre=/usr/bin/bash -c "grep 'certificate-authority-data' /etc/kubernetes/kubeconfig | awk '{print $2}' | base64 -d > /etc/kubernetes/ca.crt" ExecStartPre=-/usr/bin/rkt rm --uuid-file=/var/cache/kubelet-pod.uuid - ExecStart=/usr/lib/coreos/kubelet-wrapper \ + ExecStart=/usr/bin/rkt run \ + --uuid-file-save=/var/cache/kubelet-pod.uuid \ + --stage1-from-dir=stage1-fly.aci \ + --hosts-entry host \ + --insecure-options=image \ + --volume etc-kubernetes,kind=host,source=/etc/kubernetes,readOnly=true \ + --mount volume=etc-kubernetes,target=/etc/kubernetes \ + --volume etc-machine-id,kind=host,source=/etc/machine-id,readOnly=true \ + --mount volume=etc-machine-id,target=/etc/machine-id \ + --volume etc-os-release,kind=host,source=/usr/lib/os-release,readOnly=true \ + --mount volume=etc-os-release,target=/etc/os-release \ + --volume=etc-resolv,kind=host,source=/etc/resolv.conf,readOnly=true \ + --mount volume=etc-resolv,target=/etc/resolv.conf \ + --volume etc-ssl-certs,kind=host,source=/etc/ssl/certs,readOnly=true \ + --mount volume=etc-ssl-certs,target=/etc/ssl/certs \ + --volume lib-modules,kind=host,source=/lib/modules,readOnly=true \ + --mount volume=lib-modules,target=/lib/modules \ + --volume run,kind=host,source=/run \ + --mount volume=run,target=/run \ + --volume usr-share-certs,kind=host,source=/usr/share/ca-certificates,readOnly=true \ + --mount volume=usr-share-certs,target=/usr/share/ca-certificates \ + --volume var-lib-calico,kind=host,source=/var/lib/calico \ + --mount volume=var-lib-calico,target=/var/lib/calico \ + --volume var-lib-docker,kind=host,source=/var/lib/docker \ + --mount volume=var-lib-docker,target=/var/lib/docker \ + --volume var-lib-kubelet,kind=host,source=/var/lib/kubelet,recursive=true \ + --mount volume=var-lib-kubelet,target=/var/lib/kubelet \ + --volume var-log,kind=host,source=/var/log \ + --mount volume=var-log,target=/var/log \ + --volume opt-cni-bin,kind=host,source=/opt/cni/bin \ + --mount volume=opt-cni-bin,target=/opt/cni/bin \ + --volume etc-iscsi,kind=host,source=/etc/iscsi \ + --mount volume=etc-iscsi,target=/etc/iscsi \ + --volume usr-sbin-iscsiadm,kind=host,source=/usr/sbin/iscsiadm \ + --mount volume=usr-sbin-iscsiadm,target=/sbin/iscsiadm \ + docker://k8s.gcr.io/hyperkube:v1.17.0 \ + --exec=/usr/local/bin/kubelet -- \ --anonymous-auth=false \ --authentication-token-webhook \ --authorization-mode=Webhook \ @@ -137,14 +155,6 @@ systemd: WantedBy=multi-user.target storage: files: - - path: /etc/kubernetes/kubelet.env - filesystem: root - mode: 0644 - contents: - inline: | - KUBELET_IMAGE_URL=docker://k8s.gcr.io/hyperkube - KUBELET_IMAGE_TAG=v1.17.0 - KUBELET_IMAGE_ARGS="--exec=/usr/local/bin/kubelet" - path: /etc/hostname filesystem: root mode: 0644 diff --git a/bare-metal/container-linux/kubernetes/cl/worker.yaml b/bare-metal/container-linux/kubernetes/cl/worker.yaml index efe18ddc9..c2b78e7a3 100644 --- a/bare-metal/container-linux/kubernetes/cl/worker.yaml +++ b/bare-metal/container-linux/kubernetes/cl/worker.yaml @@ -33,33 +33,51 @@ systemd: Description=Kubelet via Hyperkube Wants=rpc-statd.service [Service] - EnvironmentFile=/etc/kubernetes/kubelet.env - Environment="RKT_RUN_ARGS=--uuid-file-save=/var/cache/kubelet-pod.uuid \ - --volume=resolv,kind=host,source=/etc/resolv.conf \ - --mount volume=resolv,target=/etc/resolv.conf \ - --volume var-lib-cni,kind=host,source=/var/lib/cni \ - --mount volume=var-lib-cni,target=/var/lib/cni \ - --volume var-lib-calico,kind=host,source=/var/lib/calico \ - --mount volume=var-lib-calico,target=/var/lib/calico \ - --volume opt-cni-bin,kind=host,source=/opt/cni/bin \ - --mount volume=opt-cni-bin,target=/opt/cni/bin \ - --volume var-log,kind=host,source=/var/log \ - --mount volume=var-log,target=/var/log \ - --volume iscsiconf,kind=host,source=/etc/iscsi/ \ - --mount volume=iscsiconf,target=/etc/iscsi/ \ - --volume iscsiadm,kind=host,source=/usr/sbin/iscsiadm \ - --mount volume=iscsiadm,target=/sbin/iscsiadm \ - --insecure-options=image" Environment=KUBELET_CGROUP_DRIVER=${cgroup_driver} ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests ExecStartPre=/bin/mkdir -p /opt/cni/bin - ExecStartPre=/bin/mkdir -p /var/lib/cni ExecStartPre=/bin/mkdir -p /var/lib/calico ExecStartPre=/bin/mkdir -p /var/lib/kubelet/volumeplugins ExecStartPre=/usr/bin/bash -c "grep 'certificate-authority-data' /etc/kubernetes/kubeconfig | awk '{print $2}' | base64 -d > /etc/kubernetes/ca.crt" ExecStartPre=-/usr/bin/rkt rm --uuid-file=/var/cache/kubelet-pod.uuid - ExecStart=/usr/lib/coreos/kubelet-wrapper \ + ExecStart=/usr/bin/rkt run \ + --uuid-file-save=/var/cache/kubelet-pod.uuid \ + --stage1-from-dir=stage1-fly.aci \ + --hosts-entry host \ + --insecure-options=image \ + --volume etc-kubernetes,kind=host,source=/etc/kubernetes,readOnly=true \ + --mount volume=etc-kubernetes,target=/etc/kubernetes \ + --volume etc-machine-id,kind=host,source=/etc/machine-id,readOnly=true \ + --mount volume=etc-machine-id,target=/etc/machine-id \ + --volume etc-os-release,kind=host,source=/usr/lib/os-release,readOnly=true \ + --mount volume=etc-os-release,target=/etc/os-release \ + --volume=etc-resolv,kind=host,source=/etc/resolv.conf,readOnly=true \ + --mount volume=etc-resolv,target=/etc/resolv.conf \ + --volume etc-ssl-certs,kind=host,source=/etc/ssl/certs,readOnly=true \ + --mount volume=etc-ssl-certs,target=/etc/ssl/certs \ + --volume lib-modules,kind=host,source=/lib/modules,readOnly=true \ + --mount volume=lib-modules,target=/lib/modules \ + --volume run,kind=host,source=/run \ + --mount volume=run,target=/run \ + --volume usr-share-certs,kind=host,source=/usr/share/ca-certificates,readOnly=true \ + --mount volume=usr-share-certs,target=/usr/share/ca-certificates \ + --volume var-lib-calico,kind=host,source=/var/lib/calico \ + --mount volume=var-lib-calico,target=/var/lib/calico \ + --volume var-lib-docker,kind=host,source=/var/lib/docker \ + --mount volume=var-lib-docker,target=/var/lib/docker \ + --volume var-lib-kubelet,kind=host,source=/var/lib/kubelet,recursive=true \ + --mount volume=var-lib-kubelet,target=/var/lib/kubelet \ + --volume var-log,kind=host,source=/var/log \ + --mount volume=var-log,target=/var/log \ + --volume opt-cni-bin,kind=host,source=/opt/cni/bin \ + --mount volume=opt-cni-bin,target=/opt/cni/bin \ + --volume etc-iscsi,kind=host,source=/etc/iscsi \ + --mount volume=etc-iscsi,target=/etc/iscsi \ + --volume usr-sbin-iscsiadm,kind=host,source=/usr/sbin/iscsiadm \ + --mount volume=usr-sbin-iscsiadm,target=/sbin/iscsiadm \ + docker://k8s.gcr.io/hyperkube:v1.17.0 \ + --exec=/usr/local/bin/kubelet -- \ --anonymous-auth=false \ --authentication-token-webhook \ --authorization-mode=Webhook \ @@ -85,14 +103,6 @@ systemd: storage: files: - - path: /etc/kubernetes/kubelet.env - filesystem: root - mode: 0644 - contents: - inline: | - KUBELET_IMAGE_URL=docker://k8s.gcr.io/hyperkube - KUBELET_IMAGE_TAG=v1.17.0 - KUBELET_IMAGE_ARGS="--exec=/usr/local/bin/kubelet" - path: /etc/hostname filesystem: root mode: 0644 diff --git a/digital-ocean/container-linux/kubernetes/cl/controller.yaml b/digital-ocean/container-linux/kubernetes/cl/controller.yaml index d51f24fe8..e654fb454 100644 --- a/digital-ocean/container-linux/kubernetes/cl/controller.yaml +++ b/digital-ocean/container-linux/kubernetes/cl/controller.yaml @@ -60,29 +60,47 @@ systemd: After=coreos-metadata.service Wants=rpc-statd.service [Service] - EnvironmentFile=/etc/kubernetes/kubelet.env EnvironmentFile=/run/metadata/coreos - Environment="RKT_RUN_ARGS=--uuid-file-save=/var/cache/kubelet-pod.uuid \ - --volume=resolv,kind=host,source=/etc/resolv.conf \ - --mount volume=resolv,target=/etc/resolv.conf \ - --volume var-lib-cni,kind=host,source=/var/lib/cni \ - --mount volume=var-lib-cni,target=/var/lib/cni \ - --volume var-lib-calico,kind=host,source=/var/lib/calico \ - --mount volume=var-lib-calico,target=/var/lib/calico \ - --volume opt-cni-bin,kind=host,source=/opt/cni/bin \ - --mount volume=opt-cni-bin,target=/opt/cni/bin \ - --volume var-log,kind=host,source=/var/log \ - --mount volume=var-log,target=/var/log \ - --insecure-options=image" ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests ExecStartPre=/bin/mkdir -p /opt/cni/bin - ExecStartPre=/bin/mkdir -p /var/lib/cni ExecStartPre=/bin/mkdir -p /var/lib/calico ExecStartPre=/bin/mkdir -p /var/lib/kubelet/volumeplugins ExecStartPre=/usr/bin/bash -c "grep 'certificate-authority-data' /etc/kubernetes/kubeconfig | awk '{print $2}' | base64 -d > /etc/kubernetes/ca.crt" ExecStartPre=-/usr/bin/rkt rm --uuid-file=/var/cache/kubelet-pod.uuid - ExecStart=/usr/lib/coreos/kubelet-wrapper \ + ExecStart=/usr/bin/rkt run \ + --uuid-file-save=/var/cache/kubelet-pod.uuid \ + --stage1-from-dir=stage1-fly.aci \ + --hosts-entry host \ + --insecure-options=image \ + --volume etc-kubernetes,kind=host,source=/etc/kubernetes,readOnly=true \ + --mount volume=etc-kubernetes,target=/etc/kubernetes \ + --volume etc-machine-id,kind=host,source=/etc/machine-id,readOnly=true \ + --mount volume=etc-machine-id,target=/etc/machine-id \ + --volume etc-os-release,kind=host,source=/usr/lib/os-release,readOnly=true \ + --mount volume=etc-os-release,target=/etc/os-release \ + --volume=etc-resolv,kind=host,source=/etc/resolv.conf,readOnly=true \ + --mount volume=etc-resolv,target=/etc/resolv.conf \ + --volume etc-ssl-certs,kind=host,source=/etc/ssl/certs,readOnly=true \ + --mount volume=etc-ssl-certs,target=/etc/ssl/certs \ + --volume lib-modules,kind=host,source=/lib/modules,readOnly=true \ + --mount volume=lib-modules,target=/lib/modules \ + --volume run,kind=host,source=/run \ + --mount volume=run,target=/run \ + --volume usr-share-certs,kind=host,source=/usr/share/ca-certificates,readOnly=true \ + --mount volume=usr-share-certs,target=/usr/share/ca-certificates \ + --volume var-lib-calico,kind=host,source=/var/lib/calico \ + --mount volume=var-lib-calico,target=/var/lib/calico \ + --volume var-lib-docker,kind=host,source=/var/lib/docker \ + --mount volume=var-lib-docker,target=/var/lib/docker \ + --volume var-lib-kubelet,kind=host,source=/var/lib/kubelet,recursive=true \ + --mount volume=var-lib-kubelet,target=/var/lib/kubelet \ + --volume var-log,kind=host,source=/var/log \ + --mount volume=var-log,target=/var/log \ + --volume opt-cni-bin,kind=host,source=/opt/cni/bin \ + --mount volume=opt-cni-bin,target=/opt/cni/bin \ + docker://k8s.gcr.io/hyperkube:v1.17.0 \ + --exec=/usr/local/bin/kubelet -- \ --anonymous-auth=false \ --authentication-token-webhook \ --authorization-mode=Webhook \ @@ -134,14 +152,6 @@ systemd: WantedBy=multi-user.target storage: files: - - path: /etc/kubernetes/kubelet.env - filesystem: root - mode: 0644 - contents: - inline: | - KUBELET_IMAGE_URL=docker://k8s.gcr.io/hyperkube - KUBELET_IMAGE_TAG=v1.17.0 - KUBELET_IMAGE_ARGS="--exec=/usr/local/bin/kubelet" - path: /opt/bootstrap/layout filesystem: root mode: 0544 diff --git a/digital-ocean/container-linux/kubernetes/cl/worker.yaml b/digital-ocean/container-linux/kubernetes/cl/worker.yaml index 4ae4603e5..007014030 100644 --- a/digital-ocean/container-linux/kubernetes/cl/worker.yaml +++ b/digital-ocean/container-linux/kubernetes/cl/worker.yaml @@ -35,29 +35,47 @@ systemd: After=coreos-metadata.service Wants=rpc-statd.service [Service] - EnvironmentFile=/etc/kubernetes/kubelet.env EnvironmentFile=/run/metadata/coreos - Environment="RKT_RUN_ARGS=--uuid-file-save=/var/cache/kubelet-pod.uuid \ - --volume=resolv,kind=host,source=/etc/resolv.conf \ - --mount volume=resolv,target=/etc/resolv.conf \ - --volume var-lib-cni,kind=host,source=/var/lib/cni \ - --mount volume=var-lib-cni,target=/var/lib/cni \ - --volume var-lib-calico,kind=host,source=/var/lib/calico \ - --mount volume=var-lib-calico,target=/var/lib/calico \ - --volume opt-cni-bin,kind=host,source=/opt/cni/bin \ - --mount volume=opt-cni-bin,target=/opt/cni/bin \ - --volume var-log,kind=host,source=/var/log \ - --mount volume=var-log,target=/var/log \ - --insecure-options=image" ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests ExecStartPre=/bin/mkdir -p /opt/cni/bin - ExecStartPre=/bin/mkdir -p /var/lib/cni ExecStartPre=/bin/mkdir -p /var/lib/calico ExecStartPre=/bin/mkdir -p /var/lib/kubelet/volumeplugins ExecStartPre=/usr/bin/bash -c "grep 'certificate-authority-data' /etc/kubernetes/kubeconfig | awk '{print $2}' | base64 -d > /etc/kubernetes/ca.crt" ExecStartPre=-/usr/bin/rkt rm --uuid-file=/var/cache/kubelet-pod.uuid - ExecStart=/usr/lib/coreos/kubelet-wrapper \ + ExecStart=/usr/bin/rkt run \ + --uuid-file-save=/var/cache/kubelet-pod.uuid \ + --stage1-from-dir=stage1-fly.aci \ + --hosts-entry host \ + --insecure-options=image \ + --volume etc-kubernetes,kind=host,source=/etc/kubernetes,readOnly=true \ + --mount volume=etc-kubernetes,target=/etc/kubernetes \ + --volume etc-machine-id,kind=host,source=/etc/machine-id,readOnly=true \ + --mount volume=etc-machine-id,target=/etc/machine-id \ + --volume etc-os-release,kind=host,source=/usr/lib/os-release,readOnly=true \ + --mount volume=etc-os-release,target=/etc/os-release \ + --volume=etc-resolv,kind=host,source=/etc/resolv.conf,readOnly=true \ + --mount volume=etc-resolv,target=/etc/resolv.conf \ + --volume etc-ssl-certs,kind=host,source=/etc/ssl/certs,readOnly=true \ + --mount volume=etc-ssl-certs,target=/etc/ssl/certs \ + --volume lib-modules,kind=host,source=/lib/modules,readOnly=true \ + --mount volume=lib-modules,target=/lib/modules \ + --volume run,kind=host,source=/run \ + --mount volume=run,target=/run \ + --volume usr-share-certs,kind=host,source=/usr/share/ca-certificates,readOnly=true \ + --mount volume=usr-share-certs,target=/usr/share/ca-certificates \ + --volume var-lib-calico,kind=host,source=/var/lib/calico \ + --mount volume=var-lib-calico,target=/var/lib/calico \ + --volume var-lib-docker,kind=host,source=/var/lib/docker \ + --mount volume=var-lib-docker,target=/var/lib/docker \ + --volume var-lib-kubelet,kind=host,source=/var/lib/kubelet,recursive=true \ + --mount volume=var-lib-kubelet,target=/var/lib/kubelet \ + --volume var-log,kind=host,source=/var/log \ + --mount volume=var-log,target=/var/log \ + --volume opt-cni-bin,kind=host,source=/opt/cni/bin \ + --mount volume=opt-cni-bin,target=/opt/cni/bin \ + docker://k8s.gcr.io/hyperkube:v1.17.0 \ + --exec=/usr/local/bin/kubelet -- \ --anonymous-auth=false \ --authentication-token-webhook \ --authorization-mode=Webhook \ @@ -93,14 +111,6 @@ systemd: WantedBy=multi-user.target storage: files: - - path: /etc/kubernetes/kubelet.env - filesystem: root - mode: 0644 - contents: - inline: | - KUBELET_IMAGE_URL=docker://k8s.gcr.io/hyperkube - KUBELET_IMAGE_TAG=v1.17.0 - KUBELET_IMAGE_ARGS="--exec=/usr/local/bin/kubelet" - path: /etc/sysctl.d/max-user-watches.conf filesystem: root contents: diff --git a/google-cloud/container-linux/kubernetes/cl/controller.yaml b/google-cloud/container-linux/kubernetes/cl/controller.yaml index b2baaa048..c48e90eb9 100644 --- a/google-cloud/container-linux/kubernetes/cl/controller.yaml +++ b/google-cloud/container-linux/kubernetes/cl/controller.yaml @@ -50,29 +50,46 @@ systemd: Description=Kubelet via Hyperkube Wants=rpc-statd.service [Service] - EnvironmentFile=/etc/kubernetes/kubelet.env - Environment="RKT_RUN_ARGS=--uuid-file-save=/var/cache/kubelet-pod.uuid \ - --volume=resolv,kind=host,source=/etc/resolv.conf \ - --mount volume=resolv,target=/etc/resolv.conf \ - --volume var-lib-cni,kind=host,source=/var/lib/cni \ - --mount volume=var-lib-cni,target=/var/lib/cni \ - --volume var-lib-calico,kind=host,source=/var/lib/calico \ - --mount volume=var-lib-calico,target=/var/lib/calico \ - --volume opt-cni-bin,kind=host,source=/opt/cni/bin \ - --mount volume=opt-cni-bin,target=/opt/cni/bin \ - --volume var-log,kind=host,source=/var/log \ - --mount volume=var-log,target=/var/log \ - --hosts-entry=host \ - --insecure-options=image" ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests ExecStartPre=/bin/mkdir -p /opt/cni/bin - ExecStartPre=/bin/mkdir -p /var/lib/cni ExecStartPre=/bin/mkdir -p /var/lib/calico ExecStartPre=/bin/mkdir -p /var/lib/kubelet/volumeplugins ExecStartPre=/usr/bin/bash -c "grep 'certificate-authority-data' /etc/kubernetes/kubeconfig | awk '{print $2}' | base64 -d > /etc/kubernetes/ca.crt" ExecStartPre=-/usr/bin/rkt rm --uuid-file=/var/cache/kubelet-pod.uuid - ExecStart=/usr/lib/coreos/kubelet-wrapper \ + ExecStart=/usr/bin/rkt run \ + --uuid-file-save=/var/cache/kubelet-pod.uuid \ + --stage1-from-dir=stage1-fly.aci \ + --hosts-entry host \ + --insecure-options=image \ + --volume etc-kubernetes,kind=host,source=/etc/kubernetes,readOnly=true \ + --mount volume=etc-kubernetes,target=/etc/kubernetes \ + --volume etc-machine-id,kind=host,source=/etc/machine-id,readOnly=true \ + --mount volume=etc-machine-id,target=/etc/machine-id \ + --volume etc-os-release,kind=host,source=/usr/lib/os-release,readOnly=true \ + --mount volume=etc-os-release,target=/etc/os-release \ + --volume=etc-resolv,kind=host,source=/etc/resolv.conf,readOnly=true \ + --mount volume=etc-resolv,target=/etc/resolv.conf \ + --volume etc-ssl-certs,kind=host,source=/etc/ssl/certs,readOnly=true \ + --mount volume=etc-ssl-certs,target=/etc/ssl/certs \ + --volume lib-modules,kind=host,source=/lib/modules,readOnly=true \ + --mount volume=lib-modules,target=/lib/modules \ + --volume run,kind=host,source=/run \ + --mount volume=run,target=/run \ + --volume usr-share-certs,kind=host,source=/usr/share/ca-certificates,readOnly=true \ + --mount volume=usr-share-certs,target=/usr/share/ca-certificates \ + --volume var-lib-calico,kind=host,source=/var/lib/calico \ + --mount volume=var-lib-calico,target=/var/lib/calico \ + --volume var-lib-docker,kind=host,source=/var/lib/docker \ + --mount volume=var-lib-docker,target=/var/lib/docker \ + --volume var-lib-kubelet,kind=host,source=/var/lib/kubelet,recursive=true \ + --mount volume=var-lib-kubelet,target=/var/lib/kubelet \ + --volume var-log,kind=host,source=/var/log \ + --mount volume=var-log,target=/var/log \ + --volume opt-cni-bin,kind=host,source=/opt/cni/bin \ + --mount volume=opt-cni-bin,target=/opt/cni/bin \ + docker://k8s.gcr.io/hyperkube:v1.17.0 \ + --exec=/usr/local/bin/kubelet -- \ --anonymous-auth=false \ --authentication-token-webhook \ --authorization-mode=Webhook \ @@ -129,14 +146,6 @@ storage: contents: inline: | ${kubeconfig} - - path: /etc/kubernetes/kubelet.env - filesystem: root - mode: 0644 - contents: - inline: | - KUBELET_IMAGE_URL=docker://k8s.gcr.io/hyperkube - KUBELET_IMAGE_TAG=v1.17.0 - KUBELET_IMAGE_ARGS="--exec=/usr/local/bin/kubelet" - path: /opt/bootstrap/layout filesystem: root mode: 0544 diff --git a/google-cloud/container-linux/kubernetes/workers/cl/worker.yaml b/google-cloud/container-linux/kubernetes/workers/cl/worker.yaml index 4d4bae02e..f6e5bd5d1 100644 --- a/google-cloud/container-linux/kubernetes/workers/cl/worker.yaml +++ b/google-cloud/container-linux/kubernetes/workers/cl/worker.yaml @@ -25,29 +25,46 @@ systemd: Description=Kubelet via Hyperkube Wants=rpc-statd.service [Service] - EnvironmentFile=/etc/kubernetes/kubelet.env - Environment="RKT_RUN_ARGS=--uuid-file-save=/var/cache/kubelet-pod.uuid \ - --volume=resolv,kind=host,source=/etc/resolv.conf \ - --mount volume=resolv,target=/etc/resolv.conf \ - --volume var-lib-cni,kind=host,source=/var/lib/cni \ - --mount volume=var-lib-cni,target=/var/lib/cni \ - --volume var-lib-calico,kind=host,source=/var/lib/calico \ - --mount volume=var-lib-calico,target=/var/lib/calico \ - --volume opt-cni-bin,kind=host,source=/opt/cni/bin \ - --mount volume=opt-cni-bin,target=/opt/cni/bin \ - --volume var-log,kind=host,source=/var/log \ - --mount volume=var-log,target=/var/log \ - --hosts-entry=host \ - --insecure-options=image" ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests ExecStartPre=/bin/mkdir -p /opt/cni/bin - ExecStartPre=/bin/mkdir -p /var/lib/cni ExecStartPre=/bin/mkdir -p /var/lib/calico ExecStartPre=/bin/mkdir -p /var/lib/kubelet/volumeplugins ExecStartPre=/usr/bin/bash -c "grep 'certificate-authority-data' /etc/kubernetes/kubeconfig | awk '{print $2}' | base64 -d > /etc/kubernetes/ca.crt" ExecStartPre=-/usr/bin/rkt rm --uuid-file=/var/cache/kubelet-pod.uuid - ExecStart=/usr/lib/coreos/kubelet-wrapper \ + ExecStart=/usr/bin/rkt run \ + --uuid-file-save=/var/cache/kubelet-pod.uuid \ + --stage1-from-dir=stage1-fly.aci \ + --hosts-entry host \ + --insecure-options=image \ + --volume etc-kubernetes,kind=host,source=/etc/kubernetes,readOnly=true \ + --mount volume=etc-kubernetes,target=/etc/kubernetes \ + --volume etc-machine-id,kind=host,source=/etc/machine-id,readOnly=true \ + --mount volume=etc-machine-id,target=/etc/machine-id \ + --volume etc-os-release,kind=host,source=/usr/lib/os-release,readOnly=true \ + --mount volume=etc-os-release,target=/etc/os-release \ + --volume=etc-resolv,kind=host,source=/etc/resolv.conf,readOnly=true \ + --mount volume=etc-resolv,target=/etc/resolv.conf \ + --volume etc-ssl-certs,kind=host,source=/etc/ssl/certs,readOnly=true \ + --mount volume=etc-ssl-certs,target=/etc/ssl/certs \ + --volume lib-modules,kind=host,source=/lib/modules,readOnly=true \ + --mount volume=lib-modules,target=/lib/modules \ + --volume run,kind=host,source=/run \ + --mount volume=run,target=/run \ + --volume usr-share-certs,kind=host,source=/usr/share/ca-certificates,readOnly=true \ + --mount volume=usr-share-certs,target=/usr/share/ca-certificates \ + --volume var-lib-calico,kind=host,source=/var/lib/calico \ + --mount volume=var-lib-calico,target=/var/lib/calico \ + --volume var-lib-docker,kind=host,source=/var/lib/docker \ + --mount volume=var-lib-docker,target=/var/lib/docker \ + --volume var-lib-kubelet,kind=host,source=/var/lib/kubelet,recursive=true \ + --mount volume=var-lib-kubelet,target=/var/lib/kubelet \ + --volume var-log,kind=host,source=/var/log \ + --mount volume=var-log,target=/var/log \ + --volume opt-cni-bin,kind=host,source=/opt/cni/bin \ + --mount volume=opt-cni-bin,target=/opt/cni/bin \ + docker://k8s.gcr.io/hyperkube:v1.17.0 \ + --exec=/usr/local/bin/kubelet -- \ --anonymous-auth=false \ --authentication-token-webhook \ --authorization-mode=Webhook \ @@ -91,14 +108,6 @@ storage: contents: inline: | ${kubeconfig} - - path: /etc/kubernetes/kubelet.env - filesystem: root - mode: 0644 - contents: - inline: | - KUBELET_IMAGE_URL=docker://k8s.gcr.io/hyperkube - KUBELET_IMAGE_TAG=v1.17.0 - KUBELET_IMAGE_ARGS="--exec=/usr/local/bin/kubelet" - path: /etc/sysctl.d/max-user-watches.conf filesystem: root contents: From b2eb3e05d04330739ab14dd7631742edf51dcb83 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Sun, 29 Dec 2019 11:20:59 -0800 Subject: [PATCH 300/523] Disable Kubelet 127.0.0.1.10248 healthz endpoint * Kubelet runs a healthz server listening on 127.0.0.1:10248 by default. Its unused by Typhoon and can be disabled * https://kubernetes.io/docs/reference/command-line-tools-reference/kubelet/ --- CHANGES.md | 1 + aws/container-linux/kubernetes/cl/controller.yaml | 1 + aws/container-linux/kubernetes/workers/cl/worker.yaml | 1 + aws/fedora-coreos/kubernetes/fcc/controller.yaml | 1 + aws/fedora-coreos/kubernetes/workers/fcc/worker.yaml | 1 + azure/container-linux/kubernetes/cl/controller.yaml | 1 + azure/container-linux/kubernetes/workers/cl/worker.yaml | 1 + bare-metal/container-linux/kubernetes/cl/controller.yaml | 1 + bare-metal/container-linux/kubernetes/cl/worker.yaml | 1 + bare-metal/fedora-coreos/kubernetes/fcc/controller.yaml | 1 + bare-metal/fedora-coreos/kubernetes/fcc/worker.yaml | 1 + digital-ocean/container-linux/kubernetes/cl/controller.yaml | 1 + digital-ocean/container-linux/kubernetes/cl/worker.yaml | 1 + google-cloud/container-linux/kubernetes/cl/controller.yaml | 1 + google-cloud/container-linux/kubernetes/workers/cl/worker.yaml | 1 + 15 files changed, 15 insertions(+) diff --git a/CHANGES.md b/CHANGES.md index e37a02006..5f9f87922 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -7,6 +7,7 @@ Notable changes between versions. * Update CoreDNS from v1.6.5 to [v1.6.6](https://coredns.io/2019/12/11/coredns-1.6.6-release/) ([#602](https://github.com/poseidon/typhoon/pull/602)) * Update Calico from v3.10.2 to v3.11.1 ([#604](https://github.com/poseidon/typhoon/pull/604)) * Inline Kubelet service on Container Linux nodes ([#606](https://github.com/poseidon/typhoon/pull/606)) +* Disable unused Kubelet `127.0.0.1:10248` healthz listener ([#607](https://github.com/poseidon/typhoon/pull/607)) #### Addons diff --git a/aws/container-linux/kubernetes/cl/controller.yaml b/aws/container-linux/kubernetes/cl/controller.yaml index 95fb29bd0..b3e086e8a 100644 --- a/aws/container-linux/kubernetes/cl/controller.yaml +++ b/aws/container-linux/kubernetes/cl/controller.yaml @@ -100,6 +100,7 @@ systemd: --cluster_domain=${cluster_domain_suffix} \ --cni-conf-dir=/etc/kubernetes/cni/net.d \ --exit-on-lock-contention \ + --healthz-port=0 \ --kubeconfig=/etc/kubernetes/kubeconfig \ --lock-file=/var/run/lock/kubelet.lock \ --network-plugin=cni \ diff --git a/aws/container-linux/kubernetes/workers/cl/worker.yaml b/aws/container-linux/kubernetes/workers/cl/worker.yaml index ba27b8293..fee468486 100644 --- a/aws/container-linux/kubernetes/workers/cl/worker.yaml +++ b/aws/container-linux/kubernetes/workers/cl/worker.yaml @@ -75,6 +75,7 @@ systemd: --cluster_domain=${cluster_domain_suffix} \ --cni-conf-dir=/etc/kubernetes/cni/net.d \ --exit-on-lock-contention \ + --healthz-port=0 \ --kubeconfig=/etc/kubernetes/kubeconfig \ --lock-file=/var/run/lock/kubelet.lock \ --network-plugin=cni \ diff --git a/aws/fedora-coreos/kubernetes/fcc/controller.yaml b/aws/fedora-coreos/kubernetes/fcc/controller.yaml index f19ba8ddf..70c2a40fc 100644 --- a/aws/fedora-coreos/kubernetes/fcc/controller.yaml +++ b/aws/fedora-coreos/kubernetes/fcc/controller.yaml @@ -92,6 +92,7 @@ systemd: --cluster_domain=${cluster_domain_suffix} \ --cni-conf-dir=/etc/kubernetes/cni/net.d \ --exit-on-lock-contention \ + --healthz-port=0 \ --kubeconfig=/etc/kubernetes/kubeconfig \ --lock-file=/var/run/lock/kubelet.lock \ --network-plugin=cni \ diff --git a/aws/fedora-coreos/kubernetes/workers/fcc/worker.yaml b/aws/fedora-coreos/kubernetes/workers/fcc/worker.yaml index d987c8707..51b9cadf5 100644 --- a/aws/fedora-coreos/kubernetes/workers/fcc/worker.yaml +++ b/aws/fedora-coreos/kubernetes/workers/fcc/worker.yaml @@ -62,6 +62,7 @@ systemd: --cluster_domain=${cluster_domain_suffix} \ --cni-conf-dir=/etc/kubernetes/cni/net.d \ --exit-on-lock-contention \ + --healthz-port=0 \ --kubeconfig=/etc/kubernetes/kubeconfig \ --lock-file=/var/run/lock/kubelet.lock \ --network-plugin=cni \ diff --git a/azure/container-linux/kubernetes/cl/controller.yaml b/azure/container-linux/kubernetes/cl/controller.yaml index 236f65e9f..4ba0d3cff 100644 --- a/azure/container-linux/kubernetes/cl/controller.yaml +++ b/azure/container-linux/kubernetes/cl/controller.yaml @@ -98,6 +98,7 @@ systemd: --cluster_domain=${cluster_domain_suffix} \ --cni-conf-dir=/etc/kubernetes/cni/net.d \ --exit-on-lock-contention \ + --healthz-port=0 \ --kubeconfig=/etc/kubernetes/kubeconfig \ --lock-file=/var/run/lock/kubelet.lock \ --network-plugin=cni \ diff --git a/azure/container-linux/kubernetes/workers/cl/worker.yaml b/azure/container-linux/kubernetes/workers/cl/worker.yaml index ab0992927..ace3b1ba4 100644 --- a/azure/container-linux/kubernetes/workers/cl/worker.yaml +++ b/azure/container-linux/kubernetes/workers/cl/worker.yaml @@ -73,6 +73,7 @@ systemd: --cluster_domain=${cluster_domain_suffix} \ --cni-conf-dir=/etc/kubernetes/cni/net.d \ --exit-on-lock-contention \ + --healthz-port=0 \ --kubeconfig=/etc/kubernetes/kubeconfig \ --lock-file=/var/run/lock/kubelet.lock \ --network-plugin=cni \ diff --git a/bare-metal/container-linux/kubernetes/cl/controller.yaml b/bare-metal/container-linux/kubernetes/cl/controller.yaml index 42e7d3965..c5bbe9ab2 100644 --- a/bare-metal/container-linux/kubernetes/cl/controller.yaml +++ b/bare-metal/container-linux/kubernetes/cl/controller.yaml @@ -112,6 +112,7 @@ systemd: --cluster_domain=${cluster_domain_suffix} \ --cni-conf-dir=/etc/kubernetes/cni/net.d \ --exit-on-lock-contention \ + --healthz-port=0 \ --hostname-override=${domain_name} \ --kubeconfig=/etc/kubernetes/kubeconfig \ --lock-file=/var/run/lock/kubelet.lock \ diff --git a/bare-metal/container-linux/kubernetes/cl/worker.yaml b/bare-metal/container-linux/kubernetes/cl/worker.yaml index c2b78e7a3..64c598bc6 100644 --- a/bare-metal/container-linux/kubernetes/cl/worker.yaml +++ b/bare-metal/container-linux/kubernetes/cl/worker.yaml @@ -87,6 +87,7 @@ systemd: --cluster_domain=${cluster_domain_suffix} \ --cni-conf-dir=/etc/kubernetes/cni/net.d \ --exit-on-lock-contention \ + --healthz-port=0 \ --hostname-override=${domain_name} \ --kubeconfig=/etc/kubernetes/kubeconfig \ --lock-file=/var/run/lock/kubelet.lock \ diff --git a/bare-metal/fedora-coreos/kubernetes/fcc/controller.yaml b/bare-metal/fedora-coreos/kubernetes/fcc/controller.yaml index 196d364cc..3d3982112 100644 --- a/bare-metal/fedora-coreos/kubernetes/fcc/controller.yaml +++ b/bare-metal/fedora-coreos/kubernetes/fcc/controller.yaml @@ -93,6 +93,7 @@ systemd: --cluster_domain=${cluster_domain_suffix} \ --cni-conf-dir=/etc/kubernetes/cni/net.d \ --exit-on-lock-contention \ + --healthz-port=0 \ --hostname-override=${domain_name} \ --kubeconfig=/etc/kubernetes/kubeconfig \ --lock-file=/var/run/lock/kubelet.lock \ diff --git a/bare-metal/fedora-coreos/kubernetes/fcc/worker.yaml b/bare-metal/fedora-coreos/kubernetes/fcc/worker.yaml index 600be9c4b..fa7fcc26e 100644 --- a/bare-metal/fedora-coreos/kubernetes/fcc/worker.yaml +++ b/bare-metal/fedora-coreos/kubernetes/fcc/worker.yaml @@ -63,6 +63,7 @@ systemd: --cluster_domain=${cluster_domain_suffix} \ --cni-conf-dir=/etc/kubernetes/cni/net.d \ --exit-on-lock-contention \ + --healthz-port=0 \ --hostname-override=${domain_name} \ --kubeconfig=/etc/kubernetes/kubeconfig \ --lock-file=/var/run/lock/kubelet.lock \ diff --git a/digital-ocean/container-linux/kubernetes/cl/controller.yaml b/digital-ocean/container-linux/kubernetes/cl/controller.yaml index e654fb454..f318d04c6 100644 --- a/digital-ocean/container-linux/kubernetes/cl/controller.yaml +++ b/digital-ocean/container-linux/kubernetes/cl/controller.yaml @@ -109,6 +109,7 @@ systemd: --cluster_domain=${cluster_domain_suffix} \ --cni-conf-dir=/etc/kubernetes/cni/net.d \ --exit-on-lock-contention \ + --healthz-port=0 \ --hostname-override=$${COREOS_DIGITALOCEAN_IPV4_PRIVATE_0} \ --kubeconfig=/etc/kubernetes/kubeconfig \ --lock-file=/var/run/lock/kubelet.lock \ diff --git a/digital-ocean/container-linux/kubernetes/cl/worker.yaml b/digital-ocean/container-linux/kubernetes/cl/worker.yaml index 007014030..073c3339d 100644 --- a/digital-ocean/container-linux/kubernetes/cl/worker.yaml +++ b/digital-ocean/container-linux/kubernetes/cl/worker.yaml @@ -84,6 +84,7 @@ systemd: --cluster_domain=${cluster_domain_suffix} \ --cni-conf-dir=/etc/kubernetes/cni/net.d \ --exit-on-lock-contention \ + --healthz-port=0 \ --hostname-override=$${COREOS_DIGITALOCEAN_IPV4_PRIVATE_0} \ --kubeconfig=/etc/kubernetes/kubeconfig \ --lock-file=/var/run/lock/kubelet.lock \ diff --git a/google-cloud/container-linux/kubernetes/cl/controller.yaml b/google-cloud/container-linux/kubernetes/cl/controller.yaml index c48e90eb9..0dc53c042 100644 --- a/google-cloud/container-linux/kubernetes/cl/controller.yaml +++ b/google-cloud/container-linux/kubernetes/cl/controller.yaml @@ -98,6 +98,7 @@ systemd: --cluster_domain=${cluster_domain_suffix} \ --cni-conf-dir=/etc/kubernetes/cni/net.d \ --exit-on-lock-contention \ + --healthz-port=0 \ --kubeconfig=/etc/kubernetes/kubeconfig \ --lock-file=/var/run/lock/kubelet.lock \ --network-plugin=cni \ diff --git a/google-cloud/container-linux/kubernetes/workers/cl/worker.yaml b/google-cloud/container-linux/kubernetes/workers/cl/worker.yaml index f6e5bd5d1..179323129 100644 --- a/google-cloud/container-linux/kubernetes/workers/cl/worker.yaml +++ b/google-cloud/container-linux/kubernetes/workers/cl/worker.yaml @@ -73,6 +73,7 @@ systemd: --cluster_domain=${cluster_domain_suffix} \ --cni-conf-dir=/etc/kubernetes/cni/net.d \ --exit-on-lock-contention \ + --healthz-port=0 \ --kubeconfig=/etc/kubernetes/kubeconfig \ --lock-file=/var/run/lock/kubelet.lock \ --network-plugin=cni \ From 43e05b9131d8ab53f7487dc83885a2752247d882 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Sun, 29 Dec 2019 12:21:49 -0800 Subject: [PATCH 301/523] Enable kube-proxy metrics and allow Prometheus scrapes * Configure kube-proxy --metrics-bind-address=0.0.0.0 (default 127.0.0.1) to serve metrics on 0.0.0.0:10249 * Add firewall rules to allow Prometheus (resides on a worker) to scrape kube-proxy service endpoints on controllers or workers * Add a clusterIP: None service for kube-proxy endpoint discovery --- CHANGES.md | 3 + .../discovery/kube-controller-manager.yaml | 2 +- addons/prometheus/discovery/kube-proxy.yaml | 19 +++++++ .../prometheus/discovery/kube-scheduler.yaml | 2 +- aws/container-linux/kubernetes/bootstrap.tf | 2 +- aws/container-linux/kubernetes/security.tf | 56 +++++++++++++------ aws/fedora-coreos/kubernetes/bootstrap.tf | 2 +- aws/fedora-coreos/kubernetes/security.tf | 34 +++++++++-- azure/container-linux/kubernetes/bootstrap.tf | 2 +- azure/container-linux/kubernetes/security.tf | 34 ++++++++++- .../container-linux/kubernetes/bootstrap.tf | 2 +- .../fedora-coreos/kubernetes/bootstrap.tf | 2 +- .../container-linux/kubernetes/bootstrap.tf | 2 +- .../container-linux/kubernetes/network.tf | 8 +++ .../container-linux/kubernetes/bootstrap.tf | 2 +- .../container-linux/kubernetes/network.tf | 14 +++++ 16 files changed, 153 insertions(+), 33 deletions(-) create mode 100644 addons/prometheus/discovery/kube-proxy.yaml diff --git a/CHANGES.md b/CHANGES.md index 5f9f87922..a2c9b3be9 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -8,10 +8,13 @@ Notable changes between versions. * Update Calico from v3.10.2 to v3.11.1 ([#604](https://github.com/poseidon/typhoon/pull/604)) * Inline Kubelet service on Container Linux nodes ([#606](https://github.com/poseidon/typhoon/pull/606)) * Disable unused Kubelet `127.0.0.1:10248` healthz listener ([#607](https://github.com/poseidon/typhoon/pull/607)) +* Enable kube-proxy metrics and allow Prometheus scrapes + * Allow TCP/10249 traffic with worker node sources #### Addons * Update Prometheus from v2.14.0 to [v2.15.1](https://github.com/prometheus/prometheus/releases/tag/v2.15.1) + * Add discovery for kube-proxy service endpoints * Update kube-state-metrics from v1.8.0 to v1.9.0 * Update Grafana from v6.5.1 to v6.5.2 diff --git a/addons/prometheus/discovery/kube-controller-manager.yaml b/addons/prometheus/discovery/kube-controller-manager.yaml index 19faa6cfa..1dabf7240 100644 --- a/addons/prometheus/discovery/kube-controller-manager.yaml +++ b/addons/prometheus/discovery/kube-controller-manager.yaml @@ -1,3 +1,4 @@ +# Allow Prometheus to scrape service endpoints apiVersion: v1 kind: Service metadata: @@ -7,7 +8,6 @@ metadata: prometheus.io/scrape: 'true' spec: type: ClusterIP - # service is created to allow prometheus to scrape endpoints clusterIP: None selector: k8s-app: kube-controller-manager diff --git a/addons/prometheus/discovery/kube-proxy.yaml b/addons/prometheus/discovery/kube-proxy.yaml new file mode 100644 index 000000000..9c49bef20 --- /dev/null +++ b/addons/prometheus/discovery/kube-proxy.yaml @@ -0,0 +1,19 @@ +# Allow Prometheus to scrape service endpoints +apiVersion: v1 +kind: Service +metadata: + name: kube-proxy + namespace: kube-system + annotations: + prometheus.io/scrape: 'true' + prometheus.io/port: '10249' +spec: + type: ClusterIP + clusterIP: None + selector: + k8s-app: kube-proxy + ports: + - name: metrics + protocol: TCP + port: 10249 + targetPort: 10249 diff --git a/addons/prometheus/discovery/kube-scheduler.yaml b/addons/prometheus/discovery/kube-scheduler.yaml index 80fd19137..0032cf1b5 100644 --- a/addons/prometheus/discovery/kube-scheduler.yaml +++ b/addons/prometheus/discovery/kube-scheduler.yaml @@ -1,3 +1,4 @@ +# Allow Prometheus to scrape service endpoints apiVersion: v1 kind: Service metadata: @@ -7,7 +8,6 @@ metadata: prometheus.io/scrape: 'true' spec: type: ClusterIP - # service is created to allow prometheus to scrape endpoints clusterIP: None selector: k8s-app: kube-scheduler diff --git a/aws/container-linux/kubernetes/bootstrap.tf b/aws/container-linux/kubernetes/bootstrap.tf index ae850c6b2..5b9f7c982 100644 --- a/aws/container-linux/kubernetes/bootstrap.tf +++ b/aws/container-linux/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=c8c21deb7682c2a83a1b86ff6ed88f3e5a20262d" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=ac4b7af57012d477cd53bd74ce632ac581e807e1" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/aws/container-linux/kubernetes/security.tf b/aws/container-linux/kubernetes/security.tf index 706f80f87..60727af85 100644 --- a/aws/container-linux/kubernetes/security.tf +++ b/aws/container-linux/kubernetes/security.tf @@ -33,6 +33,28 @@ resource "aws_security_group_rule" "controller-etcd" { self = true } +# Allow Prometheus to scrape etcd metrics +resource "aws_security_group_rule" "controller-etcd-metrics" { + security_group_id = aws_security_group.controller.id + + type = "ingress" + protocol = "tcp" + from_port = 2381 + to_port = 2381 + source_security_group_id = aws_security_group.worker.id +} + +# Allow Prometheus to scrape kube-proxy +resource "aws_security_group_rule" "kube-proxy-metrics" { + security_group_id = aws_security_group.controller.id + + type = "ingress" + protocol = "tcp" + from_port = 10249 + to_port = 10249 + source_security_group_id = aws_security_group.worker.id +} + # Allow Prometheus to scrape kube-scheduler resource "aws_security_group_rule" "controller-scheduler-metrics" { security_group_id = aws_security_group.controller.id @@ -55,17 +77,6 @@ resource "aws_security_group_rule" "controller-manager-metrics" { source_security_group_id = aws_security_group.worker.id } -# Allow Prometheus to scrape etcd metrics -resource "aws_security_group_rule" "controller-etcd-metrics" { - security_group_id = aws_security_group.controller.id - - type = "ingress" - protocol = "tcp" - from_port = 2381 - to_port = 2381 - source_security_group_id = aws_security_group.worker.id -} - resource "aws_security_group_rule" "controller-vxlan" { count = var.networking == "flannel" ? 1 : 0 @@ -281,14 +292,15 @@ resource "aws_security_group_rule" "worker-node-exporter" { self = true } -resource "aws_security_group_rule" "ingress-health" { +# Allow Prometheus to scrape kube-proxy +resource "aws_security_group_rule" "worker-kube-proxy" { security_group_id = aws_security_group.worker.id - type = "ingress" - protocol = "tcp" - from_port = 10254 - to_port = 10254 - cidr_blocks = ["0.0.0.0/0"] + type = "ingress" + protocol = "tcp" + from_port = 10249 + to_port = 10249 + self = true } # Allow apiserver to access kubelets for exec, log, port-forward @@ -313,6 +325,16 @@ resource "aws_security_group_rule" "worker-kubelet-self" { self = true } +resource "aws_security_group_rule" "ingress-health" { + security_group_id = aws_security_group.worker.id + + type = "ingress" + protocol = "tcp" + from_port = 10254 + to_port = 10254 + cidr_blocks = ["0.0.0.0/0"] +} + resource "aws_security_group_rule" "worker-bgp" { security_group_id = aws_security_group.worker.id diff --git a/aws/fedora-coreos/kubernetes/bootstrap.tf b/aws/fedora-coreos/kubernetes/bootstrap.tf index 7d60bb946..b2015fa34 100644 --- a/aws/fedora-coreos/kubernetes/bootstrap.tf +++ b/aws/fedora-coreos/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=c8c21deb7682c2a83a1b86ff6ed88f3e5a20262d" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=ac4b7af57012d477cd53bd74ce632ac581e807e1" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/aws/fedora-coreos/kubernetes/security.tf b/aws/fedora-coreos/kubernetes/security.tf index ddc4e8e47..60727af85 100644 --- a/aws/fedora-coreos/kubernetes/security.tf +++ b/aws/fedora-coreos/kubernetes/security.tf @@ -44,6 +44,17 @@ resource "aws_security_group_rule" "controller-etcd-metrics" { source_security_group_id = aws_security_group.worker.id } +# Allow Prometheus to scrape kube-proxy +resource "aws_security_group_rule" "kube-proxy-metrics" { + security_group_id = aws_security_group.controller.id + + type = "ingress" + protocol = "tcp" + from_port = 10249 + to_port = 10249 + source_security_group_id = aws_security_group.worker.id +} + # Allow Prometheus to scrape kube-scheduler resource "aws_security_group_rule" "controller-scheduler-metrics" { security_group_id = aws_security_group.controller.id @@ -281,14 +292,15 @@ resource "aws_security_group_rule" "worker-node-exporter" { self = true } -resource "aws_security_group_rule" "ingress-health" { +# Allow Prometheus to scrape kube-proxy +resource "aws_security_group_rule" "worker-kube-proxy" { security_group_id = aws_security_group.worker.id - type = "ingress" - protocol = "tcp" - from_port = 10254 - to_port = 10254 - cidr_blocks = ["0.0.0.0/0"] + type = "ingress" + protocol = "tcp" + from_port = 10249 + to_port = 10249 + self = true } # Allow apiserver to access kubelets for exec, log, port-forward @@ -313,6 +325,16 @@ resource "aws_security_group_rule" "worker-kubelet-self" { self = true } +resource "aws_security_group_rule" "ingress-health" { + security_group_id = aws_security_group.worker.id + + type = "ingress" + protocol = "tcp" + from_port = 10254 + to_port = 10254 + cidr_blocks = ["0.0.0.0/0"] +} + resource "aws_security_group_rule" "worker-bgp" { security_group_id = aws_security_group.worker.id diff --git a/azure/container-linux/kubernetes/bootstrap.tf b/azure/container-linux/kubernetes/bootstrap.tf index 218b050e4..f7607daa3 100644 --- a/azure/container-linux/kubernetes/bootstrap.tf +++ b/azure/container-linux/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=c8c21deb7682c2a83a1b86ff6ed88f3e5a20262d" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=ac4b7af57012d477cd53bd74ce632ac581e807e1" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/azure/container-linux/kubernetes/security.tf b/azure/container-linux/kubernetes/security.tf index c2e97307e..feb6fef54 100644 --- a/azure/container-linux/kubernetes/security.tf +++ b/azure/container-linux/kubernetes/security.tf @@ -53,13 +53,29 @@ resource "azurerm_network_security_rule" "controller-etcd-metrics" { destination_address_prefix = azurerm_subnet.controller.address_prefix } +# Allow Prometheus to scrape kube-proxy metrics +resource "azurerm_network_security_rule" "controller-kube-proxy" { + resource_group_name = azurerm_resource_group.cluster.name + + name = "allow-kube-proxy-metrics" + network_security_group_name = azurerm_network_security_group.controller.name + priority = "2011" + access = "Allow" + direction = "Inbound" + protocol = "Tcp" + source_port_range = "*" + destination_port_range = "10249" + source_address_prefix = azurerm_subnet.worker.address_prefix + destination_address_prefix = azurerm_subnet.controller.address_prefix +} + # Allow Prometheus to scrape kube-scheduler and kube-controller-manager metrics resource "azurerm_network_security_rule" "controller-kube-metrics" { resource_group_name = azurerm_resource_group.cluster.name name = "allow-kube-metrics" network_security_group_name = azurerm_network_security_group.controller.name - priority = "2011" + priority = "2012" access = "Allow" direction = "Inbound" protocol = "Tcp" @@ -251,6 +267,22 @@ resource "azurerm_network_security_rule" "worker-node-exporter" { destination_address_prefix = azurerm_subnet.worker.address_prefix } +# Allow Prometheus to scrape kube-proxy +resource "azurerm_network_security_rule" "worker-kube-proxy" { + resource_group_name = azurerm_resource_group.cluster.name + + name = "allow-kube-proxy" + network_security_group_name = azurerm_network_security_group.worker.name + priority = "2024" + access = "Allow" + direction = "Inbound" + protocol = "Tcp" + source_port_range = "*" + destination_port_range = "10249" + source_address_prefix = azurerm_subnet.worker.address_prefix + destination_address_prefix = azurerm_subnet.worker.address_prefix +} + # Allow apiserver to access kubelet's for exec, log, port-forward resource "azurerm_network_security_rule" "worker-kubelet" { resource_group_name = azurerm_resource_group.cluster.name diff --git a/bare-metal/container-linux/kubernetes/bootstrap.tf b/bare-metal/container-linux/kubernetes/bootstrap.tf index 7def8550c..5e8c940b4 100644 --- a/bare-metal/container-linux/kubernetes/bootstrap.tf +++ b/bare-metal/container-linux/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=c8c21deb7682c2a83a1b86ff6ed88f3e5a20262d" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=ac4b7af57012d477cd53bd74ce632ac581e807e1" cluster_name = var.cluster_name api_servers = [var.k8s_domain_name] diff --git a/bare-metal/fedora-coreos/kubernetes/bootstrap.tf b/bare-metal/fedora-coreos/kubernetes/bootstrap.tf index bc2364627..8bbb756db 100644 --- a/bare-metal/fedora-coreos/kubernetes/bootstrap.tf +++ b/bare-metal/fedora-coreos/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=c8c21deb7682c2a83a1b86ff6ed88f3e5a20262d" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=ac4b7af57012d477cd53bd74ce632ac581e807e1" cluster_name = var.cluster_name api_servers = [var.k8s_domain_name] diff --git a/digital-ocean/container-linux/kubernetes/bootstrap.tf b/digital-ocean/container-linux/kubernetes/bootstrap.tf index ebb1f80a1..7c195c7d2 100644 --- a/digital-ocean/container-linux/kubernetes/bootstrap.tf +++ b/digital-ocean/container-linux/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=c8c21deb7682c2a83a1b86ff6ed88f3e5a20262d" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=ac4b7af57012d477cd53bd74ce632ac581e807e1" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/digital-ocean/container-linux/kubernetes/network.tf b/digital-ocean/container-linux/kubernetes/network.tf index 6a8d40957..bc5434852 100644 --- a/digital-ocean/container-linux/kubernetes/network.tf +++ b/digital-ocean/container-linux/kubernetes/network.tf @@ -16,12 +16,20 @@ resource "digitalocean_firewall" "rules" { source_tags = [digitalocean_tag.controllers.name, digitalocean_tag.workers.name] } + # Allow Prometheus to scrape node-exporter inbound_rule { protocol = "tcp" port_range = "9100" source_tags = [digitalocean_tag.workers.name] } + # Allow Prometheus to scrape kube-proxy + inbound_rule { + protocol = "tcp" + port_range = "10249" + source_tags = [digitalocean_tag.workers.name] + } + inbound_rule { protocol = "tcp" port_range = "10250" diff --git a/google-cloud/container-linux/kubernetes/bootstrap.tf b/google-cloud/container-linux/kubernetes/bootstrap.tf index 9eeeb9795..f6b385f7a 100644 --- a/google-cloud/container-linux/kubernetes/bootstrap.tf +++ b/google-cloud/container-linux/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=c8c21deb7682c2a83a1b86ff6ed88f3e5a20262d" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=ac4b7af57012d477cd53bd74ce632ac581e807e1" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/google-cloud/container-linux/kubernetes/network.tf b/google-cloud/container-linux/kubernetes/network.tf index 8d0bbefc8..bd7067d7b 100644 --- a/google-cloud/container-linux/kubernetes/network.tf +++ b/google-cloud/container-linux/kubernetes/network.tf @@ -126,6 +126,20 @@ resource "google_compute_firewall" "internal-node-exporter" { target_tags = ["${var.cluster_name}-controller", "${var.cluster_name}-worker"] } +# Allow Prometheus to scrape kube-proxy metrics +resource "google_compute_firewall" "internal-kube-proxy" { + name = "${var.cluster_name}-internal-kube-proxy" + network = google_compute_network.network.name + + allow { + protocol = "tcp" + ports = [10249] + } + + source_tags = ["${var.cluster_name}-worker"] + target_tags = ["${var.cluster_name}-controller", "${var.cluster_name}-worker"] +} + # Allow apiserver to access kubelets for exec, log, port-forward resource "google_compute_firewall" "internal-kubelet" { name = "${var.cluster_name}-internal-kubelet" From bb586b60da062208c06c5f035a8962d07e42f1c2 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Mon, 6 Jan 2020 21:14:03 -0800 Subject: [PATCH 302/523] Reduce Prometheus addon's node-exporter tolerations * Change node-exporter DaemonSet tolerations from tolerating all possible NoSchedule taints to tolerating the master taint and the not ready taint (we'd like metrics regardless) * Users who add custom node taints must add their custom taints to the addon node-exporter DaemonSet. As an addon, its expected users copy and manipulate manifests out-of-band in their own systems --- CHANGES.md | 1 + addons/prometheus/exporters/node-exporter/daemonset.yaml | 4 +++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index a2c9b3be9..04b70ba8d 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -16,6 +16,7 @@ Notable changes between versions. * Update Prometheus from v2.14.0 to [v2.15.1](https://github.com/prometheus/prometheus/releases/tag/v2.15.1) * Add discovery for kube-proxy service endpoints * Update kube-state-metrics from v1.8.0 to v1.9.0 +* Reduce node-exporter DaemonSet tolerations ([#614](https://github.com/poseidon/typhoon/pull/614)) * Update Grafana from v6.5.1 to v6.5.2 ## v1.17.0 diff --git a/addons/prometheus/exporters/node-exporter/daemonset.yaml b/addons/prometheus/exporters/node-exporter/daemonset.yaml index a8e584217..7ef88739e 100644 --- a/addons/prometheus/exporters/node-exporter/daemonset.yaml +++ b/addons/prometheus/exporters/node-exporter/daemonset.yaml @@ -57,7 +57,9 @@ spec: mountPath: /host/root readOnly: true tolerations: - - effect: NoSchedule + - key: node-role.kubernetes.io/master + operator: Exists + - key: node.kubernetes.io/not-ready operator: Exists volumes: - name: proc From 0223b31e1a40ffc6e0608c7c6412ba1513de286b Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Mon, 6 Jan 2020 21:38:20 -0800 Subject: [PATCH 303/523] Ensure /etc/kubernetes exists following Kubelet inlining * Inlining the Kubelet service removed the need for the kubelet.env file declared in Ignition. However, on some platforms, this removed the guarantee that /etc/kubernetes exists. Bare-Metal and DigitalOcean distribute the kubelet kubeconfig through Terraform file provisioner (scp) and place it in (now missing) /etc/kubernetes * https://github.com/poseidon/typhoon/pull/606 * Fix bare-metal and DigitalOcean Ignition to ensure the desired directory exists following first boot from disk * Cloud platforms with worker pools distribute the kubeconfig through Ignition user data (no impact or need) --- bare-metal/container-linux/kubernetes/cl/controller.yaml | 3 +++ bare-metal/container-linux/kubernetes/cl/worker.yaml | 3 +++ digital-ocean/container-linux/kubernetes/cl/controller.yaml | 3 +++ digital-ocean/container-linux/kubernetes/cl/worker.yaml | 3 +++ 4 files changed, 12 insertions(+) diff --git a/bare-metal/container-linux/kubernetes/cl/controller.yaml b/bare-metal/container-linux/kubernetes/cl/controller.yaml index c5bbe9ab2..4289b3cfd 100644 --- a/bare-metal/container-linux/kubernetes/cl/controller.yaml +++ b/bare-metal/container-linux/kubernetes/cl/controller.yaml @@ -155,6 +155,9 @@ systemd: [Install] WantedBy=multi-user.target storage: + directories: + - path: /etc/kubernetes + filesystem: root files: - path: /etc/hostname filesystem: root diff --git a/bare-metal/container-linux/kubernetes/cl/worker.yaml b/bare-metal/container-linux/kubernetes/cl/worker.yaml index 64c598bc6..5c8ab2b03 100644 --- a/bare-metal/container-linux/kubernetes/cl/worker.yaml +++ b/bare-metal/container-linux/kubernetes/cl/worker.yaml @@ -103,6 +103,9 @@ systemd: WantedBy=multi-user.target storage: + directories: + - path: /etc/kubernetes + filesystem: root files: - path: /etc/hostname filesystem: root diff --git a/digital-ocean/container-linux/kubernetes/cl/controller.yaml b/digital-ocean/container-linux/kubernetes/cl/controller.yaml index f318d04c6..4cb2a20ad 100644 --- a/digital-ocean/container-linux/kubernetes/cl/controller.yaml +++ b/digital-ocean/container-linux/kubernetes/cl/controller.yaml @@ -152,6 +152,9 @@ systemd: [Install] WantedBy=multi-user.target storage: + directories: + - path: /etc/kubernetes + filesystem: root files: - path: /opt/bootstrap/layout filesystem: root diff --git a/digital-ocean/container-linux/kubernetes/cl/worker.yaml b/digital-ocean/container-linux/kubernetes/cl/worker.yaml index 073c3339d..8427ec473 100644 --- a/digital-ocean/container-linux/kubernetes/cl/worker.yaml +++ b/digital-ocean/container-linux/kubernetes/cl/worker.yaml @@ -111,6 +111,9 @@ systemd: [Install] WantedBy=multi-user.target storage: + directories: + - path: /etc/kubernetes + filesystem: root files: - path: /etc/sysctl.d/max-user-watches.conf filesystem: root From 73588cfad3e3a6fa6791dedf442e0c5b850f8279 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Mon, 6 Jan 2020 22:08:02 -0800 Subject: [PATCH 304/523] Update Prometheus from v2.15.1 to v2.15.2 * https://github.com/prometheus/prometheus/releases/tag/v2.15.2 --- CHANGES.md | 2 +- addons/prometheus/deployment.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 04b70ba8d..ab03b9305 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -13,7 +13,7 @@ Notable changes between versions. #### Addons -* Update Prometheus from v2.14.0 to [v2.15.1](https://github.com/prometheus/prometheus/releases/tag/v2.15.1) +* Update Prometheus from v2.14.0 to [v2.15.2](https://github.com/prometheus/prometheus/releases/tag/v2.15.2) * Add discovery for kube-proxy service endpoints * Update kube-state-metrics from v1.8.0 to v1.9.0 * Reduce node-exporter DaemonSet tolerations ([#614](https://github.com/poseidon/typhoon/pull/614)) diff --git a/addons/prometheus/deployment.yaml b/addons/prometheus/deployment.yaml index 254fa0c88..873aeaa75 100644 --- a/addons/prometheus/deployment.yaml +++ b/addons/prometheus/deployment.yaml @@ -20,7 +20,7 @@ spec: serviceAccountName: prometheus containers: - name: prometheus - image: quay.io/prometheus/prometheus:v2.15.1 + image: quay.io/prometheus/prometheus:v2.15.2 args: - --web.listen-address=0.0.0.0:9090 - --config.file=/etc/prometheus/prometheus.yaml From b1f521fc4ace0f851b3704cba96135a2696c7148 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Sat, 11 Jan 2020 13:58:23 -0800 Subject: [PATCH 305/523] Allow terraform-provider-google v3.x plugin versions * Typhoon Google Cloud is compatible with `terraform-provider-google` v3.x releases * No v3.x specific features are used, so v2.19+ provider versions are still allowed, to ease migrations --- CHANGES.md | 5 +++++ docs/cl/google-cloud.md | 2 +- google-cloud/container-linux/kubernetes/versions.tf | 2 +- 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index ab03b9305..62cd6e30f 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -11,6 +11,11 @@ Notable changes between versions. * Enable kube-proxy metrics and allow Prometheus scrapes * Allow TCP/10249 traffic with worker node sources +#### Google + +* Allow `terraform-provider-google` v3.0+ ([#617](https://github.com/poseidon/typhoon/pull/617)) + * Only enforce `v2.19+` to ease migration, as no v3.x features are used + #### Addons * Update Prometheus from v2.14.0 to [v2.15.2](https://github.com/prometheus/prometheus/releases/tag/v2.15.2) diff --git a/docs/cl/google-cloud.md b/docs/cl/google-cloud.md index 8171e35fd..392765a11 100644 --- a/docs/cl/google-cloud.md +++ b/docs/cl/google-cloud.md @@ -49,7 +49,7 @@ Configure the Google Cloud provider to use your service account key, project-id, ```tf provider "google" { - version = "2.20.0" + version = "3.4.0" project = "project-id" region = "us-central1" credentials = file("~/.config/google-cloud/terraform.json") diff --git a/google-cloud/container-linux/kubernetes/versions.tf b/google-cloud/container-linux/kubernetes/versions.tf index f28854f96..26ea74cac 100644 --- a/google-cloud/container-linux/kubernetes/versions.tf +++ b/google-cloud/container-linux/kubernetes/versions.tf @@ -3,7 +3,7 @@ terraform { required_version = "~> 0.12.6" required_providers { - google = "~> 2.19" + google = ">= 2.19, < 4.0" ct = "~> 0.3" template = "~> 2.1" null = "~> 2.1" From 0e2fc89f78f38d87feeda20f30832d250511a8db Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Sat, 11 Jan 2020 14:15:55 -0800 Subject: [PATCH 306/523] Update kube-state-metrics from v1.9.0 to v1.9.1 * https://github.com/kubernetes/kube-state-metrics/releases/tag/v1.9.1 --- CHANGES.md | 2 +- addons/prometheus/exporters/kube-state-metrics/deployment.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 62cd6e30f..8c8eb7aae 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -20,7 +20,7 @@ Notable changes between versions. * Update Prometheus from v2.14.0 to [v2.15.2](https://github.com/prometheus/prometheus/releases/tag/v2.15.2) * Add discovery for kube-proxy service endpoints -* Update kube-state-metrics from v1.8.0 to v1.9.0 +* Update kube-state-metrics from v1.8.0 to v1.9.1 * Reduce node-exporter DaemonSet tolerations ([#614](https://github.com/poseidon/typhoon/pull/614)) * Update Grafana from v6.5.1 to v6.5.2 diff --git a/addons/prometheus/exporters/kube-state-metrics/deployment.yaml b/addons/prometheus/exporters/kube-state-metrics/deployment.yaml index 8cbd151d5..da7e0129c 100644 --- a/addons/prometheus/exporters/kube-state-metrics/deployment.yaml +++ b/addons/prometheus/exporters/kube-state-metrics/deployment.yaml @@ -24,7 +24,7 @@ spec: serviceAccountName: kube-state-metrics containers: - name: kube-state-metrics - image: quay.io/coreos/kube-state-metrics:v1.9.0 + image: quay.io/coreos/kube-state-metrics:v1.9.1 ports: - name: metrics containerPort: 8080 From ce0569e03b8cd5ae1fb5d989b909d6cbe0611095 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Sat, 11 Jan 2020 15:15:39 -0800 Subject: [PATCH 307/523] Remove unneeded Kubelet /var/run mount on Fedora CoreOS * /var/run symlinks to /run (already mounted) --- aws/fedora-coreos/kubernetes/fcc/controller.yaml | 1 - aws/fedora-coreos/kubernetes/workers/fcc/worker.yaml | 1 - bare-metal/fedora-coreos/kubernetes/fcc/controller.yaml | 1 - bare-metal/fedora-coreos/kubernetes/fcc/worker.yaml | 1 - 4 files changed, 4 deletions(-) diff --git a/aws/fedora-coreos/kubernetes/fcc/controller.yaml b/aws/fedora-coreos/kubernetes/fcc/controller.yaml index 70c2a40fc..8796b0487 100644 --- a/aws/fedora-coreos/kubernetes/fcc/controller.yaml +++ b/aws/fedora-coreos/kubernetes/fcc/controller.yaml @@ -77,7 +77,6 @@ systemd: --volume /var/lib/docker:/var/lib/docker \ --volume /var/lib/kubelet:/var/lib/kubelet:rshared,z \ --volume /var/log:/var/log \ - --volume /var/run:/var/run \ --volume /var/run/lock:/var/run/lock:z \ --volume /opt/cni/bin:/opt/cni/bin:z \ k8s.gcr.io/hyperkube:v1.17.0 kubelet \ diff --git a/aws/fedora-coreos/kubernetes/workers/fcc/worker.yaml b/aws/fedora-coreos/kubernetes/workers/fcc/worker.yaml index 51b9cadf5..c631d5e57 100644 --- a/aws/fedora-coreos/kubernetes/workers/fcc/worker.yaml +++ b/aws/fedora-coreos/kubernetes/workers/fcc/worker.yaml @@ -47,7 +47,6 @@ systemd: --volume /var/lib/docker:/var/lib/docker \ --volume /var/lib/kubelet:/var/lib/kubelet:rshared,z \ --volume /var/log:/var/log \ - --volume /var/run:/var/run \ --volume /var/run/lock:/var/run/lock:z \ --volume /opt/cni/bin:/opt/cni/bin:z \ k8s.gcr.io/hyperkube:v1.17.0 kubelet \ diff --git a/bare-metal/fedora-coreos/kubernetes/fcc/controller.yaml b/bare-metal/fedora-coreos/kubernetes/fcc/controller.yaml index 3d3982112..b65c184d7 100644 --- a/bare-metal/fedora-coreos/kubernetes/fcc/controller.yaml +++ b/bare-metal/fedora-coreos/kubernetes/fcc/controller.yaml @@ -76,7 +76,6 @@ systemd: --volume /var/lib/docker:/var/lib/docker \ --volume /var/lib/kubelet:/var/lib/kubelet:rshared,z \ --volume /var/log:/var/log \ - --volume /var/run:/var/run \ --volume /var/run/lock:/var/run/lock:z \ --volume /opt/cni/bin:/opt/cni/bin:z \ --volume /etc/iscsi:/etc/iscsi \ diff --git a/bare-metal/fedora-coreos/kubernetes/fcc/worker.yaml b/bare-metal/fedora-coreos/kubernetes/fcc/worker.yaml index fa7fcc26e..0862401f3 100644 --- a/bare-metal/fedora-coreos/kubernetes/fcc/worker.yaml +++ b/bare-metal/fedora-coreos/kubernetes/fcc/worker.yaml @@ -46,7 +46,6 @@ systemd: --volume /var/lib/docker:/var/lib/docker \ --volume /var/lib/kubelet:/var/lib/kubelet:rshared,z \ --volume /var/log:/var/log \ - --volume /var/run:/var/run \ --volume /var/run/lock:/var/run/lock:z \ --volume /opt/cni/bin:/opt/cni/bin:z \ --volume /etc/iscsi:/etc/iscsi \ From 073fcb7067937e0166723d8530763e7a457fcd87 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Sun, 12 Jan 2020 14:16:00 -0800 Subject: [PATCH 308/523] Fix bare-metal instruction for watching install to disk * Original instructions were to watch install to disk by SSH'ing via port 2222 following Typhoon v1.10.1. Restore that message, since the version number in the instruction was incorrectly bumped on each release --- docs/cl/bare-metal.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/cl/bare-metal.md b/docs/cl/bare-metal.md index b265cb688..248ba80bb 100644 --- a/docs/cl/bare-metal.md +++ b/docs/cl/bare-metal.md @@ -264,9 +264,9 @@ Apply complete! Resources: 55 added, 0 changed, 0 destroyed. To watch the install to disk (until machines reboot from disk), SSH to port 2222. ``` -# before v1.17.0 +# before v1.10.1 $ ssh debug@node1.example.com -# after v1.17.0 +# after v1.10.1 $ ssh -p 2222 core@node1.example.com ``` From ac786a2efc371a61d04ea040ded4be8205fe073c Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Sat, 11 Jan 2020 16:51:08 -0800 Subject: [PATCH 309/523] Update AWS Fedora CoreOS AMI filter for fedora-coreos-31 * Select the most recent fedora-coreos-31 AMI on AWS, instead of the most recent fedora-coreos-30 AMI (Nov 27, 2019) * Evaluated with fedora-coreos-31.20200108.2.0-hvm --- aws/fedora-coreos/kubernetes/ami.tf | 4 ++-- aws/fedora-coreos/kubernetes/workers/ami.tf | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/aws/fedora-coreos/kubernetes/ami.tf b/aws/fedora-coreos/kubernetes/ami.tf index 7e0990509..c6da3b368 100644 --- a/aws/fedora-coreos/kubernetes/ami.tf +++ b/aws/fedora-coreos/kubernetes/ami.tf @@ -15,9 +15,9 @@ data "aws_ami" "fedora-coreos" { filter { name = "name" - values = ["fedora-coreos-30.*.*-hvm"] + values = ["fedora-coreos-31.*.*.*-hvm"] } # try to filter out dev images (AWS filters can't) - name_regex = "^fedora-coreos-30.[0-9]*.[0-9]*-hvm*" + name_regex = "^fedora-coreos-31.[0-9]*.[0-9]*.[0-9]*-hvm*" } diff --git a/aws/fedora-coreos/kubernetes/workers/ami.tf b/aws/fedora-coreos/kubernetes/workers/ami.tf index 7e0990509..c6da3b368 100644 --- a/aws/fedora-coreos/kubernetes/workers/ami.tf +++ b/aws/fedora-coreos/kubernetes/workers/ami.tf @@ -15,9 +15,9 @@ data "aws_ami" "fedora-coreos" { filter { name = "name" - values = ["fedora-coreos-30.*.*-hvm"] + values = ["fedora-coreos-31.*.*.*-hvm"] } # try to filter out dev images (AWS filters can't) - name_regex = "^fedora-coreos-30.[0-9]*.[0-9]*-hvm*" + name_regex = "^fedora-coreos-31.[0-9]*.[0-9]*.[0-9]*-hvm*" } From b642e3b41b9f1606d5fbbcd6908908d602d3404b Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Tue, 14 Jan 2020 20:21:36 -0800 Subject: [PATCH 310/523] Update Kubernetes from v1.17.0 to v1.17.1 * https://github.com/kubernetes/kubernetes/blob/master/CHANGELOG-1.17.md#v1171 --- README.md | 10 +++++----- aws/container-linux/kubernetes/README.md | 2 +- aws/container-linux/kubernetes/bootstrap.tf | 2 +- aws/container-linux/kubernetes/cl/controller.yaml | 4 ++-- .../kubernetes/workers/cl/worker.yaml | 4 ++-- aws/fedora-coreos/kubernetes/README.md | 2 +- aws/fedora-coreos/kubernetes/bootstrap.tf | 2 +- aws/fedora-coreos/kubernetes/fcc/controller.yaml | 4 ++-- .../kubernetes/workers/fcc/worker.yaml | 2 +- azure/container-linux/kubernetes/README.md | 2 +- azure/container-linux/kubernetes/bootstrap.tf | 2 +- .../container-linux/kubernetes/cl/controller.yaml | 4 ++-- .../kubernetes/workers/cl/worker.yaml | 4 ++-- bare-metal/container-linux/kubernetes/README.md | 2 +- bare-metal/container-linux/kubernetes/bootstrap.tf | 2 +- .../container-linux/kubernetes/cl/controller.yaml | 4 ++-- .../container-linux/kubernetes/cl/worker.yaml | 2 +- bare-metal/fedora-coreos/kubernetes/README.md | 2 +- bare-metal/fedora-coreos/kubernetes/bootstrap.tf | 2 +- .../fedora-coreos/kubernetes/fcc/controller.yaml | 4 ++-- .../fedora-coreos/kubernetes/fcc/worker.yaml | 2 +- digital-ocean/container-linux/kubernetes/README.md | 2 +- .../container-linux/kubernetes/bootstrap.tf | 2 +- .../container-linux/kubernetes/cl/controller.yaml | 4 ++-- .../container-linux/kubernetes/cl/worker.yaml | 4 ++-- docs/advanced/worker-pools.md | 14 +++++++------- docs/cl/aws.md | 10 +++++----- docs/cl/azure.md | 10 +++++----- docs/cl/bare-metal.md | 10 +++++----- docs/cl/digital-ocean.md | 10 +++++----- docs/cl/google-cloud.md | 10 +++++----- docs/fedora-coreos/aws.md | 10 +++++----- docs/fedora-coreos/bare-metal.md | 10 +++++----- docs/index.md | 10 +++++----- docs/topics/maintenance.md | 10 +++++----- google-cloud/container-linux/kubernetes/README.md | 2 +- .../container-linux/kubernetes/bootstrap.tf | 2 +- .../container-linux/kubernetes/cl/controller.yaml | 4 ++-- .../kubernetes/workers/cl/worker.yaml | 4 ++-- 39 files changed, 96 insertions(+), 96 deletions(-) diff --git a/README.md b/README.md index c1164f7dd..bec2c0c32 100644 --- a/README.md +++ b/README.md @@ -11,7 +11,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features -* Kubernetes v1.17.0 (upstream) +* Kubernetes v1.17.1 (upstream) * Single or multi-master, [Calico](https://www.projectcalico.org/) or [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/) * Advanced features like [worker pools](https://typhoon.psdn.io/advanced/worker-pools/), [preemptible](https://typhoon.psdn.io/cl/google-cloud/#preemption) workers, and [snippets](https://typhoon.psdn.io/advanced/customization/#container-linux) customization @@ -48,7 +48,7 @@ Define a Kubernetes cluster by using the Terraform module for your chosen platfo ```tf module "yavin" { - source = "git::https://github.com/poseidon/typhoon//google-cloud/container-linux/kubernetes?ref=v1.17.0" + source = "git::https://github.com/poseidon/typhoon//google-cloud/container-linux/kubernetes?ref=v1.17.1" # Google Cloud cluster_name = "yavin" @@ -87,9 +87,9 @@ In 4-8 minutes (varies by platform), the cluster will be ready. This Google Clou $ export KUBECONFIG=/home/user/.kube/configs/yavin-config $ kubectl get nodes NAME ROLES STATUS AGE VERSION -yavin-controller-0.c.example-com.internal Ready 6m v1.17.0 -yavin-worker-jrbf.c.example-com.internal Ready 5m v1.17.0 -yavin-worker-mzdm.c.example-com.internal Ready 5m v1.17.0 +yavin-controller-0.c.example-com.internal Ready 6m v1.17.1 +yavin-worker-jrbf.c.example-com.internal Ready 5m v1.17.1 +yavin-worker-mzdm.c.example-com.internal Ready 5m v1.17.1 ``` List the pods. diff --git a/aws/container-linux/kubernetes/README.md b/aws/container-linux/kubernetes/README.md index c89a8b404..82f414880 100644 --- a/aws/container-linux/kubernetes/README.md +++ b/aws/container-linux/kubernetes/README.md @@ -11,7 +11,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features -* Kubernetes v1.17.0 (upstream) +* Kubernetes v1.17.1 (upstream) * Single or multi-master, [Calico](https://www.projectcalico.org/) or [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/) * Advanced features like [worker pools](https://typhoon.psdn.io/advanced/worker-pools/), [spot](https://typhoon.psdn.io/cl/aws/#spot) workers, and [snippets](https://typhoon.psdn.io/advanced/customization/#container-linux) customization diff --git a/aws/container-linux/kubernetes/bootstrap.tf b/aws/container-linux/kubernetes/bootstrap.tf index 5b9f7c982..e03875062 100644 --- a/aws/container-linux/kubernetes/bootstrap.tf +++ b/aws/container-linux/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=ac4b7af57012d477cd53bd74ce632ac581e807e1" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=5ce4fc69537a8bfd1cdd08499e03f246d9f73ca6" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/aws/container-linux/kubernetes/cl/controller.yaml b/aws/container-linux/kubernetes/cl/controller.yaml index b3e086e8a..9a35a6c90 100644 --- a/aws/container-linux/kubernetes/cl/controller.yaml +++ b/aws/container-linux/kubernetes/cl/controller.yaml @@ -89,7 +89,7 @@ systemd: --mount volume=var-log,target=/var/log \ --volume opt-cni-bin,kind=host,source=/opt/cni/bin \ --mount volume=opt-cni-bin,target=/opt/cni/bin \ - docker://k8s.gcr.io/hyperkube:v1.17.0 \ + docker://k8s.gcr.io/hyperkube:v1.17.1 \ --exec=/usr/local/bin/kubelet -- \ --anonymous-auth=false \ --authentication-token-webhook \ @@ -134,7 +134,7 @@ systemd: --volume script,kind=host,source=/opt/bootstrap/apply \ --mount volume=script,target=/apply \ --insecure-options=image \ - docker://k8s.gcr.io/hyperkube:v1.17.0 \ + docker://k8s.gcr.io/hyperkube:v1.17.1 \ --net=host \ --dns=host \ --exec=/apply diff --git a/aws/container-linux/kubernetes/workers/cl/worker.yaml b/aws/container-linux/kubernetes/workers/cl/worker.yaml index fee468486..44e8a73c8 100644 --- a/aws/container-linux/kubernetes/workers/cl/worker.yaml +++ b/aws/container-linux/kubernetes/workers/cl/worker.yaml @@ -64,7 +64,7 @@ systemd: --mount volume=var-log,target=/var/log \ --volume opt-cni-bin,kind=host,source=/opt/cni/bin \ --mount volume=opt-cni-bin,target=/opt/cni/bin \ - docker://k8s.gcr.io/hyperkube:v1.17.0 \ + docker://k8s.gcr.io/hyperkube:v1.17.1 \ --exec=/usr/local/bin/kubelet -- \ --anonymous-auth=false \ --authentication-token-webhook \ @@ -128,7 +128,7 @@ storage: --volume config,kind=host,source=/etc/kubernetes \ --mount volume=config,target=/etc/kubernetes \ --insecure-options=image \ - docker://k8s.gcr.io/hyperkube:v1.17.0 \ + docker://k8s.gcr.io/hyperkube:v1.17.1 \ --net=host \ --dns=host \ -- \ diff --git a/aws/fedora-coreos/kubernetes/README.md b/aws/fedora-coreos/kubernetes/README.md index b389491c8..cd15fa628 100644 --- a/aws/fedora-coreos/kubernetes/README.md +++ b/aws/fedora-coreos/kubernetes/README.md @@ -11,7 +11,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features -* Kubernetes v1.17.0 (upstream) +* Kubernetes v1.17.1 (upstream) * Single or multi-master, [Calico](https://www.projectcalico.org/) or [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/) * Advanced features like [worker pools](https://typhoon.psdn.io/advanced/worker-pools/), [spot](https://typhoon.psdn.io/cl/aws/#spot) workers, and [snippets](https://typhoon.psdn.io/advanced/customization/#container-linux) customization diff --git a/aws/fedora-coreos/kubernetes/bootstrap.tf b/aws/fedora-coreos/kubernetes/bootstrap.tf index b2015fa34..f474206b4 100644 --- a/aws/fedora-coreos/kubernetes/bootstrap.tf +++ b/aws/fedora-coreos/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=ac4b7af57012d477cd53bd74ce632ac581e807e1" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=5ce4fc69537a8bfd1cdd08499e03f246d9f73ca6" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/aws/fedora-coreos/kubernetes/fcc/controller.yaml b/aws/fedora-coreos/kubernetes/fcc/controller.yaml index 8796b0487..f43123177 100644 --- a/aws/fedora-coreos/kubernetes/fcc/controller.yaml +++ b/aws/fedora-coreos/kubernetes/fcc/controller.yaml @@ -79,7 +79,7 @@ systemd: --volume /var/log:/var/log \ --volume /var/run/lock:/var/run/lock:z \ --volume /opt/cni/bin:/opt/cni/bin:z \ - k8s.gcr.io/hyperkube:v1.17.0 kubelet \ + k8s.gcr.io/hyperkube:v1.17.1 kubelet \ --anonymous-auth=false \ --authentication-token-webhook \ --authorization-mode=Webhook \ @@ -123,7 +123,7 @@ systemd: --volume /opt/bootstrap/assets:/assets:ro,Z \ --volume /opt/bootstrap/apply:/apply:ro,Z \ --entrypoint=/apply \ - k8s.gcr.io/hyperkube:v1.17.0 + k8s.gcr.io/hyperkube:v1.17.1 ExecStartPost=/bin/touch /opt/bootstrap/bootstrap.done ExecStartPost=-/usr/bin/podman stop bootstrap storage: diff --git a/aws/fedora-coreos/kubernetes/workers/fcc/worker.yaml b/aws/fedora-coreos/kubernetes/workers/fcc/worker.yaml index c631d5e57..6ba667aa3 100644 --- a/aws/fedora-coreos/kubernetes/workers/fcc/worker.yaml +++ b/aws/fedora-coreos/kubernetes/workers/fcc/worker.yaml @@ -49,7 +49,7 @@ systemd: --volume /var/log:/var/log \ --volume /var/run/lock:/var/run/lock:z \ --volume /opt/cni/bin:/opt/cni/bin:z \ - k8s.gcr.io/hyperkube:v1.17.0 kubelet \ + k8s.gcr.io/hyperkube:v1.17.1 kubelet \ --anonymous-auth=false \ --authentication-token-webhook \ --authorization-mode=Webhook \ diff --git a/azure/container-linux/kubernetes/README.md b/azure/container-linux/kubernetes/README.md index 3adc3b2a4..2df87a060 100644 --- a/azure/container-linux/kubernetes/README.md +++ b/azure/container-linux/kubernetes/README.md @@ -11,7 +11,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features -* Kubernetes v1.17.0 (upstream) +* Kubernetes v1.17.1 (upstream) * Single or multi-master, [Calico](https://www.projectcalico.org/) or [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/) * Advanced features like [worker pools](https://typhoon.psdn.io/advanced/worker-pools/), [low-priority](https://typhoon.psdn.io/cl/azure/#low-priority) workers, and [snippets](https://typhoon.psdn.io/advanced/customization/#container-linux) customization diff --git a/azure/container-linux/kubernetes/bootstrap.tf b/azure/container-linux/kubernetes/bootstrap.tf index f7607daa3..57bbe1ee3 100644 --- a/azure/container-linux/kubernetes/bootstrap.tf +++ b/azure/container-linux/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=ac4b7af57012d477cd53bd74ce632ac581e807e1" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=5ce4fc69537a8bfd1cdd08499e03f246d9f73ca6" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/azure/container-linux/kubernetes/cl/controller.yaml b/azure/container-linux/kubernetes/cl/controller.yaml index 4ba0d3cff..dbfea0c6c 100644 --- a/azure/container-linux/kubernetes/cl/controller.yaml +++ b/azure/container-linux/kubernetes/cl/controller.yaml @@ -88,7 +88,7 @@ systemd: --mount volume=var-log,target=/var/log \ --volume opt-cni-bin,kind=host,source=/opt/cni/bin \ --mount volume=opt-cni-bin,target=/opt/cni/bin \ - docker://k8s.gcr.io/hyperkube:v1.17.0 \ + docker://k8s.gcr.io/hyperkube:v1.17.1 \ --exec=/usr/local/bin/kubelet -- \ --anonymous-auth=false \ --authentication-token-webhook \ @@ -132,7 +132,7 @@ systemd: --volume script,kind=host,source=/opt/bootstrap/apply \ --mount volume=script,target=/apply \ --insecure-options=image \ - docker://k8s.gcr.io/hyperkube:v1.17.0 \ + docker://k8s.gcr.io/hyperkube:v1.17.1 \ --net=host \ --dns=host \ --exec=/apply diff --git a/azure/container-linux/kubernetes/workers/cl/worker.yaml b/azure/container-linux/kubernetes/workers/cl/worker.yaml index ace3b1ba4..69f4aacab 100644 --- a/azure/container-linux/kubernetes/workers/cl/worker.yaml +++ b/azure/container-linux/kubernetes/workers/cl/worker.yaml @@ -63,7 +63,7 @@ systemd: --mount volume=var-log,target=/var/log \ --volume opt-cni-bin,kind=host,source=/opt/cni/bin \ --mount volume=opt-cni-bin,target=/opt/cni/bin \ - docker://k8s.gcr.io/hyperkube:v1.17.0 \ + docker://k8s.gcr.io/hyperkube:v1.17.1 \ --exec=/usr/local/bin/kubelet -- \ --anonymous-auth=false \ --authentication-token-webhook \ @@ -126,7 +126,7 @@ storage: --volume config,kind=host,source=/etc/kubernetes \ --mount volume=config,target=/etc/kubernetes \ --insecure-options=image \ - docker://k8s.gcr.io/hyperkube:v1.17.0 \ + docker://k8s.gcr.io/hyperkube:v1.17.1 \ --net=host \ --dns=host \ -- \ diff --git a/bare-metal/container-linux/kubernetes/README.md b/bare-metal/container-linux/kubernetes/README.md index 821632bd6..3ec30a8db 100644 --- a/bare-metal/container-linux/kubernetes/README.md +++ b/bare-metal/container-linux/kubernetes/README.md @@ -11,7 +11,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features -* Kubernetes v1.17.0 (upstream) +* Kubernetes v1.17.1 (upstream) * Single or multi-master, [Calico](https://www.projectcalico.org/) or [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/) * Advanced features like [snippets](https://typhoon.psdn.io/advanced/customization/#container-linux) customization diff --git a/bare-metal/container-linux/kubernetes/bootstrap.tf b/bare-metal/container-linux/kubernetes/bootstrap.tf index 5e8c940b4..70a4db0cd 100644 --- a/bare-metal/container-linux/kubernetes/bootstrap.tf +++ b/bare-metal/container-linux/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=ac4b7af57012d477cd53bd74ce632ac581e807e1" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=5ce4fc69537a8bfd1cdd08499e03f246d9f73ca6" cluster_name = var.cluster_name api_servers = [var.k8s_domain_name] diff --git a/bare-metal/container-linux/kubernetes/cl/controller.yaml b/bare-metal/container-linux/kubernetes/cl/controller.yaml index 4289b3cfd..707891d65 100644 --- a/bare-metal/container-linux/kubernetes/cl/controller.yaml +++ b/bare-metal/container-linux/kubernetes/cl/controller.yaml @@ -101,7 +101,7 @@ systemd: --mount volume=etc-iscsi,target=/etc/iscsi \ --volume usr-sbin-iscsiadm,kind=host,source=/usr/sbin/iscsiadm \ --mount volume=usr-sbin-iscsiadm,target=/sbin/iscsiadm \ - docker://k8s.gcr.io/hyperkube:v1.17.0 \ + docker://k8s.gcr.io/hyperkube:v1.17.1 \ --exec=/usr/local/bin/kubelet -- \ --anonymous-auth=false \ --authentication-token-webhook \ @@ -147,7 +147,7 @@ systemd: --volume script,kind=host,source=/opt/bootstrap/apply \ --mount volume=script,target=/apply \ --insecure-options=image \ - docker://k8s.gcr.io/hyperkube:v1.17.0 \ + docker://k8s.gcr.io/hyperkube:v1.17.1 \ --net=host \ --dns=host \ --exec=/apply diff --git a/bare-metal/container-linux/kubernetes/cl/worker.yaml b/bare-metal/container-linux/kubernetes/cl/worker.yaml index 5c8ab2b03..194468121 100644 --- a/bare-metal/container-linux/kubernetes/cl/worker.yaml +++ b/bare-metal/container-linux/kubernetes/cl/worker.yaml @@ -76,7 +76,7 @@ systemd: --mount volume=etc-iscsi,target=/etc/iscsi \ --volume usr-sbin-iscsiadm,kind=host,source=/usr/sbin/iscsiadm \ --mount volume=usr-sbin-iscsiadm,target=/sbin/iscsiadm \ - docker://k8s.gcr.io/hyperkube:v1.17.0 \ + docker://k8s.gcr.io/hyperkube:v1.17.1 \ --exec=/usr/local/bin/kubelet -- \ --anonymous-auth=false \ --authentication-token-webhook \ diff --git a/bare-metal/fedora-coreos/kubernetes/README.md b/bare-metal/fedora-coreos/kubernetes/README.md index 5e1fecda3..d406ba12e 100644 --- a/bare-metal/fedora-coreos/kubernetes/README.md +++ b/bare-metal/fedora-coreos/kubernetes/README.md @@ -11,7 +11,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features -* Kubernetes v1.17.0 (upstream) +* Kubernetes v1.17.1 (upstream) * Single or multi-master, [Calico](https://www.projectcalico.org/) or [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/) * Advanced features like [snippets](https://typhoon.psdn.io/advanced/customization/#container-linux) customization diff --git a/bare-metal/fedora-coreos/kubernetes/bootstrap.tf b/bare-metal/fedora-coreos/kubernetes/bootstrap.tf index 8bbb756db..7062552ea 100644 --- a/bare-metal/fedora-coreos/kubernetes/bootstrap.tf +++ b/bare-metal/fedora-coreos/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=ac4b7af57012d477cd53bd74ce632ac581e807e1" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=5ce4fc69537a8bfd1cdd08499e03f246d9f73ca6" cluster_name = var.cluster_name api_servers = [var.k8s_domain_name] diff --git a/bare-metal/fedora-coreos/kubernetes/fcc/controller.yaml b/bare-metal/fedora-coreos/kubernetes/fcc/controller.yaml index b65c184d7..f428915f3 100644 --- a/bare-metal/fedora-coreos/kubernetes/fcc/controller.yaml +++ b/bare-metal/fedora-coreos/kubernetes/fcc/controller.yaml @@ -80,7 +80,7 @@ systemd: --volume /opt/cni/bin:/opt/cni/bin:z \ --volume /etc/iscsi:/etc/iscsi \ --volume /sbin/iscsiadm:/sbin/iscsiadm \ - k8s.gcr.io/hyperkube:v1.17.0 kubelet \ + k8s.gcr.io/hyperkube:v1.17.1 kubelet \ --anonymous-auth=false \ --authentication-token-webhook \ --authorization-mode=Webhook \ @@ -134,7 +134,7 @@ systemd: --volume /opt/bootstrap/assets:/assets:ro,Z \ --volume /opt/bootstrap/apply:/apply:ro,Z \ --entrypoint=/apply \ - k8s.gcr.io/hyperkube:v1.17.0 + k8s.gcr.io/hyperkube:v1.17.1 ExecStartPost=/bin/touch /opt/bootstrap/bootstrap.done ExecStartPost=-/usr/bin/podman stop bootstrap storage: diff --git a/bare-metal/fedora-coreos/kubernetes/fcc/worker.yaml b/bare-metal/fedora-coreos/kubernetes/fcc/worker.yaml index 0862401f3..cc39c11cb 100644 --- a/bare-metal/fedora-coreos/kubernetes/fcc/worker.yaml +++ b/bare-metal/fedora-coreos/kubernetes/fcc/worker.yaml @@ -50,7 +50,7 @@ systemd: --volume /opt/cni/bin:/opt/cni/bin:z \ --volume /etc/iscsi:/etc/iscsi \ --volume /sbin/iscsiadm:/sbin/iscsiadm \ - k8s.gcr.io/hyperkube:v1.17.0 kubelet \ + k8s.gcr.io/hyperkube:v1.17.1 kubelet \ --anonymous-auth=false \ --authentication-token-webhook \ --authorization-mode=Webhook \ diff --git a/digital-ocean/container-linux/kubernetes/README.md b/digital-ocean/container-linux/kubernetes/README.md index 1c248f766..d915164a6 100644 --- a/digital-ocean/container-linux/kubernetes/README.md +++ b/digital-ocean/container-linux/kubernetes/README.md @@ -11,7 +11,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features -* Kubernetes v1.17.0 (upstream) +* Kubernetes v1.17.1 (upstream) * Single or multi-master, [Calico](https://www.projectcalico.org/) or [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/) * Advanced features like [snippets](https://typhoon.psdn.io/advanced/customization/#container-linux) customization diff --git a/digital-ocean/container-linux/kubernetes/bootstrap.tf b/digital-ocean/container-linux/kubernetes/bootstrap.tf index 7c195c7d2..4e7e8d275 100644 --- a/digital-ocean/container-linux/kubernetes/bootstrap.tf +++ b/digital-ocean/container-linux/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=ac4b7af57012d477cd53bd74ce632ac581e807e1" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=5ce4fc69537a8bfd1cdd08499e03f246d9f73ca6" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/digital-ocean/container-linux/kubernetes/cl/controller.yaml b/digital-ocean/container-linux/kubernetes/cl/controller.yaml index 4cb2a20ad..de8de96da 100644 --- a/digital-ocean/container-linux/kubernetes/cl/controller.yaml +++ b/digital-ocean/container-linux/kubernetes/cl/controller.yaml @@ -99,7 +99,7 @@ systemd: --mount volume=var-log,target=/var/log \ --volume opt-cni-bin,kind=host,source=/opt/cni/bin \ --mount volume=opt-cni-bin,target=/opt/cni/bin \ - docker://k8s.gcr.io/hyperkube:v1.17.0 \ + docker://k8s.gcr.io/hyperkube:v1.17.1 \ --exec=/usr/local/bin/kubelet -- \ --anonymous-auth=false \ --authentication-token-webhook \ @@ -144,7 +144,7 @@ systemd: --volume script,kind=host,source=/opt/bootstrap/apply \ --mount volume=script,target=/apply \ --insecure-options=image \ - docker://k8s.gcr.io/hyperkube:v1.17.0 \ + docker://k8s.gcr.io/hyperkube:v1.17.1 \ --net=host \ --dns=host \ --exec=/apply diff --git a/digital-ocean/container-linux/kubernetes/cl/worker.yaml b/digital-ocean/container-linux/kubernetes/cl/worker.yaml index 8427ec473..3f8f485e0 100644 --- a/digital-ocean/container-linux/kubernetes/cl/worker.yaml +++ b/digital-ocean/container-linux/kubernetes/cl/worker.yaml @@ -74,7 +74,7 @@ systemd: --mount volume=var-log,target=/var/log \ --volume opt-cni-bin,kind=host,source=/opt/cni/bin \ --mount volume=opt-cni-bin,target=/opt/cni/bin \ - docker://k8s.gcr.io/hyperkube:v1.17.0 \ + docker://k8s.gcr.io/hyperkube:v1.17.1 \ --exec=/usr/local/bin/kubelet -- \ --anonymous-auth=false \ --authentication-token-webhook \ @@ -132,7 +132,7 @@ storage: --volume config,kind=host,source=/etc/kubernetes \ --mount volume=config,target=/etc/kubernetes \ --insecure-options=image \ - docker://k8s.gcr.io/hyperkube:v1.17.0 \ + docker://k8s.gcr.io/hyperkube:v1.17.1 \ --net=host \ --dns=host \ -- \ diff --git a/docs/advanced/worker-pools.md b/docs/advanced/worker-pools.md index bcb2d6381..9f9757e08 100644 --- a/docs/advanced/worker-pools.md +++ b/docs/advanced/worker-pools.md @@ -79,7 +79,7 @@ Create a cluster following the Azure [tutorial](../cl/azure.md#cluster). Define ```tf module "ramius-worker-pool" { - source = "git::https://github.com/poseidon/typhoon//azure/container-linux/kubernetes/workers?ref=v1.17.0" + source = "git::https://github.com/poseidon/typhoon//azure/container-linux/kubernetes/workers?ref=v1.17.1" # Azure region = module.ramius.region @@ -145,7 +145,7 @@ Create a cluster following the Google Cloud [tutorial](../cl/google-cloud.md#clu ```tf module "yavin-worker-pool" { - source = "git::https://github.com/poseidon/typhoon//google-cloud/container-linux/kubernetes/workers?ref=v1.17.0" + source = "git::https://github.com/poseidon/typhoon//google-cloud/container-linux/kubernetes/workers?ref=v1.17.1" # Google Cloud region = "europe-west2" @@ -176,11 +176,11 @@ Verify a managed instance group of workers joins the cluster within a few minute ``` $ kubectl get nodes NAME STATUS AGE VERSION -yavin-controller-0.c.example-com.internal Ready 6m v1.17.0 -yavin-worker-jrbf.c.example-com.internal Ready 5m v1.17.0 -yavin-worker-mzdm.c.example-com.internal Ready 5m v1.17.0 -yavin-16x-worker-jrbf.c.example-com.internal Ready 3m v1.17.0 -yavin-16x-worker-mzdm.c.example-com.internal Ready 3m v1.17.0 +yavin-controller-0.c.example-com.internal Ready 6m v1.17.1 +yavin-worker-jrbf.c.example-com.internal Ready 5m v1.17.1 +yavin-worker-mzdm.c.example-com.internal Ready 5m v1.17.1 +yavin-16x-worker-jrbf.c.example-com.internal Ready 3m v1.17.1 +yavin-16x-worker-mzdm.c.example-com.internal Ready 3m v1.17.1 ``` ### Variables diff --git a/docs/cl/aws.md b/docs/cl/aws.md index 58e1b0849..5701ab622 100644 --- a/docs/cl/aws.md +++ b/docs/cl/aws.md @@ -1,6 +1,6 @@ # AWS -In this tutorial, we'll create a Kubernetes v1.17.0 cluster on AWS with Container Linux. +In this tutorial, we'll create a Kubernetes v1.17.1 cluster on AWS with Container Linux. We'll declare a Kubernetes cluster using the Typhoon Terraform module. Then apply the changes to create a VPC, gateway, subnets, security groups, controller instances, worker auto-scaling group, network load balancer, and TLS assets. @@ -70,7 +70,7 @@ Define a Kubernetes cluster using the module `aws/container-linux/kubernetes`. ```tf module "tempest" { - source = "git::https://github.com/poseidon/typhoon//aws/container-linux/kubernetes?ref=v1.17.0" + source = "git::https://github.com/poseidon/typhoon//aws/container-linux/kubernetes?ref=v1.17.1" # AWS cluster_name = "tempest" @@ -143,9 +143,9 @@ List nodes in the cluster. $ export KUBECONFIG=/home/user/.kube/configs/tempest-config $ kubectl get nodes NAME STATUS ROLES AGE VERSION -ip-10-0-3-155 Ready 10m v1.17.0 -ip-10-0-26-65 Ready 10m v1.17.0 -ip-10-0-41-21 Ready 10m v1.17.0 +ip-10-0-3-155 Ready 10m v1.17.1 +ip-10-0-26-65 Ready 10m v1.17.1 +ip-10-0-41-21 Ready 10m v1.17.1 ``` List the pods. diff --git a/docs/cl/azure.md b/docs/cl/azure.md index b3212e5b8..a5211f6ea 100644 --- a/docs/cl/azure.md +++ b/docs/cl/azure.md @@ -3,7 +3,7 @@ !!! danger Typhoon for Azure is alpha. For production, use AWS, Google Cloud, or bare-metal. As Azure matures, check [errata](https://github.com/poseidon/typhoon/wiki/Errata) for known shortcomings. -In this tutorial, we'll create a Kubernetes v1.17.0 cluster on Azure with Container Linux. +In this tutorial, we'll create a Kubernetes v1.17.1 cluster on Azure with Container Linux. We'll declare a Kubernetes cluster using the Typhoon Terraform module. Then apply the changes to create a resource group, virtual network, subnets, security groups, controller availability set, worker scale set, load balancer, and TLS assets. @@ -66,7 +66,7 @@ Define a Kubernetes cluster using the module `azure/container-linux/kubernetes`. ```tf module "ramius" { - source = "git::https://github.com/poseidon/typhoon//azure/container-linux/kubernetes?ref=v1.17.0" + source = "git::https://github.com/poseidon/typhoon//azure/container-linux/kubernetes?ref=v1.17.1" # Azure cluster_name = "ramius" @@ -140,9 +140,9 @@ List nodes in the cluster. $ export KUBECONFIG=/home/user/.kube/configs/ramius-config $ kubectl get nodes NAME STATUS ROLES AGE VERSION -ramius-controller-0 Ready 24m v1.17.0 -ramius-worker-000001 Ready 25m v1.17.0 -ramius-worker-000002 Ready 24m v1.17.0 +ramius-controller-0 Ready 24m v1.17.1 +ramius-worker-000001 Ready 25m v1.17.1 +ramius-worker-000002 Ready 24m v1.17.1 ``` List the pods. diff --git a/docs/cl/bare-metal.md b/docs/cl/bare-metal.md index 248ba80bb..3e6dddac0 100644 --- a/docs/cl/bare-metal.md +++ b/docs/cl/bare-metal.md @@ -1,6 +1,6 @@ # Bare-Metal -In this tutorial, we'll network boot and provision a Kubernetes v1.17.0 cluster on bare-metal with Container Linux. +In this tutorial, we'll network boot and provision a Kubernetes v1.17.1 cluster on bare-metal with Container Linux. First, we'll deploy a [Matchbox](https://github.com/poseidon/matchbox) service and setup a network boot environment. Then, we'll declare a Kubernetes cluster using the Typhoon Terraform module and power on machines. On PXE boot, machines will install Container Linux to disk, reboot into the disk install, and provision themselves as Kubernetes controllers or workers via Ignition. @@ -160,7 +160,7 @@ Define a Kubernetes cluster using the module `bare-metal/container-linux/kuberne ```tf module "mercury" { - source = "git::https://github.com/poseidon/typhoon//bare-metal/container-linux/kubernetes?ref=v1.17.0" + source = "git::https://github.com/poseidon/typhoon//bare-metal/container-linux/kubernetes?ref=v1.17.1" # bare-metal cluster_name = "mercury" @@ -299,9 +299,9 @@ List nodes in the cluster. $ export KUBECONFIG=/home/user/.kube/configs/mercury-config $ kubectl get nodes NAME STATUS ROLES AGE VERSION -node1.example.com Ready 10m v1.17.0 -node2.example.com Ready 10m v1.17.0 -node3.example.com Ready 10m v1.17.0 +node1.example.com Ready 10m v1.17.1 +node2.example.com Ready 10m v1.17.1 +node3.example.com Ready 10m v1.17.1 ``` List the pods. diff --git a/docs/cl/digital-ocean.md b/docs/cl/digital-ocean.md index 49cde1b34..988c3ce32 100644 --- a/docs/cl/digital-ocean.md +++ b/docs/cl/digital-ocean.md @@ -1,6 +1,6 @@ # Digital Ocean -In this tutorial, we'll create a Kubernetes v1.17.0 cluster on DigitalOcean with Container Linux. +In this tutorial, we'll create a Kubernetes v1.17.1 cluster on DigitalOcean with Container Linux. We'll declare a Kubernetes cluster using the Typhoon Terraform module. Then apply the changes to create controller droplets, worker droplets, DNS records, tags, and TLS assets. @@ -65,7 +65,7 @@ Define a Kubernetes cluster using the module `digital-ocean/container-linux/kube ```tf module "nemo" { - source = "git::https://github.com/poseidon/typhoon//digital-ocean/container-linux/kubernetes?ref=v1.17.0" + source = "git::https://github.com/poseidon/typhoon//digital-ocean/container-linux/kubernetes?ref=v1.17.1" # Digital Ocean cluster_name = "nemo" @@ -138,9 +138,9 @@ List nodes in the cluster. $ export KUBECONFIG=/home/user/.kube/configs/nemo-config $ kubectl get nodes NAME STATUS ROLES AGE VERSION -10.132.110.130 Ready 10m v1.17.0 -10.132.115.81 Ready 10m v1.17.0 -10.132.124.107 Ready 10m v1.17.0 +10.132.110.130 Ready 10m v1.17.1 +10.132.115.81 Ready 10m v1.17.1 +10.132.124.107 Ready 10m v1.17.1 ``` List the pods. diff --git a/docs/cl/google-cloud.md b/docs/cl/google-cloud.md index 392765a11..cdc8accb1 100644 --- a/docs/cl/google-cloud.md +++ b/docs/cl/google-cloud.md @@ -1,6 +1,6 @@ # Google Cloud -In this tutorial, we'll create a Kubernetes v1.17.0 cluster on Google Compute Engine with Container Linux. +In this tutorial, we'll create a Kubernetes v1.17.1 cluster on Google Compute Engine with Container Linux. We'll declare a Kubernetes cluster using the Typhoon Terraform module. Then apply the changes to create a network, firewall rules, health checks, controller instances, worker managed instance group, load balancers, and TLS assets. @@ -71,7 +71,7 @@ Define a Kubernetes cluster using the module `google-cloud/container-linux/kuber ```tf module "yavin" { - source = "git::https://github.com/poseidon/typhoon//google-cloud/container-linux/kubernetes?ref=v1.17.0" + source = "git::https://github.com/poseidon/typhoon//google-cloud/container-linux/kubernetes?ref=v1.17.1" # Google Cloud cluster_name = "yavin" @@ -145,9 +145,9 @@ List nodes in the cluster. $ export KUBECONFIG=/home/user/.kube/configs/yavin-config $ kubectl get nodes NAME ROLES STATUS AGE VERSION -yavin-controller-0.c.example-com.internal Ready 6m v1.17.0 -yavin-worker-jrbf.c.example-com.internal Ready 5m v1.17.0 -yavin-worker-mzdm.c.example-com.internal Ready 5m v1.17.0 +yavin-controller-0.c.example-com.internal Ready 6m v1.17.1 +yavin-worker-jrbf.c.example-com.internal Ready 5m v1.17.1 +yavin-worker-mzdm.c.example-com.internal Ready 5m v1.17.1 ``` List the pods. diff --git a/docs/fedora-coreos/aws.md b/docs/fedora-coreos/aws.md index e44bbf5d4..f36424ebf 100644 --- a/docs/fedora-coreos/aws.md +++ b/docs/fedora-coreos/aws.md @@ -3,7 +3,7 @@ !!! danger Typhoon for Fedora CoreOS is an early preview! Fedora CoreOS itself is a preview! Expect bugs and design shifts. Please help both projects solve problems. Report Fedora CoreOS bugs to [Fedora](https://github.com/coreos/fedora-coreos-tracker/issues). Report Typhoon issues to Typhoon. -In this tutorial, we'll create a Kubernetes v1.17.0 cluster on AWS with Fedora CoreOS. +In this tutorial, we'll create a Kubernetes v1.17.1 cluster on AWS with Fedora CoreOS. We'll declare a Kubernetes cluster using the Typhoon Terraform module. Then apply the changes to create a VPC, gateway, subnets, security groups, controller instances, worker auto-scaling group, network load balancer, and TLS assets. @@ -73,7 +73,7 @@ Define a Kubernetes cluster using the module `aws/fedora-coreos/kubernetes`. ```tf module "tempest" { - source = "git::https://github.com/poseidon/typhoon//aws/fedora-coreos/kubernetes?ref=v1.17.0" + source = "git::https://github.com/poseidon/typhoon//aws/fedora-coreos/kubernetes?ref=v1.17.1" # AWS cluster_name = "tempest" @@ -146,9 +146,9 @@ List nodes in the cluster. $ export KUBECONFIG=/home/user/.kube/configs/tempest-config $ kubectl get nodes NAME STATUS ROLES AGE VERSION -ip-10-0-3-155 Ready 10m v1.17.0 -ip-10-0-26-65 Ready 10m v1.17.0 -ip-10-0-41-21 Ready 10m v1.17.0 +ip-10-0-3-155 Ready 10m v1.17.1 +ip-10-0-26-65 Ready 10m v1.17.1 +ip-10-0-41-21 Ready 10m v1.17.1 ``` List the pods. diff --git a/docs/fedora-coreos/bare-metal.md b/docs/fedora-coreos/bare-metal.md index 461bb232c..dbe6b9950 100644 --- a/docs/fedora-coreos/bare-metal.md +++ b/docs/fedora-coreos/bare-metal.md @@ -3,7 +3,7 @@ !!! danger Typhoon for Fedora CoreOS is an early preview! Fedora CoreOS itself is a preview! Expect bugs and design shifts. Please help both projects solve problems. Report Fedora CoreOS bugs to [Fedora](https://github.com/coreos/fedora-coreos-tracker/issues). Report Typhoon issues to Typhoon. -In this tutorial, we'll network boot and provision a Kubernetes v1.17.0 cluster on bare-metal with Fedora CoreOS. +In this tutorial, we'll network boot and provision a Kubernetes v1.17.1 cluster on bare-metal with Fedora CoreOS. First, we'll deploy a [Matchbox](https://github.com/poseidon/matchbox) service and setup a network boot environment. Then, we'll declare a Kubernetes cluster using the Typhoon Terraform module and power on machines. On PXE boot, machines will install Fedora CoreOS to disk, reboot into the disk install, and provision themselves as Kubernetes controllers or workers via Ignition. @@ -163,7 +163,7 @@ Define a Kubernetes cluster using the module `bare-metal/fedora-coreos/kubernete ```tf module "mercury" { - source = "git::https://github.com/poseidon/typhoon//bare-metal/fedora-coreos/kubernetes?ref=v1.17.0" + source = "git::https://github.com/poseidon/typhoon//bare-metal/fedora-coreos/kubernetes?ref=v1.17.1" # bare-metal cluster_name = "mercury" @@ -293,9 +293,9 @@ List nodes in the cluster. $ export KUBECONFIG=/home/user/.kube/configs/mercury-config $ kubectl get nodes NAME STATUS ROLES AGE VERSION -node1.example.com Ready 10m v1.17.0 -node2.example.com Ready 10m v1.17.0 -node3.example.com Ready 10m v1.17.0 +node1.example.com Ready 10m v1.17.1 +node2.example.com Ready 10m v1.17.1 +node3.example.com Ready 10m v1.17.1 ``` List the pods. diff --git a/docs/index.md b/docs/index.md index 144c98729..0a4b4504e 100644 --- a/docs/index.md +++ b/docs/index.md @@ -11,7 +11,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features -* Kubernetes v1.17.0 (upstream) +* Kubernetes v1.17.1 (upstream) * Single or multi-master, [Calico](https://www.projectcalico.org/) or [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/) * Advanced features like [worker pools](advanced/worker-pools/), [preemptible](cl/google-cloud/#preemption) workers, and [snippets](advanced/customization/#container-linux) customization @@ -47,7 +47,7 @@ Define a Kubernetes cluster by using the Terraform module for your chosen platfo ```tf module "yavin" { - source = "git::https://github.com/poseidon/typhoon//google-cloud/container-linux/kubernetes?ref=v1.17.0" + source = "git::https://github.com/poseidon/typhoon//google-cloud/container-linux/kubernetes?ref=v1.17.1" # Google Cloud cluster_name = "yavin" @@ -85,9 +85,9 @@ In 4-8 minutes (varies by platform), the cluster will be ready. This Google Clou $ export KUBECONFIG=/home/user/.kube/configs/yavin-config $ kubectl get nodes NAME ROLES STATUS AGE VERSION -yavin-controller-0.c.example-com.internal Ready 6m v1.17.0 -yavin-worker-jrbf.c.example-com.internal Ready 5m v1.17.0 -yavin-worker-mzdm.c.example-com.internal Ready 5m v1.17.0 +yavin-controller-0.c.example-com.internal Ready 6m v1.17.1 +yavin-worker-jrbf.c.example-com.internal Ready 5m v1.17.1 +yavin-worker-mzdm.c.example-com.internal Ready 5m v1.17.1 ``` List the pods. diff --git a/docs/topics/maintenance.md b/docs/topics/maintenance.md index bd97b62eb..2acf3a5fd 100644 --- a/docs/topics/maintenance.md +++ b/docs/topics/maintenance.md @@ -18,7 +18,7 @@ module "yavin" { } module "mercury" { - source = "git::https://github.com/poseidon/typhoon//bare-metal/container-linux/kubernetes?ref=v1.17.0" + source = "git::https://github.com/poseidon/typhoon//bare-metal/container-linux/kubernetes?ref=v1.17.1" ... } ``` @@ -279,15 +279,15 @@ Typhoon modules have been adapted for Terraform v0.12. Provider plugins requirem | Typhoon Release | Terraform version | |-------------------|---------------------| -| v1.17.0 - ? | v0.12.x | -| v1.10.3 - v1.17.0 | v0.11.x | +| v1.17.1 - ? | v0.12.x | +| v1.10.3 - v1.17.1 | v0.11.x | | v1.9.2 - v1.10.2 | v0.10.4+ or v0.11.x | | v1.7.3 - v1.9.1 | v0.10.x | | v1.6.4 - v1.7.2 | v0.9.x | ### New users -New users can start with Terraform v0.12.x and follow the docs for Typhoon v1.17.0+ without issue. +New users can start with Terraform v0.12.x and follow the docs for Typhoon v1.17.1+ without issue. ### Existing users @@ -404,7 +404,7 @@ tree . └── infraB <- new Terraform v0.12.x configs ``` -Define Typhoon clusters in the new config directory using Terraform v0.12 syntax. Follow the Typhoon v1.17.0+ docs (e.g. use `terraform12` in the `infraB` dir). See [AWS](/cl/aws), [Azure](/cl/azure), [Bare-Metal](/cl/bare-metal), [Digital Ocean](/cl/digital-ocean), or [Google-Cloud](/cl/google-cloud)) to create new clusters. Follow the usual [upgrade](/topics/maintenance/#upgrades) process to apply workloads and shift traffic. Later, switch back to the old config directory and deprovision clusters with Terraform v0.11. +Define Typhoon clusters in the new config directory using Terraform v0.12 syntax. Follow the Typhoon v1.17.1+ docs (e.g. use `terraform12` in the `infraB` dir). See [AWS](/cl/aws), [Azure](/cl/azure), [Bare-Metal](/cl/bare-metal), [Digital Ocean](/cl/digital-ocean), or [Google-Cloud](/cl/google-cloud)) to create new clusters. Follow the usual [upgrade](/topics/maintenance/#upgrades) process to apply workloads and shift traffic. Later, switch back to the old config directory and deprovision clusters with Terraform v0.11. ```shell terraform12 init diff --git a/google-cloud/container-linux/kubernetes/README.md b/google-cloud/container-linux/kubernetes/README.md index 0f96f6eb1..b94a44463 100644 --- a/google-cloud/container-linux/kubernetes/README.md +++ b/google-cloud/container-linux/kubernetes/README.md @@ -11,7 +11,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features -* Kubernetes v1.17.0 (upstream) +* Kubernetes v1.17.1 (upstream) * Single or multi-master, [Calico](https://www.projectcalico.org/) or [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/) * Advanced features like [worker pools](https://typhoon.psdn.io/advanced/worker-pools/), [preemptible](https://typhoon.psdn.io/cl/google-cloud/#preemption) workers, and [snippets](https://typhoon.psdn.io/advanced/customization/#container-linux) customization diff --git a/google-cloud/container-linux/kubernetes/bootstrap.tf b/google-cloud/container-linux/kubernetes/bootstrap.tf index f6b385f7a..9b79eca6b 100644 --- a/google-cloud/container-linux/kubernetes/bootstrap.tf +++ b/google-cloud/container-linux/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=ac4b7af57012d477cd53bd74ce632ac581e807e1" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=5ce4fc69537a8bfd1cdd08499e03f246d9f73ca6" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/google-cloud/container-linux/kubernetes/cl/controller.yaml b/google-cloud/container-linux/kubernetes/cl/controller.yaml index 0dc53c042..07fc36a26 100644 --- a/google-cloud/container-linux/kubernetes/cl/controller.yaml +++ b/google-cloud/container-linux/kubernetes/cl/controller.yaml @@ -88,7 +88,7 @@ systemd: --mount volume=var-log,target=/var/log \ --volume opt-cni-bin,kind=host,source=/opt/cni/bin \ --mount volume=opt-cni-bin,target=/opt/cni/bin \ - docker://k8s.gcr.io/hyperkube:v1.17.0 \ + docker://k8s.gcr.io/hyperkube:v1.17.1 \ --exec=/usr/local/bin/kubelet -- \ --anonymous-auth=false \ --authentication-token-webhook \ @@ -132,7 +132,7 @@ systemd: --volume script,kind=host,source=/opt/bootstrap/apply \ --mount volume=script,target=/apply \ --insecure-options=image \ - docker://k8s.gcr.io/hyperkube:v1.17.0 \ + docker://k8s.gcr.io/hyperkube:v1.17.1 \ --net=host \ --dns=host \ --exec=/apply diff --git a/google-cloud/container-linux/kubernetes/workers/cl/worker.yaml b/google-cloud/container-linux/kubernetes/workers/cl/worker.yaml index 179323129..9d70dd386 100644 --- a/google-cloud/container-linux/kubernetes/workers/cl/worker.yaml +++ b/google-cloud/container-linux/kubernetes/workers/cl/worker.yaml @@ -63,7 +63,7 @@ systemd: --mount volume=var-log,target=/var/log \ --volume opt-cni-bin,kind=host,source=/opt/cni/bin \ --mount volume=opt-cni-bin,target=/opt/cni/bin \ - docker://k8s.gcr.io/hyperkube:v1.17.0 \ + docker://k8s.gcr.io/hyperkube:v1.17.1 \ --exec=/usr/local/bin/kubelet -- \ --anonymous-auth=false \ --authentication-token-webhook \ @@ -126,7 +126,7 @@ storage: --volume config,kind=host,source=/etc/kubernetes \ --mount volume=config,target=/etc/kubernetes \ --insecure-options=image \ - docker://k8s.gcr.io/hyperkube:v1.17.0 \ + docker://k8s.gcr.io/hyperkube:v1.17.1 \ --net=host \ --dns=host \ -- \ From 7daabd28b501d704841aeb0cf10ae2bd289e85bb Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Sat, 18 Jan 2020 13:45:24 -0800 Subject: [PATCH 311/523] Update Calico from v3.11.1 to v3.11.2 * https://docs.projectcalico.org/v3.11/release-notes/ --- CHANGES.md | 3 ++- aws/container-linux/kubernetes/bootstrap.tf | 2 +- aws/fedora-coreos/kubernetes/bootstrap.tf | 2 +- azure/container-linux/kubernetes/bootstrap.tf | 2 +- bare-metal/container-linux/kubernetes/bootstrap.tf | 2 +- bare-metal/fedora-coreos/kubernetes/bootstrap.tf | 2 +- digital-ocean/container-linux/kubernetes/bootstrap.tf | 2 +- google-cloud/container-linux/kubernetes/bootstrap.tf | 2 +- 8 files changed, 9 insertions(+), 8 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 8c8eb7aae..78701ae71 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -4,8 +4,9 @@ Notable changes between versions. ## Latest +* Kubernetes [v1.17.1](https://github.com/kubernetes/kubernetes/blob/master/CHANGELOG-1.17.md#v1171) * Update CoreDNS from v1.6.5 to [v1.6.6](https://coredns.io/2019/12/11/coredns-1.6.6-release/) ([#602](https://github.com/poseidon/typhoon/pull/602)) -* Update Calico from v3.10.2 to v3.11.1 ([#604](https://github.com/poseidon/typhoon/pull/604)) +* Update Calico from v3.10.2 to v3.11.2 ([#604](https://github.com/poseidon/typhoon/pull/604)) * Inline Kubelet service on Container Linux nodes ([#606](https://github.com/poseidon/typhoon/pull/606)) * Disable unused Kubelet `127.0.0.1:10248` healthz listener ([#607](https://github.com/poseidon/typhoon/pull/607)) * Enable kube-proxy metrics and allow Prometheus scrapes diff --git a/aws/container-linux/kubernetes/bootstrap.tf b/aws/container-linux/kubernetes/bootstrap.tf index e03875062..92d1b073a 100644 --- a/aws/container-linux/kubernetes/bootstrap.tf +++ b/aws/container-linux/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=5ce4fc69537a8bfd1cdd08499e03f246d9f73ca6" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=de85f1da7df0b13dfb7488350c20a510f3090cdf" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/aws/fedora-coreos/kubernetes/bootstrap.tf b/aws/fedora-coreos/kubernetes/bootstrap.tf index f474206b4..93123c2c2 100644 --- a/aws/fedora-coreos/kubernetes/bootstrap.tf +++ b/aws/fedora-coreos/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=5ce4fc69537a8bfd1cdd08499e03f246d9f73ca6" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=de85f1da7df0b13dfb7488350c20a510f3090cdf" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/azure/container-linux/kubernetes/bootstrap.tf b/azure/container-linux/kubernetes/bootstrap.tf index 57bbe1ee3..f96f64e1c 100644 --- a/azure/container-linux/kubernetes/bootstrap.tf +++ b/azure/container-linux/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=5ce4fc69537a8bfd1cdd08499e03f246d9f73ca6" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=de85f1da7df0b13dfb7488350c20a510f3090cdf" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/bare-metal/container-linux/kubernetes/bootstrap.tf b/bare-metal/container-linux/kubernetes/bootstrap.tf index 70a4db0cd..b563ae148 100644 --- a/bare-metal/container-linux/kubernetes/bootstrap.tf +++ b/bare-metal/container-linux/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=5ce4fc69537a8bfd1cdd08499e03f246d9f73ca6" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=de85f1da7df0b13dfb7488350c20a510f3090cdf" cluster_name = var.cluster_name api_servers = [var.k8s_domain_name] diff --git a/bare-metal/fedora-coreos/kubernetes/bootstrap.tf b/bare-metal/fedora-coreos/kubernetes/bootstrap.tf index 7062552ea..8cc0d4645 100644 --- a/bare-metal/fedora-coreos/kubernetes/bootstrap.tf +++ b/bare-metal/fedora-coreos/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=5ce4fc69537a8bfd1cdd08499e03f246d9f73ca6" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=de85f1da7df0b13dfb7488350c20a510f3090cdf" cluster_name = var.cluster_name api_servers = [var.k8s_domain_name] diff --git a/digital-ocean/container-linux/kubernetes/bootstrap.tf b/digital-ocean/container-linux/kubernetes/bootstrap.tf index 4e7e8d275..7f7d7356d 100644 --- a/digital-ocean/container-linux/kubernetes/bootstrap.tf +++ b/digital-ocean/container-linux/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=5ce4fc69537a8bfd1cdd08499e03f246d9f73ca6" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=de85f1da7df0b13dfb7488350c20a510f3090cdf" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/google-cloud/container-linux/kubernetes/bootstrap.tf b/google-cloud/container-linux/kubernetes/bootstrap.tf index 9b79eca6b..883dfd3b1 100644 --- a/google-cloud/container-linux/kubernetes/bootstrap.tf +++ b/google-cloud/container-linux/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=5ce4fc69537a8bfd1cdd08499e03f246d9f73ca6" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=de85f1da7df0b13dfb7488350c20a510f3090cdf" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] From 7ddd3d096dc62ab32f68942c860f58bc591fdfed Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Sat, 18 Jan 2020 15:14:03 -0800 Subject: [PATCH 312/523] Fix link in maintenance docs * Also a fix version mention, since Terraform v0.12 was added in Typhoon v1.15.0 --- docs/topics/maintenance.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/topics/maintenance.md b/docs/topics/maintenance.md index 2acf3a5fd..524818940 100644 --- a/docs/topics/maintenance.md +++ b/docs/topics/maintenance.md @@ -395,7 +395,7 @@ Verify no changes are proposed and commit changes to version control. You've mig Alternately, continue maintaining existing clusters using Terraform v0.11.x and existing Terraform configuration directory(ies). Create new Terraform directory(ies) and move resources there to be managed with Terraform v0.12. This approach allows resources to be migrated incrementally and ensures existing resources can always be managed (e.g. emergency patches). -Create a new Terraform [config directory](/architecture/concepts#organize) for *new* resources. +Create a new Terraform [config directory](/architecture/concepts/#organize) for *new* resources. ```shell mkdir infra2 @@ -404,7 +404,7 @@ tree . └── infraB <- new Terraform v0.12.x configs ``` -Define Typhoon clusters in the new config directory using Terraform v0.12 syntax. Follow the Typhoon v1.17.1+ docs (e.g. use `terraform12` in the `infraB` dir). See [AWS](/cl/aws), [Azure](/cl/azure), [Bare-Metal](/cl/bare-metal), [Digital Ocean](/cl/digital-ocean), or [Google-Cloud](/cl/google-cloud)) to create new clusters. Follow the usual [upgrade](/topics/maintenance/#upgrades) process to apply workloads and shift traffic. Later, switch back to the old config directory and deprovision clusters with Terraform v0.11. +Define Typhoon clusters in the new config directory using Terraform v0.12 syntax. Follow the Typhoon v1.15.0+ docs (e.g. use `terraform12` in the `infraB` dir). See [AWS](/cl/aws), [Azure](/cl/azure), [Bare-Metal](/cl/bare-metal), [Digital Ocean](/cl/digital-ocean), or [Google-Cloud](/cl/google-cloud)) to create new clusters. Follow the usual [upgrade](/topics/maintenance/#upgrades) process to apply workloads and shift traffic. Later, switch back to the old config directory and deprovision clusters with Terraform v0.11. ```shell terraform12 init From 48703f9906503cfb3c6f440f34381a198aa7a3e4 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Sat, 18 Jan 2020 15:30:09 -0800 Subject: [PATCH 313/523] Update Grafana from v6.5.2 to v6.5.3 * https://github.com/grafana/grafana/releases/tag/v6.5.3 --- CHANGES.md | 2 +- addons/grafana/deployment.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 78701ae71..db6d3a170 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -23,7 +23,7 @@ Notable changes between versions. * Add discovery for kube-proxy service endpoints * Update kube-state-metrics from v1.8.0 to v1.9.1 * Reduce node-exporter DaemonSet tolerations ([#614](https://github.com/poseidon/typhoon/pull/614)) -* Update Grafana from v6.5.1 to v6.5.2 +* Update Grafana from v6.5.1 to v6.5.3 ## v1.17.0 diff --git a/addons/grafana/deployment.yaml b/addons/grafana/deployment.yaml index 93d187282..a255fd47e 100644 --- a/addons/grafana/deployment.yaml +++ b/addons/grafana/deployment.yaml @@ -23,7 +23,7 @@ spec: spec: containers: - name: grafana - image: docker.io/grafana/grafana:6.5.2 + image: docker.io/grafana/grafana:6.5.3 env: - name: GF_PATHS_CONFIG value: "/etc/grafana/custom.ini" From 03ff3a9cf34d72ee457dc2a30d07f53b4a8d0a99 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Sat, 18 Jan 2020 15:32:10 -0800 Subject: [PATCH 314/523] Update kube-state-metrics from v1.9.1 to v1.9.2 * https://github.com/kubernetes/kube-state-metrics/releases/tag/v1.9.2 --- CHANGES.md | 2 +- addons/prometheus/exporters/kube-state-metrics/deployment.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index db6d3a170..92abd6f12 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -21,7 +21,7 @@ Notable changes between versions. * Update Prometheus from v2.14.0 to [v2.15.2](https://github.com/prometheus/prometheus/releases/tag/v2.15.2) * Add discovery for kube-proxy service endpoints -* Update kube-state-metrics from v1.8.0 to v1.9.1 +* Update kube-state-metrics from v1.8.0 to v1.9.2 * Reduce node-exporter DaemonSet tolerations ([#614](https://github.com/poseidon/typhoon/pull/614)) * Update Grafana from v6.5.1 to v6.5.3 diff --git a/addons/prometheus/exporters/kube-state-metrics/deployment.yaml b/addons/prometheus/exporters/kube-state-metrics/deployment.yaml index da7e0129c..55fe0432f 100644 --- a/addons/prometheus/exporters/kube-state-metrics/deployment.yaml +++ b/addons/prometheus/exporters/kube-state-metrics/deployment.yaml @@ -24,7 +24,7 @@ spec: serviceAccountName: kube-state-metrics containers: - name: kube-state-metrics - image: quay.io/coreos/kube-state-metrics:v1.9.1 + image: quay.io/coreos/kube-state-metrics:v1.9.2 ports: - name: metrics containerPort: 8080 From dd930a2ff9c0c30605ebaf159ffa8b0a2b6e7d34 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Tue, 14 Jan 2020 00:49:34 -0800 Subject: [PATCH 315/523] Update bare-metal Fedora CoreOS image location * Use Fedora CoreOS production download streams (change) * Use live PXE kernel and initramfs images * https://getfedora.org/coreos/download/ * Update docs example to use public images (cache is still recommended at large scale) and stable stream --- CHANGES.md | 30 +++++++++++++------ .../fedora-coreos/kubernetes/profiles.tf | 8 ++--- docs/fedora-coreos/bare-metal.md | 17 +++++------ 3 files changed, 33 insertions(+), 22 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 92abd6f12..c9b88d1a7 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -4,6 +4,14 @@ Notable changes between versions. ## Latest +#### Bare-Metal + +* Update Fedora CoreOS images location + * Use Fedora CoreOS production [download](https://getfedora.org/coreos/download/) streams + * Use live PXE kernel and initramfs images + +## v1.17.1 + * Kubernetes [v1.17.1](https://github.com/kubernetes/kubernetes/blob/master/CHANGELOG-1.17.md#v1171) * Update CoreDNS from v1.6.5 to [v1.6.6](https://coredns.io/2019/12/11/coredns-1.6.6-release/) ([#602](https://github.com/poseidon/typhoon/pull/602)) * Update Calico from v3.10.2 to v3.11.2 ([#604](https://github.com/poseidon/typhoon/pull/604)) @@ -12,6 +20,10 @@ Notable changes between versions. * Enable kube-proxy metrics and allow Prometheus scrapes * Allow TCP/10249 traffic with worker node sources +#### AWS + +* Update Fedora CoreOS AMI filter for fedora-coreos-31 ([#620](https://github.com/poseidon/typhoon/pull/620)) + #### Google * Allow `terraform-provider-google` v3.0+ ([#617](https://github.com/poseidon/typhoon/pull/617)) @@ -247,7 +259,7 @@ Notable changes between versions. * Require `terraform-provider-azurerm` v1.27+ to support Terraform v0.12 (action required) * Avoid unneeded rotations of Regular priority virtual machine scale sets * Azure only allows `eviction_policy` to be set for Low priority VMs. Supporting Low priority VMs meant when Regular VMs were used, each `terraform apply` rolled workers, to set eviction_policy to null. - * Terraform v0.12 nullable variables fix the issue so plan does not produce a diff. + * Terraform v0.12 nullable variables fix the issue so plan does not produce a diff. #### Bare-Metal @@ -302,7 +314,7 @@ Notable changes between versions. * Update Grafana from v6.1.6 to v6.2.1 ## v1.14.2 - + * Kubernetes [v1.14.2](https://github.com/kubernetes/kubernetes/blob/master/CHANGELOG-1.14.md#v1142) * Update etcd from v3.3.12 to [v3.3.13](https://github.com/etcd-io/etcd/releases/tag/v3.3.13) * Upgrade Calico from v3.6.1 to [v3.7.2](https://docs.projectcalico.org/v3.7/release-notes/) @@ -373,7 +385,7 @@ Notable changes between versions. * Add ability to load balance TCP/UDP applications ([#442](https://github.com/poseidon/typhoon/pull/442)) * Add worker instances to a target pool, output as `worker_target_pool` - * Health check for workers with Ingress controllers. Forward rules don't support differing internal/external ports, but some Ingress controllers support TCP/UDP proxy as a workaround + * Health check for workers with Ingress controllers. Forward rules don't support differing internal/external ports, but some Ingress controllers support TCP/UDP proxy as a workaround * Remove Haswell minimum CPU platform requirement ([#439](https://github.com/poseidon/typhoon/pull/439)) * Google Cloud API implements `min_cpu_platform` to mean "use exactly this CPU". Revert [#405](https://github.com/poseidon/typhoon/pull/405) added in v1.13.4. * Fix error creating clusters in new regions without Haswell (e.g. europe-west2) ([#438](https://github.com/poseidon/typhoon/issues/438)) @@ -558,7 +570,7 @@ Notable changes between versions. * Update Calico from v3.3.0 to [v3.3.1](https://docs.projectcalico.org/v3.3/releases/) * Disable Felix usage reporting by default ([#345](https://github.com/poseidon/typhoon/pull/345)) * Improve flannel manifests - * [Rename](https://github.com/poseidon/terraform-render-bootkube/commit/d045a8e6b8eccfbb9d69bb51953b5a93d23f67f7) `kube-flannel` DaemonSet to `flannel` and `kube-flannel-cfg` ConfigMap to `flannel-config` + * [Rename](https://github.com/poseidon/terraform-render-bootkube/commit/d045a8e6b8eccfbb9d69bb51953b5a93d23f67f7) `kube-flannel` DaemonSet to `flannel` and `kube-flannel-cfg` ConfigMap to `flannel-config` * [Drop](https://github.com/poseidon/terraform-render-bootkube/commit/39f9afb3360ec642e5b98457c8bd07eda35b6c96) unused mounts and add a CPU resource request * Update CoreDNS from v1.2.4 to [v1.2.6](https://coredns.io/2018/11/05/coredns-1.2.6-release/) * Enable CoreDNS `loop` and `loadbalance` plugins ([#340](https://github.com/poseidon/typhoon/pull/340)) @@ -720,7 +732,7 @@ Notable changes between versions. * Force apiserver to stop listening on `127.0.0.1:8080` * Replace `kube-dns` with [CoreDNS](https://coredns.io/) ([#261](https://github.com/poseidon/typhoon/pull/261)) * Edit the `coredns` ConfigMap to [customize](https://coredns.io/plugins/) - * CoreDNS doesn't use a resizer. For large clusters, scaling may be required. + * CoreDNS doesn't use a resizer. For large clusters, scaling may be required. #### AWS @@ -765,7 +777,7 @@ Notable changes between versions. * Switch `kube-apiserver` port from 443 to 6443 ([#248](https://github.com/poseidon/typhoon/pull/248)) * Users who exposed kube-apiserver on a WAN via their router/load-balancer will need to adjust its configuration (e.g. DNAT 6443). Most apiservers are on a LAN (internal, VPN-only, etc) so if you didn't specially configure network gear for 443, no change is needed. (possible action required) -* Fix possible deadlock when provisioning clusters larger than 10 nodes ([#244](https://github.com/poseidon/typhoon/pull/244)) +* Fix possible deadlock when provisioning clusters larger than 10 nodes ([#244](https://github.com/poseidon/typhoon/pull/244)) #### DigitalOcean @@ -833,7 +845,7 @@ Notable changes between versions. * Please change values stable, beta, or alpha to coreos-stable, coreos-beta, coreos-alpha (**action required!**) * Replace `container_linux_version` variable with `os_version` * Add `network_ip_autodetection_method` variable for Calico host IPv4 address detection - * Use Calico's default "first-found" to support single NIC and bonded NIC nodes + * Use Calico's default "first-found" to support single NIC and bonded NIC nodes * Allow [alternative](https://docs.projectcalico.org/v3.1/reference/node/configuration#ip-autodetection-methods) methods for multi NIC nodes, like can-reach=IP or interface=REGEX * Deprecate `container_linux_oem` variable @@ -866,7 +878,7 @@ Notable changes between versions. #### Google Cloud * Add support for multi-controller clusters (i.e. multi-master) ([#54](https://github.com/poseidon/typhoon/issues/54), [#190](https://github.com/poseidon/typhoon/pull/190)) - * Switch from Google Cloud network load balancer to a TCP proxy load balancer. Avoid a [bug](https://issuetracker.google.com/issues/67366622) in Google network load balancers that limited clusters to only bootstrapping one controller node. + * Switch from Google Cloud network load balancer to a TCP proxy load balancer. Avoid a [bug](https://issuetracker.google.com/issues/67366622) in Google network load balancers that limited clusters to only bootstrapping one controller node. * Add TCP health check for apiserver pods on controllers. Replace kubelet check approximation. #### Addons @@ -1097,7 +1109,7 @@ Notable changes between versions. * Container Linux stable, beta, and alpha now provide Docker 17.09 (instead of 1.12) * Older clusters (with CLUO addon) auto-update Container Linux version to begin using Docker 17.09 -* Fix race where `etcd-member.service` could fail to resolve peers ([#69](https://github.com/poseidon/typhoon/pull/69)) +* Fix race where `etcd-member.service` could fail to resolve peers ([#69](https://github.com/poseidon/typhoon/pull/69)) * Add optional `cluster_domain_suffix` variable (#74) * Use kubernetes-incubator/bootkube v0.9.1 diff --git a/bare-metal/fedora-coreos/kubernetes/profiles.tf b/bare-metal/fedora-coreos/kubernetes/profiles.tf index 2c214260a..b49e3176e 100644 --- a/bare-metal/fedora-coreos/kubernetes/profiles.tf +++ b/bare-metal/fedora-coreos/kubernetes/profiles.tf @@ -1,6 +1,6 @@ locals { - remote_kernel = "https://builds.coreos.fedoraproject.org/prod/streams/${var.os_stream}/builds/${var.os_version}/x86_64/fedora-coreos-${var.os_version}-installer-kernel-x86_64" - remote_initrd = "https://builds.coreos.fedoraproject.org/prod/streams/${var.os_stream}/builds/${var.os_version}/x86_64/fedora-coreos-${var.os_version}-installer-initramfs.x86_64.img" + remote_kernel = "https://builds.coreos.fedoraproject.org/prod/streams/${var.os_stream}/builds/${var.os_version}/x86_64/fedora-coreos-${var.os_version}-live-kernel-x86_64" + remote_initrd = "https://builds.coreos.fedoraproject.org/prod/streams/${var.os_stream}/builds/${var.os_version}/x86_64/fedora-coreos-${var.os_version}-live-initramfs.x86_64.img" remote_args = [ "ip=dhcp", "rd.neednet=1", @@ -10,8 +10,8 @@ locals { "coreos.inst.install_dev=${var.install_disk}" ] - cached_kernel = "/assets/fedora-coreos/fedora-coreos-${var.os_version}-installer-kernel-x86_64" - cached_initrd = "/assets/fedora-coreos/fedora-coreos-${var.os_version}-installer-initramfs.x86_64.img" + cached_kernel = "/assets/fedora-coreos/fedora-coreos-${var.os_version}-live-kernel-x86_64" + cached_initrd = "/assets/fedora-coreos/fedora-coreos-${var.os_version}-live-initramfs.x86_64.img" cached_args = [ "ip=dhcp", "rd.neednet=1", diff --git a/docs/fedora-coreos/bare-metal.md b/docs/fedora-coreos/bare-metal.md index dbe6b9950..05ec55a5e 100644 --- a/docs/fedora-coreos/bare-metal.md +++ b/docs/fedora-coreos/bare-metal.md @@ -30,7 +30,7 @@ Configure each machine to boot from the disk through IPMI or the BIOS menu. ``` ipmitool -H node1 -U USER -P PASS chassis bootdev disk options=persistent ``` - + During provisioning, you'll explicitly set the boot device to `pxe` for the next boot only. Machines will install (overwrite) the operating system to disk on PXE boot and reboot into the disk install. !!! tip "" @@ -106,7 +106,7 @@ Read about the [many ways](https://coreos.com/matchbox/docs/latest/network-setup TFTP chainloading to modern boot firmware, like iPXE, avoids issues with old NICs and allows faster transfer protocols like HTTP to be used. !!! warning - Compile iPXE from [source](https://github.com/ipxe/ipxe) with support for [HTTPS downloads](https://ipxe.org/crypto). iPXE's pre-built firmware binaries do not enable this. Fedora does not provide images over HTTP. + Compile iPXE from [source](https://github.com/ipxe/ipxe) with support for [HTTPS downloads](https://ipxe.org/crypto). iPXE's pre-built firmware binaries do not enable this. Fedora CoreOS downloads are HTTPS-only. ## Terraform Setup @@ -164,13 +164,12 @@ Define a Kubernetes cluster using the module `bare-metal/fedora-coreos/kubernete ```tf module "mercury" { source = "git::https://github.com/poseidon/typhoon//bare-metal/fedora-coreos/kubernetes?ref=v1.17.1" - + # bare-metal cluster_name = "mercury" matchbox_http_endpoint = "http://matchbox.example.com" - os_stream = "testing" - os_version = "30.20191002.0" - cached_install = true + os_stream = "stable" + os_version = "31.20200113.3.1" # configuration k8s_domain_name = "node1.example.com" @@ -330,8 +329,8 @@ Check the [variables.tf](https://github.com/poseidon/typhoon/blob/master/bare-me |:-----|:------------|:--------| | cluster_name | Unique cluster name | "mercury" | | matchbox_http_endpoint | Matchbox HTTP read-only endpoint | "http://matchbox.example.com:port" | -| os_stream | Fedora CoreOS release stream | "testing" | -| os_version | Fedora CoreOS version to PXE and install | "30.20190716.1" | +| os_stream | Fedora CoreOS release stream | "stable" | +| os_version | Fedora CoreOS version to PXE and install | "31.20200113.3.1" | | k8s_domain_name | FQDN resolving to the controller(s) nodes. Workers and kubectl will communicate with this endpoint | "myk8s.example.com" | | ssh_authorized_key | SSH public key for user 'core' | "ssh-rsa AAAAB3Nz..." | | controllers | List of controller machine detail objects (unique name, identifying MAC address, FQDN) | `[{name="node1", mac="52:54:00:a1:9c:ae", domain="node1.example.com"}]` | @@ -345,7 +344,7 @@ Check the [variables.tf](https://github.com/poseidon/typhoon/blob/master/bare-me | cached_install | PXE boot and install from the Matchbox `/assets` cache. Admin MUST have downloaded Fedora CoreOS images into the cache | false | true | | install_disk | Disk device where Fedora CoreOS should be installed | "sda" (not "/dev/sda" like Container Linux) | "sdb" | | networking | Choice of networking provider | "calico" | "calico" or "flannel" | -| network_mtu | CNI interface MTU (calico-only) | 1480 | - | +| network_mtu | CNI interface MTU (calico-only) | 1480 | - | | snippets | Map from machine names to lists of Fedora CoreOS Config snippets | {} | UNSUPPORTED | | network_ip_autodetection_method | Method to detect host IPv4 address (calico-only) | "first-found" | "can-reach=10.0.0.1" | | pod_cidr | CIDR IPv4 range to assign to Kubernetes pods | "10.2.0.0/16" | "10.22.0.0/16" | From bda73264f716072f3fae343577523884cd96dac1 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Mon, 20 Jan 2020 15:06:26 -0800 Subject: [PATCH 316/523] Update nginx-ingress from v0.26.1 to v0.27.1 * Change runAsUser from 33 to 101 for new alpine-based image * https://github.com/kubernetes/ingress-nginx/releases/tag/nginx-0.27.0 * https://github.com/kubernetes/ingress-nginx/releases/tag/nginx-0.27.1 --- CHANGES.md | 5 +++++ addons/nginx-ingress/aws/deployment.yaml | 4 ++-- addons/nginx-ingress/azure/deployment.yaml | 4 ++-- addons/nginx-ingress/bare-metal/deployment.yaml | 4 ++-- addons/nginx-ingress/digital-ocean/daemonset.yaml | 4 ++-- addons/nginx-ingress/google-cloud/deployment.yaml | 4 ++-- 6 files changed, 15 insertions(+), 10 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index c9b88d1a7..f15b072e4 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -10,6 +10,11 @@ Notable changes between versions. * Use Fedora CoreOS production [download](https://getfedora.org/coreos/download/) streams * Use live PXE kernel and initramfs images +#### Addons + +* Update nginx-ingress from v0.26.1 to [v0.27.1](https://github.com/kubernetes/ingress-nginx/releases/tag/nginx-0.27.1) () + * Change runAsUser from 33 to 101 for alpine-based image + ## v1.17.1 * Kubernetes [v1.17.1](https://github.com/kubernetes/kubernetes/blob/master/CHANGELOG-1.17.md#v1171) diff --git a/addons/nginx-ingress/aws/deployment.yaml b/addons/nginx-ingress/aws/deployment.yaml index 3d1382e66..9a70c565e 100644 --- a/addons/nginx-ingress/aws/deployment.yaml +++ b/addons/nginx-ingress/aws/deployment.yaml @@ -22,7 +22,7 @@ spec: spec: containers: - name: nginx-ingress-controller - image: quay.io/kubernetes-ingress-controller/nginx-ingress-controller:0.26.1 + image: quay.io/kubernetes-ingress-controller/nginx-ingress-controller:0.27.1 args: - /nginx-ingress-controller - --ingress-class=public @@ -76,6 +76,6 @@ spec: - NET_BIND_SERVICE drop: - ALL - runAsUser: 33 # www-data + runAsUser: 101 # www-data restartPolicy: Always terminationGracePeriodSeconds: 300 diff --git a/addons/nginx-ingress/azure/deployment.yaml b/addons/nginx-ingress/azure/deployment.yaml index 3d1382e66..9a70c565e 100644 --- a/addons/nginx-ingress/azure/deployment.yaml +++ b/addons/nginx-ingress/azure/deployment.yaml @@ -22,7 +22,7 @@ spec: spec: containers: - name: nginx-ingress-controller - image: quay.io/kubernetes-ingress-controller/nginx-ingress-controller:0.26.1 + image: quay.io/kubernetes-ingress-controller/nginx-ingress-controller:0.27.1 args: - /nginx-ingress-controller - --ingress-class=public @@ -76,6 +76,6 @@ spec: - NET_BIND_SERVICE drop: - ALL - runAsUser: 33 # www-data + runAsUser: 101 # www-data restartPolicy: Always terminationGracePeriodSeconds: 300 diff --git a/addons/nginx-ingress/bare-metal/deployment.yaml b/addons/nginx-ingress/bare-metal/deployment.yaml index 38b85c865..b872a428c 100644 --- a/addons/nginx-ingress/bare-metal/deployment.yaml +++ b/addons/nginx-ingress/bare-metal/deployment.yaml @@ -22,7 +22,7 @@ spec: spec: containers: - name: nginx-ingress-controller - image: quay.io/kubernetes-ingress-controller/nginx-ingress-controller:0.26.1 + image: quay.io/kubernetes-ingress-controller/nginx-ingress-controller:0.27.1 args: - /nginx-ingress-controller - --ingress-class=public @@ -73,7 +73,7 @@ spec: - NET_BIND_SERVICE drop: - ALL - runAsUser: 33 # www-data + runAsUser: 101 # www-data restartPolicy: Always terminationGracePeriodSeconds: 300 diff --git a/addons/nginx-ingress/digital-ocean/daemonset.yaml b/addons/nginx-ingress/digital-ocean/daemonset.yaml index 3c2bb74de..fe9c8f6a6 100644 --- a/addons/nginx-ingress/digital-ocean/daemonset.yaml +++ b/addons/nginx-ingress/digital-ocean/daemonset.yaml @@ -22,7 +22,7 @@ spec: spec: containers: - name: nginx-ingress-controller - image: quay.io/kubernetes-ingress-controller/nginx-ingress-controller:0.26.1 + image: quay.io/kubernetes-ingress-controller/nginx-ingress-controller:0.27.1 args: - /nginx-ingress-controller - --ingress-class=public @@ -76,6 +76,6 @@ spec: - NET_BIND_SERVICE drop: - ALL - runAsUser: 33 # www-data + runAsUser: 101 # www-data restartPolicy: Always terminationGracePeriodSeconds: 300 diff --git a/addons/nginx-ingress/google-cloud/deployment.yaml b/addons/nginx-ingress/google-cloud/deployment.yaml index 3d1382e66..9a70c565e 100644 --- a/addons/nginx-ingress/google-cloud/deployment.yaml +++ b/addons/nginx-ingress/google-cloud/deployment.yaml @@ -22,7 +22,7 @@ spec: spec: containers: - name: nginx-ingress-controller - image: quay.io/kubernetes-ingress-controller/nginx-ingress-controller:0.26.1 + image: quay.io/kubernetes-ingress-controller/nginx-ingress-controller:0.27.1 args: - /nginx-ingress-controller - --ingress-class=public @@ -76,6 +76,6 @@ spec: - NET_BIND_SERVICE drop: - ALL - runAsUser: 33 # www-data + runAsUser: 101 # www-data restartPolicy: Always terminationGracePeriodSeconds: 300 From 1cda5bcd2a8fb617d08d32d36d5b71558b0b089c Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Tue, 21 Jan 2020 18:27:39 -0800 Subject: [PATCH 317/523] Update Kubernetes from v1.17.1 to v1.17.2 * https://github.com/kubernetes/kubernetes/blob/master/CHANGELOG-1.17.md#v1172 --- CHANGES.md | 4 ++++ README.md | 10 +++++----- aws/container-linux/kubernetes/README.md | 2 +- aws/container-linux/kubernetes/bootstrap.tf | 2 +- aws/container-linux/kubernetes/cl/controller.yaml | 4 ++-- .../kubernetes/workers/cl/worker.yaml | 4 ++-- aws/fedora-coreos/kubernetes/README.md | 2 +- aws/fedora-coreos/kubernetes/bootstrap.tf | 2 +- aws/fedora-coreos/kubernetes/fcc/controller.yaml | 4 ++-- .../kubernetes/workers/fcc/worker.yaml | 2 +- azure/container-linux/kubernetes/README.md | 2 +- azure/container-linux/kubernetes/bootstrap.tf | 2 +- .../container-linux/kubernetes/cl/controller.yaml | 4 ++-- .../kubernetes/workers/cl/worker.yaml | 4 ++-- bare-metal/container-linux/kubernetes/README.md | 2 +- bare-metal/container-linux/kubernetes/bootstrap.tf | 2 +- .../container-linux/kubernetes/cl/controller.yaml | 4 ++-- .../container-linux/kubernetes/cl/worker.yaml | 2 +- bare-metal/fedora-coreos/kubernetes/README.md | 2 +- bare-metal/fedora-coreos/kubernetes/bootstrap.tf | 2 +- .../fedora-coreos/kubernetes/fcc/controller.yaml | 4 ++-- .../fedora-coreos/kubernetes/fcc/worker.yaml | 2 +- digital-ocean/container-linux/kubernetes/README.md | 2 +- .../container-linux/kubernetes/bootstrap.tf | 2 +- .../container-linux/kubernetes/cl/controller.yaml | 4 ++-- .../container-linux/kubernetes/cl/worker.yaml | 4 ++-- docs/advanced/worker-pools.md | 14 +++++++------- docs/cl/aws.md | 10 +++++----- docs/cl/azure.md | 10 +++++----- docs/cl/bare-metal.md | 10 +++++----- docs/cl/digital-ocean.md | 10 +++++----- docs/cl/google-cloud.md | 10 +++++----- docs/fedora-coreos/aws.md | 10 +++++----- docs/fedora-coreos/bare-metal.md | 10 +++++----- docs/index.md | 10 +++++----- docs/topics/maintenance.md | 8 ++++---- google-cloud/container-linux/kubernetes/README.md | 2 +- .../container-linux/kubernetes/bootstrap.tf | 2 +- .../container-linux/kubernetes/cl/controller.yaml | 4 ++-- .../kubernetes/workers/cl/worker.yaml | 4 ++-- 40 files changed, 99 insertions(+), 95 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index f15b072e4..480eb62f1 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -4,6 +4,10 @@ Notable changes between versions. ## Latest +## v1.17.2 + +* Kubernetes [v1.17.2](https://github.com/kubernetes/kubernetes/blob/master/CHANGELOG-1.17.md#v1172) + #### Bare-Metal * Update Fedora CoreOS images location diff --git a/README.md b/README.md index bec2c0c32..01290424a 100644 --- a/README.md +++ b/README.md @@ -11,7 +11,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features -* Kubernetes v1.17.1 (upstream) +* Kubernetes v1.17.2 (upstream) * Single or multi-master, [Calico](https://www.projectcalico.org/) or [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/) * Advanced features like [worker pools](https://typhoon.psdn.io/advanced/worker-pools/), [preemptible](https://typhoon.psdn.io/cl/google-cloud/#preemption) workers, and [snippets](https://typhoon.psdn.io/advanced/customization/#container-linux) customization @@ -48,7 +48,7 @@ Define a Kubernetes cluster by using the Terraform module for your chosen platfo ```tf module "yavin" { - source = "git::https://github.com/poseidon/typhoon//google-cloud/container-linux/kubernetes?ref=v1.17.1" + source = "git::https://github.com/poseidon/typhoon//google-cloud/container-linux/kubernetes?ref=v1.17.2" # Google Cloud cluster_name = "yavin" @@ -87,9 +87,9 @@ In 4-8 minutes (varies by platform), the cluster will be ready. This Google Clou $ export KUBECONFIG=/home/user/.kube/configs/yavin-config $ kubectl get nodes NAME ROLES STATUS AGE VERSION -yavin-controller-0.c.example-com.internal Ready 6m v1.17.1 -yavin-worker-jrbf.c.example-com.internal Ready 5m v1.17.1 -yavin-worker-mzdm.c.example-com.internal Ready 5m v1.17.1 +yavin-controller-0.c.example-com.internal Ready 6m v1.17.2 +yavin-worker-jrbf.c.example-com.internal Ready 5m v1.17.2 +yavin-worker-mzdm.c.example-com.internal Ready 5m v1.17.2 ``` List the pods. diff --git a/aws/container-linux/kubernetes/README.md b/aws/container-linux/kubernetes/README.md index 82f414880..04a2ea3c1 100644 --- a/aws/container-linux/kubernetes/README.md +++ b/aws/container-linux/kubernetes/README.md @@ -11,7 +11,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features -* Kubernetes v1.17.1 (upstream) +* Kubernetes v1.17.2 (upstream) * Single or multi-master, [Calico](https://www.projectcalico.org/) or [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/) * Advanced features like [worker pools](https://typhoon.psdn.io/advanced/worker-pools/), [spot](https://typhoon.psdn.io/cl/aws/#spot) workers, and [snippets](https://typhoon.psdn.io/advanced/customization/#container-linux) customization diff --git a/aws/container-linux/kubernetes/bootstrap.tf b/aws/container-linux/kubernetes/bootstrap.tf index 92d1b073a..e6c9ac7d4 100644 --- a/aws/container-linux/kubernetes/bootstrap.tf +++ b/aws/container-linux/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=de85f1da7df0b13dfb7488350c20a510f3090cdf" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=05297b94a936c356851e180e4963034e0047e1c0" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/aws/container-linux/kubernetes/cl/controller.yaml b/aws/container-linux/kubernetes/cl/controller.yaml index 9a35a6c90..e8d375b70 100644 --- a/aws/container-linux/kubernetes/cl/controller.yaml +++ b/aws/container-linux/kubernetes/cl/controller.yaml @@ -89,7 +89,7 @@ systemd: --mount volume=var-log,target=/var/log \ --volume opt-cni-bin,kind=host,source=/opt/cni/bin \ --mount volume=opt-cni-bin,target=/opt/cni/bin \ - docker://k8s.gcr.io/hyperkube:v1.17.1 \ + docker://k8s.gcr.io/hyperkube:v1.17.2 \ --exec=/usr/local/bin/kubelet -- \ --anonymous-auth=false \ --authentication-token-webhook \ @@ -134,7 +134,7 @@ systemd: --volume script,kind=host,source=/opt/bootstrap/apply \ --mount volume=script,target=/apply \ --insecure-options=image \ - docker://k8s.gcr.io/hyperkube:v1.17.1 \ + docker://k8s.gcr.io/hyperkube:v1.17.2 \ --net=host \ --dns=host \ --exec=/apply diff --git a/aws/container-linux/kubernetes/workers/cl/worker.yaml b/aws/container-linux/kubernetes/workers/cl/worker.yaml index 44e8a73c8..4ee4ec34c 100644 --- a/aws/container-linux/kubernetes/workers/cl/worker.yaml +++ b/aws/container-linux/kubernetes/workers/cl/worker.yaml @@ -64,7 +64,7 @@ systemd: --mount volume=var-log,target=/var/log \ --volume opt-cni-bin,kind=host,source=/opt/cni/bin \ --mount volume=opt-cni-bin,target=/opt/cni/bin \ - docker://k8s.gcr.io/hyperkube:v1.17.1 \ + docker://k8s.gcr.io/hyperkube:v1.17.2 \ --exec=/usr/local/bin/kubelet -- \ --anonymous-auth=false \ --authentication-token-webhook \ @@ -128,7 +128,7 @@ storage: --volume config,kind=host,source=/etc/kubernetes \ --mount volume=config,target=/etc/kubernetes \ --insecure-options=image \ - docker://k8s.gcr.io/hyperkube:v1.17.1 \ + docker://k8s.gcr.io/hyperkube:v1.17.2 \ --net=host \ --dns=host \ -- \ diff --git a/aws/fedora-coreos/kubernetes/README.md b/aws/fedora-coreos/kubernetes/README.md index cd15fa628..ded0faeae 100644 --- a/aws/fedora-coreos/kubernetes/README.md +++ b/aws/fedora-coreos/kubernetes/README.md @@ -11,7 +11,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features -* Kubernetes v1.17.1 (upstream) +* Kubernetes v1.17.2 (upstream) * Single or multi-master, [Calico](https://www.projectcalico.org/) or [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/) * Advanced features like [worker pools](https://typhoon.psdn.io/advanced/worker-pools/), [spot](https://typhoon.psdn.io/cl/aws/#spot) workers, and [snippets](https://typhoon.psdn.io/advanced/customization/#container-linux) customization diff --git a/aws/fedora-coreos/kubernetes/bootstrap.tf b/aws/fedora-coreos/kubernetes/bootstrap.tf index 93123c2c2..50e08e59b 100644 --- a/aws/fedora-coreos/kubernetes/bootstrap.tf +++ b/aws/fedora-coreos/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=de85f1da7df0b13dfb7488350c20a510f3090cdf" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=05297b94a936c356851e180e4963034e0047e1c0" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/aws/fedora-coreos/kubernetes/fcc/controller.yaml b/aws/fedora-coreos/kubernetes/fcc/controller.yaml index f43123177..df9338823 100644 --- a/aws/fedora-coreos/kubernetes/fcc/controller.yaml +++ b/aws/fedora-coreos/kubernetes/fcc/controller.yaml @@ -79,7 +79,7 @@ systemd: --volume /var/log:/var/log \ --volume /var/run/lock:/var/run/lock:z \ --volume /opt/cni/bin:/opt/cni/bin:z \ - k8s.gcr.io/hyperkube:v1.17.1 kubelet \ + k8s.gcr.io/hyperkube:v1.17.2 kubelet \ --anonymous-auth=false \ --authentication-token-webhook \ --authorization-mode=Webhook \ @@ -123,7 +123,7 @@ systemd: --volume /opt/bootstrap/assets:/assets:ro,Z \ --volume /opt/bootstrap/apply:/apply:ro,Z \ --entrypoint=/apply \ - k8s.gcr.io/hyperkube:v1.17.1 + k8s.gcr.io/hyperkube:v1.17.2 ExecStartPost=/bin/touch /opt/bootstrap/bootstrap.done ExecStartPost=-/usr/bin/podman stop bootstrap storage: diff --git a/aws/fedora-coreos/kubernetes/workers/fcc/worker.yaml b/aws/fedora-coreos/kubernetes/workers/fcc/worker.yaml index 6ba667aa3..8c4ff0b84 100644 --- a/aws/fedora-coreos/kubernetes/workers/fcc/worker.yaml +++ b/aws/fedora-coreos/kubernetes/workers/fcc/worker.yaml @@ -49,7 +49,7 @@ systemd: --volume /var/log:/var/log \ --volume /var/run/lock:/var/run/lock:z \ --volume /opt/cni/bin:/opt/cni/bin:z \ - k8s.gcr.io/hyperkube:v1.17.1 kubelet \ + k8s.gcr.io/hyperkube:v1.17.2 kubelet \ --anonymous-auth=false \ --authentication-token-webhook \ --authorization-mode=Webhook \ diff --git a/azure/container-linux/kubernetes/README.md b/azure/container-linux/kubernetes/README.md index 2df87a060..c81da4954 100644 --- a/azure/container-linux/kubernetes/README.md +++ b/azure/container-linux/kubernetes/README.md @@ -11,7 +11,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features -* Kubernetes v1.17.1 (upstream) +* Kubernetes v1.17.2 (upstream) * Single or multi-master, [Calico](https://www.projectcalico.org/) or [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/) * Advanced features like [worker pools](https://typhoon.psdn.io/advanced/worker-pools/), [low-priority](https://typhoon.psdn.io/cl/azure/#low-priority) workers, and [snippets](https://typhoon.psdn.io/advanced/customization/#container-linux) customization diff --git a/azure/container-linux/kubernetes/bootstrap.tf b/azure/container-linux/kubernetes/bootstrap.tf index f96f64e1c..8402d2554 100644 --- a/azure/container-linux/kubernetes/bootstrap.tf +++ b/azure/container-linux/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=de85f1da7df0b13dfb7488350c20a510f3090cdf" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=05297b94a936c356851e180e4963034e0047e1c0" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/azure/container-linux/kubernetes/cl/controller.yaml b/azure/container-linux/kubernetes/cl/controller.yaml index dbfea0c6c..c1d19685b 100644 --- a/azure/container-linux/kubernetes/cl/controller.yaml +++ b/azure/container-linux/kubernetes/cl/controller.yaml @@ -88,7 +88,7 @@ systemd: --mount volume=var-log,target=/var/log \ --volume opt-cni-bin,kind=host,source=/opt/cni/bin \ --mount volume=opt-cni-bin,target=/opt/cni/bin \ - docker://k8s.gcr.io/hyperkube:v1.17.1 \ + docker://k8s.gcr.io/hyperkube:v1.17.2 \ --exec=/usr/local/bin/kubelet -- \ --anonymous-auth=false \ --authentication-token-webhook \ @@ -132,7 +132,7 @@ systemd: --volume script,kind=host,source=/opt/bootstrap/apply \ --mount volume=script,target=/apply \ --insecure-options=image \ - docker://k8s.gcr.io/hyperkube:v1.17.1 \ + docker://k8s.gcr.io/hyperkube:v1.17.2 \ --net=host \ --dns=host \ --exec=/apply diff --git a/azure/container-linux/kubernetes/workers/cl/worker.yaml b/azure/container-linux/kubernetes/workers/cl/worker.yaml index 69f4aacab..cb314e0f9 100644 --- a/azure/container-linux/kubernetes/workers/cl/worker.yaml +++ b/azure/container-linux/kubernetes/workers/cl/worker.yaml @@ -63,7 +63,7 @@ systemd: --mount volume=var-log,target=/var/log \ --volume opt-cni-bin,kind=host,source=/opt/cni/bin \ --mount volume=opt-cni-bin,target=/opt/cni/bin \ - docker://k8s.gcr.io/hyperkube:v1.17.1 \ + docker://k8s.gcr.io/hyperkube:v1.17.2 \ --exec=/usr/local/bin/kubelet -- \ --anonymous-auth=false \ --authentication-token-webhook \ @@ -126,7 +126,7 @@ storage: --volume config,kind=host,source=/etc/kubernetes \ --mount volume=config,target=/etc/kubernetes \ --insecure-options=image \ - docker://k8s.gcr.io/hyperkube:v1.17.1 \ + docker://k8s.gcr.io/hyperkube:v1.17.2 \ --net=host \ --dns=host \ -- \ diff --git a/bare-metal/container-linux/kubernetes/README.md b/bare-metal/container-linux/kubernetes/README.md index 3ec30a8db..cccd3eb83 100644 --- a/bare-metal/container-linux/kubernetes/README.md +++ b/bare-metal/container-linux/kubernetes/README.md @@ -11,7 +11,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features -* Kubernetes v1.17.1 (upstream) +* Kubernetes v1.17.2 (upstream) * Single or multi-master, [Calico](https://www.projectcalico.org/) or [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/) * Advanced features like [snippets](https://typhoon.psdn.io/advanced/customization/#container-linux) customization diff --git a/bare-metal/container-linux/kubernetes/bootstrap.tf b/bare-metal/container-linux/kubernetes/bootstrap.tf index b563ae148..1a7519506 100644 --- a/bare-metal/container-linux/kubernetes/bootstrap.tf +++ b/bare-metal/container-linux/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=de85f1da7df0b13dfb7488350c20a510f3090cdf" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=05297b94a936c356851e180e4963034e0047e1c0" cluster_name = var.cluster_name api_servers = [var.k8s_domain_name] diff --git a/bare-metal/container-linux/kubernetes/cl/controller.yaml b/bare-metal/container-linux/kubernetes/cl/controller.yaml index 707891d65..74f3b8fcc 100644 --- a/bare-metal/container-linux/kubernetes/cl/controller.yaml +++ b/bare-metal/container-linux/kubernetes/cl/controller.yaml @@ -101,7 +101,7 @@ systemd: --mount volume=etc-iscsi,target=/etc/iscsi \ --volume usr-sbin-iscsiadm,kind=host,source=/usr/sbin/iscsiadm \ --mount volume=usr-sbin-iscsiadm,target=/sbin/iscsiadm \ - docker://k8s.gcr.io/hyperkube:v1.17.1 \ + docker://k8s.gcr.io/hyperkube:v1.17.2 \ --exec=/usr/local/bin/kubelet -- \ --anonymous-auth=false \ --authentication-token-webhook \ @@ -147,7 +147,7 @@ systemd: --volume script,kind=host,source=/opt/bootstrap/apply \ --mount volume=script,target=/apply \ --insecure-options=image \ - docker://k8s.gcr.io/hyperkube:v1.17.1 \ + docker://k8s.gcr.io/hyperkube:v1.17.2 \ --net=host \ --dns=host \ --exec=/apply diff --git a/bare-metal/container-linux/kubernetes/cl/worker.yaml b/bare-metal/container-linux/kubernetes/cl/worker.yaml index 194468121..f07e0bb39 100644 --- a/bare-metal/container-linux/kubernetes/cl/worker.yaml +++ b/bare-metal/container-linux/kubernetes/cl/worker.yaml @@ -76,7 +76,7 @@ systemd: --mount volume=etc-iscsi,target=/etc/iscsi \ --volume usr-sbin-iscsiadm,kind=host,source=/usr/sbin/iscsiadm \ --mount volume=usr-sbin-iscsiadm,target=/sbin/iscsiadm \ - docker://k8s.gcr.io/hyperkube:v1.17.1 \ + docker://k8s.gcr.io/hyperkube:v1.17.2 \ --exec=/usr/local/bin/kubelet -- \ --anonymous-auth=false \ --authentication-token-webhook \ diff --git a/bare-metal/fedora-coreos/kubernetes/README.md b/bare-metal/fedora-coreos/kubernetes/README.md index d406ba12e..329d38bfd 100644 --- a/bare-metal/fedora-coreos/kubernetes/README.md +++ b/bare-metal/fedora-coreos/kubernetes/README.md @@ -11,7 +11,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features -* Kubernetes v1.17.1 (upstream) +* Kubernetes v1.17.2 (upstream) * Single or multi-master, [Calico](https://www.projectcalico.org/) or [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/) * Advanced features like [snippets](https://typhoon.psdn.io/advanced/customization/#container-linux) customization diff --git a/bare-metal/fedora-coreos/kubernetes/bootstrap.tf b/bare-metal/fedora-coreos/kubernetes/bootstrap.tf index 8cc0d4645..81d543fc3 100644 --- a/bare-metal/fedora-coreos/kubernetes/bootstrap.tf +++ b/bare-metal/fedora-coreos/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=de85f1da7df0b13dfb7488350c20a510f3090cdf" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=05297b94a936c356851e180e4963034e0047e1c0" cluster_name = var.cluster_name api_servers = [var.k8s_domain_name] diff --git a/bare-metal/fedora-coreos/kubernetes/fcc/controller.yaml b/bare-metal/fedora-coreos/kubernetes/fcc/controller.yaml index f428915f3..21c874805 100644 --- a/bare-metal/fedora-coreos/kubernetes/fcc/controller.yaml +++ b/bare-metal/fedora-coreos/kubernetes/fcc/controller.yaml @@ -80,7 +80,7 @@ systemd: --volume /opt/cni/bin:/opt/cni/bin:z \ --volume /etc/iscsi:/etc/iscsi \ --volume /sbin/iscsiadm:/sbin/iscsiadm \ - k8s.gcr.io/hyperkube:v1.17.1 kubelet \ + k8s.gcr.io/hyperkube:v1.17.2 kubelet \ --anonymous-auth=false \ --authentication-token-webhook \ --authorization-mode=Webhook \ @@ -134,7 +134,7 @@ systemd: --volume /opt/bootstrap/assets:/assets:ro,Z \ --volume /opt/bootstrap/apply:/apply:ro,Z \ --entrypoint=/apply \ - k8s.gcr.io/hyperkube:v1.17.1 + k8s.gcr.io/hyperkube:v1.17.2 ExecStartPost=/bin/touch /opt/bootstrap/bootstrap.done ExecStartPost=-/usr/bin/podman stop bootstrap storage: diff --git a/bare-metal/fedora-coreos/kubernetes/fcc/worker.yaml b/bare-metal/fedora-coreos/kubernetes/fcc/worker.yaml index cc39c11cb..077ca75ce 100644 --- a/bare-metal/fedora-coreos/kubernetes/fcc/worker.yaml +++ b/bare-metal/fedora-coreos/kubernetes/fcc/worker.yaml @@ -50,7 +50,7 @@ systemd: --volume /opt/cni/bin:/opt/cni/bin:z \ --volume /etc/iscsi:/etc/iscsi \ --volume /sbin/iscsiadm:/sbin/iscsiadm \ - k8s.gcr.io/hyperkube:v1.17.1 kubelet \ + k8s.gcr.io/hyperkube:v1.17.2 kubelet \ --anonymous-auth=false \ --authentication-token-webhook \ --authorization-mode=Webhook \ diff --git a/digital-ocean/container-linux/kubernetes/README.md b/digital-ocean/container-linux/kubernetes/README.md index d915164a6..0a4a61ea4 100644 --- a/digital-ocean/container-linux/kubernetes/README.md +++ b/digital-ocean/container-linux/kubernetes/README.md @@ -11,7 +11,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features -* Kubernetes v1.17.1 (upstream) +* Kubernetes v1.17.2 (upstream) * Single or multi-master, [Calico](https://www.projectcalico.org/) or [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/) * Advanced features like [snippets](https://typhoon.psdn.io/advanced/customization/#container-linux) customization diff --git a/digital-ocean/container-linux/kubernetes/bootstrap.tf b/digital-ocean/container-linux/kubernetes/bootstrap.tf index 7f7d7356d..f41d4a158 100644 --- a/digital-ocean/container-linux/kubernetes/bootstrap.tf +++ b/digital-ocean/container-linux/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=de85f1da7df0b13dfb7488350c20a510f3090cdf" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=05297b94a936c356851e180e4963034e0047e1c0" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/digital-ocean/container-linux/kubernetes/cl/controller.yaml b/digital-ocean/container-linux/kubernetes/cl/controller.yaml index de8de96da..61cde9dcf 100644 --- a/digital-ocean/container-linux/kubernetes/cl/controller.yaml +++ b/digital-ocean/container-linux/kubernetes/cl/controller.yaml @@ -99,7 +99,7 @@ systemd: --mount volume=var-log,target=/var/log \ --volume opt-cni-bin,kind=host,source=/opt/cni/bin \ --mount volume=opt-cni-bin,target=/opt/cni/bin \ - docker://k8s.gcr.io/hyperkube:v1.17.1 \ + docker://k8s.gcr.io/hyperkube:v1.17.2 \ --exec=/usr/local/bin/kubelet -- \ --anonymous-auth=false \ --authentication-token-webhook \ @@ -144,7 +144,7 @@ systemd: --volume script,kind=host,source=/opt/bootstrap/apply \ --mount volume=script,target=/apply \ --insecure-options=image \ - docker://k8s.gcr.io/hyperkube:v1.17.1 \ + docker://k8s.gcr.io/hyperkube:v1.17.2 \ --net=host \ --dns=host \ --exec=/apply diff --git a/digital-ocean/container-linux/kubernetes/cl/worker.yaml b/digital-ocean/container-linux/kubernetes/cl/worker.yaml index 3f8f485e0..c6ef316f7 100644 --- a/digital-ocean/container-linux/kubernetes/cl/worker.yaml +++ b/digital-ocean/container-linux/kubernetes/cl/worker.yaml @@ -74,7 +74,7 @@ systemd: --mount volume=var-log,target=/var/log \ --volume opt-cni-bin,kind=host,source=/opt/cni/bin \ --mount volume=opt-cni-bin,target=/opt/cni/bin \ - docker://k8s.gcr.io/hyperkube:v1.17.1 \ + docker://k8s.gcr.io/hyperkube:v1.17.2 \ --exec=/usr/local/bin/kubelet -- \ --anonymous-auth=false \ --authentication-token-webhook \ @@ -132,7 +132,7 @@ storage: --volume config,kind=host,source=/etc/kubernetes \ --mount volume=config,target=/etc/kubernetes \ --insecure-options=image \ - docker://k8s.gcr.io/hyperkube:v1.17.1 \ + docker://k8s.gcr.io/hyperkube:v1.17.2 \ --net=host \ --dns=host \ -- \ diff --git a/docs/advanced/worker-pools.md b/docs/advanced/worker-pools.md index 9f9757e08..c81ca0cbb 100644 --- a/docs/advanced/worker-pools.md +++ b/docs/advanced/worker-pools.md @@ -79,7 +79,7 @@ Create a cluster following the Azure [tutorial](../cl/azure.md#cluster). Define ```tf module "ramius-worker-pool" { - source = "git::https://github.com/poseidon/typhoon//azure/container-linux/kubernetes/workers?ref=v1.17.1" + source = "git::https://github.com/poseidon/typhoon//azure/container-linux/kubernetes/workers?ref=v1.17.2" # Azure region = module.ramius.region @@ -145,7 +145,7 @@ Create a cluster following the Google Cloud [tutorial](../cl/google-cloud.md#clu ```tf module "yavin-worker-pool" { - source = "git::https://github.com/poseidon/typhoon//google-cloud/container-linux/kubernetes/workers?ref=v1.17.1" + source = "git::https://github.com/poseidon/typhoon//google-cloud/container-linux/kubernetes/workers?ref=v1.17.2" # Google Cloud region = "europe-west2" @@ -176,11 +176,11 @@ Verify a managed instance group of workers joins the cluster within a few minute ``` $ kubectl get nodes NAME STATUS AGE VERSION -yavin-controller-0.c.example-com.internal Ready 6m v1.17.1 -yavin-worker-jrbf.c.example-com.internal Ready 5m v1.17.1 -yavin-worker-mzdm.c.example-com.internal Ready 5m v1.17.1 -yavin-16x-worker-jrbf.c.example-com.internal Ready 3m v1.17.1 -yavin-16x-worker-mzdm.c.example-com.internal Ready 3m v1.17.1 +yavin-controller-0.c.example-com.internal Ready 6m v1.17.2 +yavin-worker-jrbf.c.example-com.internal Ready 5m v1.17.2 +yavin-worker-mzdm.c.example-com.internal Ready 5m v1.17.2 +yavin-16x-worker-jrbf.c.example-com.internal Ready 3m v1.17.2 +yavin-16x-worker-mzdm.c.example-com.internal Ready 3m v1.17.2 ``` ### Variables diff --git a/docs/cl/aws.md b/docs/cl/aws.md index 5701ab622..ce5a9eeda 100644 --- a/docs/cl/aws.md +++ b/docs/cl/aws.md @@ -1,6 +1,6 @@ # AWS -In this tutorial, we'll create a Kubernetes v1.17.1 cluster on AWS with Container Linux. +In this tutorial, we'll create a Kubernetes v1.17.2 cluster on AWS with Container Linux. We'll declare a Kubernetes cluster using the Typhoon Terraform module. Then apply the changes to create a VPC, gateway, subnets, security groups, controller instances, worker auto-scaling group, network load balancer, and TLS assets. @@ -70,7 +70,7 @@ Define a Kubernetes cluster using the module `aws/container-linux/kubernetes`. ```tf module "tempest" { - source = "git::https://github.com/poseidon/typhoon//aws/container-linux/kubernetes?ref=v1.17.1" + source = "git::https://github.com/poseidon/typhoon//aws/container-linux/kubernetes?ref=v1.17.2" # AWS cluster_name = "tempest" @@ -143,9 +143,9 @@ List nodes in the cluster. $ export KUBECONFIG=/home/user/.kube/configs/tempest-config $ kubectl get nodes NAME STATUS ROLES AGE VERSION -ip-10-0-3-155 Ready 10m v1.17.1 -ip-10-0-26-65 Ready 10m v1.17.1 -ip-10-0-41-21 Ready 10m v1.17.1 +ip-10-0-3-155 Ready 10m v1.17.2 +ip-10-0-26-65 Ready 10m v1.17.2 +ip-10-0-41-21 Ready 10m v1.17.2 ``` List the pods. diff --git a/docs/cl/azure.md b/docs/cl/azure.md index a5211f6ea..3c191fdc8 100644 --- a/docs/cl/azure.md +++ b/docs/cl/azure.md @@ -3,7 +3,7 @@ !!! danger Typhoon for Azure is alpha. For production, use AWS, Google Cloud, or bare-metal. As Azure matures, check [errata](https://github.com/poseidon/typhoon/wiki/Errata) for known shortcomings. -In this tutorial, we'll create a Kubernetes v1.17.1 cluster on Azure with Container Linux. +In this tutorial, we'll create a Kubernetes v1.17.2 cluster on Azure with Container Linux. We'll declare a Kubernetes cluster using the Typhoon Terraform module. Then apply the changes to create a resource group, virtual network, subnets, security groups, controller availability set, worker scale set, load balancer, and TLS assets. @@ -66,7 +66,7 @@ Define a Kubernetes cluster using the module `azure/container-linux/kubernetes`. ```tf module "ramius" { - source = "git::https://github.com/poseidon/typhoon//azure/container-linux/kubernetes?ref=v1.17.1" + source = "git::https://github.com/poseidon/typhoon//azure/container-linux/kubernetes?ref=v1.17.2" # Azure cluster_name = "ramius" @@ -140,9 +140,9 @@ List nodes in the cluster. $ export KUBECONFIG=/home/user/.kube/configs/ramius-config $ kubectl get nodes NAME STATUS ROLES AGE VERSION -ramius-controller-0 Ready 24m v1.17.1 -ramius-worker-000001 Ready 25m v1.17.1 -ramius-worker-000002 Ready 24m v1.17.1 +ramius-controller-0 Ready 24m v1.17.2 +ramius-worker-000001 Ready 25m v1.17.2 +ramius-worker-000002 Ready 24m v1.17.2 ``` List the pods. diff --git a/docs/cl/bare-metal.md b/docs/cl/bare-metal.md index 3e6dddac0..e1b2990c9 100644 --- a/docs/cl/bare-metal.md +++ b/docs/cl/bare-metal.md @@ -1,6 +1,6 @@ # Bare-Metal -In this tutorial, we'll network boot and provision a Kubernetes v1.17.1 cluster on bare-metal with Container Linux. +In this tutorial, we'll network boot and provision a Kubernetes v1.17.2 cluster on bare-metal with Container Linux. First, we'll deploy a [Matchbox](https://github.com/poseidon/matchbox) service and setup a network boot environment. Then, we'll declare a Kubernetes cluster using the Typhoon Terraform module and power on machines. On PXE boot, machines will install Container Linux to disk, reboot into the disk install, and provision themselves as Kubernetes controllers or workers via Ignition. @@ -160,7 +160,7 @@ Define a Kubernetes cluster using the module `bare-metal/container-linux/kuberne ```tf module "mercury" { - source = "git::https://github.com/poseidon/typhoon//bare-metal/container-linux/kubernetes?ref=v1.17.1" + source = "git::https://github.com/poseidon/typhoon//bare-metal/container-linux/kubernetes?ref=v1.17.2" # bare-metal cluster_name = "mercury" @@ -299,9 +299,9 @@ List nodes in the cluster. $ export KUBECONFIG=/home/user/.kube/configs/mercury-config $ kubectl get nodes NAME STATUS ROLES AGE VERSION -node1.example.com Ready 10m v1.17.1 -node2.example.com Ready 10m v1.17.1 -node3.example.com Ready 10m v1.17.1 +node1.example.com Ready 10m v1.17.2 +node2.example.com Ready 10m v1.17.2 +node3.example.com Ready 10m v1.17.2 ``` List the pods. diff --git a/docs/cl/digital-ocean.md b/docs/cl/digital-ocean.md index 988c3ce32..2e9c61005 100644 --- a/docs/cl/digital-ocean.md +++ b/docs/cl/digital-ocean.md @@ -1,6 +1,6 @@ # Digital Ocean -In this tutorial, we'll create a Kubernetes v1.17.1 cluster on DigitalOcean with Container Linux. +In this tutorial, we'll create a Kubernetes v1.17.2 cluster on DigitalOcean with Container Linux. We'll declare a Kubernetes cluster using the Typhoon Terraform module. Then apply the changes to create controller droplets, worker droplets, DNS records, tags, and TLS assets. @@ -65,7 +65,7 @@ Define a Kubernetes cluster using the module `digital-ocean/container-linux/kube ```tf module "nemo" { - source = "git::https://github.com/poseidon/typhoon//digital-ocean/container-linux/kubernetes?ref=v1.17.1" + source = "git::https://github.com/poseidon/typhoon//digital-ocean/container-linux/kubernetes?ref=v1.17.2" # Digital Ocean cluster_name = "nemo" @@ -138,9 +138,9 @@ List nodes in the cluster. $ export KUBECONFIG=/home/user/.kube/configs/nemo-config $ kubectl get nodes NAME STATUS ROLES AGE VERSION -10.132.110.130 Ready 10m v1.17.1 -10.132.115.81 Ready 10m v1.17.1 -10.132.124.107 Ready 10m v1.17.1 +10.132.110.130 Ready 10m v1.17.2 +10.132.115.81 Ready 10m v1.17.2 +10.132.124.107 Ready 10m v1.17.2 ``` List the pods. diff --git a/docs/cl/google-cloud.md b/docs/cl/google-cloud.md index cdc8accb1..31867f134 100644 --- a/docs/cl/google-cloud.md +++ b/docs/cl/google-cloud.md @@ -1,6 +1,6 @@ # Google Cloud -In this tutorial, we'll create a Kubernetes v1.17.1 cluster on Google Compute Engine with Container Linux. +In this tutorial, we'll create a Kubernetes v1.17.2 cluster on Google Compute Engine with Container Linux. We'll declare a Kubernetes cluster using the Typhoon Terraform module. Then apply the changes to create a network, firewall rules, health checks, controller instances, worker managed instance group, load balancers, and TLS assets. @@ -71,7 +71,7 @@ Define a Kubernetes cluster using the module `google-cloud/container-linux/kuber ```tf module "yavin" { - source = "git::https://github.com/poseidon/typhoon//google-cloud/container-linux/kubernetes?ref=v1.17.1" + source = "git::https://github.com/poseidon/typhoon//google-cloud/container-linux/kubernetes?ref=v1.17.2" # Google Cloud cluster_name = "yavin" @@ -145,9 +145,9 @@ List nodes in the cluster. $ export KUBECONFIG=/home/user/.kube/configs/yavin-config $ kubectl get nodes NAME ROLES STATUS AGE VERSION -yavin-controller-0.c.example-com.internal Ready 6m v1.17.1 -yavin-worker-jrbf.c.example-com.internal Ready 5m v1.17.1 -yavin-worker-mzdm.c.example-com.internal Ready 5m v1.17.1 +yavin-controller-0.c.example-com.internal Ready 6m v1.17.2 +yavin-worker-jrbf.c.example-com.internal Ready 5m v1.17.2 +yavin-worker-mzdm.c.example-com.internal Ready 5m v1.17.2 ``` List the pods. diff --git a/docs/fedora-coreos/aws.md b/docs/fedora-coreos/aws.md index f36424ebf..88881b80e 100644 --- a/docs/fedora-coreos/aws.md +++ b/docs/fedora-coreos/aws.md @@ -3,7 +3,7 @@ !!! danger Typhoon for Fedora CoreOS is an early preview! Fedora CoreOS itself is a preview! Expect bugs and design shifts. Please help both projects solve problems. Report Fedora CoreOS bugs to [Fedora](https://github.com/coreos/fedora-coreos-tracker/issues). Report Typhoon issues to Typhoon. -In this tutorial, we'll create a Kubernetes v1.17.1 cluster on AWS with Fedora CoreOS. +In this tutorial, we'll create a Kubernetes v1.17.2 cluster on AWS with Fedora CoreOS. We'll declare a Kubernetes cluster using the Typhoon Terraform module. Then apply the changes to create a VPC, gateway, subnets, security groups, controller instances, worker auto-scaling group, network load balancer, and TLS assets. @@ -73,7 +73,7 @@ Define a Kubernetes cluster using the module `aws/fedora-coreos/kubernetes`. ```tf module "tempest" { - source = "git::https://github.com/poseidon/typhoon//aws/fedora-coreos/kubernetes?ref=v1.17.1" + source = "git::https://github.com/poseidon/typhoon//aws/fedora-coreos/kubernetes?ref=v1.17.2" # AWS cluster_name = "tempest" @@ -146,9 +146,9 @@ List nodes in the cluster. $ export KUBECONFIG=/home/user/.kube/configs/tempest-config $ kubectl get nodes NAME STATUS ROLES AGE VERSION -ip-10-0-3-155 Ready 10m v1.17.1 -ip-10-0-26-65 Ready 10m v1.17.1 -ip-10-0-41-21 Ready 10m v1.17.1 +ip-10-0-3-155 Ready 10m v1.17.2 +ip-10-0-26-65 Ready 10m v1.17.2 +ip-10-0-41-21 Ready 10m v1.17.2 ``` List the pods. diff --git a/docs/fedora-coreos/bare-metal.md b/docs/fedora-coreos/bare-metal.md index 05ec55a5e..7cdac24c5 100644 --- a/docs/fedora-coreos/bare-metal.md +++ b/docs/fedora-coreos/bare-metal.md @@ -3,7 +3,7 @@ !!! danger Typhoon for Fedora CoreOS is an early preview! Fedora CoreOS itself is a preview! Expect bugs and design shifts. Please help both projects solve problems. Report Fedora CoreOS bugs to [Fedora](https://github.com/coreos/fedora-coreos-tracker/issues). Report Typhoon issues to Typhoon. -In this tutorial, we'll network boot and provision a Kubernetes v1.17.1 cluster on bare-metal with Fedora CoreOS. +In this tutorial, we'll network boot and provision a Kubernetes v1.17.2 cluster on bare-metal with Fedora CoreOS. First, we'll deploy a [Matchbox](https://github.com/poseidon/matchbox) service and setup a network boot environment. Then, we'll declare a Kubernetes cluster using the Typhoon Terraform module and power on machines. On PXE boot, machines will install Fedora CoreOS to disk, reboot into the disk install, and provision themselves as Kubernetes controllers or workers via Ignition. @@ -163,7 +163,7 @@ Define a Kubernetes cluster using the module `bare-metal/fedora-coreos/kubernete ```tf module "mercury" { - source = "git::https://github.com/poseidon/typhoon//bare-metal/fedora-coreos/kubernetes?ref=v1.17.1" + source = "git::https://github.com/poseidon/typhoon//bare-metal/fedora-coreos/kubernetes?ref=v1.17.2" # bare-metal cluster_name = "mercury" @@ -292,9 +292,9 @@ List nodes in the cluster. $ export KUBECONFIG=/home/user/.kube/configs/mercury-config $ kubectl get nodes NAME STATUS ROLES AGE VERSION -node1.example.com Ready 10m v1.17.1 -node2.example.com Ready 10m v1.17.1 -node3.example.com Ready 10m v1.17.1 +node1.example.com Ready 10m v1.17.2 +node2.example.com Ready 10m v1.17.2 +node3.example.com Ready 10m v1.17.2 ``` List the pods. diff --git a/docs/index.md b/docs/index.md index 0a4b4504e..30e745a31 100644 --- a/docs/index.md +++ b/docs/index.md @@ -11,7 +11,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features -* Kubernetes v1.17.1 (upstream) +* Kubernetes v1.17.2 (upstream) * Single or multi-master, [Calico](https://www.projectcalico.org/) or [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/) * Advanced features like [worker pools](advanced/worker-pools/), [preemptible](cl/google-cloud/#preemption) workers, and [snippets](advanced/customization/#container-linux) customization @@ -47,7 +47,7 @@ Define a Kubernetes cluster by using the Terraform module for your chosen platfo ```tf module "yavin" { - source = "git::https://github.com/poseidon/typhoon//google-cloud/container-linux/kubernetes?ref=v1.17.1" + source = "git::https://github.com/poseidon/typhoon//google-cloud/container-linux/kubernetes?ref=v1.17.2" # Google Cloud cluster_name = "yavin" @@ -85,9 +85,9 @@ In 4-8 minutes (varies by platform), the cluster will be ready. This Google Clou $ export KUBECONFIG=/home/user/.kube/configs/yavin-config $ kubectl get nodes NAME ROLES STATUS AGE VERSION -yavin-controller-0.c.example-com.internal Ready 6m v1.17.1 -yavin-worker-jrbf.c.example-com.internal Ready 5m v1.17.1 -yavin-worker-mzdm.c.example-com.internal Ready 5m v1.17.1 +yavin-controller-0.c.example-com.internal Ready 6m v1.17.2 +yavin-worker-jrbf.c.example-com.internal Ready 5m v1.17.2 +yavin-worker-mzdm.c.example-com.internal Ready 5m v1.17.2 ``` List the pods. diff --git a/docs/topics/maintenance.md b/docs/topics/maintenance.md index 524818940..b583d1be1 100644 --- a/docs/topics/maintenance.md +++ b/docs/topics/maintenance.md @@ -18,7 +18,7 @@ module "yavin" { } module "mercury" { - source = "git::https://github.com/poseidon/typhoon//bare-metal/container-linux/kubernetes?ref=v1.17.1" + source = "git::https://github.com/poseidon/typhoon//bare-metal/container-linux/kubernetes?ref=v1.17.2" ... } ``` @@ -279,15 +279,15 @@ Typhoon modules have been adapted for Terraform v0.12. Provider plugins requirem | Typhoon Release | Terraform version | |-------------------|---------------------| -| v1.17.1 - ? | v0.12.x | -| v1.10.3 - v1.17.1 | v0.11.x | +| v1.17.2 - ? | v0.12.x | +| v1.10.3 - v1.17.2 | v0.11.x | | v1.9.2 - v1.10.2 | v0.10.4+ or v0.11.x | | v1.7.3 - v1.9.1 | v0.10.x | | v1.6.4 - v1.7.2 | v0.9.x | ### New users -New users can start with Terraform v0.12.x and follow the docs for Typhoon v1.17.1+ without issue. +New users can start with Terraform v0.12.x and follow the docs for Typhoon v1.17.2+ without issue. ### Existing users diff --git a/google-cloud/container-linux/kubernetes/README.md b/google-cloud/container-linux/kubernetes/README.md index b94a44463..cfb94a30f 100644 --- a/google-cloud/container-linux/kubernetes/README.md +++ b/google-cloud/container-linux/kubernetes/README.md @@ -11,7 +11,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features -* Kubernetes v1.17.1 (upstream) +* Kubernetes v1.17.2 (upstream) * Single or multi-master, [Calico](https://www.projectcalico.org/) or [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/) * Advanced features like [worker pools](https://typhoon.psdn.io/advanced/worker-pools/), [preemptible](https://typhoon.psdn.io/cl/google-cloud/#preemption) workers, and [snippets](https://typhoon.psdn.io/advanced/customization/#container-linux) customization diff --git a/google-cloud/container-linux/kubernetes/bootstrap.tf b/google-cloud/container-linux/kubernetes/bootstrap.tf index 883dfd3b1..56ebceeaa 100644 --- a/google-cloud/container-linux/kubernetes/bootstrap.tf +++ b/google-cloud/container-linux/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=de85f1da7df0b13dfb7488350c20a510f3090cdf" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=05297b94a936c356851e180e4963034e0047e1c0" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/google-cloud/container-linux/kubernetes/cl/controller.yaml b/google-cloud/container-linux/kubernetes/cl/controller.yaml index 07fc36a26..8e005c6c3 100644 --- a/google-cloud/container-linux/kubernetes/cl/controller.yaml +++ b/google-cloud/container-linux/kubernetes/cl/controller.yaml @@ -88,7 +88,7 @@ systemd: --mount volume=var-log,target=/var/log \ --volume opt-cni-bin,kind=host,source=/opt/cni/bin \ --mount volume=opt-cni-bin,target=/opt/cni/bin \ - docker://k8s.gcr.io/hyperkube:v1.17.1 \ + docker://k8s.gcr.io/hyperkube:v1.17.2 \ --exec=/usr/local/bin/kubelet -- \ --anonymous-auth=false \ --authentication-token-webhook \ @@ -132,7 +132,7 @@ systemd: --volume script,kind=host,source=/opt/bootstrap/apply \ --mount volume=script,target=/apply \ --insecure-options=image \ - docker://k8s.gcr.io/hyperkube:v1.17.1 \ + docker://k8s.gcr.io/hyperkube:v1.17.2 \ --net=host \ --dns=host \ --exec=/apply diff --git a/google-cloud/container-linux/kubernetes/workers/cl/worker.yaml b/google-cloud/container-linux/kubernetes/workers/cl/worker.yaml index 9d70dd386..1cfb34f74 100644 --- a/google-cloud/container-linux/kubernetes/workers/cl/worker.yaml +++ b/google-cloud/container-linux/kubernetes/workers/cl/worker.yaml @@ -63,7 +63,7 @@ systemd: --mount volume=var-log,target=/var/log \ --volume opt-cni-bin,kind=host,source=/opt/cni/bin \ --mount volume=opt-cni-bin,target=/opt/cni/bin \ - docker://k8s.gcr.io/hyperkube:v1.17.1 \ + docker://k8s.gcr.io/hyperkube:v1.17.2 \ --exec=/usr/local/bin/kubelet -- \ --anonymous-auth=false \ --authentication-token-webhook \ @@ -126,7 +126,7 @@ storage: --volume config,kind=host,source=/etc/kubernetes \ --mount volume=config,target=/etc/kubernetes \ --insecure-options=image \ - docker://k8s.gcr.io/hyperkube:v1.17.1 \ + docker://k8s.gcr.io/hyperkube:v1.17.2 \ --net=host \ --dns=host \ -- \ From d5b7ce8f2773d9c530a352c3884c4b32f4076eb4 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Thu, 23 Jan 2020 00:03:16 -0800 Subject: [PATCH 318/523] Update kube-state-metrics from v1.9.2 to v1.9.3 * https://github.com/kubernetes/kube-state-metrics/releases/tag/v1.9.3 --- CHANGES.md | 3 ++- addons/prometheus/exporters/kube-state-metrics/deployment.yaml | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 480eb62f1..f901d55b5 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -16,8 +16,9 @@ Notable changes between versions. #### Addons -* Update nginx-ingress from v0.26.1 to [v0.27.1](https://github.com/kubernetes/ingress-nginx/releases/tag/nginx-0.27.1) () +* Update nginx-ingress from v0.26.1 to [v0.27.1](https://github.com/kubernetes/ingress-nginx/releases/tag/nginx-0.27.1) ([#625](https://github.com/poseidon/typhoon/pull/625)) * Change runAsUser from 33 to 101 for alpine-based image +* Update kube-state-metrics from v1.9.2 to v1.9.3 ## v1.17.1 diff --git a/addons/prometheus/exporters/kube-state-metrics/deployment.yaml b/addons/prometheus/exporters/kube-state-metrics/deployment.yaml index 55fe0432f..ab7c62394 100644 --- a/addons/prometheus/exporters/kube-state-metrics/deployment.yaml +++ b/addons/prometheus/exporters/kube-state-metrics/deployment.yaml @@ -24,7 +24,7 @@ spec: serviceAccountName: kube-state-metrics containers: - name: kube-state-metrics - image: quay.io/coreos/kube-state-metrics:v1.9.2 + image: quay.io/coreos/kube-state-metrics:v1.9.3 ports: - name: metrics containerPort: 8080 From 5643ad525fce51a399972d6a0b0750e564732061 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Thu, 23 Jan 2020 01:09:09 -0800 Subject: [PATCH 319/523] Promote Fedora CoreOS from preview to alpha in docs * Add an announcement to the website as well --- CHANGES.md | 5 +++++ README.md | 8 ++++---- docs/announce.md | 14 +++++++++++--- docs/fedora-coreos/aws.md | 2 +- docs/fedora-coreos/bare-metal.md | 2 +- docs/index.md | 8 ++++---- 6 files changed, 26 insertions(+), 13 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index f901d55b5..c70ac8ff2 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -8,8 +8,13 @@ Notable changes between versions. * Kubernetes [v1.17.2](https://github.com/kubernetes/kubernetes/blob/master/CHANGELOG-1.17.md#v1172) +#### AWS + +* Promote Fedora CoreOS from preview to alpha + #### Bare-Metal +* Promote Fedora CoreOS from preview to alpha * Update Fedora CoreOS images location * Use Fedora CoreOS production [download](https://getfedora.org/coreos/download/) streams * Use live PXE kernel and initramfs images diff --git a/README.md b/README.md index 01290424a..db1847f13 100644 --- a/README.md +++ b/README.md @@ -29,12 +29,12 @@ Typhoon provides a Terraform Module for each supported operating system and plat | Digital Ocean | Container Linux | [digital-ocean/container-linux/kubernetes](digital-ocean/container-linux/kubernetes) | beta | | Google Cloud | Container Linux | [google-cloud/container-linux/kubernetes](google-cloud/container-linux/kubernetes) | stable | -A preview of Typhoon for [Fedora CoreOS](https://getfedora.org/coreos/) is available for testing. +Typhoon is available for [Fedora CoreOS](https://getfedora.org/coreos/) in alpha. | Platform | Operating System | Terraform Module | Status | |---------------|------------------|------------------|--------| -| AWS | Fedora CoreOS | [aws/fedora-coreos/kubernetes](aws/fedora-coreos/kubernetes) | preview | -| Bare-Metal | Fedora CoreOS | [bare-metal/fedora-coreos/kubernetes](bare-metal/fedora-coreos/kubernetes) | preview | +| AWS | Fedora CoreOS | [aws/fedora-coreos/kubernetes](aws/fedora-coreos/kubernetes) | alpha | +| Bare-Metal | Fedora CoreOS | [bare-metal/fedora-coreos/kubernetes](bare-metal/fedora-coreos/kubernetes) | alpha | ## Documentation @@ -58,7 +58,7 @@ module "yavin" { # configuration ssh_authorized_key = "ssh-rsa AAAAB3Nz..." - + # optional worker_count = 2 worker_preemptible = true diff --git a/docs/announce.md b/docs/announce.md index 864bade71..4b41f4721 100644 --- a/docs/announce.md +++ b/docs/announce.md @@ -1,5 +1,15 @@ # Announce +## Jan 23rd, 2020 + +Typhoon for Fedora CoreOS promoted to alpha! + +Last summer, Typhoon released the first preview of Kubernetes on Fedora CoreOS for bare-metal and AWS, developing many ideas and patterns from Typhoon for Container Linux and Fedora Atomic. Since then, Typhoon for Fedora CoreOS has evolved and gained features alongside Typhoon, while Fedora CoreOS itself has evolved and improved too. + +Fedora recently [announced](https://fedoramagazine.org/fedora-coreos-out-of-preview/) that Fedora CoreOS is available for general use. To align with that change and to better indicate the maturing status, Typhoon for Fedora CoreOS has been promoted to alpha. Many thanks to folks who have worked to make this possbile! + +About: For newcomers, Typhoon is a minimal and free (cost and freedom) Kubernetes distribution providing upstream Kubernetes, declarative configuration via Terraform, and support for AWS, Azure, Google Cloud, DigitalOcean, and bare-metal. It is run by former CoreOS engineer [@dghubble](https://twitter.com/dghubble) to power his clusters, with freedom [motivations](https://typhoon.psdn.io/#motivation). + ## Jul 18, 2019 Introducing a preview of Typhoon Kubernetes clusters with Fedora CoreOS! @@ -8,8 +18,6 @@ Fedora recently [announced](https://lists.fedoraproject.org/archives/list/coreos While Typhoon uses Container Linux (or Flatcar Linux) for stable modules, the project hasn't been a stranger to Fedora ideas, once developing a [Fedora Atomic](https://typhoon.psdn.io/announce/#april-26-2018) variant in 2018. That makes the Fedora CoreOS fushion both exciting and familiar. Typhoon with Fedora CoreOS uses Ignition v3 for provisioning, uses rpm-ostree for layering and updates, tries swapping system containers for podman, and brings SELinux enforcement ([table](https://typhoon.psdn.io/architecture/operating-systems/)). This is an early preview (don't go to prod), but do try it out and help identify and solve issues (getting started links above). -About: For newcomers, Typhoon is a minimal and free (cost and freedom) Kubernetes distribution providing upstream Kubernetes, declarative configuration via Terraform, and support for AWS, Azure, Google Cloud, DigitalOcean, and bare-metal. It is run by former CoreOS engineer [@dghubble](https://twitter.com/dghubble) to power his clusters with freedom [motivations](https://typhoon.psdn.io/#motivation). - ## March 27, 2019 Last April, Typhoon [introduced](#april-26-2018) alpha support for creating Kubernetes clusters with Fedora Atomic on AWS, Google Cloud, DigitalOcean, and bare-metal. Fedora Atomic shared many of Container Linux's aims for a container-optimized operating system, introduced novel ideas, and provided technical diversification for an uncertain future. However, Project Atomic efforts were merged into Fedora CoreOS and future Fedora Atomic releases are [not expected](http://www.projectatomic.io/blog/2018/06/welcome-to-fedora-coreos/). *Typhoon modules for Fedora Atomic will not be updated much beyond Kubernetes v1.13*. They may later be removed. @@ -46,7 +54,7 @@ Typhoon for Fedora Atomic reflects many of the same principles that created Typh Meanwhile, Fedora Atomic adds some promising new low-level technologies: -* [ostree](https://github.com/ostreedev/ostree) & [rpm-ostree](https://github.com/projectatomic/rpm-ostree) - a hybrid, layered, image and package system that lets you perform atomic updates and rollbacks, layer on packages, "rebase" your system, or manage a remote tree repo. See Dusty Mabe's great [intro](https://dustymabe.com/2017/09/01/atomic-host-101-lab-part-3-rebase-upgrade-rollback/). +* [ostree](https://github.com/ostreedev/ostree) & [rpm-ostree](https://github.com/projectatomic/rpm-ostree) - a hybrid, layered, image and package system that lets you perform atomic updates and rollbacks, layer on packages, "rebase" your system, or manage a remote tree repo. See Dusty Mabe's great [intro](https://dustymabe.com/2017/09/01/atomic-host-101-lab-part-3-rebase-upgrade-rollback/). * [system containers](http://www.projectatomic.io/blog/2016/09/intro-to-system-containers/) - OCI container images that embed systemd and runc metadata for starting low-level host services before container runtimes are ready. Typhoon uses system containers under runc for `etcd`, `kubelet`, and `bootkube` on Fedora Atomic (instead of rkt-fly). diff --git a/docs/fedora-coreos/aws.md b/docs/fedora-coreos/aws.md index 88881b80e..5676db0ed 100644 --- a/docs/fedora-coreos/aws.md +++ b/docs/fedora-coreos/aws.md @@ -1,7 +1,7 @@ # AWS !!! danger - Typhoon for Fedora CoreOS is an early preview! Fedora CoreOS itself is a preview! Expect bugs and design shifts. Please help both projects solve problems. Report Fedora CoreOS bugs to [Fedora](https://github.com/coreos/fedora-coreos-tracker/issues). Report Typhoon issues to Typhoon. + Typhoon for Fedora CoreOS is an alpha. Please report Fedora CoreOS bugs to [Fedora](https://github.com/coreos/fedora-coreos-tracker/issues) and Typhoon issues to Typhoon. In this tutorial, we'll create a Kubernetes v1.17.2 cluster on AWS with Fedora CoreOS. diff --git a/docs/fedora-coreos/bare-metal.md b/docs/fedora-coreos/bare-metal.md index 7cdac24c5..580c5022a 100644 --- a/docs/fedora-coreos/bare-metal.md +++ b/docs/fedora-coreos/bare-metal.md @@ -1,7 +1,7 @@ # Bare-Metal !!! danger - Typhoon for Fedora CoreOS is an early preview! Fedora CoreOS itself is a preview! Expect bugs and design shifts. Please help both projects solve problems. Report Fedora CoreOS bugs to [Fedora](https://github.com/coreos/fedora-coreos-tracker/issues). Report Typhoon issues to Typhoon. + Typhoon for Fedora CoreOS is an alpha. Please report Fedora CoreOS bugs to [Fedora](https://github.com/coreos/fedora-coreos-tracker/issues) and Typhoon issues to Typhoon. In this tutorial, we'll network boot and provision a Kubernetes v1.17.2 cluster on bare-metal with Fedora CoreOS. diff --git a/docs/index.md b/docs/index.md index 30e745a31..f5658363f 100644 --- a/docs/index.md +++ b/docs/index.md @@ -29,12 +29,12 @@ Typhoon provides a Terraform Module for each supported operating system and plat | Digital Ocean | Container Linux | [digital-ocean/container-linux/kubernetes](cl/digital-ocean.md) | beta | | Google Cloud | Container Linux | [google-cloud/container-linux/kubernetes](cl/google-cloud.md) | stable | -A preview of Typhoon for [Fedora CoreOS](https://getfedora.org/coreos/) is available for testing. +Typhoon is available for [Fedora CoreOS](https://getfedora.org/coreos/) in alpha. | Platform | Operating System | Terraform Module | Status | |---------------|------------------|------------------|--------| -| AWS | Fedora CoreOS | [aws/fedora-coreos/kubernetes](fedora-coreos/aws.md) | preview | -| Bare-Metal | Fedora CoreOS | [bare-metal/fedora-coreos/kubernetes](fedora-coreos/bare-metal.md) | preview | +| AWS | Fedora CoreOS | [aws/fedora-coreos/kubernetes](fedora-coreos/aws.md) | alpha | +| Bare-Metal | Fedora CoreOS | [bare-metal/fedora-coreos/kubernetes](fedora-coreos/bare-metal.md) | alpha | ## Documentation @@ -57,7 +57,7 @@ module "yavin" { # configuration ssh_authorized_key = "ssh-rsa AAAAB3Nz..." - + # optional worker_count = 2 } From 02a470d2f2f3e2bda6624829606f32f5035c56ed Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Thu, 23 Jan 2020 08:57:01 -0800 Subject: [PATCH 320/523] Fix minor typo in announcement date --- docs/announce.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/announce.md b/docs/announce.md index 4b41f4721..8e54bd0db 100644 --- a/docs/announce.md +++ b/docs/announce.md @@ -1,6 +1,6 @@ # Announce -## Jan 23rd, 2020 +## Jan 23, 2020 Typhoon for Fedora CoreOS promoted to alpha! From d127a7345cd367e092225456e8571ee082e2cda2 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Mon, 27 Jan 2020 20:46:32 -0800 Subject: [PATCH 321/523] Update Grafana from v6.5.3 to v6.6.0 * https://github.com/grafana/grafana/releases/tag/v6.6.0 --- CHANGES.md | 4 ++++ addons/grafana/deployment.yaml | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index c70ac8ff2..864af8408 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -4,6 +4,10 @@ Notable changes between versions. ## Latest +#### Addons + +* Update Grafana from v6.5.3 to v6.6.0 + ## v1.17.2 * Kubernetes [v1.17.2](https://github.com/kubernetes/kubernetes/blob/master/CHANGELOG-1.17.md#v1172) diff --git a/addons/grafana/deployment.yaml b/addons/grafana/deployment.yaml index a255fd47e..1639b42e3 100644 --- a/addons/grafana/deployment.yaml +++ b/addons/grafana/deployment.yaml @@ -23,7 +23,7 @@ spec: spec: containers: - name: grafana - image: docker.io/grafana/grafana:6.5.3 + image: docker.io/grafana/grafana:6.6.0 env: - name: GF_PATHS_CONFIG value: "/etc/grafana/custom.ini" From b19ba16afa6f80a39304afb9cc8139343c71d54b Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Thu, 30 Jan 2020 18:00:23 -0800 Subject: [PATCH 322/523] Update nginx-ingress from v0.27.1 to v0.28.0 * https://github.com/kubernetes/ingress-nginx/releases/tag/nginx-0.28.0 --- CHANGES.md | 1 + addons/nginx-ingress/aws/deployment.yaml | 2 +- addons/nginx-ingress/azure/deployment.yaml | 2 +- addons/nginx-ingress/bare-metal/deployment.yaml | 2 +- addons/nginx-ingress/digital-ocean/daemonset.yaml | 2 +- addons/nginx-ingress/google-cloud/deployment.yaml | 2 +- 6 files changed, 6 insertions(+), 5 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 864af8408..ca455b5dd 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -6,6 +6,7 @@ Notable changes between versions. #### Addons +* Update nginx-ingress from v0.27.1 to v0.28.0 * Update Grafana from v6.5.3 to v6.6.0 ## v1.17.2 diff --git a/addons/nginx-ingress/aws/deployment.yaml b/addons/nginx-ingress/aws/deployment.yaml index 9a70c565e..5bfd2df3c 100644 --- a/addons/nginx-ingress/aws/deployment.yaml +++ b/addons/nginx-ingress/aws/deployment.yaml @@ -22,7 +22,7 @@ spec: spec: containers: - name: nginx-ingress-controller - image: quay.io/kubernetes-ingress-controller/nginx-ingress-controller:0.27.1 + image: quay.io/kubernetes-ingress-controller/nginx-ingress-controller:0.28.0 args: - /nginx-ingress-controller - --ingress-class=public diff --git a/addons/nginx-ingress/azure/deployment.yaml b/addons/nginx-ingress/azure/deployment.yaml index 9a70c565e..5bfd2df3c 100644 --- a/addons/nginx-ingress/azure/deployment.yaml +++ b/addons/nginx-ingress/azure/deployment.yaml @@ -22,7 +22,7 @@ spec: spec: containers: - name: nginx-ingress-controller - image: quay.io/kubernetes-ingress-controller/nginx-ingress-controller:0.27.1 + image: quay.io/kubernetes-ingress-controller/nginx-ingress-controller:0.28.0 args: - /nginx-ingress-controller - --ingress-class=public diff --git a/addons/nginx-ingress/bare-metal/deployment.yaml b/addons/nginx-ingress/bare-metal/deployment.yaml index b872a428c..7d342024f 100644 --- a/addons/nginx-ingress/bare-metal/deployment.yaml +++ b/addons/nginx-ingress/bare-metal/deployment.yaml @@ -22,7 +22,7 @@ spec: spec: containers: - name: nginx-ingress-controller - image: quay.io/kubernetes-ingress-controller/nginx-ingress-controller:0.27.1 + image: quay.io/kubernetes-ingress-controller/nginx-ingress-controller:0.28.0 args: - /nginx-ingress-controller - --ingress-class=public diff --git a/addons/nginx-ingress/digital-ocean/daemonset.yaml b/addons/nginx-ingress/digital-ocean/daemonset.yaml index fe9c8f6a6..0f2167dc1 100644 --- a/addons/nginx-ingress/digital-ocean/daemonset.yaml +++ b/addons/nginx-ingress/digital-ocean/daemonset.yaml @@ -22,7 +22,7 @@ spec: spec: containers: - name: nginx-ingress-controller - image: quay.io/kubernetes-ingress-controller/nginx-ingress-controller:0.27.1 + image: quay.io/kubernetes-ingress-controller/nginx-ingress-controller:0.28.0 args: - /nginx-ingress-controller - --ingress-class=public diff --git a/addons/nginx-ingress/google-cloud/deployment.yaml b/addons/nginx-ingress/google-cloud/deployment.yaml index 9a70c565e..5bfd2df3c 100644 --- a/addons/nginx-ingress/google-cloud/deployment.yaml +++ b/addons/nginx-ingress/google-cloud/deployment.yaml @@ -22,7 +22,7 @@ spec: spec: containers: - name: nginx-ingress-controller - image: quay.io/kubernetes-ingress-controller/nginx-ingress-controller:0.27.1 + image: quay.io/kubernetes-ingress-controller/nginx-ingress-controller:0.28.0 args: - /nginx-ingress-controller - --ingress-class=public From 8cc303c9ac685bae2bca6ba0e34fc88779f321c2 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Sat, 1 Feb 2020 15:10:40 -0800 Subject: [PATCH 323/523] Add module for Fedora CoreOS on Google Cloud * Add Typhoon Fedora CoreOS on Google Cloud as alpha * Add docs on uploading the Fedora CoreOS GCP gzipped tarball to Google Cloud storage to create a boot disk image --- CHANGES.md | 4 + README.md | 1 + docs/fedora-coreos/google-cloud.md | 254 ++++++++++++++++++ docs/index.md | 1 + google-cloud/fedora-coreos/kubernetes/LICENSE | 23 ++ .../fedora-coreos/kubernetes/README.md | 23 ++ .../fedora-coreos/kubernetes/apiserver.tf | 93 +++++++ .../fedora-coreos/kubernetes/bootstrap.tf | 22 ++ .../fedora-coreos/kubernetes/controllers.tf | 103 +++++++ .../kubernetes/fcc/controller.yaml | 216 +++++++++++++++ .../fedora-coreos/kubernetes/ingress.tf | 123 +++++++++ .../fedora-coreos/kubernetes/network.tf | 193 +++++++++++++ .../fedora-coreos/kubernetes/outputs.tf | 44 +++ google-cloud/fedora-coreos/kubernetes/ssh.tf | 58 ++++ .../fedora-coreos/kubernetes/variables.tf | 138 ++++++++++ .../fedora-coreos/kubernetes/versions.tf | 11 + .../fedora-coreos/kubernetes/workers.tf | 23 ++ .../kubernetes/workers/fcc/worker.yaml | 110 ++++++++ .../kubernetes/workers/outputs.tf | 14 + .../kubernetes/workers/target_pool.tf | 23 ++ .../kubernetes/workers/variables.tf | 106 ++++++++ .../kubernetes/workers/versions.tf | 4 + .../kubernetes/workers/workers.tf | 91 +++++++ google-cloud/ignore/.gitkeep | 0 mkdocs.yml | 7 +- 25 files changed, 1682 insertions(+), 3 deletions(-) create mode 100644 docs/fedora-coreos/google-cloud.md create mode 100644 google-cloud/fedora-coreos/kubernetes/LICENSE create mode 100644 google-cloud/fedora-coreos/kubernetes/README.md create mode 100644 google-cloud/fedora-coreos/kubernetes/apiserver.tf create mode 100644 google-cloud/fedora-coreos/kubernetes/bootstrap.tf create mode 100644 google-cloud/fedora-coreos/kubernetes/controllers.tf create mode 100644 google-cloud/fedora-coreos/kubernetes/fcc/controller.yaml create mode 100644 google-cloud/fedora-coreos/kubernetes/ingress.tf create mode 100644 google-cloud/fedora-coreos/kubernetes/network.tf create mode 100644 google-cloud/fedora-coreos/kubernetes/outputs.tf create mode 100644 google-cloud/fedora-coreos/kubernetes/ssh.tf create mode 100644 google-cloud/fedora-coreos/kubernetes/variables.tf create mode 100644 google-cloud/fedora-coreos/kubernetes/versions.tf create mode 100644 google-cloud/fedora-coreos/kubernetes/workers.tf create mode 100644 google-cloud/fedora-coreos/kubernetes/workers/fcc/worker.yaml create mode 100644 google-cloud/fedora-coreos/kubernetes/workers/outputs.tf create mode 100644 google-cloud/fedora-coreos/kubernetes/workers/target_pool.tf create mode 100644 google-cloud/fedora-coreos/kubernetes/workers/variables.tf create mode 100644 google-cloud/fedora-coreos/kubernetes/workers/versions.tf create mode 100644 google-cloud/fedora-coreos/kubernetes/workers/workers.tf delete mode 100644 google-cloud/ignore/.gitkeep diff --git a/CHANGES.md b/CHANGES.md index ca455b5dd..7f8d34cfd 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -4,6 +4,10 @@ Notable changes between versions. ## Latest +#### Google Cloud + +* Add Terraform module for Fedora CoreOS ([#632](https://github.com/poseidon/typhoon/pull/632)) + #### Addons * Update nginx-ingress from v0.27.1 to v0.28.0 diff --git a/README.md b/README.md index db1847f13..6d06e9c01 100644 --- a/README.md +++ b/README.md @@ -35,6 +35,7 @@ Typhoon is available for [Fedora CoreOS](https://getfedora.org/coreos/) in alpha |---------------|------------------|------------------|--------| | AWS | Fedora CoreOS | [aws/fedora-coreos/kubernetes](aws/fedora-coreos/kubernetes) | alpha | | Bare-Metal | Fedora CoreOS | [bare-metal/fedora-coreos/kubernetes](bare-metal/fedora-coreos/kubernetes) | alpha | +| Google Cloud | Fedora CoreOS | [google-cloud/fedora-coreos/kubernetes](google-cloud/fedora-coreos/kubernetes) | alpha | ## Documentation diff --git a/docs/fedora-coreos/google-cloud.md b/docs/fedora-coreos/google-cloud.md new file mode 100644 index 000000000..5df392675 --- /dev/null +++ b/docs/fedora-coreos/google-cloud.md @@ -0,0 +1,254 @@ +# Google Cloud + +!!! danger + Typhoon for Fedora CoreOS is an alpha. Please report Fedora CoreOS bugs to [Fedora](https://github.com/coreos/fedora-coreos-tracker/issues) and Typhoon issues to Typhoon. + +In this tutorial, we'll create a Kubernetes v1.17.2 cluster on Google Compute Engine with Fedora CoreOS. + +We'll declare a Kubernetes cluster using the Typhoon Terraform module. Then apply the changes to create a network, firewall rules, health checks, controller instances, worker managed instance group, load balancers, and TLS assets. + +Controller hosts are provisioned to run an `etcd-member` peer and a `kubelet` service. Worker hosts run a `kubelet` service. Controller nodes run `kube-apiserver`, `kube-scheduler`, `kube-controller-manager`, and `coredns`, while `kube-proxy` and `calico` (or `flannel`) run on every node. A generated `kubeconfig` provides `kubectl` access to the cluster. + +## Requirements + +* Google Cloud Account and Service Account +* Google Cloud DNS Zone (registered Domain Name or delegated subdomain) +* Terraform v0.12.6+ and [terraform-provider-ct](https://github.com/poseidon/terraform-provider-ct) installed locally + +## Terraform Setup + +Install [Terraform](https://www.terraform.io/downloads.html) v0.12.6+ on your system. + +```sh +$ terraform version +Terraform v0.12.16 +``` + +Add the [terraform-provider-ct](https://github.com/poseidon/terraform-provider-ct) plugin binary for your system to `~/.terraform.d/plugins/`, noting the final name. + +```sh +wget https://github.com/poseidon/terraform-provider-ct/releases/download/v0.4.0/terraform-provider-ct-v0.4.0-linux-amd64.tar.gz +tar xzf terraform-provider-ct-v0.4.0-linux-amd64.tar.gz +mv terraform-provider-ct-v0.4.0-linux-amd64/terraform-provider-ct ~/.terraform.d/plugins/terraform-provider-ct_v0.4.0 +``` + +Read [concepts](/architecture/concepts/) to learn about Terraform, modules, and organizing resources. Change to your infrastructure repository (e.g. `infra`). + +``` +cd infra/clusters +``` + +## Provider + +Login to your Google Console [API Manager](https://console.cloud.google.com/apis/dashboard) and select a project, or [signup](https://cloud.google.com/free/) if you don't have an account. + +Select "Credentials" and create a service account key. Choose the "Compute Engine Admin" and "DNS Administrator" roles and save the JSON private key to a file that can be referenced in configs. + +```sh +mv ~/Downloads/project-id-43048204.json ~/.config/google-cloud/terraform.json +``` + +Configure the Google Cloud provider to use your service account key, project-id, and region in a `providers.tf` file. + +```tf +provider "google" { + version = "3.4.0" + project = "project-id" + region = "us-central1" + credentials = file("~/.config/google-cloud/terraform.json") +} + +provider "ct" { + version = "0.4.0" +} +``` + +Additional configuration options are described in the `google` provider [docs](https://www.terraform.io/docs/providers/google/index.html). + +!!! tip + Regions are listed in [docs](https://cloud.google.com/compute/docs/regions-zones/regions-zones) or with `gcloud compute regions list`. A project may contain multiple clusters across different regions. + +## Fedora CoreOS Images + +Fedora CoreOS publishes images for Google Cloud, but does not yet upload them. Google Cloud allows [custom boot images](https://cloud.google.com/compute/docs/images/import-existing-image) to be uploaded to a bucket and imported into your project. + +[Download](https://getfedora.org/coreos/download/) the Fedora CoreOS GCP gzipped tarball. Then upload the file to a GCS storage bucket. + +``` +gsutil list +gsutil cp fedora-coreos-31.20200113.3.1-gcp.x86_64.tar.gz gs://BUCKET_NAME +``` + +Create a Google Compute Engine image from the bucket file. + +``` +gcloud compute images create fedora-coreos-31-20200113-3-1 --source-uri gs://BUCKET/fedora-coreos-31.20200113.3.1-gcp.x86_64.tar.gz +``` + +## Cluster + +Define a Kubernetes cluster using the module `google-cloud/fedora-coreos/kubernetes`. + +```tf +module "yavin" { + source = "git::https://github.com/poseidon/typhoon//google-cloud/fedora-coreos/kubernetes?ref=development-sha" + + # Google Cloud + cluster_name = "yavin" + region = "us-central1" + dns_zone = "example.com" + dns_zone_name = "example-zone" + # temporary + os_image = "fedora-coreos-31-20200113-3-1" + + # configuration + ssh_authorized_key = "ssh-rsa AAAAB3Nz..." + + # optional + worker_count = 2 +} +``` + +Reference the [variables docs](#variables) or the [variables.tf](https://github.com/poseidon/typhoon/blob/master/google-cloud/container-linux/kubernetes/variables.tf) source. + +## ssh-agent + +Initial bootstrapping requires `bootstrap.service` be started on one controller node. Terraform uses `ssh-agent` to automate this step. Add your SSH private key to `ssh-agent`. + +```sh +ssh-add ~/.ssh/id_rsa +ssh-add -L +``` + +## Apply + +Initialize the config directory if this is the first use with Terraform. + +```sh +terraform init +``` + +Plan the resources to be created. + +```sh +$ terraform plan +Plan: 64 to add, 0 to change, 0 to destroy. +``` + +Apply the changes to create the cluster. + +```sh +$ terraform apply +module.yavin.null_resource.bootstrap: Still creating... (10s elapsed) +... +module.yavin.null_resource.bootstrap: Still creating... (5m30s elapsed) +module.yavin.null_resource.bootstrap: Still creating... (5m40s elapsed) +module.yavin.null_resource.bootstrap: Creation complete (ID: 5768638456220583358) + +Apply complete! Resources: 62 added, 0 changed, 0 destroyed. +``` + +In 4-8 minutes, the Kubernetes cluster will be ready. + +## Verify + +[Install kubectl](https://kubernetes.io/docs/tasks/tools/install-kubectl/) on your system. Obtain the generated cluster `kubeconfig` from module outputs (e.g. write to a local file). + +``` +resource "local_file" "kubeconfig-yavin" { + content = module.yavin.kubeconfig-admin + filename = "/home/user/.kube/configs/yavin-config" +} +``` + +List nodes in the cluster. + +``` +$ export KUBECONFIG=/home/user/.kube/configs/yavin-config +$ kubectl get nodes +NAME ROLES STATUS AGE VERSION +yavin-controller-0.c.example-com.internal Ready 6m v1.17.2 +yavin-worker-jrbf.c.example-com.internal Ready 5m v1.17.2 +yavin-worker-mzdm.c.example-com.internal Ready 5m v1.17.2 +``` + +List the pods. + +``` +$ kubectl get pods --all-namespaces +NAMESPACE NAME READY STATUS RESTARTS AGE +kube-system calico-node-1cs8z 2/2 Running 0 6m +kube-system calico-node-d1l5b 2/2 Running 0 6m +kube-system calico-node-sp9ps 2/2 Running 0 6m +kube-system coredns-1187388186-dkh3o 1/1 Running 0 6m +kube-system coredns-1187388186-zj5dl 1/1 Running 0 6m +kube-system kube-apiserver-controller-0 1/1 Running 0 6m +kube-system kube-controller-manager-controller-0 1/1 Running 0 6m +kube-system kube-proxy-117v6 1/1 Running 0 6m +kube-system kube-proxy-9886n 1/1 Running 0 6m +kube-system kube-proxy-njn47 1/1 Running 0 6m +kube-system kube-scheduler-controller-0 1/1 Running 0 6m +``` + +## Going Further + +Learn about [maintenance](/topics/maintenance/) and [addons](/addons/overview/). + +## Variables + +Check the [variables.tf](https://github.com/poseidon/typhoon/blob/master/google-cloud/container-linux/kubernetes/variables.tf) source. + +### Required + +| Name | Description | Example | +|:-----|:------------|:--------| +| cluster_name | Unique cluster name (prepended to dns_zone) | "yavin" | +| region | Google Cloud region | "us-central1" | +| dns_zone | Google Cloud DNS zone | "google-cloud.example.com" | +| dns_zone_name | Google Cloud DNS zone name | "example-zone" | +| ssh_authorized_key | SSH public key for user 'core' | "ssh-rsa AAAAB3NZ..." | + +Check the list of valid [regions](https://cloud.google.com/compute/docs/regions-zones/regions-zones) and list Fedora CoreOS [images](https://cloud.google.com/compute/docs/images) with `gcloud compute images list | grep fedora-coreos`. + +#### DNS Zone + +Clusters create a DNS A record `${cluster_name}.${dns_zone}` to resolve a TCP proxy load balancer backed by controller instances. This FQDN is used by workers and `kubectl` to access the apiserver(s). In this example, the cluster's apiserver would be accessible at `yavin.google-cloud.example.com`. + +You'll need a registered domain name or delegated subdomain on Google Cloud DNS. You can set this up once and create many clusters with unique names. + +```tf +resource "google_dns_managed_zone" "zone-for-clusters" { + dns_name = "google-cloud.example.com." + name = "example-zone" + description = "Production DNS zone" +} +``` + +!!! tip "" + If you have an existing domain name with a zone file elsewhere, just delegate a subdomain that can be managed on Google Cloud (e.g. google-cloud.mydomain.com) and [update nameservers](https://cloud.google.com/dns/update-name-servers). + +### Optional + +| Name | Description | Default | Example | +|:-----|:------------|:--------|:--------| +| asset_dir | Absolute path to a directory where generated assets should be placed (contains secrets) | "" (disabled) | "/home/user/.secrets/clusters/yavin" | +| controller_count | Number of controllers (i.e. masters) | 1 | 3 | +| worker_count | Number of workers | 1 | 3 | +| controller_type | Machine type for controllers | "n1-standard-1" | See below | +| worker_type | Machine type for workers | "n1-standard-1" | See below | +| os_image | Fedora CoreOS image for compute instances | "" | "fedora-coreos-31-20200113-3-1" | +| disk_size | Size of the disk in GB | 40 | 100 | +| worker_preemptible | If enabled, Compute Engine will terminate workers randomly within 24 hours | false | true | +| controller_snippets | Controller Fedora CoreOS Config snippets | [] | UNSUPPORTED | +| worker_snippets | Worker Fedora CoreOS Config snippets | [] | UNSUPPORTED | +| networking | Choice of networking provider | "calico" | "calico" or "flannel" | +| pod_cidr | CIDR IPv4 range to assign to Kubernetes pods | "10.2.0.0/16" | "10.22.0.0/16" | +| service_cidr | CIDR IPv4 range to assign to Kubernetes services | "10.3.0.0/16" | "10.3.0.0/24" | +| worker_node_labels | List of initial worker node labels | [] | ["worker-pool=default"] | + +Check the list of valid [machine types](https://cloud.google.com/compute/docs/machine-types). + +#### Preemption + +Add `worker_preemptible = "true"` to allow worker nodes to be [preempted](https://cloud.google.com/compute/docs/instances/preemptible) at random, but pay [significantly](https://cloud.google.com/compute/pricing) less. Clusters tolerate stopping instances fairly well (reschedules pods, but cannot drain) and preemption provides a nice reward for running fault-tolerant cluster systems.` + diff --git a/docs/index.md b/docs/index.md index f5658363f..038440f08 100644 --- a/docs/index.md +++ b/docs/index.md @@ -35,6 +35,7 @@ Typhoon is available for [Fedora CoreOS](https://getfedora.org/coreos/) in alpha |---------------|------------------|------------------|--------| | AWS | Fedora CoreOS | [aws/fedora-coreos/kubernetes](fedora-coreos/aws.md) | alpha | | Bare-Metal | Fedora CoreOS | [bare-metal/fedora-coreos/kubernetes](fedora-coreos/bare-metal.md) | alpha | +| Google Cloud | Fedora CoreOS | [google-cloud/fedora-coreos/kubernetes](google-cloud/fedora-coreos/kubernetes) | alpha | ## Documentation diff --git a/google-cloud/fedora-coreos/kubernetes/LICENSE b/google-cloud/fedora-coreos/kubernetes/LICENSE new file mode 100644 index 000000000..658b1c469 --- /dev/null +++ b/google-cloud/fedora-coreos/kubernetes/LICENSE @@ -0,0 +1,23 @@ +The MIT License (MIT) + +Copyright (c) 2020 Typhoon Authors +Copyright (c) 2020 Dalton Hubble + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. + diff --git a/google-cloud/fedora-coreos/kubernetes/README.md b/google-cloud/fedora-coreos/kubernetes/README.md new file mode 100644 index 000000000..cfb94a30f --- /dev/null +++ b/google-cloud/fedora-coreos/kubernetes/README.md @@ -0,0 +1,23 @@ +# Typhoon + +Typhoon is a minimal and free Kubernetes distribution. + +* Minimal, stable base Kubernetes distribution +* Declarative infrastructure and configuration +* Free (freedom and cost) and privacy-respecting +* Practical for labs, datacenters, and clouds + +Typhoon distributes upstream Kubernetes, architectural conventions, and cluster addons, much like a GNU/Linux distribution provides the Linux kernel and userspace components. + +## Features + +* Kubernetes v1.17.2 (upstream) +* Single or multi-master, [Calico](https://www.projectcalico.org/) or [flannel](https://github.com/coreos/flannel) networking +* On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/) +* Advanced features like [worker pools](https://typhoon.psdn.io/advanced/worker-pools/), [preemptible](https://typhoon.psdn.io/cl/google-cloud/#preemption) workers, and [snippets](https://typhoon.psdn.io/advanced/customization/#container-linux) customization +* Ready for Ingress, Prometheus, Grafana, and other optional [addons](https://typhoon.psdn.io/addons/overview/) + +## Docs + +Please see the [official docs](https://typhoon.psdn.io) and the Google Cloud [tutorial](https://typhoon.psdn.io/cl/google-cloud/). + diff --git a/google-cloud/fedora-coreos/kubernetes/apiserver.tf b/google-cloud/fedora-coreos/kubernetes/apiserver.tf new file mode 100644 index 000000000..318d9c1fb --- /dev/null +++ b/google-cloud/fedora-coreos/kubernetes/apiserver.tf @@ -0,0 +1,93 @@ +# TCP Proxy load balancer DNS record +resource "google_dns_record_set" "apiserver" { + # DNS Zone name where record should be created + managed_zone = var.dns_zone_name + + # DNS record + name = format("%s.%s.", var.cluster_name, var.dns_zone) + type = "A" + ttl = 300 + + # IPv4 address of apiserver TCP Proxy load balancer + rrdatas = [google_compute_global_address.apiserver-ipv4.address] +} + +# Static IPv4 address for the TCP Proxy Load Balancer +resource "google_compute_global_address" "apiserver-ipv4" { + name = "${var.cluster_name}-apiserver-ip" + ip_version = "IPV4" +} + +# Forward IPv4 TCP traffic to the TCP proxy load balancer +resource "google_compute_global_forwarding_rule" "apiserver" { + name = "${var.cluster_name}-apiserver" + ip_address = google_compute_global_address.apiserver-ipv4.address + ip_protocol = "TCP" + port_range = "443" + target = google_compute_target_tcp_proxy.apiserver.self_link +} + +# Global TCP Proxy Load Balancer for apiservers +resource "google_compute_target_tcp_proxy" "apiserver" { + name = "${var.cluster_name}-apiserver" + description = "Distribute TCP load across ${var.cluster_name} controllers" + backend_service = google_compute_backend_service.apiserver.self_link +} + +# Global backend service backed by unmanaged instance groups +resource "google_compute_backend_service" "apiserver" { + name = "${var.cluster_name}-apiserver" + description = "${var.cluster_name} apiserver service" + + protocol = "TCP" + port_name = "apiserver" + session_affinity = "NONE" + timeout_sec = "300" + + # controller(s) spread across zonal instance groups + dynamic "backend" { + for_each = google_compute_instance_group.controllers + content { + group = backend.value.self_link + } + } + + health_checks = [google_compute_health_check.apiserver.self_link] +} + +# Instance group of heterogeneous (unmanged) controller instances +resource "google_compute_instance_group" "controllers" { + count = min(var.controller_count, length(local.zones)) + + name = format("%s-controllers-%s", var.cluster_name, element(local.zones, count.index)) + zone = element(local.zones, count.index) + + named_port { + name = "apiserver" + port = "6443" + } + + # add instances in the zone into the instance group + instances = matchkeys( + google_compute_instance.controllers.*.self_link, + google_compute_instance.controllers.*.zone, + [element(local.zones, count.index)], + ) +} + +# TCP health check for apiserver +resource "google_compute_health_check" "apiserver" { + name = "${var.cluster_name}-apiserver-tcp-health" + description = "TCP health check for kube-apiserver" + + timeout_sec = 5 + check_interval_sec = 5 + + healthy_threshold = 1 + unhealthy_threshold = 3 + + tcp_health_check { + port = "6443" + } +} + diff --git a/google-cloud/fedora-coreos/kubernetes/bootstrap.tf b/google-cloud/fedora-coreos/kubernetes/bootstrap.tf new file mode 100644 index 000000000..4bb6217aa --- /dev/null +++ b/google-cloud/fedora-coreos/kubernetes/bootstrap.tf @@ -0,0 +1,22 @@ +# Kubernetes assets (kubeconfig, manifests) +module "bootstrap" { + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=05297b94a936c356851e180e4963034e0047e1c0" + + cluster_name = var.cluster_name + api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] + etcd_servers = google_dns_record_set.etcds.*.name + asset_dir = var.asset_dir + networking = var.networking + network_mtu = 1440 + pod_cidr = var.pod_cidr + service_cidr = var.service_cidr + cluster_domain_suffix = var.cluster_domain_suffix + enable_reporting = var.enable_reporting + enable_aggregation = var.enable_aggregation + + trusted_certs_dir = "/etc/pki/tls/certs" + + // temporary + external_apiserver_port = 443 +} + diff --git a/google-cloud/fedora-coreos/kubernetes/controllers.tf b/google-cloud/fedora-coreos/kubernetes/controllers.tf new file mode 100644 index 000000000..b2cde4341 --- /dev/null +++ b/google-cloud/fedora-coreos/kubernetes/controllers.tf @@ -0,0 +1,103 @@ +# Discrete DNS records for each controller's private IPv4 for etcd usage +resource "google_dns_record_set" "etcds" { + count = var.controller_count + + # DNS Zone name where record should be created + managed_zone = var.dns_zone_name + + # DNS record + name = format("%s-etcd%d.%s.", var.cluster_name, count.index, var.dns_zone) + type = "A" + ttl = 300 + + # private IPv4 address for etcd + rrdatas = [google_compute_instance.controllers.*.network_interface.0.network_ip[count.index]] +} + +# Zones in the region +data "google_compute_zones" "all" { + region = var.region +} + +locals { + zones = data.google_compute_zones.all.names + + controllers_ipv4_public = google_compute_instance.controllers.*.network_interface.0.access_config.0.nat_ip +} + +# Controller instances +resource "google_compute_instance" "controllers" { + count = var.controller_count + + name = "${var.cluster_name}-controller-${count.index}" + # use a zone in the region and wrap around (e.g. controllers > zones) + zone = element(local.zones, count.index) + machine_type = var.controller_type + + metadata = { + user-data = data.ct_config.controller-ignitions.*.rendered[count.index] + } + + boot_disk { + auto_delete = true + + initialize_params { + image = var.os_image + size = var.disk_size + } + } + + network_interface { + network = google_compute_network.network.name + + # Ephemeral external IP + access_config { + } + } + + can_ip_forward = true + tags = ["${var.cluster_name}-controller"] + + lifecycle { + ignore_changes = [metadata] + } +} + +# Controller Ignition configs +data "ct_config" "controller-ignitions" { + count = var.controller_count + content = data.template_file.controller-configs.*.rendered[count.index] + strict = true + snippets = var.controller_snippets +} + +# Controller Fedora CoreOS configs +data "template_file" "controller-configs" { + count = var.controller_count + + template = file("${path.module}/fcc/controller.yaml") + + vars = { + # Cannot use cyclic dependencies on controllers or their DNS records + etcd_name = "etcd${count.index}" + etcd_domain = "${var.cluster_name}-etcd${count.index}.${var.dns_zone}" + # etcd0=https://cluster-etcd0.example.com,etcd1=https://cluster-etcd1.example.com,... + etcd_initial_cluster = join(",", data.template_file.etcds.*.rendered) + kubeconfig = indent(10, module.bootstrap.kubeconfig-kubelet) + ssh_authorized_key = var.ssh_authorized_key + cluster_dns_service_ip = cidrhost(var.service_cidr, 10) + cluster_domain_suffix = var.cluster_domain_suffix + } +} + +data "template_file" "etcds" { + count = var.controller_count + template = "etcd$${index}=https://$${cluster_name}-etcd$${index}.$${dns_zone}:2380" + + vars = { + index = count.index + cluster_name = var.cluster_name + dns_zone = var.dns_zone + } +} + diff --git a/google-cloud/fedora-coreos/kubernetes/fcc/controller.yaml b/google-cloud/fedora-coreos/kubernetes/fcc/controller.yaml new file mode 100644 index 000000000..df9338823 --- /dev/null +++ b/google-cloud/fedora-coreos/kubernetes/fcc/controller.yaml @@ -0,0 +1,216 @@ +--- +variant: fcos +version: 1.0.0 +systemd: + units: + - name: etcd-member.service + enabled: true + contents: | + [Unit] + Description=etcd (System Container) + Documentation=https://github.com/coreos/etcd + Wants=network-online.target network.target + After=network-online.target + [Service] + # https://github.com/opencontainers/runc/pull/1807 + # Type=notify + # NotifyAccess=exec + Type=exec + Restart=on-failure + RestartSec=10s + TimeoutStartSec=0 + LimitNOFILE=40000 + ExecStartPre=/bin/mkdir -p /var/lib/etcd + ExecStartPre=-/usr/bin/podman rm etcd + #--volume $${NOTIFY_SOCKET}:/run/systemd/notify \ + ExecStart=/usr/bin/podman run --name etcd \ + --env-file /etc/etcd/etcd.env \ + --network host \ + --volume /var/lib/etcd:/var/lib/etcd:rw,Z \ + --volume /etc/ssl/etcd:/etc/ssl/certs:ro,Z \ + quay.io/coreos/etcd:v3.4.3 + ExecStop=/usr/bin/podman stop etcd + [Install] + WantedBy=multi-user.target + - name: docker.service + enabled: true + - name: wait-for-dns.service + enabled: true + contents: | + [Unit] + Description=Wait for DNS entries + Before=kubelet.service + [Service] + Type=oneshot + RemainAfterExit=true + ExecStart=/bin/sh -c 'while ! /usr/bin/grep '^[^#[:space:]]' /etc/resolv.conf > /dev/null; do sleep 1; done' + [Install] + RequiredBy=kubelet.service + RequiredBy=etcd-member.service + - name: kubelet.service + enabled: true + contents: | + [Unit] + Description=Kubelet via Hyperkube (System Container) + Wants=rpc-statd.service + [Service] + ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d + ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests + ExecStartPre=/bin/mkdir -p /opt/cni/bin + ExecStartPre=/bin/mkdir -p /var/lib/calico + ExecStartPre=/bin/mkdir -p /var/lib/kubelet/volumeplugins + ExecStartPre=/usr/bin/bash -c "grep 'certificate-authority-data' /etc/kubernetes/kubeconfig | awk '{print $2}' | base64 -d > /etc/kubernetes/ca.crt" + ExecStartPre=-/usr/bin/podman rm kubelet + ExecStart=/usr/bin/podman run --name kubelet \ + --privileged \ + --pid host \ + --network host \ + --volume /etc/kubernetes:/etc/kubernetes:ro,z \ + --volume /usr/lib/os-release:/etc/os-release:ro \ + --volume /etc/ssl/certs:/etc/ssl/certs:ro \ + --volume /lib/modules:/lib/modules:ro \ + --volume /run:/run \ + --volume /sys/fs/cgroup:/sys/fs/cgroup:ro \ + --volume /sys/fs/cgroup/systemd:/sys/fs/cgroup/systemd \ + --volume /etc/pki/tls/certs:/usr/share/ca-certificates:ro \ + --volume /var/lib/calico:/var/lib/calico \ + --volume /var/lib/docker:/var/lib/docker \ + --volume /var/lib/kubelet:/var/lib/kubelet:rshared,z \ + --volume /var/log:/var/log \ + --volume /var/run/lock:/var/run/lock:z \ + --volume /opt/cni/bin:/opt/cni/bin:z \ + k8s.gcr.io/hyperkube:v1.17.2 kubelet \ + --anonymous-auth=false \ + --authentication-token-webhook \ + --authorization-mode=Webhook \ + --cgroup-driver=systemd \ + --cgroups-per-qos=true \ + --enforce-node-allocatable=pods \ + --client-ca-file=/etc/kubernetes/ca.crt \ + --cluster_dns=${cluster_dns_service_ip} \ + --cluster_domain=${cluster_domain_suffix} \ + --cni-conf-dir=/etc/kubernetes/cni/net.d \ + --exit-on-lock-contention \ + --healthz-port=0 \ + --kubeconfig=/etc/kubernetes/kubeconfig \ + --lock-file=/var/run/lock/kubelet.lock \ + --network-plugin=cni \ + --node-labels=node.kubernetes.io/master \ + --node-labels=node.kubernetes.io/controller="true" \ + --pod-manifest-path=/etc/kubernetes/manifests \ + --read-only-port=0 \ + --register-with-taints=node-role.kubernetes.io/master=:NoSchedule \ + --volume-plugin-dir=/var/lib/kubelet/volumeplugins + ExecStop=-/usr/bin/podman stop kubelet + Delegate=yes + Restart=always + RestartSec=10 + [Install] + WantedBy=multi-user.target + - name: bootstrap.service + contents: | + [Unit] + Description=Kubernetes control plane + ConditionPathExists=!/opt/bootstrap/bootstrap.done + [Service] + Type=oneshot + RemainAfterExit=true + WorkingDirectory=/opt/bootstrap + ExecStartPre=-/usr/bin/bash -c 'set -x && [ -n "$(ls /opt/bootstrap/assets/manifests-*/* 2>/dev/null)" ] && mv /opt/bootstrap/assets/manifests-*/* /opt/bootstrap/assets/manifests && rm -rf /opt/bootstrap/assets/manifests-*' + ExecStart=/usr/bin/podman run --name bootstrap \ + --network host \ + --volume /etc/kubernetes/bootstrap-secrets:/etc/kubernetes/secrets:ro,Z \ + --volume /opt/bootstrap/assets:/assets:ro,Z \ + --volume /opt/bootstrap/apply:/apply:ro,Z \ + --entrypoint=/apply \ + k8s.gcr.io/hyperkube:v1.17.2 + ExecStartPost=/bin/touch /opt/bootstrap/bootstrap.done + ExecStartPost=-/usr/bin/podman stop bootstrap +storage: + directories: + - path: /etc/kubernetes + - path: /opt/bootstrap + files: + - path: /etc/kubernetes/kubeconfig + mode: 0644 + contents: + inline: | + ${kubeconfig} + - path: /opt/bootstrap/layout + mode: 0544 + contents: + inline: | + #!/bin/bash -e + mkdir -p -- auth tls/etcd tls/k8s static-manifests manifests/coredns manifests-networking + awk '/#####/ {filename=$2; next} {print > filename}' assets + mkdir -p /etc/ssl/etcd/etcd + mkdir -p /etc/kubernetes/bootstrap-secrets + mv tls/etcd/{peer*,server*} /etc/ssl/etcd/etcd/ + mv tls/etcd/etcd-client* /etc/kubernetes/bootstrap-secrets/ + chown -R etcd:etcd /etc/ssl/etcd + chmod -R 500 /etc/ssl/etcd + mv auth/kubeconfig /etc/kubernetes/bootstrap-secrets/ + mv tls/k8s/* /etc/kubernetes/bootstrap-secrets/ + sudo mkdir -p /etc/kubernetes/manifests + sudo mv static-manifests/* /etc/kubernetes/manifests/ + sudo mkdir -p /opt/bootstrap/assets + sudo mv manifests /opt/bootstrap/assets/manifests + sudo mv manifests-networking /opt/bootstrap/assets/manifests-networking + rm -rf assets auth static-manifests tls + - path: /opt/bootstrap/apply + mode: 0544 + contents: + inline: | + #!/bin/bash -e + export KUBECONFIG=/etc/kubernetes/secrets/kubeconfig + until kubectl version; do + echo "Waiting for static pod control plane" + sleep 5 + done + until kubectl apply -f /assets/manifests -R; do + echo "Retry applying manifests" + sleep 5 + done + - path: /etc/sysctl.d/reverse-path-filter.conf + contents: + inline: | + net.ipv4.conf.all.rp_filter=1 + - path: /etc/sysctl.d/max-user-watches.conf + contents: + inline: | + fs.inotify.max_user_watches=16184 + - path: /etc/systemd/system.conf.d/accounting.conf + contents: + inline: | + [Manager] + DefaultCPUAccounting=yes + DefaultMemoryAccounting=yes + DefaultBlockIOAccounting=yes + - path: /etc/etcd/etcd.env + mode: 0644 + contents: + inline: | + # TODO: Use a systemd dropin once podman v1.4.5 is avail. + NOTIFY_SOCKET=/run/systemd/notify + ETCD_NAME=${etcd_name} + ETCD_DATA_DIR=/var/lib/etcd + ETCD_ADVERTISE_CLIENT_URLS=https://${etcd_domain}:2379 + ETCD_INITIAL_ADVERTISE_PEER_URLS=https://${etcd_domain}:2380 + ETCD_LISTEN_CLIENT_URLS=https://0.0.0.0:2379 + ETCD_LISTEN_PEER_URLS=https://0.0.0.0:2380 + ETCD_LISTEN_METRICS_URLS=http://0.0.0.0:2381 + ETCD_INITIAL_CLUSTER=${etcd_initial_cluster} + ETCD_STRICT_RECONFIG_CHECK=true + ETCD_TRUSTED_CA_FILE=/etc/ssl/certs/etcd/server-ca.crt + ETCD_CERT_FILE=/etc/ssl/certs/etcd/server.crt + ETCD_KEY_FILE=/etc/ssl/certs/etcd/server.key + ETCD_CLIENT_CERT_AUTH=true + ETCD_PEER_TRUSTED_CA_FILE=/etc/ssl/certs/etcd/peer-ca.crt + ETCD_PEER_CERT_FILE=/etc/ssl/certs/etcd/peer.crt + ETCD_PEER_KEY_FILE=/etc/ssl/certs/etcd/peer.key + ETCD_PEER_CLIENT_CERT_AUTH=true +passwd: + users: + - name: core + ssh_authorized_keys: + - ${ssh_authorized_key} diff --git a/google-cloud/fedora-coreos/kubernetes/ingress.tf b/google-cloud/fedora-coreos/kubernetes/ingress.tf new file mode 100644 index 000000000..71b57a082 --- /dev/null +++ b/google-cloud/fedora-coreos/kubernetes/ingress.tf @@ -0,0 +1,123 @@ +# Static IPv4 address for Ingress Load Balancing +resource "google_compute_global_address" "ingress-ipv4" { + name = "${var.cluster_name}-ingress-ipv4" + ip_version = "IPV4" +} + +# Static IPv6 address for Ingress Load Balancing +resource "google_compute_global_address" "ingress-ipv6" { + name = "${var.cluster_name}-ingress-ipv6" + ip_version = "IPV6" +} + +# Forward IPv4 TCP traffic to the HTTP proxy load balancer +# Google Cloud does not allow TCP proxies for port 80. Must use HTTP proxy. +resource "google_compute_global_forwarding_rule" "ingress-http-ipv4" { + name = "${var.cluster_name}-ingress-http-ipv4" + ip_address = google_compute_global_address.ingress-ipv4.address + ip_protocol = "TCP" + port_range = "80" + target = google_compute_target_http_proxy.ingress-http.self_link +} + +# Forward IPv4 TCP traffic to the TCP proxy load balancer +resource "google_compute_global_forwarding_rule" "ingress-https-ipv4" { + name = "${var.cluster_name}-ingress-https-ipv4" + ip_address = google_compute_global_address.ingress-ipv4.address + ip_protocol = "TCP" + port_range = "443" + target = google_compute_target_tcp_proxy.ingress-https.self_link +} + +# Forward IPv6 TCP traffic to the HTTP proxy load balancer +# Google Cloud does not allow TCP proxies for port 80. Must use HTTP proxy. +resource "google_compute_global_forwarding_rule" "ingress-http-ipv6" { + name = "${var.cluster_name}-ingress-http-ipv6" + ip_address = google_compute_global_address.ingress-ipv6.address + ip_protocol = "TCP" + port_range = "80" + target = google_compute_target_http_proxy.ingress-http.self_link +} + +# Forward IPv6 TCP traffic to the TCP proxy load balancer +resource "google_compute_global_forwarding_rule" "ingress-https-ipv6" { + name = "${var.cluster_name}-ingress-https-ipv6" + ip_address = google_compute_global_address.ingress-ipv6.address + ip_protocol = "TCP" + port_range = "443" + target = google_compute_target_tcp_proxy.ingress-https.self_link +} + +# HTTP proxy load balancer for ingress controllers +resource "google_compute_target_http_proxy" "ingress-http" { + name = "${var.cluster_name}-ingress-http" + description = "Distribute HTTP load across ${var.cluster_name} workers" + url_map = google_compute_url_map.ingress-http.self_link +} + +# TCP proxy load balancer for ingress controllers +resource "google_compute_target_tcp_proxy" "ingress-https" { + name = "${var.cluster_name}-ingress-https" + description = "Distribute HTTPS load across ${var.cluster_name} workers" + backend_service = google_compute_backend_service.ingress-https.self_link +} + +# HTTP URL Map (required) +resource "google_compute_url_map" "ingress-http" { + name = "${var.cluster_name}-ingress-http" + + # Do not add host/path rules for applications here. Use Ingress resources. + default_service = google_compute_backend_service.ingress-http.self_link +} + +# Backend service backed by managed instance group of workers +resource "google_compute_backend_service" "ingress-http" { + name = "${var.cluster_name}-ingress-http" + description = "${var.cluster_name} ingress service" + + protocol = "HTTP" + port_name = "http" + session_affinity = "NONE" + timeout_sec = "60" + + backend { + group = module.workers.instance_group + } + + health_checks = [google_compute_health_check.ingress.self_link] +} + +# Backend service backed by managed instance group of workers +resource "google_compute_backend_service" "ingress-https" { + name = "${var.cluster_name}-ingress-https" + description = "${var.cluster_name} ingress service" + + protocol = "TCP" + port_name = "https" + session_affinity = "NONE" + timeout_sec = "60" + + backend { + group = module.workers.instance_group + } + + health_checks = [google_compute_health_check.ingress.self_link] +} + +# Ingress HTTP Health Check +resource "google_compute_health_check" "ingress" { + name = "${var.cluster_name}-ingress-health" + description = "Health check for Ingress controller" + + timeout_sec = 5 + check_interval_sec = 5 + + healthy_threshold = 2 + unhealthy_threshold = 4 + + http_health_check { + port = 10254 + request_path = "/healthz" + } +} + diff --git a/google-cloud/fedora-coreos/kubernetes/network.tf b/google-cloud/fedora-coreos/kubernetes/network.tf new file mode 100644 index 000000000..bd7067d7b --- /dev/null +++ b/google-cloud/fedora-coreos/kubernetes/network.tf @@ -0,0 +1,193 @@ +resource "google_compute_network" "network" { + name = var.cluster_name + description = "Network for the ${var.cluster_name} cluster" + auto_create_subnetworks = true + + timeouts { + delete = "6m" + } +} + +resource "google_compute_firewall" "allow-ssh" { + name = "${var.cluster_name}-allow-ssh" + network = google_compute_network.network.name + + allow { + protocol = "tcp" + ports = [22] + } + + source_ranges = ["0.0.0.0/0"] + target_tags = ["${var.cluster_name}-controller", "${var.cluster_name}-worker"] +} + +resource "google_compute_firewall" "internal-etcd" { + name = "${var.cluster_name}-internal-etcd" + network = google_compute_network.network.name + + allow { + protocol = "tcp" + ports = [2379, 2380] + } + + source_tags = ["${var.cluster_name}-controller"] + target_tags = ["${var.cluster_name}-controller"] +} + +# Allow Prometheus to scrape etcd metrics +resource "google_compute_firewall" "internal-etcd-metrics" { + name = "${var.cluster_name}-internal-etcd-metrics" + network = google_compute_network.network.name + + allow { + protocol = "tcp" + ports = [2381] + } + + source_tags = ["${var.cluster_name}-worker"] + target_tags = ["${var.cluster_name}-controller"] +} + +# Allow Prometheus to scrape kube-scheduler and kube-controller-manager metrics +resource "google_compute_firewall" "internal-kube-metrics" { + name = "${var.cluster_name}-internal-kube-metrics" + network = google_compute_network.network.name + + allow { + protocol = "tcp" + ports = [10251, 10252] + } + + source_tags = ["${var.cluster_name}-worker"] + target_tags = ["${var.cluster_name}-controller"] +} + +resource "google_compute_firewall" "allow-apiserver" { + name = "${var.cluster_name}-allow-apiserver" + network = google_compute_network.network.name + + allow { + protocol = "tcp" + ports = [6443] + } + + source_ranges = ["0.0.0.0/0"] + target_tags = ["${var.cluster_name}-controller"] +} + +# BGP and IPIP +# https://docs.projectcalico.org/latest/reference/public-cloud/gce +resource "google_compute_firewall" "internal-bgp" { + count = var.networking != "flannel" ? 1 : 0 + + name = "${var.cluster_name}-internal-bgp" + network = google_compute_network.network.name + + allow { + protocol = "tcp" + ports = ["179"] + } + + allow { + protocol = "ipip" + } + + source_tags = ["${var.cluster_name}-controller", "${var.cluster_name}-worker"] + target_tags = ["${var.cluster_name}-controller", "${var.cluster_name}-worker"] +} + +# flannel VXLAN +resource "google_compute_firewall" "internal-vxlan" { + count = var.networking == "flannel" ? 1 : 0 + + name = "${var.cluster_name}-internal-vxlan" + network = google_compute_network.network.name + + allow { + protocol = "udp" + ports = [4789] + } + + source_tags = ["${var.cluster_name}-controller", "${var.cluster_name}-worker"] + target_tags = ["${var.cluster_name}-controller", "${var.cluster_name}-worker"] +} + +# Allow Prometheus to scrape node-exporter daemonset +resource "google_compute_firewall" "internal-node-exporter" { + name = "${var.cluster_name}-internal-node-exporter" + network = google_compute_network.network.name + + allow { + protocol = "tcp" + ports = [9100] + } + + source_tags = ["${var.cluster_name}-worker"] + target_tags = ["${var.cluster_name}-controller", "${var.cluster_name}-worker"] +} + +# Allow Prometheus to scrape kube-proxy metrics +resource "google_compute_firewall" "internal-kube-proxy" { + name = "${var.cluster_name}-internal-kube-proxy" + network = google_compute_network.network.name + + allow { + protocol = "tcp" + ports = [10249] + } + + source_tags = ["${var.cluster_name}-worker"] + target_tags = ["${var.cluster_name}-controller", "${var.cluster_name}-worker"] +} + +# Allow apiserver to access kubelets for exec, log, port-forward +resource "google_compute_firewall" "internal-kubelet" { + name = "${var.cluster_name}-internal-kubelet" + network = google_compute_network.network.name + + allow { + protocol = "tcp" + ports = [10250] + } + + # allow Prometheus to scrape kubelet metrics too + source_tags = ["${var.cluster_name}-controller", "${var.cluster_name}-worker"] + target_tags = ["${var.cluster_name}-controller", "${var.cluster_name}-worker"] +} + +# Workers + +resource "google_compute_firewall" "allow-ingress" { + name = "${var.cluster_name}-allow-ingress" + network = google_compute_network.network.name + + allow { + protocol = "tcp" + ports = [80, 443] + } + + source_ranges = ["0.0.0.0/0"] + target_tags = ["${var.cluster_name}-worker"] +} + +resource "google_compute_firewall" "google-ingress-health-checks" { + name = "${var.cluster_name}-ingress-health" + network = google_compute_network.network.name + + allow { + protocol = "tcp" + ports = [10254] + } + + # https://cloud.google.com/load-balancing/docs/health-check-concepts#method + source_ranges = [ + "35.191.0.0/16", + "130.211.0.0/22", + "35.191.0.0/16", + "209.85.152.0/22", + "209.85.204.0/22", + ] + + target_tags = ["${var.cluster_name}-worker"] +} + diff --git a/google-cloud/fedora-coreos/kubernetes/outputs.tf b/google-cloud/fedora-coreos/kubernetes/outputs.tf new file mode 100644 index 000000000..0de725025 --- /dev/null +++ b/google-cloud/fedora-coreos/kubernetes/outputs.tf @@ -0,0 +1,44 @@ +output "kubeconfig-admin" { + value = module.bootstrap.kubeconfig-admin +} + +# Outputs for Kubernetes Ingress + +output "ingress_static_ipv4" { + description = "Global IPv4 address for proxy load balancing to the nearest Ingress controller" + value = google_compute_global_address.ingress-ipv4.address +} + +output "ingress_static_ipv6" { + description = "Global IPv6 address for proxy load balancing to the nearest Ingress controller" + value = google_compute_global_address.ingress-ipv6.address +} + +# Outputs for worker pools + +output "network_name" { + value = google_compute_network.network.name +} + +output "kubeconfig" { + value = module.bootstrap.kubeconfig-kubelet +} + +# Outputs for custom firewalling + +output "network_self_link" { + value = google_compute_network.network.self_link +} + +# Outputs for custom load balancing + +output "worker_instance_group" { + description = "Worker managed instance group full URL" + value = module.workers.instance_group +} + +output "worker_target_pool" { + description = "Worker target pool self link" + value = module.workers.target_pool +} + diff --git a/google-cloud/fedora-coreos/kubernetes/ssh.tf b/google-cloud/fedora-coreos/kubernetes/ssh.tf new file mode 100644 index 000000000..6f80c5589 --- /dev/null +++ b/google-cloud/fedora-coreos/kubernetes/ssh.tf @@ -0,0 +1,58 @@ +locals { + # format assets for distribution + assets_bundle = [ + # header with the unpack location + for key, value in module.bootstrap.assets_dist : + format("##### %s\n%s", key, value) + ] +} + +# Secure copy assets to controllers. +resource "null_resource" "copy-controller-secrets" { + count = var.controller_count + + depends_on = [ + module.bootstrap, + ] + + connection { + type = "ssh" + host = local.controllers_ipv4_public[count.index] + user = "core" + timeout = "15m" + } + + provisioner "file" { + content = join("\n", local.assets_bundle) + destination = "$HOME/assets" + } + + provisioner "remote-exec" { + inline = [ + "sudo /opt/bootstrap/layout", + ] + } +} + +# Connect to a controller to perform one-time cluster bootstrap. +resource "null_resource" "bootstrap" { + depends_on = [ + null_resource.copy-controller-secrets, + module.workers, + google_dns_record_set.apiserver, + ] + + connection { + type = "ssh" + host = local.controllers_ipv4_public[0] + user = "core" + timeout = "15m" + } + + provisioner "remote-exec" { + inline = [ + "sudo systemctl start bootstrap", + ] + } +} + diff --git a/google-cloud/fedora-coreos/kubernetes/variables.tf b/google-cloud/fedora-coreos/kubernetes/variables.tf new file mode 100644 index 000000000..c7be56d0a --- /dev/null +++ b/google-cloud/fedora-coreos/kubernetes/variables.tf @@ -0,0 +1,138 @@ +variable "cluster_name" { + type = string + description = "Unique cluster name (prepended to dns_zone)" +} + +# Google Cloud + +variable "region" { + type = string + description = "Google Cloud Region (e.g. us-central1, see `gcloud compute regions list`)" +} + +variable "dns_zone" { + type = string + description = "Google Cloud DNS Zone (e.g. google-cloud.example.com)" +} + +variable "dns_zone_name" { + type = string + description = "Google Cloud DNS Zone name (e.g. example-zone)" +} + +# instances + +variable "controller_count" { + type = number + description = "Number of controllers (i.e. masters)" + default = 1 +} + +variable "worker_count" { + type = number + description = "Number of workers" + default = 1 +} + +variable "controller_type" { + type = string + description = "Machine type for controllers (see `gcloud compute machine-types list`)" + default = "n1-standard-1" +} + +variable "worker_type" { + type = string + description = "Machine type for controllers (see `gcloud compute machine-types list`)" + default = "n1-standard-1" +} + +variable "os_image" { + type = string + description = "Fedora CoreOS image for compute instances (e.g. fedora-coreos)" +} + +variable "disk_size" { + type = number + description = "Size of the disk in GB" + default = 40 +} + +variable "worker_preemptible" { + type = bool + description = "If enabled, Compute Engine will terminate workers randomly within 24 hours" + default = false +} + +variable "controller_snippets" { + type = list(string) + description = "Controller Fedora CoreOS Config snippets" + default = [] +} + +variable "worker_snippets" { + type = list(string) + description = "Worker Fedora CoreOS Config snippets" + default = [] +} + +# configuration + +variable "ssh_authorized_key" { + type = string + description = "SSH public key for user 'core'" +} + +variable "asset_dir" { + type = string + description = "Absolute path to a directory where generated assets should be placed (contains secrets)" + default = "" +} + +variable "networking" { + type = string + description = "Choice of networking provider (flannel or calico)" + default = "calico" +} + +variable "pod_cidr" { + type = string + description = "CIDR IPv4 range to assign Kubernetes pods" + default = "10.2.0.0/16" +} + +variable "service_cidr" { + type = string + description = < /dev/null; do sleep 1; done' + [Install] + RequiredBy=kubelet.service + - name: kubelet.service + enabled: true + contents: | + [Unit] + Description=Kubelet via Hyperkube (System Container) + Wants=rpc-statd.service + [Service] + ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d + ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests + ExecStartPre=/bin/mkdir -p /opt/cni/bin + ExecStartPre=/bin/mkdir -p /var/lib/calico + ExecStartPre=/bin/mkdir -p /var/lib/kubelet/volumeplugins + ExecStartPre=/usr/bin/bash -c "grep 'certificate-authority-data' /etc/kubernetes/kubeconfig | awk '{print $2}' | base64 -d > /etc/kubernetes/ca.crt" + ExecStartPre=-/usr/bin/podman rm kubelet + ExecStart=/usr/bin/podman run --name kubelet \ + --privileged \ + --pid host \ + --network host \ + --volume /etc/kubernetes:/etc/kubernetes:ro,z \ + --volume /usr/lib/os-release:/etc/os-release:ro \ + --volume /etc/ssl/certs:/etc/ssl/certs:ro \ + --volume /lib/modules:/lib/modules:ro \ + --volume /run:/run \ + --volume /sys/fs/cgroup:/sys/fs/cgroup:ro \ + --volume /sys/fs/cgroup/systemd:/sys/fs/cgroup/systemd \ + --volume /etc/pki/tls/certs:/usr/share/ca-certificates:ro \ + --volume /var/lib/calico:/var/lib/calico \ + --volume /var/lib/docker:/var/lib/docker \ + --volume /var/lib/kubelet:/var/lib/kubelet:rshared,z \ + --volume /var/log:/var/log \ + --volume /var/run/lock:/var/run/lock:z \ + --volume /opt/cni/bin:/opt/cni/bin:z \ + k8s.gcr.io/hyperkube:v1.17.2 kubelet \ + --anonymous-auth=false \ + --authentication-token-webhook \ + --authorization-mode=Webhook \ + --cgroup-driver=systemd \ + --cgroups-per-qos=true \ + --enforce-node-allocatable=pods \ + --client-ca-file=/etc/kubernetes/ca.crt \ + --cluster_dns=${cluster_dns_service_ip} \ + --cluster_domain=${cluster_domain_suffix} \ + --cni-conf-dir=/etc/kubernetes/cni/net.d \ + --exit-on-lock-contention \ + --healthz-port=0 \ + --kubeconfig=/etc/kubernetes/kubeconfig \ + --lock-file=/var/run/lock/kubelet.lock \ + --network-plugin=cni \ + --node-labels=node.kubernetes.io/node \ + %{ for label in split(",", node_labels) } + --node-labels=${label} \ + %{ endfor ~} + --pod-manifest-path=/etc/kubernetes/manifests \ + --read-only-port=0 \ + --volume-plugin-dir=/var/lib/kubelet/volumeplugins + ExecStop=-/usr/bin/podman stop kubelet + Delegate=yes + Restart=always + RestartSec=10 + [Install] + WantedBy=multi-user.target +storage: + directories: + - path: /etc/kubernetes + files: + - path: /etc/kubernetes/kubeconfig + mode: 0644 + contents: + inline: | + ${kubeconfig} + - path: /etc/sysctl.d/reverse-path-filter.conf + contents: + inline: | + net.ipv4.conf.all.rp_filter=1 + - path: /etc/sysctl.d/max-user-watches.conf + contents: + inline: | + fs.inotify.max_user_watches=16184 + - path: /etc/systemd/system.conf.d/accounting.conf + contents: + inline: | + [Manager] + DefaultCPUAccounting=yes + DefaultMemoryAccounting=yes + DefaultBlockIOAccounting=yes +passwd: + users: + - name: core + ssh_authorized_keys: + - ${ssh_authorized_key} + diff --git a/google-cloud/fedora-coreos/kubernetes/workers/outputs.tf b/google-cloud/fedora-coreos/kubernetes/workers/outputs.tf new file mode 100644 index 000000000..b73c42af7 --- /dev/null +++ b/google-cloud/fedora-coreos/kubernetes/workers/outputs.tf @@ -0,0 +1,14 @@ +# Outputs for global load balancing + +output "instance_group" { + description = "Worker managed instance group full URL" + value = google_compute_region_instance_group_manager.workers.instance_group +} + +# Outputs for regional load balancing + +output "target_pool" { + description = "Worker target pool self link" + value = google_compute_target_pool.workers.self_link +} + diff --git a/google-cloud/fedora-coreos/kubernetes/workers/target_pool.tf b/google-cloud/fedora-coreos/kubernetes/workers/target_pool.tf new file mode 100644 index 000000000..a07855bfc --- /dev/null +++ b/google-cloud/fedora-coreos/kubernetes/workers/target_pool.tf @@ -0,0 +1,23 @@ +# Target pool for TCP/UDP load balancing +resource "google_compute_target_pool" "workers" { + name = "${var.name}-worker-pool" + region = var.region + session_affinity = "NONE" + + health_checks = [ + google_compute_http_health_check.workers.name, + ] +} + +# HTTP Health Check (for TCP/UDP load balancing) +# Forward rules (regional) to target pools don't support different external +# and internal ports. Health check for nodes with Ingress controllers that +# may support proxying or otherwise satisfy the check. +resource "google_compute_http_health_check" "workers" { + name = "${var.name}-target-pool-health" + description = "Health check for the worker target pool" + + port = 10254 + request_path = "/healthz" +} + diff --git a/google-cloud/fedora-coreos/kubernetes/workers/variables.tf b/google-cloud/fedora-coreos/kubernetes/workers/variables.tf new file mode 100644 index 000000000..8f1ef933b --- /dev/null +++ b/google-cloud/fedora-coreos/kubernetes/workers/variables.tf @@ -0,0 +1,106 @@ +variable "name" { + type = string + description = "Unique name for the worker pool" +} + +variable "cluster_name" { + type = string + description = "Must be set to `cluster_name of cluster`" +} + +# Google Cloud + +variable "region" { + type = string + description = "Must be set to `region` of cluster" +} + +variable "network" { + type = string + description = "Must be set to `network_name` output by cluster" +} + +# instances + +variable "worker_count" { + type = number + description = "Number of worker compute instances the instance group should manage" + default = 1 +} + +variable "machine_type" { + type = string + description = "Machine type for compute instances (e.g. gcloud compute machine-types list)" + default = "n1-standard-1" +} + +variable "os_image" { + type = string + description = "Fedora CoreOS image for compute instanges (e.g. gcloud compute images list)" +} + +variable "disk_size" { + type = number + description = "Size of the disk in GB" + default = 40 +} + +variable "preemptible" { + type = bool + description = "If enabled, Compute Engine will terminate instances randomly within 24 hours" + default = false +} + +variable "snippets" { + type = list(string) + description = "Fedora CoreOS Config snippets" + default = [] +} + +# configuration + +variable "kubeconfig" { + type = string + description = "Must be set to `kubeconfig` output by cluster" +} + +variable "ssh_authorized_key" { + type = string + description = "SSH public key for user 'core'" +} + +variable "service_cidr" { + type = string + description = < Date: Tue, 4 Feb 2020 21:32:53 -0800 Subject: [PATCH 324/523] Update kube-state-metrics from v1.9.3 to v1.9.4 * https://github.com/kubernetes/kube-state-metrics/releases/tag/v1.9.4 --- CHANGES.md | 2 ++ addons/prometheus/exporters/kube-state-metrics/deployment.yaml | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index 7f8d34cfd..cdf2a853a 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -11,8 +11,10 @@ Notable changes between versions. #### Addons * Update nginx-ingress from v0.27.1 to v0.28.0 +* Update kube-state-metrics from v1.9.3 to v1.9.4 * Update Grafana from v6.5.3 to v6.6.0 + ## v1.17.2 * Kubernetes [v1.17.2](https://github.com/kubernetes/kubernetes/blob/master/CHANGELOG-1.17.md#v1172) diff --git a/addons/prometheus/exporters/kube-state-metrics/deployment.yaml b/addons/prometheus/exporters/kube-state-metrics/deployment.yaml index ab7c62394..efef4d1a9 100644 --- a/addons/prometheus/exporters/kube-state-metrics/deployment.yaml +++ b/addons/prometheus/exporters/kube-state-metrics/deployment.yaml @@ -24,7 +24,7 @@ spec: serviceAccountName: kube-state-metrics containers: - name: kube-state-metrics - image: quay.io/coreos/kube-state-metrics:v1.9.3 + image: quay.io/coreos/kube-state-metrics:v1.9.4 ports: - name: metrics containerPort: 8080 From ca96a1335c0fe5c412e1c1219f5d1e47dd6881b5 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Thu, 6 Feb 2020 00:29:41 -0800 Subject: [PATCH 325/523] Update Calico from v3.11.2 to v3.12.0 * https://docs.projectcalico.org/release-notes/#v3120 * Remove reverse packet filter override, since Calico no longer relies on the setting * https://github.com/coreos/fedora-coreos-tracker/issues/219 * https://github.com/projectcalico/felix/pull/2189 --- CHANGES.md | 2 ++ aws/container-linux/kubernetes/bootstrap.tf | 2 +- aws/fedora-coreos/kubernetes/bootstrap.tf | 2 +- aws/fedora-coreos/kubernetes/fcc/controller.yaml | 4 ---- aws/fedora-coreos/kubernetes/workers/fcc/worker.yaml | 4 ---- azure/container-linux/kubernetes/bootstrap.tf | 2 +- bare-metal/container-linux/kubernetes/bootstrap.tf | 2 +- bare-metal/fedora-coreos/kubernetes/bootstrap.tf | 2 +- bare-metal/fedora-coreos/kubernetes/fcc/controller.yaml | 4 ---- bare-metal/fedora-coreos/kubernetes/fcc/worker.yaml | 4 ---- digital-ocean/container-linux/kubernetes/bootstrap.tf | 2 +- google-cloud/container-linux/kubernetes/bootstrap.tf | 2 +- google-cloud/fedora-coreos/kubernetes/bootstrap.tf | 2 +- google-cloud/fedora-coreos/kubernetes/fcc/controller.yaml | 4 ---- google-cloud/fedora-coreos/kubernetes/workers/fcc/worker.yaml | 4 ---- 15 files changed, 10 insertions(+), 32 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index cdf2a853a..5eb42d574 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -4,6 +4,8 @@ Notable changes between versions. ## Latest +* Update Calico from v3.11.2 to v3.12.0 + #### Google Cloud * Add Terraform module for Fedora CoreOS ([#632](https://github.com/poseidon/typhoon/pull/632)) diff --git a/aws/container-linux/kubernetes/bootstrap.tf b/aws/container-linux/kubernetes/bootstrap.tf index e6c9ac7d4..435a7ab8e 100644 --- a/aws/container-linux/kubernetes/bootstrap.tf +++ b/aws/container-linux/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=05297b94a936c356851e180e4963034e0047e1c0" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=1ea8fe7a85e87f7e851b4cd4ab8c4e9902b4a40c" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/aws/fedora-coreos/kubernetes/bootstrap.tf b/aws/fedora-coreos/kubernetes/bootstrap.tf index 50e08e59b..60bc59980 100644 --- a/aws/fedora-coreos/kubernetes/bootstrap.tf +++ b/aws/fedora-coreos/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=05297b94a936c356851e180e4963034e0047e1c0" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=1ea8fe7a85e87f7e851b4cd4ab8c4e9902b4a40c" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/aws/fedora-coreos/kubernetes/fcc/controller.yaml b/aws/fedora-coreos/kubernetes/fcc/controller.yaml index df9338823..1b97d933d 100644 --- a/aws/fedora-coreos/kubernetes/fcc/controller.yaml +++ b/aws/fedora-coreos/kubernetes/fcc/controller.yaml @@ -171,10 +171,6 @@ storage: echo "Retry applying manifests" sleep 5 done - - path: /etc/sysctl.d/reverse-path-filter.conf - contents: - inline: | - net.ipv4.conf.all.rp_filter=1 - path: /etc/sysctl.d/max-user-watches.conf contents: inline: | diff --git a/aws/fedora-coreos/kubernetes/workers/fcc/worker.yaml b/aws/fedora-coreos/kubernetes/workers/fcc/worker.yaml index 8c4ff0b84..3e3b6edc9 100644 --- a/aws/fedora-coreos/kubernetes/workers/fcc/worker.yaml +++ b/aws/fedora-coreos/kubernetes/workers/fcc/worker.yaml @@ -87,10 +87,6 @@ storage: contents: inline: | ${kubeconfig} - - path: /etc/sysctl.d/reverse-path-filter.conf - contents: - inline: | - net.ipv4.conf.all.rp_filter=1 - path: /etc/sysctl.d/max-user-watches.conf contents: inline: | diff --git a/azure/container-linux/kubernetes/bootstrap.tf b/azure/container-linux/kubernetes/bootstrap.tf index 8402d2554..077b7dcc5 100644 --- a/azure/container-linux/kubernetes/bootstrap.tf +++ b/azure/container-linux/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=05297b94a936c356851e180e4963034e0047e1c0" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=1ea8fe7a85e87f7e851b4cd4ab8c4e9902b4a40c" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/bare-metal/container-linux/kubernetes/bootstrap.tf b/bare-metal/container-linux/kubernetes/bootstrap.tf index 1a7519506..f9f94b491 100644 --- a/bare-metal/container-linux/kubernetes/bootstrap.tf +++ b/bare-metal/container-linux/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=05297b94a936c356851e180e4963034e0047e1c0" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=1ea8fe7a85e87f7e851b4cd4ab8c4e9902b4a40c" cluster_name = var.cluster_name api_servers = [var.k8s_domain_name] diff --git a/bare-metal/fedora-coreos/kubernetes/bootstrap.tf b/bare-metal/fedora-coreos/kubernetes/bootstrap.tf index 81d543fc3..52c3e5f9c 100644 --- a/bare-metal/fedora-coreos/kubernetes/bootstrap.tf +++ b/bare-metal/fedora-coreos/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=05297b94a936c356851e180e4963034e0047e1c0" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=1ea8fe7a85e87f7e851b4cd4ab8c4e9902b4a40c" cluster_name = var.cluster_name api_servers = [var.k8s_domain_name] diff --git a/bare-metal/fedora-coreos/kubernetes/fcc/controller.yaml b/bare-metal/fedora-coreos/kubernetes/fcc/controller.yaml index 21c874805..5f9d34143 100644 --- a/bare-metal/fedora-coreos/kubernetes/fcc/controller.yaml +++ b/bare-metal/fedora-coreos/kubernetes/fcc/controller.yaml @@ -182,10 +182,6 @@ storage: echo "Retry applying manifests" sleep 5 done - - path: /etc/sysctl.d/reverse-path-filter.conf - contents: - inline: | - net.ipv4.conf.all.rp_filter=1 - path: /etc/sysctl.d/max-user-watches.conf contents: inline: | diff --git a/bare-metal/fedora-coreos/kubernetes/fcc/worker.yaml b/bare-metal/fedora-coreos/kubernetes/fcc/worker.yaml index 077ca75ce..0360a68ba 100644 --- a/bare-metal/fedora-coreos/kubernetes/fcc/worker.yaml +++ b/bare-metal/fedora-coreos/kubernetes/fcc/worker.yaml @@ -95,10 +95,6 @@ storage: contents: inline: ${domain_name} - - path: /etc/sysctl.d/reverse-path-filter.conf - contents: - inline: | - net.ipv4.conf.all.rp_filter=1 - path: /etc/sysctl.d/max-user-watches.conf contents: inline: | diff --git a/digital-ocean/container-linux/kubernetes/bootstrap.tf b/digital-ocean/container-linux/kubernetes/bootstrap.tf index f41d4a158..c2811e89c 100644 --- a/digital-ocean/container-linux/kubernetes/bootstrap.tf +++ b/digital-ocean/container-linux/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=05297b94a936c356851e180e4963034e0047e1c0" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=1ea8fe7a85e87f7e851b4cd4ab8c4e9902b4a40c" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/google-cloud/container-linux/kubernetes/bootstrap.tf b/google-cloud/container-linux/kubernetes/bootstrap.tf index 56ebceeaa..ebebc82f7 100644 --- a/google-cloud/container-linux/kubernetes/bootstrap.tf +++ b/google-cloud/container-linux/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=05297b94a936c356851e180e4963034e0047e1c0" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=1ea8fe7a85e87f7e851b4cd4ab8c4e9902b4a40c" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/google-cloud/fedora-coreos/kubernetes/bootstrap.tf b/google-cloud/fedora-coreos/kubernetes/bootstrap.tf index 4bb6217aa..96576ffed 100644 --- a/google-cloud/fedora-coreos/kubernetes/bootstrap.tf +++ b/google-cloud/fedora-coreos/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=05297b94a936c356851e180e4963034e0047e1c0" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=1ea8fe7a85e87f7e851b4cd4ab8c4e9902b4a40c" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/google-cloud/fedora-coreos/kubernetes/fcc/controller.yaml b/google-cloud/fedora-coreos/kubernetes/fcc/controller.yaml index df9338823..1b97d933d 100644 --- a/google-cloud/fedora-coreos/kubernetes/fcc/controller.yaml +++ b/google-cloud/fedora-coreos/kubernetes/fcc/controller.yaml @@ -171,10 +171,6 @@ storage: echo "Retry applying manifests" sleep 5 done - - path: /etc/sysctl.d/reverse-path-filter.conf - contents: - inline: | - net.ipv4.conf.all.rp_filter=1 - path: /etc/sysctl.d/max-user-watches.conf contents: inline: | diff --git a/google-cloud/fedora-coreos/kubernetes/workers/fcc/worker.yaml b/google-cloud/fedora-coreos/kubernetes/workers/fcc/worker.yaml index 8c4ff0b84..3e3b6edc9 100644 --- a/google-cloud/fedora-coreos/kubernetes/workers/fcc/worker.yaml +++ b/google-cloud/fedora-coreos/kubernetes/workers/fcc/worker.yaml @@ -87,10 +87,6 @@ storage: contents: inline: | ${kubeconfig} - - path: /etc/sysctl.d/reverse-path-filter.conf - contents: - inline: | - net.ipv4.conf.all.rp_filter=1 - path: /etc/sysctl.d/max-user-watches.conf contents: inline: | From 34c3d7cc39fa782e632e092ad396953a381ea5b8 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Sat, 8 Feb 2020 14:50:33 -0800 Subject: [PATCH 326/523] Update Grafana from v6.6.0 to v6.6.1 * https://github.com/grafana/grafana/releases/tag/v6.6.1 --- CHANGES.md | 3 +-- addons/grafana/deployment.yaml | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 5eb42d574..00064a4b9 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -14,8 +14,7 @@ Notable changes between versions. * Update nginx-ingress from v0.27.1 to v0.28.0 * Update kube-state-metrics from v1.9.3 to v1.9.4 -* Update Grafana from v6.5.3 to v6.6.0 - +* Update Grafana from v6.5.3 to v6.6.1 ## v1.17.2 diff --git a/addons/grafana/deployment.yaml b/addons/grafana/deployment.yaml index 1639b42e3..a5b35a2d7 100644 --- a/addons/grafana/deployment.yaml +++ b/addons/grafana/deployment.yaml @@ -23,7 +23,7 @@ spec: spec: containers: - name: grafana - image: docker.io/grafana/grafana:6.6.0 + image: docker.io/grafana/grafana:6.6.1 env: - name: GF_PATHS_CONFIG value: "/etc/grafana/custom.ini" From b49a1d715d0a89546233a8c6e8ee344ff2c3c0a7 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Sat, 8 Feb 2020 14:55:15 -0800 Subject: [PATCH 327/523] Update docs generation packages * Update mkdocs-material from v4.6.0 to v4.6.2 --- requirements.txt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/requirements.txt b/requirements.txt index 9cd287ae0..b6e5cc8ce 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ mkdocs==1.0.4 -mkdocs-material==4.6.0 -pygments==2.4.2 -pymdown-extensions==6.2.0 +mkdocs-material==4.6.2 +pygments==2.5.2 +pymdown-extensions==6.3.0 From ba84f86dc7432022235fc3fc36408d63a3bf3d32 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Tue, 11 Feb 2020 00:12:54 -0800 Subject: [PATCH 328/523] Add guide for Typhoon with Flatcar Linux on Google Cloud * Add docs on manually uploading a Flatcar Linux GCE/GCP gzipped tarball image as a Compute Engine image for use with the Typhoon container-linux module * Set status of Flatcar Linux on Google Cloud to alpha --- CHANGES.md | 1 + README.md | 14 +++++++++++--- docs/cl/google-cloud.md | 29 +++++++++++++++++++++++++++-- docs/fedora-coreos/google-cloud.md | 9 +++++---- docs/index.md | 14 +++++++++++--- 5 files changed, 55 insertions(+), 12 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 00064a4b9..5740f3809 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -9,6 +9,7 @@ Notable changes between versions. #### Google Cloud * Add Terraform module for Fedora CoreOS ([#632](https://github.com/poseidon/typhoon/pull/632)) +* Add support for Flatcar Container Linux ([#639](https://github.com/poseidon/typhoon/pull/639)) #### Addons diff --git a/README.md b/README.md index 6d06e9c01..822d36692 100644 --- a/README.md +++ b/README.md @@ -23,13 +23,13 @@ Typhoon provides a Terraform Module for each supported operating system and plat | Platform | Operating System | Terraform Module | Status | |---------------|------------------|------------------|--------| -| AWS | Container Linux / Flatcar Linux | [aws/container-linux/kubernetes](aws/container-linux/kubernetes) | stable | +| AWS | Container Linux | [aws/container-linux/kubernetes](aws/container-linux/kubernetes) | stable | | Azure | Container Linux | [azure/container-linux/kubernetes](azure/container-linux/kubernetes) | alpha | -| Bare-Metal | Container Linux / Flatcar Linux | [bare-metal/container-linux/kubernetes](bare-metal/container-linux/kubernetes) | stable | +| Bare-Metal | Container Linux | [bare-metal/container-linux/kubernetes](bare-metal/container-linux/kubernetes) | stable | | Digital Ocean | Container Linux | [digital-ocean/container-linux/kubernetes](digital-ocean/container-linux/kubernetes) | beta | | Google Cloud | Container Linux | [google-cloud/container-linux/kubernetes](google-cloud/container-linux/kubernetes) | stable | -Typhoon is available for [Fedora CoreOS](https://getfedora.org/coreos/) in alpha. +Typhoon is available for [Fedora CoreOS](https://getfedora.org/coreos/). | Platform | Operating System | Terraform Module | Status | |---------------|------------------|------------------|--------| @@ -37,6 +37,14 @@ Typhoon is available for [Fedora CoreOS](https://getfedora.org/coreos/) in alpha | Bare-Metal | Fedora CoreOS | [bare-metal/fedora-coreos/kubernetes](bare-metal/fedora-coreos/kubernetes) | alpha | | Google Cloud | Fedora CoreOS | [google-cloud/fedora-coreos/kubernetes](google-cloud/fedora-coreos/kubernetes) | alpha | +Typhoon is available for [Flatcar Container Linux](https://www.flatcar-linux.org/releases/). + +| Platform | Operating System | Terraform Module | Status | +|---------------|------------------|------------------|--------| +| AWS | Flatcar Linux | [aws/container-linux/kubernetes](aws/container-linux/kubernetes) | stable | +| Bare-Metal | Flatcar Linux | [bare-metal/container-linux/kubernetes](bare-metal/container-linux/kubernetes) | stable | +| Google Cloud | Flatcar Linux | [google-cloud/container-linux/kubernetes](google-cloud/container-linux/kubernetes) | alpha | + ## Documentation * [Docs](https://typhoon.psdn.io) diff --git a/docs/cl/google-cloud.md b/docs/cl/google-cloud.md index 31867f134..00f00a35b 100644 --- a/docs/cl/google-cloud.md +++ b/docs/cl/google-cloud.md @@ -81,7 +81,7 @@ module "yavin" { # configuration ssh_authorized_key = "ssh-rsa AAAAB3Nz..." - + # optional worker_count = 2 } @@ -89,6 +89,31 @@ module "yavin" { Reference the [variables docs](#variables) or the [variables.tf](https://github.com/poseidon/typhoon/blob/master/google-cloud/container-linux/kubernetes/variables.tf) source. +### Flatcar Linux Images + +!!! success + Skip this section when using CoreOS Container Linux (default). CoreOS Container Linux publishes official images to Google Cloud. + +!!! danger + Typhoon for Flatcar Linux on Google Cloud is alpha. + +Flatcar Linux publishes images for Google Cloud, but does not yet upload them. Google Cloud allows [custom boot images](https://cloud.google.com/compute/docs/images/import-existing-image) to be uploaded to a bucket and imported into your project. + +[Download](https://www.flatcar-linux.org/releases/) the Flatcar Linux GCE gzipped tarball and upload it to a Google Cloud storage bucket. + +``` +gsutil list +gsutil cp flatcar_production_gce.tar.gz gs://BUCKET +``` + +Create a Compute Engine image from the file. + +``` +gcloud compute images create flatcar-linux-2303-4-0 --source-uri gs://BUCKET_NAME/flatcar_production_gce.tar.gz +``` + +Set the [os_image](#variables) to the image name (e.g. `flatcar-linux-2303-4-0`) + ## ssh-agent Initial bootstrapping requires `bootstrap.service` be started on one controller node. Terraform uses `ssh-agent` to automate this step. Add your SSH private key to `ssh-agent`. @@ -217,7 +242,7 @@ resource "google_dns_managed_zone" "zone-for-clusters" { | worker_count | Number of workers | 1 | 3 | | controller_type | Machine type for controllers | "n1-standard-1" | See below | | worker_type | Machine type for workers | "n1-standard-1" | See below | -| os_image | Container Linux image for compute instances | "coreos-stable" | "coreos-stable-1632-3-0-v20180215" | +| os_image | Container Linux image for compute instances | "coreos-stable" | "flatcar-linux-2303-4-0" | | disk_size | Size of the disk in GB | 40 | 100 | | worker_preemptible | If enabled, Compute Engine will terminate workers randomly within 24 hours | false | true | | controller_clc_snippets | Controller Container Linux Config snippets | [] | [example](/advanced/customization/) | diff --git a/docs/fedora-coreos/google-cloud.md b/docs/fedora-coreos/google-cloud.md index 5df392675..14e2eaf10 100644 --- a/docs/fedora-coreos/google-cloud.md +++ b/docs/fedora-coreos/google-cloud.md @@ -72,14 +72,14 @@ Additional configuration options are described in the `google` provider [docs](h Fedora CoreOS publishes images for Google Cloud, but does not yet upload them. Google Cloud allows [custom boot images](https://cloud.google.com/compute/docs/images/import-existing-image) to be uploaded to a bucket and imported into your project. -[Download](https://getfedora.org/coreos/download/) the Fedora CoreOS GCP gzipped tarball. Then upload the file to a GCS storage bucket. +[Download](https://getfedora.org/coreos/download/) a Fedora CoreOS GCP gzipped tarball and upload it to a Google Cloud storage bucket. ``` gsutil list -gsutil cp fedora-coreos-31.20200113.3.1-gcp.x86_64.tar.gz gs://BUCKET_NAME +gsutil cp fedora-coreos-31.20200113.3.1-gcp.x86_64.tar.gz gs://BUCKET ``` -Create a Google Compute Engine image from the bucket file. +Create a Compute Engine image from the file. ``` gcloud compute images create fedora-coreos-31-20200113-3-1 --source-uri gs://BUCKET/fedora-coreos-31.20200113.3.1-gcp.x86_64.tar.gz @@ -98,7 +98,8 @@ module "yavin" { region = "us-central1" dns_zone = "example.com" dns_zone_name = "example-zone" - # temporary + + # custom image name from above os_image = "fedora-coreos-31-20200113-3-1" # configuration diff --git a/docs/index.md b/docs/index.md index 038440f08..addaf5175 100644 --- a/docs/index.md +++ b/docs/index.md @@ -23,13 +23,13 @@ Typhoon provides a Terraform Module for each supported operating system and plat | Platform | Operating System | Terraform Module | Status | |---------------|------------------|------------------|--------| -| AWS | Container Linux / Flatcar Linux | [aws/container-linux/kubernetes](cl/aws.md) | stable | +| AWS | Container Linux | [aws/container-linux/kubernetes](cl/aws.md) | stable | | Azure | Container Linux | [azure/container-linux/kubernetes](cl/azure.md) | alpha | -| Bare-Metal | Container Linux / Flatcar Linux | [bare-metal/container-linux/kubernetes](cl/bare-metal.md) | stable | +| Bare-Metal | Container Linux | [bare-metal/container-linux/kubernetes](cl/bare-metal.md) | stable | | Digital Ocean | Container Linux | [digital-ocean/container-linux/kubernetes](cl/digital-ocean.md) | beta | | Google Cloud | Container Linux | [google-cloud/container-linux/kubernetes](cl/google-cloud.md) | stable | -Typhoon is available for [Fedora CoreOS](https://getfedora.org/coreos/) in alpha. +Typhoon is available for [Fedora CoreOS](https://getfedora.org/coreos/). | Platform | Operating System | Terraform Module | Status | |---------------|------------------|------------------|--------| @@ -37,6 +37,14 @@ Typhoon is available for [Fedora CoreOS](https://getfedora.org/coreos/) in alpha | Bare-Metal | Fedora CoreOS | [bare-metal/fedora-coreos/kubernetes](fedora-coreos/bare-metal.md) | alpha | | Google Cloud | Fedora CoreOS | [google-cloud/fedora-coreos/kubernetes](google-cloud/fedora-coreos/kubernetes) | alpha | +Typhoon is available for [Flatcar Container Linux](https://www.flatcar-linux.org/releases/). + +| Platform | Operating System | Terraform Module | Status | +|---------------|------------------|------------------|--------| +| AWS | Flatcar Linux | [aws/container-linux/kubernetes](cl/aws.md) | stable | +| Bare-Metal | Flatcar Linux | [bare-metal/container-linux/kubernetes](cl/bare-metal.md) | stable | +| Google Cloud | Flatcar Linux | [google-cloud/container-linux/kubernetes](cl/google-cloud.md) | alpha | + ## Documentation * Architecture [concepts](architecture/concepts.md) and [operating-systems](architecture/operating-systems.md) From 846f11097f4011dd17832a39c55487e2087a3533 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Tue, 11 Feb 2020 19:54:10 -0800 Subject: [PATCH 329/523] Update Fedora CoreOS kernel arguments to align with upstream * Align bare-metal kernel arguments with upstream docs * Add missing initrd argument which can cause issues if not present. Fix #638 * Add tty0 and ttyS0 consoles (matches Container Linux) * Remove unused coreos.inst=yes Related: https://docs.fedoraproject.org/en-US/fedora-coreos/bare-metal/ --- CHANGES.md | 4 ++++ bare-metal/fedora-coreos/kubernetes/profiles.tf | 12 ++++++++---- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 5740f3809..4d45b4942 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -6,6 +6,10 @@ Notable changes between versions. * Update Calico from v3.11.2 to v3.12.0 +#### Bare-Metal + +* Add Fedora CoreOS kernel arguments initrd and console ([#640](https://github.com/poseidon/typhoon/pull/640)) + #### Google Cloud * Add Terraform module for Fedora CoreOS ([#632](https://github.com/poseidon/typhoon/pull/632)) diff --git a/bare-metal/fedora-coreos/kubernetes/profiles.tf b/bare-metal/fedora-coreos/kubernetes/profiles.tf index b49e3176e..fefed8852 100644 --- a/bare-metal/fedora-coreos/kubernetes/profiles.tf +++ b/bare-metal/fedora-coreos/kubernetes/profiles.tf @@ -4,10 +4,12 @@ locals { remote_args = [ "ip=dhcp", "rd.neednet=1", - "coreos.inst=yes", + "initrd=fedora-coreos-${var.os_version}-live-initramfs.x86_64.img", "coreos.inst.image_url=https://builds.coreos.fedoraproject.org/prod/streams/${var.os_stream}/builds/${var.os_version}/x86_64/fedora-coreos-${var.os_version}-metal.x86_64.raw.xz", "coreos.inst.ignition_url=${var.matchbox_http_endpoint}/ignition?uuid=$${uuid}&mac=$${mac:hexhyp}", - "coreos.inst.install_dev=${var.install_disk}" + "coreos.inst.install_dev=${var.install_disk}", + "console=tty0", + "console=ttyS0", ] cached_kernel = "/assets/fedora-coreos/fedora-coreos-${var.os_version}-live-kernel-x86_64" @@ -15,10 +17,12 @@ locals { cached_args = [ "ip=dhcp", "rd.neednet=1", - "coreos.inst=yes", + "initrd=fedora-coreos-${var.os_version}-live-initramfs.x86_64.img", "coreos.inst.image_url=${var.matchbox_http_endpoint}/assets/fedora-coreos/fedora-coreos-${var.os_version}-metal.x86_64.raw.xz", "coreos.inst.ignition_url=${var.matchbox_http_endpoint}/ignition?uuid=$${uuid}&mac=$${mac:hexhyp}", - "coreos.inst.install_dev=${var.install_disk}" + "coreos.inst.install_dev=${var.install_disk}", + "console=tty0", + "console=ttyS0", ] kernel = var.cached_install ? local.cached_kernel : local.remote_kernel From 1243f395d17dcabcff795137ea13b26fc7c61f3c Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Tue, 11 Feb 2020 20:22:14 -0800 Subject: [PATCH 330/523] Update Kubernetes from v1.17.2 to v1.17.3 * https://github.com/kubernetes/kubernetes/blob/master/CHANGELOG/CHANGELOG-1.17.md#v1173 --- CHANGES.md | 1 + README.md | 10 +++++----- aws/container-linux/kubernetes/README.md | 2 +- aws/container-linux/kubernetes/bootstrap.tf | 2 +- aws/container-linux/kubernetes/cl/controller.yaml | 4 ++-- .../kubernetes/workers/cl/worker.yaml | 4 ++-- aws/fedora-coreos/kubernetes/README.md | 2 +- aws/fedora-coreos/kubernetes/bootstrap.tf | 2 +- aws/fedora-coreos/kubernetes/fcc/controller.yaml | 4 ++-- .../kubernetes/workers/fcc/worker.yaml | 2 +- azure/container-linux/kubernetes/README.md | 2 +- azure/container-linux/kubernetes/bootstrap.tf | 2 +- .../container-linux/kubernetes/cl/controller.yaml | 4 ++-- .../kubernetes/workers/cl/worker.yaml | 4 ++-- bare-metal/container-linux/kubernetes/README.md | 2 +- bare-metal/container-linux/kubernetes/bootstrap.tf | 2 +- .../container-linux/kubernetes/cl/controller.yaml | 4 ++-- .../container-linux/kubernetes/cl/worker.yaml | 2 +- bare-metal/fedora-coreos/kubernetes/README.md | 2 +- bare-metal/fedora-coreos/kubernetes/bootstrap.tf | 2 +- .../fedora-coreos/kubernetes/fcc/controller.yaml | 4 ++-- .../fedora-coreos/kubernetes/fcc/worker.yaml | 2 +- digital-ocean/container-linux/kubernetes/README.md | 2 +- .../container-linux/kubernetes/bootstrap.tf | 2 +- .../container-linux/kubernetes/cl/controller.yaml | 4 ++-- .../container-linux/kubernetes/cl/worker.yaml | 4 ++-- docs/advanced/worker-pools.md | 14 +++++++------- docs/cl/aws.md | 10 +++++----- docs/cl/azure.md | 10 +++++----- docs/cl/bare-metal.md | 10 +++++----- docs/cl/digital-ocean.md | 10 +++++----- docs/cl/google-cloud.md | 10 +++++----- docs/fedora-coreos/aws.md | 10 +++++----- docs/fedora-coreos/bare-metal.md | 10 +++++----- docs/fedora-coreos/google-cloud.md | 8 ++++---- docs/index.md | 10 +++++----- docs/topics/maintenance.md | 8 ++++---- google-cloud/container-linux/kubernetes/README.md | 2 +- .../container-linux/kubernetes/bootstrap.tf | 2 +- .../container-linux/kubernetes/cl/controller.yaml | 4 ++-- .../kubernetes/workers/cl/worker.yaml | 4 ++-- google-cloud/fedora-coreos/kubernetes/README.md | 2 +- google-cloud/fedora-coreos/kubernetes/bootstrap.tf | 2 +- .../fedora-coreos/kubernetes/fcc/controller.yaml | 4 ++-- .../kubernetes/workers/fcc/worker.yaml | 2 +- 45 files changed, 105 insertions(+), 104 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 4d45b4942..74751ae5e 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -4,6 +4,7 @@ Notable changes between versions. ## Latest +* Kubernetes [v1.17.3](https://github.com/kubernetes/kubernetes/blob/master/CHANGELOG/CHANGELOG-1.17.md#v1173) * Update Calico from v3.11.2 to v3.12.0 #### Bare-Metal diff --git a/README.md b/README.md index 822d36692..f7d26020a 100644 --- a/README.md +++ b/README.md @@ -11,7 +11,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features -* Kubernetes v1.17.2 (upstream) +* Kubernetes v1.17.3 (upstream) * Single or multi-master, [Calico](https://www.projectcalico.org/) or [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/) * Advanced features like [worker pools](https://typhoon.psdn.io/advanced/worker-pools/), [preemptible](https://typhoon.psdn.io/cl/google-cloud/#preemption) workers, and [snippets](https://typhoon.psdn.io/advanced/customization/#container-linux) customization @@ -57,7 +57,7 @@ Define a Kubernetes cluster by using the Terraform module for your chosen platfo ```tf module "yavin" { - source = "git::https://github.com/poseidon/typhoon//google-cloud/container-linux/kubernetes?ref=v1.17.2" + source = "git::https://github.com/poseidon/typhoon//google-cloud/container-linux/kubernetes?ref=v1.17.3" # Google Cloud cluster_name = "yavin" @@ -96,9 +96,9 @@ In 4-8 minutes (varies by platform), the cluster will be ready. This Google Clou $ export KUBECONFIG=/home/user/.kube/configs/yavin-config $ kubectl get nodes NAME ROLES STATUS AGE VERSION -yavin-controller-0.c.example-com.internal Ready 6m v1.17.2 -yavin-worker-jrbf.c.example-com.internal Ready 5m v1.17.2 -yavin-worker-mzdm.c.example-com.internal Ready 5m v1.17.2 +yavin-controller-0.c.example-com.internal Ready 6m v1.17.3 +yavin-worker-jrbf.c.example-com.internal Ready 5m v1.17.3 +yavin-worker-mzdm.c.example-com.internal Ready 5m v1.17.3 ``` List the pods. diff --git a/aws/container-linux/kubernetes/README.md b/aws/container-linux/kubernetes/README.md index 04a2ea3c1..fffa6688e 100644 --- a/aws/container-linux/kubernetes/README.md +++ b/aws/container-linux/kubernetes/README.md @@ -11,7 +11,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features -* Kubernetes v1.17.2 (upstream) +* Kubernetes v1.17.3 (upstream) * Single or multi-master, [Calico](https://www.projectcalico.org/) or [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/) * Advanced features like [worker pools](https://typhoon.psdn.io/advanced/worker-pools/), [spot](https://typhoon.psdn.io/cl/aws/#spot) workers, and [snippets](https://typhoon.psdn.io/advanced/customization/#container-linux) customization diff --git a/aws/container-linux/kubernetes/bootstrap.tf b/aws/container-linux/kubernetes/bootstrap.tf index 435a7ab8e..6a2c30958 100644 --- a/aws/container-linux/kubernetes/bootstrap.tf +++ b/aws/container-linux/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=1ea8fe7a85e87f7e851b4cd4ab8c4e9902b4a40c" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=796194583426593d1a62b6f1bf3f7ffed8fca140" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/aws/container-linux/kubernetes/cl/controller.yaml b/aws/container-linux/kubernetes/cl/controller.yaml index e8d375b70..670bb39ff 100644 --- a/aws/container-linux/kubernetes/cl/controller.yaml +++ b/aws/container-linux/kubernetes/cl/controller.yaml @@ -89,7 +89,7 @@ systemd: --mount volume=var-log,target=/var/log \ --volume opt-cni-bin,kind=host,source=/opt/cni/bin \ --mount volume=opt-cni-bin,target=/opt/cni/bin \ - docker://k8s.gcr.io/hyperkube:v1.17.2 \ + docker://k8s.gcr.io/hyperkube:v1.17.3 \ --exec=/usr/local/bin/kubelet -- \ --anonymous-auth=false \ --authentication-token-webhook \ @@ -134,7 +134,7 @@ systemd: --volume script,kind=host,source=/opt/bootstrap/apply \ --mount volume=script,target=/apply \ --insecure-options=image \ - docker://k8s.gcr.io/hyperkube:v1.17.2 \ + docker://k8s.gcr.io/hyperkube:v1.17.3 \ --net=host \ --dns=host \ --exec=/apply diff --git a/aws/container-linux/kubernetes/workers/cl/worker.yaml b/aws/container-linux/kubernetes/workers/cl/worker.yaml index 4ee4ec34c..cd8fc92cc 100644 --- a/aws/container-linux/kubernetes/workers/cl/worker.yaml +++ b/aws/container-linux/kubernetes/workers/cl/worker.yaml @@ -64,7 +64,7 @@ systemd: --mount volume=var-log,target=/var/log \ --volume opt-cni-bin,kind=host,source=/opt/cni/bin \ --mount volume=opt-cni-bin,target=/opt/cni/bin \ - docker://k8s.gcr.io/hyperkube:v1.17.2 \ + docker://k8s.gcr.io/hyperkube:v1.17.3 \ --exec=/usr/local/bin/kubelet -- \ --anonymous-auth=false \ --authentication-token-webhook \ @@ -128,7 +128,7 @@ storage: --volume config,kind=host,source=/etc/kubernetes \ --mount volume=config,target=/etc/kubernetes \ --insecure-options=image \ - docker://k8s.gcr.io/hyperkube:v1.17.2 \ + docker://k8s.gcr.io/hyperkube:v1.17.3 \ --net=host \ --dns=host \ -- \ diff --git a/aws/fedora-coreos/kubernetes/README.md b/aws/fedora-coreos/kubernetes/README.md index ded0faeae..4cd36d5d6 100644 --- a/aws/fedora-coreos/kubernetes/README.md +++ b/aws/fedora-coreos/kubernetes/README.md @@ -11,7 +11,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features -* Kubernetes v1.17.2 (upstream) +* Kubernetes v1.17.3 (upstream) * Single or multi-master, [Calico](https://www.projectcalico.org/) or [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/) * Advanced features like [worker pools](https://typhoon.psdn.io/advanced/worker-pools/), [spot](https://typhoon.psdn.io/cl/aws/#spot) workers, and [snippets](https://typhoon.psdn.io/advanced/customization/#container-linux) customization diff --git a/aws/fedora-coreos/kubernetes/bootstrap.tf b/aws/fedora-coreos/kubernetes/bootstrap.tf index 60bc59980..d9dd742ce 100644 --- a/aws/fedora-coreos/kubernetes/bootstrap.tf +++ b/aws/fedora-coreos/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=1ea8fe7a85e87f7e851b4cd4ab8c4e9902b4a40c" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=796194583426593d1a62b6f1bf3f7ffed8fca140" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/aws/fedora-coreos/kubernetes/fcc/controller.yaml b/aws/fedora-coreos/kubernetes/fcc/controller.yaml index 1b97d933d..e0131f0fb 100644 --- a/aws/fedora-coreos/kubernetes/fcc/controller.yaml +++ b/aws/fedora-coreos/kubernetes/fcc/controller.yaml @@ -79,7 +79,7 @@ systemd: --volume /var/log:/var/log \ --volume /var/run/lock:/var/run/lock:z \ --volume /opt/cni/bin:/opt/cni/bin:z \ - k8s.gcr.io/hyperkube:v1.17.2 kubelet \ + k8s.gcr.io/hyperkube:v1.17.3 kubelet \ --anonymous-auth=false \ --authentication-token-webhook \ --authorization-mode=Webhook \ @@ -123,7 +123,7 @@ systemd: --volume /opt/bootstrap/assets:/assets:ro,Z \ --volume /opt/bootstrap/apply:/apply:ro,Z \ --entrypoint=/apply \ - k8s.gcr.io/hyperkube:v1.17.2 + k8s.gcr.io/hyperkube:v1.17.3 ExecStartPost=/bin/touch /opt/bootstrap/bootstrap.done ExecStartPost=-/usr/bin/podman stop bootstrap storage: diff --git a/aws/fedora-coreos/kubernetes/workers/fcc/worker.yaml b/aws/fedora-coreos/kubernetes/workers/fcc/worker.yaml index 3e3b6edc9..8a1d02cfc 100644 --- a/aws/fedora-coreos/kubernetes/workers/fcc/worker.yaml +++ b/aws/fedora-coreos/kubernetes/workers/fcc/worker.yaml @@ -49,7 +49,7 @@ systemd: --volume /var/log:/var/log \ --volume /var/run/lock:/var/run/lock:z \ --volume /opt/cni/bin:/opt/cni/bin:z \ - k8s.gcr.io/hyperkube:v1.17.2 kubelet \ + k8s.gcr.io/hyperkube:v1.17.3 kubelet \ --anonymous-auth=false \ --authentication-token-webhook \ --authorization-mode=Webhook \ diff --git a/azure/container-linux/kubernetes/README.md b/azure/container-linux/kubernetes/README.md index c81da4954..af5359873 100644 --- a/azure/container-linux/kubernetes/README.md +++ b/azure/container-linux/kubernetes/README.md @@ -11,7 +11,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features -* Kubernetes v1.17.2 (upstream) +* Kubernetes v1.17.3 (upstream) * Single or multi-master, [Calico](https://www.projectcalico.org/) or [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/) * Advanced features like [worker pools](https://typhoon.psdn.io/advanced/worker-pools/), [low-priority](https://typhoon.psdn.io/cl/azure/#low-priority) workers, and [snippets](https://typhoon.psdn.io/advanced/customization/#container-linux) customization diff --git a/azure/container-linux/kubernetes/bootstrap.tf b/azure/container-linux/kubernetes/bootstrap.tf index 077b7dcc5..5c089e89e 100644 --- a/azure/container-linux/kubernetes/bootstrap.tf +++ b/azure/container-linux/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=1ea8fe7a85e87f7e851b4cd4ab8c4e9902b4a40c" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=796194583426593d1a62b6f1bf3f7ffed8fca140" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/azure/container-linux/kubernetes/cl/controller.yaml b/azure/container-linux/kubernetes/cl/controller.yaml index c1d19685b..4f18d6bb6 100644 --- a/azure/container-linux/kubernetes/cl/controller.yaml +++ b/azure/container-linux/kubernetes/cl/controller.yaml @@ -88,7 +88,7 @@ systemd: --mount volume=var-log,target=/var/log \ --volume opt-cni-bin,kind=host,source=/opt/cni/bin \ --mount volume=opt-cni-bin,target=/opt/cni/bin \ - docker://k8s.gcr.io/hyperkube:v1.17.2 \ + docker://k8s.gcr.io/hyperkube:v1.17.3 \ --exec=/usr/local/bin/kubelet -- \ --anonymous-auth=false \ --authentication-token-webhook \ @@ -132,7 +132,7 @@ systemd: --volume script,kind=host,source=/opt/bootstrap/apply \ --mount volume=script,target=/apply \ --insecure-options=image \ - docker://k8s.gcr.io/hyperkube:v1.17.2 \ + docker://k8s.gcr.io/hyperkube:v1.17.3 \ --net=host \ --dns=host \ --exec=/apply diff --git a/azure/container-linux/kubernetes/workers/cl/worker.yaml b/azure/container-linux/kubernetes/workers/cl/worker.yaml index cb314e0f9..994c8f4c5 100644 --- a/azure/container-linux/kubernetes/workers/cl/worker.yaml +++ b/azure/container-linux/kubernetes/workers/cl/worker.yaml @@ -63,7 +63,7 @@ systemd: --mount volume=var-log,target=/var/log \ --volume opt-cni-bin,kind=host,source=/opt/cni/bin \ --mount volume=opt-cni-bin,target=/opt/cni/bin \ - docker://k8s.gcr.io/hyperkube:v1.17.2 \ + docker://k8s.gcr.io/hyperkube:v1.17.3 \ --exec=/usr/local/bin/kubelet -- \ --anonymous-auth=false \ --authentication-token-webhook \ @@ -126,7 +126,7 @@ storage: --volume config,kind=host,source=/etc/kubernetes \ --mount volume=config,target=/etc/kubernetes \ --insecure-options=image \ - docker://k8s.gcr.io/hyperkube:v1.17.2 \ + docker://k8s.gcr.io/hyperkube:v1.17.3 \ --net=host \ --dns=host \ -- \ diff --git a/bare-metal/container-linux/kubernetes/README.md b/bare-metal/container-linux/kubernetes/README.md index cccd3eb83..a85c3aafe 100644 --- a/bare-metal/container-linux/kubernetes/README.md +++ b/bare-metal/container-linux/kubernetes/README.md @@ -11,7 +11,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features -* Kubernetes v1.17.2 (upstream) +* Kubernetes v1.17.3 (upstream) * Single or multi-master, [Calico](https://www.projectcalico.org/) or [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/) * Advanced features like [snippets](https://typhoon.psdn.io/advanced/customization/#container-linux) customization diff --git a/bare-metal/container-linux/kubernetes/bootstrap.tf b/bare-metal/container-linux/kubernetes/bootstrap.tf index f9f94b491..94f54d381 100644 --- a/bare-metal/container-linux/kubernetes/bootstrap.tf +++ b/bare-metal/container-linux/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=1ea8fe7a85e87f7e851b4cd4ab8c4e9902b4a40c" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=796194583426593d1a62b6f1bf3f7ffed8fca140" cluster_name = var.cluster_name api_servers = [var.k8s_domain_name] diff --git a/bare-metal/container-linux/kubernetes/cl/controller.yaml b/bare-metal/container-linux/kubernetes/cl/controller.yaml index 74f3b8fcc..5fe9d7cc2 100644 --- a/bare-metal/container-linux/kubernetes/cl/controller.yaml +++ b/bare-metal/container-linux/kubernetes/cl/controller.yaml @@ -101,7 +101,7 @@ systemd: --mount volume=etc-iscsi,target=/etc/iscsi \ --volume usr-sbin-iscsiadm,kind=host,source=/usr/sbin/iscsiadm \ --mount volume=usr-sbin-iscsiadm,target=/sbin/iscsiadm \ - docker://k8s.gcr.io/hyperkube:v1.17.2 \ + docker://k8s.gcr.io/hyperkube:v1.17.3 \ --exec=/usr/local/bin/kubelet -- \ --anonymous-auth=false \ --authentication-token-webhook \ @@ -147,7 +147,7 @@ systemd: --volume script,kind=host,source=/opt/bootstrap/apply \ --mount volume=script,target=/apply \ --insecure-options=image \ - docker://k8s.gcr.io/hyperkube:v1.17.2 \ + docker://k8s.gcr.io/hyperkube:v1.17.3 \ --net=host \ --dns=host \ --exec=/apply diff --git a/bare-metal/container-linux/kubernetes/cl/worker.yaml b/bare-metal/container-linux/kubernetes/cl/worker.yaml index f07e0bb39..5a1db8ac5 100644 --- a/bare-metal/container-linux/kubernetes/cl/worker.yaml +++ b/bare-metal/container-linux/kubernetes/cl/worker.yaml @@ -76,7 +76,7 @@ systemd: --mount volume=etc-iscsi,target=/etc/iscsi \ --volume usr-sbin-iscsiadm,kind=host,source=/usr/sbin/iscsiadm \ --mount volume=usr-sbin-iscsiadm,target=/sbin/iscsiadm \ - docker://k8s.gcr.io/hyperkube:v1.17.2 \ + docker://k8s.gcr.io/hyperkube:v1.17.3 \ --exec=/usr/local/bin/kubelet -- \ --anonymous-auth=false \ --authentication-token-webhook \ diff --git a/bare-metal/fedora-coreos/kubernetes/README.md b/bare-metal/fedora-coreos/kubernetes/README.md index 329d38bfd..46a9ed4e9 100644 --- a/bare-metal/fedora-coreos/kubernetes/README.md +++ b/bare-metal/fedora-coreos/kubernetes/README.md @@ -11,7 +11,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features -* Kubernetes v1.17.2 (upstream) +* Kubernetes v1.17.3 (upstream) * Single or multi-master, [Calico](https://www.projectcalico.org/) or [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/) * Advanced features like [snippets](https://typhoon.psdn.io/advanced/customization/#container-linux) customization diff --git a/bare-metal/fedora-coreos/kubernetes/bootstrap.tf b/bare-metal/fedora-coreos/kubernetes/bootstrap.tf index 52c3e5f9c..bb5cc0880 100644 --- a/bare-metal/fedora-coreos/kubernetes/bootstrap.tf +++ b/bare-metal/fedora-coreos/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=1ea8fe7a85e87f7e851b4cd4ab8c4e9902b4a40c" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=796194583426593d1a62b6f1bf3f7ffed8fca140" cluster_name = var.cluster_name api_servers = [var.k8s_domain_name] diff --git a/bare-metal/fedora-coreos/kubernetes/fcc/controller.yaml b/bare-metal/fedora-coreos/kubernetes/fcc/controller.yaml index 5f9d34143..d38e9c899 100644 --- a/bare-metal/fedora-coreos/kubernetes/fcc/controller.yaml +++ b/bare-metal/fedora-coreos/kubernetes/fcc/controller.yaml @@ -80,7 +80,7 @@ systemd: --volume /opt/cni/bin:/opt/cni/bin:z \ --volume /etc/iscsi:/etc/iscsi \ --volume /sbin/iscsiadm:/sbin/iscsiadm \ - k8s.gcr.io/hyperkube:v1.17.2 kubelet \ + k8s.gcr.io/hyperkube:v1.17.3 kubelet \ --anonymous-auth=false \ --authentication-token-webhook \ --authorization-mode=Webhook \ @@ -134,7 +134,7 @@ systemd: --volume /opt/bootstrap/assets:/assets:ro,Z \ --volume /opt/bootstrap/apply:/apply:ro,Z \ --entrypoint=/apply \ - k8s.gcr.io/hyperkube:v1.17.2 + k8s.gcr.io/hyperkube:v1.17.3 ExecStartPost=/bin/touch /opt/bootstrap/bootstrap.done ExecStartPost=-/usr/bin/podman stop bootstrap storage: diff --git a/bare-metal/fedora-coreos/kubernetes/fcc/worker.yaml b/bare-metal/fedora-coreos/kubernetes/fcc/worker.yaml index 0360a68ba..c95bea1e3 100644 --- a/bare-metal/fedora-coreos/kubernetes/fcc/worker.yaml +++ b/bare-metal/fedora-coreos/kubernetes/fcc/worker.yaml @@ -50,7 +50,7 @@ systemd: --volume /opt/cni/bin:/opt/cni/bin:z \ --volume /etc/iscsi:/etc/iscsi \ --volume /sbin/iscsiadm:/sbin/iscsiadm \ - k8s.gcr.io/hyperkube:v1.17.2 kubelet \ + k8s.gcr.io/hyperkube:v1.17.3 kubelet \ --anonymous-auth=false \ --authentication-token-webhook \ --authorization-mode=Webhook \ diff --git a/digital-ocean/container-linux/kubernetes/README.md b/digital-ocean/container-linux/kubernetes/README.md index 0a4a61ea4..b5845e625 100644 --- a/digital-ocean/container-linux/kubernetes/README.md +++ b/digital-ocean/container-linux/kubernetes/README.md @@ -11,7 +11,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features -* Kubernetes v1.17.2 (upstream) +* Kubernetes v1.17.3 (upstream) * Single or multi-master, [Calico](https://www.projectcalico.org/) or [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/) * Advanced features like [snippets](https://typhoon.psdn.io/advanced/customization/#container-linux) customization diff --git a/digital-ocean/container-linux/kubernetes/bootstrap.tf b/digital-ocean/container-linux/kubernetes/bootstrap.tf index c2811e89c..481b44189 100644 --- a/digital-ocean/container-linux/kubernetes/bootstrap.tf +++ b/digital-ocean/container-linux/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=1ea8fe7a85e87f7e851b4cd4ab8c4e9902b4a40c" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=796194583426593d1a62b6f1bf3f7ffed8fca140" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/digital-ocean/container-linux/kubernetes/cl/controller.yaml b/digital-ocean/container-linux/kubernetes/cl/controller.yaml index 61cde9dcf..c011427d8 100644 --- a/digital-ocean/container-linux/kubernetes/cl/controller.yaml +++ b/digital-ocean/container-linux/kubernetes/cl/controller.yaml @@ -99,7 +99,7 @@ systemd: --mount volume=var-log,target=/var/log \ --volume opt-cni-bin,kind=host,source=/opt/cni/bin \ --mount volume=opt-cni-bin,target=/opt/cni/bin \ - docker://k8s.gcr.io/hyperkube:v1.17.2 \ + docker://k8s.gcr.io/hyperkube:v1.17.3 \ --exec=/usr/local/bin/kubelet -- \ --anonymous-auth=false \ --authentication-token-webhook \ @@ -144,7 +144,7 @@ systemd: --volume script,kind=host,source=/opt/bootstrap/apply \ --mount volume=script,target=/apply \ --insecure-options=image \ - docker://k8s.gcr.io/hyperkube:v1.17.2 \ + docker://k8s.gcr.io/hyperkube:v1.17.3 \ --net=host \ --dns=host \ --exec=/apply diff --git a/digital-ocean/container-linux/kubernetes/cl/worker.yaml b/digital-ocean/container-linux/kubernetes/cl/worker.yaml index c6ef316f7..94714efbc 100644 --- a/digital-ocean/container-linux/kubernetes/cl/worker.yaml +++ b/digital-ocean/container-linux/kubernetes/cl/worker.yaml @@ -74,7 +74,7 @@ systemd: --mount volume=var-log,target=/var/log \ --volume opt-cni-bin,kind=host,source=/opt/cni/bin \ --mount volume=opt-cni-bin,target=/opt/cni/bin \ - docker://k8s.gcr.io/hyperkube:v1.17.2 \ + docker://k8s.gcr.io/hyperkube:v1.17.3 \ --exec=/usr/local/bin/kubelet -- \ --anonymous-auth=false \ --authentication-token-webhook \ @@ -132,7 +132,7 @@ storage: --volume config,kind=host,source=/etc/kubernetes \ --mount volume=config,target=/etc/kubernetes \ --insecure-options=image \ - docker://k8s.gcr.io/hyperkube:v1.17.2 \ + docker://k8s.gcr.io/hyperkube:v1.17.3 \ --net=host \ --dns=host \ -- \ diff --git a/docs/advanced/worker-pools.md b/docs/advanced/worker-pools.md index c81ca0cbb..07d022a1f 100644 --- a/docs/advanced/worker-pools.md +++ b/docs/advanced/worker-pools.md @@ -79,7 +79,7 @@ Create a cluster following the Azure [tutorial](../cl/azure.md#cluster). Define ```tf module "ramius-worker-pool" { - source = "git::https://github.com/poseidon/typhoon//azure/container-linux/kubernetes/workers?ref=v1.17.2" + source = "git::https://github.com/poseidon/typhoon//azure/container-linux/kubernetes/workers?ref=v1.17.3" # Azure region = module.ramius.region @@ -145,7 +145,7 @@ Create a cluster following the Google Cloud [tutorial](../cl/google-cloud.md#clu ```tf module "yavin-worker-pool" { - source = "git::https://github.com/poseidon/typhoon//google-cloud/container-linux/kubernetes/workers?ref=v1.17.2" + source = "git::https://github.com/poseidon/typhoon//google-cloud/container-linux/kubernetes/workers?ref=v1.17.3" # Google Cloud region = "europe-west2" @@ -176,11 +176,11 @@ Verify a managed instance group of workers joins the cluster within a few minute ``` $ kubectl get nodes NAME STATUS AGE VERSION -yavin-controller-0.c.example-com.internal Ready 6m v1.17.2 -yavin-worker-jrbf.c.example-com.internal Ready 5m v1.17.2 -yavin-worker-mzdm.c.example-com.internal Ready 5m v1.17.2 -yavin-16x-worker-jrbf.c.example-com.internal Ready 3m v1.17.2 -yavin-16x-worker-mzdm.c.example-com.internal Ready 3m v1.17.2 +yavin-controller-0.c.example-com.internal Ready 6m v1.17.3 +yavin-worker-jrbf.c.example-com.internal Ready 5m v1.17.3 +yavin-worker-mzdm.c.example-com.internal Ready 5m v1.17.3 +yavin-16x-worker-jrbf.c.example-com.internal Ready 3m v1.17.3 +yavin-16x-worker-mzdm.c.example-com.internal Ready 3m v1.17.3 ``` ### Variables diff --git a/docs/cl/aws.md b/docs/cl/aws.md index ce5a9eeda..d388b323f 100644 --- a/docs/cl/aws.md +++ b/docs/cl/aws.md @@ -1,6 +1,6 @@ # AWS -In this tutorial, we'll create a Kubernetes v1.17.2 cluster on AWS with Container Linux. +In this tutorial, we'll create a Kubernetes v1.17.3 cluster on AWS with Container Linux. We'll declare a Kubernetes cluster using the Typhoon Terraform module. Then apply the changes to create a VPC, gateway, subnets, security groups, controller instances, worker auto-scaling group, network load balancer, and TLS assets. @@ -70,7 +70,7 @@ Define a Kubernetes cluster using the module `aws/container-linux/kubernetes`. ```tf module "tempest" { - source = "git::https://github.com/poseidon/typhoon//aws/container-linux/kubernetes?ref=v1.17.2" + source = "git::https://github.com/poseidon/typhoon//aws/container-linux/kubernetes?ref=v1.17.3" # AWS cluster_name = "tempest" @@ -143,9 +143,9 @@ List nodes in the cluster. $ export KUBECONFIG=/home/user/.kube/configs/tempest-config $ kubectl get nodes NAME STATUS ROLES AGE VERSION -ip-10-0-3-155 Ready 10m v1.17.2 -ip-10-0-26-65 Ready 10m v1.17.2 -ip-10-0-41-21 Ready 10m v1.17.2 +ip-10-0-3-155 Ready 10m v1.17.3 +ip-10-0-26-65 Ready 10m v1.17.3 +ip-10-0-41-21 Ready 10m v1.17.3 ``` List the pods. diff --git a/docs/cl/azure.md b/docs/cl/azure.md index 3c191fdc8..7761929fa 100644 --- a/docs/cl/azure.md +++ b/docs/cl/azure.md @@ -3,7 +3,7 @@ !!! danger Typhoon for Azure is alpha. For production, use AWS, Google Cloud, or bare-metal. As Azure matures, check [errata](https://github.com/poseidon/typhoon/wiki/Errata) for known shortcomings. -In this tutorial, we'll create a Kubernetes v1.17.2 cluster on Azure with Container Linux. +In this tutorial, we'll create a Kubernetes v1.17.3 cluster on Azure with Container Linux. We'll declare a Kubernetes cluster using the Typhoon Terraform module. Then apply the changes to create a resource group, virtual network, subnets, security groups, controller availability set, worker scale set, load balancer, and TLS assets. @@ -66,7 +66,7 @@ Define a Kubernetes cluster using the module `azure/container-linux/kubernetes`. ```tf module "ramius" { - source = "git::https://github.com/poseidon/typhoon//azure/container-linux/kubernetes?ref=v1.17.2" + source = "git::https://github.com/poseidon/typhoon//azure/container-linux/kubernetes?ref=v1.17.3" # Azure cluster_name = "ramius" @@ -140,9 +140,9 @@ List nodes in the cluster. $ export KUBECONFIG=/home/user/.kube/configs/ramius-config $ kubectl get nodes NAME STATUS ROLES AGE VERSION -ramius-controller-0 Ready 24m v1.17.2 -ramius-worker-000001 Ready 25m v1.17.2 -ramius-worker-000002 Ready 24m v1.17.2 +ramius-controller-0 Ready 24m v1.17.3 +ramius-worker-000001 Ready 25m v1.17.3 +ramius-worker-000002 Ready 24m v1.17.3 ``` List the pods. diff --git a/docs/cl/bare-metal.md b/docs/cl/bare-metal.md index e1b2990c9..c63500bb5 100644 --- a/docs/cl/bare-metal.md +++ b/docs/cl/bare-metal.md @@ -1,6 +1,6 @@ # Bare-Metal -In this tutorial, we'll network boot and provision a Kubernetes v1.17.2 cluster on bare-metal with Container Linux. +In this tutorial, we'll network boot and provision a Kubernetes v1.17.3 cluster on bare-metal with Container Linux. First, we'll deploy a [Matchbox](https://github.com/poseidon/matchbox) service and setup a network boot environment. Then, we'll declare a Kubernetes cluster using the Typhoon Terraform module and power on machines. On PXE boot, machines will install Container Linux to disk, reboot into the disk install, and provision themselves as Kubernetes controllers or workers via Ignition. @@ -160,7 +160,7 @@ Define a Kubernetes cluster using the module `bare-metal/container-linux/kuberne ```tf module "mercury" { - source = "git::https://github.com/poseidon/typhoon//bare-metal/container-linux/kubernetes?ref=v1.17.2" + source = "git::https://github.com/poseidon/typhoon//bare-metal/container-linux/kubernetes?ref=v1.17.3" # bare-metal cluster_name = "mercury" @@ -299,9 +299,9 @@ List nodes in the cluster. $ export KUBECONFIG=/home/user/.kube/configs/mercury-config $ kubectl get nodes NAME STATUS ROLES AGE VERSION -node1.example.com Ready 10m v1.17.2 -node2.example.com Ready 10m v1.17.2 -node3.example.com Ready 10m v1.17.2 +node1.example.com Ready 10m v1.17.3 +node2.example.com Ready 10m v1.17.3 +node3.example.com Ready 10m v1.17.3 ``` List the pods. diff --git a/docs/cl/digital-ocean.md b/docs/cl/digital-ocean.md index 2e9c61005..07774b835 100644 --- a/docs/cl/digital-ocean.md +++ b/docs/cl/digital-ocean.md @@ -1,6 +1,6 @@ # Digital Ocean -In this tutorial, we'll create a Kubernetes v1.17.2 cluster on DigitalOcean with Container Linux. +In this tutorial, we'll create a Kubernetes v1.17.3 cluster on DigitalOcean with Container Linux. We'll declare a Kubernetes cluster using the Typhoon Terraform module. Then apply the changes to create controller droplets, worker droplets, DNS records, tags, and TLS assets. @@ -65,7 +65,7 @@ Define a Kubernetes cluster using the module `digital-ocean/container-linux/kube ```tf module "nemo" { - source = "git::https://github.com/poseidon/typhoon//digital-ocean/container-linux/kubernetes?ref=v1.17.2" + source = "git::https://github.com/poseidon/typhoon//digital-ocean/container-linux/kubernetes?ref=v1.17.3" # Digital Ocean cluster_name = "nemo" @@ -138,9 +138,9 @@ List nodes in the cluster. $ export KUBECONFIG=/home/user/.kube/configs/nemo-config $ kubectl get nodes NAME STATUS ROLES AGE VERSION -10.132.110.130 Ready 10m v1.17.2 -10.132.115.81 Ready 10m v1.17.2 -10.132.124.107 Ready 10m v1.17.2 +10.132.110.130 Ready 10m v1.17.3 +10.132.115.81 Ready 10m v1.17.3 +10.132.124.107 Ready 10m v1.17.3 ``` List the pods. diff --git a/docs/cl/google-cloud.md b/docs/cl/google-cloud.md index 00f00a35b..661cdac9f 100644 --- a/docs/cl/google-cloud.md +++ b/docs/cl/google-cloud.md @@ -1,6 +1,6 @@ # Google Cloud -In this tutorial, we'll create a Kubernetes v1.17.2 cluster on Google Compute Engine with Container Linux. +In this tutorial, we'll create a Kubernetes v1.17.3 cluster on Google Compute Engine with Container Linux. We'll declare a Kubernetes cluster using the Typhoon Terraform module. Then apply the changes to create a network, firewall rules, health checks, controller instances, worker managed instance group, load balancers, and TLS assets. @@ -71,7 +71,7 @@ Define a Kubernetes cluster using the module `google-cloud/container-linux/kuber ```tf module "yavin" { - source = "git::https://github.com/poseidon/typhoon//google-cloud/container-linux/kubernetes?ref=v1.17.2" + source = "git::https://github.com/poseidon/typhoon//google-cloud/container-linux/kubernetes?ref=v1.17.3" # Google Cloud cluster_name = "yavin" @@ -170,9 +170,9 @@ List nodes in the cluster. $ export KUBECONFIG=/home/user/.kube/configs/yavin-config $ kubectl get nodes NAME ROLES STATUS AGE VERSION -yavin-controller-0.c.example-com.internal Ready 6m v1.17.2 -yavin-worker-jrbf.c.example-com.internal Ready 5m v1.17.2 -yavin-worker-mzdm.c.example-com.internal Ready 5m v1.17.2 +yavin-controller-0.c.example-com.internal Ready 6m v1.17.3 +yavin-worker-jrbf.c.example-com.internal Ready 5m v1.17.3 +yavin-worker-mzdm.c.example-com.internal Ready 5m v1.17.3 ``` List the pods. diff --git a/docs/fedora-coreos/aws.md b/docs/fedora-coreos/aws.md index 5676db0ed..52913b0f5 100644 --- a/docs/fedora-coreos/aws.md +++ b/docs/fedora-coreos/aws.md @@ -3,7 +3,7 @@ !!! danger Typhoon for Fedora CoreOS is an alpha. Please report Fedora CoreOS bugs to [Fedora](https://github.com/coreos/fedora-coreos-tracker/issues) and Typhoon issues to Typhoon. -In this tutorial, we'll create a Kubernetes v1.17.2 cluster on AWS with Fedora CoreOS. +In this tutorial, we'll create a Kubernetes v1.17.3 cluster on AWS with Fedora CoreOS. We'll declare a Kubernetes cluster using the Typhoon Terraform module. Then apply the changes to create a VPC, gateway, subnets, security groups, controller instances, worker auto-scaling group, network load balancer, and TLS assets. @@ -73,7 +73,7 @@ Define a Kubernetes cluster using the module `aws/fedora-coreos/kubernetes`. ```tf module "tempest" { - source = "git::https://github.com/poseidon/typhoon//aws/fedora-coreos/kubernetes?ref=v1.17.2" + source = "git::https://github.com/poseidon/typhoon//aws/fedora-coreos/kubernetes?ref=v1.17.3" # AWS cluster_name = "tempest" @@ -146,9 +146,9 @@ List nodes in the cluster. $ export KUBECONFIG=/home/user/.kube/configs/tempest-config $ kubectl get nodes NAME STATUS ROLES AGE VERSION -ip-10-0-3-155 Ready 10m v1.17.2 -ip-10-0-26-65 Ready 10m v1.17.2 -ip-10-0-41-21 Ready 10m v1.17.2 +ip-10-0-3-155 Ready 10m v1.17.3 +ip-10-0-26-65 Ready 10m v1.17.3 +ip-10-0-41-21 Ready 10m v1.17.3 ``` List the pods. diff --git a/docs/fedora-coreos/bare-metal.md b/docs/fedora-coreos/bare-metal.md index 580c5022a..54e3faf9d 100644 --- a/docs/fedora-coreos/bare-metal.md +++ b/docs/fedora-coreos/bare-metal.md @@ -3,7 +3,7 @@ !!! danger Typhoon for Fedora CoreOS is an alpha. Please report Fedora CoreOS bugs to [Fedora](https://github.com/coreos/fedora-coreos-tracker/issues) and Typhoon issues to Typhoon. -In this tutorial, we'll network boot and provision a Kubernetes v1.17.2 cluster on bare-metal with Fedora CoreOS. +In this tutorial, we'll network boot and provision a Kubernetes v1.17.3 cluster on bare-metal with Fedora CoreOS. First, we'll deploy a [Matchbox](https://github.com/poseidon/matchbox) service and setup a network boot environment. Then, we'll declare a Kubernetes cluster using the Typhoon Terraform module and power on machines. On PXE boot, machines will install Fedora CoreOS to disk, reboot into the disk install, and provision themselves as Kubernetes controllers or workers via Ignition. @@ -163,7 +163,7 @@ Define a Kubernetes cluster using the module `bare-metal/fedora-coreos/kubernete ```tf module "mercury" { - source = "git::https://github.com/poseidon/typhoon//bare-metal/fedora-coreos/kubernetes?ref=v1.17.2" + source = "git::https://github.com/poseidon/typhoon//bare-metal/fedora-coreos/kubernetes?ref=v1.17.3" # bare-metal cluster_name = "mercury" @@ -292,9 +292,9 @@ List nodes in the cluster. $ export KUBECONFIG=/home/user/.kube/configs/mercury-config $ kubectl get nodes NAME STATUS ROLES AGE VERSION -node1.example.com Ready 10m v1.17.2 -node2.example.com Ready 10m v1.17.2 -node3.example.com Ready 10m v1.17.2 +node1.example.com Ready 10m v1.17.3 +node2.example.com Ready 10m v1.17.3 +node3.example.com Ready 10m v1.17.3 ``` List the pods. diff --git a/docs/fedora-coreos/google-cloud.md b/docs/fedora-coreos/google-cloud.md index 14e2eaf10..9e05041dd 100644 --- a/docs/fedora-coreos/google-cloud.md +++ b/docs/fedora-coreos/google-cloud.md @@ -3,7 +3,7 @@ !!! danger Typhoon for Fedora CoreOS is an alpha. Please report Fedora CoreOS bugs to [Fedora](https://github.com/coreos/fedora-coreos-tracker/issues) and Typhoon issues to Typhoon. -In this tutorial, we'll create a Kubernetes v1.17.2 cluster on Google Compute Engine with Fedora CoreOS. +In this tutorial, we'll create a Kubernetes v1.17.3 cluster on Google Compute Engine with Fedora CoreOS. We'll declare a Kubernetes cluster using the Typhoon Terraform module. Then apply the changes to create a network, firewall rules, health checks, controller instances, worker managed instance group, load balancers, and TLS assets. @@ -168,9 +168,9 @@ List nodes in the cluster. $ export KUBECONFIG=/home/user/.kube/configs/yavin-config $ kubectl get nodes NAME ROLES STATUS AGE VERSION -yavin-controller-0.c.example-com.internal Ready 6m v1.17.2 -yavin-worker-jrbf.c.example-com.internal Ready 5m v1.17.2 -yavin-worker-mzdm.c.example-com.internal Ready 5m v1.17.2 +yavin-controller-0.c.example-com.internal Ready 6m v1.17.3 +yavin-worker-jrbf.c.example-com.internal Ready 5m v1.17.3 +yavin-worker-mzdm.c.example-com.internal Ready 5m v1.17.3 ``` List the pods. diff --git a/docs/index.md b/docs/index.md index addaf5175..0bbc41b4b 100644 --- a/docs/index.md +++ b/docs/index.md @@ -11,7 +11,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features -* Kubernetes v1.17.2 (upstream) +* Kubernetes v1.17.3 (upstream) * Single or multi-master, [Calico](https://www.projectcalico.org/) or [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/) * Advanced features like [worker pools](advanced/worker-pools/), [preemptible](cl/google-cloud/#preemption) workers, and [snippets](advanced/customization/#container-linux) customization @@ -56,7 +56,7 @@ Define a Kubernetes cluster by using the Terraform module for your chosen platfo ```tf module "yavin" { - source = "git::https://github.com/poseidon/typhoon//google-cloud/container-linux/kubernetes?ref=v1.17.2" + source = "git::https://github.com/poseidon/typhoon//google-cloud/container-linux/kubernetes?ref=v1.17.3" # Google Cloud cluster_name = "yavin" @@ -94,9 +94,9 @@ In 4-8 minutes (varies by platform), the cluster will be ready. This Google Clou $ export KUBECONFIG=/home/user/.kube/configs/yavin-config $ kubectl get nodes NAME ROLES STATUS AGE VERSION -yavin-controller-0.c.example-com.internal Ready 6m v1.17.2 -yavin-worker-jrbf.c.example-com.internal Ready 5m v1.17.2 -yavin-worker-mzdm.c.example-com.internal Ready 5m v1.17.2 +yavin-controller-0.c.example-com.internal Ready 6m v1.17.3 +yavin-worker-jrbf.c.example-com.internal Ready 5m v1.17.3 +yavin-worker-mzdm.c.example-com.internal Ready 5m v1.17.3 ``` List the pods. diff --git a/docs/topics/maintenance.md b/docs/topics/maintenance.md index b583d1be1..114bfc93d 100644 --- a/docs/topics/maintenance.md +++ b/docs/topics/maintenance.md @@ -18,7 +18,7 @@ module "yavin" { } module "mercury" { - source = "git::https://github.com/poseidon/typhoon//bare-metal/container-linux/kubernetes?ref=v1.17.2" + source = "git::https://github.com/poseidon/typhoon//bare-metal/container-linux/kubernetes?ref=v1.17.3" ... } ``` @@ -279,15 +279,15 @@ Typhoon modules have been adapted for Terraform v0.12. Provider plugins requirem | Typhoon Release | Terraform version | |-------------------|---------------------| -| v1.17.2 - ? | v0.12.x | -| v1.10.3 - v1.17.2 | v0.11.x | +| v1.17.3 - ? | v0.12.x | +| v1.10.3 - v1.17.3 | v0.11.x | | v1.9.2 - v1.10.2 | v0.10.4+ or v0.11.x | | v1.7.3 - v1.9.1 | v0.10.x | | v1.6.4 - v1.7.2 | v0.9.x | ### New users -New users can start with Terraform v0.12.x and follow the docs for Typhoon v1.17.2+ without issue. +New users can start with Terraform v0.12.x and follow the docs for Typhoon v1.17.3+ without issue. ### Existing users diff --git a/google-cloud/container-linux/kubernetes/README.md b/google-cloud/container-linux/kubernetes/README.md index cfb94a30f..fe81ec8f4 100644 --- a/google-cloud/container-linux/kubernetes/README.md +++ b/google-cloud/container-linux/kubernetes/README.md @@ -11,7 +11,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features -* Kubernetes v1.17.2 (upstream) +* Kubernetes v1.17.3 (upstream) * Single or multi-master, [Calico](https://www.projectcalico.org/) or [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/) * Advanced features like [worker pools](https://typhoon.psdn.io/advanced/worker-pools/), [preemptible](https://typhoon.psdn.io/cl/google-cloud/#preemption) workers, and [snippets](https://typhoon.psdn.io/advanced/customization/#container-linux) customization diff --git a/google-cloud/container-linux/kubernetes/bootstrap.tf b/google-cloud/container-linux/kubernetes/bootstrap.tf index ebebc82f7..3bd0ea094 100644 --- a/google-cloud/container-linux/kubernetes/bootstrap.tf +++ b/google-cloud/container-linux/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=1ea8fe7a85e87f7e851b4cd4ab8c4e9902b4a40c" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=796194583426593d1a62b6f1bf3f7ffed8fca140" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/google-cloud/container-linux/kubernetes/cl/controller.yaml b/google-cloud/container-linux/kubernetes/cl/controller.yaml index 8e005c6c3..d7f0a744d 100644 --- a/google-cloud/container-linux/kubernetes/cl/controller.yaml +++ b/google-cloud/container-linux/kubernetes/cl/controller.yaml @@ -88,7 +88,7 @@ systemd: --mount volume=var-log,target=/var/log \ --volume opt-cni-bin,kind=host,source=/opt/cni/bin \ --mount volume=opt-cni-bin,target=/opt/cni/bin \ - docker://k8s.gcr.io/hyperkube:v1.17.2 \ + docker://k8s.gcr.io/hyperkube:v1.17.3 \ --exec=/usr/local/bin/kubelet -- \ --anonymous-auth=false \ --authentication-token-webhook \ @@ -132,7 +132,7 @@ systemd: --volume script,kind=host,source=/opt/bootstrap/apply \ --mount volume=script,target=/apply \ --insecure-options=image \ - docker://k8s.gcr.io/hyperkube:v1.17.2 \ + docker://k8s.gcr.io/hyperkube:v1.17.3 \ --net=host \ --dns=host \ --exec=/apply diff --git a/google-cloud/container-linux/kubernetes/workers/cl/worker.yaml b/google-cloud/container-linux/kubernetes/workers/cl/worker.yaml index 1cfb34f74..766fe98d7 100644 --- a/google-cloud/container-linux/kubernetes/workers/cl/worker.yaml +++ b/google-cloud/container-linux/kubernetes/workers/cl/worker.yaml @@ -63,7 +63,7 @@ systemd: --mount volume=var-log,target=/var/log \ --volume opt-cni-bin,kind=host,source=/opt/cni/bin \ --mount volume=opt-cni-bin,target=/opt/cni/bin \ - docker://k8s.gcr.io/hyperkube:v1.17.2 \ + docker://k8s.gcr.io/hyperkube:v1.17.3 \ --exec=/usr/local/bin/kubelet -- \ --anonymous-auth=false \ --authentication-token-webhook \ @@ -126,7 +126,7 @@ storage: --volume config,kind=host,source=/etc/kubernetes \ --mount volume=config,target=/etc/kubernetes \ --insecure-options=image \ - docker://k8s.gcr.io/hyperkube:v1.17.2 \ + docker://k8s.gcr.io/hyperkube:v1.17.3 \ --net=host \ --dns=host \ -- \ diff --git a/google-cloud/fedora-coreos/kubernetes/README.md b/google-cloud/fedora-coreos/kubernetes/README.md index cfb94a30f..fe81ec8f4 100644 --- a/google-cloud/fedora-coreos/kubernetes/README.md +++ b/google-cloud/fedora-coreos/kubernetes/README.md @@ -11,7 +11,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features -* Kubernetes v1.17.2 (upstream) +* Kubernetes v1.17.3 (upstream) * Single or multi-master, [Calico](https://www.projectcalico.org/) or [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/) * Advanced features like [worker pools](https://typhoon.psdn.io/advanced/worker-pools/), [preemptible](https://typhoon.psdn.io/cl/google-cloud/#preemption) workers, and [snippets](https://typhoon.psdn.io/advanced/customization/#container-linux) customization diff --git a/google-cloud/fedora-coreos/kubernetes/bootstrap.tf b/google-cloud/fedora-coreos/kubernetes/bootstrap.tf index 96576ffed..9d9a5155d 100644 --- a/google-cloud/fedora-coreos/kubernetes/bootstrap.tf +++ b/google-cloud/fedora-coreos/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=1ea8fe7a85e87f7e851b4cd4ab8c4e9902b4a40c" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=796194583426593d1a62b6f1bf3f7ffed8fca140" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/google-cloud/fedora-coreos/kubernetes/fcc/controller.yaml b/google-cloud/fedora-coreos/kubernetes/fcc/controller.yaml index 1b97d933d..e0131f0fb 100644 --- a/google-cloud/fedora-coreos/kubernetes/fcc/controller.yaml +++ b/google-cloud/fedora-coreos/kubernetes/fcc/controller.yaml @@ -79,7 +79,7 @@ systemd: --volume /var/log:/var/log \ --volume /var/run/lock:/var/run/lock:z \ --volume /opt/cni/bin:/opt/cni/bin:z \ - k8s.gcr.io/hyperkube:v1.17.2 kubelet \ + k8s.gcr.io/hyperkube:v1.17.3 kubelet \ --anonymous-auth=false \ --authentication-token-webhook \ --authorization-mode=Webhook \ @@ -123,7 +123,7 @@ systemd: --volume /opt/bootstrap/assets:/assets:ro,Z \ --volume /opt/bootstrap/apply:/apply:ro,Z \ --entrypoint=/apply \ - k8s.gcr.io/hyperkube:v1.17.2 + k8s.gcr.io/hyperkube:v1.17.3 ExecStartPost=/bin/touch /opt/bootstrap/bootstrap.done ExecStartPost=-/usr/bin/podman stop bootstrap storage: diff --git a/google-cloud/fedora-coreos/kubernetes/workers/fcc/worker.yaml b/google-cloud/fedora-coreos/kubernetes/workers/fcc/worker.yaml index 3e3b6edc9..8a1d02cfc 100644 --- a/google-cloud/fedora-coreos/kubernetes/workers/fcc/worker.yaml +++ b/google-cloud/fedora-coreos/kubernetes/workers/fcc/worker.yaml @@ -49,7 +49,7 @@ systemd: --volume /var/log:/var/log \ --volume /var/run/lock:/var/run/lock:z \ --volume /opt/cni/bin:/opt/cni/bin:z \ - k8s.gcr.io/hyperkube:v1.17.2 kubelet \ + k8s.gcr.io/hyperkube:v1.17.3 kubelet \ --anonymous-auth=false \ --authentication-token-webhook \ --authorization-mode=Webhook \ From 49d3b9e6b3b2ba30e6833bda2620024c23bd7ff8 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Tue, 11 Feb 2020 21:13:58 -0800 Subject: [PATCH 331/523] Set docker log driver to json-file on Fedora CoreOS * Fix the last minor issue for Fedora CoreOS clusters to pass CNCF's Kubernetes conformance tests * Kubelet supports a seldom used feature `kubectl logs --limit-bytes=N` to trim a log stream to a desired length. Kubelet handles this in the CRI driver. The Kubelet docker shim only supports the limit bytes feature when Docker is configured with the default `json-file` logging driver * CNCF conformance tests started requiring limit-bytes be supported, indirectly forcing the log driver choice until either the Kubelet or the conformance tests are fixed * Fedora CoreOS defaults Docker to use `journald` (desired). For now, as a workaround to offer conformant clusters, the log driver can be set back to `json-file`. RHEL CoreOS likely won't have noticed the non-conformance since its using crio runtime * https://github.com/kubernetes/kubernetes/issues/86367 Note: When upstream has a fix, the aim is to drop the docker config override and use the journald default --- CHANGES.md | 6 ++++-- aws/fedora-coreos/kubernetes/fcc/controller.yaml | 13 +++++++++++++ .../kubernetes/workers/fcc/worker.yaml | 13 +++++++++++++ .../fedora-coreos/kubernetes/fcc/controller.yaml | 13 +++++++++++++ bare-metal/fedora-coreos/kubernetes/fcc/worker.yaml | 13 +++++++++++++ .../fedora-coreos/kubernetes/fcc/controller.yaml | 13 +++++++++++++ .../kubernetes/workers/fcc/worker.yaml | 13 +++++++++++++ 7 files changed, 82 insertions(+), 2 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 74751ae5e..f8e3df23a 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -6,6 +6,8 @@ Notable changes between versions. * Kubernetes [v1.17.3](https://github.com/kubernetes/kubernetes/blob/master/CHANGELOG/CHANGELOG-1.17.md#v1173) * Update Calico from v3.11.2 to v3.12.0 +* Allow Fedora CoreOS clusters to pass CNCF conformance suite + * Set Docker log driver to `json-file` as a workaround #### Bare-Metal @@ -13,8 +15,8 @@ Notable changes between versions. #### Google Cloud -* Add Terraform module for Fedora CoreOS ([#632](https://github.com/poseidon/typhoon/pull/632)) -* Add support for Flatcar Container Linux ([#639](https://github.com/poseidon/typhoon/pull/639)) +* Add initial Terraform module for Fedora CoreOS ([#632](https://github.com/poseidon/typhoon/pull/632)) +* Add initial support for Flatcar Container Linux ([#639](https://github.com/poseidon/typhoon/pull/639)) #### Addons diff --git a/aws/fedora-coreos/kubernetes/fcc/controller.yaml b/aws/fedora-coreos/kubernetes/fcc/controller.yaml index e0131f0fb..f9aaf6be8 100644 --- a/aws/fedora-coreos/kubernetes/fcc/controller.yaml +++ b/aws/fedora-coreos/kubernetes/fcc/controller.yaml @@ -182,6 +182,19 @@ storage: DefaultCPUAccounting=yes DefaultMemoryAccounting=yes DefaultBlockIOAccounting=yes + - path: /etc/sysconfig/docker + mode: 0644 + overwrite: true + contents: + inline: | + # Modify these options if you want to change the way the docker daemon runs + OPTIONS="--selinux-enabled \ + --log-driver=json-file \ + --live-restore \ + --default-ulimit nofile=1024:1024 \ + --init-path /usr/libexec/docker/docker-init \ + --userland-proxy-path /usr/libexec/docker/docker-proxy \ + " - path: /etc/etcd/etcd.env mode: 0644 contents: diff --git a/aws/fedora-coreos/kubernetes/workers/fcc/worker.yaml b/aws/fedora-coreos/kubernetes/workers/fcc/worker.yaml index 8a1d02cfc..7d1d536d2 100644 --- a/aws/fedora-coreos/kubernetes/workers/fcc/worker.yaml +++ b/aws/fedora-coreos/kubernetes/workers/fcc/worker.yaml @@ -98,6 +98,19 @@ storage: DefaultCPUAccounting=yes DefaultMemoryAccounting=yes DefaultBlockIOAccounting=yes + - path: /etc/sysconfig/docker + mode: 0644 + overwrite: true + contents: + inline: | + # Modify these options if you want to change the way the docker daemon runs + OPTIONS="--selinux-enabled \ + --log-driver=json-file \ + --live-restore \ + --default-ulimit nofile=1024:1024 \ + --init-path /usr/libexec/docker/docker-init \ + --userland-proxy-path /usr/libexec/docker/docker-proxy \ + " passwd: users: - name: core diff --git a/bare-metal/fedora-coreos/kubernetes/fcc/controller.yaml b/bare-metal/fedora-coreos/kubernetes/fcc/controller.yaml index d38e9c899..c8cc09792 100644 --- a/bare-metal/fedora-coreos/kubernetes/fcc/controller.yaml +++ b/bare-metal/fedora-coreos/kubernetes/fcc/controller.yaml @@ -193,6 +193,19 @@ storage: DefaultCPUAccounting=yes DefaultMemoryAccounting=yes DefaultBlockIOAccounting=yes + - path: /etc/sysconfig/docker + mode: 0644 + overwrite: true + contents: + inline: | + # Modify these options if you want to change the way the docker daemon runs + OPTIONS="--selinux-enabled \ + --log-driver=json-file \ + --live-restore \ + --default-ulimit nofile=1024:1024 \ + --init-path /usr/libexec/docker/docker-init \ + --userland-proxy-path /usr/libexec/docker/docker-proxy \ + " - path: /etc/etcd/etcd.env mode: 0644 contents: diff --git a/bare-metal/fedora-coreos/kubernetes/fcc/worker.yaml b/bare-metal/fedora-coreos/kubernetes/fcc/worker.yaml index c95bea1e3..49b05d538 100644 --- a/bare-metal/fedora-coreos/kubernetes/fcc/worker.yaml +++ b/bare-metal/fedora-coreos/kubernetes/fcc/worker.yaml @@ -106,6 +106,19 @@ storage: DefaultCPUAccounting=yes DefaultMemoryAccounting=yes DefaultBlockIOAccounting=yes + - path: /etc/sysconfig/docker + mode: 0644 + overwrite: true + contents: + inline: | + # Modify these options if you want to change the way the docker daemon runs + OPTIONS="--selinux-enabled \ + --log-driver=json-file \ + --live-restore \ + --default-ulimit nofile=1024:1024 \ + --init-path /usr/libexec/docker/docker-init \ + --userland-proxy-path /usr/libexec/docker/docker-proxy \ + " passwd: users: - name: core diff --git a/google-cloud/fedora-coreos/kubernetes/fcc/controller.yaml b/google-cloud/fedora-coreos/kubernetes/fcc/controller.yaml index e0131f0fb..f9aaf6be8 100644 --- a/google-cloud/fedora-coreos/kubernetes/fcc/controller.yaml +++ b/google-cloud/fedora-coreos/kubernetes/fcc/controller.yaml @@ -182,6 +182,19 @@ storage: DefaultCPUAccounting=yes DefaultMemoryAccounting=yes DefaultBlockIOAccounting=yes + - path: /etc/sysconfig/docker + mode: 0644 + overwrite: true + contents: + inline: | + # Modify these options if you want to change the way the docker daemon runs + OPTIONS="--selinux-enabled \ + --log-driver=json-file \ + --live-restore \ + --default-ulimit nofile=1024:1024 \ + --init-path /usr/libexec/docker/docker-init \ + --userland-proxy-path /usr/libexec/docker/docker-proxy \ + " - path: /etc/etcd/etcd.env mode: 0644 contents: diff --git a/google-cloud/fedora-coreos/kubernetes/workers/fcc/worker.yaml b/google-cloud/fedora-coreos/kubernetes/workers/fcc/worker.yaml index 8a1d02cfc..7d1d536d2 100644 --- a/google-cloud/fedora-coreos/kubernetes/workers/fcc/worker.yaml +++ b/google-cloud/fedora-coreos/kubernetes/workers/fcc/worker.yaml @@ -98,6 +98,19 @@ storage: DefaultCPUAccounting=yes DefaultMemoryAccounting=yes DefaultBlockIOAccounting=yes + - path: /etc/sysconfig/docker + mode: 0644 + overwrite: true + contents: + inline: | + # Modify these options if you want to change the way the docker daemon runs + OPTIONS="--selinux-enabled \ + --log-driver=json-file \ + --live-restore \ + --default-ulimit nofile=1024:1024 \ + --init-path /usr/libexec/docker/docker-init \ + --userland-proxy-path /usr/libexec/docker/docker-proxy \ + " passwd: users: - name: core From 008817b0aa542d49a77587e0afdb5805fde6cd18 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Thu, 13 Feb 2020 14:25:22 -0800 Subject: [PATCH 332/523] Promote Fedora CoreOS AWS/bare-metal to beta * Remove alpha warnings from docs headers --- CHANGES.md | 5 +++++ README.md | 4 ++-- docs/fedora-coreos/aws.md | 3 --- docs/fedora-coreos/bare-metal.md | 3 --- docs/index.md | 4 ++-- 5 files changed, 9 insertions(+), 10 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index f8e3df23a..a954b117c 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -9,8 +9,13 @@ Notable changes between versions. * Allow Fedora CoreOS clusters to pass CNCF conformance suite * Set Docker log driver to `json-file` as a workaround +#### AWS + +* Promote Fedora CoreOS to beta + #### Bare-Metal +* Promote Fedora CoreOS to beta * Add Fedora CoreOS kernel arguments initrd and console ([#640](https://github.com/poseidon/typhoon/pull/640)) #### Google Cloud diff --git a/README.md b/README.md index f7d26020a..eb4999e7c 100644 --- a/README.md +++ b/README.md @@ -33,8 +33,8 @@ Typhoon is available for [Fedora CoreOS](https://getfedora.org/coreos/). | Platform | Operating System | Terraform Module | Status | |---------------|------------------|------------------|--------| -| AWS | Fedora CoreOS | [aws/fedora-coreos/kubernetes](aws/fedora-coreos/kubernetes) | alpha | -| Bare-Metal | Fedora CoreOS | [bare-metal/fedora-coreos/kubernetes](bare-metal/fedora-coreos/kubernetes) | alpha | +| AWS | Fedora CoreOS | [aws/fedora-coreos/kubernetes](aws/fedora-coreos/kubernetes) | beta | +| Bare-Metal | Fedora CoreOS | [bare-metal/fedora-coreos/kubernetes](bare-metal/fedora-coreos/kubernetes) | beta | | Google Cloud | Fedora CoreOS | [google-cloud/fedora-coreos/kubernetes](google-cloud/fedora-coreos/kubernetes) | alpha | Typhoon is available for [Flatcar Container Linux](https://www.flatcar-linux.org/releases/). diff --git a/docs/fedora-coreos/aws.md b/docs/fedora-coreos/aws.md index 52913b0f5..cb708c2c2 100644 --- a/docs/fedora-coreos/aws.md +++ b/docs/fedora-coreos/aws.md @@ -1,8 +1,5 @@ # AWS -!!! danger - Typhoon for Fedora CoreOS is an alpha. Please report Fedora CoreOS bugs to [Fedora](https://github.com/coreos/fedora-coreos-tracker/issues) and Typhoon issues to Typhoon. - In this tutorial, we'll create a Kubernetes v1.17.3 cluster on AWS with Fedora CoreOS. We'll declare a Kubernetes cluster using the Typhoon Terraform module. Then apply the changes to create a VPC, gateway, subnets, security groups, controller instances, worker auto-scaling group, network load balancer, and TLS assets. diff --git a/docs/fedora-coreos/bare-metal.md b/docs/fedora-coreos/bare-metal.md index 54e3faf9d..a29557b46 100644 --- a/docs/fedora-coreos/bare-metal.md +++ b/docs/fedora-coreos/bare-metal.md @@ -1,8 +1,5 @@ # Bare-Metal -!!! danger - Typhoon for Fedora CoreOS is an alpha. Please report Fedora CoreOS bugs to [Fedora](https://github.com/coreos/fedora-coreos-tracker/issues) and Typhoon issues to Typhoon. - In this tutorial, we'll network boot and provision a Kubernetes v1.17.3 cluster on bare-metal with Fedora CoreOS. First, we'll deploy a [Matchbox](https://github.com/poseidon/matchbox) service and setup a network boot environment. Then, we'll declare a Kubernetes cluster using the Typhoon Terraform module and power on machines. On PXE boot, machines will install Fedora CoreOS to disk, reboot into the disk install, and provision themselves as Kubernetes controllers or workers via Ignition. diff --git a/docs/index.md b/docs/index.md index 0bbc41b4b..50ff8be80 100644 --- a/docs/index.md +++ b/docs/index.md @@ -33,8 +33,8 @@ Typhoon is available for [Fedora CoreOS](https://getfedora.org/coreos/). | Platform | Operating System | Terraform Module | Status | |---------------|------------------|------------------|--------| -| AWS | Fedora CoreOS | [aws/fedora-coreos/kubernetes](fedora-coreos/aws.md) | alpha | -| Bare-Metal | Fedora CoreOS | [bare-metal/fedora-coreos/kubernetes](fedora-coreos/bare-metal.md) | alpha | +| AWS | Fedora CoreOS | [aws/fedora-coreos/kubernetes](fedora-coreos/aws.md) | beta | +| Bare-Metal | Fedora CoreOS | [bare-metal/fedora-coreos/kubernetes](fedora-coreos/bare-metal.md) | beta | | Google Cloud | Fedora CoreOS | [google-cloud/fedora-coreos/kubernetes](google-cloud/fedora-coreos/kubernetes) | alpha | Typhoon is available for [Flatcar Container Linux](https://www.flatcar-linux.org/releases/). From 0c53ad52e4f93caaf2d8cd92222cac1b12bb960f Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Thu, 13 Feb 2020 14:39:48 -0800 Subject: [PATCH 333/523] Update recommended Terraform versions and providers * Sync the documented Terraform versions and provider plugin versions to those that are actively used/tested by the author --- docs/cl/aws.md | 28 ++++++++++++++-------------- docs/cl/azure.md | 12 ++++++------ docs/cl/bare-metal.md | 10 +++++----- docs/cl/digital-ocean.md | 14 +++++++------- docs/cl/google-cloud.md | 6 +++--- docs/fedora-coreos/aws.md | 4 ++-- docs/fedora-coreos/bare-metal.md | 2 +- docs/fedora-coreos/google-cloud.md | 4 ++-- 8 files changed, 40 insertions(+), 40 deletions(-) diff --git a/docs/cl/aws.md b/docs/cl/aws.md index d388b323f..f46a76a8e 100644 --- a/docs/cl/aws.md +++ b/docs/cl/aws.md @@ -1,6 +1,6 @@ # AWS -In this tutorial, we'll create a Kubernetes v1.17.3 cluster on AWS with Container Linux. +In this tutorial, we'll create a Kubernetes v1.17.3 cluster on AWS with CoreOS Container Linux or Flatcar Linux. We'll declare a Kubernetes cluster using the Typhoon Terraform module. Then apply the changes to create a VPC, gateway, subnets, security groups, controller instances, worker auto-scaling group, network load balancer, and TLS assets. @@ -18,7 +18,7 @@ Install [Terraform](https://www.terraform.io/downloads.html) v0.12.6+ on your sy ```sh $ terraform version -Terraform v0.12.16 +Terraform v0.12.20 ``` Add the [terraform-provider-ct](https://github.com/poseidon/terraform-provider-ct) plugin binary for your system to `~/.terraform.d/plugins/`, noting the final name. @@ -49,7 +49,7 @@ Configure the AWS provider to use your access key credentials in a `providers.tf ```tf provider "aws" { - version = "2.41.0" + version = "2.48.0" region = "eu-central-1" shared_credentials_file = "/home/user/.config/aws/credentials" } @@ -152,18 +152,18 @@ List the pods. ``` $ kubectl get pods --all-namespaces -NAMESPACE NAME READY STATUS RESTARTS AGE -kube-system calico-node-1m5bf 2/2 Running 0 34m -kube-system calico-node-7jmr1 2/2 Running 0 34m -kube-system calico-node-bknc8 2/2 Running 0 34m -kube-system coredns-1187388186-wx1lg 1/1 Running 0 34m +NAMESPACE NAME READY STATUS RESTARTS AGE +kube-system calico-node-1m5bf 2/2 Running 0 34m +kube-system calico-node-7jmr1 2/2 Running 0 34m +kube-system calico-node-bknc8 2/2 Running 0 34m +kube-system coredns-1187388186-wx1lg 1/1 Running 0 34m kube-system coredns-1187388186-qjnvp 1/1 Running 0 34m -kube-system kube-apiserver-ip-10-0-3-155 1/1 Running 0 34m -kube-system kube-controller-manager-ip-10-0-3-155 1/1 Running 0 34m -kube-system kube-proxy-14wxv 1/1 Running 0 34m -kube-system kube-proxy-9vxh2 1/1 Running 0 34m -kube-system kube-proxy-sbbsh 1/1 Running 0 34m -kube-system kube-scheduler-ip-10-0-3-155 1/1 Running 1 34m +kube-system kube-apiserver-ip-10-0-3-155 1/1 Running 0 34m +kube-system kube-controller-manager-ip-10-0-3-155 1/1 Running 0 34m +kube-system kube-proxy-14wxv 1/1 Running 0 34m +kube-system kube-proxy-9vxh2 1/1 Running 0 34m +kube-system kube-proxy-sbbsh 1/1 Running 0 34m +kube-system kube-scheduler-ip-10-0-3-155 1/1 Running 1 34m ``` ## Going Further diff --git a/docs/cl/azure.md b/docs/cl/azure.md index 7761929fa..3bf311c9c 100644 --- a/docs/cl/azure.md +++ b/docs/cl/azure.md @@ -3,7 +3,7 @@ !!! danger Typhoon for Azure is alpha. For production, use AWS, Google Cloud, or bare-metal. As Azure matures, check [errata](https://github.com/poseidon/typhoon/wiki/Errata) for known shortcomings. -In this tutorial, we'll create a Kubernetes v1.17.3 cluster on Azure with Container Linux. +In this tutorial, we'll create a Kubernetes v1.17.3 cluster on Azure with CoreOS Container Linux or Flatcar Linux. We'll declare a Kubernetes cluster using the Typhoon Terraform module. Then apply the changes to create a resource group, virtual network, subnets, security groups, controller availability set, worker scale set, load balancer, and TLS assets. @@ -21,7 +21,7 @@ Install [Terraform](https://www.terraform.io/downloads.html) v0.12.6+ on your sy ```sh $ terraform version -Terraform v0.12.16 +Terraform v0.12.20 ``` Add the [terraform-provider-ct](https://github.com/poseidon/terraform-provider-ct) plugin binary for your system to `~/.terraform.d/plugins/`, noting the final name. @@ -50,7 +50,7 @@ Configure the Azure provider in a `providers.tf` file. ```tf provider "azurerm" { - version = "1.38.0" + version = "1.43.0" } provider "ct" { @@ -152,9 +152,9 @@ $ kubectl get pods --all-namespaces NAMESPACE NAME READY STATUS RESTARTS AGE kube-system coredns-7c6fbb4f4b-b6qzx 1/1 Running 0 26m kube-system coredns-7c6fbb4f4b-j2k3d 1/1 Running 0 26m -kube-system calico-node-1m5bf 2/2 Running 0 26m -kube-system calico-node-7jmr1 2/2 Running 0 26m -kube-system calico-node-bknc8 2/2 Running 0 26m +kube-system calico-node-1m5bf 2/2 Running 0 26m +kube-system calico-node-7jmr1 2/2 Running 0 26m +kube-system calico-node-bknc8 2/2 Running 0 26m kube-system kube-apiserver-ramius-controller-0 1/1 Running 0 26m kube-system kube-controller-manager-ramius-controller-0 1/1 Running 0 26m kube-system kube-proxy-j4vpq 1/1 Running 0 26m diff --git a/docs/cl/bare-metal.md b/docs/cl/bare-metal.md index c63500bb5..775870271 100644 --- a/docs/cl/bare-metal.md +++ b/docs/cl/bare-metal.md @@ -1,6 +1,6 @@ # Bare-Metal -In this tutorial, we'll network boot and provision a Kubernetes v1.17.3 cluster on bare-metal with Container Linux. +In this tutorial, we'll network boot and provision a Kubernetes v1.17.3 cluster on bare-metal with CoreOS Container Linux or Flatcar Linux. First, we'll deploy a [Matchbox](https://github.com/poseidon/matchbox) service and setup a network boot environment. Then, we'll declare a Kubernetes cluster using the Typhoon Terraform module and power on machines. On PXE boot, machines will install Container Linux to disk, reboot into the disk install, and provision themselves as Kubernetes controllers or workers via Ignition. @@ -27,7 +27,7 @@ Configure each machine to boot from the disk through IPMI or the BIOS menu. ``` ipmitool -H node1 -U USER -P PASS chassis bootdev disk options=persistent ``` - + During provisioning, you'll explicitly set the boot device to `pxe` for the next boot only. Machines will install (overwrite) the operating system to disk on PXE boot and reboot into the disk install. !!! tip "" @@ -111,7 +111,7 @@ Install [Terraform](https://www.terraform.io/downloads.html) v0.12.6+ on your sy ```sh $ terraform version -Terraform v0.12.16 +Terraform v0.12.20 ``` Add the [terraform-provider-matchbox](https://github.com/poseidon/terraform-provider-matchbox) plugin binary for your system to `~/.terraform.d/plugins/`, noting the final name. @@ -161,7 +161,7 @@ Define a Kubernetes cluster using the module `bare-metal/container-linux/kuberne ```tf module "mercury" { source = "git::https://github.com/poseidon/typhoon//bare-metal/container-linux/kubernetes?ref=v1.17.3" - + # bare-metal cluster_name = "mercury" matchbox_http_endpoint = "http://matchbox.example.com" @@ -355,7 +355,7 @@ Check the [variables.tf](https://github.com/poseidon/typhoon/blob/master/bare-me | cached_install | PXE boot and install from the Matchbox `/assets` cache. Admin MUST have downloaded Container Linux or Flatcar images into the cache | false | true | | install_disk | Disk device where Container Linux should be installed | "/dev/sda" | "/dev/sdb" | | networking | Choice of networking provider | "calico" | "calico" or "flannel" | -| network_mtu | CNI interface MTU (calico-only) | 1480 | - | +| network_mtu | CNI interface MTU (calico-only) | 1480 | - | | clc_snippets | Map from machine names to lists of Container Linux Config snippets | {} | [example](/advanced/customization/#usage) | | network_ip_autodetection_method | Method to detect host IPv4 address (calico-only) | "first-found" | "can-reach=10.0.0.1" | | pod_cidr | CIDR IPv4 range to assign to Kubernetes pods | "10.2.0.0/16" | "10.22.0.0/16" | diff --git a/docs/cl/digital-ocean.md b/docs/cl/digital-ocean.md index 07774b835..f2e5078fa 100644 --- a/docs/cl/digital-ocean.md +++ b/docs/cl/digital-ocean.md @@ -1,6 +1,6 @@ # Digital Ocean -In this tutorial, we'll create a Kubernetes v1.17.3 cluster on DigitalOcean with Container Linux. +In this tutorial, we'll create a Kubernetes v1.17.3 cluster on DigitalOcean with CoreOS Container Linux or Flatcar Linux. We'll declare a Kubernetes cluster using the Typhoon Terraform module. Then apply the changes to create controller droplets, worker droplets, DNS records, tags, and TLS assets. @@ -18,7 +18,7 @@ Install [Terraform](https://www.terraform.io/downloads.html) v0.12.6+ on your sy ```sh $ terraform version -Terraform v0.12.16 +Terraform v0.12.20 ``` Add the [terraform-provider-ct](https://github.com/poseidon/terraform-provider-ct) plugin binary for your system to `~/.terraform.d/plugins/`, noting the final name. @@ -50,7 +50,7 @@ Configure the DigitalOcean provider to use your token in a `providers.tf` file. ```tf provider "digitalocean" { - version = "1.11.0" + version = "1.14.0" token = "${chomp(file("~/.config/digital-ocean/token"))}" } @@ -74,7 +74,7 @@ module "nemo" { # configuration ssh_fingerprints = ["d7:9d:79:ae:56:32:73:79:95:88:e3:a2:ab:5d:45:e7"] - + # optional worker_count = 2 } @@ -149,9 +149,9 @@ List the pods. NAMESPACE NAME READY STATUS RESTARTS AGE kube-system coredns-1187388186-ld1j7 1/1 Running 0 11m kube-system coredns-1187388186-rdhf7 1/1 Running 0 11m -kube-system calico-node-1m5bf 2/2 Running 0 11m -kube-system calico-node-7jmr1 2/2 Running 0 11m -kube-system calico-node-bknc8 2/2 Running 0 11m +kube-system calico-node-1m5bf 2/2 Running 0 11m +kube-system calico-node-7jmr1 2/2 Running 0 11m +kube-system calico-node-bknc8 2/2 Running 0 11m kube-system kube-apiserver-ip-10.132.115.81 1/1 Running 0 11m kube-system kube-controller-manager-ip-10.132.115.81 1/1 Running 0 11m kube-system kube-proxy-6kxjf 1/1 Running 0 11m diff --git a/docs/cl/google-cloud.md b/docs/cl/google-cloud.md index 661cdac9f..eefd2ebbc 100644 --- a/docs/cl/google-cloud.md +++ b/docs/cl/google-cloud.md @@ -1,6 +1,6 @@ # Google Cloud -In this tutorial, we'll create a Kubernetes v1.17.3 cluster on Google Compute Engine with Container Linux. +In this tutorial, we'll create a Kubernetes v1.17.3 cluster on Google Compute Engine with CoreOS Container Linux or Flatcar Linux. We'll declare a Kubernetes cluster using the Typhoon Terraform module. Then apply the changes to create a network, firewall rules, health checks, controller instances, worker managed instance group, load balancers, and TLS assets. @@ -18,7 +18,7 @@ Install [Terraform](https://www.terraform.io/downloads.html) v0.12.6+ on your sy ```sh $ terraform version -Terraform v0.12.16 +Terraform v0.12.20 ``` Add the [terraform-provider-ct](https://github.com/poseidon/terraform-provider-ct) plugin binary for your system to `~/.terraform.d/plugins/`, noting the final name. @@ -49,7 +49,7 @@ Configure the Google Cloud provider to use your service account key, project-id, ```tf provider "google" { - version = "3.4.0" + version = "3.7.0" project = "project-id" region = "us-central1" credentials = file("~/.config/google-cloud/terraform.json") diff --git a/docs/fedora-coreos/aws.md b/docs/fedora-coreos/aws.md index cb708c2c2..61876769e 100644 --- a/docs/fedora-coreos/aws.md +++ b/docs/fedora-coreos/aws.md @@ -18,7 +18,7 @@ Install [Terraform](https://www.terraform.io/downloads.html) v0.12.6+ on your sy ```sh $ terraform version -Terraform v0.12.16 +Terraform v0.12.20 ``` Add the [terraform-provider-ct](https://github.com/poseidon/terraform-provider-ct) plugin binary for your system to `~/.terraform.d/plugins/`, noting the final name. @@ -49,7 +49,7 @@ Configure the AWS provider to use your access key credentials in a `providers.tf ```tf provider "aws" { - version = "2.41.0" + version = "2.48.0" region = "eu-central-1" shared_credentials_file = "/home/user/.config/aws/credentials" } diff --git a/docs/fedora-coreos/bare-metal.md b/docs/fedora-coreos/bare-metal.md index a29557b46..42974f6fd 100644 --- a/docs/fedora-coreos/bare-metal.md +++ b/docs/fedora-coreos/bare-metal.md @@ -111,7 +111,7 @@ Install [Terraform](https://www.terraform.io/downloads.html) v0.12.6+ on your sy ```sh $ terraform version -Terraform v0.12.16 +Terraform v0.12.20 ``` Add the [terraform-provider-matchbox](https://github.com/poseidon/terraform-provider-matchbox) plugin binary for your system to `~/.terraform.d/plugins/`, noting the final name. diff --git a/docs/fedora-coreos/google-cloud.md b/docs/fedora-coreos/google-cloud.md index 9e05041dd..80d255695 100644 --- a/docs/fedora-coreos/google-cloud.md +++ b/docs/fedora-coreos/google-cloud.md @@ -21,7 +21,7 @@ Install [Terraform](https://www.terraform.io/downloads.html) v0.12.6+ on your sy ```sh $ terraform version -Terraform v0.12.16 +Terraform v0.12.20 ``` Add the [terraform-provider-ct](https://github.com/poseidon/terraform-provider-ct) plugin binary for your system to `~/.terraform.d/plugins/`, noting the final name. @@ -52,7 +52,7 @@ Configure the Google Cloud provider to use your service account key, project-id, ```tf provider "google" { - version = "3.4.0" + version = "3.7.0" project = "project-id" region = "us-central1" credentials = file("~/.config/google-cloud/terraform.json") From 32db59b9ebec24e31d61f3e4efac87f9440c268b Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Fri, 14 Feb 2020 12:05:51 -0800 Subject: [PATCH 334/523] Update CHANGELOG sections and links --- CHANGES.md | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index a954b117c..ab164a9ab 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -4,18 +4,21 @@ Notable changes between versions. ## Latest +## v1.17.3 + * Kubernetes [v1.17.3](https://github.com/kubernetes/kubernetes/blob/master/CHANGELOG/CHANGELOG-1.17.md#v1173) * Update Calico from v3.11.2 to v3.12.0 * Allow Fedora CoreOS clusters to pass CNCF conformance suite * Set Docker log driver to `json-file` as a workaround +* Try Fedora CoreOS or Flatcar Linux alongside CoreOS [Container Linux](https://coreos.com/os/eol/) clusters (recommended) #### AWS -* Promote Fedora CoreOS to beta +* Promote Fedora CoreOS to beta ([#645](https://github.com/poseidon/typhoon/pull/645)) #### Bare-Metal -* Promote Fedora CoreOS to beta +* Promote Fedora CoreOS to beta ([#645](https://github.com/poseidon/typhoon/pull/645)) * Add Fedora CoreOS kernel arguments initrd and console ([#640](https://github.com/poseidon/typhoon/pull/640)) #### Google Cloud From 362b3fac5c5305d1f5c0b5e8c0a983702b0cdf54 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Thu, 13 Feb 2020 14:18:41 -0800 Subject: [PATCH 335/523] Add guide for Typhoon with Flatcar Linux on DigitalOcean * Add docs on manually uploading a Flatcar Linux DigitalOcean bin image as a custom image and using a data reference * Set status of Flatcar Linux on DigitalOcean to alpha * IPv6 is not supported for DigitalOcean custom images --- CHANGES.md | 8 ++++-- README.md | 1 + .../container-linux/kubernetes/controllers.tf | 8 +++++- .../container-linux/kubernetes/workers.tf | 6 +++-- docs/cl/digital-ocean.md | 25 ++++++++++++++++++- docs/cl/google-cloud.md | 9 +++---- docs/index.md | 1 + 7 files changed, 46 insertions(+), 12 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index ab164a9ab..90cba7158 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -4,6 +4,10 @@ Notable changes between versions. ## Latest +#### DigitalOcean + +* Add support for Flatcar Container Linux ([#644](https://github.com/poseidon/typhoon/pull/644)) + ## v1.17.3 * Kubernetes [v1.17.3](https://github.com/kubernetes/kubernetes/blob/master/CHANGELOG/CHANGELOG-1.17.md#v1173) @@ -23,8 +27,8 @@ Notable changes between versions. #### Google Cloud -* Add initial Terraform module for Fedora CoreOS ([#632](https://github.com/poseidon/typhoon/pull/632)) -* Add initial support for Flatcar Container Linux ([#639](https://github.com/poseidon/typhoon/pull/639)) +* Add Terraform module for Fedora CoreOS ([#632](https://github.com/poseidon/typhoon/pull/632)) +* Add support for Flatcar Container Linux ([#639](https://github.com/poseidon/typhoon/pull/639)) #### Addons diff --git a/README.md b/README.md index eb4999e7c..69b987f7a 100644 --- a/README.md +++ b/README.md @@ -44,6 +44,7 @@ Typhoon is available for [Flatcar Container Linux](https://www.flatcar-linux.org | AWS | Flatcar Linux | [aws/container-linux/kubernetes](aws/container-linux/kubernetes) | stable | | Bare-Metal | Flatcar Linux | [bare-metal/container-linux/kubernetes](bare-metal/container-linux/kubernetes) | stable | | Google Cloud | Flatcar Linux | [google-cloud/container-linux/kubernetes](google-cloud/container-linux/kubernetes) | alpha | +| Digital Ocean | Flatcar Linux | [digital-ocean/container-linux/kubernetes](digital-ocean/container-linux/kubernetes) | alpha | ## Documentation diff --git a/digital-ocean/container-linux/kubernetes/controllers.tf b/digital-ocean/container-linux/kubernetes/controllers.tf index f020e3595..f64e3f4a2 100644 --- a/digital-ocean/container-linux/kubernetes/controllers.tf +++ b/digital-ocean/container-linux/kubernetes/controllers.tf @@ -1,3 +1,8 @@ +locals { + official_images = ["coreos-stable", "coreos-beta", "coreos-alpha"] + is_official_image = contains(local.official_images, var.image) +} + # Controller Instance DNS records resource "digitalocean_record" "controllers" { count = var.controller_count @@ -41,7 +46,8 @@ resource "digitalocean_droplet" "controllers" { size = var.controller_type # network - ipv6 = true + # only official DigitalOcean images support IPv6 + ipv6 = local.is_official_image private_networking = true user_data = data.ct_config.controller-ignitions.*.rendered[count.index] diff --git a/digital-ocean/container-linux/kubernetes/workers.tf b/digital-ocean/container-linux/kubernetes/workers.tf index 6a1c927f4..2fe95fac6 100644 --- a/digital-ocean/container-linux/kubernetes/workers.tf +++ b/digital-ocean/container-linux/kubernetes/workers.tf @@ -12,7 +12,8 @@ resource "digitalocean_record" "workers-record-a" { } resource "digitalocean_record" "workers-record-aaaa" { - count = var.worker_count + # only official DigitalOcean images support IPv6 + count = local.is_official_image ? var.worker_count : 0 # DNS zone where record should be created domain = var.dns_zone @@ -34,7 +35,8 @@ resource "digitalocean_droplet" "workers" { size = var.worker_type # network - ipv6 = true + # only official DigitalOcean images support IPv6 + ipv6 = local.is_official_image private_networking = true user_data = data.ct_config.worker-ignition.rendered diff --git a/docs/cl/digital-ocean.md b/docs/cl/digital-ocean.md index f2e5078fa..ead7fc7ea 100644 --- a/docs/cl/digital-ocean.md +++ b/docs/cl/digital-ocean.md @@ -71,6 +71,7 @@ module "nemo" { cluster_name = "nemo" region = "nyc3" dns_zone = "digital-ocean.example.com" + image = "coreos-stable" # configuration ssh_fingerprints = ["d7:9d:79:ae:56:32:73:79:95:88:e3:a2:ab:5d:45:e7"] @@ -82,6 +83,28 @@ module "nemo" { Reference the [variables docs](#variables) or the [variables.tf](https://github.com/poseidon/typhoon/blob/master/digital-ocean/container-linux/kubernetes/variables.tf) source. +### Flatcar Linux Only + +!!! warning + Typhoon for Flatcar Linux on DigitalOcean is alpha. Also IPv6 is unsupported with DigitalOcean custom images. + +Flatcar Linux publishes DigitalOcean images, but does not upload them. DigitalOcean allows [custom boot images](https://blog.digitalocean.com/custom-images/) by file or URL. + +[Download](https://www.flatcar-linux.org/releases/) the Flatcar Linux DigitalOcean bin image (or copy the URL) and [upload](https://cloud.digitalocean.com/images/custom_images) it as a custom image. Rename the image with the channel and version to refer to these images over time. + +```tf +module "nemo" { + ... + image = data.digitalocean_image.flatcar-stable.id +} + +data "digitalocean_image" "flatcar-stable" { + name = "flatcar-stable-2303.4.0.bin.bz2" +} +``` + +Set the [image](#variables) to the custom image id. + ## ssh-agent Initial bootstrapping requires `bootstrap.service` be started on one controller node. Terraform uses `ssh-agent` to automate this step. Add your SSH private key to `ssh-agent`. @@ -224,7 +247,7 @@ Digital Ocean requires the SSH public key be uploaded to your account, so you ma | worker_count | Number of workers | 1 | 3 | | controller_type | Droplet type for controllers | "s-2vcpu-2gb" | s-2vcpu-2gb, s-2vcpu-4gb, s-4vcpu-8gb, ... | | worker_type | Droplet type for workers | "s-1vcpu-2gb" | s-1vcpu-2gb, s-2vcpu-2gb, ... | -| image | Container Linux image for instances | "coreos-stable" | coreos-stable, coreos-beta, coreos-alpha | +| image | Container Linux image for instances | "coreos-stable" | coreos-stable, coreos-beta, coreos-alpha, "custom-image-id" | | controller_clc_snippets | Controller Container Linux Config snippets | [] | [example](/advanced/customization/) | | worker_clc_snippets | Worker Container Linux Config snippets | [] | [example](/advanced/customization/) | | networking | Choice of networking provider | "calico" | "flannel" or "calico" | diff --git a/docs/cl/google-cloud.md b/docs/cl/google-cloud.md index eefd2ebbc..08bed675d 100644 --- a/docs/cl/google-cloud.md +++ b/docs/cl/google-cloud.md @@ -89,15 +89,12 @@ module "yavin" { Reference the [variables docs](#variables) or the [variables.tf](https://github.com/poseidon/typhoon/blob/master/google-cloud/container-linux/kubernetes/variables.tf) source. -### Flatcar Linux Images +### Flatcar Linux Only -!!! success - Skip this section when using CoreOS Container Linux (default). CoreOS Container Linux publishes official images to Google Cloud. - -!!! danger +!!! warning Typhoon for Flatcar Linux on Google Cloud is alpha. -Flatcar Linux publishes images for Google Cloud, but does not yet upload them. Google Cloud allows [custom boot images](https://cloud.google.com/compute/docs/images/import-existing-image) to be uploaded to a bucket and imported into your project. +Flatcar Linux publishes Google Cloud images, but does not upload them. Google Cloud allows [custom boot images](https://cloud.google.com/compute/docs/images/import-existing-image) to be uploaded to a bucket and imported into a project. [Download](https://www.flatcar-linux.org/releases/) the Flatcar Linux GCE gzipped tarball and upload it to a Google Cloud storage bucket. diff --git a/docs/index.md b/docs/index.md index 50ff8be80..8122b0556 100644 --- a/docs/index.md +++ b/docs/index.md @@ -44,6 +44,7 @@ Typhoon is available for [Flatcar Container Linux](https://www.flatcar-linux.org | AWS | Flatcar Linux | [aws/container-linux/kubernetes](cl/aws.md) | stable | | Bare-Metal | Flatcar Linux | [bare-metal/container-linux/kubernetes](cl/bare-metal.md) | stable | | Google Cloud | Flatcar Linux | [google-cloud/container-linux/kubernetes](cl/google-cloud.md) | alpha | +| Digital Ocean | Flatcar Linux | [digital-ocean/container-linux/kubernetes](cl/digital-ocean.md) | alpha | ## Documentation From 7ca03e5219fe81d055e1e3bb84ce16e1489e0e2b Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Fri, 14 Feb 2020 12:10:56 -0800 Subject: [PATCH 336/523] Update Prometheus from v1.15.2 to v1.16.0 * https://github.com/prometheus/prometheus/releases/tag/v2.16.0 --- CHANGES.md | 4 ++++ addons/prometheus/deployment.yaml | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index 90cba7158..3db121061 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -8,6 +8,10 @@ Notable changes between versions. * Add support for Flatcar Container Linux ([#644](https://github.com/poseidon/typhoon/pull/644)) +#### Addons + +* Update Prometheus from v2.15.2 to [v2.16.0](https://github.com/prometheus/prometheus/releases/tag/v2.16.0) + ## v1.17.3 * Kubernetes [v1.17.3](https://github.com/kubernetes/kubernetes/blob/master/CHANGELOG/CHANGELOG-1.17.md#v1173) diff --git a/addons/prometheus/deployment.yaml b/addons/prometheus/deployment.yaml index 873aeaa75..02c2e2810 100644 --- a/addons/prometheus/deployment.yaml +++ b/addons/prometheus/deployment.yaml @@ -20,7 +20,7 @@ spec: serviceAccountName: prometheus containers: - name: prometheus - image: quay.io/prometheus/prometheus:v2.15.2 + image: quay.io/prometheus/prometheus:v2.16.0 args: - --web.listen-address=0.0.0.0:9090 - --config.file=/etc/prometheus/prometheus.yaml From c4e64a9d1b6fb731026201d03d87d84b638a0ab1 Mon Sep 17 00:00:00 2001 From: Suraj Deshmukh Date: Wed, 19 Feb 2020 11:10:58 +0530 Subject: [PATCH 337/523] Change Kubelet /var/lib/calico mount to read-only (#643) * Kubelet only requires read access to /var/lib/calico Signed-off-by: Suraj Deshmukh --- aws/container-linux/kubernetes/cl/controller.yaml | 2 +- aws/container-linux/kubernetes/workers/cl/worker.yaml | 2 +- aws/fedora-coreos/kubernetes/fcc/controller.yaml | 2 +- aws/fedora-coreos/kubernetes/workers/fcc/worker.yaml | 2 +- azure/container-linux/kubernetes/cl/controller.yaml | 2 +- azure/container-linux/kubernetes/workers/cl/worker.yaml | 2 +- bare-metal/container-linux/kubernetes/cl/controller.yaml | 2 +- bare-metal/container-linux/kubernetes/cl/worker.yaml | 2 +- bare-metal/fedora-coreos/kubernetes/fcc/controller.yaml | 2 +- bare-metal/fedora-coreos/kubernetes/fcc/worker.yaml | 2 +- digital-ocean/container-linux/kubernetes/cl/controller.yaml | 2 +- digital-ocean/container-linux/kubernetes/cl/worker.yaml | 2 +- google-cloud/container-linux/kubernetes/cl/controller.yaml | 2 +- google-cloud/container-linux/kubernetes/workers/cl/worker.yaml | 2 +- google-cloud/fedora-coreos/kubernetes/fcc/controller.yaml | 2 +- google-cloud/fedora-coreos/kubernetes/workers/fcc/worker.yaml | 2 +- 16 files changed, 16 insertions(+), 16 deletions(-) diff --git a/aws/container-linux/kubernetes/cl/controller.yaml b/aws/container-linux/kubernetes/cl/controller.yaml index 670bb39ff..3e2a777be 100644 --- a/aws/container-linux/kubernetes/cl/controller.yaml +++ b/aws/container-linux/kubernetes/cl/controller.yaml @@ -79,7 +79,7 @@ systemd: --mount volume=run,target=/run \ --volume usr-share-certs,kind=host,source=/usr/share/ca-certificates,readOnly=true \ --mount volume=usr-share-certs,target=/usr/share/ca-certificates \ - --volume var-lib-calico,kind=host,source=/var/lib/calico \ + --volume var-lib-calico,kind=host,source=/var/lib/calico,readOnly=true \ --mount volume=var-lib-calico,target=/var/lib/calico \ --volume var-lib-docker,kind=host,source=/var/lib/docker \ --mount volume=var-lib-docker,target=/var/lib/docker \ diff --git a/aws/container-linux/kubernetes/workers/cl/worker.yaml b/aws/container-linux/kubernetes/workers/cl/worker.yaml index cd8fc92cc..1eac9085f 100644 --- a/aws/container-linux/kubernetes/workers/cl/worker.yaml +++ b/aws/container-linux/kubernetes/workers/cl/worker.yaml @@ -54,7 +54,7 @@ systemd: --mount volume=run,target=/run \ --volume usr-share-certs,kind=host,source=/usr/share/ca-certificates,readOnly=true \ --mount volume=usr-share-certs,target=/usr/share/ca-certificates \ - --volume var-lib-calico,kind=host,source=/var/lib/calico \ + --volume var-lib-calico,kind=host,source=/var/lib/calico,readOnly=true \ --mount volume=var-lib-calico,target=/var/lib/calico \ --volume var-lib-docker,kind=host,source=/var/lib/docker \ --mount volume=var-lib-docker,target=/var/lib/docker \ diff --git a/aws/fedora-coreos/kubernetes/fcc/controller.yaml b/aws/fedora-coreos/kubernetes/fcc/controller.yaml index f9aaf6be8..a90558c7b 100644 --- a/aws/fedora-coreos/kubernetes/fcc/controller.yaml +++ b/aws/fedora-coreos/kubernetes/fcc/controller.yaml @@ -73,7 +73,7 @@ systemd: --volume /sys/fs/cgroup:/sys/fs/cgroup:ro \ --volume /sys/fs/cgroup/systemd:/sys/fs/cgroup/systemd \ --volume /etc/pki/tls/certs:/usr/share/ca-certificates:ro \ - --volume /var/lib/calico:/var/lib/calico \ + --volume /var/lib/calico:/var/lib/calico:ro \ --volume /var/lib/docker:/var/lib/docker \ --volume /var/lib/kubelet:/var/lib/kubelet:rshared,z \ --volume /var/log:/var/log \ diff --git a/aws/fedora-coreos/kubernetes/workers/fcc/worker.yaml b/aws/fedora-coreos/kubernetes/workers/fcc/worker.yaml index 7d1d536d2..992cb604b 100644 --- a/aws/fedora-coreos/kubernetes/workers/fcc/worker.yaml +++ b/aws/fedora-coreos/kubernetes/workers/fcc/worker.yaml @@ -43,7 +43,7 @@ systemd: --volume /sys/fs/cgroup:/sys/fs/cgroup:ro \ --volume /sys/fs/cgroup/systemd:/sys/fs/cgroup/systemd \ --volume /etc/pki/tls/certs:/usr/share/ca-certificates:ro \ - --volume /var/lib/calico:/var/lib/calico \ + --volume /var/lib/calico:/var/lib/calico:ro \ --volume /var/lib/docker:/var/lib/docker \ --volume /var/lib/kubelet:/var/lib/kubelet:rshared,z \ --volume /var/log:/var/log \ diff --git a/azure/container-linux/kubernetes/cl/controller.yaml b/azure/container-linux/kubernetes/cl/controller.yaml index 4f18d6bb6..850c7173b 100644 --- a/azure/container-linux/kubernetes/cl/controller.yaml +++ b/azure/container-linux/kubernetes/cl/controller.yaml @@ -78,7 +78,7 @@ systemd: --mount volume=run,target=/run \ --volume usr-share-certs,kind=host,source=/usr/share/ca-certificates,readOnly=true \ --mount volume=usr-share-certs,target=/usr/share/ca-certificates \ - --volume var-lib-calico,kind=host,source=/var/lib/calico \ + --volume var-lib-calico,kind=host,source=/var/lib/calico,readOnly=true \ --mount volume=var-lib-calico,target=/var/lib/calico \ --volume var-lib-docker,kind=host,source=/var/lib/docker \ --mount volume=var-lib-docker,target=/var/lib/docker \ diff --git a/azure/container-linux/kubernetes/workers/cl/worker.yaml b/azure/container-linux/kubernetes/workers/cl/worker.yaml index 994c8f4c5..fe7fb84c8 100644 --- a/azure/container-linux/kubernetes/workers/cl/worker.yaml +++ b/azure/container-linux/kubernetes/workers/cl/worker.yaml @@ -53,7 +53,7 @@ systemd: --mount volume=run,target=/run \ --volume usr-share-certs,kind=host,source=/usr/share/ca-certificates,readOnly=true \ --mount volume=usr-share-certs,target=/usr/share/ca-certificates \ - --volume var-lib-calico,kind=host,source=/var/lib/calico \ + --volume var-lib-calico,kind=host,source=/var/lib/calico,readOnly=true \ --mount volume=var-lib-calico,target=/var/lib/calico \ --volume var-lib-docker,kind=host,source=/var/lib/docker \ --mount volume=var-lib-docker,target=/var/lib/docker \ diff --git a/bare-metal/container-linux/kubernetes/cl/controller.yaml b/bare-metal/container-linux/kubernetes/cl/controller.yaml index 5fe9d7cc2..4f5dd370b 100644 --- a/bare-metal/container-linux/kubernetes/cl/controller.yaml +++ b/bare-metal/container-linux/kubernetes/cl/controller.yaml @@ -87,7 +87,7 @@ systemd: --mount volume=run,target=/run \ --volume usr-share-certs,kind=host,source=/usr/share/ca-certificates,readOnly=true \ --mount volume=usr-share-certs,target=/usr/share/ca-certificates \ - --volume var-lib-calico,kind=host,source=/var/lib/calico \ + --volume var-lib-calico,kind=host,source=/var/lib/calico,readOnly=true \ --mount volume=var-lib-calico,target=/var/lib/calico \ --volume var-lib-docker,kind=host,source=/var/lib/docker \ --mount volume=var-lib-docker,target=/var/lib/docker \ diff --git a/bare-metal/container-linux/kubernetes/cl/worker.yaml b/bare-metal/container-linux/kubernetes/cl/worker.yaml index 5a1db8ac5..8f6e984ff 100644 --- a/bare-metal/container-linux/kubernetes/cl/worker.yaml +++ b/bare-metal/container-linux/kubernetes/cl/worker.yaml @@ -62,7 +62,7 @@ systemd: --mount volume=run,target=/run \ --volume usr-share-certs,kind=host,source=/usr/share/ca-certificates,readOnly=true \ --mount volume=usr-share-certs,target=/usr/share/ca-certificates \ - --volume var-lib-calico,kind=host,source=/var/lib/calico \ + --volume var-lib-calico,kind=host,source=/var/lib/calico,readOnly=true \ --mount volume=var-lib-calico,target=/var/lib/calico \ --volume var-lib-docker,kind=host,source=/var/lib/docker \ --mount volume=var-lib-docker,target=/var/lib/docker \ diff --git a/bare-metal/fedora-coreos/kubernetes/fcc/controller.yaml b/bare-metal/fedora-coreos/kubernetes/fcc/controller.yaml index c8cc09792..6a6f74e20 100644 --- a/bare-metal/fedora-coreos/kubernetes/fcc/controller.yaml +++ b/bare-metal/fedora-coreos/kubernetes/fcc/controller.yaml @@ -72,7 +72,7 @@ systemd: --volume /sys/fs/cgroup:/sys/fs/cgroup:ro \ --volume /sys/fs/cgroup/systemd:/sys/fs/cgroup/systemd \ --volume /etc/pki/tls/certs:/usr/share/ca-certificates:ro \ - --volume /var/lib/calico:/var/lib/calico \ + --volume /var/lib/calico:/var/lib/calico:ro \ --volume /var/lib/docker:/var/lib/docker \ --volume /var/lib/kubelet:/var/lib/kubelet:rshared,z \ --volume /var/log:/var/log \ diff --git a/bare-metal/fedora-coreos/kubernetes/fcc/worker.yaml b/bare-metal/fedora-coreos/kubernetes/fcc/worker.yaml index 49b05d538..c0e09b58c 100644 --- a/bare-metal/fedora-coreos/kubernetes/fcc/worker.yaml +++ b/bare-metal/fedora-coreos/kubernetes/fcc/worker.yaml @@ -42,7 +42,7 @@ systemd: --volume /sys/fs/cgroup:/sys/fs/cgroup:ro \ --volume /sys/fs/cgroup/systemd:/sys/fs/cgroup/systemd \ --volume /etc/pki/tls/certs:/usr/share/ca-certificates:ro \ - --volume /var/lib/calico:/var/lib/calico \ + --volume /var/lib/calico:/var/lib/calico:ro \ --volume /var/lib/docker:/var/lib/docker \ --volume /var/lib/kubelet:/var/lib/kubelet:rshared,z \ --volume /var/log:/var/log \ diff --git a/digital-ocean/container-linux/kubernetes/cl/controller.yaml b/digital-ocean/container-linux/kubernetes/cl/controller.yaml index c011427d8..f751e6824 100644 --- a/digital-ocean/container-linux/kubernetes/cl/controller.yaml +++ b/digital-ocean/container-linux/kubernetes/cl/controller.yaml @@ -89,7 +89,7 @@ systemd: --mount volume=run,target=/run \ --volume usr-share-certs,kind=host,source=/usr/share/ca-certificates,readOnly=true \ --mount volume=usr-share-certs,target=/usr/share/ca-certificates \ - --volume var-lib-calico,kind=host,source=/var/lib/calico \ + --volume var-lib-calico,kind=host,source=/var/lib/calico,readOnly=true \ --mount volume=var-lib-calico,target=/var/lib/calico \ --volume var-lib-docker,kind=host,source=/var/lib/docker \ --mount volume=var-lib-docker,target=/var/lib/docker \ diff --git a/digital-ocean/container-linux/kubernetes/cl/worker.yaml b/digital-ocean/container-linux/kubernetes/cl/worker.yaml index 94714efbc..0a23c0cce 100644 --- a/digital-ocean/container-linux/kubernetes/cl/worker.yaml +++ b/digital-ocean/container-linux/kubernetes/cl/worker.yaml @@ -64,7 +64,7 @@ systemd: --mount volume=run,target=/run \ --volume usr-share-certs,kind=host,source=/usr/share/ca-certificates,readOnly=true \ --mount volume=usr-share-certs,target=/usr/share/ca-certificates \ - --volume var-lib-calico,kind=host,source=/var/lib/calico \ + --volume var-lib-calico,kind=host,source=/var/lib/calico,readOnly=true \ --mount volume=var-lib-calico,target=/var/lib/calico \ --volume var-lib-docker,kind=host,source=/var/lib/docker \ --mount volume=var-lib-docker,target=/var/lib/docker \ diff --git a/google-cloud/container-linux/kubernetes/cl/controller.yaml b/google-cloud/container-linux/kubernetes/cl/controller.yaml index d7f0a744d..4f286eee5 100644 --- a/google-cloud/container-linux/kubernetes/cl/controller.yaml +++ b/google-cloud/container-linux/kubernetes/cl/controller.yaml @@ -78,7 +78,7 @@ systemd: --mount volume=run,target=/run \ --volume usr-share-certs,kind=host,source=/usr/share/ca-certificates,readOnly=true \ --mount volume=usr-share-certs,target=/usr/share/ca-certificates \ - --volume var-lib-calico,kind=host,source=/var/lib/calico \ + --volume var-lib-calico,kind=host,source=/var/lib/calico,readOnly=true \ --mount volume=var-lib-calico,target=/var/lib/calico \ --volume var-lib-docker,kind=host,source=/var/lib/docker \ --mount volume=var-lib-docker,target=/var/lib/docker \ diff --git a/google-cloud/container-linux/kubernetes/workers/cl/worker.yaml b/google-cloud/container-linux/kubernetes/workers/cl/worker.yaml index 766fe98d7..26dffd3e5 100644 --- a/google-cloud/container-linux/kubernetes/workers/cl/worker.yaml +++ b/google-cloud/container-linux/kubernetes/workers/cl/worker.yaml @@ -53,7 +53,7 @@ systemd: --mount volume=run,target=/run \ --volume usr-share-certs,kind=host,source=/usr/share/ca-certificates,readOnly=true \ --mount volume=usr-share-certs,target=/usr/share/ca-certificates \ - --volume var-lib-calico,kind=host,source=/var/lib/calico \ + --volume var-lib-calico,kind=host,source=/var/lib/calico,readOnly=true \ --mount volume=var-lib-calico,target=/var/lib/calico \ --volume var-lib-docker,kind=host,source=/var/lib/docker \ --mount volume=var-lib-docker,target=/var/lib/docker \ diff --git a/google-cloud/fedora-coreos/kubernetes/fcc/controller.yaml b/google-cloud/fedora-coreos/kubernetes/fcc/controller.yaml index f9aaf6be8..a90558c7b 100644 --- a/google-cloud/fedora-coreos/kubernetes/fcc/controller.yaml +++ b/google-cloud/fedora-coreos/kubernetes/fcc/controller.yaml @@ -73,7 +73,7 @@ systemd: --volume /sys/fs/cgroup:/sys/fs/cgroup:ro \ --volume /sys/fs/cgroup/systemd:/sys/fs/cgroup/systemd \ --volume /etc/pki/tls/certs:/usr/share/ca-certificates:ro \ - --volume /var/lib/calico:/var/lib/calico \ + --volume /var/lib/calico:/var/lib/calico:ro \ --volume /var/lib/docker:/var/lib/docker \ --volume /var/lib/kubelet:/var/lib/kubelet:rshared,z \ --volume /var/log:/var/log \ diff --git a/google-cloud/fedora-coreos/kubernetes/workers/fcc/worker.yaml b/google-cloud/fedora-coreos/kubernetes/workers/fcc/worker.yaml index 7d1d536d2..992cb604b 100644 --- a/google-cloud/fedora-coreos/kubernetes/workers/fcc/worker.yaml +++ b/google-cloud/fedora-coreos/kubernetes/workers/fcc/worker.yaml @@ -43,7 +43,7 @@ systemd: --volume /sys/fs/cgroup:/sys/fs/cgroup:ro \ --volume /sys/fs/cgroup/systemd:/sys/fs/cgroup/systemd \ --volume /etc/pki/tls/certs:/usr/share/ca-certificates:ro \ - --volume /var/lib/calico:/var/lib/calico \ + --volume /var/lib/calico:/var/lib/calico:ro \ --volume /var/lib/docker:/var/lib/docker \ --volume /var/lib/kubelet:/var/lib/kubelet:rshared,z \ --volume /var/log:/var/log \ From 4a38fb5927173049930ad472cc9075138ef0e092 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Tue, 18 Feb 2020 21:45:14 -0800 Subject: [PATCH 338/523] Update CoreDNS from v1.6.6 to v1.6.7 * https://coredns.io/2020/01/28/coredns-1.6.7-release/ --- CHANGES.md | 2 ++ aws/container-linux/kubernetes/bootstrap.tf | 2 +- aws/fedora-coreos/kubernetes/bootstrap.tf | 2 +- azure/container-linux/kubernetes/bootstrap.tf | 2 +- bare-metal/container-linux/kubernetes/bootstrap.tf | 2 +- bare-metal/fedora-coreos/kubernetes/bootstrap.tf | 2 +- digital-ocean/container-linux/kubernetes/bootstrap.tf | 2 +- google-cloud/container-linux/kubernetes/bootstrap.tf | 2 +- google-cloud/fedora-coreos/kubernetes/bootstrap.tf | 2 +- 9 files changed, 10 insertions(+), 8 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 3db121061..784561abb 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -4,6 +4,8 @@ Notable changes between versions. ## Latest +* Update CoreDNS from v1.6.6 to v1.6.7 ([#648](https://github.com/poseidon/typhoon/pull/648)) + #### DigitalOcean * Add support for Flatcar Container Linux ([#644](https://github.com/poseidon/typhoon/pull/644)) diff --git a/aws/container-linux/kubernetes/bootstrap.tf b/aws/container-linux/kubernetes/bootstrap.tf index 6a2c30958..3ee5bdda4 100644 --- a/aws/container-linux/kubernetes/bootstrap.tf +++ b/aws/container-linux/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=796194583426593d1a62b6f1bf3f7ffed8fca140" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=d1831e626a06d1aa63a75ca90a670ca594657fbf" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/aws/fedora-coreos/kubernetes/bootstrap.tf b/aws/fedora-coreos/kubernetes/bootstrap.tf index d9dd742ce..7511d2d08 100644 --- a/aws/fedora-coreos/kubernetes/bootstrap.tf +++ b/aws/fedora-coreos/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=796194583426593d1a62b6f1bf3f7ffed8fca140" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=d1831e626a06d1aa63a75ca90a670ca594657fbf" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/azure/container-linux/kubernetes/bootstrap.tf b/azure/container-linux/kubernetes/bootstrap.tf index 5c089e89e..07404d67e 100644 --- a/azure/container-linux/kubernetes/bootstrap.tf +++ b/azure/container-linux/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=796194583426593d1a62b6f1bf3f7ffed8fca140" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=d1831e626a06d1aa63a75ca90a670ca594657fbf" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/bare-metal/container-linux/kubernetes/bootstrap.tf b/bare-metal/container-linux/kubernetes/bootstrap.tf index 94f54d381..2810938c8 100644 --- a/bare-metal/container-linux/kubernetes/bootstrap.tf +++ b/bare-metal/container-linux/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=796194583426593d1a62b6f1bf3f7ffed8fca140" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=d1831e626a06d1aa63a75ca90a670ca594657fbf" cluster_name = var.cluster_name api_servers = [var.k8s_domain_name] diff --git a/bare-metal/fedora-coreos/kubernetes/bootstrap.tf b/bare-metal/fedora-coreos/kubernetes/bootstrap.tf index bb5cc0880..99a4729f0 100644 --- a/bare-metal/fedora-coreos/kubernetes/bootstrap.tf +++ b/bare-metal/fedora-coreos/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=796194583426593d1a62b6f1bf3f7ffed8fca140" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=d1831e626a06d1aa63a75ca90a670ca594657fbf" cluster_name = var.cluster_name api_servers = [var.k8s_domain_name] diff --git a/digital-ocean/container-linux/kubernetes/bootstrap.tf b/digital-ocean/container-linux/kubernetes/bootstrap.tf index 481b44189..3985411ab 100644 --- a/digital-ocean/container-linux/kubernetes/bootstrap.tf +++ b/digital-ocean/container-linux/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=796194583426593d1a62b6f1bf3f7ffed8fca140" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=d1831e626a06d1aa63a75ca90a670ca594657fbf" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/google-cloud/container-linux/kubernetes/bootstrap.tf b/google-cloud/container-linux/kubernetes/bootstrap.tf index 3bd0ea094..ad011ed77 100644 --- a/google-cloud/container-linux/kubernetes/bootstrap.tf +++ b/google-cloud/container-linux/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=796194583426593d1a62b6f1bf3f7ffed8fca140" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=d1831e626a06d1aa63a75ca90a670ca594657fbf" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/google-cloud/fedora-coreos/kubernetes/bootstrap.tf b/google-cloud/fedora-coreos/kubernetes/bootstrap.tf index 9d9a5155d..24197be98 100644 --- a/google-cloud/fedora-coreos/kubernetes/bootstrap.tf +++ b/google-cloud/fedora-coreos/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=796194583426593d1a62b6f1bf3f7ffed8fca140" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=d1831e626a06d1aa63a75ca90a670ca594657fbf" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] From 947c2c181553d8e9519e66aa7edcc6bfbeedb2c2 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Tue, 18 Feb 2020 21:59:17 -0800 Subject: [PATCH 339/523] Update mkdocs-material from v4.6.2 to v4.6.3 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index b6e5cc8ce..e8189b359 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ mkdocs==1.0.4 -mkdocs-material==4.6.2 +mkdocs-material==4.6.3 pygments==2.5.2 pymdown-extensions==6.3.0 From e4d977bfcd67a87f4666a8ff3fc558f8659df2cb Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Sat, 22 Feb 2020 15:01:57 -0800 Subject: [PATCH 340/523] Fix worker_node_labels for initial Fedora CoreOS * Add Terraform strip markers to consume beginning and trailing whitespace in templated Kubelet arguments for podman (Fedora CoreOS only) * Fix initial `worker_node_labels` being quietly ignored on Fedora CoreOS cloud platforms that offer the feature * Close https://github.com/poseidon/typhoon/issues/650 --- CHANGES.md | 8 ++++++++ aws/fedora-coreos/kubernetes/workers/fcc/worker.yaml | 4 ++-- .../fedora-coreos/kubernetes/workers/fcc/worker.yaml | 4 ++-- 3 files changed, 12 insertions(+), 4 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 784561abb..50450f9c7 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -6,6 +6,14 @@ Notable changes between versions. * Update CoreDNS from v1.6.6 to v1.6.7 ([#648](https://github.com/poseidon/typhoon/pull/648)) +#### AWS + +* Fix `worker_node_labels` for setting initial worker node labels on Fedora CoreOS ([#651](https://github.com/poseidon/typhoon/pull/651)) + +#### Google Cloud + +* Fix `worker_node_labels` for setting initial worker node labels on Fedora CoreOS ([#651](https://github.com/poseidon/typhoon/pull/651)) + #### DigitalOcean * Add support for Flatcar Container Linux ([#644](https://github.com/poseidon/typhoon/pull/644)) diff --git a/aws/fedora-coreos/kubernetes/workers/fcc/worker.yaml b/aws/fedora-coreos/kubernetes/workers/fcc/worker.yaml index 992cb604b..6e6fd2fef 100644 --- a/aws/fedora-coreos/kubernetes/workers/fcc/worker.yaml +++ b/aws/fedora-coreos/kubernetes/workers/fcc/worker.yaml @@ -66,9 +66,9 @@ systemd: --lock-file=/var/run/lock/kubelet.lock \ --network-plugin=cni \ --node-labels=node.kubernetes.io/node \ - %{ for label in split(",", node_labels) } + %{~ for label in split(",", node_labels) ~} --node-labels=${label} \ - %{ endfor ~} + %{~ endfor ~} --pod-manifest-path=/etc/kubernetes/manifests \ --read-only-port=0 \ --volume-plugin-dir=/var/lib/kubelet/volumeplugins diff --git a/google-cloud/fedora-coreos/kubernetes/workers/fcc/worker.yaml b/google-cloud/fedora-coreos/kubernetes/workers/fcc/worker.yaml index 992cb604b..6e6fd2fef 100644 --- a/google-cloud/fedora-coreos/kubernetes/workers/fcc/worker.yaml +++ b/google-cloud/fedora-coreos/kubernetes/workers/fcc/worker.yaml @@ -66,9 +66,9 @@ systemd: --lock-file=/var/run/lock/kubelet.lock \ --network-plugin=cni \ --node-labels=node.kubernetes.io/node \ - %{ for label in split(",", node_labels) } + %{~ for label in split(",", node_labels) ~} --node-labels=${label} \ - %{ endfor ~} + %{~ endfor ~} --pod-manifest-path=/etc/kubernetes/manifests \ --read-only-port=0 \ --volume-plugin-dir=/var/lib/kubelet/volumeplugins From 1fbd6835f2ae7391eaf0032c532668edfe04c99c Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Sat, 22 Feb 2020 15:19:24 -0800 Subject: [PATCH 341/523] Update Grafana from v6.6.1 to v6.6.2 * https://github.com/grafana/grafana/releases/tag/v6.6.2 --- CHANGES.md | 1 + addons/grafana/deployment.yaml | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index 50450f9c7..16ce2907c 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -21,6 +21,7 @@ Notable changes between versions. #### Addons * Update Prometheus from v2.15.2 to [v2.16.0](https://github.com/prometheus/prometheus/releases/tag/v2.16.0) +* Update Grafana from v6.6.1 to v6.6.2 ## v1.17.3 diff --git a/addons/grafana/deployment.yaml b/addons/grafana/deployment.yaml index a5b35a2d7..813e2dd80 100644 --- a/addons/grafana/deployment.yaml +++ b/addons/grafana/deployment.yaml @@ -23,7 +23,7 @@ spec: spec: containers: - name: grafana - image: docker.io/grafana/grafana:6.6.1 + image: docker.io/grafana/grafana:6.6.2 env: - name: GF_PATHS_CONFIG value: "/etc/grafana/custom.ini" From 4c964b56a020b65ee32819271542288105593ca6 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Sat, 22 Feb 2020 15:21:10 -0800 Subject: [PATCH 342/523] Update kube-state-metrics from v1.9.4 to v1.9.5 * https://github.com/kubernetes/kube-state-metrics/releases/tag/v1.9.5 --- CHANGES.md | 1 + addons/prometheus/exporters/kube-state-metrics/deployment.yaml | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index 16ce2907c..87740084e 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -21,6 +21,7 @@ Notable changes between versions. #### Addons * Update Prometheus from v2.15.2 to [v2.16.0](https://github.com/prometheus/prometheus/releases/tag/v2.16.0) + * Update kube-state-metrics from v1.9.4 to v1.9.5 * Update Grafana from v6.6.1 to v6.6.2 ## v1.17.3 diff --git a/addons/prometheus/exporters/kube-state-metrics/deployment.yaml b/addons/prometheus/exporters/kube-state-metrics/deployment.yaml index efef4d1a9..bf9274fe9 100644 --- a/addons/prometheus/exporters/kube-state-metrics/deployment.yaml +++ b/addons/prometheus/exporters/kube-state-metrics/deployment.yaml @@ -24,7 +24,7 @@ spec: serviceAccountName: kube-state-metrics containers: - name: kube-state-metrics - image: quay.io/coreos/kube-state-metrics:v1.9.4 + image: quay.io/coreos/kube-state-metrics:v1.9.5 ports: - name: metrics containerPort: 8080 From 60c7eb85ee42cc7d4564cd20ee55b099cfab4de8 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Sat, 22 Feb 2020 15:57:59 -0800 Subject: [PATCH 343/523] Update nginx-ingress from v0.28.0 to v0.29.0 * https://github.com/kubernetes/ingress-nginx/releases/tag/nginx-0.29.0 --- CHANGES.md | 1 + addons/nginx-ingress/aws/deployment.yaml | 2 +- addons/nginx-ingress/azure/deployment.yaml | 2 +- addons/nginx-ingress/bare-metal/deployment.yaml | 2 +- addons/nginx-ingress/digital-ocean/daemonset.yaml | 2 +- addons/nginx-ingress/google-cloud/deployment.yaml | 2 +- 6 files changed, 6 insertions(+), 5 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 87740084e..414cfcb36 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -20,6 +20,7 @@ Notable changes between versions. #### Addons +* Update nginx-ingress from v0.28.0 to [v0.29.0](https://github.com/kubernetes/ingress-nginx/releases/tag/nginx-0.29.0) * Update Prometheus from v2.15.2 to [v2.16.0](https://github.com/prometheus/prometheus/releases/tag/v2.16.0) * Update kube-state-metrics from v1.9.4 to v1.9.5 * Update Grafana from v6.6.1 to v6.6.2 diff --git a/addons/nginx-ingress/aws/deployment.yaml b/addons/nginx-ingress/aws/deployment.yaml index 5bfd2df3c..98af919f1 100644 --- a/addons/nginx-ingress/aws/deployment.yaml +++ b/addons/nginx-ingress/aws/deployment.yaml @@ -22,7 +22,7 @@ spec: spec: containers: - name: nginx-ingress-controller - image: quay.io/kubernetes-ingress-controller/nginx-ingress-controller:0.28.0 + image: quay.io/kubernetes-ingress-controller/nginx-ingress-controller:0.29.0 args: - /nginx-ingress-controller - --ingress-class=public diff --git a/addons/nginx-ingress/azure/deployment.yaml b/addons/nginx-ingress/azure/deployment.yaml index 5bfd2df3c..98af919f1 100644 --- a/addons/nginx-ingress/azure/deployment.yaml +++ b/addons/nginx-ingress/azure/deployment.yaml @@ -22,7 +22,7 @@ spec: spec: containers: - name: nginx-ingress-controller - image: quay.io/kubernetes-ingress-controller/nginx-ingress-controller:0.28.0 + image: quay.io/kubernetes-ingress-controller/nginx-ingress-controller:0.29.0 args: - /nginx-ingress-controller - --ingress-class=public diff --git a/addons/nginx-ingress/bare-metal/deployment.yaml b/addons/nginx-ingress/bare-metal/deployment.yaml index 7d342024f..75f66d6da 100644 --- a/addons/nginx-ingress/bare-metal/deployment.yaml +++ b/addons/nginx-ingress/bare-metal/deployment.yaml @@ -22,7 +22,7 @@ spec: spec: containers: - name: nginx-ingress-controller - image: quay.io/kubernetes-ingress-controller/nginx-ingress-controller:0.28.0 + image: quay.io/kubernetes-ingress-controller/nginx-ingress-controller:0.29.0 args: - /nginx-ingress-controller - --ingress-class=public diff --git a/addons/nginx-ingress/digital-ocean/daemonset.yaml b/addons/nginx-ingress/digital-ocean/daemonset.yaml index 0f2167dc1..694945ef3 100644 --- a/addons/nginx-ingress/digital-ocean/daemonset.yaml +++ b/addons/nginx-ingress/digital-ocean/daemonset.yaml @@ -22,7 +22,7 @@ spec: spec: containers: - name: nginx-ingress-controller - image: quay.io/kubernetes-ingress-controller/nginx-ingress-controller:0.28.0 + image: quay.io/kubernetes-ingress-controller/nginx-ingress-controller:0.29.0 args: - /nginx-ingress-controller - --ingress-class=public diff --git a/addons/nginx-ingress/google-cloud/deployment.yaml b/addons/nginx-ingress/google-cloud/deployment.yaml index 5bfd2df3c..98af919f1 100644 --- a/addons/nginx-ingress/google-cloud/deployment.yaml +++ b/addons/nginx-ingress/google-cloud/deployment.yaml @@ -22,7 +22,7 @@ spec: spec: containers: - name: nginx-ingress-controller - image: quay.io/kubernetes-ingress-controller/nginx-ingress-controller:0.28.0 + image: quay.io/kubernetes-ingress-controller/nginx-ingress-controller:0.29.0 args: - /nginx-ingress-controller - --ingress-class=public From d9219a67221b1cc8716b119ecbe41e417ad923e8 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Tue, 25 Feb 2020 22:11:59 -0800 Subject: [PATCH 344/523] Update nginx-ingress from v0.29.0 to v0.30.0 * https://github.com/kubernetes/ingress-nginx/releases/tag/nginx-0.30.0 --- CHANGES.md | 2 +- addons/nginx-ingress/aws/deployment.yaml | 2 +- addons/nginx-ingress/azure/deployment.yaml | 2 +- addons/nginx-ingress/bare-metal/deployment.yaml | 2 +- addons/nginx-ingress/digital-ocean/daemonset.yaml | 2 +- addons/nginx-ingress/google-cloud/deployment.yaml | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 414cfcb36..389bac593 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -20,7 +20,7 @@ Notable changes between versions. #### Addons -* Update nginx-ingress from v0.28.0 to [v0.29.0](https://github.com/kubernetes/ingress-nginx/releases/tag/nginx-0.29.0) +* Update nginx-ingress from v0.28.0 to [v0.30.0](https://github.com/kubernetes/ingress-nginx/releases/tag/nginx-0.30.0) * Update Prometheus from v2.15.2 to [v2.16.0](https://github.com/prometheus/prometheus/releases/tag/v2.16.0) * Update kube-state-metrics from v1.9.4 to v1.9.5 * Update Grafana from v6.6.1 to v6.6.2 diff --git a/addons/nginx-ingress/aws/deployment.yaml b/addons/nginx-ingress/aws/deployment.yaml index 98af919f1..56b74e882 100644 --- a/addons/nginx-ingress/aws/deployment.yaml +++ b/addons/nginx-ingress/aws/deployment.yaml @@ -22,7 +22,7 @@ spec: spec: containers: - name: nginx-ingress-controller - image: quay.io/kubernetes-ingress-controller/nginx-ingress-controller:0.29.0 + image: quay.io/kubernetes-ingress-controller/nginx-ingress-controller:0.30.0 args: - /nginx-ingress-controller - --ingress-class=public diff --git a/addons/nginx-ingress/azure/deployment.yaml b/addons/nginx-ingress/azure/deployment.yaml index 98af919f1..56b74e882 100644 --- a/addons/nginx-ingress/azure/deployment.yaml +++ b/addons/nginx-ingress/azure/deployment.yaml @@ -22,7 +22,7 @@ spec: spec: containers: - name: nginx-ingress-controller - image: quay.io/kubernetes-ingress-controller/nginx-ingress-controller:0.29.0 + image: quay.io/kubernetes-ingress-controller/nginx-ingress-controller:0.30.0 args: - /nginx-ingress-controller - --ingress-class=public diff --git a/addons/nginx-ingress/bare-metal/deployment.yaml b/addons/nginx-ingress/bare-metal/deployment.yaml index 75f66d6da..ac86bd5fe 100644 --- a/addons/nginx-ingress/bare-metal/deployment.yaml +++ b/addons/nginx-ingress/bare-metal/deployment.yaml @@ -22,7 +22,7 @@ spec: spec: containers: - name: nginx-ingress-controller - image: quay.io/kubernetes-ingress-controller/nginx-ingress-controller:0.29.0 + image: quay.io/kubernetes-ingress-controller/nginx-ingress-controller:0.30.0 args: - /nginx-ingress-controller - --ingress-class=public diff --git a/addons/nginx-ingress/digital-ocean/daemonset.yaml b/addons/nginx-ingress/digital-ocean/daemonset.yaml index 694945ef3..1bf474d66 100644 --- a/addons/nginx-ingress/digital-ocean/daemonset.yaml +++ b/addons/nginx-ingress/digital-ocean/daemonset.yaml @@ -22,7 +22,7 @@ spec: spec: containers: - name: nginx-ingress-controller - image: quay.io/kubernetes-ingress-controller/nginx-ingress-controller:0.29.0 + image: quay.io/kubernetes-ingress-controller/nginx-ingress-controller:0.30.0 args: - /nginx-ingress-controller - --ingress-class=public diff --git a/addons/nginx-ingress/google-cloud/deployment.yaml b/addons/nginx-ingress/google-cloud/deployment.yaml index 98af919f1..56b74e882 100644 --- a/addons/nginx-ingress/google-cloud/deployment.yaml +++ b/addons/nginx-ingress/google-cloud/deployment.yaml @@ -22,7 +22,7 @@ spec: spec: containers: - name: nginx-ingress-controller - image: quay.io/kubernetes-ingress-controller/nginx-ingress-controller:0.29.0 + image: quay.io/kubernetes-ingress-controller/nginx-ingress-controller:0.30.0 args: - /nginx-ingress-controller - --ingress-class=public From f4d260645c0909ea606c68f8e49d75228310c7ce Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Tue, 25 Feb 2020 22:26:24 -0800 Subject: [PATCH 345/523] Update node-exporter from v0.18.1 to v1.0.0-rc.0 * Update mdadm alert rule; node-exporter adds `state` label to `node_md_disks` and removes `node_md_disks_active` * https://github.com/prometheus/node_exporter/releases/tag/v1.0.0-rc.0 --- CHANGES.md | 1 + addons/prometheus/exporters/node-exporter/daemonset.yaml | 2 +- addons/prometheus/rules.yaml | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 389bac593..251456e31 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -23,6 +23,7 @@ Notable changes between versions. * Update nginx-ingress from v0.28.0 to [v0.30.0](https://github.com/kubernetes/ingress-nginx/releases/tag/nginx-0.30.0) * Update Prometheus from v2.15.2 to [v2.16.0](https://github.com/prometheus/prometheus/releases/tag/v2.16.0) * Update kube-state-metrics from v1.9.4 to v1.9.5 + * Update node-exporter from v0.18.1 to [v1.0.0-rc.0](https://github.com/prometheus/node_exporter/releases/tag/v1.0.0-rc.0) * Update Grafana from v6.6.1 to v6.6.2 ## v1.17.3 diff --git a/addons/prometheus/exporters/node-exporter/daemonset.yaml b/addons/prometheus/exporters/node-exporter/daemonset.yaml index 7ef88739e..da3f723af 100644 --- a/addons/prometheus/exporters/node-exporter/daemonset.yaml +++ b/addons/prometheus/exporters/node-exporter/daemonset.yaml @@ -28,7 +28,7 @@ spec: hostPID: true containers: - name: node-exporter - image: quay.io/prometheus/node-exporter:v0.18.1 + image: quay.io/prometheus/node-exporter:v1.0.0-rc.0 args: - --path.procfs=/host/proc - --path.sysfs=/host/sys diff --git a/addons/prometheus/rules.yaml b/addons/prometheus/rules.yaml index 78746f236..804f9e718 100644 --- a/addons/prometheus/rules.yaml +++ b/addons/prometheus/rules.yaml @@ -1212,7 +1212,7 @@ data: "annotations": { "message": "{{ $value }} RAID disk(s) on node {{ $labels.instance }} are inactive." }, - "expr": "node_md_disks - node_md_disks_active > 0", + "expr": "node_md_disks{state=\"failed\"} > 0", "for": "10m", "labels": { "severity": "warning" From 3250994c95eed489cf2a8b09ca899df37bedf2bf Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Tue, 25 Feb 2020 23:12:19 -0800 Subject: [PATCH 346/523] Use a route table with separate (rather than inline) routes * Allow users to extend the route table using a data reference and adding route resources (e.g. unusual peering setups) * Note: Internally connecting AWS clusters can reduce cross-cloud flexibility and inhibits blue-green cluster patterns. It is not recommended --- CHANGES.md | 1 + aws/container-linux/kubernetes/network.tf | 22 ++++++++++++---------- aws/fedora-coreos/kubernetes/network.tf | 22 ++++++++++++---------- docs/architecture/aws.md | 17 +++++++++++++++++ 4 files changed, 42 insertions(+), 20 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 251456e31..b39c5602c 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -9,6 +9,7 @@ Notable changes between versions. #### AWS * Fix `worker_node_labels` for setting initial worker node labels on Fedora CoreOS ([#651](https://github.com/poseidon/typhoon/pull/651)) +* Allow VPC route table extension via reference ([#654](https://github.com/poseidon/typhoon/pull/654)) #### Google Cloud diff --git a/aws/container-linux/kubernetes/network.tf b/aws/container-linux/kubernetes/network.tf index a93b3f0cb..f8ea0cec8 100644 --- a/aws/container-linux/kubernetes/network.tf +++ b/aws/container-linux/kubernetes/network.tf @@ -25,21 +25,23 @@ resource "aws_internet_gateway" "gateway" { resource "aws_route_table" "default" { vpc_id = aws_vpc.network.id - route { - cidr_block = "0.0.0.0/0" - gateway_id = aws_internet_gateway.gateway.id - } - - route { - ipv6_cidr_block = "::/0" - gateway_id = aws_internet_gateway.gateway.id - } - tags = { "Name" = var.cluster_name } } +resource "aws_route" "egress-ipv4" { + route_table_id = aws_route_table.default.id + destination_cidr_block = "0.0.0.0/0" + gateway_id = aws_internet_gateway.gateway.id +} + +resource "aws_route" "egress-ipv6" { + route_table_id = aws_route_table.default.id + destination_ipv6_cidr_block = "::/0" + gateway_id = aws_internet_gateway.gateway.id +} + # Subnets (one per availability zone) resource "aws_subnet" "public" { diff --git a/aws/fedora-coreos/kubernetes/network.tf b/aws/fedora-coreos/kubernetes/network.tf index a93b3f0cb..f8ea0cec8 100644 --- a/aws/fedora-coreos/kubernetes/network.tf +++ b/aws/fedora-coreos/kubernetes/network.tf @@ -25,21 +25,23 @@ resource "aws_internet_gateway" "gateway" { resource "aws_route_table" "default" { vpc_id = aws_vpc.network.id - route { - cidr_block = "0.0.0.0/0" - gateway_id = aws_internet_gateway.gateway.id - } - - route { - ipv6_cidr_block = "::/0" - gateway_id = aws_internet_gateway.gateway.id - } - tags = { "Name" = var.cluster_name } } +resource "aws_route" "egress-ipv4" { + route_table_id = aws_route_table.default.id + destination_cidr_block = "0.0.0.0/0" + gateway_id = aws_internet_gateway.gateway.id +} + +resource "aws_route" "egress-ipv6" { + route_table_id = aws_route_table.default.id + destination_ipv6_cidr_block = "::/0" + gateway_id = aws_internet_gateway.gateway.id +} + # Subnets (one per availability zone) resource "aws_subnet" "public" { diff --git a/docs/architecture/aws.md b/docs/architecture/aws.md index bdcc6aa14..2edf5085c 100644 --- a/docs/architecture/aws.md +++ b/docs/architecture/aws.md @@ -79,6 +79,23 @@ resource "aws_security_group_rule" "some-app" { } ``` +## Routes + +Add a custom [route](https://www.terraform.io/docs/providers/aws/r/route.html) to the VPC route table. + +```tf +data "aws_route_table" "default" { + vpc_id = module.temptest.vpc_id + subnet_id = module.tempest.subnet_ids[0] +} + +resource "aws_route" "peering" { + route_table_id = data.aws_route_table.default.id + destination_cidr_block = "192.168.4.0/24" + ... +} +``` + ## IPv6 AWS Network Load Balancers do not support `dualstack`. From 6de5cf5a55ef4c08554d828fd436a71b40b3edad Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Tue, 25 Feb 2020 23:36:33 -0800 Subject: [PATCH 347/523] Update etcd from v3.4.3 to v3.4.4 * https://github.com/etcd-io/etcd/releases/tag/v3.4.4 --- CHANGES.md | 1 + aws/container-linux/kubernetes/cl/controller.yaml | 2 +- aws/fedora-coreos/kubernetes/fcc/controller.yaml | 2 +- azure/container-linux/kubernetes/cl/controller.yaml | 2 +- bare-metal/container-linux/kubernetes/cl/controller.yaml | 2 +- bare-metal/fedora-coreos/kubernetes/fcc/controller.yaml | 2 +- digital-ocean/container-linux/kubernetes/cl/controller.yaml | 2 +- google-cloud/container-linux/kubernetes/cl/controller.yaml | 2 +- google-cloud/fedora-coreos/kubernetes/fcc/controller.yaml | 2 +- 9 files changed, 9 insertions(+), 8 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index b39c5602c..9b621f65d 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -4,6 +4,7 @@ Notable changes between versions. ## Latest +* Update etcd from v3.4.3 to [v3.4.4](https://github.com/etcd-io/etcd/releases/tag/v3.4.4) * Update CoreDNS from v1.6.6 to v1.6.7 ([#648](https://github.com/poseidon/typhoon/pull/648)) #### AWS diff --git a/aws/container-linux/kubernetes/cl/controller.yaml b/aws/container-linux/kubernetes/cl/controller.yaml index 3e2a777be..fd24cef53 100644 --- a/aws/container-linux/kubernetes/cl/controller.yaml +++ b/aws/container-linux/kubernetes/cl/controller.yaml @@ -7,7 +7,7 @@ systemd: - name: 40-etcd-cluster.conf contents: | [Service] - Environment="ETCD_IMAGE_TAG=v3.4.3" + Environment="ETCD_IMAGE_TAG=v3.4.4" Environment="ETCD_NAME=${etcd_name}" Environment="ETCD_ADVERTISE_CLIENT_URLS=https://${etcd_domain}:2379" Environment="ETCD_INITIAL_ADVERTISE_PEER_URLS=https://${etcd_domain}:2380" diff --git a/aws/fedora-coreos/kubernetes/fcc/controller.yaml b/aws/fedora-coreos/kubernetes/fcc/controller.yaml index a90558c7b..9c4e272ef 100644 --- a/aws/fedora-coreos/kubernetes/fcc/controller.yaml +++ b/aws/fedora-coreos/kubernetes/fcc/controller.yaml @@ -28,7 +28,7 @@ systemd: --network host \ --volume /var/lib/etcd:/var/lib/etcd:rw,Z \ --volume /etc/ssl/etcd:/etc/ssl/certs:ro,Z \ - quay.io/coreos/etcd:v3.4.3 + quay.io/coreos/etcd:v3.4.4 ExecStop=/usr/bin/podman stop etcd [Install] WantedBy=multi-user.target diff --git a/azure/container-linux/kubernetes/cl/controller.yaml b/azure/container-linux/kubernetes/cl/controller.yaml index 850c7173b..dda5ccc71 100644 --- a/azure/container-linux/kubernetes/cl/controller.yaml +++ b/azure/container-linux/kubernetes/cl/controller.yaml @@ -7,7 +7,7 @@ systemd: - name: 40-etcd-cluster.conf contents: | [Service] - Environment="ETCD_IMAGE_TAG=v3.4.3" + Environment="ETCD_IMAGE_TAG=v3.4.4" Environment="ETCD_NAME=${etcd_name}" Environment="ETCD_ADVERTISE_CLIENT_URLS=https://${etcd_domain}:2379" Environment="ETCD_INITIAL_ADVERTISE_PEER_URLS=https://${etcd_domain}:2380" diff --git a/bare-metal/container-linux/kubernetes/cl/controller.yaml b/bare-metal/container-linux/kubernetes/cl/controller.yaml index 4f5dd370b..96f1b2c6e 100644 --- a/bare-metal/container-linux/kubernetes/cl/controller.yaml +++ b/bare-metal/container-linux/kubernetes/cl/controller.yaml @@ -7,7 +7,7 @@ systemd: - name: 40-etcd-cluster.conf contents: | [Service] - Environment="ETCD_IMAGE_TAG=v3.4.3" + Environment="ETCD_IMAGE_TAG=v3.4.4" Environment="ETCD_NAME=${etcd_name}" Environment="ETCD_ADVERTISE_CLIENT_URLS=https://${domain_name}:2379" Environment="ETCD_INITIAL_ADVERTISE_PEER_URLS=https://${domain_name}:2380" diff --git a/bare-metal/fedora-coreos/kubernetes/fcc/controller.yaml b/bare-metal/fedora-coreos/kubernetes/fcc/controller.yaml index 6a6f74e20..72585414f 100644 --- a/bare-metal/fedora-coreos/kubernetes/fcc/controller.yaml +++ b/bare-metal/fedora-coreos/kubernetes/fcc/controller.yaml @@ -28,7 +28,7 @@ systemd: --network host \ --volume /var/lib/etcd:/var/lib/etcd:rw,Z \ --volume /etc/ssl/etcd:/etc/ssl/certs:ro,Z \ - quay.io/coreos/etcd:v3.4.3 + quay.io/coreos/etcd:v3.4.4 ExecStop=/usr/bin/podman stop etcd [Install] WantedBy=multi-user.target diff --git a/digital-ocean/container-linux/kubernetes/cl/controller.yaml b/digital-ocean/container-linux/kubernetes/cl/controller.yaml index f751e6824..0e2bac646 100644 --- a/digital-ocean/container-linux/kubernetes/cl/controller.yaml +++ b/digital-ocean/container-linux/kubernetes/cl/controller.yaml @@ -7,7 +7,7 @@ systemd: - name: 40-etcd-cluster.conf contents: | [Service] - Environment="ETCD_IMAGE_TAG=v3.4.3" + Environment="ETCD_IMAGE_TAG=v3.4.4" Environment="ETCD_NAME=${etcd_name}" Environment="ETCD_ADVERTISE_CLIENT_URLS=https://${etcd_domain}:2379" Environment="ETCD_INITIAL_ADVERTISE_PEER_URLS=https://${etcd_domain}:2380" diff --git a/google-cloud/container-linux/kubernetes/cl/controller.yaml b/google-cloud/container-linux/kubernetes/cl/controller.yaml index 4f286eee5..d790dd379 100644 --- a/google-cloud/container-linux/kubernetes/cl/controller.yaml +++ b/google-cloud/container-linux/kubernetes/cl/controller.yaml @@ -7,7 +7,7 @@ systemd: - name: 40-etcd-cluster.conf contents: | [Service] - Environment="ETCD_IMAGE_TAG=v3.4.3" + Environment="ETCD_IMAGE_TAG=v3.4.4" Environment="ETCD_NAME=${etcd_name}" Environment="ETCD_ADVERTISE_CLIENT_URLS=https://${etcd_domain}:2379" Environment="ETCD_INITIAL_ADVERTISE_PEER_URLS=https://${etcd_domain}:2380" diff --git a/google-cloud/fedora-coreos/kubernetes/fcc/controller.yaml b/google-cloud/fedora-coreos/kubernetes/fcc/controller.yaml index a90558c7b..9c4e272ef 100644 --- a/google-cloud/fedora-coreos/kubernetes/fcc/controller.yaml +++ b/google-cloud/fedora-coreos/kubernetes/fcc/controller.yaml @@ -28,7 +28,7 @@ systemd: --network host \ --volume /var/lib/etcd:/var/lib/etcd:rw,Z \ --volume /etc/ssl/etcd:/etc/ssl/certs:ro,Z \ - quay.io/coreos/etcd:v3.4.3 + quay.io/coreos/etcd:v3.4.4 ExecStop=/usr/bin/podman stop etcd [Install] WantedBy=multi-user.target From 87f9a2fc35f881a310aeeaa54347884b6b97e083 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Sat, 29 Feb 2020 20:12:06 -0800 Subject: [PATCH 348/523] Add automatic worker deletion on Fedora CoreOS clouds * On clouds where workers can scale down or be preempted (AWS, GCP, Azure), shutdown runs delete-node.service to remove a node a prevent NotReady nodes from lingering * Add the delete-node.service that wasn't carried over from Container Linux and port it to use podman --- CHANGES.md | 6 ++++-- aws/fedora-coreos/kubernetes/workers/fcc/worker.yaml | 12 ++++++++++++ .../fedora-coreos/kubernetes/workers/fcc/worker.yaml | 12 ++++++++++++ 3 files changed, 28 insertions(+), 2 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 9b621f65d..051dc8130 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -9,12 +9,14 @@ Notable changes between versions. #### AWS -* Fix `worker_node_labels` for setting initial worker node labels on Fedora CoreOS ([#651](https://github.com/poseidon/typhoon/pull/651)) * Allow VPC route table extension via reference ([#654](https://github.com/poseidon/typhoon/pull/654)) +* Fix `worker_node_labels` on Fedora CoreOS ([#651](https://github.com/poseidon/typhoon/pull/651)) +* Fix automatic worker node delete on shutdown on Fedora CoreOS ([#657](https://github.com/poseidon/typhoon/pull/657)) #### Google Cloud -* Fix `worker_node_labels` for setting initial worker node labels on Fedora CoreOS ([#651](https://github.com/poseidon/typhoon/pull/651)) +* Fix `worker_node_labels` on Fedora CoreOS ([#651](https://github.com/poseidon/typhoon/pull/651)) +* Fix automatic worker node delete on shutdown on Fedora CoreOS ([#657](https://github.com/poseidon/typhoon/pull/657)) #### DigitalOcean diff --git a/aws/fedora-coreos/kubernetes/workers/fcc/worker.yaml b/aws/fedora-coreos/kubernetes/workers/fcc/worker.yaml index 6e6fd2fef..e2b8e0363 100644 --- a/aws/fedora-coreos/kubernetes/workers/fcc/worker.yaml +++ b/aws/fedora-coreos/kubernetes/workers/fcc/worker.yaml @@ -78,6 +78,18 @@ systemd: RestartSec=10 [Install] WantedBy=multi-user.target + - name: delete-node.service + enabled: true + contents: | + [Unit] + Description=Delete Kubernetes node on shutdown + [Service] + Type=oneshot + RemainAfterExit=true + ExecStart=/bin/true + ExecStop=/bin/bash -c '/usr/bin/podman run --volume /etc/kubernetes:/etc/kubernetes:ro,z k8s.gcr.io/hyperkube:v1.17.3 kubectl --kubeconfig=/etc/kubernetes/kubeconfig delete node $HOSTNAME' + [Install] + WantedBy=multi-user.target storage: directories: - path: /etc/kubernetes diff --git a/google-cloud/fedora-coreos/kubernetes/workers/fcc/worker.yaml b/google-cloud/fedora-coreos/kubernetes/workers/fcc/worker.yaml index 6e6fd2fef..e2b8e0363 100644 --- a/google-cloud/fedora-coreos/kubernetes/workers/fcc/worker.yaml +++ b/google-cloud/fedora-coreos/kubernetes/workers/fcc/worker.yaml @@ -78,6 +78,18 @@ systemd: RestartSec=10 [Install] WantedBy=multi-user.target + - name: delete-node.service + enabled: true + contents: | + [Unit] + Description=Delete Kubernetes node on shutdown + [Service] + Type=oneshot + RemainAfterExit=true + ExecStart=/bin/true + ExecStop=/bin/bash -c '/usr/bin/podman run --volume /etc/kubernetes:/etc/kubernetes:ro,z k8s.gcr.io/hyperkube:v1.17.3 kubectl --kubeconfig=/etc/kubernetes/kubeconfig delete node $HOSTNAME' + [Install] + WantedBy=multi-user.target storage: directories: - path: /etc/kubernetes From 51cee6d5a486e08a25a1c25679a7afa18ba57387 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Mon, 2 Mar 2020 08:11:15 -0800 Subject: [PATCH 349/523] Change Container Linux etcd-member to fetch with docker:// * Quay has historically generated ACI signatures for images to facilitate rkt's notions of verification (it allowed authors to actually sign images, though `--trust-keys-from-https` is in use since etcd and most authors don't sign images). OCI standardization didn't adopt verification ideas and checking signatures has fallen out of favor. * Fix an issue where Quay no longer seems to be generating ACI signatures for new images (e.g. quay.io/coreos/etcd:v.3.4.4) * Don't be alarmed by rkt `--insecure-options=image`. It refers to disabling image signature checking (i.e. docker pull doesn't check signatures either) * System containers for Kubelet and bootstrap have transitioned to the docker:// transport, so there is precedent and this brings all the system containers on Container Linux controllers into alignment --- CHANGES.md | 1 + aws/container-linux/kubernetes/cl/controller.yaml | 2 ++ azure/container-linux/kubernetes/cl/controller.yaml | 2 ++ bare-metal/container-linux/kubernetes/cl/controller.yaml | 2 ++ digital-ocean/container-linux/kubernetes/cl/controller.yaml | 2 ++ google-cloud/container-linux/kubernetes/cl/controller.yaml | 2 ++ 6 files changed, 11 insertions(+) diff --git a/CHANGES.md b/CHANGES.md index 051dc8130..f21beb51b 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -5,6 +5,7 @@ Notable changes between versions. ## Latest * Update etcd from v3.4.3 to [v3.4.4](https://github.com/etcd-io/etcd/releases/tag/v3.4.4) + * On Container Linux, fetch using the docker transport format ([#659](https://github.com/poseidon/typhoon/pull/659)) * Update CoreDNS from v1.6.6 to v1.6.7 ([#648](https://github.com/poseidon/typhoon/pull/648)) #### AWS diff --git a/aws/container-linux/kubernetes/cl/controller.yaml b/aws/container-linux/kubernetes/cl/controller.yaml index fd24cef53..41c3319c0 100644 --- a/aws/container-linux/kubernetes/cl/controller.yaml +++ b/aws/container-linux/kubernetes/cl/controller.yaml @@ -8,6 +8,8 @@ systemd: contents: | [Service] Environment="ETCD_IMAGE_TAG=v3.4.4" + Environment="ETCD_IMAGE_URL=docker://quay.io/coreos/etcd" + Environment="RKT_RUN_ARGS=--insecure-options=image" Environment="ETCD_NAME=${etcd_name}" Environment="ETCD_ADVERTISE_CLIENT_URLS=https://${etcd_domain}:2379" Environment="ETCD_INITIAL_ADVERTISE_PEER_URLS=https://${etcd_domain}:2380" diff --git a/azure/container-linux/kubernetes/cl/controller.yaml b/azure/container-linux/kubernetes/cl/controller.yaml index dda5ccc71..a9b834d0a 100644 --- a/azure/container-linux/kubernetes/cl/controller.yaml +++ b/azure/container-linux/kubernetes/cl/controller.yaml @@ -8,6 +8,8 @@ systemd: contents: | [Service] Environment="ETCD_IMAGE_TAG=v3.4.4" + Environment="ETCD_IMAGE_URL=docker://quay.io/coreos/etcd" + Environment="RKT_RUN_ARGS=--insecure-options=image" Environment="ETCD_NAME=${etcd_name}" Environment="ETCD_ADVERTISE_CLIENT_URLS=https://${etcd_domain}:2379" Environment="ETCD_INITIAL_ADVERTISE_PEER_URLS=https://${etcd_domain}:2380" diff --git a/bare-metal/container-linux/kubernetes/cl/controller.yaml b/bare-metal/container-linux/kubernetes/cl/controller.yaml index 96f1b2c6e..569205504 100644 --- a/bare-metal/container-linux/kubernetes/cl/controller.yaml +++ b/bare-metal/container-linux/kubernetes/cl/controller.yaml @@ -8,6 +8,8 @@ systemd: contents: | [Service] Environment="ETCD_IMAGE_TAG=v3.4.4" + Environment="ETCD_IMAGE_URL=docker://quay.io/coreos/etcd" + Environment="RKT_RUN_ARGS=--insecure-options=image" Environment="ETCD_NAME=${etcd_name}" Environment="ETCD_ADVERTISE_CLIENT_URLS=https://${domain_name}:2379" Environment="ETCD_INITIAL_ADVERTISE_PEER_URLS=https://${domain_name}:2380" diff --git a/digital-ocean/container-linux/kubernetes/cl/controller.yaml b/digital-ocean/container-linux/kubernetes/cl/controller.yaml index 0e2bac646..3950c7dc3 100644 --- a/digital-ocean/container-linux/kubernetes/cl/controller.yaml +++ b/digital-ocean/container-linux/kubernetes/cl/controller.yaml @@ -8,6 +8,8 @@ systemd: contents: | [Service] Environment="ETCD_IMAGE_TAG=v3.4.4" + Environment="ETCD_IMAGE_URL=docker://quay.io/coreos/etcd" + Environment="RKT_RUN_ARGS=--insecure-options=image" Environment="ETCD_NAME=${etcd_name}" Environment="ETCD_ADVERTISE_CLIENT_URLS=https://${etcd_domain}:2379" Environment="ETCD_INITIAL_ADVERTISE_PEER_URLS=https://${etcd_domain}:2380" diff --git a/google-cloud/container-linux/kubernetes/cl/controller.yaml b/google-cloud/container-linux/kubernetes/cl/controller.yaml index d790dd379..e57ee4290 100644 --- a/google-cloud/container-linux/kubernetes/cl/controller.yaml +++ b/google-cloud/container-linux/kubernetes/cl/controller.yaml @@ -8,6 +8,8 @@ systemd: contents: | [Service] Environment="ETCD_IMAGE_TAG=v3.4.4" + Environment="ETCD_IMAGE_URL=docker://quay.io/coreos/etcd" + Environment="RKT_RUN_ARGS=--insecure-options=image" Environment="ETCD_NAME=${etcd_name}" Environment="ETCD_ADVERTISE_CLIENT_URLS=https://${etcd_domain}:2379" Environment="ETCD_INITIAL_ADVERTISE_PEER_URLS=https://${etcd_domain}:2380" From c4683c5bad8c8bbacee7d8913fd5f39559fa50ae Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Sun, 1 Mar 2020 23:20:33 -0800 Subject: [PATCH 350/523] Refresh Prometheus alerts and Grafana dashboards * Add 2 min wait before KubeNodeUnreachable to be less noisy on premeptible clusters * Add a BlackboxProbeFailure alert for any failing probes for services annotated `prometheus.io/probe: true` --- CHANGES.md | 3 + addons/grafana/dashboards-k8s-nodes.yaml | 194 +- .../grafana/dashboards-k8s-resources-1.yaml | 622 ++++- .../grafana/dashboards-k8s-resources-2.yaml | 527 +++- addons/grafana/dashboards-k8s.yaml | 2481 ++++------------- addons/grafana/dashboards-prom.yaml | 878 +++++- addons/prometheus/rules.yaml | 303 +- 7 files changed, 2539 insertions(+), 2469 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index f21beb51b..71da23729 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -27,9 +27,12 @@ Notable changes between versions. * Update nginx-ingress from v0.28.0 to [v0.30.0](https://github.com/kubernetes/ingress-nginx/releases/tag/nginx-0.30.0) * Update Prometheus from v2.15.2 to [v2.16.0](https://github.com/prometheus/prometheus/releases/tag/v2.16.0) + * Refresh Prometheus rules and alerts + * Add a BlackboxProbeFailure alert * Update kube-state-metrics from v1.9.4 to v1.9.5 * Update node-exporter from v0.18.1 to [v1.0.0-rc.0](https://github.com/prometheus/node_exporter/releases/tag/v1.0.0-rc.0) * Update Grafana from v6.6.1 to v6.6.2 + * Refresh Grafana dashboards ## v1.17.3 diff --git a/addons/grafana/dashboards-k8s-nodes.yaml b/addons/grafana/dashboards-k8s-nodes.yaml index 1b42cd429..9a67f088f 100644 --- a/addons/grafana/dashboards-k8s-nodes.yaml +++ b/addons/grafana/dashboards-k8s-nodes.yaml @@ -21,7 +21,7 @@ data: "links": [ ], - "refresh": "", + "refresh": "10s", "rows": [ { "collapse": false, @@ -558,15 +558,15 @@ data: }, "id": 8, "legend": { - "alignAsTable": "true", + "alignAsTable": true, "avg": false, - "current": "true", + "current": true, "max": false, "min": false, - "rightSide": "true", - "show": "true", + "rightSide": true, + "show": true, "total": false, - "values": "true" + "values": true }, "lines": true, "linewidth": 1, @@ -649,15 +649,15 @@ data: }, "id": 9, "legend": { - "alignAsTable": "true", + "alignAsTable": true, "avg": false, - "current": "true", + "current": true, "max": false, "min": false, - "rightSide": "true", - "show": "true", + "rightSide": true, + "show": true, "total": false, - "values": "true" + "values": true }, "lines": true, "linewidth": 1, @@ -753,15 +753,15 @@ data: }, "id": 10, "legend": { - "alignAsTable": "true", + "alignAsTable": true, "avg": false, - "current": "true", + "current": true, "max": false, "min": false, - "rightSide": "true", - "show": "true", + "rightSide": true, + "show": true, "total": false, - "values": "true" + "values": true }, "lines": true, "linewidth": 1, @@ -857,15 +857,15 @@ data: }, "id": 11, "legend": { - "alignAsTable": "true", + "alignAsTable": true, "avg": false, - "current": "true", + "current": true, "max": false, "min": false, - "rightSide": "true", - "show": "true", + "rightSide": true, + "show": true, "total": false, - "values": "true" + "values": true }, "lines": true, "linewidth": 1, @@ -955,15 +955,15 @@ data: }, "id": 12, "legend": { - "alignAsTable": "true", + "alignAsTable": true, "avg": false, - "current": "true", + "current": true, "max": false, "min": false, - "rightSide": "true", - "show": "true", + "rightSide": true, + "show": true, "total": false, - "values": "true" + "values": true }, "lines": true, "linewidth": 1, @@ -1066,17 +1066,17 @@ data: }, "id": 13, "legend": { - "alignAsTable": "true", + "alignAsTable": true, "avg": false, - "current": "true", - "hideEmpty": "true", - "hideZero": "true", + "current": true, + "hideEmpty": true, + "hideZero": true, "max": false, "min": false, - "rightSide": "true", - "show": "true", + "rightSide": true, + "show": true, "total": false, - "values": "true" + "values": true }, "lines": true, "linewidth": 1, @@ -1159,17 +1159,17 @@ data: }, "id": 14, "legend": { - "alignAsTable": "true", + "alignAsTable": true, "avg": false, - "current": "true", - "hideEmpty": "true", - "hideZero": "true", + "current": true, + "hideEmpty": true, + "hideZero": true, "max": false, "min": false, - "rightSide": "true", - "show": "true", + "rightSide": true, + "show": true, "total": false, - "values": "true" + "values": true }, "lines": true, "linewidth": 1, @@ -1265,17 +1265,17 @@ data: }, "id": 15, "legend": { - "alignAsTable": "true", + "alignAsTable": true, "avg": false, - "current": "true", - "hideEmpty": "true", - "hideZero": "true", + "current": true, + "hideEmpty": true, + "hideZero": true, "max": false, "min": false, - "rightSide": "true", + "rightSide": true, "show": true, "total": false, - "values": "true" + "values": true }, "lines": true, "linewidth": 1, @@ -1371,15 +1371,15 @@ data: }, "id": 16, "legend": { - "alignAsTable": "true", + "alignAsTable": true, "avg": false, - "current": "true", + "current": true, "max": false, "min": false, - "rightSide": "true", - "show": "true", + "rightSide": true, + "show": true, "total": false, - "values": "true" + "values": true }, "lines": true, "linewidth": 1, @@ -1462,15 +1462,15 @@ data: }, "id": 17, "legend": { - "alignAsTable": "true", + "alignAsTable": true, "avg": false, - "current": "true", + "current": true, "max": false, "min": false, - "rightSide": "true", - "show": "true", + "rightSide": true, + "show": true, "total": false, - "values": "true" + "values": true }, "lines": true, "linewidth": 1, @@ -1567,15 +1567,15 @@ data: }, "id": 18, "legend": { - "alignAsTable": "true", + "alignAsTable": true, "avg": false, - "current": "true", + "current": true, "max": false, "min": false, - "rightSide": "true", - "show": "true", + "rightSide": true, + "show": true, "total": false, - "values": "true" + "values": true }, "lines": true, "linewidth": 1, @@ -1658,15 +1658,15 @@ data: }, "id": 19, "legend": { - "alignAsTable": "true", + "alignAsTable": true, "avg": false, - "current": "true", + "current": true, "max": false, "min": false, - "rightSide": "true", - "show": "true", + "rightSide": true, + "show": true, "total": false, - "values": "true" + "values": true }, "lines": true, "linewidth": 1, @@ -1762,15 +1762,15 @@ data: }, "id": 20, "legend": { - "alignAsTable": "true", + "alignAsTable": true, "avg": false, - "current": "true", + "current": true, "max": false, "min": false, - "rightSide": "true", - "show": "true", + "rightSide": true, + "show": true, "total": false, - "values": "true" + "values": true }, "lines": true, "linewidth": 1, @@ -1991,15 +1991,15 @@ data: }, "id": 22, "legend": { - "alignAsTable": "true", + "alignAsTable": true, "avg": false, - "current": "true", + "current": true, "max": false, "min": false, - "rightSide": "true", - "show": "true", + "rightSide": true, + "show": true, "total": false, - "values": "true" + "values": true }, "lines": true, "linewidth": 1, @@ -2373,8 +2373,8 @@ data: "list": [ { "current": { - "text": "Prometheus", - "value": "Prometheus" + "text": "default", + "value": "default" }, "hide": 0, "label": null, @@ -2427,7 +2427,7 @@ data: "options": [ ], - "query": "label_values(kubelet_runtime_operations{cluster=\"$cluster\", job=\"kubelet\"}, instance)", + "query": "label_values(kubelet_runtime_operations_total{cluster=\"$cluster\", job=\"kubelet\"}, instance)", "refresh": 2, "regex": "", "sort": 1, @@ -2496,7 +2496,7 @@ data: "links": [ ], - "refresh": "", + "refresh": "10s", "rows": [ { "collapse": false, @@ -2691,15 +2691,15 @@ data: }, "id": 4, "legend": { - "alignAsTable": "true", + "alignAsTable": true, "avg": false, - "current": "true", + "current": true, "max": false, "min": false, - "rightSide": "true", - "show": "true", + "rightSide": true, + "show": true, "total": false, - "values": "true" + "values": true }, "lines": true, "linewidth": 1, @@ -2886,15 +2886,15 @@ data: }, "id": 6, "legend": { - "alignAsTable": "true", + "alignAsTable": true, "avg": false, - "current": "true", + "current": true, "max": false, "min": false, - "rightSide": "true", - "show": "true", + "rightSide": true, + "show": true, "total": false, - "values": "true" + "values": true }, "lines": true, "linewidth": 1, @@ -3206,15 +3206,15 @@ data: }, "id": 9, "legend": { - "alignAsTable": "true", + "alignAsTable": true, "avg": false, - "current": "true", + "current": true, "max": false, "min": false, - "rightSide": "true", - "show": "true", + "rightSide": true, + "show": true, "total": false, - "values": "true" + "values": true }, "lines": true, "linewidth": 1, @@ -3588,8 +3588,8 @@ data: "list": [ { "current": { - "text": "Prometheus", - "value": "Prometheus" + "text": "default", + "value": "default" }, "hide": 0, "label": null, diff --git a/addons/grafana/dashboards-k8s-resources-1.yaml b/addons/grafana/dashboards-k8s-resources-1.yaml index 110f2f338..b59a48be2 100644 --- a/addons/grafana/dashboards-k8s-resources-1.yaml +++ b/addons/grafana/dashboards-k8s-resources-1.yaml @@ -2458,8 +2458,8 @@ data: "list": [ { "current": { - "text": "Prometheus", - "value": "Prometheus" + "text": "default", + "value": "default" }, "hide": 0, "label": null, @@ -2508,7 +2508,7 @@ data: "text": "5m", "value": "5m" }, - "datasource": "prometheus", + "datasource": "$datasource", "hide": 2, "includeAll": false, "label": null, @@ -2533,6 +2533,33 @@ data: "tagsQuery": "", "type": "interval", "useTags": false + }, + { + "allValue": null, + "current": { + "text": "", + "value": "" + }, + "datasource": "$datasource", + "hide": 2, + "includeAll": false, + "label": null, + "multi": false, + "name": "cluster", + "options": [ + + ], + "query": "label_values(node_cpu_seconds_total, cluster)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false } ] }, @@ -2575,17 +2602,365 @@ data: "annotations": { "list": [ - ] - }, - "editable": true, - "gnetId": null, - "graphTooltip": 0, - "hideControls": false, - "links": [ + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "links": [ + + ], + "refresh": "10s", + "rows": [ + { + "collapse": false, + "height": "100px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "format": "percentunit", + "id": 1, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 3, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\"}) / sum(kube_pod_container_resource_requests_cpu_cores{cluster=\"$cluster\", namespace=\"$namespace\"})", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "refId": "A" + } + ], + "thresholds": "70,80", + "timeFrom": null, + "timeShift": null, + "title": "CPU Utilisation (from requests)", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "singlestat", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "format": "percentunit", + "id": 2, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 3, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\"}) / sum(kube_pod_container_resource_limits_cpu_cores{cluster=\"$cluster\", namespace=\"$namespace\"})", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "refId": "A" + } + ], + "thresholds": "70,80", + "timeFrom": null, + "timeShift": null, + "title": "CPU Utilisation (from limits)", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "singlestat", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "format": "percentunit", + "id": 3, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 3, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(container_memory_working_set_bytes{cluster=\"$cluster\", namespace=\"$namespace\",container!=\"\"}) / sum(kube_pod_container_resource_requests_memory_bytes{namespace=\"$namespace\"})", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "refId": "A" + } + ], + "thresholds": "70,80", + "timeFrom": null, + "timeShift": null, + "title": "Memory Utilization (from requests)", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "singlestat", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "format": "percentunit", + "id": 4, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ - ], - "refresh": "10s", - "rows": [ + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 3, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(container_memory_working_set_bytes{cluster=\"$cluster\", namespace=\"$namespace\",container!=\"\"}) / sum(kube_pod_container_resource_limits_memory_bytes{namespace=\"$namespace\"})", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "refId": "A" + } + ], + "thresholds": "70,80", + "timeFrom": null, + "timeShift": null, + "title": "Memory Utilisation (from limits)", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "singlestat", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Headlines", + "titleSize": "h6" + }, { "collapse": false, "height": "250px", @@ -2599,7 +2974,7 @@ data: "dashes": false, "datasource": "$datasource", "fill": 10, - "id": 1, + "id": 5, "legend": { "avg": false, "current": false, @@ -2620,7 +2995,26 @@ data: "points": false, "renderer": "flot", "seriesOverrides": [ - + { + "alias": "quota - requests", + "color": "#F2495C", + "dashes": true, + "fill": 0, + "hideTooltip": true, + "legend": false, + "linewidth": 2, + "stack": false + }, + { + "alias": "quota - limits", + "color": "#FF9830", + "dashes": true, + "fill": 0, + "hideTooltip": true, + "legend": false, + "linewidth": 2, + "stack": false + } ], "spaceLength": 10, "span": 12, @@ -2634,6 +3028,22 @@ data: "legendFormat": "{{pod}}", "legendLink": null, "step": 10 + }, + { + "expr": "scalar(kube_resourcequota{cluster=\"$cluster\", namespace=\"$namespace\", type=\"hard\",resource=\"requests.cpu\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "quota - requests", + "legendLink": null, + "step": 10 + }, + { + "expr": "scalar(kube_resourcequota{cluster=\"$cluster\", namespace=\"$namespace\", type=\"hard\",resource=\"limits.cpu\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "quota - limits", + "legendLink": null, + "step": 10 } ], "thresholds": [ @@ -2697,7 +3107,7 @@ data: "dashes": false, "datasource": "$datasource", "fill": 1, - "id": 2, + "id": 6, "legend": { "avg": false, "current": false, @@ -2964,7 +3374,7 @@ data: "dashes": false, "datasource": "$datasource", "fill": 10, - "id": 3, + "id": 7, "legend": { "avg": false, "current": false, @@ -2985,7 +3395,26 @@ data: "points": false, "renderer": "flot", "seriesOverrides": [ - + { + "alias": "quota - requests", + "color": "#F2495C", + "dashes": true, + "fill": 0, + "hideTooltip": true, + "legend": false, + "linewidth": 2, + "stack": false + }, + { + "alias": "quota - limits", + "color": "#FF9830", + "dashes": true, + "fill": 0, + "hideTooltip": true, + "legend": false, + "linewidth": 2, + "stack": false + } ], "spaceLength": 10, "span": 12, @@ -2999,6 +3428,22 @@ data: "legendFormat": "{{pod}}", "legendLink": null, "step": 10 + }, + { + "expr": "scalar(kube_resourcequota{cluster=\"$cluster\", namespace=\"$namespace\", type=\"hard\",resource=\"requests.memory\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "quota - requests", + "legendLink": null, + "step": 10 + }, + { + "expr": "scalar(kube_resourcequota{cluster=\"$cluster\", namespace=\"$namespace\", type=\"hard\",resource=\"limits.memory\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "quota - limits", + "legendLink": null, + "step": 10 } ], "thresholds": [ @@ -3062,7 +3507,7 @@ data: "dashes": false, "datasource": "$datasource", "fill": 1, - "id": 4, + "id": 8, "legend": { "avg": false, "current": false, @@ -3410,7 +3855,7 @@ data: "dashes": false, "datasource": "$datasource", "fill": 1, - "id": 5, + "id": 9, "legend": { "avg": false, "current": false, @@ -3704,7 +4149,7 @@ data: "dashes": false, "datasource": "$datasource", "fill": 10, - "id": 6, + "id": 10, "legend": { "avg": false, "current": false, @@ -3802,7 +4247,7 @@ data: "dashes": false, "datasource": "$datasource", "fill": 10, - "id": 7, + "id": 11, "legend": { "avg": false, "current": false, @@ -3900,7 +4345,7 @@ data: "dashes": false, "datasource": "$datasource", "fill": 10, - "id": 8, + "id": 12, "legend": { "avg": false, "current": false, @@ -3998,7 +4443,7 @@ data: "dashes": false, "datasource": "$datasource", "fill": 10, - "id": 9, + "id": 13, "legend": { "avg": false, "current": false, @@ -4096,7 +4541,7 @@ data: "dashes": false, "datasource": "$datasource", "fill": 10, - "id": 10, + "id": 14, "legend": { "avg": false, "current": false, @@ -4194,7 +4639,7 @@ data: "dashes": false, "datasource": "$datasource", "fill": 10, - "id": 11, + "id": 15, "legend": { "avg": false, "current": false, @@ -4289,8 +4734,8 @@ data: "list": [ { "current": { - "text": "Prometheus", - "value": "Prometheus" + "text": "default", + "value": "default" }, "hide": 0, "label": null, @@ -4305,50 +4750,58 @@ data: }, { "allValue": null, + "auto": false, + "auto_count": 30, + "auto_min": "10s", "current": { - "text": "prod", - "value": "prod" + "text": "5m", + "value": "5m" }, "datasource": "$datasource", "hide": 2, "includeAll": false, - "label": "cluster", + "label": null, "multi": false, - "name": "cluster", + "name": "interval", "options": [ - + { + "selected": true, + "text": "4h", + "value": "4h" + } ], - "query": "label_values(kube_pod_info, cluster)", - "refresh": 1, + "query": "4h", + "refresh": 2, "regex": "", - "sort": 2, + "skipUrlSync": false, + "sort": 1, "tagValuesQuery": "", "tags": [ ], "tagsQuery": "", - "type": "query", + "type": "interval", "useTags": false }, { "allValue": null, "current": { - "text": "prod", - "value": "prod" + "text": "", + "value": "" }, "datasource": "$datasource", - "hide": 0, + "hide": 2, "includeAll": false, - "label": "namespace", + "label": null, "multi": false, - "name": "namespace", + "name": "cluster", "options": [ ], - "query": "label_values(kube_pod_info{cluster=\"$cluster\"}, namespace)", + "query": "label_values(kube_pod_info, cluster)", "refresh": 1, "regex": "", - "sort": 2, + "sort": 1, "tagValuesQuery": "", "tags": [ @@ -4359,37 +4812,29 @@ data: }, { "allValue": null, - "auto": false, - "auto_count": 30, - "auto_min": "10s", "current": { - "text": "5m", - "value": "5m" + "text": "", + "value": "" }, - "datasource": "prometheus", - "hide": 2, + "datasource": "$datasource", + "hide": 0, "includeAll": false, "label": null, "multi": false, - "name": "interval", + "name": "namespace", "options": [ - { - "selected": true, - "text": "4h", - "value": "4h" - } + ], - "query": "4h", - "refresh": 2, + "query": "label_values(kube_pod_info{cluster=\"$cluster\"}, namespace)", + "refresh": 1, "regex": "", - "skipUrlSync": false, "sort": 1, "tagValuesQuery": "", "tags": [ ], "tagsQuery": "", - "type": "interval", + "type": "query", "useTags": false } ] @@ -5265,8 +5710,8 @@ data: "list": [ { "current": { - "text": "Prometheus", - "value": "Prometheus" + "text": "default", + "value": "default" }, "hide": 0, "label": null, @@ -5281,14 +5726,49 @@ data: }, { "allValue": null, + "auto": false, + "auto_count": 30, + "auto_min": "10s", "current": { - "text": "prod", - "value": "prod" + "text": "5m", + "value": "5m" }, "datasource": "$datasource", "hide": 2, "includeAll": false, - "label": "cluster", + "label": null, + "multi": false, + "name": "interval", + "options": [ + { + "selected": true, + "text": "4h", + "value": "4h" + } + ], + "query": "4h", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "interval", + "useTags": false + }, + { + "allValue": null, + "current": { + "text": "", + "value": "" + }, + "datasource": "$datasource", + "hide": 2, + "includeAll": false, + "label": null, "multi": false, "name": "cluster", "options": [ @@ -5297,7 +5777,7 @@ data: "query": "label_values(kube_pod_info, cluster)", "refresh": 1, "regex": "", - "sort": 2, + "sort": 1, "tagValuesQuery": "", "tags": [ @@ -5309,13 +5789,13 @@ data: { "allValue": null, "current": { - "text": "prod", - "value": "prod" + "text": "", + "value": "" }, "datasource": "$datasource", "hide": 0, "includeAll": false, - "label": "node", + "label": null, "multi": false, "name": "node", "options": [ @@ -5324,7 +5804,7 @@ data: "query": "label_values(kube_pod_info{cluster=\"$cluster\"}, node)", "refresh": 1, "regex": "", - "sort": 2, + "sort": 1, "tagValuesQuery": "", "tags": [ diff --git a/addons/grafana/dashboards-k8s-resources-2.yaml b/addons/grafana/dashboards-k8s-resources-2.yaml index 60dcfec49..2475dbac3 100644 --- a/addons/grafana/dashboards-k8s-resources-2.yaml +++ b/addons/grafana/dashboards-k8s-resources-2.yaml @@ -50,7 +50,24 @@ data: "points": false, "renderer": "flot", "seriesOverrides": [ - + { + "alias": "requests", + "color": "#F2495C", + "fill": 0, + "hideTooltip": true, + "legend": true, + "linewidth": 2, + "stack": false + }, + { + "alias": "limits", + "color": "#FF9830", + "fill": 0, + "hideTooltip": true, + "legend": true, + "linewidth": 2, + "stack": false + } ], "spaceLength": 10, "span": 12, @@ -64,6 +81,22 @@ data: "legendFormat": "{{container}}", "legendLink": null, "step": 10 + }, + { + "expr": "sum(\n kube_pod_container_resource_requests_cpu_cores{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"})\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "requests", + "legendLink": null, + "step": 10 + }, + { + "expr": "sum(\n kube_pod_container_resource_limits_cpu_cores{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"})\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "limits", + "legendLink": null, + "step": 10 } ], "thresholds": [ @@ -126,8 +159,113 @@ data: "dashLength": 10, "dashes": false, "datasource": "$datasource", - "fill": 1, + "fill": 10, "id": 2, + "legend": { + "avg": false, + "current": true, + "max": true, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(increase(container_cpu_cfs_throttled_periods_total{namespace=\"$namespace\", pod=\"$pod\", container!=\"POD\", cluster=\"$cluster\"}[5m])) by (container) /sum(increase(container_cpu_cfs_periods_total{namespace=\"$namespace\", pod=\"$pod\", container!=\"POD\", cluster=\"$cluster\"}[5m])) by (container)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{container}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + { + "colorMode": "critical", + "fill": true, + "line": true, + "op": "gt", + "value": 1, + "yaxis": "left" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU Throttling", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": 1, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "CPU Throttling", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 3, "legend": { "avg": false, "current": false, @@ -394,7 +532,7 @@ data: "dashes": false, "datasource": "$datasource", "fill": 10, - "id": 3, + "id": 4, "legend": { "avg": false, "current": false, @@ -415,7 +553,26 @@ data: "points": false, "renderer": "flot", "seriesOverrides": [ - + { + "alias": "requests", + "color": "#F2495C", + "dashes": true, + "fill": 0, + "hideTooltip": true, + "legend": false, + "linewidth": 2, + "stack": false + }, + { + "alias": "limits", + "color": "#FF9830", + "dashes": true, + "fill": 0, + "hideTooltip": true, + "legend": false, + "linewidth": 2, + "stack": false + } ], "spaceLength": 10, "span": 12, @@ -423,26 +580,26 @@ data: "steppedLine": false, "targets": [ { - "expr": "sum(container_memory_rss{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\", container!=\"POD\", container!=\"\"}) by (container)", + "expr": "sum(container_memory_working_set_bytes{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\", container!=\"POD\", container!=\"\"}) by (container)", "format": "time_series", "intervalFactor": 2, - "legendFormat": "{{container}} (RSS)", + "legendFormat": "{{container}}", "legendLink": null, "step": 10 }, { - "expr": "sum(container_memory_cache{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\", container!=\"POD\", container!=\"\"}) by (container)", + "expr": "sum(\n kube_pod_container_resource_requests_memory_bytes{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"})\n", "format": "time_series", "intervalFactor": 2, - "legendFormat": "{{container}} (Cache)", + "legendFormat": "requests", "legendLink": null, "step": 10 }, { - "expr": "sum(container_memory_swap{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\", container!=\"POD\", container!=\"\"}) by (container)", + "expr": "sum(\n kube_pod_container_resource_limits_memory_bytes{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"})\n", "format": "time_series", "intervalFactor": 2, - "legendFormat": "{{container}} (Swap)", + "legendFormat": "limits", "legendLink": null, "step": 10 } @@ -508,7 +665,7 @@ data: "dashes": false, "datasource": "$datasource", "fill": 1, - "id": 4, + "id": 5, "legend": { "avg": false, "current": false, @@ -856,7 +1013,7 @@ data: "dashes": false, "datasource": "$datasource", "fill": 10, - "id": 5, + "id": 6, "legend": { "avg": false, "current": false, @@ -954,7 +1111,7 @@ data: "dashes": false, "datasource": "$datasource", "fill": 10, - "id": 6, + "id": 7, "legend": { "avg": false, "current": false, @@ -1052,7 +1209,7 @@ data: "dashes": false, "datasource": "$datasource", "fill": 10, - "id": 7, + "id": 8, "legend": { "avg": false, "current": false, @@ -1150,7 +1307,7 @@ data: "dashes": false, "datasource": "$datasource", "fill": 10, - "id": 8, + "id": 9, "legend": { "avg": false, "current": false, @@ -1248,7 +1405,7 @@ data: "dashes": false, "datasource": "$datasource", "fill": 10, - "id": 9, + "id": 10, "legend": { "avg": false, "current": false, @@ -1346,7 +1503,7 @@ data: "dashes": false, "datasource": "$datasource", "fill": 10, - "id": 10, + "id": 11, "legend": { "avg": false, "current": false, @@ -1441,8 +1598,8 @@ data: "list": [ { "current": { - "text": "Prometheus", - "value": "Prometheus" + "text": "default", + "value": "default" }, "hide": 0, "label": null, @@ -1457,50 +1614,58 @@ data: }, { "allValue": null, + "auto": false, + "auto_count": 30, + "auto_min": "10s", "current": { - "text": "prod", - "value": "prod" + "text": "5m", + "value": "5m" }, "datasource": "$datasource", "hide": 2, "includeAll": false, - "label": "cluster", + "label": null, "multi": false, - "name": "cluster", + "name": "interval", "options": [ - + { + "selected": true, + "text": "4h", + "value": "4h" + } ], - "query": "label_values(kube_pod_info, cluster)", - "refresh": 1, + "query": "4h", + "refresh": 2, "regex": "", - "sort": 2, + "skipUrlSync": false, + "sort": 1, "tagValuesQuery": "", "tags": [ ], "tagsQuery": "", - "type": "query", + "type": "interval", "useTags": false }, { "allValue": null, "current": { - "text": "prod", - "value": "prod" + "text": "", + "value": "" }, "datasource": "$datasource", - "hide": 0, + "hide": 2, "includeAll": false, - "label": "namespace", + "label": null, "multi": false, - "name": "namespace", + "name": "cluster", "options": [ ], - "query": "label_values(kube_pod_info{cluster=\"$cluster\"}, namespace)", + "query": "label_values(kube_pod_info, cluster)", "refresh": 1, "regex": "", - "sort": 2, + "sort": 1, "tagValuesQuery": "", "tags": [ @@ -1512,22 +1677,22 @@ data: { "allValue": null, "current": { - "text": "prod", - "value": "prod" + "text": "", + "value": "" }, "datasource": "$datasource", "hide": 0, "includeAll": false, - "label": "pod", + "label": null, "multi": false, - "name": "pod", + "name": "namespace", "options": [ ], - "query": "label_values(kube_pod_info{cluster=\"$cluster\", namespace=\"$namespace\"}, pod)", + "query": "label_values(kube_pod_info{cluster=\"$cluster\"}, namespace)", "refresh": 1, "regex": "", - "sort": 2, + "sort": 1, "tagValuesQuery": "", "tags": [ @@ -1538,37 +1703,29 @@ data: }, { "allValue": null, - "auto": false, - "auto_count": 30, - "auto_min": "10s", "current": { - "text": "5m", - "value": "5m" + "text": "", + "value": "" }, - "datasource": "prometheus", - "hide": 2, + "datasource": "$datasource", + "hide": 0, "includeAll": false, "label": null, "multi": false, - "name": "interval", + "name": "pod", "options": [ - { - "selected": true, - "text": "4h", - "value": "4h" - } + ], - "query": "4h", + "query": "label_values(kube_pod_info{cluster=\"$cluster\", namespace=\"$namespace\"}, pod)", "refresh": 2, "regex": "", - "skipUrlSync": false, "sort": 1, "tagValuesQuery": "", "tags": [ ], "tagsQuery": "", - "type": "interval", + "type": "query", "useTags": false } ] @@ -3441,8 +3598,8 @@ data: "list": [ { "current": { - "text": "Prometheus", - "value": "Prometheus" + "text": "default", + "value": "default" }, "hide": 0, "label": null, @@ -3457,50 +3614,58 @@ data: }, { "allValue": null, + "auto": false, + "auto_count": 30, + "auto_min": "10s", "current": { - "text": "prod", - "value": "prod" + "text": "5m", + "value": "5m" }, "datasource": "$datasource", "hide": 2, "includeAll": false, - "label": "cluster", + "label": null, "multi": false, - "name": "cluster", + "name": "interval", "options": [ - + { + "selected": true, + "text": "4h", + "value": "4h" + } ], - "query": "label_values(kube_pod_info, cluster)", - "refresh": 1, + "query": "4h", + "refresh": 2, "regex": "", - "sort": 2, + "skipUrlSync": false, + "sort": 1, "tagValuesQuery": "", "tags": [ ], "tagsQuery": "", - "type": "query", + "type": "interval", "useTags": false }, { "allValue": null, "current": { - "text": "prod", - "value": "prod" + "text": "", + "value": "" }, "datasource": "$datasource", - "hide": 0, + "hide": 2, "includeAll": false, - "label": "namespace", + "label": null, "multi": false, - "name": "namespace", + "name": "cluster", "options": [ ], - "query": "label_values(kube_pod_info{cluster=\"$cluster\"}, namespace)", + "query": "label_values(kube_pod_info, cluster)", "refresh": 1, "regex": "", - "sort": 2, + "sort": 1, "tagValuesQuery": "", "tags": [ @@ -3512,22 +3677,22 @@ data: { "allValue": null, "current": { - "text": "prod", - "value": "prod" + "text": "", + "value": "" }, "datasource": "$datasource", "hide": 0, "includeAll": false, - "label": "workload", + "label": null, "multi": false, - "name": "workload", + "name": "namespace", "options": [ ], - "query": "label_values(mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\"}, workload)", + "query": "label_values(kube_pod_info{cluster=\"$cluster\"}, namespace)", "refresh": 1, "regex": "", - "sort": 2, + "sort": 1, "tagValuesQuery": "", "tags": [ @@ -3539,22 +3704,22 @@ data: { "allValue": null, "current": { - "text": "prod", - "value": "prod" + "text": "", + "value": "" }, "datasource": "$datasource", - "hide": 0, + "hide": 2, "includeAll": false, - "label": "type", + "label": null, "multi": false, - "name": "type", + "name": "workload", "options": [ ], - "query": "label_values(mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\"}, workload_type)", + "query": "label_values(mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\"}, workload)", "refresh": 1, "regex": "", - "sort": 2, + "sort": 1, "tagValuesQuery": "", "tags": [ @@ -3565,37 +3730,29 @@ data: }, { "allValue": null, - "auto": false, - "auto_count": 30, - "auto_min": "10s", "current": { - "text": "5m", - "value": "5m" + "text": "", + "value": "" }, - "datasource": "prometheus", - "hide": 2, + "datasource": "$datasource", + "hide": 0, "includeAll": false, "label": null, "multi": false, - "name": "interval", + "name": "type", "options": [ - { - "selected": true, - "text": "4h", - "value": "4h" - } + ], - "query": "4h", - "refresh": 2, + "query": "label_values(mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\"}, workload_type)", + "refresh": 1, "regex": "", - "skipUrlSync": false, "sort": 1, "tagValuesQuery": "", "tags": [ ], "tagsQuery": "", - "type": "interval", + "type": "query", "useTags": false } ] @@ -3684,7 +3841,26 @@ data: "points": false, "renderer": "flot", "seriesOverrides": [ - + { + "alias": "quota - requests", + "color": "#F2495C", + "dashes": true, + "fill": 0, + "hideTooltip": true, + "legend": false, + "linewidth": 2, + "stack": false + }, + { + "alias": "quota - limits", + "color": "#FF9830", + "dashes": true, + "fill": 0, + "hideTooltip": true, + "legend": false, + "linewidth": 2, + "stack": false + } ], "spaceLength": 10, "span": 12, @@ -3698,6 +3874,22 @@ data: "legendFormat": "{{workload}} - {{workload_type}}", "legendLink": null, "step": 10 + }, + { + "expr": "scalar(kube_resourcequota{cluster=\"$cluster\", namespace=\"$namespace\", type=\"hard\",resource=\"requests.cpu\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "quota - requests", + "legendLink": null, + "step": 10 + }, + { + "expr": "scalar(kube_resourcequota{cluster=\"$cluster\", namespace=\"$namespace\", type=\"hard\",resource=\"limits.cpu\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "quota - limits", + "legendLink": null, + "step": 10 } ], "thresholds": [ @@ -4094,7 +4286,26 @@ data: "points": false, "renderer": "flot", "seriesOverrides": [ - + { + "alias": "quota - requests", + "color": "#F2495C", + "dashes": true, + "fill": 0, + "hideTooltip": true, + "legend": false, + "linewidth": 2, + "stack": false + }, + { + "alias": "quota - limits", + "color": "#FF9830", + "dashes": true, + "fill": 0, + "hideTooltip": true, + "legend": false, + "linewidth": 2, + "stack": false + } ], "spaceLength": 10, "span": 12, @@ -4108,6 +4319,22 @@ data: "legendFormat": "{{workload}} - {{workload_type}}", "legendLink": null, "step": 10 + }, + { + "expr": "scalar(kube_resourcequota{cluster=\"$cluster\", namespace=\"$namespace\", type=\"hard\",resource=\"requests.memory\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "quota - requests", + "legendLink": null, + "step": 10 + }, + { + "expr": "scalar(kube_resourcequota{cluster=\"$cluster\", namespace=\"$namespace\", type=\"hard\",resource=\"limits.memory\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "quota - limits", + "legendLink": null, + "step": 10 } ], "thresholds": [ @@ -5576,8 +5803,8 @@ data: "list": [ { "current": { - "text": "Prometheus", - "value": "Prometheus" + "text": "default", + "value": "default" }, "hide": 0, "label": null, @@ -5592,50 +5819,63 @@ data: }, { "allValue": null, + "auto": false, + "auto_count": 30, + "auto_min": "10s", "current": { - "text": "prod", - "value": "prod" + "text": "5m", + "value": "5m" }, "datasource": "$datasource", "hide": 2, "includeAll": false, - "label": "cluster", + "label": null, "multi": false, - "name": "cluster", + "name": "interval", "options": [ - + { + "selected": true, + "text": "4h", + "value": "4h" + } ], - "query": "label_values(kube_pod_info, cluster)", - "refresh": 1, + "query": "4h", + "refresh": 2, "regex": "", - "sort": 2, + "skipUrlSync": false, + "sort": 1, "tagValuesQuery": "", "tags": [ ], "tagsQuery": "", - "type": "query", + "type": "interval", "useTags": false }, { "allValue": null, + "auto": false, + "auto_count": 30, + "auto_min": "10s", "current": { - "text": "prod", - "value": "prod" + "text": "deployment", + "value": "deployment" }, "datasource": "$datasource", + "definition": "label_values(mixin_pod_workload{namespace=~\"$namespace\", workload=~\".+\"}, workload_type)", "hide": 0, "includeAll": false, - "label": "namespace", + "label": null, "multi": false, - "name": "namespace", + "name": "type", "options": [ ], - "query": "label_values(kube_pod_info{cluster=\"$cluster\"}, namespace)", + "query": "label_values(mixin_pod_workload{namespace=~\"$namespace\", workload=~\".+\"}, workload_type)", "refresh": 1, "regex": "", - "sort": 2, + "skipUrlSync": false, + "sort": 0, "tagValuesQuery": "", "tags": [ @@ -5646,63 +5886,50 @@ data: }, { "allValue": null, - "auto": false, - "auto_count": 30, - "auto_min": "10s", "current": { - "text": "5m", - "value": "5m" + "text": "", + "value": "" }, - "datasource": "prometheus", + "datasource": "$datasource", "hide": 2, "includeAll": false, "label": null, "multi": false, - "name": "interval", + "name": "cluster", "options": [ - { - "selected": true, - "text": "4h", - "value": "4h" - } + ], - "query": "4h", - "refresh": 2, + "query": "label_values(kube_pod_info, cluster)", + "refresh": 1, "regex": "", - "skipUrlSync": false, "sort": 1, "tagValuesQuery": "", "tags": [ ], "tagsQuery": "", - "type": "interval", + "type": "query", "useTags": false }, { "allValue": null, - "auto": false, - "auto_count": 30, - "auto_min": "10s", "current": { - "text": "deployment", - "value": "deployment" + "text": "", + "value": "" }, "datasource": "$datasource", - "definition": "label_values(mixin_pod_workload{namespace=~\"$namespace\", workload=~\".+\"}, workload_type)", "hide": 0, "includeAll": false, "label": null, "multi": false, - "name": "type", + "name": "namespace", "options": [ ], - "query": "label_values(mixin_pod_workload{namespace=~\"$namespace\", workload=~\".+\"}, workload_type)", + "query": "label_values(kube_pod_info{cluster=\"$cluster\"}, namespace)", "refresh": 1, "regex": "", - "skipUrlSync": false, - "sort": 0, + "sort": 1, "tagValuesQuery": "", "tags": [ diff --git a/addons/grafana/dashboards-k8s.yaml b/addons/grafana/dashboards-k8s.yaml index 56a2e5bca..744fa39f9 100644 --- a/addons/grafana/dashboards-k8s.yaml +++ b/addons/grafana/dashboards-k8s.yaml @@ -21,7 +21,7 @@ data: "links": [ ], - "refresh": "", + "refresh": "10s", "rows": [ { "collapse": false, @@ -88,7 +88,7 @@ data: "tableColumn": "", "targets": [ { - "expr": "sum(up{job=\"apiserver\"})", + "expr": "sum(up{job=\"apiserver\", cluster=\"$cluster\"})", "format": "time_series", "intervalFactor": 2, "legendFormat": "", @@ -155,28 +155,28 @@ data: "steppedLine": false, "targets": [ { - "expr": "sum(rate(apiserver_request_total{job=\"apiserver\", instance=~\"$instance\",code=~\"2..\"}[5m]))", + "expr": "sum(rate(apiserver_request_total{job=\"apiserver\", instance=~\"$instance\",code=~\"2..\", cluster=\"$cluster\"}[5m]))", "format": "time_series", "intervalFactor": 2, "legendFormat": "2xx", "refId": "A" }, { - "expr": "sum(rate(apiserver_request_total{job=\"apiserver\", instance=~\"$instance\",code=~\"3..\"}[5m]))", + "expr": "sum(rate(apiserver_request_total{job=\"apiserver\", instance=~\"$instance\",code=~\"3..\", cluster=\"$cluster\"}[5m]))", "format": "time_series", "intervalFactor": 2, "legendFormat": "3xx", "refId": "B" }, { - "expr": "sum(rate(apiserver_request_total{job=\"apiserver\", instance=~\"$instance\",code=~\"4..\"}[5m]))", + "expr": "sum(rate(apiserver_request_total{job=\"apiserver\", instance=~\"$instance\",code=~\"4..\", cluster=\"$cluster\"}[5m]))", "format": "time_series", "intervalFactor": 2, "legendFormat": "4xx", "refId": "C" }, { - "expr": "sum(rate(apiserver_request_total{job=\"apiserver\", instance=~\"$instance\",code=~\"5..\"}[5m]))", + "expr": "sum(rate(apiserver_request_total{job=\"apiserver\", instance=~\"$instance\",code=~\"5..\", cluster=\"$cluster\"}[5m]))", "format": "time_series", "intervalFactor": 2, "legendFormat": "5xx", @@ -237,15 +237,15 @@ data: }, "id": 4, "legend": { - "alignAsTable": "true", + "alignAsTable": true, "avg": false, - "current": "true", + "current": true, "max": false, "min": false, - "rightSide": "true", - "show": "true", + "rightSide": true, + "show": true, "total": false, - "values": "true" + "values": true }, "lines": true, "linewidth": 1, @@ -267,7 +267,7 @@ data: "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.99, sum(rate(apiserver_request_duration_seconds_bucket{job=\"apiserver\", instance=~\"$instance\"}[5m])) by (verb, le))", + "expr": "histogram_quantile(0.99, sum(rate(apiserver_request_duration_seconds_bucket{job=\"apiserver\", instance=~\"$instance\", verb!=\"WATCH\", cluster=\"$cluster\"}[5m])) by (verb, le))", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{verb}}", @@ -371,7 +371,7 @@ data: "steppedLine": false, "targets": [ { - "expr": "sum(rate(workqueue_adds_total{job=\"apiserver\", instance=~\"$instance\"}[5m])) by (instance, name)", + "expr": "sum(rate(workqueue_adds_total{job=\"apiserver\", instance=~\"$instance\", cluster=\"$cluster\"}[5m])) by (instance, name)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{instance}} {{name}}", @@ -462,7 +462,7 @@ data: "steppedLine": false, "targets": [ { - "expr": "sum(rate(workqueue_depth{job=\"apiserver\", instance=~\"$instance\"}[5m])) by (instance, name)", + "expr": "sum(rate(workqueue_depth{job=\"apiserver\", instance=~\"$instance\", cluster=\"$cluster\"}[5m])) by (instance, name)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{instance}} {{name}}", @@ -523,15 +523,15 @@ data: }, "id": 7, "legend": { - "alignAsTable": "true", + "alignAsTable": true, "avg": false, - "current": "true", + "current": true, "max": false, "min": false, - "rightSide": "true", - "show": "true", + "rightSide": true, + "show": true, "total": false, - "values": "true" + "values": true }, "lines": true, "linewidth": 1, @@ -553,7 +553,7 @@ data: "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.99, sum(rate(workqueue_queue_duration_seconds_bucket{job=\"apiserver\", instance=~\"$instance\"}[5m])) by (instance, name, le))", + "expr": "histogram_quantile(0.99, sum(rate(workqueue_queue_duration_seconds_bucket{job=\"apiserver\", instance=~\"$instance\", cluster=\"$cluster\"}[5m])) by (instance, name, le))", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{instance}} {{name}}", @@ -657,7 +657,7 @@ data: "steppedLine": false, "targets": [ { - "expr": "etcd_helper_cache_entry_total{job=\"apiserver\", instance=~\"$instance\"}", + "expr": "etcd_helper_cache_entry_total{job=\"apiserver\", instance=~\"$instance\", cluster=\"$cluster\"}", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{instance}}", @@ -748,14 +748,14 @@ data: "steppedLine": false, "targets": [ { - "expr": "sum(rate(etcd_helper_cache_hit_total{job=\"apiserver\",instance=~\"$instance\"}[5m])) by (intance)", + "expr": "sum(rate(etcd_helper_cache_hit_total{job=\"apiserver\",instance=~\"$instance\", cluster=\"$cluster\"}[5m])) by (instance)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{instance}} hit", "refId": "A" }, { - "expr": "sum(rate(etcd_helper_cache_miss_total{job=\"apiserver\",instance=~\"$instance\"}[5m])) by (instance)", + "expr": "sum(rate(etcd_helper_cache_miss_total{job=\"apiserver\",instance=~\"$instance\", cluster=\"$cluster\"}[5m])) by (instance)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{instance}} miss", @@ -846,14 +846,14 @@ data: "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.99,sum(rate(etcd_request_cache_get_duration_seconds_bucket{job=\"apiserver\",instance=~\"$instance\"}[5m])) by (instance, le))", + "expr": "histogram_quantile(0.99,sum(rate(etcd_request_cache_get_duration_seconds_bucket{job=\"apiserver\",instance=~\"$instance\", cluster=\"$cluster\"}[5m])) by (instance, le))", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{instance}} get", "refId": "A" }, { - "expr": "histogram_quantile(0.99,sum(rate(etcd_request_cache_add_duration_seconds_bucket{job=\"apiserver\",instance=~\"$instance\"}[5m])) by (instance, le))", + "expr": "histogram_quantile(0.99,sum(rate(etcd_request_cache_add_duration_seconds_bucket{job=\"apiserver\",instance=~\"$instance\", cluster=\"$cluster\"}[5m])) by (instance, le))", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{instance}} miss", @@ -957,7 +957,7 @@ data: "steppedLine": false, "targets": [ { - "expr": "process_resident_memory_bytes{job=\"apiserver\",instance=~\"$instance\"}", + "expr": "process_resident_memory_bytes{job=\"apiserver\",instance=~\"$instance\", cluster=\"$cluster\"}", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{instance}}", @@ -1048,7 +1048,7 @@ data: "steppedLine": false, "targets": [ { - "expr": "rate(process_cpu_seconds_total{job=\"apiserver\",instance=~\"$instance\"}[5m])", + "expr": "rate(process_cpu_seconds_total{job=\"apiserver\",instance=~\"$instance\", cluster=\"$cluster\"}[5m])", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{instance}}", @@ -1139,7 +1139,7 @@ data: "steppedLine": false, "targets": [ { - "expr": "go_goroutines{job=\"apiserver\",instance=~\"$instance\"}", + "expr": "go_goroutines{job=\"apiserver\",instance=~\"$instance\", cluster=\"$cluster\"}", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{instance}}", @@ -1205,8 +1205,8 @@ data: "list": [ { "current": { - "text": "Prometheus", - "value": "Prometheus" + "text": "default", + "value": "default" }, "hide": 0, "label": null, @@ -1219,6 +1219,33 @@ data: "regex": "", "type": "datasource" }, + { + "allValue": null, + "current": { + "text": "prod", + "value": "prod" + }, + "datasource": "$datasource", + "hide": 2, + "includeAll": false, + "label": null, + "multi": false, + "name": "cluster", + "options": [ + + ], + "query": "label_values(apiserver_request_total, cluster)", + "refresh": 1, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, { "allValue": null, "current": { @@ -1233,7 +1260,7 @@ data: "options": [ ], - "query": "label_values(apiserver_request_total{job=\"apiserver\"}, instance)", + "query": "label_values(apiserver_request_total{job=\"apiserver\", cluster=\"$cluster\"}, instance)", "refresh": 2, "regex": "", "sort": 1, @@ -1302,7 +1329,7 @@ data: "links": [ ], - "refresh": "", + "refresh": "10s", "rows": [ { "collapse": false, @@ -1406,15 +1433,15 @@ data: }, "id": 3, "legend": { - "alignAsTable": "true", + "alignAsTable": true, "avg": false, - "current": "true", + "current": true, "max": false, "min": false, - "rightSide": "true", - "show": "true", + "rightSide": true, + "show": true, "total": false, - "values": "true" + "values": true }, "lines": true, "linewidth": 1, @@ -1510,15 +1537,15 @@ data: }, "id": 4, "legend": { - "alignAsTable": "true", + "alignAsTable": true, "avg": false, - "current": "true", + "current": true, "max": false, "min": false, - "rightSide": "true", - "show": "true", + "rightSide": true, + "show": true, "total": false, - "values": "true" + "values": true }, "lines": true, "linewidth": 1, @@ -1614,15 +1641,15 @@ data: }, "id": 5, "legend": { - "alignAsTable": "true", + "alignAsTable": true, "avg": false, - "current": "true", + "current": true, "max": false, "min": false, - "rightSide": "true", - "show": "true", + "rightSide": true, + "show": true, "total": false, - "values": "true" + "values": true }, "lines": true, "linewidth": 1, @@ -1934,15 +1961,15 @@ data: }, "id": 8, "legend": { - "alignAsTable": "true", + "alignAsTable": true, "avg": false, - "current": "true", + "current": true, "max": false, "min": false, - "rightSide": "true", - "show": "true", + "rightSide": true, + "show": true, "total": false, - "values": "true" + "values": true }, "lines": true, "linewidth": 1, @@ -2316,8 +2343,8 @@ data: "list": [ { "current": { - "text": "Prometheus", - "value": "Prometheus" + "text": "default", + "value": "default" }, "hide": 0, "label": null, @@ -2413,7 +2440,7 @@ data: "links": [ ], - "refresh": "", + "refresh": "10s", "rows": [ { "collapse": false, @@ -2815,8 +2842,8 @@ data: "list": [ { "current": { - "text": "Prometheus", - "value": "Prometheus" + "text": "default", + "value": "default" }, "hide": 0, "label": null, @@ -2943,7 +2970,7 @@ data: "uid": "919b92a8e8041bd567af9edab12c840c", "version": 0 } - pods.json: |- + scheduler.json: |- { "__inputs": [ @@ -2953,20 +2980,7 @@ data: ], "annotations": { "list": [ - { - "builtIn": 1, - "datasource": "$datasource", - "enable": true, - "expr": "time() == BOOL timestamp(rate(kube_pod_container_status_restarts_total{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}[2m]) > 0)", - "hide": false, - "iconColor": "rgba(215, 44, 44, 1)", - "name": "Restarts", - "showIn": 0, - "tags": [ - "restart" - ], - "type": "rows" - } + ] }, "editable": false, @@ -2977,12 +2991,96 @@ data: "links": [ ], - "refresh": "", + "refresh": "10s", "rows": [ { "collapse": false, "collapsed": false, "panels": [ + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "$datasource", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + + }, + "id": 2, + "interval": null, + "links": [ + + ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "span": 2, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum(up{job=\"kube-scheduler\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": "", + "title": "Up", + "tooltip": { + "shared": false + }, + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "min" + }, { "aliasColors": { @@ -2995,17 +3093,17 @@ data: "gridPos": { }, - "id": 2, + "id": 3, "legend": { "alignAsTable": true, - "avg": true, + "avg": false, "current": true, "max": false, "min": false, "rightSide": true, "show": true, "total": false, - "values": false + "values": true }, "lines": true, "linewidth": 1, @@ -3022,36 +3120,36 @@ data: ], "spaceLength": 10, - "span": 12, + "span": 5, "stack": false, "steppedLine": false, "targets": [ { - "expr": "sum by(container) (container_memory_usage_bytes{job=\"kubernetes-cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\", container=~\"$container\", container!=\"POD\"})", + "expr": "sum(rate(scheduler_e2e_scheduling_duration_seconds_count{job=\"kube-scheduler\", instance=~\"$instance\"}[5m])) by (instance)", "format": "time_series", "intervalFactor": 2, - "legendFormat": "Current: {{ container }}", + "legendFormat": "{{instance}} e2e", "refId": "A" }, { - "expr": "sum by(container) (kube_pod_container_resource_requests{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"memory\", pod=\"$pod\", container=~\"$container\"})", + "expr": "sum(rate(scheduler_binding_duration_seconds_count{job=\"kube-scheduler\", instance=~\"$instance\"}[5m])) by (instance)", "format": "time_series", "intervalFactor": 2, - "legendFormat": "Requested: {{ container }}", + "legendFormat": "{{instance}} binding", "refId": "B" }, { - "expr": "sum by(container) (kube_pod_container_resource_limits{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"memory\", pod=\"$pod\", container=~\"$container\"})", + "expr": "sum(rate(scheduler_scheduling_algorithm_duration_seconds_count{job=\"kube-scheduler\", instance=~\"$instance\"}[5m])) by (instance)", "format": "time_series", "intervalFactor": 2, - "legendFormat": "Limit: {{ container }}", + "legendFormat": "{{instance}} scheduling algorithm", "refId": "C" }, { - "expr": "sum by(container) (container_memory_cache{job=\"kubernetes-cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"$pod\", container=~\"$container\", container!=\"POD\"})", + "expr": "sum(rate(scheduler_volume_scheduling_duration_seconds_count{job=\"kube-scheduler\", instance=~\"$instance\"}[5m])) by (instance)", "format": "time_series", "intervalFactor": 2, - "legendFormat": "Cache: {{ container }}", + "legendFormat": "{{instance}} volume", "refId": "D" } ], @@ -3060,7 +3158,7 @@ data: ], "timeFrom": null, "timeShift": null, - "title": "Memory Usage", + "title": "Scheduling Rate", "tooltip": { "shared": false, "sort": 0, @@ -3078,7 +3176,7 @@ data: }, "yaxes": [ { - "format": "bytes", + "format": "ops", "label": null, "logBase": 1, "max": null, @@ -3086,7 +3184,7 @@ data: "show": true }, { - "format": "bytes", + "format": "ops", "label": null, "logBase": 1, "max": null, @@ -3094,20 +3192,7 @@ data: "show": true } ] - } - ], - "repeat": null, - "repeatIteration": null, - "repeatRowId": null, - "showTitle": false, - "title": "Dashboard Row", - "titleSize": "h6", - "type": "row" - }, - { - "collapse": false, - "collapsed": false, - "panels": [ + }, { "aliasColors": { @@ -3120,17 +3205,17 @@ data: "gridPos": { }, - "id": 3, + "id": 4, "legend": { "alignAsTable": true, - "avg": true, + "avg": false, "current": true, "max": false, "min": false, "rightSide": true, "show": true, "total": false, - "values": false + "values": true }, "lines": true, "linewidth": 1, @@ -3147,30 +3232,37 @@ data: ], "spaceLength": 10, - "span": 12, + "span": 5, "stack": false, "steppedLine": false, "targets": [ { - "expr": "sum by (container) (irate(container_cpu_usage_seconds_total{job=\"kubernetes-cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\", image!=\"\", pod=\"$pod\", container=~\"$container\", container!=\"POD\"}[4m]))", + "expr": "histogram_quantile(0.99, sum(rate(scheduler_e2e_scheduling_duration_seconds_bucket{job=\"kube-scheduler\",instance=~\"$instance\"}[5m])) by (instance, le))", "format": "time_series", "intervalFactor": 2, - "legendFormat": "Current: {{ container }}", + "legendFormat": "{{instance}} e2e", "refId": "A" }, { - "expr": "sum by(container) (kube_pod_container_resource_requests{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"cpu\", pod=\"$pod\", container=~\"$container\"})", + "expr": "histogram_quantile(0.99, sum(rate(scheduler_binding_duration_seconds_bucket{job=\"kube-scheduler\",instance=~\"$instance\"}[5m])) by (instance, le))", "format": "time_series", "intervalFactor": 2, - "legendFormat": "Requested: {{ container }}", + "legendFormat": "{{instance}} binding", "refId": "B" }, { - "expr": "sum by(container) (kube_pod_container_resource_limits{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"cpu\", pod=\"$pod\", container=~\"$container\"})", + "expr": "histogram_quantile(0.99, sum(rate(scheduler_scheduling_algorithm_duration_seconds_bucket{job=\"kube-scheduler\",instance=~\"$instance\"}[5m])) by (instance, le))", "format": "time_series", "intervalFactor": 2, - "legendFormat": "Limit: {{ container }}", + "legendFormat": "{{instance}} scheduling algorithm", "refId": "C" + }, + { + "expr": "histogram_quantile(0.99, sum(rate(scheduler_volume_scheduling_duration_seconds_bucket{job=\"kube-scheduler\",instance=~\"$instance\"}[5m])) by (instance, le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{instance}} volume", + "refId": "D" } ], "thresholds": [ @@ -3178,7 +3270,7 @@ data: ], "timeFrom": null, "timeShift": null, - "title": "CPU Usage", + "title": "Scheduling latency 99th Quantile", "tooltip": { "shared": false, "sort": 0, @@ -3196,7 +3288,7 @@ data: }, "yaxes": [ { - "format": "short", + "format": "s", "label": null, "logBase": 1, "max": null, @@ -3204,7 +3296,7 @@ data: "show": true }, { - "format": "short", + "format": "s", "label": null, "logBase": 1, "max": null, @@ -3238,14 +3330,14 @@ data: "gridPos": { }, - "id": 4, + "id": 5, "legend": { - "alignAsTable": true, - "avg": true, - "current": true, + "alignAsTable": false, + "avg": false, + "current": false, "max": false, "min": false, - "rightSide": true, + "rightSide": false, "show": true, "total": false, "values": false @@ -3265,23 +3357,37 @@ data: ], "spaceLength": 10, - "span": 12, + "span": 4, "stack": false, "steppedLine": false, "targets": [ { - "expr": "sort_desc(sum by (pod) (irate(container_network_receive_bytes_total{job=\"kubernetes-cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}[4m])))", + "expr": "sum(rate(rest_client_requests_total{job=\"kube-scheduler\", instance=~\"$instance\",code=~\"2..\"}[5m]))", "format": "time_series", "intervalFactor": 2, - "legendFormat": "RX: {{ pod }}", + "legendFormat": "2xx", "refId": "A" }, { - "expr": "sort_desc(sum by (pod) (irate(container_network_transmit_bytes_total{job=\"kubernetes-cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}[4m])))", + "expr": "sum(rate(rest_client_requests_total{job=\"kube-scheduler\", instance=~\"$instance\",code=~\"3..\"}[5m]))", "format": "time_series", "intervalFactor": 2, - "legendFormat": "TX: {{ pod }}", + "legendFormat": "3xx", "refId": "B" + }, + { + "expr": "sum(rate(rest_client_requests_total{job=\"kube-scheduler\", instance=~\"$instance\",code=~\"4..\"}[5m]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "4xx", + "refId": "C" + }, + { + "expr": "sum(rate(rest_client_requests_total{job=\"kube-scheduler\", instance=~\"$instance\",code=~\"5..\"}[5m]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "5xx", + "refId": "D" } ], "thresholds": [ @@ -3289,7 +3395,7 @@ data: ], "timeFrom": null, "timeShift": null, - "title": "Network I/O", + "title": "Kube API Request Rate", "tooltip": { "shared": false, "sort": 0, @@ -3307,7 +3413,7 @@ data: }, "yaxes": [ { - "format": "bytes", + "format": "ops", "label": null, "logBase": 1, "max": null, @@ -3315,7 +3421,7 @@ data: "show": true }, { - "format": "bytes", + "format": "ops", "label": null, "logBase": 1, "max": null, @@ -3323,20 +3429,7 @@ data: "show": true } ] - } - ], - "repeat": null, - "repeatIteration": null, - "repeatRowId": null, - "showTitle": false, - "title": "Dashboard Row", - "titleSize": "h6", - "type": "row" - }, - { - "collapse": false, - "collapsed": false, - "panels": [ + }, { "aliasColors": { @@ -3349,14 +3442,14 @@ data: "gridPos": { }, - "id": 5, + "id": 6, "legend": { - "alignAsTable": true, - "avg": true, - "current": true, + "alignAsTable": false, + "avg": false, + "current": false, "max": false, "min": false, - "rightSide": true, + "rightSide": false, "show": true, "total": false, "values": false @@ -3376,15 +3469,15 @@ data: ], "spaceLength": 10, - "span": 12, + "span": 8, "stack": false, "steppedLine": false, "targets": [ { - "expr": "max by (container) (kube_pod_container_status_restarts_total{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\", container=~\"$container\"})", + "expr": "histogram_quantile(0.99, sum(rate(rest_client_request_latency_seconds_bucket{job=\"kube-scheduler\", instance=~\"$instance\", verb=\"POST\"}[5m])) by (verb, url, le))", "format": "time_series", "intervalFactor": 2, - "legendFormat": "Restarts: {{ container }}", + "legendFormat": "{{verb}} {{url}}", "refId": "A" } ], @@ -3393,7 +3486,7 @@ data: ], "timeFrom": null, "timeShift": null, - "title": "Total Restarts Per Container", + "title": "Post Request Latency 99th Quantile", "tooltip": { "shared": false, "sort": 0, @@ -3411,7 +3504,7 @@ data: }, "yaxes": [ { - "format": "short", + "format": "s", "label": null, "logBase": 1, "max": null, @@ -3419,7 +3512,7 @@ data: "show": true }, { - "format": "short", + "format": "s", "label": null, "logBase": 1, "max": null, @@ -3436,464 +3529,60 @@ data: "title": "Dashboard Row", "titleSize": "h6", "type": "row" - } - ], - "schemaVersion": 14, - "style": "dark", - "tags": [ - "kubernetes-mixin" - ], - "templating": { - "list": [ - { - "current": { - "text": "Prometheus", - "value": "Prometheus" - }, - "hide": 0, - "label": null, - "name": "datasource", - "options": [ - - ], - "query": "prometheus", - "refresh": 1, - "regex": "", - "type": "datasource" - }, - { - "allValue": null, - "current": { - - }, - "datasource": "$datasource", - "hide": 2, - "includeAll": false, - "label": "cluster", - "multi": false, - "name": "cluster", - "options": [ - - ], - "query": "label_values(kube_pod_info, cluster)", - "refresh": 2, - "regex": "", - "sort": 1, - "tagValuesQuery": "", - "tags": [ - - ], - "tagsQuery": "", - "type": "query", - "useTags": false - }, - { - "allValue": null, - "current": { - - }, - "datasource": "$datasource", - "hide": 0, - "includeAll": false, - "label": "Namespace", - "multi": false, - "name": "namespace", - "options": [ - - ], - "query": "label_values(kube_pod_info{cluster=\"$cluster\"}, namespace)", - "refresh": 2, - "regex": "", - "sort": 1, - "tagValuesQuery": "", - "tags": [ - - ], - "tagsQuery": "", - "type": "query", - "useTags": false - }, - { - "allValue": null, - "current": { - - }, - "datasource": "$datasource", - "hide": 0, - "includeAll": false, - "label": "Pod", - "multi": false, - "name": "pod", - "options": [ - - ], - "query": "label_values(kube_pod_info{cluster=\"$cluster\", namespace=~\"$namespace\"}, pod)", - "refresh": 2, - "regex": "", - "sort": 1, - "tagValuesQuery": "", - "tags": [ - - ], - "tagsQuery": "", - "type": "query", - "useTags": false - }, - { - "allValue": null, - "current": { - - }, - "datasource": "$datasource", - "hide": 0, - "includeAll": true, - "label": "Container", - "multi": false, - "name": "container", - "options": [ - - ], - "query": "label_values(kube_pod_container_info{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}, container)", - "refresh": 2, - "regex": "", - "sort": 1, - "tagValuesQuery": "", - "tags": [ - - ], - "tagsQuery": "", - "type": "query", - "useTags": false - } - ] - }, - "time": { - "from": "now-1h", - "to": "now" - }, - "timepicker": { - "refresh_intervals": [ - "5s", - "10s", - "30s", - "1m", - "5m", - "15m", - "30m", - "1h", - "2h", - "1d" - ], - "time_options": [ - "5m", - "15m", - "1h", - "6h", - "12h", - "24h", - "2d", - "7d", - "30d" - ] - }, - "timezone": "", - "title": "Kubernetes / Pods", - "uid": "ab4f13a9892a76a4d21ce8c2445bf4ea", - "version": 0 - } - scheduler.json: |- - { - "__inputs": [ - - ], - "__requires": [ - - ], - "annotations": { - "list": [ - - ] - }, - "editable": false, - "gnetId": null, - "graphTooltip": 0, - "hideControls": false, - "id": null, - "links": [ - - ], - "refresh": "", - "rows": [ + }, { "collapse": false, "collapsed": false, "panels": [ { - "cacheTimeout": null, - "colorBackground": false, - "colorValue": false, - "colors": [ - "#299c46", - "rgba(237, 129, 40, 0.89)", - "#d44a3a" - ], - "datasource": "$datasource", - "format": "none", - "gauge": { - "maxValue": 100, - "minValue": 0, - "show": false, - "thresholdLabels": false, - "thresholdMarkers": true + "aliasColors": { + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, "gridPos": { }, - "id": 2, - "interval": null, + "id": 7, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, "links": [ ], - "mappingType": 1, - "mappingTypes": [ - { - "name": "value to text", - "value": 1 - }, - { - "name": "range to text", - "value": 2 - } - ], - "maxDataPoints": 100, - "nullPointMode": "connected", - "nullText": null, - "postfix": "", - "postfixFontSize": "50%", - "prefix": "", - "prefixFontSize": "50%", - "rangeMaps": [ - { - "from": "null", - "text": "N/A", - "to": "null" - } - ], - "span": 2, - "sparkline": { - "fillColor": "rgba(31, 118, 189, 0.18)", - "full": false, - "lineColor": "rgb(31, 120, 193)", - "show": false - }, - "tableColumn": "", - "targets": [ - { - "expr": "sum(up{job=\"kube-scheduler\"})", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "", - "refId": "A" - } - ], - "thresholds": "", - "title": "Up", - "tooltip": { - "shared": false - }, - "type": "singlestat", - "valueFontSize": "80%", - "valueMaps": [ - { - "op": "=", - "text": "N/A", - "value": "null" - } - ], - "valueName": "min" - }, - { - "aliasColors": { - - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 1, - "gridPos": { - - }, - "id": 3, - "legend": { - "alignAsTable": "true", - "avg": false, - "current": "true", - "max": false, - "min": false, - "rightSide": "true", - "show": "true", - "total": false, - "values": "true" - }, - "lines": true, - "linewidth": 1, - "links": [ - - ], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "repeat": null, - "seriesOverrides": [ - - ], - "spaceLength": 10, - "span": 5, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum(rate(scheduler_e2e_scheduling_duration_seconds_count{job=\"kube-scheduler\", instance=~\"$instance\"}[5m])) by (instance)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{instance}} e2e", - "refId": "A" - }, - { - "expr": "sum(rate(scheduler_binding_duration_seconds_count{job=\"kube-scheduler\", instance=~\"$instance\"}[5m])) by (instance)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{instance}} binding", - "refId": "B" - }, - { - "expr": "sum(rate(scheduler_scheduling_algorithm_duration_seconds_count{job=\"kube-scheduler\", instance=~\"$instance\"}[5m])) by (instance)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{instance}} scheduling algorithm", - "refId": "C" - }, - { - "expr": "sum(rate(scheduler_volume_scheduling_duration_seconds_count{job=\"kube-scheduler\", instance=~\"$instance\"}[5m])) by (instance)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{instance}} volume", - "refId": "D" - } - ], - "thresholds": [ - - ], - "timeFrom": null, - "timeShift": null, - "title": "Scheduling Rate", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ - - ] - }, - "yaxes": [ - { - "format": "ops", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "ops", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - } - ] - }, - { - "aliasColors": { - - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 1, - "gridPos": { - - }, - "id": 4, - "legend": { - "alignAsTable": "true", - "avg": false, - "current": "true", - "max": false, - "min": false, - "rightSide": "true", - "show": "true", - "total": false, - "values": "true" - }, - "lines": true, - "linewidth": 1, - "links": [ - - ], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "repeat": null, - "seriesOverrides": [ - + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + ], "spaceLength": 10, - "span": 5, + "span": 12, "stack": false, "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.99, sum(rate(scheduler_e2e_scheduling_duration_seconds_bucket{job=\"kube-scheduler\",instance=~\"$instance\"}[5m])) by (instance, le))", + "expr": "histogram_quantile(0.99, sum(rate(rest_client_request_latency_seconds_bucket{job=\"kube-scheduler\", instance=~\"$instance\", verb=\"GET\"}[5m])) by (verb, url, le))", "format": "time_series", "intervalFactor": 2, - "legendFormat": "{{instance}} e2e", + "legendFormat": "{{verb}} {{url}}", "refId": "A" - }, - { - "expr": "histogram_quantile(0.99, sum(rate(scheduler_binding_duration_seconds_bucket{job=\"kube-scheduler\",instance=~\"$instance\"}[5m])) by (instance, le))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{instance}} binding", - "refId": "B" - }, - { - "expr": "histogram_quantile(0.99, sum(rate(scheduler_scheduling_algorithm_duration_seconds_bucket{job=\"kube-scheduler\",instance=~\"$instance\"}[5m])) by (instance, le))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{instance}} scheduling algorithm", - "refId": "C" - }, - { - "expr": "histogram_quantile(0.99, sum(rate(scheduler_volume_scheduling_duration_seconds_bucket{job=\"kube-scheduler\",instance=~\"$instance\"}[5m])) by (instance, le))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{instance}} volume", - "refId": "D" } ], "thresholds": [ @@ -3901,7 +3590,7 @@ data: ], "timeFrom": null, "timeShift": null, - "title": "Scheduling latency 99th Quantile", + "title": "Get Request Latency 99th Quantile", "tooltip": { "shared": false, "sort": 0, @@ -3925,1343 +3614,16 @@ data: "max": null, "min": 0, "show": true - }, - { - "format": "s", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - } - ] - } - ], - "repeat": null, - "repeatIteration": null, - "repeatRowId": null, - "showTitle": false, - "title": "Dashboard Row", - "titleSize": "h6", - "type": "row" - }, - { - "collapse": false, - "collapsed": false, - "panels": [ - { - "aliasColors": { - - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 1, - "gridPos": { - - }, - "id": 5, - "legend": { - "alignAsTable": false, - "avg": false, - "current": false, - "max": false, - "min": false, - "rightSide": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [ - - ], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "repeat": null, - "seriesOverrides": [ - - ], - "spaceLength": 10, - "span": 4, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum(rate(rest_client_requests_total{job=\"kube-scheduler\", instance=~\"$instance\",code=~\"2..\"}[5m]))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "2xx", - "refId": "A" - }, - { - "expr": "sum(rate(rest_client_requests_total{job=\"kube-scheduler\", instance=~\"$instance\",code=~\"3..\"}[5m]))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "3xx", - "refId": "B" - }, - { - "expr": "sum(rate(rest_client_requests_total{job=\"kube-scheduler\", instance=~\"$instance\",code=~\"4..\"}[5m]))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "4xx", - "refId": "C" - }, - { - "expr": "sum(rate(rest_client_requests_total{job=\"kube-scheduler\", instance=~\"$instance\",code=~\"5..\"}[5m]))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "5xx", - "refId": "D" - } - ], - "thresholds": [ - - ], - "timeFrom": null, - "timeShift": null, - "title": "Kube API Request Rate", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ - - ] - }, - "yaxes": [ - { - "format": "ops", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "ops", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - } - ] - }, - { - "aliasColors": { - - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 1, - "gridPos": { - - }, - "id": 6, - "legend": { - "alignAsTable": false, - "avg": false, - "current": false, - "max": false, - "min": false, - "rightSide": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [ - - ], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "repeat": null, - "seriesOverrides": [ - - ], - "spaceLength": 10, - "span": 8, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "histogram_quantile(0.99, sum(rate(rest_client_request_latency_seconds_bucket{job=\"kube-scheduler\", instance=~\"$instance\", verb=\"POST\"}[5m])) by (verb, url, le))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{verb}} {{url}}", - "refId": "A" - } - ], - "thresholds": [ - - ], - "timeFrom": null, - "timeShift": null, - "title": "Post Request Latency 99th Quantile", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ - - ] - }, - "yaxes": [ - { - "format": "s", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "s", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - } - ] - } - ], - "repeat": null, - "repeatIteration": null, - "repeatRowId": null, - "showTitle": false, - "title": "Dashboard Row", - "titleSize": "h6", - "type": "row" - }, - { - "collapse": false, - "collapsed": false, - "panels": [ - { - "aliasColors": { - - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 1, - "gridPos": { - - }, - "id": 7, - "legend": { - "alignAsTable": "true", - "avg": false, - "current": "true", - "max": false, - "min": false, - "rightSide": "true", - "show": "true", - "total": false, - "values": "true" - }, - "lines": true, - "linewidth": 1, - "links": [ - - ], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "repeat": null, - "seriesOverrides": [ - - ], - "spaceLength": 10, - "span": 12, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "histogram_quantile(0.99, sum(rate(rest_client_request_latency_seconds_bucket{job=\"kube-scheduler\", instance=~\"$instance\", verb=\"GET\"}[5m])) by (verb, url, le))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{verb}} {{url}}", - "refId": "A" - } - ], - "thresholds": [ - - ], - "timeFrom": null, - "timeShift": null, - "title": "Get Request Latency 99th Quantile", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ - - ] - }, - "yaxes": [ - { - "format": "s", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "s", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - } - ] - } - ], - "repeat": null, - "repeatIteration": null, - "repeatRowId": null, - "showTitle": false, - "title": "Dashboard Row", - "titleSize": "h6", - "type": "row" - }, - { - "collapse": false, - "collapsed": false, - "panels": [ - { - "aliasColors": { - - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 1, - "gridPos": { - - }, - "id": 8, - "legend": { - "alignAsTable": false, - "avg": false, - "current": false, - "max": false, - "min": false, - "rightSide": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [ - - ], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "repeat": null, - "seriesOverrides": [ - - ], - "spaceLength": 10, - "span": 4, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "process_resident_memory_bytes{job=\"kube-scheduler\", instance=~\"$instance\"}", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{instance}}", - "refId": "A" - } - ], - "thresholds": [ - - ], - "timeFrom": null, - "timeShift": null, - "title": "Memory", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ - - ] - }, - "yaxes": [ - { - "format": "bytes", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "bytes", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] - }, - { - "aliasColors": { - - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 1, - "gridPos": { - - }, - "id": 9, - "legend": { - "alignAsTable": false, - "avg": false, - "current": false, - "max": false, - "min": false, - "rightSide": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [ - - ], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "repeat": null, - "seriesOverrides": [ - - ], - "spaceLength": 10, - "span": 4, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "rate(process_cpu_seconds_total{job=\"kube-scheduler\", instance=~\"$instance\"}[5m])", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{instance}}", - "refId": "A" - } - ], - "thresholds": [ - - ], - "timeFrom": null, - "timeShift": null, - "title": "CPU usage", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ - - ] - }, - "yaxes": [ - { - "format": "bytes", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "bytes", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - } - ] - }, - { - "aliasColors": { - - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 1, - "gridPos": { - - }, - "id": 10, - "legend": { - "alignAsTable": false, - "avg": false, - "current": false, - "max": false, - "min": false, - "rightSide": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [ - - ], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "repeat": null, - "seriesOverrides": [ - - ], - "spaceLength": 10, - "span": 4, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "go_goroutines{job=\"kube-scheduler\",instance=~\"$instance\"}", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{instance}}", - "refId": "A" - } - ], - "thresholds": [ - - ], - "timeFrom": null, - "timeShift": null, - "title": "Goroutines", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ - - ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] - } - ], - "repeat": null, - "repeatIteration": null, - "repeatRowId": null, - "showTitle": false, - "title": "Dashboard Row", - "titleSize": "h6", - "type": "row" - } - ], - "schemaVersion": 14, - "style": "dark", - "tags": [ - "kubernetes-mixin" - ], - "templating": { - "list": [ - { - "current": { - "text": "Prometheus", - "value": "Prometheus" - }, - "hide": 0, - "label": null, - "name": "datasource", - "options": [ - - ], - "query": "prometheus", - "refresh": 1, - "regex": "", - "type": "datasource" - }, - { - "allValue": null, - "current": { - - }, - "datasource": "$datasource", - "hide": 0, - "includeAll": true, - "label": null, - "multi": false, - "name": "instance", - "options": [ - - ], - "query": "label_values(process_cpu_seconds_total{job=\"kube-scheduler\"}, instance)", - "refresh": 2, - "regex": "", - "sort": 1, - "tagValuesQuery": "", - "tags": [ - - ], - "tagsQuery": "", - "type": "query", - "useTags": false - } - ] - }, - "time": { - "from": "now-1h", - "to": "now" - }, - "timepicker": { - "refresh_intervals": [ - "5s", - "10s", - "30s", - "1m", - "5m", - "15m", - "30m", - "1h", - "2h", - "1d" - ], - "time_options": [ - "5m", - "15m", - "1h", - "6h", - "12h", - "24h", - "2d", - "7d", - "30d" - ] - }, - "timezone": "", - "title": "Kubernetes / Scheduler", - "uid": "2e6b6a3b4bddf1427b3a55aa1311c656", - "version": 0 - } - statefulset.json: |- - { - "__inputs": [ - - ], - "__requires": [ - - ], - "annotations": { - "list": [ - - ] - }, - "editable": false, - "gnetId": null, - "graphTooltip": 0, - "hideControls": false, - "id": null, - "links": [ - - ], - "refresh": "", - "rows": [ - { - "collapse": false, - "collapsed": false, - "panels": [ - { - "cacheTimeout": null, - "colorBackground": false, - "colorValue": false, - "colors": [ - "#299c46", - "rgba(237, 129, 40, 0.89)", - "#d44a3a" - ], - "datasource": "$datasource", - "format": "none", - "gauge": { - "maxValue": 100, - "minValue": 0, - "show": false, - "thresholdLabels": false, - "thresholdMarkers": true - }, - "gridPos": { - - }, - "id": 2, - "interval": null, - "links": [ - - ], - "mappingType": 1, - "mappingTypes": [ - { - "name": "value to text", - "value": 1 - }, - { - "name": "range to text", - "value": 2 - } - ], - "maxDataPoints": 100, - "nullPointMode": "connected", - "nullText": null, - "postfix": "cores", - "postfixFontSize": "50%", - "prefix": "", - "prefixFontSize": "50%", - "rangeMaps": [ - { - "from": "null", - "text": "N/A", - "to": "null" - } - ], - "span": 4, - "sparkline": { - "fillColor": "rgba(31, 118, 189, 0.18)", - "lineColor": "rgb(31, 120, 193)", - "show": true - }, - "tableColumn": "", - "targets": [ - { - "expr": "sum(rate(container_cpu_usage_seconds_total{job=\"kubernetes-cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"$statefulset.*\"}[3m]))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "", - "refId": "A" - } - ], - "thresholds": "", - "title": "CPU", - "tooltip": { - "shared": false - }, - "type": "singlestat", - "valueFontSize": "80%", - "valueMaps": [ - { - "op": "=", - "text": "0", - "value": "null" - } - ], - "valueName": "current" - }, - { - "cacheTimeout": null, - "colorBackground": false, - "colorValue": false, - "colors": [ - "#299c46", - "rgba(237, 129, 40, 0.89)", - "#d44a3a" - ], - "datasource": "$datasource", - "format": "none", - "gauge": { - "maxValue": 100, - "minValue": 0, - "show": false, - "thresholdLabels": false, - "thresholdMarkers": true - }, - "gridPos": { - - }, - "id": 3, - "interval": null, - "links": [ - - ], - "mappingType": 1, - "mappingTypes": [ - { - "name": "value to text", - "value": 1 - }, - { - "name": "range to text", - "value": 2 - } - ], - "maxDataPoints": 100, - "nullPointMode": "connected", - "nullText": null, - "postfix": "GB", - "postfixFontSize": "50%", - "prefix": "", - "prefixFontSize": "50%", - "rangeMaps": [ - { - "from": "null", - "text": "N/A", - "to": "null" - } - ], - "span": 4, - "sparkline": { - "fillColor": "rgba(31, 118, 189, 0.18)", - "lineColor": "rgb(31, 120, 193)", - "show": true - }, - "tableColumn": "", - "targets": [ - { - "expr": "sum(container_memory_usage_bytes{job=\"kubernetes-cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"$statefulset.*\"}) / 1024^3", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "", - "refId": "A" - } - ], - "thresholds": "", - "title": "Memory", - "tooltip": { - "shared": false - }, - "type": "singlestat", - "valueFontSize": "80%", - "valueMaps": [ - { - "op": "=", - "text": "0", - "value": "null" - } - ], - "valueName": "current" - }, - { - "cacheTimeout": null, - "colorBackground": false, - "colorValue": false, - "colors": [ - "#299c46", - "rgba(237, 129, 40, 0.89)", - "#d44a3a" - ], - "datasource": "$datasource", - "format": "none", - "gauge": { - "maxValue": 100, - "minValue": 0, - "show": false, - "thresholdLabels": false, - "thresholdMarkers": true - }, - "gridPos": { - - }, - "id": 4, - "interval": null, - "links": [ - - ], - "mappingType": 1, - "mappingTypes": [ - { - "name": "value to text", - "value": 1 - }, - { - "name": "range to text", - "value": 2 - } - ], - "maxDataPoints": 100, - "nullPointMode": "connected", - "nullText": null, - "postfix": "Bps", - "postfixFontSize": "50%", - "prefix": "", - "prefixFontSize": "50%", - "rangeMaps": [ - { - "from": "null", - "text": "N/A", - "to": "null" - } - ], - "span": 4, - "sparkline": { - "fillColor": "rgba(31, 118, 189, 0.18)", - "lineColor": "rgb(31, 120, 193)", - "show": true - }, - "tableColumn": "", - "targets": [ - { - "expr": "sum(rate(container_network_transmit_bytes_total{job=\"kubernetes-cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"$statefulset.*\"}[3m])) + sum(rate(container_network_receive_bytes_total{cluster=\"$cluster\", namespace=\"$namespace\",pod=~\"$statefulset.*\"}[3m]))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "", - "refId": "A" - } - ], - "thresholds": "", - "title": "Network", - "tooltip": { - "shared": false - }, - "type": "singlestat", - "valueFontSize": "80%", - "valueMaps": [ - { - "op": "=", - "text": "0", - "value": "null" - } - ], - "valueName": "current" - } - ], - "repeat": null, - "repeatIteration": null, - "repeatRowId": null, - "showTitle": false, - "title": "Dashboard Row", - "titleSize": "h6", - "type": "row" - }, - { - "collapse": false, - "collapsed": false, - "height": "100px", - "panels": [ - { - "cacheTimeout": null, - "colorBackground": false, - "colorValue": false, - "colors": [ - "#299c46", - "rgba(237, 129, 40, 0.89)", - "#d44a3a" - ], - "datasource": "$datasource", - "format": "none", - "gauge": { - "maxValue": 100, - "minValue": 0, - "show": false, - "thresholdLabels": false, - "thresholdMarkers": true - }, - "gridPos": { - - }, - "id": 5, - "interval": null, - "links": [ - - ], - "mappingType": 1, - "mappingTypes": [ - { - "name": "value to text", - "value": 1 - }, - { - "name": "range to text", - "value": 2 - } - ], - "maxDataPoints": 100, - "nullPointMode": "connected", - "nullText": null, - "postfix": "", - "postfixFontSize": "50%", - "prefix": "", - "prefixFontSize": "50%", - "rangeMaps": [ - { - "from": "null", - "text": "N/A", - "to": "null" - } - ], - "span": 3, - "sparkline": { - "fillColor": "rgba(31, 118, 189, 0.18)", - "full": false, - "lineColor": "rgb(31, 120, 193)", - "show": false - }, - "tableColumn": "", - "targets": [ - { - "expr": "max(kube_statefulset_replicas{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", statefulset=\"$statefulset\"}) without (instance, pod)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "", - "refId": "A" - } - ], - "thresholds": "", - "title": "Desired Replicas", - "tooltip": { - "shared": false - }, - "type": "singlestat", - "valueFontSize": "80%", - "valueMaps": [ - { - "op": "=", - "text": "0", - "value": "null" - } - ], - "valueName": "current" - }, - { - "cacheTimeout": null, - "colorBackground": false, - "colorValue": false, - "colors": [ - "#299c46", - "rgba(237, 129, 40, 0.89)", - "#d44a3a" - ], - "datasource": "$datasource", - "format": "none", - "gauge": { - "maxValue": 100, - "minValue": 0, - "show": false, - "thresholdLabels": false, - "thresholdMarkers": true - }, - "gridPos": { - - }, - "id": 6, - "interval": null, - "links": [ - - ], - "mappingType": 1, - "mappingTypes": [ - { - "name": "value to text", - "value": 1 - }, - { - "name": "range to text", - "value": 2 - } - ], - "maxDataPoints": 100, - "nullPointMode": "connected", - "nullText": null, - "postfix": "", - "postfixFontSize": "50%", - "prefix": "", - "prefixFontSize": "50%", - "rangeMaps": [ - { - "from": "null", - "text": "N/A", - "to": "null" - } - ], - "span": 3, - "sparkline": { - "fillColor": "rgba(31, 118, 189, 0.18)", - "full": false, - "lineColor": "rgb(31, 120, 193)", - "show": false - }, - "tableColumn": "", - "targets": [ - { - "expr": "min(kube_statefulset_status_replicas_current{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", statefulset=\"$statefulset\"}) without (instance, pod)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "", - "refId": "A" - } - ], - "thresholds": "", - "title": "Replicas of current version", - "tooltip": { - "shared": false - }, - "type": "singlestat", - "valueFontSize": "80%", - "valueMaps": [ - { - "op": "=", - "text": "0", - "value": "null" - } - ], - "valueName": "current" - }, - { - "cacheTimeout": null, - "colorBackground": false, - "colorValue": false, - "colors": [ - "#299c46", - "rgba(237, 129, 40, 0.89)", - "#d44a3a" - ], - "datasource": "$datasource", - "format": "none", - "gauge": { - "maxValue": 100, - "minValue": 0, - "show": false, - "thresholdLabels": false, - "thresholdMarkers": true - }, - "gridPos": { - - }, - "id": 7, - "interval": null, - "links": [ - - ], - "mappingType": 1, - "mappingTypes": [ - { - "name": "value to text", - "value": 1 - }, - { - "name": "range to text", - "value": 2 - } - ], - "maxDataPoints": 100, - "nullPointMode": "connected", - "nullText": null, - "postfix": "", - "postfixFontSize": "50%", - "prefix": "", - "prefixFontSize": "50%", - "rangeMaps": [ - { - "from": "null", - "text": "N/A", - "to": "null" - } - ], - "span": 3, - "sparkline": { - "fillColor": "rgba(31, 118, 189, 0.18)", - "full": false, - "lineColor": "rgb(31, 120, 193)", - "show": false - }, - "tableColumn": "", - "targets": [ - { - "expr": "max(kube_statefulset_status_observed_generation{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", statefulset=\"$statefulset\"}) without (instance, pod)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "", - "refId": "A" - } - ], - "thresholds": "", - "title": "Observed Generation", - "tooltip": { - "shared": false - }, - "type": "singlestat", - "valueFontSize": "80%", - "valueMaps": [ - { - "op": "=", - "text": "0", - "value": "null" - } - ], - "valueName": "current" - }, - { - "cacheTimeout": null, - "colorBackground": false, - "colorValue": false, - "colors": [ - "#299c46", - "rgba(237, 129, 40, 0.89)", - "#d44a3a" - ], - "datasource": "$datasource", - "format": "none", - "gauge": { - "maxValue": 100, - "minValue": 0, - "show": false, - "thresholdLabels": false, - "thresholdMarkers": true - }, - "gridPos": { - - }, - "id": 8, - "interval": null, - "links": [ - - ], - "mappingType": 1, - "mappingTypes": [ - { - "name": "value to text", - "value": 1 - }, - { - "name": "range to text", - "value": 2 - } - ], - "maxDataPoints": 100, - "nullPointMode": "connected", - "nullText": null, - "postfix": "", - "postfixFontSize": "50%", - "prefix": "", - "prefixFontSize": "50%", - "rangeMaps": [ - { - "from": "null", - "text": "N/A", - "to": "null" - } - ], - "span": 3, - "sparkline": { - "fillColor": "rgba(31, 118, 189, 0.18)", - "full": false, - "lineColor": "rgb(31, 120, 193)", - "show": false - }, - "tableColumn": "", - "targets": [ - { - "expr": "max(kube_statefulset_metadata_generation{job=\"kube-state-metrics\", statefulset=\"$statefulset\", cluster=\"$cluster\", namespace=\"$namespace\"}) without (instance, pod)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "", - "refId": "A" - } - ], - "thresholds": "", - "title": "Metadata Generation", - "tooltip": { - "shared": false - }, - "type": "singlestat", - "valueFontSize": "80%", - "valueMaps": [ + }, { - "op": "=", - "text": "0", - "value": "null" + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true } - ], - "valueName": "current" + ] } ], "repeat": null, @@ -5288,7 +3650,7 @@ data: "gridPos": { }, - "id": 9, + "id": 8, "legend": { "alignAsTable": false, "avg": false, @@ -5315,43 +3677,198 @@ data: ], "spaceLength": 10, + "span": 4, "stack": false, "steppedLine": false, "targets": [ { - "expr": "max(kube_statefulset_replicas{job=\"kube-state-metrics\", statefulset=\"$statefulset\", cluster=\"$cluster\", namespace=\"$namespace\"}) without (instance, pod)", + "expr": "process_resident_memory_bytes{job=\"kube-scheduler\", instance=~\"$instance\"}", "format": "time_series", "intervalFactor": 2, - "legendFormat": "replicas specified", + "legendFormat": "{{instance}}", "refId": "A" - }, + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Memory", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ { - "expr": "max(kube_statefulset_status_replicas{job=\"kube-state-metrics\", statefulset=\"$statefulset\", cluster=\"$cluster\", namespace=\"$namespace\"}) without (instance, pod)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "replicas created", - "refId": "B" + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true }, { - "expr": "min(kube_statefulset_status_replicas_ready{job=\"kube-state-metrics\", statefulset=\"$statefulset\", cluster=\"$cluster\", namespace=\"$namespace\"}) without (instance, pod)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "ready", - "refId": "C" - }, + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "gridPos": { + + }, + "id": 9, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ { - "expr": "min(kube_statefulset_status_replicas_current{job=\"kube-state-metrics\", statefulset=\"$statefulset\", cluster=\"$cluster\", namespace=\"$namespace\"}) without (instance, pod)", + "expr": "rate(process_cpu_seconds_total{job=\"kube-scheduler\", instance=~\"$instance\"}[5m])", "format": "time_series", "intervalFactor": 2, - "legendFormat": "replicas of current version", - "refId": "D" + "legendFormat": "{{instance}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU usage", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true }, { - "expr": "min(kube_statefulset_status_replicas_updated{job=\"kube-state-metrics\", statefulset=\"$statefulset\", cluster=\"$cluster\", namespace=\"$namespace\"}) without (instance, pod)", + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "gridPos": { + + }, + "id": 10, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "go_goroutines{job=\"kube-scheduler\",instance=~\"$instance\"}", "format": "time_series", "intervalFactor": 2, - "legendFormat": "updated", - "refId": "E" + "legendFormat": "{{instance}}", + "refId": "A" } ], "thresholds": [ @@ -5359,7 +3876,7 @@ data: ], "timeFrom": null, "timeShift": null, - "title": "Replicas", + "title": "Goroutines", "tooltip": { "shared": false, "sort": 0, @@ -5413,8 +3930,8 @@ data: "list": [ { "current": { - "text": "Prometheus", - "value": "Prometheus" + "text": "default", + "value": "default" }, "hide": 0, "label": null, @@ -5431,69 +3948,17 @@ data: "allValue": null, "current": { - }, - "datasource": "$datasource", - "hide": 2, - "includeAll": false, - "label": "cluster", - "multi": false, - "name": "cluster", - "options": [ - - ], - "query": "label_values(kube_statefulset_metadata_generation, cluster)", - "refresh": 2, - "regex": "", - "sort": 1, - "tagValuesQuery": "", - "tags": [ - - ], - "tagsQuery": "", - "type": "query", - "useTags": false - }, - { - "allValue": null, - "current": { - }, "datasource": "$datasource", "hide": 0, - "includeAll": false, - "label": "Namespace", - "multi": false, - "name": "namespace", - "options": [ - - ], - "query": "label_values(kube_statefulset_metadata_generation{job=\"kube-state-metrics\", cluster=\"$cluster\"}, namespace)", - "refresh": 2, - "regex": "", - "sort": 1, - "tagValuesQuery": "", - "tags": [ - - ], - "tagsQuery": "", - "type": "query", - "useTags": false - }, - { - "allValue": null, - "current": { - - }, - "datasource": "$datasource", - "hide": 0, - "includeAll": false, - "label": "Name", + "includeAll": true, + "label": null, "multi": false, - "name": "statefulset", + "name": "instance", "options": [ ], - "query": "label_values(kube_statefulset_metadata_generation{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\"}, statefulset)", + "query": "label_values(process_cpu_seconds_total{job=\"kube-scheduler\"}, instance)", "refresh": 2, "regex": "", "sort": 1, @@ -5537,8 +4002,8 @@ data: ] }, "timezone": "", - "title": "Kubernetes / StatefulSets", - "uid": "a31c1f46e6f727cb37c0d731a7245005", + "title": "Kubernetes / Scheduler", + "uid": "2e6b6a3b4bddf1427b3a55aa1311c656", "version": 0 } kind: ConfigMap diff --git a/addons/grafana/dashboards-prom.yaml b/addons/grafana/dashboards-prom.yaml index 0c97844cb..3eb0c0308 100644 --- a/addons/grafana/dashboards-prom.yaml +++ b/addons/grafana/dashboards-prom.yaml @@ -2,6 +2,12 @@ apiVersion: v1 data: prometheus-remote-write.json: |- { + "__inputs": [ + + ], + "__requires": [ + + ], "annotations": { "list": [ @@ -11,14 +17,15 @@ data: "gnetId": null, "graphTooltip": 0, "hideControls": false, + "id": null, "links": [ ], - "refresh": "10s", + "refresh": "", "rows": [ { "collapse": false, - "height": "250px", + "collapsed": false, "panels": [ { "aliasColors": { @@ -29,12 +36,499 @@ data: "dashes": false, "datasource": "$datasource", "fill": 1, - "id": 1, + "gridPos": { + + }, + "id": 2, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "(\n prometheus_remote_storage_highest_timestamp_in_seconds{cluster=~\"$cluster\", instance=~\"$instance\"} \n- \n ignoring(queue) group_right(instance) prometheus_remote_storage_queue_highest_sent_timestamp_seconds{cluster=~\"$cluster\", instance=~\"$instance\"}\n)\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{cluster}}:{{instance}}-{{queue}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Highest Timestamp In vs. Highest Timestamp Sent", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "gridPos": { + + }, + "id": 3, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "(\n rate(prometheus_remote_storage_highest_timestamp_in_seconds{cluster=~\"$cluster\", instance=~\"$instance\"}[5m]) \n- \n ignoring (queue) group_right(instance) rate(prometheus_remote_storage_queue_highest_sent_timestamp_seconds{cluster=~\"$cluster\", instance=~\"$instance\"}[5m])\n)\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{cluster}}:{{instance}}-{{queue}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate[5m]", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Timestamps", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "gridPos": { + + }, + "id": 4, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(\n prometheus_remote_storage_samples_in_total{cluster=~\"$cluster\", instance=~\"$instance\"}[5m])\n- \n ignoring(queue) group_right(instance) rate(prometheus_remote_storage_succeeded_samples_total{cluster=~\"$cluster\", instance=~\"$instance\"}[5m]) \n- \n rate(prometheus_remote_storage_dropped_samples_total{cluster=~\"$cluster\", instance=~\"$instance\"}[5m])\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{cluster}}:{{instance}}-{{queue}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate, in vs. succeeded or dropped [5m]", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Samples", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "gridPos": { + + }, + "id": 5, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "minSpan": 6, + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "prometheus_remote_storage_shards{cluster=~\"$cluster\", instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{cluster}}:{{instance}}-{{queue}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Current Shards", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "gridPos": { + + }, + "id": 6, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "prometheus_remote_storage_shards_max{cluster=~\"$cluster\", instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{cluster}}:{{instance}}-{{queue}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Max Shards", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "gridPos": { + + }, + "id": 7, "legend": { + "alignAsTable": false, "avg": false, "current": false, "max": false, "min": false, + "rightSide": false, "show": true, "total": false, "values": false @@ -44,26 +538,26 @@ data: "links": [ ], - "nullPointMode": "null as zero", + "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", + "repeat": null, "seriesOverrides": [ ], "spaceLength": 10, - "span": 6, + "span": 4, "stack": false, "steppedLine": false, "targets": [ { - "expr": "prometheus_remote_storage_highest_timestamp_in_seconds{cluster=~\"$cluster\", instance=~\"$instance\"} - ignoring(queue) group_right(instance) prometheus_remote_storage_queue_highest_sent_timestamp_seconds{cluster=~\"$cluster\", instance=~\"$instance\"}", + "expr": "prometheus_remote_storage_shards_min{cluster=~\"$cluster\", instance=~\"$instance\"}", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{cluster}}:{{instance}}-{{queue}}", - "legendLink": null, - "step": 10 + "refId": "A" } ], "thresholds": [ @@ -71,7 +565,7 @@ data: ], "timeFrom": null, "timeShift": null, - "title": "Highest Timestamp In vs. Highest Timestamp Sent", + "title": "Min Shards", "tooltip": { "shared": true, "sort": 0, @@ -89,11 +583,11 @@ data: }, "yaxes": [ { - "format": "s", + "format": "short", "label": null, "logBase": 1, "max": null, - "min": 0, + "min": null, "show": true }, { @@ -102,7 +596,7 @@ data: "logBase": 1, "max": null, "min": null, - "show": false + "show": true } ] }, @@ -115,12 +609,17 @@ data: "dashes": false, "datasource": "$datasource", "fill": 1, - "id": 2, + "gridPos": { + + }, + "id": 8, "legend": { + "alignAsTable": false, "avg": false, "current": false, "max": false, "min": false, + "rightSide": false, "show": true, "total": false, "values": false @@ -130,26 +629,26 @@ data: "links": [ ], - "nullPointMode": "null as zero", + "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", + "repeat": null, "seriesOverrides": [ ], "spaceLength": 10, - "span": 6, + "span": 4, "stack": false, "steppedLine": false, "targets": [ { - "expr": "rate(prometheus_remote_storage_highest_timestamp_in_seconds{cluster=~\"$cluster\", instance=~\"$instance\"}[5m]) - ignoring (queue) group_right(instance) rate(prometheus_remote_storage_queue_highest_sent_timestamp_seconds{cluster=~\"$cluster\", instance=~\"$instance\"}[5m])", + "expr": "prometheus_remote_storage_shards_desired{cluster=~\"$cluster\", instance=~\"$instance\"}", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{cluster}}:{{instance}}-{{queue}}", - "legendLink": null, - "step": 10 + "refId": "A" } ], "thresholds": [ @@ -157,7 +656,7 @@ data: ], "timeFrom": null, "timeShift": null, - "title": "Rate[5m]", + "title": "Desired Shards", "tooltip": { "shared": true, "sort": 0, @@ -179,7 +678,7 @@ data: "label": null, "logBase": 1, "max": null, - "min": 0, + "min": null, "show": true }, { @@ -188,7 +687,7 @@ data: "logBase": 1, "max": null, "min": null, - "show": false + "show": true } ] } @@ -197,12 +696,13 @@ data: "repeatIteration": null, "repeatRowId": null, "showTitle": true, - "title": "Timestamps", - "titleSize": "h6" + "title": "Shards", + "titleSize": "h6", + "type": "row" }, { "collapse": false, - "height": "250px", + "collapsed": false, "panels": [ { "aliasColors": { @@ -213,12 +713,17 @@ data: "dashes": false, "datasource": "$datasource", "fill": 1, - "id": 3, + "gridPos": { + + }, + "id": 9, "legend": { + "alignAsTable": false, "avg": false, "current": false, "max": false, "min": false, + "rightSide": false, "show": true, "total": false, "values": false @@ -228,26 +733,26 @@ data: "links": [ ], - "nullPointMode": "null as zero", + "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", + "repeat": null, "seriesOverrides": [ ], "spaceLength": 10, - "span": 12, + "span": 6, "stack": false, "steppedLine": false, "targets": [ { - "expr": "rate(prometheus_remote_storage_samples_in_total{cluster=~\"$cluster\", instance=~\"$instance\"}[5m])- ignoring(queue) group_right(instance) rate(prometheus_remote_storage_succeeded_samples_total{cluster=~\"$cluster\", instance=~\"$instance\"}[5m]) - rate(prometheus_remote_storage_dropped_samples_total{cluster=~\"$cluster\", instance=~\"$instance\"}[5m])", + "expr": "prometheus_remote_storage_shard_capacity{cluster=~\"$cluster\", instance=~\"$instance\"}", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{cluster}}:{{instance}}-{{queue}}", - "legendLink": null, - "step": 10 + "refId": "A" } ], "thresholds": [ @@ -255,7 +760,7 @@ data: ], "timeFrom": null, "timeShift": null, - "title": "Rate, in vs. succeeded or dropped [5m]", + "title": "Shard Capacity", "tooltip": { "shared": true, "sort": 0, @@ -277,7 +782,7 @@ data: "label": null, "logBase": 1, "max": null, - "min": 0, + "min": null, "show": true }, { @@ -286,7 +791,98 @@ data: "logBase": 1, "max": null, "min": null, - "show": false + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "gridPos": { + + }, + "id": 10, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "prometheus_remote_storage_pending_samples{cluster=~\"$cluster\", instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{cluster}}:{{instance}}-{{queue}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Pending Samples", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true } ] } @@ -295,12 +891,13 @@ data: "repeatIteration": null, "repeatRowId": null, "showTitle": true, - "title": "Samples", - "titleSize": "h6" + "title": "Shard Details", + "titleSize": "h6", + "type": "row" }, { "collapse": false, - "height": "250px", + "collapsed": false, "panels": [ { "aliasColors": { @@ -311,12 +908,17 @@ data: "dashes": false, "datasource": "$datasource", "fill": 1, - "id": 4, + "gridPos": { + + }, + "id": 11, "legend": { + "alignAsTable": false, "avg": false, "current": false, "max": false, "min": false, + "rightSide": false, "show": true, "total": false, "values": false @@ -326,11 +928,12 @@ data: "links": [ ], - "nullPointMode": "null as zero", + "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", + "repeat": null, "seriesOverrides": [ ], @@ -340,12 +943,11 @@ data: "steppedLine": false, "targets": [ { - "expr": "prometheus_remote_storage_shards{cluster=~\"$cluster\", instance=~\"$instance\"}", + "expr": "prometheus_tsdb_wal_segment_current{cluster=~\"$cluster\", instance=~\"$instance\"}", "format": "time_series", "intervalFactor": 2, - "legendFormat": "{{cluster}}:{{instance}}-{{queue}}", - "legendLink": null, - "step": 10 + "legendFormat": "{{cluster}}:{{instance}}", + "refId": "A" } ], "thresholds": [ @@ -353,7 +955,7 @@ data: ], "timeFrom": null, "timeShift": null, - "title": "Num. Shards", + "title": "TSDB Current Segment", "tooltip": { "shared": true, "sort": 0, @@ -371,11 +973,11 @@ data: }, "yaxes": [ { - "format": "short", + "format": "none", "label": null, "logBase": 1, "max": null, - "min": 0, + "min": null, "show": true }, { @@ -384,7 +986,7 @@ data: "logBase": 1, "max": null, "min": null, - "show": false + "show": true } ] }, @@ -397,12 +999,17 @@ data: "dashes": false, "datasource": "$datasource", "fill": 1, - "id": 5, + "gridPos": { + + }, + "id": 12, "legend": { + "alignAsTable": false, "avg": false, "current": false, "max": false, "min": false, + "rightSide": false, "show": true, "total": false, "values": false @@ -412,11 +1019,12 @@ data: "links": [ ], - "nullPointMode": "null as zero", + "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", + "repeat": null, "seriesOverrides": [ ], @@ -426,12 +1034,11 @@ data: "steppedLine": false, "targets": [ { - "expr": "prometheus_remote_storage_shard_capacity{cluster=~\"$cluster\", instance=~\"$instance\"}", + "expr": "prometheus_wal_watcher_current_segment{cluster=~\"$cluster\", instance=~\"$instance\"}", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{cluster}}:{{instance}}-{{queue}}", - "legendLink": null, - "step": 10 + "refId": "A" } ], "thresholds": [ @@ -439,7 +1046,7 @@ data: ], "timeFrom": null, "timeShift": null, - "title": "Capacity", + "title": "Remote Write Current Segment", "tooltip": { "shared": true, "sort": 0, @@ -457,11 +1064,11 @@ data: }, "yaxes": [ { - "format": "short", + "format": "none", "label": null, "logBase": 1, "max": null, - "min": 0, + "min": null, "show": true }, { @@ -470,7 +1077,7 @@ data: "logBase": 1, "max": null, "min": null, - "show": false + "show": true } ] } @@ -479,12 +1086,13 @@ data: "repeatIteration": null, "repeatRowId": null, "showTitle": true, - "title": "Shards", - "titleSize": "h6" + "title": "Segments", + "titleSize": "h6", + "type": "row" }, { "collapse": false, - "height": "250px", + "collapsed": false, "panels": [ { "aliasColors": { @@ -495,12 +1103,17 @@ data: "dashes": false, "datasource": "$datasource", "fill": 1, - "id": 6, + "gridPos": { + + }, + "id": 13, "legend": { + "alignAsTable": false, "avg": false, "current": false, "max": false, "min": false, + "rightSide": false, "show": true, "total": false, "values": false @@ -510,11 +1123,12 @@ data: "links": [ ], - "nullPointMode": "null as zero", + "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", + "repeat": null, "seriesOverrides": [ ], @@ -528,8 +1142,7 @@ data: "format": "time_series", "intervalFactor": 2, "legendFormat": "{{cluster}}:{{instance}}-{{queue}}", - "legendLink": null, - "step": 10 + "refId": "A" } ], "thresholds": [ @@ -559,7 +1172,7 @@ data: "label": null, "logBase": 1, "max": null, - "min": 0, + "min": null, "show": true }, { @@ -568,7 +1181,7 @@ data: "logBase": 1, "max": null, "min": null, - "show": false + "show": true } ] }, @@ -581,12 +1194,17 @@ data: "dashes": false, "datasource": "$datasource", "fill": 1, - "id": 7, + "gridPos": { + + }, + "id": 14, "legend": { + "alignAsTable": false, "avg": false, "current": false, "max": false, "min": false, + "rightSide": false, "show": true, "total": false, "values": false @@ -596,11 +1214,12 @@ data: "links": [ ], - "nullPointMode": "null as zero", + "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", + "repeat": null, "seriesOverrides": [ ], @@ -614,8 +1233,7 @@ data: "format": "time_series", "intervalFactor": 2, "legendFormat": "{{cluster}}:{{instance}}-{{queue}}", - "legendLink": null, - "step": 10 + "refId": "A" } ], "thresholds": [ @@ -645,7 +1263,7 @@ data: "label": null, "logBase": 1, "max": null, - "min": 0, + "min": null, "show": true }, { @@ -654,7 +1272,7 @@ data: "logBase": 1, "max": null, "min": null, - "show": false + "show": true } ] }, @@ -667,12 +1285,17 @@ data: "dashes": false, "datasource": "$datasource", "fill": 1, - "id": 8, + "gridPos": { + + }, + "id": 15, "legend": { + "alignAsTable": false, "avg": false, "current": false, "max": false, "min": false, + "rightSide": false, "show": true, "total": false, "values": false @@ -682,11 +1305,12 @@ data: "links": [ ], - "nullPointMode": "null as zero", + "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", + "repeat": null, "seriesOverrides": [ ], @@ -700,8 +1324,7 @@ data: "format": "time_series", "intervalFactor": 2, "legendFormat": "{{cluster}}:{{instance}}-{{queue}}", - "legendLink": null, - "step": 10 + "refId": "A" } ], "thresholds": [ @@ -731,7 +1354,7 @@ data: "label": null, "logBase": 1, "max": null, - "min": 0, + "min": null, "show": true }, { @@ -740,7 +1363,7 @@ data: "logBase": 1, "max": null, "min": null, - "show": false + "show": true } ] }, @@ -753,12 +1376,17 @@ data: "dashes": false, "datasource": "$datasource", "fill": 1, - "id": 9, + "gridPos": { + + }, + "id": 16, "legend": { + "alignAsTable": false, "avg": false, "current": false, "max": false, "min": false, + "rightSide": false, "show": true, "total": false, "values": false @@ -768,11 +1396,12 @@ data: "links": [ ], - "nullPointMode": "null as zero", + "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", + "repeat": null, "seriesOverrides": [ ], @@ -786,8 +1415,7 @@ data: "format": "time_series", "intervalFactor": 2, "legendFormat": "{{cluster}}:{{instance}}-{{queue}}", - "legendLink": null, - "step": 10 + "refId": "A" } ], "thresholds": [ @@ -817,7 +1445,7 @@ data: "label": null, "logBase": 1, "max": null, - "min": 0, + "min": null, "show": true }, { @@ -826,7 +1454,7 @@ data: "logBase": 1, "max": null, "min": null, - "show": false + "show": true } ] } @@ -835,8 +1463,9 @@ data: "repeatIteration": null, "repeatRowId": null, "showTitle": true, - "title": "Misc Rates.", - "titleSize": "h6" + "title": "Misc. Rates", + "titleSize": "h6", + "type": "row" } ], "schemaVersion": 14, @@ -847,10 +1476,6 @@ data: "templating": { "list": [ { - "current": { - "text": "Prometheus", - "value": "Prometheus" - }, "hide": 0, "label": null, "name": "datasource", @@ -865,23 +1490,30 @@ data: { "allValue": null, "current": { - "selected": true, - "text": "All", - "value": "$__all" + "text": { + "selected": true, + "text": "All", + "value": "$__all" + }, + "value": { + "selected": true, + "text": "All", + "value": "$__all" + } }, "datasource": "$datasource", "hide": 0, "includeAll": true, - "label": "instance", - "multi": true, + "label": null, + "multi": false, "name": "instance", "options": [ ], "query": "label_values(prometheus_build_info, instance)", - "refresh": 1, + "refresh": 2, "regex": "", - "sort": 2, + "sort": 0, "tagValuesQuery": "", "tags": [ @@ -893,23 +1525,56 @@ data: { "allValue": null, "current": { - "selected": true, - "text": "All", - "value": "$__all" + "text": { + "selected": true, + "text": "All", + "value": "$__all" + }, + "value": { + "selected": true, + "text": "All", + "value": "$__all" + } }, "datasource": "$datasource", "hide": 0, "includeAll": true, - "label": "cluster", - "multi": true, + "label": null, + "multi": false, "name": "cluster", "options": [ ], "query": "label_values(kube_pod_container_info{image=~\".*prometheus.*\"}, cluster)", - "refresh": 1, + "refresh": 2, "regex": "", - "sort": 2, + "sort": 0, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": true, + "label": null, + "multi": false, + "name": "queue", + "options": [ + + ], + "query": "label_values(prometheus_remote_storage_shards{cluster=~\"$cluster\", instance=~\"$instance\"}, queue)", + "refresh": 2, + "regex": "", + "sort": 0, "tagValuesQuery": "", "tags": [ @@ -921,7 +1586,7 @@ data: ] }, "time": { - "from": "now-1h", + "from": "now-6h", "to": "now" }, "timepicker": { @@ -949,9 +1614,8 @@ data: "30d" ] }, - "timezone": "utc", + "timezone": "browser", "title": "Prometheus Remote Write", - "uid": "", "version": 0 } prometheus.json: |- @@ -2048,8 +2712,8 @@ data: "list": [ { "current": { - "text": "Prometheus", - "value": "Prometheus" + "text": "default", + "value": "default" }, "hide": 0, "label": null, diff --git a/addons/prometheus/rules.yaml b/addons/prometheus/rules.yaml index 804f9e718..e1ffbc6f0 100644 --- a/addons/prometheus/rules.yaml +++ b/addons/prometheus/rules.yaml @@ -42,10 +42,10 @@ data: { "alert": "etcdHighNumberOfLeaderChanges", "annotations": { - "message": "etcd cluster \"{{ $labels.job }}\": instance {{ $labels.instance }} has seen {{ $value }} leader changes within the last 30 minutes." + "message": "etcd cluster \"{{ $labels.job }}\": {{ $value }} leader changes within the last 15 minutes. Frequent elections may be a sign of insufficient resources, high network latency, or disruptions by other components and should be investigated." }, - "expr": "rate(etcd_server_leader_changes_seen_total{job=~\".*etcd.*\"}[15m]) > 3\n", - "for": "15m", + "expr": "increase((max by (job) (etcd_server_leader_changes_seen_total{job=~\".*etcd.*\"}) or 0*absent(etcd_server_leader_changes_seen_total{job=~\".*etcd.*\"}))[15m:1m]) >= 3\n", + "for": "5m", "labels": { "severity": "warning" } @@ -145,25 +145,132 @@ data: kube.yaml: |- { "groups": [ + { + "name": "kube-apiserver-error", + "rules": [ + { + "expr": "sum by (status_class) (\n label_replace(\n rate(apiserver_request_total{job=\"apiserver\"}[5m]\n ), \"status_class\", \"${1}xx\", \"code\", \"([0-9])..\")\n)\n", + "labels": { + "job": "apiserver" + }, + "record": "status_class:apiserver_request_total:rate5m" + }, + { + "expr": "sum by (status_class) (\n label_replace(\n rate(apiserver_request_total{job=\"apiserver\"}[30m]\n ), \"status_class\", \"${1}xx\", \"code\", \"([0-9])..\")\n)\n", + "labels": { + "job": "apiserver" + }, + "record": "status_class:apiserver_request_total:rate30m" + }, + { + "expr": "sum by (status_class) (\n label_replace(\n rate(apiserver_request_total{job=\"apiserver\"}[1h]\n ), \"status_class\", \"${1}xx\", \"code\", \"([0-9])..\")\n)\n", + "labels": { + "job": "apiserver" + }, + "record": "status_class:apiserver_request_total:rate1h" + }, + { + "expr": "sum by (status_class) (\n label_replace(\n rate(apiserver_request_total{job=\"apiserver\"}[2h]\n ), \"status_class\", \"${1}xx\", \"code\", \"([0-9])..\")\n)\n", + "labels": { + "job": "apiserver" + }, + "record": "status_class:apiserver_request_total:rate2h" + }, + { + "expr": "sum by (status_class) (\n label_replace(\n rate(apiserver_request_total{job=\"apiserver\"}[6h]\n ), \"status_class\", \"${1}xx\", \"code\", \"([0-9])..\")\n)\n", + "labels": { + "job": "apiserver" + }, + "record": "status_class:apiserver_request_total:rate6h" + }, + { + "expr": "sum by (status_class) (\n label_replace(\n rate(apiserver_request_total{job=\"apiserver\"}[1d]\n ), \"status_class\", \"${1}xx\", \"code\", \"([0-9])..\")\n)\n", + "labels": { + "job": "apiserver" + }, + "record": "status_class:apiserver_request_total:rate1d" + }, + { + "expr": "sum by (status_class) (\n label_replace(\n rate(apiserver_request_total{job=\"apiserver\"}[3d]\n ), \"status_class\", \"${1}xx\", \"code\", \"([0-9])..\")\n)\n", + "labels": { + "job": "apiserver" + }, + "record": "status_class:apiserver_request_total:rate3d" + }, + { + "expr": "sum(status_class:apiserver_request_total:rate5m{job=\"apiserver\",status_class=\"5xx\"})\n/\nsum(status_class:apiserver_request_total:rate5m{job=\"apiserver\"})\n", + "labels": { + "job": "apiserver" + }, + "record": "status_class_5xx:apiserver_request_total:ratio_rate5m" + }, + { + "expr": "sum(status_class:apiserver_request_total:rate30m{job=\"apiserver\",status_class=\"5xx\"})\n/\nsum(status_class:apiserver_request_total:rate30m{job=\"apiserver\"})\n", + "labels": { + "job": "apiserver" + }, + "record": "status_class_5xx:apiserver_request_total:ratio_rate30m" + }, + { + "expr": "sum(status_class:apiserver_request_total:rate1h{job=\"apiserver\",status_class=\"5xx\"})\n/\nsum(status_class:apiserver_request_total:rate1h{job=\"apiserver\"})\n", + "labels": { + "job": "apiserver" + }, + "record": "status_class_5xx:apiserver_request_total:ratio_rate1h" + }, + { + "expr": "sum(status_class:apiserver_request_total:rate2h{job=\"apiserver\",status_class=\"5xx\"})\n/\nsum(status_class:apiserver_request_total:rate2h{job=\"apiserver\"})\n", + "labels": { + "job": "apiserver" + }, + "record": "status_class_5xx:apiserver_request_total:ratio_rate2h" + }, + { + "expr": "sum(status_class:apiserver_request_total:rate6h{job=\"apiserver\",status_class=\"5xx\"})\n/\nsum(status_class:apiserver_request_total:rate6h{job=\"apiserver\"})\n", + "labels": { + "job": "apiserver" + }, + "record": "status_class_5xx:apiserver_request_total:ratio_rate6h" + }, + { + "expr": "sum(status_class:apiserver_request_total:rate1d{job=\"apiserver\",status_class=\"5xx\"})\n/\nsum(status_class:apiserver_request_total:rate1d{job=\"apiserver\"})\n", + "labels": { + "job": "apiserver" + }, + "record": "status_class_5xx:apiserver_request_total:ratio_rate1d" + }, + { + "expr": "sum(status_class:apiserver_request_total:rate3d{job=\"apiserver\",status_class=\"5xx\"})\n/\nsum(status_class:apiserver_request_total:rate3d{job=\"apiserver\"})\n", + "labels": { + "job": "apiserver" + }, + "record": "status_class_5xx:apiserver_request_total:ratio_rate3d" + } + ] + }, { "name": "kube-apiserver.rules", "rules": [ { - "expr": "histogram_quantile(0.99, sum(rate(apiserver_request_duration_seconds_bucket{job=\"apiserver\"}[5m])) without(instance, pod))\n", + "expr": "sum(rate(apiserver_request_duration_seconds_sum{subresource!=\"log\",verb!~\"LIST|WATCH|WATCHLIST|PROXY|CONNECT\"}[5m])) without(instance, pod)\n/\nsum(rate(apiserver_request_duration_seconds_count{subresource!=\"log\",verb!~\"LIST|WATCH|WATCHLIST|PROXY|CONNECT\"}[5m])) without(instance, pod)\n", + "record": "cluster:apiserver_request_duration_seconds:mean5m" + }, + { + "expr": "histogram_quantile(0.99, sum(rate(apiserver_request_duration_seconds_bucket{job=\"apiserver\",subresource!=\"log\",verb!~\"LIST|WATCH|WATCHLIST|PROXY|CONNECT\"}[5m])) without(instance, pod))\n", "labels": { "quantile": "0.99" }, "record": "cluster_quantile:apiserver_request_duration_seconds:histogram_quantile" }, { - "expr": "histogram_quantile(0.9, sum(rate(apiserver_request_duration_seconds_bucket{job=\"apiserver\"}[5m])) without(instance, pod))\n", + "expr": "histogram_quantile(0.9, sum(rate(apiserver_request_duration_seconds_bucket{job=\"apiserver\",subresource!=\"log\",verb!~\"LIST|WATCH|WATCHLIST|PROXY|CONNECT\"}[5m])) without(instance, pod))\n", "labels": { "quantile": "0.9" }, "record": "cluster_quantile:apiserver_request_duration_seconds:histogram_quantile" }, { - "expr": "histogram_quantile(0.5, sum(rate(apiserver_request_duration_seconds_bucket{job=\"apiserver\"}[5m])) without(instance, pod))\n", + "expr": "histogram_quantile(0.5, sum(rate(apiserver_request_duration_seconds_bucket{job=\"apiserver\",subresource!=\"log\",verb!~\"LIST|WATCH|WATCHLIST|PROXY|CONNECT\"}[5m])) without(instance, pod))\n", "labels": { "quantile": "0.5" }, @@ -179,23 +286,23 @@ data: "record": "namespace:container_cpu_usage_seconds_total:sum_rate" }, { - "expr": "sum by (namespace, pod, container) (\n rate(container_cpu_usage_seconds_total{job=\"kubernetes-cadvisor\", image!=\"\", container!=\"POD\"}[5m])\n) * on (namespace, pod) group_left(node) max by(namespace, pod, node) (kube_pod_info)\n", + "expr": "sum by (cluster, namespace, pod, container) (\n rate(container_cpu_usage_seconds_total{job=\"kubernetes-cadvisor\", image!=\"\", container!=\"POD\"}[5m])\n) * on (cluster, namespace, pod) group_left(node) topk by (cluster, namespace, pod) (\n 1, max by(cluster, namespace, pod, node) (kube_pod_info)\n)\n", "record": "node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate" }, { - "expr": "container_memory_working_set_bytes{job=\"kubernetes-cadvisor\", image!=\"\"}\n* on (namespace, pod) group_left(node) max by(namespace, pod, node) (kube_pod_info)\n", + "expr": "container_memory_working_set_bytes{job=\"kubernetes-cadvisor\", image!=\"\"}\n* on (namespace, pod) group_left(node) topk by(namespace, pod) (1,\n max by(namespace, pod, node) (kube_pod_info)\n)\n", "record": "node_namespace_pod_container:container_memory_working_set_bytes" }, { - "expr": "container_memory_rss{job=\"kubernetes-cadvisor\", image!=\"\"}\n* on (namespace, pod) group_left(node) max by(namespace, pod, node) (kube_pod_info)\n", + "expr": "container_memory_rss{job=\"kubernetes-cadvisor\", image!=\"\"}\n* on (namespace, pod) group_left(node) topk by(namespace, pod) (1,\n max by(namespace, pod, node) (kube_pod_info)\n)\n", "record": "node_namespace_pod_container:container_memory_rss" }, { - "expr": "container_memory_cache{job=\"kubernetes-cadvisor\", image!=\"\"}\n* on (namespace, pod) group_left(node) max by(namespace, pod, node) (kube_pod_info)\n", + "expr": "container_memory_cache{job=\"kubernetes-cadvisor\", image!=\"\"}\n* on (namespace, pod) group_left(node) topk by(namespace, pod) (1,\n max by(namespace, pod, node) (kube_pod_info)\n)\n", "record": "node_namespace_pod_container:container_memory_cache" }, { - "expr": "container_memory_swap{job=\"kubernetes-cadvisor\", image!=\"\"}\n* on (namespace, pod) group_left(node) max by(namespace, pod, node) (kube_pod_info)\n", + "expr": "container_memory_swap{job=\"kubernetes-cadvisor\", image!=\"\"}\n* on (namespace, pod) group_left(node) topk by(namespace, pod) (1,\n max by(namespace, pod, node) (kube_pod_info)\n)\n", "record": "node_namespace_pod_container:container_memory_swap" }, { @@ -203,29 +310,29 @@ data: "record": "namespace:container_memory_usage_bytes:sum" }, { - "expr": "sum by (namespace, label_name) (\n sum(kube_pod_container_resource_requests_memory_bytes{job=\"kube-state-metrics\"} * on (endpoint, instance, job, namespace, pod, service) group_left(phase) (kube_pod_status_phase{phase=~\"Pending|Running\"} == 1)) by (namespace, pod)\n * on (namespace, pod)\n group_left(label_name) kube_pod_labels{job=\"kube-state-metrics\"}\n)\n", + "expr": "sum by (namespace) (\n sum by (namespace, pod) (\n max by (namespace, pod, container) (\n kube_pod_container_resource_requests_memory_bytes{job=\"kube-state-metrics\"}\n ) * on(namespace, pod) group_left() max by (namespace, pod) (\n kube_pod_status_phase{phase=~\"Pending|Running\"} == 1\n )\n )\n)\n", "record": "namespace:kube_pod_container_resource_requests_memory_bytes:sum" }, { - "expr": "sum by (namespace, label_name) (\n sum(kube_pod_container_resource_requests_cpu_cores{job=\"kube-state-metrics\"} * on (endpoint, instance, job, namespace, pod, service) group_left(phase) (kube_pod_status_phase{phase=~\"Pending|Running\"} == 1)) by (namespace, pod)\n * on (namespace, pod)\n group_left(label_name) kube_pod_labels{job=\"kube-state-metrics\"}\n)\n", + "expr": "sum by (namespace) (\n sum by (namespace, pod) (\n max by (namespace, pod, container) (\n kube_pod_container_resource_requests_cpu_cores{job=\"kube-state-metrics\"}\n ) * on(namespace, pod) group_left() max by (namespace, pod) (\n kube_pod_status_phase{phase=~\"Pending|Running\"} == 1\n )\n )\n)\n", "record": "namespace:kube_pod_container_resource_requests_cpu_cores:sum" }, { - "expr": "sum(\n label_replace(\n label_replace(\n kube_pod_owner{job=\"kube-state-metrics\", owner_kind=\"ReplicaSet\"},\n \"replicaset\", \"$1\", \"owner_name\", \"(.*)\"\n ) * on(replicaset, namespace) group_left(owner_name) kube_replicaset_owner{job=\"kube-state-metrics\"},\n \"workload\", \"$1\", \"owner_name\", \"(.*)\"\n )\n) by (namespace, workload, pod)\n", + "expr": "max by (cluster, namespace, workload, pod) (\n label_replace(\n label_replace(\n kube_pod_owner{job=\"kube-state-metrics\", owner_kind=\"ReplicaSet\"},\n \"replicaset\", \"$1\", \"owner_name\", \"(.*)\"\n ) * on(replicaset, namespace) group_left(owner_name) topk by(replicaset, namespace) (\n 1, max by (replicaset, namespace, owner_name) (\n kube_replicaset_owner{job=\"kube-state-metrics\"}\n )\n ),\n \"workload\", \"$1\", \"owner_name\", \"(.*)\"\n )\n)\n", "labels": { "workload_type": "deployment" }, "record": "mixin_pod_workload" }, { - "expr": "sum(\n label_replace(\n kube_pod_owner{job=\"kube-state-metrics\", owner_kind=\"DaemonSet\"},\n \"workload\", \"$1\", \"owner_name\", \"(.*)\"\n )\n) by (namespace, workload, pod)\n", + "expr": "max by (cluster, namespace, workload, pod) (\n label_replace(\n kube_pod_owner{job=\"kube-state-metrics\", owner_kind=\"DaemonSet\"},\n \"workload\", \"$1\", \"owner_name\", \"(.*)\"\n )\n)\n", "labels": { "workload_type": "daemonset" }, "record": "mixin_pod_workload" }, { - "expr": "sum(\n label_replace(\n kube_pod_owner{job=\"kube-state-metrics\", owner_kind=\"StatefulSet\"},\n \"workload\", \"$1\", \"owner_name\", \"(.*)\"\n )\n) by (namespace, workload, pod)\n", + "expr": "max by (cluster, namespace, workload, pod) (\n label_replace(\n kube_pod_owner{job=\"kube-state-metrics\", owner_kind=\"StatefulSet\"},\n \"workload\", \"$1\", \"owner_name\", \"(.*)\"\n )\n)\n", "labels": { "workload_type": "statefulset" }, @@ -305,23 +412,49 @@ data: "name": "node.rules", "rules": [ { - "expr": "sum(min(kube_pod_info) by (node))", + "expr": "sum(min(kube_pod_info) by (cluster, node))\n", "record": ":kube_pod_info_node_count:" }, { - "expr": "max(label_replace(kube_pod_info{job=\"kube-state-metrics\"}, \"pod\", \"$1\", \"pod\", \"(.*)\")) by (node, namespace, pod)\n", + "expr": "topk by(namespace, pod) (1,\n max by (node, namespace, pod) (\n label_replace(kube_pod_info{job=\"kube-state-metrics\"}, \"pod\", \"$1\", \"pod\", \"(.*)\")\n))\n", "record": "node_namespace_pod:kube_pod_info:" }, { - "expr": "count by (node) (sum by (node, cpu) (\n node_cpu_seconds_total{job=\"node-exporter\"}\n* on (namespace, pod) group_left(node)\n node_namespace_pod:kube_pod_info:\n))\n", + "expr": "count by (cluster, node) (sum by (node, cpu) (\n node_cpu_seconds_total{job=\"node-exporter\"}\n* on (namespace, pod) group_left(node)\n node_namespace_pod:kube_pod_info:\n))\n", "record": "node:node_num_cpu:sum" }, { - "expr": "sum(\n node_memory_MemAvailable_bytes{job=\"node-exporter\"} or\n (\n node_memory_Buffers_bytes{job=\"node-exporter\"} +\n node_memory_Cached_bytes{job=\"node-exporter\"} +\n node_memory_MemFree_bytes{job=\"node-exporter\"} +\n node_memory_Slab_bytes{job=\"node-exporter\"}\n )\n)\n", + "expr": "sum(\n node_memory_MemAvailable_bytes{job=\"node-exporter\"} or\n (\n node_memory_Buffers_bytes{job=\"node-exporter\"} +\n node_memory_Cached_bytes{job=\"node-exporter\"} +\n node_memory_MemFree_bytes{job=\"node-exporter\"} +\n node_memory_Slab_bytes{job=\"node-exporter\"}\n )\n) by (cluster)\n", "record": ":node_memory_MemAvailable_bytes:sum" } ] }, + { + "name": "kubelet.rules", + "rules": [ + { + "expr": "histogram_quantile(0.99, sum(rate(kubelet_pleg_relist_duration_seconds_bucket[5m])) by (instance, le) * on(instance) group_left(node) kubelet_node_name{job=\"kubelet\"})\n", + "labels": { + "quantile": "0.99" + }, + "record": "node_quantile:kubelet_pleg_relist_duration_seconds:histogram_quantile" + }, + { + "expr": "histogram_quantile(0.9, sum(rate(kubelet_pleg_relist_duration_seconds_bucket[5m])) by (instance, le) * on(instance) group_left(node) kubelet_node_name{job=\"kubelet\"})\n", + "labels": { + "quantile": "0.9" + }, + "record": "node_quantile:kubelet_pleg_relist_duration_seconds:histogram_quantile" + }, + { + "expr": "histogram_quantile(0.5, sum(rate(kubelet_pleg_relist_duration_seconds_bucket[5m])) by (instance, le) * on(instance) group_left(node) kubelet_node_name{job=\"kubelet\"})\n", + "labels": { + "quantile": "0.5" + }, + "record": "node_quantile:kubelet_pleg_relist_duration_seconds:histogram_quantile" + } + ] + }, { "name": "kubernetes-apps", "rules": [ @@ -343,7 +476,7 @@ data: "message": "Pod {{ $labels.namespace }}/{{ $labels.pod }} has been in a non-ready state for longer than 15 minutes.", "runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubepodnotready" }, - "expr": "sum by (namespace, pod) (kube_pod_status_phase{job=\"kube-state-metrics\", phase=~\"Failed|Pending|Unknown\"} * on(namespace, pod) group_left(owner_kind) kube_pod_owner{owner_kind!=\"Job\"}) > 0\n", + "expr": "sum by (namespace, pod) (max by(namespace, pod) (kube_pod_status_phase{job=\"kube-state-metrics\", phase=~\"Pending|Unknown\"}) * on(namespace, pod) group_left(owner_kind) max by(namespace, pod, owner_kind) (kube_pod_owner{owner_kind!=\"Job\"})) > 0\n", "for": "15m", "labels": { "severity": "critical" @@ -367,7 +500,7 @@ data: "message": "Deployment {{ $labels.namespace }}/{{ $labels.deployment }} has not matched the expected number of replicas for longer than 15 minutes.", "runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubedeploymentreplicasmismatch" }, - "expr": "kube_deployment_spec_replicas{job=\"kube-state-metrics\"}\n !=\nkube_deployment_status_replicas_available{job=\"kube-state-metrics\"}\n", + "expr": "(\n kube_deployment_spec_replicas{job=\"kube-state-metrics\"}\n !=\n kube_deployment_status_replicas_available{job=\"kube-state-metrics\"}\n) and (\n changes(kube_deployment_status_replicas_updated{job=\"kube-state-metrics\"}[5m])\n ==\n 0\n)\n", "for": "15m", "labels": { "severity": "critical" @@ -379,7 +512,7 @@ data: "message": "StatefulSet {{ $labels.namespace }}/{{ $labels.statefulset }} has not matched the expected number of replicas for longer than 15 minutes.", "runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubestatefulsetreplicasmismatch" }, - "expr": "kube_statefulset_status_replicas_ready{job=\"kube-state-metrics\"}\n !=\nkube_statefulset_status_replicas{job=\"kube-state-metrics\"}\n", + "expr": "(\n kube_statefulset_status_replicas_ready{job=\"kube-state-metrics\"}\n !=\n kube_statefulset_status_replicas{job=\"kube-state-metrics\"}\n) and (\n changes(kube_statefulset_status_replicas_updated{job=\"kube-state-metrics\"}[5m])\n ==\n 0\n)\n", "for": "15m", "labels": { "severity": "critical" @@ -528,7 +661,7 @@ data: "message": "Cluster has overcommitted CPU resource requests for Pods and cannot tolerate node failure.", "runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubecpuovercommit" }, - "expr": "sum(namespace:kube_pod_container_resource_requests_cpu_cores:sum)\n /\nsum(kube_node_status_allocatable_cpu_cores)\n >\n(count(kube_node_status_allocatable_cpu_cores)-1) / count(kube_node_status_allocatable_cpu_cores)\n", + "expr": "sum(namespace:kube_pod_container_resource_requests_cpu_cores:sum{})\n /\nsum(kube_node_status_allocatable_cpu_cores)\n >\n(count(kube_node_status_allocatable_cpu_cores)-1) / count(kube_node_status_allocatable_cpu_cores)\n", "for": "5m", "labels": { "severity": "warning" @@ -540,7 +673,7 @@ data: "message": "Cluster has overcommitted memory resource requests for Pods and cannot tolerate node failure.", "runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubememovercommit" }, - "expr": "sum(namespace:kube_pod_container_resource_requests_memory_bytes:sum)\n /\nsum(kube_node_status_allocatable_memory_bytes)\n >\n(count(kube_node_status_allocatable_memory_bytes)-1)\n /\ncount(kube_node_status_allocatable_memory_bytes)\n", + "expr": "sum(namespace:kube_pod_container_resource_requests_memory_bytes:sum{})\n /\nsum(kube_node_status_allocatable_memory_bytes)\n >\n(count(kube_node_status_allocatable_memory_bytes)-1)\n /\ncount(kube_node_status_allocatable_memory_bytes)\n", "for": "5m", "labels": { "severity": "warning" @@ -618,7 +751,7 @@ data: "runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubepersistentvolumefullinfourdays" }, "expr": "(\n kubelet_volume_stats_available_bytes{job=\"kubelet\"}\n /\n kubelet_volume_stats_capacity_bytes{job=\"kubelet\"}\n) < 0.15\nand\npredict_linear(kubelet_volume_stats_available_bytes{job=\"kubelet\"}[6h], 4 * 24 * 3600) < 0\n", - "for": "5m", + "for": "1h", "labels": { "severity": "critical" } @@ -666,17 +799,44 @@ data: } ] }, + { + "name": "kube-apiserver-error-alerts", + "rules": [ + { + "alert": "ErrorBudgetBurn", + "annotations": { + "runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-errorbudgetburn" + }, + "expr": "(\n status_class_5xx:apiserver_request_total:ratio_rate1h{job=\"apiserver\"} > (14.4*0.010000)\n and\n status_class_5xx:apiserver_request_total:ratio_rate5m{job=\"apiserver\"} > (14.4*0.010000)\n)\nor\n(\n status_class_5xx:apiserver_request_total:ratio_rate6h{job=\"apiserver\"} > (6*0.010000)\n and\n status_class_5xx:apiserver_request_total:ratio_rate30m{job=\"apiserver\"} > (6*0.010000)\n)\n", + "labels": { + "job": "apiserver", + "severity": "critical" + } + }, + { + "alert": "ErrorBudgetBurn", + "annotations": { + "runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-errorbudgetburn" + }, + "expr": "(\n status_class_5xx:apiserver_request_total:ratio_rate1d{job=\"apiserver\"} > (3*0.010000)\n and\n status_class_5xx:apiserver_request_total:ratio_rate2h{job=\"apiserver\"} > (3*0.010000)\n)\nor\n(\n status_class_5xx:apiserver_request_total:ratio_rate3d{job=\"apiserver\"} > (0.010000)\n and\n status_class_5xx:apiserver_request_total:ratio_rate6h{job=\"apiserver\"} > (0.010000)\n)\n", + "labels": { + "job": "apiserver", + "severity": "warning" + } + } + ] + }, { "name": "kubernetes-system-apiserver", "rules": [ { "alert": "KubeAPILatencyHigh", "annotations": { - "message": "The API server has a 99th percentile latency of {{ $value }} seconds for {{ $labels.verb }} {{ $labels.resource }}.", + "message": "The API server has an abnormal latency of {{ $value }} seconds for {{ $labels.verb }} {{ $labels.resource }}.", "runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubeapilatencyhigh" }, - "expr": "cluster_quantile:apiserver_request_duration_seconds:histogram_quantile{job=\"apiserver\",quantile=\"0.99\",subresource!=\"log\",verb!~\"LIST|WATCH|WATCHLIST|PROXY|CONNECT\"} > 1\n", - "for": "10m", + "expr": "(\n cluster:apiserver_request_duration_seconds:mean5m{job=\"apiserver\"}\n >\n on (verb) group_left()\n (\n avg by (verb) (cluster:apiserver_request_duration_seconds:mean5m{job=\"apiserver\"} >= 0)\n +\n 2*stddev by (verb) (cluster:apiserver_request_duration_seconds:mean5m{job=\"apiserver\"} >= 0)\n )\n) > on (verb) group_left()\n1.2 * avg by (verb) (cluster:apiserver_request_duration_seconds:mean5m{job=\"apiserver\"} >= 0)\nand on (verb,resource)\ncluster_quantile:apiserver_request_duration_seconds:histogram_quantile{job=\"apiserver\",quantile=\"0.99\"}\n>\n1\n", + "for": "5m", "labels": { "severity": "warning" } @@ -687,7 +847,7 @@ data: "message": "The API server has a 99th percentile latency of {{ $value }} seconds for {{ $labels.verb }} {{ $labels.resource }}.", "runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubeapilatencyhigh" }, - "expr": "cluster_quantile:apiserver_request_duration_seconds:histogram_quantile{job=\"apiserver\",quantile=\"0.99\",subresource!=\"log\",verb!~\"LIST|WATCH|WATCHLIST|PROXY|CONNECT\"} > 4\n", + "expr": "cluster_quantile:apiserver_request_duration_seconds:histogram_quantile{job=\"apiserver\",quantile=\"0.99\"} > 4\n", "for": "10m", "labels": { "severity": "critical" @@ -747,7 +907,7 @@ data: "message": "A client certificate used to authenticate to the apiserver is expiring in less than 7.0 days.", "runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubeclientcertificateexpiration" }, - "expr": "apiserver_client_certificate_expiration_seconds_count{job=\"apiserver\"} > 0 and histogram_quantile(0.01, sum by (job, le) (rate(apiserver_client_certificate_expiration_seconds_bucket{job=\"apiserver\"}[5m]))) < 604800\n", + "expr": "apiserver_client_certificate_expiration_seconds_count{job=\"apiserver\"} > 0 and on(job) histogram_quantile(0.01, sum by (job, le) (rate(apiserver_client_certificate_expiration_seconds_bucket{job=\"apiserver\"}[5m]))) < 604800\n", "labels": { "severity": "warning" } @@ -758,11 +918,34 @@ data: "message": "A client certificate used to authenticate to the apiserver is expiring in less than 24.0 hours.", "runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubeclientcertificateexpiration" }, - "expr": "apiserver_client_certificate_expiration_seconds_count{job=\"apiserver\"} > 0 and histogram_quantile(0.01, sum by (job, le) (rate(apiserver_client_certificate_expiration_seconds_bucket{job=\"apiserver\"}[5m]))) < 86400\n", + "expr": "apiserver_client_certificate_expiration_seconds_count{job=\"apiserver\"} > 0 and on(job) histogram_quantile(0.01, sum by (job, le) (rate(apiserver_client_certificate_expiration_seconds_bucket{job=\"apiserver\"}[5m]))) < 86400\n", "labels": { "severity": "critical" } }, + { + "alert": "AggregatedAPIErrors", + "annotations": { + "message": "An aggregated API {{ $labels.name }}/{{ $labels.namespace }} has reported errors. The number of errors have increased for it in the past five minutes. High values indicate that the availability of the service changes too often.", + "runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-aggregatedapierrors" + }, + "expr": "sum by(name, namespace)(increase(aggregator_unavailable_apiservice_count[5m])) > 2\n", + "labels": { + "severity": "warning" + } + }, + { + "alert": "AggregatedAPIDown", + "annotations": { + "message": "An aggregated API {{ $labels.name }}/{{ $labels.namespace }} is down. It has not been available at least for the past five minutes.", + "runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-aggregatedapidown" + }, + "expr": "sum by(name, namespace)(sum_over_time(aggregator_unavailable_apiservice[5m])) > 0\n", + "for": "5m", + "labels": { + "severity": "warning" + } + }, { "alert": "KubeAPIDown", "annotations": { @@ -799,6 +982,7 @@ data: "runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubenodeunreachable" }, "expr": "kube_node_spec_taint{job=\"kube-state-metrics\",key=\"node.kubernetes.io/unreachable\",effect=\"NoSchedule\"} == 1\n", + "for": "2m", "labels": { "severity": "warning" } @@ -815,6 +999,42 @@ data: "severity": "warning" } }, + { + "alert": "KubeNodeReadinessFlapping", + "annotations": { + "message": "The readiness status of node {{ $labels.node }} has changed {{ $value }} times in the last 15 minutes.", + "runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubenodereadinessflapping" + }, + "expr": "sum(changes(kube_node_status_condition{status=\"true\",condition=\"Ready\"}[15m])) by (node) > 2\n", + "for": "15m", + "labels": { + "severity": "warning" + } + }, + { + "alert": "KubeletPlegDurationHigh", + "annotations": { + "message": "The Kubelet Pod Lifecycle Event Generator has a 99th percentile duration of {{ $value }} seconds on node {{ $labels.node }}.", + "runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubeletplegdurationhigh" + }, + "expr": "node_quantile:kubelet_pleg_relist_duration_seconds:histogram_quantile{quantile=\"0.99\"} >= 10\n", + "for": "5m", + "labels": { + "severity": "warning" + } + }, + { + "alert": "KubeletPodStartUpLatencyHigh", + "annotations": { + "message": "Kubelet Pod startup 99th percentile latency is {{ $value }} seconds on node {{ $labels.node }}.", + "runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubeletpodstartuplatencyhigh" + }, + "expr": "histogram_quantile(0.99, sum(rate(kubelet_pod_worker_duration_seconds_bucket{job=\"kubelet\"}[5m])) by (instance, le)) * on(instance) group_left(node) kubelet_node_name > 5\n", + "for": "15m", + "labels": { + "severity": "warning" + } + }, { "alert": "KubeletDown", "annotations": { @@ -1124,7 +1344,7 @@ data: { "alert": "PrometheusRemoteStorageFailures", "annotations": { - "description": "Prometheus {{$labels.instance}} failed to send {{ printf \"%.1f\" $value }}% of the samples to queue {{$labels.queue}}.", + "description": "Prometheus {{$labels.instance}} failed to send {{ printf \"%.1f\" $value }}% of the samples to {{ if $labels.queue }}{{ $labels.queue }}{{ else }}{{ $labels.url }}{{ end }}.", "summary": "Prometheus fails to send samples to remote storage." }, "expr": "(\n rate(prometheus_remote_storage_failed_samples_total{job=\"prometheus\"}[5m])\n/\n (\n rate(prometheus_remote_storage_failed_samples_total{job=\"prometheus\"}[5m])\n +\n rate(prometheus_remote_storage_succeeded_samples_total{job=\"prometheus\"}[5m])\n )\n)\n* 100\n> 1\n", @@ -1136,7 +1356,7 @@ data: { "alert": "PrometheusRemoteWriteBehind", "annotations": { - "description": "Prometheus {{$labels.instance}} remote write is {{ printf \"%.1f\" $value }}s behind for queue {{$labels.queue}}.", + "description": "Prometheus {{$labels.instance}} remote write is {{ printf \"%.1f\" $value }}s behind for {{ if $labels.queue }}{{ $labels.queue }}{{ else }}{{ $labels.url }}{{ end }}.", "summary": "Prometheus remote write is behind." }, "expr": "# Without max_over_time, failed scrapes could create false negatives, see\n# https://www.robustperception.io/alerting-on-gauges-in-prometheus-2-0 for details.\n(\n max_over_time(prometheus_remote_storage_highest_timestamp_in_seconds{job=\"prometheus\"}[5m])\n- on(job, instance) group_right\n max_over_time(prometheus_remote_storage_queue_highest_sent_timestamp_seconds{job=\"prometheus\"}[5m])\n)\n> 120\n", @@ -1148,10 +1368,10 @@ data: { "alert": "PrometheusRemoteWriteDesiredShards", "annotations": { - "description": "Prometheus {{$labels.instance}} remote write desired shards calculation wants to run {{ printf $value }} shards, which is more than the max of {{ printf `prometheus_remote_storage_shards_max{instance=\"%s\",job=\"prometheus\"}` $labels.instance | query | first | value }}.", + "description": "Prometheus {{$labels.instance}} remote write desired shards calculation wants to run {{ $value }} shards, which is more than the max of {{ printf `prometheus_remote_storage_shards_max{instance=\"%s\",job=\"prometheus\"}` $labels.instance | query | first | value }}.", "summary": "Prometheus remote write desired shards calculation wants to run more than configured max shards." }, - "expr": "# Without max_over_time, failed scrapes could create false negatives, see\n# https://www.robustperception.io/alerting-on-gauges-in-prometheus-2-0 for details.\n(\n max_over_time(prometheus_remote_storage_shards_desired{job=\"prometheus\"}[5m])\n> on(job, instance) group_right\n max_over_time(prometheus_remote_storage_shards_max{job=\"prometheus\"}[5m])\n)\n", + "expr": "# Without max_over_time, failed scrapes could create false negatives, see\n# https://www.robustperception.io/alerting-on-gauges-in-prometheus-2-0 for details.\n(\n max_over_time(prometheus_remote_storage_shards_desired{job=\"prometheus\"}[5m])\n>\n max_over_time(prometheus_remote_storage_shards_max{job=\"prometheus\"}[5m])\n)\n", "for": "15m", "labels": { "severity": "warning" @@ -1201,6 +1421,17 @@ data: "labels": { "severity": "warning" } + }, + { + "alert": "BlackboxProbeFailure", + "annotations": { + "message": "Blackbox probe {{$labels.instance}} failed" + }, + "expr": "probe_success == 0", + "for": "2m", + "labels": { + "severity": "critical" + } } ] }, From 7b0ea23cdc4fec2d68d91067585c7bddaf591d1d Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Sat, 7 Mar 2020 18:40:39 -0800 Subject: [PATCH 351/523] Upgrade terraform-provider-azurerm to v2.0+ * Add support for `terraform-provider-azurerm` v2.0+. Require `terraform-provider-azurerm` v2.0+ and drop v1.x support since the Azure provider major release is not backwards compatible * Use Azure's new Linux VM and Linux VM Scale Set resources * Change controller's Azure disk caching to None * Associate subnets (in addition to NICs) with security groups (aesthetic) * If set, change `worker_priority` from `Low` to `Spot` (action required) Related: * https://www.terraform.io/docs/providers/azurerm/guides/2.0-upgrade-guide.html --- CHANGES.md | 8 ++ .../container-linux/kubernetes/controllers.tf | 102 +++++++++--------- azure/container-linux/kubernetes/network.tf | 10 ++ azure/container-linux/kubernetes/ssh.tf | 2 +- azure/container-linux/kubernetes/versions.tf | 2 +- azure/container-linux/kubernetes/workers.tf | 1 - .../kubernetes/workers/workers.tf | 57 ++++------ docs/cl/azure.md | 8 +- 8 files changed, 94 insertions(+), 96 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 71da23729..265bd8af3 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -14,6 +14,14 @@ Notable changes between versions. * Fix `worker_node_labels` on Fedora CoreOS ([#651](https://github.com/poseidon/typhoon/pull/651)) * Fix automatic worker node delete on shutdown on Fedora CoreOS ([#657](https://github.com/poseidon/typhoon/pull/657)) +#### Azure + +* Upgrade to `terraform-provider-azurerm` [v2.0+](https://www.terraform.io/docs/providers/azurerm/guides/2.0-upgrade-guide.html) (action required) + * Switch to Azure's new Linux VM and Linux VM Scale Set resources + * If set, change `worker_priority` from `Low` to `Spot` (action required) + * Set controller's Azure disk caching to None + * Associate subnets (in addition to NICs) with security groups (aesthetic) + #### Google Cloud * Fix `worker_node_labels` on Fedora CoreOS ([#651](https://github.com/poseidon/typhoon/pull/651)) diff --git a/azure/container-linux/kubernetes/controllers.tf b/azure/container-linux/kubernetes/controllers.tf index 6012ad336..ea45c93ad 100644 --- a/azure/container-linux/kubernetes/controllers.tf +++ b/azure/container-linux/kubernetes/controllers.tf @@ -32,84 +32,89 @@ resource "azurerm_availability_set" "controllers" { } # Controller instances -resource "azurerm_virtual_machine" "controllers" { +resource "azurerm_linux_virtual_machine" "controllers" { count = var.controller_count resource_group_name = azurerm_resource_group.cluster.name name = "${var.cluster_name}-controller-${count.index}" location = var.region availability_set_id = azurerm_availability_set.controllers.id - vm_size = var.controller_type - # boot - storage_image_reference { + size = var.controller_type + custom_data = base64encode(data.ct_config.controller-ignitions.*.rendered[count.index]) + + # storage + os_disk { + name = "${var.cluster_name}-controller-${count.index}" + caching = "None" + disk_size_gb = var.disk_size + storage_account_type = "Premium_LRS" + } + + source_image_reference { publisher = "CoreOS" offer = "CoreOS" sku = local.channel version = "latest" } - # storage - storage_os_disk { - name = "${var.cluster_name}-controller-${count.index}" - create_option = "FromImage" - caching = "ReadWrite" - disk_size_gb = var.disk_size - os_type = "Linux" - managed_disk_type = "Premium_LRS" - } - # network - network_interface_ids = [azurerm_network_interface.controllers.*.id[count.index]] - - os_profile { - computer_name = "${var.cluster_name}-controller-${count.index}" - admin_username = "core" - custom_data = data.ct_config.controller-ignitions.*.rendered[count.index] - } - - # Azure mandates setting an ssh_key, even though Ignition custom_data handles it too - os_profile_linux_config { - disable_password_authentication = true - - ssh_keys { - path = "/home/core/.ssh/authorized_keys" - key_data = var.ssh_authorized_key - } + network_interface_ids = [ + azurerm_network_interface.controllers.*.id[count.index] + ] + + # Azure requires setting admin_ssh_key, though Ignition custom_data handles it too + admin_username = "core" + admin_ssh_key { + username = "core" + public_key = var.ssh_authorized_key } - # lifecycle - delete_os_disk_on_termination = true - delete_data_disks_on_termination = true - lifecycle { ignore_changes = [ - storage_os_disk, - os_profile, + os_disk, + custom_data, ] } } +# Controller public IPv4 addresses +resource "azurerm_public_ip" "controllers" { + count = var.controller_count + resource_group_name = azurerm_resource_group.cluster.name + + name = "${var.cluster_name}-controller-${count.index}" + location = azurerm_resource_group.cluster.location + sku = "Standard" + allocation_method = "Static" +} + # Controller NICs with public and private IPv4 resource "azurerm_network_interface" "controllers" { count = var.controller_count resource_group_name = azurerm_resource_group.cluster.name - name = "${var.cluster_name}-controller-${count.index}" - location = azurerm_resource_group.cluster.location - network_security_group_id = azurerm_network_security_group.controller.id + name = "${var.cluster_name}-controller-${count.index}" + location = azurerm_resource_group.cluster.location ip_configuration { name = "ip0" subnet_id = azurerm_subnet.controller.id - private_ip_address_allocation = "dynamic" - - # public IPv4 + private_ip_address_allocation = "Dynamic" + # instance public IPv4 public_ip_address_id = azurerm_public_ip.controllers.*.id[count.index] } } -# Add controller NICs to the controller backend address pool +# Associate controller network interface with controller security group +resource "azurerm_network_interface_security_group_association" "controllers" { + count = var.controller_count + + network_interface_id = azurerm_network_interface.controllers[count.index].id + network_security_group_id = azurerm_network_security_group.controller.id +} + +# Associate controller network interface with controller backend address pool resource "azurerm_network_interface_backend_address_pool_association" "controllers" { count = var.controller_count @@ -118,17 +123,6 @@ resource "azurerm_network_interface_backend_address_pool_association" "controlle backend_address_pool_id = azurerm_lb_backend_address_pool.controller.id } -# Controller public IPv4 addresses -resource "azurerm_public_ip" "controllers" { - count = var.controller_count - resource_group_name = azurerm_resource_group.cluster.name - - name = "${var.cluster_name}-controller-${count.index}" - location = azurerm_resource_group.cluster.location - sku = "Standard" - allocation_method = "Static" -} - # Controller Ignition configs data "ct_config" "controller-ignitions" { count = var.controller_count diff --git a/azure/container-linux/kubernetes/network.tf b/azure/container-linux/kubernetes/network.tf index aa9157b0b..ea92a5a7d 100644 --- a/azure/container-linux/kubernetes/network.tf +++ b/azure/container-linux/kubernetes/network.tf @@ -24,6 +24,11 @@ resource "azurerm_subnet" "controller" { address_prefix = cidrsubnet(var.host_cidr, 1, 0) } +resource "azurerm_subnet_network_security_group_association" "controller" { + subnet_id = azurerm_subnet.controller.id + network_security_group_id = azurerm_network_security_group.controller.id +} + resource "azurerm_subnet" "worker" { resource_group_name = azurerm_resource_group.cluster.name @@ -32,3 +37,8 @@ resource "azurerm_subnet" "worker" { address_prefix = cidrsubnet(var.host_cidr, 1, 1) } +resource "azurerm_subnet_network_security_group_association" "worker" { + subnet_id = azurerm_subnet.worker.id + network_security_group_id = azurerm_network_security_group.worker.id +} + diff --git a/azure/container-linux/kubernetes/ssh.tf b/azure/container-linux/kubernetes/ssh.tf index 5ff47b35f..04f54393e 100644 --- a/azure/container-linux/kubernetes/ssh.tf +++ b/azure/container-linux/kubernetes/ssh.tf @@ -13,7 +13,7 @@ resource "null_resource" "copy-controller-secrets" { depends_on = [ module.bootstrap, - azurerm_virtual_machine.controllers + azurerm_linux_virtual_machine.controllers ] connection { diff --git a/azure/container-linux/kubernetes/versions.tf b/azure/container-linux/kubernetes/versions.tf index 098f81942..f9653cab3 100644 --- a/azure/container-linux/kubernetes/versions.tf +++ b/azure/container-linux/kubernetes/versions.tf @@ -3,7 +3,7 @@ terraform { required_version = "~> 0.12.6" required_providers { - azurerm = "~> 1.27" + azurerm = "~> 2.0" ct = "~> 0.3" template = "~> 2.1" null = "~> 2.1" diff --git a/azure/container-linux/kubernetes/workers.tf b/azure/container-linux/kubernetes/workers.tf index 11b77c50f..a99db98dd 100644 --- a/azure/container-linux/kubernetes/workers.tf +++ b/azure/container-linux/kubernetes/workers.tf @@ -22,4 +22,3 @@ module "workers" { clc_snippets = var.worker_clc_snippets node_labels = var.worker_node_labels } - diff --git a/azure/container-linux/kubernetes/workers/workers.tf b/azure/container-linux/kubernetes/workers/workers.tf index 7acdf63e9..ea41a0cac 100644 --- a/azure/container-linux/kubernetes/workers/workers.tf +++ b/azure/container-linux/kubernetes/workers/workers.tf @@ -5,53 +5,40 @@ locals { } # Workers scale set -resource "azurerm_virtual_machine_scale_set" "workers" { +resource "azurerm_linux_virtual_machine_scale_set" "workers" { resource_group_name = var.resource_group_name - name = "${var.name}-workers" + name = "${var.name}-worker" location = var.region + sku = var.vm_type + instances = var.worker_count + # instance name prefix for instances in the set + computer_name_prefix = "${var.name}-worker" single_placement_group = false + custom_data = base64encode(data.ct_config.worker-ignition.rendered) - sku { - name = var.vm_type - tier = "standard" - capacity = var.worker_count + # storage + os_disk { + storage_account_type = "Standard_LRS" + caching = "ReadWrite" } - # boot - storage_profile_image_reference { + source_image_reference { publisher = "CoreOS" offer = "CoreOS" sku = local.channel version = "latest" } - # storage - storage_profile_os_disk { - create_option = "FromImage" - caching = "ReadWrite" - os_type = "linux" - managed_disk_type = "Standard_LRS" - } - - os_profile { - computer_name_prefix = "${var.name}-worker-" - admin_username = "core" - custom_data = data.ct_config.worker-ignition.rendered - } - - # Azure mandates setting an ssh_key, even though Ignition custom_data handles it too - os_profile_linux_config { - disable_password_authentication = true - - ssh_keys { - path = "/home/core/.ssh/authorized_keys" - key_data = var.ssh_authorized_key - } + # Azure requires setting admin_ssh_key, though Ignition custom_data handles it too + admin_username = "core" + admin_ssh_key { + username = "core" + public_key = var.ssh_authorized_key } # network - network_profile { + network_interface { name = "nic0" primary = true network_security_group_id = var.security_group_id @@ -67,10 +54,10 @@ resource "azurerm_virtual_machine_scale_set" "workers" { } # lifecycle - upgrade_policy_mode = "Manual" - # eviction policy may only be set when priority is Low + upgrade_mode = "Manual" + # eviction policy may only be set when priority is Spot priority = var.priority - eviction_policy = var.priority == "Low" ? "Delete" : null + eviction_policy = var.priority == "Spot" ? "Delete" : null } # Scale up or down to maintain desired number, tolerating deallocations. @@ -82,7 +69,7 @@ resource "azurerm_monitor_autoscale_setting" "workers" { # autoscale enabled = true - target_resource_id = azurerm_virtual_machine_scale_set.workers.id + target_resource_id = azurerm_linux_virtual_machine_scale_set.workers.id profile { name = "default" diff --git a/docs/cl/azure.md b/docs/cl/azure.md index 3bf311c9c..0ece179a5 100644 --- a/docs/cl/azure.md +++ b/docs/cl/azure.md @@ -50,7 +50,7 @@ Configure the Azure provider in a `providers.tf` file. ```tf provider "azurerm" { - version = "1.43.0" + version = "2.0.0" } provider "ct" { @@ -225,7 +225,7 @@ Reference the DNS zone with `azurerm_dns_zone.clusters.name` and its resource gr | worker_type | Machine type for workers | "Standard_DS1_v2" | See below | | os_image | Channel for a Container Linux derivative | "coreos-stable" | coreos-stable, coreos-beta, coreos-alpha | | disk_size | Size of the disk in GB | 40 | 100 | -| worker_priority | Set priority to Low to use reduced cost surplus capacity, with the tradeoff that instances can be deallocated at any time | Regular | Low | +| worker_priority | Set priority to Spot to use reduced cost surplus capacity, with the tradeoff that instances can be deallocated at any time | Regular | Spot | | controller_clc_snippets | Controller Container Linux Config snippets | [] | [example](/advanced/customization/#usage) | | worker_clc_snippets | Worker Container Linux Config snippets | [] | [example](/advanced/customization/#usage) | | networking | Choice of networking provider | "calico" | "flannel" or "calico" | @@ -242,6 +242,6 @@ Check the list of valid [machine types](https://azure.microsoft.com/en-us/pricin !!! warning Do not choose a `controller_type` smaller than `Standard_B2s`. Smaller instances are not sufficient for running a controller. -#### Low Priority +#### Spot Priority -Add `worker_priority=Low` to use [Low Priority](https://docs.microsoft.com/en-us/azure/virtual-machine-scale-sets/virtual-machine-scale-sets-use-low-priority) workers that run on Azure's surplus capacity at lower cost, but with the tradeoff that they can be deallocated at random. Low priority VMs are Azure's analog to AWS spot instances or GCP premptible instances. +Add `worker_priority=Spot` to use [Spot Priority](https://docs.microsoft.com/en-us/azure/virtual-machines/linux/spot-vms) workers that run on Azure's surplus capacity at lower cost, but with the tradeoff that they can be deallocated at random. Spot priority VMs are Azure's analog to AWS spot instances or GCP premptible instances. From ab7913a0613abdb03bde00bdca1ec385d144380c Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Sun, 8 Mar 2020 20:39:18 -0700 Subject: [PATCH 352/523] Accept initial worker node labels and taints map on bare-metal * Add `worker_node_labels` map from node name to a list of initial node label strings * Add `worker_node_taints` map from node name to a list of initial node taint strings * Unlike cloud platforms, bare-metal node labels and taints are defined via a map from node name to list of labels/taints. Bare-metal clusters may have heterogeneous hardware so per node labels and taints are accepted * Only worker node names are allowed. Workloads are not scheduled on controller nodes so altering their labels/taints isn't suitable ``` module "mercury" { ... worker_node_labels = { "node2" = ["role=special"] } worker_node_taints = { "node2" = ["role=special:NoSchedule"] } } ``` Related: https://github.com/poseidon/typhoon/issues/429 --- CHANGES.md | 7 ++++++- bare-metal/container-linux/kubernetes/cl/worker.yaml | 6 ++++++ bare-metal/container-linux/kubernetes/profiles.tf | 2 ++ bare-metal/container-linux/kubernetes/variables.tf | 12 ++++++++++++ bare-metal/fedora-coreos/kubernetes/fcc/worker.yaml | 6 ++++++ bare-metal/fedora-coreos/kubernetes/profiles.tf | 2 ++ bare-metal/fedora-coreos/kubernetes/variables.tf | 12 ++++++++++++ docs/cl/bare-metal.md | 2 ++ docs/fedora-coreos/bare-metal.md | 2 ++ 9 files changed, 50 insertions(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index 265bd8af3..5c22af18e 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -17,11 +17,16 @@ Notable changes between versions. #### Azure * Upgrade to `terraform-provider-azurerm` [v2.0+](https://www.terraform.io/docs/providers/azurerm/guides/2.0-upgrade-guide.html) (action required) + * Change `worker_priority` from `Low` to `Spot` if used (action required) * Switch to Azure's new Linux VM and Linux VM Scale Set resources - * If set, change `worker_priority` from `Low` to `Spot` (action required) * Set controller's Azure disk caching to None * Associate subnets (in addition to NICs) with security groups (aesthetic) +#### Bare-Metal + +* Add `worker_node_labels` map variable for per-worker node labels ([#663](https://github.com/poseidon/typhoon/pull/663)) +* Add `worker_node_taints` map variable for per-worker node taints ([#663](https://github.com/poseidon/typhoon/pull/663)) + #### Google Cloud * Fix `worker_node_labels` on Fedora CoreOS ([#651](https://github.com/poseidon/typhoon/pull/651)) diff --git a/bare-metal/container-linux/kubernetes/cl/worker.yaml b/bare-metal/container-linux/kubernetes/cl/worker.yaml index 8f6e984ff..8ffb1d2e8 100644 --- a/bare-metal/container-linux/kubernetes/cl/worker.yaml +++ b/bare-metal/container-linux/kubernetes/cl/worker.yaml @@ -93,6 +93,12 @@ systemd: --lock-file=/var/run/lock/kubelet.lock \ --network-plugin=cni \ --node-labels=node.kubernetes.io/node \ + %{~ for label in compact(split(",", node_labels)) ~} + --node-labels=${label} \ + %{~ endfor ~} + %{~ for taint in compact(split(",", node_taints)) ~} + --register-with-taints=${taint} \ + %{~ endfor ~} --pod-manifest-path=/etc/kubernetes/manifests \ --read-only-port=0 \ --volume-plugin-dir=/var/lib/kubelet/volumeplugins diff --git a/bare-metal/container-linux/kubernetes/profiles.tf b/bare-metal/container-linux/kubernetes/profiles.tf index 6bbaa1d5f..98efdcc36 100644 --- a/bare-metal/container-linux/kubernetes/profiles.tf +++ b/bare-metal/container-linux/kubernetes/profiles.tf @@ -188,6 +188,8 @@ data "template_file" "worker-configs" { cluster_dns_service_ip = module.bootstrap.cluster_dns_service_ip cluster_domain_suffix = var.cluster_domain_suffix ssh_authorized_key = var.ssh_authorized_key + node_labels = join(",", lookup(var.worker_node_labels, var.workers.*.name[count.index], [])) + node_taints = join(",", lookup(var.worker_node_taints, var.workers.*.name[count.index], [])) } } diff --git a/bare-metal/container-linux/kubernetes/variables.tf b/bare-metal/container-linux/kubernetes/variables.tf index 80dd70bcf..27b8ac8e7 100644 --- a/bare-metal/container-linux/kubernetes/variables.tf +++ b/bare-metal/container-linux/kubernetes/variables.tf @@ -55,6 +55,18 @@ variable "clc_snippets" { default = {} } +variable "worker_node_labels" { + type = map(list(string)) + description = "Map from worker names to lists of initial node labels" + default = {} +} + +variable "worker_node_taints" { + type = map(list(string)) + description = "Map from worker names to lists of initial node taints" + default = {} +} + # configuration variable "k8s_domain_name" { diff --git a/bare-metal/fedora-coreos/kubernetes/fcc/worker.yaml b/bare-metal/fedora-coreos/kubernetes/fcc/worker.yaml index c0e09b58c..d1d5e8aa8 100644 --- a/bare-metal/fedora-coreos/kubernetes/fcc/worker.yaml +++ b/bare-metal/fedora-coreos/kubernetes/fcc/worker.yaml @@ -68,6 +68,12 @@ systemd: --lock-file=/var/run/lock/kubelet.lock \ --network-plugin=cni \ --node-labels=node.kubernetes.io/node \ + %{~ for label in compact(split(",", node_labels)) ~} + --node-labels=${label} \ + %{~ endfor ~} + %{~ for taint in compact(split(",", node_taints)) ~} + --register-with-taints=${taint} \ + %{~ endfor ~} --pod-manifest-path=/etc/kubernetes/manifests \ --read-only-port=0 \ --volume-plugin-dir=/var/lib/kubelet/volumeplugins diff --git a/bare-metal/fedora-coreos/kubernetes/profiles.tf b/bare-metal/fedora-coreos/kubernetes/profiles.tf index fefed8852..8d75b8307 100644 --- a/bare-metal/fedora-coreos/kubernetes/profiles.tf +++ b/bare-metal/fedora-coreos/kubernetes/profiles.tf @@ -96,6 +96,8 @@ data "template_file" "worker-configs" { cluster_dns_service_ip = module.bootstrap.cluster_dns_service_ip cluster_domain_suffix = var.cluster_domain_suffix ssh_authorized_key = var.ssh_authorized_key + node_labels = join(",", lookup(var.worker_node_labels, var.workers.*.name[count.index], [])) + node_taints = join(",", lookup(var.worker_node_taints, var.workers.*.name[count.index], [])) } } diff --git a/bare-metal/fedora-coreos/kubernetes/variables.tf b/bare-metal/fedora-coreos/kubernetes/variables.tf index 8e956754b..45fc0341c 100644 --- a/bare-metal/fedora-coreos/kubernetes/variables.tf +++ b/bare-metal/fedora-coreos/kubernetes/variables.tf @@ -56,6 +56,18 @@ variable "snippets" { default = {} } +variable "worker_node_labels" { + type = map(list(string)) + description = "Map from worker names to lists of initial node labels" + default = {} +} + +variable "worker_node_taints" { + type = map(list(string)) + description = "Map from worker names to lists of initial node taints" + default = {} +} + # configuration variable "k8s_domain_name" { diff --git a/docs/cl/bare-metal.md b/docs/cl/bare-metal.md index 775870271..5cf4f7775 100644 --- a/docs/cl/bare-metal.md +++ b/docs/cl/bare-metal.md @@ -361,4 +361,6 @@ Check the [variables.tf](https://github.com/poseidon/typhoon/blob/master/bare-me | pod_cidr | CIDR IPv4 range to assign to Kubernetes pods | "10.2.0.0/16" | "10.22.0.0/16" | | service_cidr | CIDR IPv4 range to assign to Kubernetes services | "10.3.0.0/16" | "10.3.0.0/24" | | kernel_args | Additional kernel args to provide at PXE boot | [] | ["kvm-intel.nested=1"] | +| worker_node_labels | Map from worker name to list of initial node labels | {} | {"node2" = ["role=special"]} | +| worker_node_taints | Map from worker name to list of initial node taints | {} | {"node2" = ["role=special:NoSchedule"]} | diff --git a/docs/fedora-coreos/bare-metal.md b/docs/fedora-coreos/bare-metal.md index 42974f6fd..c95fbf3d5 100644 --- a/docs/fedora-coreos/bare-metal.md +++ b/docs/fedora-coreos/bare-metal.md @@ -347,4 +347,6 @@ Check the [variables.tf](https://github.com/poseidon/typhoon/blob/master/bare-me | pod_cidr | CIDR IPv4 range to assign to Kubernetes pods | "10.2.0.0/16" | "10.22.0.0/16" | | service_cidr | CIDR IPv4 range to assign to Kubernetes services | "10.3.0.0/16" | "10.3.0.0/24" | | kernel_args | Additional kernel args to provide at PXE boot | [] | ["kvm-intel.nested=1"] | +| worker_node_labels | Map from worker name to list of initial node labels | {} | {"node2" = ["role=special"]} | +| worker_node_taints | Map from worker name to list of initial node taints | {} | {"node2" = ["role=special:NoSchedule"]} | From 4e1b8f22dff92e80e002f6a1dc5e45496051f6ab Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Tue, 10 Mar 2020 23:55:23 -0700 Subject: [PATCH 353/523] Add support for Flatcar Linux on Azure * Accept `os_image` "flatcar-stable" and "flatcar-beta" to use Kinvolk's Flatcar Linux images from the Azure Marketplace Note: Flatcar Linux Azure Marketplace images require terms be accepted before use --- CHANGES.md | 2 ++ README.md | 1 + .../container-linux/kubernetes/controllers.tf | 19 ++++++++++++++++--- azure/container-linux/kubernetes/variables.tf | 2 +- .../kubernetes/workers/workers.tf | 18 +++++++++++++++--- docs/cl/azure.md | 11 ++++++++++- docs/index.md | 1 + 7 files changed, 46 insertions(+), 8 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 5c22af18e..570ccd768 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -21,6 +21,8 @@ Notable changes between versions. * Switch to Azure's new Linux VM and Linux VM Scale Set resources * Set controller's Azure disk caching to None * Associate subnets (in addition to NICs) with security groups (aesthetic) +* Add support for Flatcar Container Linux ([#664](https://github.com/poseidon/typhoon/pull/664)) + * Requires accepting Flatcar Linux Azure Marketplace terms #### Bare-Metal diff --git a/README.md b/README.md index 69b987f7a..73bc89caf 100644 --- a/README.md +++ b/README.md @@ -42,6 +42,7 @@ Typhoon is available for [Flatcar Container Linux](https://www.flatcar-linux.org | Platform | Operating System | Terraform Module | Status | |---------------|------------------|------------------|--------| | AWS | Flatcar Linux | [aws/container-linux/kubernetes](aws/container-linux/kubernetes) | stable | +| Azure | Flatcar Linux | [azure/container-linux/kubernetes](azure/container-linux/kubernetes) | alpha | | Bare-Metal | Flatcar Linux | [bare-metal/container-linux/kubernetes](bare-metal/container-linux/kubernetes) | stable | | Google Cloud | Flatcar Linux | [google-cloud/container-linux/kubernetes](google-cloud/container-linux/kubernetes) | alpha | | Digital Ocean | Flatcar Linux | [digital-ocean/container-linux/kubernetes](digital-ocean/container-linux/kubernetes) | alpha | diff --git a/azure/container-linux/kubernetes/controllers.tf b/azure/container-linux/kubernetes/controllers.tf index ea45c93ad..f535ffcd0 100644 --- a/azure/container-linux/kubernetes/controllers.tf +++ b/azure/container-linux/kubernetes/controllers.tf @@ -15,8 +15,10 @@ resource "azurerm_dns_a_record" "etcds" { } locals { - # Channel for a Container Linux derivative + # Container Linux derivative # coreos-stable -> Container Linux Stable + # flatcar-stable -> Flatcar Linux Stable + flavor = split("-", var.os_image)[0] channel = split("-", var.os_image)[1] } @@ -52,12 +54,23 @@ resource "azurerm_linux_virtual_machine" "controllers" { } source_image_reference { - publisher = "CoreOS" - offer = "CoreOS" + publisher = local.flavor == "flatcar" ? "Kinvolk" : "CoreOS" + offer = local.flavor == "flatcar" ? "flatcar-container-linux" : "CoreOS" sku = local.channel version = "latest" } + # Gross hack just for Flatcar Linux + dynamic "plan" { + for_each = local.flavor == "flatcar" ? [1] : [] + + content { + name = local.channel + publisher = "kinvolk" + product = "flatcar-container-linux" + } + } + # network network_interface_ids = [ azurerm_network_interface.controllers.*.id[count.index] diff --git a/azure/container-linux/kubernetes/variables.tf b/azure/container-linux/kubernetes/variables.tf index c86ab9aa1..7200e82eb 100644 --- a/azure/container-linux/kubernetes/variables.tf +++ b/azure/container-linux/kubernetes/variables.tf @@ -49,7 +49,7 @@ variable "worker_type" { variable "os_image" { type = string default = "coreos-stable" - description = "Channel for a Container Linux derivative (coreos-stable, coreos-beta, coreos-alpha)" + description = "Channel for a Container Linux derivative (coreos-stable, coreos-beta, coreos-alpha, flatcar-stable, flatcar-beta)" } variable "disk_size" { diff --git a/azure/container-linux/kubernetes/workers/workers.tf b/azure/container-linux/kubernetes/workers/workers.tf index ea41a0cac..1a102bd56 100644 --- a/azure/container-linux/kubernetes/workers/workers.tf +++ b/azure/container-linux/kubernetes/workers/workers.tf @@ -1,6 +1,7 @@ locals { - # Channel for a Container Linux derivative # coreos-stable -> Container Linux Stable + # flatcar-stable -> Flatcar Linux Stable + flavor = split("-", var.os_image)[0] channel = split("-", var.os_image)[1] } @@ -24,12 +25,23 @@ resource "azurerm_linux_virtual_machine_scale_set" "workers" { } source_image_reference { - publisher = "CoreOS" - offer = "CoreOS" + publisher = local.flavor == "flatcar" ? "Kinvolk" : "CoreOS" + offer = local.flavor == "flatcar" ? "flatcar-container-linux" : "CoreOS" sku = local.channel version = "latest" } + # Gross hack just for Flatcar Linux + dynamic "plan" { + for_each = local.flavor == "flatcar" ? [1] : [] + + content { + name = local.channel + publisher = "kinvolk" + product = "flatcar-container-linux" + } + } + # Azure requires setting admin_ssh_key, though Ignition custom_data handles it too admin_username = "core" admin_ssh_key { diff --git a/docs/cl/azure.md b/docs/cl/azure.md index 0ece179a5..deb4b0a1d 100644 --- a/docs/cl/azure.md +++ b/docs/cl/azure.md @@ -85,6 +85,15 @@ module "ramius" { Reference the [variables docs](#variables) or the [variables.tf](https://github.com/poseidon/typhoon/blob/master/azure/container-linux/kubernetes/variables.tf) source. +### Flatcar Linux Only + +Flatcar Linux publishes images to the Azure Marketplace and requires accepting their legal terms. + +``` +az vm image terms show --publish kinvolk --offer flatcar-container-linux --plan stable +az vm image terms accept --publish kinvolk --offer flatcar-container-linux --plan stable +``` + ## ssh-agent Initial bootstrapping requires `bootstrap.service` be started on one controller node. Terraform uses `ssh-agent` to automate this step. Add your SSH private key to `ssh-agent`. @@ -223,7 +232,7 @@ Reference the DNS zone with `azurerm_dns_zone.clusters.name` and its resource gr | worker_count | Number of workers | 1 | 3 | | controller_type | Machine type for controllers | "Standard_B2s" | See below | | worker_type | Machine type for workers | "Standard_DS1_v2" | See below | -| os_image | Channel for a Container Linux derivative | "coreos-stable" | coreos-stable, coreos-beta, coreos-alpha | +| os_image | Channel for a Container Linux derivative | "coreos-stable" | coreos-stable, coreos-beta, coreos-alpha, flatcar-stable, flatcar-beta | | disk_size | Size of the disk in GB | 40 | 100 | | worker_priority | Set priority to Spot to use reduced cost surplus capacity, with the tradeoff that instances can be deallocated at any time | Regular | Spot | | controller_clc_snippets | Controller Container Linux Config snippets | [] | [example](/advanced/customization/#usage) | diff --git a/docs/index.md b/docs/index.md index 8122b0556..22ec89311 100644 --- a/docs/index.md +++ b/docs/index.md @@ -42,6 +42,7 @@ Typhoon is available for [Flatcar Container Linux](https://www.flatcar-linux.org | Platform | Operating System | Terraform Module | Status | |---------------|------------------|------------------|--------| | AWS | Flatcar Linux | [aws/container-linux/kubernetes](cl/aws.md) | stable | +| Azure | Flatcar Linux | [azure/container-linux/kubernetes](cl/azure.md) | alpha | | Bare-Metal | Flatcar Linux | [bare-metal/container-linux/kubernetes](cl/bare-metal.md) | stable | | Google Cloud | Flatcar Linux | [google-cloud/container-linux/kubernetes](cl/google-cloud.md) | alpha | | Digital Ocean | Flatcar Linux | [digital-ocean/container-linux/kubernetes](cl/digital-ocean.md) | alpha | From 70bf39bb9a44af703a60bd495733154363200434 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Thu, 12 Mar 2020 22:59:44 -0700 Subject: [PATCH 354/523] Update Calico from v3.12.0 to v3.13.1 * https://docs.projectcalico.org/v3.13/release-notes/ --- CHANGES.md | 1 + aws/container-linux/kubernetes/bootstrap.tf | 2 +- aws/fedora-coreos/kubernetes/bootstrap.tf | 2 +- azure/container-linux/kubernetes/bootstrap.tf | 2 +- bare-metal/container-linux/kubernetes/bootstrap.tf | 2 +- bare-metal/fedora-coreos/kubernetes/bootstrap.tf | 2 +- digital-ocean/container-linux/kubernetes/bootstrap.tf | 2 +- google-cloud/container-linux/kubernetes/bootstrap.tf | 2 +- google-cloud/fedora-coreos/kubernetes/bootstrap.tf | 2 +- 9 files changed, 9 insertions(+), 8 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 570ccd768..883682098 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -7,6 +7,7 @@ Notable changes between versions. * Update etcd from v3.4.3 to [v3.4.4](https://github.com/etcd-io/etcd/releases/tag/v3.4.4) * On Container Linux, fetch using the docker transport format ([#659](https://github.com/poseidon/typhoon/pull/659)) * Update CoreDNS from v1.6.6 to v1.6.7 ([#648](https://github.com/poseidon/typhoon/pull/648)) +* Update Calico from v3.12.0 to [v3.13.1](https://docs.projectcalico.org/v3.13/release-notes/) #### AWS diff --git a/aws/container-linux/kubernetes/bootstrap.tf b/aws/container-linux/kubernetes/bootstrap.tf index 3ee5bdda4..36062932d 100644 --- a/aws/container-linux/kubernetes/bootstrap.tf +++ b/aws/container-linux/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=d1831e626a06d1aa63a75ca90a670ca594657fbf" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=804029edd5a6a5cf02e0db4919e7f2bb601d0089" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/aws/fedora-coreos/kubernetes/bootstrap.tf b/aws/fedora-coreos/kubernetes/bootstrap.tf index 7511d2d08..0a2f9d52b 100644 --- a/aws/fedora-coreos/kubernetes/bootstrap.tf +++ b/aws/fedora-coreos/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=d1831e626a06d1aa63a75ca90a670ca594657fbf" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=804029edd5a6a5cf02e0db4919e7f2bb601d0089" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/azure/container-linux/kubernetes/bootstrap.tf b/azure/container-linux/kubernetes/bootstrap.tf index 07404d67e..9b2254b1b 100644 --- a/azure/container-linux/kubernetes/bootstrap.tf +++ b/azure/container-linux/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=d1831e626a06d1aa63a75ca90a670ca594657fbf" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=804029edd5a6a5cf02e0db4919e7f2bb601d0089" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/bare-metal/container-linux/kubernetes/bootstrap.tf b/bare-metal/container-linux/kubernetes/bootstrap.tf index 2810938c8..d3c9540f4 100644 --- a/bare-metal/container-linux/kubernetes/bootstrap.tf +++ b/bare-metal/container-linux/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=d1831e626a06d1aa63a75ca90a670ca594657fbf" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=804029edd5a6a5cf02e0db4919e7f2bb601d0089" cluster_name = var.cluster_name api_servers = [var.k8s_domain_name] diff --git a/bare-metal/fedora-coreos/kubernetes/bootstrap.tf b/bare-metal/fedora-coreos/kubernetes/bootstrap.tf index 99a4729f0..bd5283baa 100644 --- a/bare-metal/fedora-coreos/kubernetes/bootstrap.tf +++ b/bare-metal/fedora-coreos/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=d1831e626a06d1aa63a75ca90a670ca594657fbf" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=804029edd5a6a5cf02e0db4919e7f2bb601d0089" cluster_name = var.cluster_name api_servers = [var.k8s_domain_name] diff --git a/digital-ocean/container-linux/kubernetes/bootstrap.tf b/digital-ocean/container-linux/kubernetes/bootstrap.tf index 3985411ab..647c6da8e 100644 --- a/digital-ocean/container-linux/kubernetes/bootstrap.tf +++ b/digital-ocean/container-linux/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=d1831e626a06d1aa63a75ca90a670ca594657fbf" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=804029edd5a6a5cf02e0db4919e7f2bb601d0089" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/google-cloud/container-linux/kubernetes/bootstrap.tf b/google-cloud/container-linux/kubernetes/bootstrap.tf index ad011ed77..3cad1dc5e 100644 --- a/google-cloud/container-linux/kubernetes/bootstrap.tf +++ b/google-cloud/container-linux/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=d1831e626a06d1aa63a75ca90a670ca594657fbf" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=804029edd5a6a5cf02e0db4919e7f2bb601d0089" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/google-cloud/fedora-coreos/kubernetes/bootstrap.tf b/google-cloud/fedora-coreos/kubernetes/bootstrap.tf index 24197be98..3ef495e90 100644 --- a/google-cloud/fedora-coreos/kubernetes/bootstrap.tf +++ b/google-cloud/fedora-coreos/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=d1831e626a06d1aa63a75ca90a670ca594657fbf" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=804029edd5a6a5cf02e0db4919e7f2bb601d0089" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] From bc7902f40abb7b0d18458368c65fba76455226a3 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Thu, 12 Mar 2020 23:02:06 -0700 Subject: [PATCH 355/523] Update Kubernetes from v1.17.3 to v1.17.4 * https://github.com/kubernetes/kubernetes/blob/master/CHANGELOG/CHANGELOG-1.17.md#v1174 --- CHANGES.md | 8 ++++---- README.md | 10 +++++----- aws/container-linux/kubernetes/README.md | 2 +- aws/container-linux/kubernetes/bootstrap.tf | 2 +- aws/container-linux/kubernetes/cl/controller.yaml | 4 ++-- .../kubernetes/workers/cl/worker.yaml | 4 ++-- aws/fedora-coreos/kubernetes/README.md | 2 +- aws/fedora-coreos/kubernetes/bootstrap.tf | 2 +- aws/fedora-coreos/kubernetes/fcc/controller.yaml | 4 ++-- .../kubernetes/workers/fcc/worker.yaml | 4 ++-- azure/container-linux/kubernetes/README.md | 2 +- azure/container-linux/kubernetes/bootstrap.tf | 2 +- .../container-linux/kubernetes/cl/controller.yaml | 4 ++-- .../kubernetes/workers/cl/worker.yaml | 4 ++-- bare-metal/container-linux/kubernetes/README.md | 2 +- bare-metal/container-linux/kubernetes/bootstrap.tf | 2 +- .../container-linux/kubernetes/cl/controller.yaml | 4 ++-- .../container-linux/kubernetes/cl/worker.yaml | 2 +- bare-metal/fedora-coreos/kubernetes/README.md | 2 +- bare-metal/fedora-coreos/kubernetes/bootstrap.tf | 2 +- .../fedora-coreos/kubernetes/fcc/controller.yaml | 4 ++-- .../fedora-coreos/kubernetes/fcc/worker.yaml | 2 +- digital-ocean/container-linux/kubernetes/README.md | 2 +- .../container-linux/kubernetes/bootstrap.tf | 2 +- .../container-linux/kubernetes/cl/controller.yaml | 4 ++-- .../container-linux/kubernetes/cl/worker.yaml | 4 ++-- docs/advanced/worker-pools.md | 14 +++++++------- docs/cl/aws.md | 10 +++++----- docs/cl/azure.md | 10 +++++----- docs/cl/bare-metal.md | 10 +++++----- docs/cl/digital-ocean.md | 10 +++++----- docs/cl/google-cloud.md | 10 +++++----- docs/fedora-coreos/aws.md | 10 +++++----- docs/fedora-coreos/bare-metal.md | 10 +++++----- docs/fedora-coreos/google-cloud.md | 8 ++++---- docs/index.md | 10 +++++----- docs/topics/maintenance.md | 8 ++++---- google-cloud/container-linux/kubernetes/README.md | 2 +- .../container-linux/kubernetes/bootstrap.tf | 2 +- .../container-linux/kubernetes/cl/controller.yaml | 4 ++-- .../kubernetes/workers/cl/worker.yaml | 4 ++-- google-cloud/fedora-coreos/kubernetes/README.md | 2 +- google-cloud/fedora-coreos/kubernetes/bootstrap.tf | 2 +- .../fedora-coreos/kubernetes/fcc/controller.yaml | 4 ++-- .../kubernetes/workers/fcc/worker.yaml | 4 ++-- 45 files changed, 110 insertions(+), 110 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 883682098..403f382ea 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -30,15 +30,15 @@ Notable changes between versions. * Add `worker_node_labels` map variable for per-worker node labels ([#663](https://github.com/poseidon/typhoon/pull/663)) * Add `worker_node_taints` map variable for per-worker node taints ([#663](https://github.com/poseidon/typhoon/pull/663)) +#### DigitalOcean + +* Add support for Flatcar Container Linux ([#644](https://github.com/poseidon/typhoon/pull/644)) + #### Google Cloud * Fix `worker_node_labels` on Fedora CoreOS ([#651](https://github.com/poseidon/typhoon/pull/651)) * Fix automatic worker node delete on shutdown on Fedora CoreOS ([#657](https://github.com/poseidon/typhoon/pull/657)) -#### DigitalOcean - -* Add support for Flatcar Container Linux ([#644](https://github.com/poseidon/typhoon/pull/644)) - #### Addons * Update nginx-ingress from v0.28.0 to [v0.30.0](https://github.com/kubernetes/ingress-nginx/releases/tag/nginx-0.30.0) diff --git a/README.md b/README.md index 73bc89caf..590009a10 100644 --- a/README.md +++ b/README.md @@ -11,7 +11,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features -* Kubernetes v1.17.3 (upstream) +* Kubernetes v1.17.4 (upstream) * Single or multi-master, [Calico](https://www.projectcalico.org/) or [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/) * Advanced features like [worker pools](https://typhoon.psdn.io/advanced/worker-pools/), [preemptible](https://typhoon.psdn.io/cl/google-cloud/#preemption) workers, and [snippets](https://typhoon.psdn.io/advanced/customization/#container-linux) customization @@ -59,7 +59,7 @@ Define a Kubernetes cluster by using the Terraform module for your chosen platfo ```tf module "yavin" { - source = "git::https://github.com/poseidon/typhoon//google-cloud/container-linux/kubernetes?ref=v1.17.3" + source = "git::https://github.com/poseidon/typhoon//google-cloud/container-linux/kubernetes?ref=v1.17.4" # Google Cloud cluster_name = "yavin" @@ -98,9 +98,9 @@ In 4-8 minutes (varies by platform), the cluster will be ready. This Google Clou $ export KUBECONFIG=/home/user/.kube/configs/yavin-config $ kubectl get nodes NAME ROLES STATUS AGE VERSION -yavin-controller-0.c.example-com.internal Ready 6m v1.17.3 -yavin-worker-jrbf.c.example-com.internal Ready 5m v1.17.3 -yavin-worker-mzdm.c.example-com.internal Ready 5m v1.17.3 +yavin-controller-0.c.example-com.internal Ready 6m v1.17.4 +yavin-worker-jrbf.c.example-com.internal Ready 5m v1.17.4 +yavin-worker-mzdm.c.example-com.internal Ready 5m v1.17.4 ``` List the pods. diff --git a/aws/container-linux/kubernetes/README.md b/aws/container-linux/kubernetes/README.md index fffa6688e..fcd9f1a07 100644 --- a/aws/container-linux/kubernetes/README.md +++ b/aws/container-linux/kubernetes/README.md @@ -11,7 +11,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features -* Kubernetes v1.17.3 (upstream) +* Kubernetes v1.17.4 (upstream) * Single or multi-master, [Calico](https://www.projectcalico.org/) or [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/) * Advanced features like [worker pools](https://typhoon.psdn.io/advanced/worker-pools/), [spot](https://typhoon.psdn.io/cl/aws/#spot) workers, and [snippets](https://typhoon.psdn.io/advanced/customization/#container-linux) customization diff --git a/aws/container-linux/kubernetes/bootstrap.tf b/aws/container-linux/kubernetes/bootstrap.tf index 36062932d..f55e011d1 100644 --- a/aws/container-linux/kubernetes/bootstrap.tf +++ b/aws/container-linux/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=804029edd5a6a5cf02e0db4919e7f2bb601d0089" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=73784c1b2c791d9ba586a1478979ac34dd324dad" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/aws/container-linux/kubernetes/cl/controller.yaml b/aws/container-linux/kubernetes/cl/controller.yaml index 41c3319c0..ee96a5768 100644 --- a/aws/container-linux/kubernetes/cl/controller.yaml +++ b/aws/container-linux/kubernetes/cl/controller.yaml @@ -91,7 +91,7 @@ systemd: --mount volume=var-log,target=/var/log \ --volume opt-cni-bin,kind=host,source=/opt/cni/bin \ --mount volume=opt-cni-bin,target=/opt/cni/bin \ - docker://k8s.gcr.io/hyperkube:v1.17.3 \ + docker://k8s.gcr.io/hyperkube:v1.17.4 \ --exec=/usr/local/bin/kubelet -- \ --anonymous-auth=false \ --authentication-token-webhook \ @@ -136,7 +136,7 @@ systemd: --volume script,kind=host,source=/opt/bootstrap/apply \ --mount volume=script,target=/apply \ --insecure-options=image \ - docker://k8s.gcr.io/hyperkube:v1.17.3 \ + docker://k8s.gcr.io/hyperkube:v1.17.4 \ --net=host \ --dns=host \ --exec=/apply diff --git a/aws/container-linux/kubernetes/workers/cl/worker.yaml b/aws/container-linux/kubernetes/workers/cl/worker.yaml index 1eac9085f..a41dc8538 100644 --- a/aws/container-linux/kubernetes/workers/cl/worker.yaml +++ b/aws/container-linux/kubernetes/workers/cl/worker.yaml @@ -64,7 +64,7 @@ systemd: --mount volume=var-log,target=/var/log \ --volume opt-cni-bin,kind=host,source=/opt/cni/bin \ --mount volume=opt-cni-bin,target=/opt/cni/bin \ - docker://k8s.gcr.io/hyperkube:v1.17.3 \ + docker://k8s.gcr.io/hyperkube:v1.17.4 \ --exec=/usr/local/bin/kubelet -- \ --anonymous-auth=false \ --authentication-token-webhook \ @@ -128,7 +128,7 @@ storage: --volume config,kind=host,source=/etc/kubernetes \ --mount volume=config,target=/etc/kubernetes \ --insecure-options=image \ - docker://k8s.gcr.io/hyperkube:v1.17.3 \ + docker://k8s.gcr.io/hyperkube:v1.17.4 \ --net=host \ --dns=host \ -- \ diff --git a/aws/fedora-coreos/kubernetes/README.md b/aws/fedora-coreos/kubernetes/README.md index 4cd36d5d6..5b29531d9 100644 --- a/aws/fedora-coreos/kubernetes/README.md +++ b/aws/fedora-coreos/kubernetes/README.md @@ -11,7 +11,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features -* Kubernetes v1.17.3 (upstream) +* Kubernetes v1.17.4 (upstream) * Single or multi-master, [Calico](https://www.projectcalico.org/) or [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/) * Advanced features like [worker pools](https://typhoon.psdn.io/advanced/worker-pools/), [spot](https://typhoon.psdn.io/cl/aws/#spot) workers, and [snippets](https://typhoon.psdn.io/advanced/customization/#container-linux) customization diff --git a/aws/fedora-coreos/kubernetes/bootstrap.tf b/aws/fedora-coreos/kubernetes/bootstrap.tf index 0a2f9d52b..556cc760f 100644 --- a/aws/fedora-coreos/kubernetes/bootstrap.tf +++ b/aws/fedora-coreos/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=804029edd5a6a5cf02e0db4919e7f2bb601d0089" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=73784c1b2c791d9ba586a1478979ac34dd324dad" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/aws/fedora-coreos/kubernetes/fcc/controller.yaml b/aws/fedora-coreos/kubernetes/fcc/controller.yaml index 9c4e272ef..aeed1df67 100644 --- a/aws/fedora-coreos/kubernetes/fcc/controller.yaml +++ b/aws/fedora-coreos/kubernetes/fcc/controller.yaml @@ -79,7 +79,7 @@ systemd: --volume /var/log:/var/log \ --volume /var/run/lock:/var/run/lock:z \ --volume /opt/cni/bin:/opt/cni/bin:z \ - k8s.gcr.io/hyperkube:v1.17.3 kubelet \ + k8s.gcr.io/hyperkube:v1.17.4 kubelet \ --anonymous-auth=false \ --authentication-token-webhook \ --authorization-mode=Webhook \ @@ -123,7 +123,7 @@ systemd: --volume /opt/bootstrap/assets:/assets:ro,Z \ --volume /opt/bootstrap/apply:/apply:ro,Z \ --entrypoint=/apply \ - k8s.gcr.io/hyperkube:v1.17.3 + k8s.gcr.io/hyperkube:v1.17.4 ExecStartPost=/bin/touch /opt/bootstrap/bootstrap.done ExecStartPost=-/usr/bin/podman stop bootstrap storage: diff --git a/aws/fedora-coreos/kubernetes/workers/fcc/worker.yaml b/aws/fedora-coreos/kubernetes/workers/fcc/worker.yaml index e2b8e0363..07492c3e9 100644 --- a/aws/fedora-coreos/kubernetes/workers/fcc/worker.yaml +++ b/aws/fedora-coreos/kubernetes/workers/fcc/worker.yaml @@ -49,7 +49,7 @@ systemd: --volume /var/log:/var/log \ --volume /var/run/lock:/var/run/lock:z \ --volume /opt/cni/bin:/opt/cni/bin:z \ - k8s.gcr.io/hyperkube:v1.17.3 kubelet \ + k8s.gcr.io/hyperkube:v1.17.4 kubelet \ --anonymous-auth=false \ --authentication-token-webhook \ --authorization-mode=Webhook \ @@ -87,7 +87,7 @@ systemd: Type=oneshot RemainAfterExit=true ExecStart=/bin/true - ExecStop=/bin/bash -c '/usr/bin/podman run --volume /etc/kubernetes:/etc/kubernetes:ro,z k8s.gcr.io/hyperkube:v1.17.3 kubectl --kubeconfig=/etc/kubernetes/kubeconfig delete node $HOSTNAME' + ExecStop=/bin/bash -c '/usr/bin/podman run --volume /etc/kubernetes:/etc/kubernetes:ro,z k8s.gcr.io/hyperkube:v1.17.4 kubectl --kubeconfig=/etc/kubernetes/kubeconfig delete node $HOSTNAME' [Install] WantedBy=multi-user.target storage: diff --git a/azure/container-linux/kubernetes/README.md b/azure/container-linux/kubernetes/README.md index af5359873..27d1002eb 100644 --- a/azure/container-linux/kubernetes/README.md +++ b/azure/container-linux/kubernetes/README.md @@ -11,7 +11,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features -* Kubernetes v1.17.3 (upstream) +* Kubernetes v1.17.4 (upstream) * Single or multi-master, [Calico](https://www.projectcalico.org/) or [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/) * Advanced features like [worker pools](https://typhoon.psdn.io/advanced/worker-pools/), [low-priority](https://typhoon.psdn.io/cl/azure/#low-priority) workers, and [snippets](https://typhoon.psdn.io/advanced/customization/#container-linux) customization diff --git a/azure/container-linux/kubernetes/bootstrap.tf b/azure/container-linux/kubernetes/bootstrap.tf index 9b2254b1b..d7656e2c8 100644 --- a/azure/container-linux/kubernetes/bootstrap.tf +++ b/azure/container-linux/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=804029edd5a6a5cf02e0db4919e7f2bb601d0089" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=73784c1b2c791d9ba586a1478979ac34dd324dad" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/azure/container-linux/kubernetes/cl/controller.yaml b/azure/container-linux/kubernetes/cl/controller.yaml index a9b834d0a..b5035dc0d 100644 --- a/azure/container-linux/kubernetes/cl/controller.yaml +++ b/azure/container-linux/kubernetes/cl/controller.yaml @@ -90,7 +90,7 @@ systemd: --mount volume=var-log,target=/var/log \ --volume opt-cni-bin,kind=host,source=/opt/cni/bin \ --mount volume=opt-cni-bin,target=/opt/cni/bin \ - docker://k8s.gcr.io/hyperkube:v1.17.3 \ + docker://k8s.gcr.io/hyperkube:v1.17.4 \ --exec=/usr/local/bin/kubelet -- \ --anonymous-auth=false \ --authentication-token-webhook \ @@ -134,7 +134,7 @@ systemd: --volume script,kind=host,source=/opt/bootstrap/apply \ --mount volume=script,target=/apply \ --insecure-options=image \ - docker://k8s.gcr.io/hyperkube:v1.17.3 \ + docker://k8s.gcr.io/hyperkube:v1.17.4 \ --net=host \ --dns=host \ --exec=/apply diff --git a/azure/container-linux/kubernetes/workers/cl/worker.yaml b/azure/container-linux/kubernetes/workers/cl/worker.yaml index fe7fb84c8..749cb9bc7 100644 --- a/azure/container-linux/kubernetes/workers/cl/worker.yaml +++ b/azure/container-linux/kubernetes/workers/cl/worker.yaml @@ -63,7 +63,7 @@ systemd: --mount volume=var-log,target=/var/log \ --volume opt-cni-bin,kind=host,source=/opt/cni/bin \ --mount volume=opt-cni-bin,target=/opt/cni/bin \ - docker://k8s.gcr.io/hyperkube:v1.17.3 \ + docker://k8s.gcr.io/hyperkube:v1.17.4 \ --exec=/usr/local/bin/kubelet -- \ --anonymous-auth=false \ --authentication-token-webhook \ @@ -126,7 +126,7 @@ storage: --volume config,kind=host,source=/etc/kubernetes \ --mount volume=config,target=/etc/kubernetes \ --insecure-options=image \ - docker://k8s.gcr.io/hyperkube:v1.17.3 \ + docker://k8s.gcr.io/hyperkube:v1.17.4 \ --net=host \ --dns=host \ -- \ diff --git a/bare-metal/container-linux/kubernetes/README.md b/bare-metal/container-linux/kubernetes/README.md index a85c3aafe..1cc1b3fff 100644 --- a/bare-metal/container-linux/kubernetes/README.md +++ b/bare-metal/container-linux/kubernetes/README.md @@ -11,7 +11,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features -* Kubernetes v1.17.3 (upstream) +* Kubernetes v1.17.4 (upstream) * Single or multi-master, [Calico](https://www.projectcalico.org/) or [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/) * Advanced features like [snippets](https://typhoon.psdn.io/advanced/customization/#container-linux) customization diff --git a/bare-metal/container-linux/kubernetes/bootstrap.tf b/bare-metal/container-linux/kubernetes/bootstrap.tf index d3c9540f4..25c4f6863 100644 --- a/bare-metal/container-linux/kubernetes/bootstrap.tf +++ b/bare-metal/container-linux/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=804029edd5a6a5cf02e0db4919e7f2bb601d0089" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=73784c1b2c791d9ba586a1478979ac34dd324dad" cluster_name = var.cluster_name api_servers = [var.k8s_domain_name] diff --git a/bare-metal/container-linux/kubernetes/cl/controller.yaml b/bare-metal/container-linux/kubernetes/cl/controller.yaml index 569205504..5589f76de 100644 --- a/bare-metal/container-linux/kubernetes/cl/controller.yaml +++ b/bare-metal/container-linux/kubernetes/cl/controller.yaml @@ -103,7 +103,7 @@ systemd: --mount volume=etc-iscsi,target=/etc/iscsi \ --volume usr-sbin-iscsiadm,kind=host,source=/usr/sbin/iscsiadm \ --mount volume=usr-sbin-iscsiadm,target=/sbin/iscsiadm \ - docker://k8s.gcr.io/hyperkube:v1.17.3 \ + docker://k8s.gcr.io/hyperkube:v1.17.4 \ --exec=/usr/local/bin/kubelet -- \ --anonymous-auth=false \ --authentication-token-webhook \ @@ -149,7 +149,7 @@ systemd: --volume script,kind=host,source=/opt/bootstrap/apply \ --mount volume=script,target=/apply \ --insecure-options=image \ - docker://k8s.gcr.io/hyperkube:v1.17.3 \ + docker://k8s.gcr.io/hyperkube:v1.17.4 \ --net=host \ --dns=host \ --exec=/apply diff --git a/bare-metal/container-linux/kubernetes/cl/worker.yaml b/bare-metal/container-linux/kubernetes/cl/worker.yaml index 8ffb1d2e8..f4b923d37 100644 --- a/bare-metal/container-linux/kubernetes/cl/worker.yaml +++ b/bare-metal/container-linux/kubernetes/cl/worker.yaml @@ -76,7 +76,7 @@ systemd: --mount volume=etc-iscsi,target=/etc/iscsi \ --volume usr-sbin-iscsiadm,kind=host,source=/usr/sbin/iscsiadm \ --mount volume=usr-sbin-iscsiadm,target=/sbin/iscsiadm \ - docker://k8s.gcr.io/hyperkube:v1.17.3 \ + docker://k8s.gcr.io/hyperkube:v1.17.4 \ --exec=/usr/local/bin/kubelet -- \ --anonymous-auth=false \ --authentication-token-webhook \ diff --git a/bare-metal/fedora-coreos/kubernetes/README.md b/bare-metal/fedora-coreos/kubernetes/README.md index 46a9ed4e9..088e4e865 100644 --- a/bare-metal/fedora-coreos/kubernetes/README.md +++ b/bare-metal/fedora-coreos/kubernetes/README.md @@ -11,7 +11,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features -* Kubernetes v1.17.3 (upstream) +* Kubernetes v1.17.4 (upstream) * Single or multi-master, [Calico](https://www.projectcalico.org/) or [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/) * Advanced features like [snippets](https://typhoon.psdn.io/advanced/customization/#container-linux) customization diff --git a/bare-metal/fedora-coreos/kubernetes/bootstrap.tf b/bare-metal/fedora-coreos/kubernetes/bootstrap.tf index bd5283baa..077df9a43 100644 --- a/bare-metal/fedora-coreos/kubernetes/bootstrap.tf +++ b/bare-metal/fedora-coreos/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=804029edd5a6a5cf02e0db4919e7f2bb601d0089" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=73784c1b2c791d9ba586a1478979ac34dd324dad" cluster_name = var.cluster_name api_servers = [var.k8s_domain_name] diff --git a/bare-metal/fedora-coreos/kubernetes/fcc/controller.yaml b/bare-metal/fedora-coreos/kubernetes/fcc/controller.yaml index 72585414f..89b778001 100644 --- a/bare-metal/fedora-coreos/kubernetes/fcc/controller.yaml +++ b/bare-metal/fedora-coreos/kubernetes/fcc/controller.yaml @@ -80,7 +80,7 @@ systemd: --volume /opt/cni/bin:/opt/cni/bin:z \ --volume /etc/iscsi:/etc/iscsi \ --volume /sbin/iscsiadm:/sbin/iscsiadm \ - k8s.gcr.io/hyperkube:v1.17.3 kubelet \ + k8s.gcr.io/hyperkube:v1.17.4 kubelet \ --anonymous-auth=false \ --authentication-token-webhook \ --authorization-mode=Webhook \ @@ -134,7 +134,7 @@ systemd: --volume /opt/bootstrap/assets:/assets:ro,Z \ --volume /opt/bootstrap/apply:/apply:ro,Z \ --entrypoint=/apply \ - k8s.gcr.io/hyperkube:v1.17.3 + k8s.gcr.io/hyperkube:v1.17.4 ExecStartPost=/bin/touch /opt/bootstrap/bootstrap.done ExecStartPost=-/usr/bin/podman stop bootstrap storage: diff --git a/bare-metal/fedora-coreos/kubernetes/fcc/worker.yaml b/bare-metal/fedora-coreos/kubernetes/fcc/worker.yaml index d1d5e8aa8..54395326a 100644 --- a/bare-metal/fedora-coreos/kubernetes/fcc/worker.yaml +++ b/bare-metal/fedora-coreos/kubernetes/fcc/worker.yaml @@ -50,7 +50,7 @@ systemd: --volume /opt/cni/bin:/opt/cni/bin:z \ --volume /etc/iscsi:/etc/iscsi \ --volume /sbin/iscsiadm:/sbin/iscsiadm \ - k8s.gcr.io/hyperkube:v1.17.3 kubelet \ + k8s.gcr.io/hyperkube:v1.17.4 kubelet \ --anonymous-auth=false \ --authentication-token-webhook \ --authorization-mode=Webhook \ diff --git a/digital-ocean/container-linux/kubernetes/README.md b/digital-ocean/container-linux/kubernetes/README.md index b5845e625..5fd66520a 100644 --- a/digital-ocean/container-linux/kubernetes/README.md +++ b/digital-ocean/container-linux/kubernetes/README.md @@ -11,7 +11,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features -* Kubernetes v1.17.3 (upstream) +* Kubernetes v1.17.4 (upstream) * Single or multi-master, [Calico](https://www.projectcalico.org/) or [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/) * Advanced features like [snippets](https://typhoon.psdn.io/advanced/customization/#container-linux) customization diff --git a/digital-ocean/container-linux/kubernetes/bootstrap.tf b/digital-ocean/container-linux/kubernetes/bootstrap.tf index 647c6da8e..bfec3b981 100644 --- a/digital-ocean/container-linux/kubernetes/bootstrap.tf +++ b/digital-ocean/container-linux/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=804029edd5a6a5cf02e0db4919e7f2bb601d0089" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=73784c1b2c791d9ba586a1478979ac34dd324dad" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/digital-ocean/container-linux/kubernetes/cl/controller.yaml b/digital-ocean/container-linux/kubernetes/cl/controller.yaml index 3950c7dc3..40930c031 100644 --- a/digital-ocean/container-linux/kubernetes/cl/controller.yaml +++ b/digital-ocean/container-linux/kubernetes/cl/controller.yaml @@ -101,7 +101,7 @@ systemd: --mount volume=var-log,target=/var/log \ --volume opt-cni-bin,kind=host,source=/opt/cni/bin \ --mount volume=opt-cni-bin,target=/opt/cni/bin \ - docker://k8s.gcr.io/hyperkube:v1.17.3 \ + docker://k8s.gcr.io/hyperkube:v1.17.4 \ --exec=/usr/local/bin/kubelet -- \ --anonymous-auth=false \ --authentication-token-webhook \ @@ -146,7 +146,7 @@ systemd: --volume script,kind=host,source=/opt/bootstrap/apply \ --mount volume=script,target=/apply \ --insecure-options=image \ - docker://k8s.gcr.io/hyperkube:v1.17.3 \ + docker://k8s.gcr.io/hyperkube:v1.17.4 \ --net=host \ --dns=host \ --exec=/apply diff --git a/digital-ocean/container-linux/kubernetes/cl/worker.yaml b/digital-ocean/container-linux/kubernetes/cl/worker.yaml index 0a23c0cce..230e05a8e 100644 --- a/digital-ocean/container-linux/kubernetes/cl/worker.yaml +++ b/digital-ocean/container-linux/kubernetes/cl/worker.yaml @@ -74,7 +74,7 @@ systemd: --mount volume=var-log,target=/var/log \ --volume opt-cni-bin,kind=host,source=/opt/cni/bin \ --mount volume=opt-cni-bin,target=/opt/cni/bin \ - docker://k8s.gcr.io/hyperkube:v1.17.3 \ + docker://k8s.gcr.io/hyperkube:v1.17.4 \ --exec=/usr/local/bin/kubelet -- \ --anonymous-auth=false \ --authentication-token-webhook \ @@ -132,7 +132,7 @@ storage: --volume config,kind=host,source=/etc/kubernetes \ --mount volume=config,target=/etc/kubernetes \ --insecure-options=image \ - docker://k8s.gcr.io/hyperkube:v1.17.3 \ + docker://k8s.gcr.io/hyperkube:v1.17.4 \ --net=host \ --dns=host \ -- \ diff --git a/docs/advanced/worker-pools.md b/docs/advanced/worker-pools.md index 07d022a1f..2eb295bca 100644 --- a/docs/advanced/worker-pools.md +++ b/docs/advanced/worker-pools.md @@ -79,7 +79,7 @@ Create a cluster following the Azure [tutorial](../cl/azure.md#cluster). Define ```tf module "ramius-worker-pool" { - source = "git::https://github.com/poseidon/typhoon//azure/container-linux/kubernetes/workers?ref=v1.17.3" + source = "git::https://github.com/poseidon/typhoon//azure/container-linux/kubernetes/workers?ref=v1.17.4" # Azure region = module.ramius.region @@ -145,7 +145,7 @@ Create a cluster following the Google Cloud [tutorial](../cl/google-cloud.md#clu ```tf module "yavin-worker-pool" { - source = "git::https://github.com/poseidon/typhoon//google-cloud/container-linux/kubernetes/workers?ref=v1.17.3" + source = "git::https://github.com/poseidon/typhoon//google-cloud/container-linux/kubernetes/workers?ref=v1.17.4" # Google Cloud region = "europe-west2" @@ -176,11 +176,11 @@ Verify a managed instance group of workers joins the cluster within a few minute ``` $ kubectl get nodes NAME STATUS AGE VERSION -yavin-controller-0.c.example-com.internal Ready 6m v1.17.3 -yavin-worker-jrbf.c.example-com.internal Ready 5m v1.17.3 -yavin-worker-mzdm.c.example-com.internal Ready 5m v1.17.3 -yavin-16x-worker-jrbf.c.example-com.internal Ready 3m v1.17.3 -yavin-16x-worker-mzdm.c.example-com.internal Ready 3m v1.17.3 +yavin-controller-0.c.example-com.internal Ready 6m v1.17.4 +yavin-worker-jrbf.c.example-com.internal Ready 5m v1.17.4 +yavin-worker-mzdm.c.example-com.internal Ready 5m v1.17.4 +yavin-16x-worker-jrbf.c.example-com.internal Ready 3m v1.17.4 +yavin-16x-worker-mzdm.c.example-com.internal Ready 3m v1.17.4 ``` ### Variables diff --git a/docs/cl/aws.md b/docs/cl/aws.md index f46a76a8e..a6c8b3370 100644 --- a/docs/cl/aws.md +++ b/docs/cl/aws.md @@ -1,6 +1,6 @@ # AWS -In this tutorial, we'll create a Kubernetes v1.17.3 cluster on AWS with CoreOS Container Linux or Flatcar Linux. +In this tutorial, we'll create a Kubernetes v1.17.4 cluster on AWS with CoreOS Container Linux or Flatcar Linux. We'll declare a Kubernetes cluster using the Typhoon Terraform module. Then apply the changes to create a VPC, gateway, subnets, security groups, controller instances, worker auto-scaling group, network load balancer, and TLS assets. @@ -70,7 +70,7 @@ Define a Kubernetes cluster using the module `aws/container-linux/kubernetes`. ```tf module "tempest" { - source = "git::https://github.com/poseidon/typhoon//aws/container-linux/kubernetes?ref=v1.17.3" + source = "git::https://github.com/poseidon/typhoon//aws/container-linux/kubernetes?ref=v1.17.4" # AWS cluster_name = "tempest" @@ -143,9 +143,9 @@ List nodes in the cluster. $ export KUBECONFIG=/home/user/.kube/configs/tempest-config $ kubectl get nodes NAME STATUS ROLES AGE VERSION -ip-10-0-3-155 Ready 10m v1.17.3 -ip-10-0-26-65 Ready 10m v1.17.3 -ip-10-0-41-21 Ready 10m v1.17.3 +ip-10-0-3-155 Ready 10m v1.17.4 +ip-10-0-26-65 Ready 10m v1.17.4 +ip-10-0-41-21 Ready 10m v1.17.4 ``` List the pods. diff --git a/docs/cl/azure.md b/docs/cl/azure.md index deb4b0a1d..5f985e7c4 100644 --- a/docs/cl/azure.md +++ b/docs/cl/azure.md @@ -3,7 +3,7 @@ !!! danger Typhoon for Azure is alpha. For production, use AWS, Google Cloud, or bare-metal. As Azure matures, check [errata](https://github.com/poseidon/typhoon/wiki/Errata) for known shortcomings. -In this tutorial, we'll create a Kubernetes v1.17.3 cluster on Azure with CoreOS Container Linux or Flatcar Linux. +In this tutorial, we'll create a Kubernetes v1.17.4 cluster on Azure with CoreOS Container Linux or Flatcar Linux. We'll declare a Kubernetes cluster using the Typhoon Terraform module. Then apply the changes to create a resource group, virtual network, subnets, security groups, controller availability set, worker scale set, load balancer, and TLS assets. @@ -66,7 +66,7 @@ Define a Kubernetes cluster using the module `azure/container-linux/kubernetes`. ```tf module "ramius" { - source = "git::https://github.com/poseidon/typhoon//azure/container-linux/kubernetes?ref=v1.17.3" + source = "git::https://github.com/poseidon/typhoon//azure/container-linux/kubernetes?ref=v1.17.4" # Azure cluster_name = "ramius" @@ -149,9 +149,9 @@ List nodes in the cluster. $ export KUBECONFIG=/home/user/.kube/configs/ramius-config $ kubectl get nodes NAME STATUS ROLES AGE VERSION -ramius-controller-0 Ready 24m v1.17.3 -ramius-worker-000001 Ready 25m v1.17.3 -ramius-worker-000002 Ready 24m v1.17.3 +ramius-controller-0 Ready 24m v1.17.4 +ramius-worker-000001 Ready 25m v1.17.4 +ramius-worker-000002 Ready 24m v1.17.4 ``` List the pods. diff --git a/docs/cl/bare-metal.md b/docs/cl/bare-metal.md index 5cf4f7775..d12884132 100644 --- a/docs/cl/bare-metal.md +++ b/docs/cl/bare-metal.md @@ -1,6 +1,6 @@ # Bare-Metal -In this tutorial, we'll network boot and provision a Kubernetes v1.17.3 cluster on bare-metal with CoreOS Container Linux or Flatcar Linux. +In this tutorial, we'll network boot and provision a Kubernetes v1.17.4 cluster on bare-metal with CoreOS Container Linux or Flatcar Linux. First, we'll deploy a [Matchbox](https://github.com/poseidon/matchbox) service and setup a network boot environment. Then, we'll declare a Kubernetes cluster using the Typhoon Terraform module and power on machines. On PXE boot, machines will install Container Linux to disk, reboot into the disk install, and provision themselves as Kubernetes controllers or workers via Ignition. @@ -160,7 +160,7 @@ Define a Kubernetes cluster using the module `bare-metal/container-linux/kuberne ```tf module "mercury" { - source = "git::https://github.com/poseidon/typhoon//bare-metal/container-linux/kubernetes?ref=v1.17.3" + source = "git::https://github.com/poseidon/typhoon//bare-metal/container-linux/kubernetes?ref=v1.17.4" # bare-metal cluster_name = "mercury" @@ -299,9 +299,9 @@ List nodes in the cluster. $ export KUBECONFIG=/home/user/.kube/configs/mercury-config $ kubectl get nodes NAME STATUS ROLES AGE VERSION -node1.example.com Ready 10m v1.17.3 -node2.example.com Ready 10m v1.17.3 -node3.example.com Ready 10m v1.17.3 +node1.example.com Ready 10m v1.17.4 +node2.example.com Ready 10m v1.17.4 +node3.example.com Ready 10m v1.17.4 ``` List the pods. diff --git a/docs/cl/digital-ocean.md b/docs/cl/digital-ocean.md index ead7fc7ea..7b796b3bc 100644 --- a/docs/cl/digital-ocean.md +++ b/docs/cl/digital-ocean.md @@ -1,6 +1,6 @@ # Digital Ocean -In this tutorial, we'll create a Kubernetes v1.17.3 cluster on DigitalOcean with CoreOS Container Linux or Flatcar Linux. +In this tutorial, we'll create a Kubernetes v1.17.4 cluster on DigitalOcean with CoreOS Container Linux or Flatcar Linux. We'll declare a Kubernetes cluster using the Typhoon Terraform module. Then apply the changes to create controller droplets, worker droplets, DNS records, tags, and TLS assets. @@ -65,7 +65,7 @@ Define a Kubernetes cluster using the module `digital-ocean/container-linux/kube ```tf module "nemo" { - source = "git::https://github.com/poseidon/typhoon//digital-ocean/container-linux/kubernetes?ref=v1.17.3" + source = "git::https://github.com/poseidon/typhoon//digital-ocean/container-linux/kubernetes?ref=v1.17.4" # Digital Ocean cluster_name = "nemo" @@ -161,9 +161,9 @@ List nodes in the cluster. $ export KUBECONFIG=/home/user/.kube/configs/nemo-config $ kubectl get nodes NAME STATUS ROLES AGE VERSION -10.132.110.130 Ready 10m v1.17.3 -10.132.115.81 Ready 10m v1.17.3 -10.132.124.107 Ready 10m v1.17.3 +10.132.110.130 Ready 10m v1.17.4 +10.132.115.81 Ready 10m v1.17.4 +10.132.124.107 Ready 10m v1.17.4 ``` List the pods. diff --git a/docs/cl/google-cloud.md b/docs/cl/google-cloud.md index 08bed675d..6c50577d9 100644 --- a/docs/cl/google-cloud.md +++ b/docs/cl/google-cloud.md @@ -1,6 +1,6 @@ # Google Cloud -In this tutorial, we'll create a Kubernetes v1.17.3 cluster on Google Compute Engine with CoreOS Container Linux or Flatcar Linux. +In this tutorial, we'll create a Kubernetes v1.17.4 cluster on Google Compute Engine with CoreOS Container Linux or Flatcar Linux. We'll declare a Kubernetes cluster using the Typhoon Terraform module. Then apply the changes to create a network, firewall rules, health checks, controller instances, worker managed instance group, load balancers, and TLS assets. @@ -71,7 +71,7 @@ Define a Kubernetes cluster using the module `google-cloud/container-linux/kuber ```tf module "yavin" { - source = "git::https://github.com/poseidon/typhoon//google-cloud/container-linux/kubernetes?ref=v1.17.3" + source = "git::https://github.com/poseidon/typhoon//google-cloud/container-linux/kubernetes?ref=v1.17.4" # Google Cloud cluster_name = "yavin" @@ -167,9 +167,9 @@ List nodes in the cluster. $ export KUBECONFIG=/home/user/.kube/configs/yavin-config $ kubectl get nodes NAME ROLES STATUS AGE VERSION -yavin-controller-0.c.example-com.internal Ready 6m v1.17.3 -yavin-worker-jrbf.c.example-com.internal Ready 5m v1.17.3 -yavin-worker-mzdm.c.example-com.internal Ready 5m v1.17.3 +yavin-controller-0.c.example-com.internal Ready 6m v1.17.4 +yavin-worker-jrbf.c.example-com.internal Ready 5m v1.17.4 +yavin-worker-mzdm.c.example-com.internal Ready 5m v1.17.4 ``` List the pods. diff --git a/docs/fedora-coreos/aws.md b/docs/fedora-coreos/aws.md index 61876769e..928d47683 100644 --- a/docs/fedora-coreos/aws.md +++ b/docs/fedora-coreos/aws.md @@ -1,6 +1,6 @@ # AWS -In this tutorial, we'll create a Kubernetes v1.17.3 cluster on AWS with Fedora CoreOS. +In this tutorial, we'll create a Kubernetes v1.17.4 cluster on AWS with Fedora CoreOS. We'll declare a Kubernetes cluster using the Typhoon Terraform module. Then apply the changes to create a VPC, gateway, subnets, security groups, controller instances, worker auto-scaling group, network load balancer, and TLS assets. @@ -70,7 +70,7 @@ Define a Kubernetes cluster using the module `aws/fedora-coreos/kubernetes`. ```tf module "tempest" { - source = "git::https://github.com/poseidon/typhoon//aws/fedora-coreos/kubernetes?ref=v1.17.3" + source = "git::https://github.com/poseidon/typhoon//aws/fedora-coreos/kubernetes?ref=v1.17.4" # AWS cluster_name = "tempest" @@ -143,9 +143,9 @@ List nodes in the cluster. $ export KUBECONFIG=/home/user/.kube/configs/tempest-config $ kubectl get nodes NAME STATUS ROLES AGE VERSION -ip-10-0-3-155 Ready 10m v1.17.3 -ip-10-0-26-65 Ready 10m v1.17.3 -ip-10-0-41-21 Ready 10m v1.17.3 +ip-10-0-3-155 Ready 10m v1.17.4 +ip-10-0-26-65 Ready 10m v1.17.4 +ip-10-0-41-21 Ready 10m v1.17.4 ``` List the pods. diff --git a/docs/fedora-coreos/bare-metal.md b/docs/fedora-coreos/bare-metal.md index c95fbf3d5..3acd118db 100644 --- a/docs/fedora-coreos/bare-metal.md +++ b/docs/fedora-coreos/bare-metal.md @@ -1,6 +1,6 @@ # Bare-Metal -In this tutorial, we'll network boot and provision a Kubernetes v1.17.3 cluster on bare-metal with Fedora CoreOS. +In this tutorial, we'll network boot and provision a Kubernetes v1.17.4 cluster on bare-metal with Fedora CoreOS. First, we'll deploy a [Matchbox](https://github.com/poseidon/matchbox) service and setup a network boot environment. Then, we'll declare a Kubernetes cluster using the Typhoon Terraform module and power on machines. On PXE boot, machines will install Fedora CoreOS to disk, reboot into the disk install, and provision themselves as Kubernetes controllers or workers via Ignition. @@ -160,7 +160,7 @@ Define a Kubernetes cluster using the module `bare-metal/fedora-coreos/kubernete ```tf module "mercury" { - source = "git::https://github.com/poseidon/typhoon//bare-metal/fedora-coreos/kubernetes?ref=v1.17.3" + source = "git::https://github.com/poseidon/typhoon//bare-metal/fedora-coreos/kubernetes?ref=v1.17.4" # bare-metal cluster_name = "mercury" @@ -289,9 +289,9 @@ List nodes in the cluster. $ export KUBECONFIG=/home/user/.kube/configs/mercury-config $ kubectl get nodes NAME STATUS ROLES AGE VERSION -node1.example.com Ready 10m v1.17.3 -node2.example.com Ready 10m v1.17.3 -node3.example.com Ready 10m v1.17.3 +node1.example.com Ready 10m v1.17.4 +node2.example.com Ready 10m v1.17.4 +node3.example.com Ready 10m v1.17.4 ``` List the pods. diff --git a/docs/fedora-coreos/google-cloud.md b/docs/fedora-coreos/google-cloud.md index 80d255695..4a059d9fb 100644 --- a/docs/fedora-coreos/google-cloud.md +++ b/docs/fedora-coreos/google-cloud.md @@ -3,7 +3,7 @@ !!! danger Typhoon for Fedora CoreOS is an alpha. Please report Fedora CoreOS bugs to [Fedora](https://github.com/coreos/fedora-coreos-tracker/issues) and Typhoon issues to Typhoon. -In this tutorial, we'll create a Kubernetes v1.17.3 cluster on Google Compute Engine with Fedora CoreOS. +In this tutorial, we'll create a Kubernetes v1.17.4 cluster on Google Compute Engine with Fedora CoreOS. We'll declare a Kubernetes cluster using the Typhoon Terraform module. Then apply the changes to create a network, firewall rules, health checks, controller instances, worker managed instance group, load balancers, and TLS assets. @@ -168,9 +168,9 @@ List nodes in the cluster. $ export KUBECONFIG=/home/user/.kube/configs/yavin-config $ kubectl get nodes NAME ROLES STATUS AGE VERSION -yavin-controller-0.c.example-com.internal Ready 6m v1.17.3 -yavin-worker-jrbf.c.example-com.internal Ready 5m v1.17.3 -yavin-worker-mzdm.c.example-com.internal Ready 5m v1.17.3 +yavin-controller-0.c.example-com.internal Ready 6m v1.17.4 +yavin-worker-jrbf.c.example-com.internal Ready 5m v1.17.4 +yavin-worker-mzdm.c.example-com.internal Ready 5m v1.17.4 ``` List the pods. diff --git a/docs/index.md b/docs/index.md index 22ec89311..df5bf6675 100644 --- a/docs/index.md +++ b/docs/index.md @@ -11,7 +11,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features -* Kubernetes v1.17.3 (upstream) +* Kubernetes v1.17.4 (upstream) * Single or multi-master, [Calico](https://www.projectcalico.org/) or [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/) * Advanced features like [worker pools](advanced/worker-pools/), [preemptible](cl/google-cloud/#preemption) workers, and [snippets](advanced/customization/#container-linux) customization @@ -58,7 +58,7 @@ Define a Kubernetes cluster by using the Terraform module for your chosen platfo ```tf module "yavin" { - source = "git::https://github.com/poseidon/typhoon//google-cloud/container-linux/kubernetes?ref=v1.17.3" + source = "git::https://github.com/poseidon/typhoon//google-cloud/container-linux/kubernetes?ref=v1.17.4" # Google Cloud cluster_name = "yavin" @@ -96,9 +96,9 @@ In 4-8 minutes (varies by platform), the cluster will be ready. This Google Clou $ export KUBECONFIG=/home/user/.kube/configs/yavin-config $ kubectl get nodes NAME ROLES STATUS AGE VERSION -yavin-controller-0.c.example-com.internal Ready 6m v1.17.3 -yavin-worker-jrbf.c.example-com.internal Ready 5m v1.17.3 -yavin-worker-mzdm.c.example-com.internal Ready 5m v1.17.3 +yavin-controller-0.c.example-com.internal Ready 6m v1.17.4 +yavin-worker-jrbf.c.example-com.internal Ready 5m v1.17.4 +yavin-worker-mzdm.c.example-com.internal Ready 5m v1.17.4 ``` List the pods. diff --git a/docs/topics/maintenance.md b/docs/topics/maintenance.md index 114bfc93d..2b3126a17 100644 --- a/docs/topics/maintenance.md +++ b/docs/topics/maintenance.md @@ -18,7 +18,7 @@ module "yavin" { } module "mercury" { - source = "git::https://github.com/poseidon/typhoon//bare-metal/container-linux/kubernetes?ref=v1.17.3" + source = "git::https://github.com/poseidon/typhoon//bare-metal/container-linux/kubernetes?ref=v1.17.4" ... } ``` @@ -279,15 +279,15 @@ Typhoon modules have been adapted for Terraform v0.12. Provider plugins requirem | Typhoon Release | Terraform version | |-------------------|---------------------| -| v1.17.3 - ? | v0.12.x | -| v1.10.3 - v1.17.3 | v0.11.x | +| v1.17.4 - ? | v0.12.x | +| v1.10.3 - v1.17.4 | v0.11.x | | v1.9.2 - v1.10.2 | v0.10.4+ or v0.11.x | | v1.7.3 - v1.9.1 | v0.10.x | | v1.6.4 - v1.7.2 | v0.9.x | ### New users -New users can start with Terraform v0.12.x and follow the docs for Typhoon v1.17.3+ without issue. +New users can start with Terraform v0.12.x and follow the docs for Typhoon v1.17.4+ without issue. ### Existing users diff --git a/google-cloud/container-linux/kubernetes/README.md b/google-cloud/container-linux/kubernetes/README.md index fe81ec8f4..ff8a08a96 100644 --- a/google-cloud/container-linux/kubernetes/README.md +++ b/google-cloud/container-linux/kubernetes/README.md @@ -11,7 +11,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features -* Kubernetes v1.17.3 (upstream) +* Kubernetes v1.17.4 (upstream) * Single or multi-master, [Calico](https://www.projectcalico.org/) or [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/) * Advanced features like [worker pools](https://typhoon.psdn.io/advanced/worker-pools/), [preemptible](https://typhoon.psdn.io/cl/google-cloud/#preemption) workers, and [snippets](https://typhoon.psdn.io/advanced/customization/#container-linux) customization diff --git a/google-cloud/container-linux/kubernetes/bootstrap.tf b/google-cloud/container-linux/kubernetes/bootstrap.tf index 3cad1dc5e..f3244b99e 100644 --- a/google-cloud/container-linux/kubernetes/bootstrap.tf +++ b/google-cloud/container-linux/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=804029edd5a6a5cf02e0db4919e7f2bb601d0089" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=73784c1b2c791d9ba586a1478979ac34dd324dad" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/google-cloud/container-linux/kubernetes/cl/controller.yaml b/google-cloud/container-linux/kubernetes/cl/controller.yaml index e57ee4290..5966bff1d 100644 --- a/google-cloud/container-linux/kubernetes/cl/controller.yaml +++ b/google-cloud/container-linux/kubernetes/cl/controller.yaml @@ -90,7 +90,7 @@ systemd: --mount volume=var-log,target=/var/log \ --volume opt-cni-bin,kind=host,source=/opt/cni/bin \ --mount volume=opt-cni-bin,target=/opt/cni/bin \ - docker://k8s.gcr.io/hyperkube:v1.17.3 \ + docker://k8s.gcr.io/hyperkube:v1.17.4 \ --exec=/usr/local/bin/kubelet -- \ --anonymous-auth=false \ --authentication-token-webhook \ @@ -134,7 +134,7 @@ systemd: --volume script,kind=host,source=/opt/bootstrap/apply \ --mount volume=script,target=/apply \ --insecure-options=image \ - docker://k8s.gcr.io/hyperkube:v1.17.3 \ + docker://k8s.gcr.io/hyperkube:v1.17.4 \ --net=host \ --dns=host \ --exec=/apply diff --git a/google-cloud/container-linux/kubernetes/workers/cl/worker.yaml b/google-cloud/container-linux/kubernetes/workers/cl/worker.yaml index 26dffd3e5..4cf23d9b8 100644 --- a/google-cloud/container-linux/kubernetes/workers/cl/worker.yaml +++ b/google-cloud/container-linux/kubernetes/workers/cl/worker.yaml @@ -63,7 +63,7 @@ systemd: --mount volume=var-log,target=/var/log \ --volume opt-cni-bin,kind=host,source=/opt/cni/bin \ --mount volume=opt-cni-bin,target=/opt/cni/bin \ - docker://k8s.gcr.io/hyperkube:v1.17.3 \ + docker://k8s.gcr.io/hyperkube:v1.17.4 \ --exec=/usr/local/bin/kubelet -- \ --anonymous-auth=false \ --authentication-token-webhook \ @@ -126,7 +126,7 @@ storage: --volume config,kind=host,source=/etc/kubernetes \ --mount volume=config,target=/etc/kubernetes \ --insecure-options=image \ - docker://k8s.gcr.io/hyperkube:v1.17.3 \ + docker://k8s.gcr.io/hyperkube:v1.17.4 \ --net=host \ --dns=host \ -- \ diff --git a/google-cloud/fedora-coreos/kubernetes/README.md b/google-cloud/fedora-coreos/kubernetes/README.md index fe81ec8f4..ff8a08a96 100644 --- a/google-cloud/fedora-coreos/kubernetes/README.md +++ b/google-cloud/fedora-coreos/kubernetes/README.md @@ -11,7 +11,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features -* Kubernetes v1.17.3 (upstream) +* Kubernetes v1.17.4 (upstream) * Single or multi-master, [Calico](https://www.projectcalico.org/) or [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/) * Advanced features like [worker pools](https://typhoon.psdn.io/advanced/worker-pools/), [preemptible](https://typhoon.psdn.io/cl/google-cloud/#preemption) workers, and [snippets](https://typhoon.psdn.io/advanced/customization/#container-linux) customization diff --git a/google-cloud/fedora-coreos/kubernetes/bootstrap.tf b/google-cloud/fedora-coreos/kubernetes/bootstrap.tf index 3ef495e90..fb05144eb 100644 --- a/google-cloud/fedora-coreos/kubernetes/bootstrap.tf +++ b/google-cloud/fedora-coreos/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=804029edd5a6a5cf02e0db4919e7f2bb601d0089" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=73784c1b2c791d9ba586a1478979ac34dd324dad" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/google-cloud/fedora-coreos/kubernetes/fcc/controller.yaml b/google-cloud/fedora-coreos/kubernetes/fcc/controller.yaml index 9c4e272ef..aeed1df67 100644 --- a/google-cloud/fedora-coreos/kubernetes/fcc/controller.yaml +++ b/google-cloud/fedora-coreos/kubernetes/fcc/controller.yaml @@ -79,7 +79,7 @@ systemd: --volume /var/log:/var/log \ --volume /var/run/lock:/var/run/lock:z \ --volume /opt/cni/bin:/opt/cni/bin:z \ - k8s.gcr.io/hyperkube:v1.17.3 kubelet \ + k8s.gcr.io/hyperkube:v1.17.4 kubelet \ --anonymous-auth=false \ --authentication-token-webhook \ --authorization-mode=Webhook \ @@ -123,7 +123,7 @@ systemd: --volume /opt/bootstrap/assets:/assets:ro,Z \ --volume /opt/bootstrap/apply:/apply:ro,Z \ --entrypoint=/apply \ - k8s.gcr.io/hyperkube:v1.17.3 + k8s.gcr.io/hyperkube:v1.17.4 ExecStartPost=/bin/touch /opt/bootstrap/bootstrap.done ExecStartPost=-/usr/bin/podman stop bootstrap storage: diff --git a/google-cloud/fedora-coreos/kubernetes/workers/fcc/worker.yaml b/google-cloud/fedora-coreos/kubernetes/workers/fcc/worker.yaml index e2b8e0363..07492c3e9 100644 --- a/google-cloud/fedora-coreos/kubernetes/workers/fcc/worker.yaml +++ b/google-cloud/fedora-coreos/kubernetes/workers/fcc/worker.yaml @@ -49,7 +49,7 @@ systemd: --volume /var/log:/var/log \ --volume /var/run/lock:/var/run/lock:z \ --volume /opt/cni/bin:/opt/cni/bin:z \ - k8s.gcr.io/hyperkube:v1.17.3 kubelet \ + k8s.gcr.io/hyperkube:v1.17.4 kubelet \ --anonymous-auth=false \ --authentication-token-webhook \ --authorization-mode=Webhook \ @@ -87,7 +87,7 @@ systemd: Type=oneshot RemainAfterExit=true ExecStart=/bin/true - ExecStop=/bin/bash -c '/usr/bin/podman run --volume /etc/kubernetes:/etc/kubernetes:ro,z k8s.gcr.io/hyperkube:v1.17.3 kubectl --kubeconfig=/etc/kubernetes/kubeconfig delete node $HOSTNAME' + ExecStop=/bin/bash -c '/usr/bin/podman run --volume /etc/kubernetes:/etc/kubernetes:ro,z k8s.gcr.io/hyperkube:v1.17.4 kubectl --kubeconfig=/etc/kubernetes/kubeconfig delete node $HOSTNAME' [Install] WantedBy=multi-user.target storage: From 1a139ef6f107f0364f5792a6e2e6525600855dc7 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Mon, 16 Mar 2020 21:40:52 -0700 Subject: [PATCH 356/523] Update recommended Terraform versions and providers * Sync the documented Terraform versions and provider plugin versions to those that are actively used/tested by the author --- docs/cl/aws.md | 4 ++-- docs/cl/azure.md | 4 ++-- docs/cl/bare-metal.md | 2 +- docs/cl/digital-ocean.md | 2 +- docs/cl/google-cloud.md | 4 ++-- docs/fedora-coreos/aws.md | 4 ++-- docs/fedora-coreos/bare-metal.md | 2 +- docs/fedora-coreos/google-cloud.md | 4 ++-- 8 files changed, 13 insertions(+), 13 deletions(-) diff --git a/docs/cl/aws.md b/docs/cl/aws.md index a6c8b3370..c67700804 100644 --- a/docs/cl/aws.md +++ b/docs/cl/aws.md @@ -18,7 +18,7 @@ Install [Terraform](https://www.terraform.io/downloads.html) v0.12.6+ on your sy ```sh $ terraform version -Terraform v0.12.20 +Terraform v0.12.21 ``` Add the [terraform-provider-ct](https://github.com/poseidon/terraform-provider-ct) plugin binary for your system to `~/.terraform.d/plugins/`, noting the final name. @@ -49,7 +49,7 @@ Configure the AWS provider to use your access key credentials in a `providers.tf ```tf provider "aws" { - version = "2.48.0" + version = "2.53.0" region = "eu-central-1" shared_credentials_file = "/home/user/.config/aws/credentials" } diff --git a/docs/cl/azure.md b/docs/cl/azure.md index 5f985e7c4..84da4c946 100644 --- a/docs/cl/azure.md +++ b/docs/cl/azure.md @@ -21,7 +21,7 @@ Install [Terraform](https://www.terraform.io/downloads.html) v0.12.6+ on your sy ```sh $ terraform version -Terraform v0.12.20 +Terraform v0.12.21 ``` Add the [terraform-provider-ct](https://github.com/poseidon/terraform-provider-ct) plugin binary for your system to `~/.terraform.d/plugins/`, noting the final name. @@ -50,7 +50,7 @@ Configure the Azure provider in a `providers.tf` file. ```tf provider "azurerm" { - version = "2.0.0" + version = "2.1.0" } provider "ct" { diff --git a/docs/cl/bare-metal.md b/docs/cl/bare-metal.md index d12884132..a9a24941c 100644 --- a/docs/cl/bare-metal.md +++ b/docs/cl/bare-metal.md @@ -111,7 +111,7 @@ Install [Terraform](https://www.terraform.io/downloads.html) v0.12.6+ on your sy ```sh $ terraform version -Terraform v0.12.20 +Terraform v0.12.21 ``` Add the [terraform-provider-matchbox](https://github.com/poseidon/terraform-provider-matchbox) plugin binary for your system to `~/.terraform.d/plugins/`, noting the final name. diff --git a/docs/cl/digital-ocean.md b/docs/cl/digital-ocean.md index 7b796b3bc..5777eb279 100644 --- a/docs/cl/digital-ocean.md +++ b/docs/cl/digital-ocean.md @@ -18,7 +18,7 @@ Install [Terraform](https://www.terraform.io/downloads.html) v0.12.6+ on your sy ```sh $ terraform version -Terraform v0.12.20 +Terraform v0.12.21 ``` Add the [terraform-provider-ct](https://github.com/poseidon/terraform-provider-ct) plugin binary for your system to `~/.terraform.d/plugins/`, noting the final name. diff --git a/docs/cl/google-cloud.md b/docs/cl/google-cloud.md index 6c50577d9..3126d2e4d 100644 --- a/docs/cl/google-cloud.md +++ b/docs/cl/google-cloud.md @@ -18,7 +18,7 @@ Install [Terraform](https://www.terraform.io/downloads.html) v0.12.6+ on your sy ```sh $ terraform version -Terraform v0.12.20 +Terraform v0.12.21 ``` Add the [terraform-provider-ct](https://github.com/poseidon/terraform-provider-ct) plugin binary for your system to `~/.terraform.d/plugins/`, noting the final name. @@ -49,7 +49,7 @@ Configure the Google Cloud provider to use your service account key, project-id, ```tf provider "google" { - version = "3.7.0" + version = "3.12.0" project = "project-id" region = "us-central1" credentials = file("~/.config/google-cloud/terraform.json") diff --git a/docs/fedora-coreos/aws.md b/docs/fedora-coreos/aws.md index 928d47683..01c2d698e 100644 --- a/docs/fedora-coreos/aws.md +++ b/docs/fedora-coreos/aws.md @@ -18,7 +18,7 @@ Install [Terraform](https://www.terraform.io/downloads.html) v0.12.6+ on your sy ```sh $ terraform version -Terraform v0.12.20 +Terraform v0.12.21 ``` Add the [terraform-provider-ct](https://github.com/poseidon/terraform-provider-ct) plugin binary for your system to `~/.terraform.d/plugins/`, noting the final name. @@ -49,7 +49,7 @@ Configure the AWS provider to use your access key credentials in a `providers.tf ```tf provider "aws" { - version = "2.48.0" + version = "2.53.0" region = "eu-central-1" shared_credentials_file = "/home/user/.config/aws/credentials" } diff --git a/docs/fedora-coreos/bare-metal.md b/docs/fedora-coreos/bare-metal.md index 3acd118db..e194732f8 100644 --- a/docs/fedora-coreos/bare-metal.md +++ b/docs/fedora-coreos/bare-metal.md @@ -111,7 +111,7 @@ Install [Terraform](https://www.terraform.io/downloads.html) v0.12.6+ on your sy ```sh $ terraform version -Terraform v0.12.20 +Terraform v0.12.21 ``` Add the [terraform-provider-matchbox](https://github.com/poseidon/terraform-provider-matchbox) plugin binary for your system to `~/.terraform.d/plugins/`, noting the final name. diff --git a/docs/fedora-coreos/google-cloud.md b/docs/fedora-coreos/google-cloud.md index 4a059d9fb..126c79ed0 100644 --- a/docs/fedora-coreos/google-cloud.md +++ b/docs/fedora-coreos/google-cloud.md @@ -21,7 +21,7 @@ Install [Terraform](https://www.terraform.io/downloads.html) v0.12.6+ on your sy ```sh $ terraform version -Terraform v0.12.20 +Terraform v0.12.21 ``` Add the [terraform-provider-ct](https://github.com/poseidon/terraform-provider-ct) plugin binary for your system to `~/.terraform.d/plugins/`, noting the final name. @@ -52,7 +52,7 @@ Configure the Google Cloud provider to use your service account key, project-id, ```tf provider "google" { - version = "3.7.0" + version = "3.12.0" project = "project-id" region = "us-central1" credentials = file("~/.config/google-cloud/terraform.json") From 75fb4e5d11e3abda948207a00002ec8ac3d42965 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Mon, 16 Mar 2020 21:57:45 -0700 Subject: [PATCH 357/523] Remove Container Linux Update Operator (CLUO) addon * Stop providing example manifests for the Container Linux Update Operator (CLUO) * CLUO requires patches to support Kubernetes v1.16+, but the project and push access is rather unowned * CLUO hasn't been in active use in our clusters and won't be relevant beyond Container Linux. Not to say folks can't patch it and run it on their own. Examples just aren't provided here Related: https://github.com/coreos/container-linux-update-operator/pull/197 --- CHANGES.md | 2 + addons/cluo/0-namespace.yaml | 4 -- addons/cluo/cluster-role-binding.yaml | 12 ----- addons/cluo/cluster-role.yaml | 45 ------------------ addons/cluo/update-agent.yaml | 68 --------------------------- addons/cluo/update-operator.yaml | 39 --------------- docs/addons/cluo.md | 29 ------------ docs/addons/overview.md | 1 - docs/cl/aws.md | 3 -- docs/cl/azure.md | 3 -- docs/cl/bare-metal.md | 3 -- docs/cl/digital-ocean.md | 3 -- docs/cl/google-cloud.md | 3 -- mkdocs.yml | 1 - 14 files changed, 2 insertions(+), 214 deletions(-) delete mode 100644 addons/cluo/0-namespace.yaml delete mode 100644 addons/cluo/cluster-role-binding.yaml delete mode 100644 addons/cluo/cluster-role.yaml delete mode 100644 addons/cluo/update-agent.yaml delete mode 100644 addons/cluo/update-operator.yaml delete mode 100644 docs/addons/cluo.md diff --git a/CHANGES.md b/CHANGES.md index 403f382ea..cc5257941 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -49,6 +49,8 @@ Notable changes between versions. * Update node-exporter from v0.18.1 to [v1.0.0-rc.0](https://github.com/prometheus/node_exporter/releases/tag/v1.0.0-rc.0) * Update Grafana from v6.6.1 to v6.6.2 * Refresh Grafana dashboards +* Remove Container Linux Update Operator (CLUO) addon + * CLUO doesn't support Kubernetes v1.16+, hasn't been in active use, and won't be relevant beyond Container Linux ## v1.17.3 diff --git a/addons/cluo/0-namespace.yaml b/addons/cluo/0-namespace.yaml deleted file mode 100644 index ab77e1f2e..000000000 --- a/addons/cluo/0-namespace.yaml +++ /dev/null @@ -1,4 +0,0 @@ -apiVersion: v1 -kind: Namespace -metadata: - name: reboot-coordinator diff --git a/addons/cluo/cluster-role-binding.yaml b/addons/cluo/cluster-role-binding.yaml deleted file mode 100644 index adf8f83b4..000000000 --- a/addons/cluo/cluster-role-binding.yaml +++ /dev/null @@ -1,12 +0,0 @@ -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: reboot-coordinator -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: reboot-coordinator -subjects: - - kind: ServiceAccount - namespace: reboot-coordinator - name: default diff --git a/addons/cluo/cluster-role.yaml b/addons/cluo/cluster-role.yaml deleted file mode 100644 index bbd86dd04..000000000 --- a/addons/cluo/cluster-role.yaml +++ /dev/null @@ -1,45 +0,0 @@ -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: reboot-coordinator -rules: - - apiGroups: - - "" - resources: - - nodes - verbs: - - get - - list - - watch - - update - - apiGroups: - - "" - resources: - - configmaps - verbs: - - create - - get - - update - - list - - watch - - apiGroups: - - "" - resources: - - events - verbs: - - create - - watch - - apiGroups: - - "" - resources: - - pods - verbs: - - get - - list - - delete - - apiGroups: - - "extensions" - resources: - - daemonsets - verbs: - - get diff --git a/addons/cluo/update-agent.yaml b/addons/cluo/update-agent.yaml deleted file mode 100644 index 1d0bef0b6..000000000 --- a/addons/cluo/update-agent.yaml +++ /dev/null @@ -1,68 +0,0 @@ -apiVersion: apps/v1 -kind: DaemonSet -metadata: - name: container-linux-update-agent - namespace: reboot-coordinator -spec: - updateStrategy: - type: RollingUpdate - rollingUpdate: - maxUnavailable: 1 - selector: - matchLabels: - name: container-linux-update-agent - template: - metadata: - labels: - name: container-linux-update-agent - annotations: - seccomp.security.alpha.kubernetes.io/pod: 'docker/default' - spec: - tolerations: - - key: node-role.kubernetes.io/master - operator: Exists - effect: NoSchedule - containers: - - name: update-agent - image: quay.io/coreos/container-linux-update-operator:v0.7.0 - command: - - "/bin/update-agent" - env: - # read by update-agent as the node name to manage reboots for - - name: UPDATE_AGENT_NODE - valueFrom: - fieldRef: - fieldPath: spec.nodeName - - name: POD_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - resources: - requests: - cpu: 10m - memory: 20Mi - limits: - cpu: 20m - memory: 40Mi - volumeMounts: - - mountPath: /var/run/dbus - name: var-run-dbus - - mountPath: /etc/coreos - name: etc-coreos - - mountPath: /usr/share/coreos - name: usr-share-coreos - - mountPath: /etc/os-release - name: etc-os-release - volumes: - - name: var-run-dbus - hostPath: - path: /var/run/dbus - - name: etc-coreos - hostPath: - path: /etc/coreos - - name: usr-share-coreos - hostPath: - path: /usr/share/coreos - - name: etc-os-release - hostPath: - path: /etc/os-release diff --git a/addons/cluo/update-operator.yaml b/addons/cluo/update-operator.yaml deleted file mode 100644 index d814626b7..000000000 --- a/addons/cluo/update-operator.yaml +++ /dev/null @@ -1,39 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: container-linux-update-operator - namespace: reboot-coordinator -spec: - replicas: 1 - selector: - matchLabels: - name: container-linux-update-operator - template: - metadata: - labels: - name: container-linux-update-operator - annotations: - seccomp.security.alpha.kubernetes.io/pod: 'docker/default' - spec: - tolerations: - - key: node-role.kubernetes.io/master - operator: Exists - effect: NoSchedule - containers: - - name: update-operator - image: quay.io/coreos/container-linux-update-operator:v0.7.0 - command: - - "/bin/update-operator" - env: - - name: POD_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - resources: - requests: - cpu: 10m - memory: 20Mi - limits: - cpu: 20m - memory: 40Mi - diff --git a/docs/addons/cluo.md b/docs/addons/cluo.md deleted file mode 100644 index e96ebde51..000000000 --- a/docs/addons/cluo.md +++ /dev/null @@ -1,29 +0,0 @@ -# Container Linux Update Operator - -The [Container Linux Update Operator](https://github.com/coreos/container-linux-update-operator) (i.e. CLUO) coordinates reboots of auto-updating Container Linux nodes so that one node reboots at a time and nodes are drained before reboot. CLUO enables the auto-update behavior Container Linux clusters are known for, but does so in a Kubernetes native way. - -## Create - -Create the `update-operator` deployment and `update-agent` DaemonSet. - -```sh -kubectl apply -f addons/cluo -R -``` - -## Usage - -`update-agent` runs as a DaemonSet and annotates a node when `update-engine.service` indicates an update has been installed and a reboot is needed. It also adds additional labels and annotations to nodes. - -``` -$ kubectl get nodes --show-labels -... -container-linux-update.v1.coreos.com/group=stable -container-linux-update.v1.coreos.com/version=1632.3.0 -``` - -`update-operator` ensures one node reboots at a time and that pods are drained prior to reboot. - -!!! note "" - CLUO replaces `locksmithd` reboot coordination. The `update_engine` systemd unit on hosts still performs the Container Linux update check, download, and install to the inactive partition. - - diff --git a/docs/addons/overview.md b/docs/addons/overview.md index c56fbd54d..e15988b7e 100644 --- a/docs/addons/overview.md +++ b/docs/addons/overview.md @@ -2,7 +2,6 @@ Every Typhoon cluster is verified to work well with several post-install addons. -* [CLUO](cluo.md) (Container Linux only) * Nginx [Ingress Controller](ingress.md) * [Prometheus](prometheus.md) * [Grafana](grafana.md) diff --git a/docs/cl/aws.md b/docs/cl/aws.md index c67700804..75a0f77b9 100644 --- a/docs/cl/aws.md +++ b/docs/cl/aws.md @@ -170,9 +170,6 @@ kube-system kube-scheduler-ip-10-0-3-155 1/1 Running 1 Learn about [maintenance](/topics/maintenance/) and [addons](/addons/overview/). -!!! note - On Container Linux clusters, install the `CLUO` addon to coordinate reboots and drains when nodes auto-update. Otherwise, updates may not be applied until the next reboot. - ## Variables Check the [variables.tf](https://github.com/poseidon/typhoon/blob/master/aws/container-linux/kubernetes/variables.tf) source. diff --git a/docs/cl/azure.md b/docs/cl/azure.md index 84da4c946..9ca478fb3 100644 --- a/docs/cl/azure.md +++ b/docs/cl/azure.md @@ -176,9 +176,6 @@ kube-system kube-scheduler-ramius-controller-0 1/1 Running 0 Learn about [maintenance](/topics/maintenance/) and [addons](/addons/overview/). -!!! note - On Container Linux clusters, install the `CLUO` addon to coordinate reboots and drains when nodes auto-update. Otherwise, updates may not be applied until the next reboot. - ## Variables Check the [variables.tf](https://github.com/poseidon/typhoon/blob/master/azure/container-linux/kubernetes/variables.tf) source. diff --git a/docs/cl/bare-metal.md b/docs/cl/bare-metal.md index a9a24941c..a1907319b 100644 --- a/docs/cl/bare-metal.md +++ b/docs/cl/bare-metal.md @@ -326,9 +326,6 @@ kube-system kube-scheduler-node1.example.com 1/1 Running 0 Learn about [maintenance](/topics/maintenance/) and [addons](/addons/overview/). -!!! note - On Container Linux clusters, install the `CLUO` addon to coordinate reboots and drains when nodes auto-update. Otherwise, updates may not be applied until the next reboot. - ## Variables Check the [variables.tf](https://github.com/poseidon/typhoon/blob/master/bare-metal/container-linux/kubernetes/variables.tf) source. diff --git a/docs/cl/digital-ocean.md b/docs/cl/digital-ocean.md index 5777eb279..9ca49a625 100644 --- a/docs/cl/digital-ocean.md +++ b/docs/cl/digital-ocean.md @@ -187,9 +187,6 @@ kube-system kube-scheduler-ip-10.132.115.81 1/1 Running 0 Learn about [maintenance](/topics/maintenance/) and [addons](/addons/overview/). -!!! note - On Container Linux clusters, install the `CLUO` addon to coordinate reboots and drains when nodes auto-update. Otherwise, updates may not be applied until the next reboot. - ## Variables Check the [variables.tf](https://github.com/poseidon/typhoon/blob/master/digital-ocean/container-linux/kubernetes/variables.tf) source. diff --git a/docs/cl/google-cloud.md b/docs/cl/google-cloud.md index 3126d2e4d..5b041d1c3 100644 --- a/docs/cl/google-cloud.md +++ b/docs/cl/google-cloud.md @@ -194,9 +194,6 @@ kube-system kube-scheduler-controller-0 1/1 Running 0 Learn about [maintenance](/topics/maintenance/) and [addons](/addons/overview/). -!!! note - On Container Linux clusters, install the `CLUO` addon to coordinate reboots and drains when nodes auto-update. Otherwise, updates may not be applied until the next reboot. - ## Variables Check the [variables.tf](https://github.com/poseidon/typhoon/blob/master/google-cloud/container-linux/kubernetes/variables.tf) source. diff --git a/mkdocs.yml b/mkdocs.yml index aef695cc7..45d35c0c3 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -75,7 +75,6 @@ nav: - 'Worker Pools': 'advanced/worker-pools.md' - 'Addons': - 'Overview': 'addons/overview.md' - - 'CLUO': 'addons/cluo.md' - 'Nginx Ingress': 'addons/ingress.md' - 'Prometheus': 'addons/prometheus.md' - 'Grafana': 'addons/grafana.md' From 2a5dddeb9d7a26a730fa76707db8bef105d1f8b0 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Mon, 16 Mar 2020 22:10:34 -0700 Subject: [PATCH 358/523] Promote Fedora CoreOS AWS and Google Cloud * Promote Fedora CoreOS AWS to stable * Promote Fedora CoreOS GCP to beta --- CHANGES.md | 7 +++++-- README.md | 4 ++-- docs/index.md | 4 ++-- 3 files changed, 9 insertions(+), 6 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index cc5257941..3faf0a50f 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -11,6 +11,7 @@ Notable changes between versions. #### AWS +* Promote Fedora CoreOS to stable ([#668](https://github.com/poseidon/typhoon/pull/668)) * Allow VPC route table extension via reference ([#654](https://github.com/poseidon/typhoon/pull/654)) * Fix `worker_node_labels` on Fedora CoreOS ([#651](https://github.com/poseidon/typhoon/pull/651)) * Fix automatic worker node delete on shutdown on Fedora CoreOS ([#657](https://github.com/poseidon/typhoon/pull/657)) @@ -36,6 +37,7 @@ Notable changes between versions. #### Google Cloud +* Promote Fedora CoreOS to beta ([#668](https://github.com/poseidon/typhoon/pull/668)) * Fix `worker_node_labels` on Fedora CoreOS ([#651](https://github.com/poseidon/typhoon/pull/651)) * Fix automatic worker node delete on shutdown on Fedora CoreOS ([#657](https://github.com/poseidon/typhoon/pull/657)) @@ -49,8 +51,9 @@ Notable changes between versions. * Update node-exporter from v0.18.1 to [v1.0.0-rc.0](https://github.com/prometheus/node_exporter/releases/tag/v1.0.0-rc.0) * Update Grafana from v6.6.1 to v6.6.2 * Refresh Grafana dashboards -* Remove Container Linux Update Operator (CLUO) addon - * CLUO doesn't support Kubernetes v1.16+, hasn't been in active use, and won't be relevant beyond Container Linux +* Remove Container Linux Update Operator (CLUO) addon example ([#667](https://github.com/poseidon/typhoon/pull/667)) + * CLUO hasn't been in active use in our clusters and won't be relevant + beyond Container Linux. Requires patches for use on Kubernetes v1.16+ ## v1.17.3 diff --git a/README.md b/README.md index 590009a10..9cb9f908a 100644 --- a/README.md +++ b/README.md @@ -33,9 +33,9 @@ Typhoon is available for [Fedora CoreOS](https://getfedora.org/coreos/). | Platform | Operating System | Terraform Module | Status | |---------------|------------------|------------------|--------| -| AWS | Fedora CoreOS | [aws/fedora-coreos/kubernetes](aws/fedora-coreos/kubernetes) | beta | +| AWS | Fedora CoreOS | [aws/fedora-coreos/kubernetes](aws/fedora-coreos/kubernetes) | stable | | Bare-Metal | Fedora CoreOS | [bare-metal/fedora-coreos/kubernetes](bare-metal/fedora-coreos/kubernetes) | beta | -| Google Cloud | Fedora CoreOS | [google-cloud/fedora-coreos/kubernetes](google-cloud/fedora-coreos/kubernetes) | alpha | +| Google Cloud | Fedora CoreOS | [google-cloud/fedora-coreos/kubernetes](google-cloud/fedora-coreos/kubernetes) | beta | Typhoon is available for [Flatcar Container Linux](https://www.flatcar-linux.org/releases/). diff --git a/docs/index.md b/docs/index.md index df5bf6675..3da5d32d2 100644 --- a/docs/index.md +++ b/docs/index.md @@ -33,9 +33,9 @@ Typhoon is available for [Fedora CoreOS](https://getfedora.org/coreos/). | Platform | Operating System | Terraform Module | Status | |---------------|------------------|------------------|--------| -| AWS | Fedora CoreOS | [aws/fedora-coreos/kubernetes](fedora-coreos/aws.md) | beta | +| AWS | Fedora CoreOS | [aws/fedora-coreos/kubernetes](fedora-coreos/aws.md) | stable | | Bare-Metal | Fedora CoreOS | [bare-metal/fedora-coreos/kubernetes](fedora-coreos/bare-metal.md) | beta | -| Google Cloud | Fedora CoreOS | [google-cloud/fedora-coreos/kubernetes](google-cloud/fedora-coreos/kubernetes) | alpha | +| Google Cloud | Fedora CoreOS | [google-cloud/fedora-coreos/kubernetes](google-cloud/fedora-coreos/kubernetes) | beta | Typhoon is available for [Flatcar Container Linux](https://www.flatcar-linux.org/releases/). From c3ef21dbf5d2887034bb8fb8ac56dd90b622dc2e Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Wed, 18 Mar 2020 20:50:41 -0700 Subject: [PATCH 359/523] Update etcd from v3.4.4 to v3.4.5 * https://github.com/etcd-io/etcd/releases/tag/v3.4.5 --- CHANGES.md | 4 ++++ aws/container-linux/kubernetes/cl/controller.yaml | 2 +- aws/fedora-coreos/kubernetes/fcc/controller.yaml | 2 +- azure/container-linux/kubernetes/cl/controller.yaml | 2 +- bare-metal/container-linux/kubernetes/cl/controller.yaml | 2 +- bare-metal/fedora-coreos/kubernetes/fcc/controller.yaml | 2 +- digital-ocean/container-linux/kubernetes/cl/controller.yaml | 2 +- google-cloud/container-linux/kubernetes/cl/controller.yaml | 2 +- google-cloud/fedora-coreos/kubernetes/fcc/controller.yaml | 2 +- 9 files changed, 12 insertions(+), 8 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 3faf0a50f..44a84db57 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -4,6 +4,10 @@ Notable changes between versions. ## Latest +* Update etcd from v3.4.4 to [v3.4.5](https://github.com/etcd-io/etcd/releases/tag/v3.4.5) + +## v1.17.4 + * Update etcd from v3.4.3 to [v3.4.4](https://github.com/etcd-io/etcd/releases/tag/v3.4.4) * On Container Linux, fetch using the docker transport format ([#659](https://github.com/poseidon/typhoon/pull/659)) * Update CoreDNS from v1.6.6 to v1.6.7 ([#648](https://github.com/poseidon/typhoon/pull/648)) diff --git a/aws/container-linux/kubernetes/cl/controller.yaml b/aws/container-linux/kubernetes/cl/controller.yaml index ee96a5768..b0a4258c7 100644 --- a/aws/container-linux/kubernetes/cl/controller.yaml +++ b/aws/container-linux/kubernetes/cl/controller.yaml @@ -7,7 +7,7 @@ systemd: - name: 40-etcd-cluster.conf contents: | [Service] - Environment="ETCD_IMAGE_TAG=v3.4.4" + Environment="ETCD_IMAGE_TAG=v3.4.5" Environment="ETCD_IMAGE_URL=docker://quay.io/coreos/etcd" Environment="RKT_RUN_ARGS=--insecure-options=image" Environment="ETCD_NAME=${etcd_name}" diff --git a/aws/fedora-coreos/kubernetes/fcc/controller.yaml b/aws/fedora-coreos/kubernetes/fcc/controller.yaml index aeed1df67..93297d7d0 100644 --- a/aws/fedora-coreos/kubernetes/fcc/controller.yaml +++ b/aws/fedora-coreos/kubernetes/fcc/controller.yaml @@ -28,7 +28,7 @@ systemd: --network host \ --volume /var/lib/etcd:/var/lib/etcd:rw,Z \ --volume /etc/ssl/etcd:/etc/ssl/certs:ro,Z \ - quay.io/coreos/etcd:v3.4.4 + quay.io/coreos/etcd:v3.4.5 ExecStop=/usr/bin/podman stop etcd [Install] WantedBy=multi-user.target diff --git a/azure/container-linux/kubernetes/cl/controller.yaml b/azure/container-linux/kubernetes/cl/controller.yaml index b5035dc0d..4cafaa006 100644 --- a/azure/container-linux/kubernetes/cl/controller.yaml +++ b/azure/container-linux/kubernetes/cl/controller.yaml @@ -7,7 +7,7 @@ systemd: - name: 40-etcd-cluster.conf contents: | [Service] - Environment="ETCD_IMAGE_TAG=v3.4.4" + Environment="ETCD_IMAGE_TAG=v3.4.5" Environment="ETCD_IMAGE_URL=docker://quay.io/coreos/etcd" Environment="RKT_RUN_ARGS=--insecure-options=image" Environment="ETCD_NAME=${etcd_name}" diff --git a/bare-metal/container-linux/kubernetes/cl/controller.yaml b/bare-metal/container-linux/kubernetes/cl/controller.yaml index 5589f76de..4f90796a5 100644 --- a/bare-metal/container-linux/kubernetes/cl/controller.yaml +++ b/bare-metal/container-linux/kubernetes/cl/controller.yaml @@ -7,7 +7,7 @@ systemd: - name: 40-etcd-cluster.conf contents: | [Service] - Environment="ETCD_IMAGE_TAG=v3.4.4" + Environment="ETCD_IMAGE_TAG=v3.4.5" Environment="ETCD_IMAGE_URL=docker://quay.io/coreos/etcd" Environment="RKT_RUN_ARGS=--insecure-options=image" Environment="ETCD_NAME=${etcd_name}" diff --git a/bare-metal/fedora-coreos/kubernetes/fcc/controller.yaml b/bare-metal/fedora-coreos/kubernetes/fcc/controller.yaml index 89b778001..1fd0d53be 100644 --- a/bare-metal/fedora-coreos/kubernetes/fcc/controller.yaml +++ b/bare-metal/fedora-coreos/kubernetes/fcc/controller.yaml @@ -28,7 +28,7 @@ systemd: --network host \ --volume /var/lib/etcd:/var/lib/etcd:rw,Z \ --volume /etc/ssl/etcd:/etc/ssl/certs:ro,Z \ - quay.io/coreos/etcd:v3.4.4 + quay.io/coreos/etcd:v3.4.5 ExecStop=/usr/bin/podman stop etcd [Install] WantedBy=multi-user.target diff --git a/digital-ocean/container-linux/kubernetes/cl/controller.yaml b/digital-ocean/container-linux/kubernetes/cl/controller.yaml index 40930c031..acd643afa 100644 --- a/digital-ocean/container-linux/kubernetes/cl/controller.yaml +++ b/digital-ocean/container-linux/kubernetes/cl/controller.yaml @@ -7,7 +7,7 @@ systemd: - name: 40-etcd-cluster.conf contents: | [Service] - Environment="ETCD_IMAGE_TAG=v3.4.4" + Environment="ETCD_IMAGE_TAG=v3.4.5" Environment="ETCD_IMAGE_URL=docker://quay.io/coreos/etcd" Environment="RKT_RUN_ARGS=--insecure-options=image" Environment="ETCD_NAME=${etcd_name}" diff --git a/google-cloud/container-linux/kubernetes/cl/controller.yaml b/google-cloud/container-linux/kubernetes/cl/controller.yaml index 5966bff1d..cdf1c57ed 100644 --- a/google-cloud/container-linux/kubernetes/cl/controller.yaml +++ b/google-cloud/container-linux/kubernetes/cl/controller.yaml @@ -7,7 +7,7 @@ systemd: - name: 40-etcd-cluster.conf contents: | [Service] - Environment="ETCD_IMAGE_TAG=v3.4.4" + Environment="ETCD_IMAGE_TAG=v3.4.5" Environment="ETCD_IMAGE_URL=docker://quay.io/coreos/etcd" Environment="RKT_RUN_ARGS=--insecure-options=image" Environment="ETCD_NAME=${etcd_name}" diff --git a/google-cloud/fedora-coreos/kubernetes/fcc/controller.yaml b/google-cloud/fedora-coreos/kubernetes/fcc/controller.yaml index aeed1df67..93297d7d0 100644 --- a/google-cloud/fedora-coreos/kubernetes/fcc/controller.yaml +++ b/google-cloud/fedora-coreos/kubernetes/fcc/controller.yaml @@ -28,7 +28,7 @@ systemd: --network host \ --volume /var/lib/etcd:/var/lib/etcd:rw,Z \ --volume /etc/ssl/etcd:/etc/ssl/certs:ro,Z \ - quay.io/coreos/etcd:v3.4.4 + quay.io/coreos/etcd:v3.4.5 ExecStop=/usr/bin/podman stop etcd [Install] WantedBy=multi-user.target From 61557e89a66248f846250f0faa7dd5138f54f9f7 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Thu, 19 Mar 2020 22:38:05 -0700 Subject: [PATCH 360/523] Update Prometheus from v2.16.0 to v2.17.0-rc.3 * https://github.com/prometheus/prometheus/releases/tag/v2.17.0-rc.3 --- CHANGES.md | 4 ++++ addons/prometheus/deployment.yaml | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index 44a84db57..7e55b4295 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -6,6 +6,10 @@ Notable changes between versions. * Update etcd from v3.4.4 to [v3.4.5](https://github.com/etcd-io/etcd/releases/tag/v3.4.5) +#### Addons + +* Update Prometheus from v2.16.0 to [v2.17.0-rc.3](https://github.com/prometheus/prometheus/releases/tag/v2.17.0-rc.3) + ## v1.17.4 * Update etcd from v3.4.3 to [v3.4.4](https://github.com/etcd-io/etcd/releases/tag/v3.4.4) diff --git a/addons/prometheus/deployment.yaml b/addons/prometheus/deployment.yaml index 02c2e2810..742fad8aa 100644 --- a/addons/prometheus/deployment.yaml +++ b/addons/prometheus/deployment.yaml @@ -20,7 +20,7 @@ spec: serviceAccountName: prometheus containers: - name: prometheus - image: quay.io/prometheus/prometheus:v2.16.0 + image: quay.io/prometheus/prometheus:v2.17.0-rc.3 args: - --web.listen-address=0.0.0.0:9090 - --config.file=/etc/prometheus/prometheus.yaml From ddc1ff53488c7d8862b09708174c8df659019eef Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Thu, 19 Mar 2020 22:58:59 -0700 Subject: [PATCH 361/523] Update Grafana from v6.6.2 to v6.7.1 * https://github.com/grafana/grafana/releases/tag/v6.7.1 --- CHANGES.md | 1 + addons/grafana/deployment.yaml | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index 7e55b4295..3bd9663e4 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -9,6 +9,7 @@ Notable changes between versions. #### Addons * Update Prometheus from v2.16.0 to [v2.17.0-rc.3](https://github.com/prometheus/prometheus/releases/tag/v2.17.0-rc.3) +* Update Grafana from v6.6.2 to [v6.7.1](https://github.com/grafana/grafana/releases/tag/v6.7.1) ## v1.17.4 diff --git a/addons/grafana/deployment.yaml b/addons/grafana/deployment.yaml index 813e2dd80..369bc4941 100644 --- a/addons/grafana/deployment.yaml +++ b/addons/grafana/deployment.yaml @@ -23,7 +23,7 @@ spec: spec: containers: - name: grafana - image: docker.io/grafana/grafana:6.6.2 + image: docker.io/grafana/grafana:6.7.1 env: - name: GF_PATHS_CONFIG value: "/etc/grafana/custom.ini" From 590d941f50dc77815c9e4c3b6ec27678c3638325 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Mon, 16 Mar 2020 21:21:41 -0700 Subject: [PATCH 362/523] Switch from upstream hyperkube image to individual images * Kubernetes plans to stop releasing the hyperkube container image * Upstream will continue to publish `kube-apiserver`, `kube-controller-manager`, `kube-scheduler`, and `kube-proxy` container images to `k8s.gcr.io` * Upstream will publish Kubelet only as a binary for distros to package, either as a DEB/RPM on traditional distros or a container image on container-optimized operating systems * Typhoon will package the upstream Kubelet (checksummed) and its dependencies as a container image for use on CoreOS Container Linux, Flatcar Linux, and Fedora CoreOS * Update the Typhoon container image security policy to list `quay.io/poseidon/kubelet`as an official distributed artifact Hyperkube: https://github.com/kubernetes/kubernetes/pull/88676 Kubelet Container Image: https://github.com/poseidon/kubelet Kubelet Quay Repo: https://quay.io/repository/poseidon/kubelet --- CHANGES.md | 9 +++++++++ aws/container-linux/kubernetes/bootstrap.tf | 2 +- aws/container-linux/kubernetes/cl/controller.yaml | 5 ++--- aws/container-linux/kubernetes/workers/cl/worker.yaml | 11 +++++------ aws/fedora-coreos/kubernetes/bootstrap.tf | 2 +- aws/fedora-coreos/kubernetes/fcc/controller.yaml | 4 ++-- aws/fedora-coreos/kubernetes/workers/fcc/worker.yaml | 4 ++-- azure/container-linux/kubernetes/bootstrap.tf | 2 +- azure/container-linux/kubernetes/cl/controller.yaml | 5 ++--- .../container-linux/kubernetes/workers/cl/worker.yaml | 11 +++++------ bare-metal/container-linux/kubernetes/bootstrap.tf | 2 +- .../container-linux/kubernetes/cl/controller.yaml | 5 ++--- bare-metal/container-linux/kubernetes/cl/worker.yaml | 3 +-- bare-metal/fedora-coreos/kubernetes/bootstrap.tf | 2 +- .../fedora-coreos/kubernetes/fcc/controller.yaml | 4 ++-- bare-metal/fedora-coreos/kubernetes/fcc/worker.yaml | 2 +- digital-ocean/container-linux/kubernetes/bootstrap.tf | 2 +- .../container-linux/kubernetes/cl/controller.yaml | 5 ++--- .../container-linux/kubernetes/cl/worker.yaml | 7 +++---- docs/architecture/operating-systems.md | 4 ++-- docs/topics/security.md | 9 +++++++-- google-cloud/container-linux/kubernetes/bootstrap.tf | 2 +- .../container-linux/kubernetes/cl/controller.yaml | 5 ++--- .../container-linux/kubernetes/workers/cl/worker.yaml | 11 +++++------ google-cloud/fedora-coreos/kubernetes/bootstrap.tf | 2 +- .../fedora-coreos/kubernetes/fcc/controller.yaml | 4 ++-- .../fedora-coreos/kubernetes/workers/fcc/worker.yaml | 4 ++-- 27 files changed, 66 insertions(+), 62 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 3bd9663e4..f19f4305d 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -5,6 +5,15 @@ Notable changes between versions. ## Latest * Update etcd from v3.4.4 to [v3.4.5](https://github.com/etcd-io/etcd/releases/tag/v3.4.5) +* Switch from upstream hyperkube image to individual images ([#669](https://github.com/poseidon/typhoon/pull/669)) + * Use upstream `k8s.gcr.io` `kube-apiserver`, `kube-controller-manager`, `kube-scheduler`, and `kube-proxy` container images + * Use [poseidon/kubelet](https://github.com/poseidon/kubelet) to package the upstream Kubelet binary (checksummed) and + its dependencies as an automated build container image [quay.io/poseidon/kubelet](https://quay.io/repository/poseidon/kubelet) + * Update base images for control plane and Kubelet images used by Typhoon from upstream's debian 9 (stretch) to debian 10 + (buster) base + * Update Typhoon container image security policy to list `quay.io/poseidon/kubelet`as an official distributed artifact + * Background: Kubernetes will [stop releasing](https://github.com/kubernetes/kubernetes/pull/88676) the hyperkube container + image and provide the Kubelet as a binary for distros to package #### Addons diff --git a/aws/container-linux/kubernetes/bootstrap.tf b/aws/container-linux/kubernetes/bootstrap.tf index f55e011d1..654312479 100644 --- a/aws/container-linux/kubernetes/bootstrap.tf +++ b/aws/container-linux/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=73784c1b2c791d9ba586a1478979ac34dd324dad" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=e76f0a09fa9e6421a9cf697ee03714c6224e2581" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/aws/container-linux/kubernetes/cl/controller.yaml b/aws/container-linux/kubernetes/cl/controller.yaml index b0a4258c7..59b634f80 100644 --- a/aws/container-linux/kubernetes/cl/controller.yaml +++ b/aws/container-linux/kubernetes/cl/controller.yaml @@ -91,8 +91,7 @@ systemd: --mount volume=var-log,target=/var/log \ --volume opt-cni-bin,kind=host,source=/opt/cni/bin \ --mount volume=opt-cni-bin,target=/opt/cni/bin \ - docker://k8s.gcr.io/hyperkube:v1.17.4 \ - --exec=/usr/local/bin/kubelet -- \ + docker://quay.io/poseidon/kubelet:v1.17.4 -- \ --anonymous-auth=false \ --authentication-token-webhook \ --authorization-mode=Webhook \ @@ -136,7 +135,7 @@ systemd: --volume script,kind=host,source=/opt/bootstrap/apply \ --mount volume=script,target=/apply \ --insecure-options=image \ - docker://k8s.gcr.io/hyperkube:v1.17.4 \ + docker://quay.io/poseidon/kubelet:v1.17.4 \ --net=host \ --dns=host \ --exec=/apply diff --git a/aws/container-linux/kubernetes/workers/cl/worker.yaml b/aws/container-linux/kubernetes/workers/cl/worker.yaml index a41dc8538..a7733b7e5 100644 --- a/aws/container-linux/kubernetes/workers/cl/worker.yaml +++ b/aws/container-linux/kubernetes/workers/cl/worker.yaml @@ -64,8 +64,7 @@ systemd: --mount volume=var-log,target=/var/log \ --volume opt-cni-bin,kind=host,source=/opt/cni/bin \ --mount volume=opt-cni-bin,target=/opt/cni/bin \ - docker://k8s.gcr.io/hyperkube:v1.17.4 \ - --exec=/usr/local/bin/kubelet -- \ + docker://quay.io/poseidon/kubelet:v1.17.4 -- \ --anonymous-auth=false \ --authentication-token-webhook \ --authorization-mode=Webhook \ @@ -80,9 +79,9 @@ systemd: --lock-file=/var/run/lock/kubelet.lock \ --network-plugin=cni \ --node-labels=node.kubernetes.io/node \ - %{ for label in split(",", node_labels) } + %{~ for label in split(",", node_labels) ~} --node-labels=${label} \ - %{ endfor ~} + %{~ endfor ~} --pod-manifest-path=/etc/kubernetes/manifests \ --read-only-port=0 \ --volume-plugin-dir=/var/lib/kubelet/volumeplugins @@ -128,11 +127,11 @@ storage: --volume config,kind=host,source=/etc/kubernetes \ --mount volume=config,target=/etc/kubernetes \ --insecure-options=image \ - docker://k8s.gcr.io/hyperkube:v1.17.4 \ + docker://quay.io/poseidon/kubelet:612b947 \ --net=host \ --dns=host \ -- \ - kubectl --kubeconfig=/etc/kubernetes/kubeconfig delete node $(hostname) + /usr/local/bin/kubectl --kubeconfig=/etc/kubernetes/kubeconfig delete node $(hostname) passwd: users: - name: core diff --git a/aws/fedora-coreos/kubernetes/bootstrap.tf b/aws/fedora-coreos/kubernetes/bootstrap.tf index 556cc760f..5036643ef 100644 --- a/aws/fedora-coreos/kubernetes/bootstrap.tf +++ b/aws/fedora-coreos/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=73784c1b2c791d9ba586a1478979ac34dd324dad" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=e76f0a09fa9e6421a9cf697ee03714c6224e2581" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/aws/fedora-coreos/kubernetes/fcc/controller.yaml b/aws/fedora-coreos/kubernetes/fcc/controller.yaml index 93297d7d0..b9cac354f 100644 --- a/aws/fedora-coreos/kubernetes/fcc/controller.yaml +++ b/aws/fedora-coreos/kubernetes/fcc/controller.yaml @@ -79,7 +79,7 @@ systemd: --volume /var/log:/var/log \ --volume /var/run/lock:/var/run/lock:z \ --volume /opt/cni/bin:/opt/cni/bin:z \ - k8s.gcr.io/hyperkube:v1.17.4 kubelet \ + quay.io/poseidon/kubelet:v1.17.4 \ --anonymous-auth=false \ --authentication-token-webhook \ --authorization-mode=Webhook \ @@ -123,7 +123,7 @@ systemd: --volume /opt/bootstrap/assets:/assets:ro,Z \ --volume /opt/bootstrap/apply:/apply:ro,Z \ --entrypoint=/apply \ - k8s.gcr.io/hyperkube:v1.17.4 + quay.io/poseidon/kubelet:v1.17.4 ExecStartPost=/bin/touch /opt/bootstrap/bootstrap.done ExecStartPost=-/usr/bin/podman stop bootstrap storage: diff --git a/aws/fedora-coreos/kubernetes/workers/fcc/worker.yaml b/aws/fedora-coreos/kubernetes/workers/fcc/worker.yaml index 07492c3e9..c840da1d9 100644 --- a/aws/fedora-coreos/kubernetes/workers/fcc/worker.yaml +++ b/aws/fedora-coreos/kubernetes/workers/fcc/worker.yaml @@ -49,7 +49,7 @@ systemd: --volume /var/log:/var/log \ --volume /var/run/lock:/var/run/lock:z \ --volume /opt/cni/bin:/opt/cni/bin:z \ - k8s.gcr.io/hyperkube:v1.17.4 kubelet \ + quay.io/poseidon/kubelet:v1.17.4 \ --anonymous-auth=false \ --authentication-token-webhook \ --authorization-mode=Webhook \ @@ -87,7 +87,7 @@ systemd: Type=oneshot RemainAfterExit=true ExecStart=/bin/true - ExecStop=/bin/bash -c '/usr/bin/podman run --volume /etc/kubernetes:/etc/kubernetes:ro,z k8s.gcr.io/hyperkube:v1.17.4 kubectl --kubeconfig=/etc/kubernetes/kubeconfig delete node $HOSTNAME' + ExecStop=/bin/bash -c '/usr/bin/podman run --volume /etc/kubernetes:/etc/kubernetes:ro,z --entrypoint /usr/local/bin/kubectl quay.io/poseidon/kubelet:v1.17.4 --kubeconfig=/etc/kubernetes/kubeconfig delete node $HOSTNAME' [Install] WantedBy=multi-user.target storage: diff --git a/azure/container-linux/kubernetes/bootstrap.tf b/azure/container-linux/kubernetes/bootstrap.tf index d7656e2c8..3f8118a67 100644 --- a/azure/container-linux/kubernetes/bootstrap.tf +++ b/azure/container-linux/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=73784c1b2c791d9ba586a1478979ac34dd324dad" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=e76f0a09fa9e6421a9cf697ee03714c6224e2581" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/azure/container-linux/kubernetes/cl/controller.yaml b/azure/container-linux/kubernetes/cl/controller.yaml index 4cafaa006..a8dc507d1 100644 --- a/azure/container-linux/kubernetes/cl/controller.yaml +++ b/azure/container-linux/kubernetes/cl/controller.yaml @@ -90,8 +90,7 @@ systemd: --mount volume=var-log,target=/var/log \ --volume opt-cni-bin,kind=host,source=/opt/cni/bin \ --mount volume=opt-cni-bin,target=/opt/cni/bin \ - docker://k8s.gcr.io/hyperkube:v1.17.4 \ - --exec=/usr/local/bin/kubelet -- \ + docker://quay.io/poseidon/kubelet:v1.17.4 -- \ --anonymous-auth=false \ --authentication-token-webhook \ --authorization-mode=Webhook \ @@ -134,7 +133,7 @@ systemd: --volume script,kind=host,source=/opt/bootstrap/apply \ --mount volume=script,target=/apply \ --insecure-options=image \ - docker://k8s.gcr.io/hyperkube:v1.17.4 \ + docker://quay.io/poseidon/kubelet:v1.17.4 \ --net=host \ --dns=host \ --exec=/apply diff --git a/azure/container-linux/kubernetes/workers/cl/worker.yaml b/azure/container-linux/kubernetes/workers/cl/worker.yaml index 749cb9bc7..6ca5794fe 100644 --- a/azure/container-linux/kubernetes/workers/cl/worker.yaml +++ b/azure/container-linux/kubernetes/workers/cl/worker.yaml @@ -63,8 +63,7 @@ systemd: --mount volume=var-log,target=/var/log \ --volume opt-cni-bin,kind=host,source=/opt/cni/bin \ --mount volume=opt-cni-bin,target=/opt/cni/bin \ - docker://k8s.gcr.io/hyperkube:v1.17.4 \ - --exec=/usr/local/bin/kubelet -- \ + docker://quay.io/poseidon/kubelet:v1.17.4 -- \ --anonymous-auth=false \ --authentication-token-webhook \ --authorization-mode=Webhook \ @@ -78,9 +77,9 @@ systemd: --lock-file=/var/run/lock/kubelet.lock \ --network-plugin=cni \ --node-labels=node.kubernetes.io/node \ - %{ for label in split(",", node_labels) } + %{~ for label in split(",", node_labels) ~} --node-labels=${label} \ - %{ endfor ~} + %{~ endfor ~} --pod-manifest-path=/etc/kubernetes/manifests \ --read-only-port=0 \ --volume-plugin-dir=/var/lib/kubelet/volumeplugins @@ -126,11 +125,11 @@ storage: --volume config,kind=host,source=/etc/kubernetes \ --mount volume=config,target=/etc/kubernetes \ --insecure-options=image \ - docker://k8s.gcr.io/hyperkube:v1.17.4 \ + docker://quay.io/poseidon/kubelet:v1.17.4 \ --net=host \ --dns=host \ -- \ - kubectl -- --kubeconfig=/etc/kubernetes/kubeconfig delete node $(hostname | tr '[:upper:]' '[:lower:]') + /usr/local/bin/kubectl -- --kubeconfig=/etc/kubernetes/kubeconfig delete node $(hostname | tr '[:upper:]' '[:lower:]') passwd: users: - name: core diff --git a/bare-metal/container-linux/kubernetes/bootstrap.tf b/bare-metal/container-linux/kubernetes/bootstrap.tf index 25c4f6863..f60e7b21f 100644 --- a/bare-metal/container-linux/kubernetes/bootstrap.tf +++ b/bare-metal/container-linux/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=73784c1b2c791d9ba586a1478979ac34dd324dad" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=e76f0a09fa9e6421a9cf697ee03714c6224e2581" cluster_name = var.cluster_name api_servers = [var.k8s_domain_name] diff --git a/bare-metal/container-linux/kubernetes/cl/controller.yaml b/bare-metal/container-linux/kubernetes/cl/controller.yaml index 4f90796a5..db0b430bb 100644 --- a/bare-metal/container-linux/kubernetes/cl/controller.yaml +++ b/bare-metal/container-linux/kubernetes/cl/controller.yaml @@ -103,8 +103,7 @@ systemd: --mount volume=etc-iscsi,target=/etc/iscsi \ --volume usr-sbin-iscsiadm,kind=host,source=/usr/sbin/iscsiadm \ --mount volume=usr-sbin-iscsiadm,target=/sbin/iscsiadm \ - docker://k8s.gcr.io/hyperkube:v1.17.4 \ - --exec=/usr/local/bin/kubelet -- \ + docker://quay.io/poseidon/kubelet:v1.17.4 -- \ --anonymous-auth=false \ --authentication-token-webhook \ --authorization-mode=Webhook \ @@ -149,7 +148,7 @@ systemd: --volume script,kind=host,source=/opt/bootstrap/apply \ --mount volume=script,target=/apply \ --insecure-options=image \ - docker://k8s.gcr.io/hyperkube:v1.17.4 \ + docker://quay.io/poseidon/kubelet:v1.17.4 \ --net=host \ --dns=host \ --exec=/apply diff --git a/bare-metal/container-linux/kubernetes/cl/worker.yaml b/bare-metal/container-linux/kubernetes/cl/worker.yaml index f4b923d37..094504356 100644 --- a/bare-metal/container-linux/kubernetes/cl/worker.yaml +++ b/bare-metal/container-linux/kubernetes/cl/worker.yaml @@ -76,8 +76,7 @@ systemd: --mount volume=etc-iscsi,target=/etc/iscsi \ --volume usr-sbin-iscsiadm,kind=host,source=/usr/sbin/iscsiadm \ --mount volume=usr-sbin-iscsiadm,target=/sbin/iscsiadm \ - docker://k8s.gcr.io/hyperkube:v1.17.4 \ - --exec=/usr/local/bin/kubelet -- \ + docker://quay.io/poseidon/kubelet:v1.17.4 -- \ --anonymous-auth=false \ --authentication-token-webhook \ --authorization-mode=Webhook \ diff --git a/bare-metal/fedora-coreos/kubernetes/bootstrap.tf b/bare-metal/fedora-coreos/kubernetes/bootstrap.tf index 077df9a43..6941dd269 100644 --- a/bare-metal/fedora-coreos/kubernetes/bootstrap.tf +++ b/bare-metal/fedora-coreos/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=73784c1b2c791d9ba586a1478979ac34dd324dad" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=e76f0a09fa9e6421a9cf697ee03714c6224e2581" cluster_name = var.cluster_name api_servers = [var.k8s_domain_name] diff --git a/bare-metal/fedora-coreos/kubernetes/fcc/controller.yaml b/bare-metal/fedora-coreos/kubernetes/fcc/controller.yaml index 1fd0d53be..69176ebc2 100644 --- a/bare-metal/fedora-coreos/kubernetes/fcc/controller.yaml +++ b/bare-metal/fedora-coreos/kubernetes/fcc/controller.yaml @@ -80,7 +80,7 @@ systemd: --volume /opt/cni/bin:/opt/cni/bin:z \ --volume /etc/iscsi:/etc/iscsi \ --volume /sbin/iscsiadm:/sbin/iscsiadm \ - k8s.gcr.io/hyperkube:v1.17.4 kubelet \ + quay.io/poseidon/kubelet:v1.17.4 \ --anonymous-auth=false \ --authentication-token-webhook \ --authorization-mode=Webhook \ @@ -134,7 +134,7 @@ systemd: --volume /opt/bootstrap/assets:/assets:ro,Z \ --volume /opt/bootstrap/apply:/apply:ro,Z \ --entrypoint=/apply \ - k8s.gcr.io/hyperkube:v1.17.4 + quay.io/poseidon/kubelet:v1.17.4 ExecStartPost=/bin/touch /opt/bootstrap/bootstrap.done ExecStartPost=-/usr/bin/podman stop bootstrap storage: diff --git a/bare-metal/fedora-coreos/kubernetes/fcc/worker.yaml b/bare-metal/fedora-coreos/kubernetes/fcc/worker.yaml index 54395326a..e02083d16 100644 --- a/bare-metal/fedora-coreos/kubernetes/fcc/worker.yaml +++ b/bare-metal/fedora-coreos/kubernetes/fcc/worker.yaml @@ -50,7 +50,7 @@ systemd: --volume /opt/cni/bin:/opt/cni/bin:z \ --volume /etc/iscsi:/etc/iscsi \ --volume /sbin/iscsiadm:/sbin/iscsiadm \ - k8s.gcr.io/hyperkube:v1.17.4 kubelet \ + quay.io/poseidon/kubelet:v1.17.4 \ --anonymous-auth=false \ --authentication-token-webhook \ --authorization-mode=Webhook \ diff --git a/digital-ocean/container-linux/kubernetes/bootstrap.tf b/digital-ocean/container-linux/kubernetes/bootstrap.tf index bfec3b981..e35dd0e6f 100644 --- a/digital-ocean/container-linux/kubernetes/bootstrap.tf +++ b/digital-ocean/container-linux/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=73784c1b2c791d9ba586a1478979ac34dd324dad" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=e76f0a09fa9e6421a9cf697ee03714c6224e2581" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/digital-ocean/container-linux/kubernetes/cl/controller.yaml b/digital-ocean/container-linux/kubernetes/cl/controller.yaml index acd643afa..cb190e39a 100644 --- a/digital-ocean/container-linux/kubernetes/cl/controller.yaml +++ b/digital-ocean/container-linux/kubernetes/cl/controller.yaml @@ -101,8 +101,7 @@ systemd: --mount volume=var-log,target=/var/log \ --volume opt-cni-bin,kind=host,source=/opt/cni/bin \ --mount volume=opt-cni-bin,target=/opt/cni/bin \ - docker://k8s.gcr.io/hyperkube:v1.17.4 \ - --exec=/usr/local/bin/kubelet -- \ + docker://quay.io/poseidon/kubelet:v1.17.4 -- \ --anonymous-auth=false \ --authentication-token-webhook \ --authorization-mode=Webhook \ @@ -146,7 +145,7 @@ systemd: --volume script,kind=host,source=/opt/bootstrap/apply \ --mount volume=script,target=/apply \ --insecure-options=image \ - docker://k8s.gcr.io/hyperkube:v1.17.4 \ + docker://quay.io/poseidon/kubelet:v1.17.4 \ --net=host \ --dns=host \ --exec=/apply diff --git a/digital-ocean/container-linux/kubernetes/cl/worker.yaml b/digital-ocean/container-linux/kubernetes/cl/worker.yaml index 230e05a8e..2a36572c3 100644 --- a/digital-ocean/container-linux/kubernetes/cl/worker.yaml +++ b/digital-ocean/container-linux/kubernetes/cl/worker.yaml @@ -74,8 +74,7 @@ systemd: --mount volume=var-log,target=/var/log \ --volume opt-cni-bin,kind=host,source=/opt/cni/bin \ --mount volume=opt-cni-bin,target=/opt/cni/bin \ - docker://k8s.gcr.io/hyperkube:v1.17.4 \ - --exec=/usr/local/bin/kubelet -- \ + docker://quay.io/poseidon/kubelet:v1.17.4 -- \ --anonymous-auth=false \ --authentication-token-webhook \ --authorization-mode=Webhook \ @@ -132,8 +131,8 @@ storage: --volume config,kind=host,source=/etc/kubernetes \ --mount volume=config,target=/etc/kubernetes \ --insecure-options=image \ - docker://k8s.gcr.io/hyperkube:v1.17.4 \ + docker://quay.io/poseidon/kubelet:v1.17.4 \ --net=host \ --dns=host \ -- \ - kubectl -- --kubeconfig=/etc/kubernetes/kubeconfig delete node $(hostname) + /usr/local/bin/kubectl -- --kubeconfig=/etc/kubernetes/kubeconfig delete node $(hostname) diff --git a/docs/architecture/operating-systems.md b/docs/architecture/operating-systems.md index 9db886494..50f30a6df 100644 --- a/docs/architecture/operating-systems.md +++ b/docs/architecture/operating-systems.md @@ -31,8 +31,8 @@ Together, they diversify Typhoon to support a range of container technologies. | single-master | all platforms | all platforms | | multi-master | all platforms | all platforms | | control plane | static pods | static pods | -| kubelet image | upstream hyperkube | upstream hyperkube | -| control plane images | upstream hyperkube | upstream hyperkube | +| kubelet image | kubelet [image](https://github.com/poseidon/kubelet) with upstream binary | kubelet [image](https://github.com/poseidon/kubelet) with upstream binary | +| control plane images | upstream images | upstream images | | on-host etcd | rkt-fly | podman | | on-host kubelet | rkt-fly | podman | | CNI plugins | calico or flannel | calico or flannel | diff --git a/docs/topics/security.md b/docs/topics/security.md index 60934e4c2..fc4cbb420 100644 --- a/docs/topics/security.md +++ b/docs/topics/security.md @@ -40,9 +40,14 @@ Typhoon limits exposure to many security threats, but it is not a silver bullet. * Do not give untrusted users a shell behind your firewall * Define network policies for your namespaces -## OpenPGP Signing +## Container Images -Typhoon uses upstream container images and binaries. We do not distribute artifacts of our own. If you find artifacts claiming to be from Typhoon, please send a note. +Typhoon uses upstream container images (where possible) and upstream binaries. + +!!! note + Kubernetes releases `kubelet` as a binary for distros to package, either as a DEB/RPM on traditional distros or as a container image for container-optimized operating systems. + +Typhoon [packages](https://github.com/poseidon/kubelet) the upstream Kubelet and its dependencies as a [container image](https://quay.io/repository/poseidon/kubelet) for use in Typhoon. The upstream Kubelet binary is checksummed and packaged directly. Quay automated builds provide verifiability and confidence in image contents. ## Disclosures diff --git a/google-cloud/container-linux/kubernetes/bootstrap.tf b/google-cloud/container-linux/kubernetes/bootstrap.tf index f3244b99e..83cd7ba7a 100644 --- a/google-cloud/container-linux/kubernetes/bootstrap.tf +++ b/google-cloud/container-linux/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=73784c1b2c791d9ba586a1478979ac34dd324dad" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=e76f0a09fa9e6421a9cf697ee03714c6224e2581" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/google-cloud/container-linux/kubernetes/cl/controller.yaml b/google-cloud/container-linux/kubernetes/cl/controller.yaml index cdf1c57ed..a8d97a54b 100644 --- a/google-cloud/container-linux/kubernetes/cl/controller.yaml +++ b/google-cloud/container-linux/kubernetes/cl/controller.yaml @@ -90,8 +90,7 @@ systemd: --mount volume=var-log,target=/var/log \ --volume opt-cni-bin,kind=host,source=/opt/cni/bin \ --mount volume=opt-cni-bin,target=/opt/cni/bin \ - docker://k8s.gcr.io/hyperkube:v1.17.4 \ - --exec=/usr/local/bin/kubelet -- \ + docker://quay.io/poseidon/kubelet:v1.17.4 -- \ --anonymous-auth=false \ --authentication-token-webhook \ --authorization-mode=Webhook \ @@ -134,7 +133,7 @@ systemd: --volume script,kind=host,source=/opt/bootstrap/apply \ --mount volume=script,target=/apply \ --insecure-options=image \ - docker://k8s.gcr.io/hyperkube:v1.17.4 \ + docker://quay.io/poseidon/kubelet:v1.17.4 \ --net=host \ --dns=host \ --exec=/apply diff --git a/google-cloud/container-linux/kubernetes/workers/cl/worker.yaml b/google-cloud/container-linux/kubernetes/workers/cl/worker.yaml index 4cf23d9b8..6ccd52548 100644 --- a/google-cloud/container-linux/kubernetes/workers/cl/worker.yaml +++ b/google-cloud/container-linux/kubernetes/workers/cl/worker.yaml @@ -63,8 +63,7 @@ systemd: --mount volume=var-log,target=/var/log \ --volume opt-cni-bin,kind=host,source=/opt/cni/bin \ --mount volume=opt-cni-bin,target=/opt/cni/bin \ - docker://k8s.gcr.io/hyperkube:v1.17.4 \ - --exec=/usr/local/bin/kubelet -- \ + docker://quay.io/poseidon/kubelet:v1.17.4 -- \ --anonymous-auth=false \ --authentication-token-webhook \ --authorization-mode=Webhook \ @@ -78,9 +77,9 @@ systemd: --lock-file=/var/run/lock/kubelet.lock \ --network-plugin=cni \ --node-labels=node.kubernetes.io/node \ - %{ for label in split(",", node_labels) } + %{~ for label in split(",", node_labels) ~} --node-labels=${label} \ - %{ endfor ~} + %{~ endfor ~} --pod-manifest-path=/etc/kubernetes/manifests \ --read-only-port=0 \ --volume-plugin-dir=/var/lib/kubelet/volumeplugins @@ -126,11 +125,11 @@ storage: --volume config,kind=host,source=/etc/kubernetes \ --mount volume=config,target=/etc/kubernetes \ --insecure-options=image \ - docker://k8s.gcr.io/hyperkube:v1.17.4 \ + docker://quay.io/poseidon/kubelet:v1.17.4 \ --net=host \ --dns=host \ -- \ - kubectl -- --kubeconfig=/etc/kubernetes/kubeconfig delete node $(hostname) + /usr/local/bin/kubectl -- --kubeconfig=/etc/kubernetes/kubeconfig delete node $(hostname) passwd: users: - name: core diff --git a/google-cloud/fedora-coreos/kubernetes/bootstrap.tf b/google-cloud/fedora-coreos/kubernetes/bootstrap.tf index fb05144eb..b8d9c1b5b 100644 --- a/google-cloud/fedora-coreos/kubernetes/bootstrap.tf +++ b/google-cloud/fedora-coreos/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=73784c1b2c791d9ba586a1478979ac34dd324dad" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=e76f0a09fa9e6421a9cf697ee03714c6224e2581" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/google-cloud/fedora-coreos/kubernetes/fcc/controller.yaml b/google-cloud/fedora-coreos/kubernetes/fcc/controller.yaml index 93297d7d0..b9cac354f 100644 --- a/google-cloud/fedora-coreos/kubernetes/fcc/controller.yaml +++ b/google-cloud/fedora-coreos/kubernetes/fcc/controller.yaml @@ -79,7 +79,7 @@ systemd: --volume /var/log:/var/log \ --volume /var/run/lock:/var/run/lock:z \ --volume /opt/cni/bin:/opt/cni/bin:z \ - k8s.gcr.io/hyperkube:v1.17.4 kubelet \ + quay.io/poseidon/kubelet:v1.17.4 \ --anonymous-auth=false \ --authentication-token-webhook \ --authorization-mode=Webhook \ @@ -123,7 +123,7 @@ systemd: --volume /opt/bootstrap/assets:/assets:ro,Z \ --volume /opt/bootstrap/apply:/apply:ro,Z \ --entrypoint=/apply \ - k8s.gcr.io/hyperkube:v1.17.4 + quay.io/poseidon/kubelet:v1.17.4 ExecStartPost=/bin/touch /opt/bootstrap/bootstrap.done ExecStartPost=-/usr/bin/podman stop bootstrap storage: diff --git a/google-cloud/fedora-coreos/kubernetes/workers/fcc/worker.yaml b/google-cloud/fedora-coreos/kubernetes/workers/fcc/worker.yaml index 07492c3e9..17a19c4fe 100644 --- a/google-cloud/fedora-coreos/kubernetes/workers/fcc/worker.yaml +++ b/google-cloud/fedora-coreos/kubernetes/workers/fcc/worker.yaml @@ -49,7 +49,7 @@ systemd: --volume /var/log:/var/log \ --volume /var/run/lock:/var/run/lock:z \ --volume /opt/cni/bin:/opt/cni/bin:z \ - k8s.gcr.io/hyperkube:v1.17.4 kubelet \ + quay.io/poseidon/kubelet:v1.17.4 \ --anonymous-auth=false \ --authentication-token-webhook \ --authorization-mode=Webhook \ @@ -87,7 +87,7 @@ systemd: Type=oneshot RemainAfterExit=true ExecStart=/bin/true - ExecStop=/bin/bash -c '/usr/bin/podman run --volume /etc/kubernetes:/etc/kubernetes:ro,z k8s.gcr.io/hyperkube:v1.17.4 kubectl --kubeconfig=/etc/kubernetes/kubeconfig delete node $HOSTNAME' + ExecStop=/bin/bash -c '/usr/bin/podman run --volume /etc/kubernetes:/etc/kubernetes:ro,z --entrypoint /usr/local/bin/kubectl quay.io/poseidon/kubelet:v1.17.4 kubectl --kubeconfig=/etc/kubernetes/kubeconfig delete node $HOSTNAME' [Install] WantedBy=multi-user.target storage: From 1bf4f3b80117a73add0dfe26ad7b1c0e2a856f66 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Sat, 21 Mar 2020 15:44:33 -0700 Subject: [PATCH 363/523] Fix image tag for Container Linux AWS workers * #669 left one reference to the original SHA tagged image before the v1.17.4 image tag was applied --- aws/container-linux/kubernetes/workers/cl/worker.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aws/container-linux/kubernetes/workers/cl/worker.yaml b/aws/container-linux/kubernetes/workers/cl/worker.yaml index a7733b7e5..b2bc6ad65 100644 --- a/aws/container-linux/kubernetes/workers/cl/worker.yaml +++ b/aws/container-linux/kubernetes/workers/cl/worker.yaml @@ -127,7 +127,7 @@ storage: --volume config,kind=host,source=/etc/kubernetes \ --mount volume=config,target=/etc/kubernetes \ --insecure-options=image \ - docker://quay.io/poseidon/kubelet:612b947 \ + docker://quay.io/poseidon/kubelet:v1.17.4 \ --net=host \ --dns=host \ -- \ From e556bc216724a117fe2ab7b890c230288a442037 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Tue, 24 Mar 2020 23:14:31 -0700 Subject: [PATCH 364/523] Update Prometheus from v2.17.0-rc.3 to v2.17.0 * https://github.com/prometheus/prometheus/releases/tag/v2.17.0 --- CHANGES.md | 2 +- addons/prometheus/deployment.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index f19f4305d..eff55a1af 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -17,7 +17,7 @@ Notable changes between versions. #### Addons -* Update Prometheus from v2.16.0 to [v2.17.0-rc.3](https://github.com/prometheus/prometheus/releases/tag/v2.17.0-rc.3) +* Update Prometheus from v2.16.0 to [v2.17.0](https://github.com/prometheus/prometheus/releases/tag/v2.17.0) * Update Grafana from v6.6.2 to [v6.7.1](https://github.com/grafana/grafana/releases/tag/v6.7.1) ## v1.17.4 diff --git a/addons/prometheus/deployment.yaml b/addons/prometheus/deployment.yaml index 742fad8aa..30972f00f 100644 --- a/addons/prometheus/deployment.yaml +++ b/addons/prometheus/deployment.yaml @@ -20,7 +20,7 @@ spec: serviceAccountName: prometheus containers: - name: prometheus - image: quay.io/prometheus/prometheus:v2.17.0-rc.3 + image: quay.io/prometheus/prometheus:v2.17.0 args: - --web.listen-address=0.0.0.0:9090 - --config.file=/etc/prometheus/prometheus.yaml From 9f702c72d2ac36ca4e4404406a878817070ee342 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Tue, 24 Mar 2020 23:45:31 -0700 Subject: [PATCH 365/523] Rename DigitalOcean image variable to os_image * Rename variable `image` to `os_image` to match the naming used for the same purpose on other supported platforms (e.g. AWS, Azure, GCP) --- CHANGES.md | 4 ++++ digital-ocean/container-linux/kubernetes/controllers.tf | 4 ++-- digital-ocean/container-linux/kubernetes/variables.tf | 2 +- digital-ocean/container-linux/kubernetes/workers.tf | 2 +- docs/cl/digital-ocean.md | 8 ++++---- 5 files changed, 12 insertions(+), 8 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index eff55a1af..736e910d8 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -15,6 +15,10 @@ Notable changes between versions. * Background: Kubernetes will [stop releasing](https://github.com/kubernetes/kubernetes/pull/88676) the hyperkube container image and provide the Kubelet as a binary for distros to package +#### DigitalOcean + +* Rename `image` variable to `os_image` for consistency ([#677](https://github.com/poseidon/typhoon/pull/677)) (action required) + #### Addons * Update Prometheus from v2.16.0 to [v2.17.0](https://github.com/prometheus/prometheus/releases/tag/v2.17.0) diff --git a/digital-ocean/container-linux/kubernetes/controllers.tf b/digital-ocean/container-linux/kubernetes/controllers.tf index f64e3f4a2..dd9562c2b 100644 --- a/digital-ocean/container-linux/kubernetes/controllers.tf +++ b/digital-ocean/container-linux/kubernetes/controllers.tf @@ -1,6 +1,6 @@ locals { official_images = ["coreos-stable", "coreos-beta", "coreos-alpha"] - is_official_image = contains(local.official_images, var.image) + is_official_image = contains(local.official_images, var.os_image) } # Controller Instance DNS records @@ -42,7 +42,7 @@ resource "digitalocean_droplet" "controllers" { name = "${var.cluster_name}-controller-${count.index}" region = var.region - image = var.image + image = var.os_image size = var.controller_type # network diff --git a/digital-ocean/container-linux/kubernetes/variables.tf b/digital-ocean/container-linux/kubernetes/variables.tf index 0c25d379d..04386941c 100644 --- a/digital-ocean/container-linux/kubernetes/variables.tf +++ b/digital-ocean/container-linux/kubernetes/variables.tf @@ -41,7 +41,7 @@ variable "worker_type" { default = "s-1vcpu-2gb" } -variable "image" { +variable "os_image" { type = string description = "Container Linux image for instances (e.g. coreos-stable)" default = "coreos-stable" diff --git a/digital-ocean/container-linux/kubernetes/workers.tf b/digital-ocean/container-linux/kubernetes/workers.tf index 2fe95fac6..02858bcb1 100644 --- a/digital-ocean/container-linux/kubernetes/workers.tf +++ b/digital-ocean/container-linux/kubernetes/workers.tf @@ -31,7 +31,7 @@ resource "digitalocean_droplet" "workers" { name = "${var.cluster_name}-worker-${count.index}" region = var.region - image = var.image + image = var.os_image size = var.worker_type # network diff --git a/docs/cl/digital-ocean.md b/docs/cl/digital-ocean.md index 9ca49a625..062421e66 100644 --- a/docs/cl/digital-ocean.md +++ b/docs/cl/digital-ocean.md @@ -71,7 +71,7 @@ module "nemo" { cluster_name = "nemo" region = "nyc3" dns_zone = "digital-ocean.example.com" - image = "coreos-stable" + os_image = "coreos-stable" # configuration ssh_fingerprints = ["d7:9d:79:ae:56:32:73:79:95:88:e3:a2:ab:5d:45:e7"] @@ -95,7 +95,7 @@ Flatcar Linux publishes DigitalOcean images, but does not upload them. DigitalOc ```tf module "nemo" { ... - image = data.digitalocean_image.flatcar-stable.id + os_image = data.digitalocean_image.flatcar-stable.id } data "digitalocean_image" "flatcar-stable" { @@ -103,7 +103,7 @@ data "digitalocean_image" "flatcar-stable" { } ``` -Set the [image](#variables) to the custom image id. +Set the [os_image](#variables) to the custom image id. ## ssh-agent @@ -244,7 +244,7 @@ Digital Ocean requires the SSH public key be uploaded to your account, so you ma | worker_count | Number of workers | 1 | 3 | | controller_type | Droplet type for controllers | "s-2vcpu-2gb" | s-2vcpu-2gb, s-2vcpu-4gb, s-4vcpu-8gb, ... | | worker_type | Droplet type for workers | "s-1vcpu-2gb" | s-1vcpu-2gb, s-2vcpu-2gb, ... | -| image | Container Linux image for instances | "coreos-stable" | coreos-stable, coreos-beta, coreos-alpha, "custom-image-id" | +| os_image | Container Linux image for instances | "coreos-stable" | coreos-stable, coreos-beta, coreos-alpha, "custom-image-id" | | controller_clc_snippets | Controller Container Linux Config snippets | [] | [example](/advanced/customization/) | | worker_clc_snippets | Worker Container Linux Config snippets | [] | [example](/advanced/customization/) | | networking | Choice of networking provider | "calico" | "flannel" or "calico" | From 5d1e4ad33304b529f4479f20b8693a1667e8de85 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Tue, 24 Mar 2020 23:53:06 -0700 Subject: [PATCH 366/523] Deprecate asset_dir variable and remove docs * Remove docs for the `asset_dir` variable and deprecate it in CHANGES. It will be removed in an upcoming release * Typhoon v1.17.0 introduced a new mechanism for managing and distributing generated assets that stopped relying on writing out to disk. `asset_dir` became optional and defaulted to being unset / off (recommended) --- CHANGES.md | 1 + docs/cl/aws.md | 1 - docs/cl/azure.md | 1 - docs/cl/bare-metal.md | 1 - docs/cl/digital-ocean.md | 1 - docs/cl/google-cloud.md | 1 - docs/fedora-coreos/aws.md | 1 - docs/fedora-coreos/bare-metal.md | 1 - docs/fedora-coreos/google-cloud.md | 1 - 9 files changed, 1 insertion(+), 8 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 736e910d8..3b33ac4a0 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -14,6 +14,7 @@ Notable changes between versions. * Update Typhoon container image security policy to list `quay.io/poseidon/kubelet`as an official distributed artifact * Background: Kubernetes will [stop releasing](https://github.com/kubernetes/kubernetes/pull/88676) the hyperkube container image and provide the Kubelet as a binary for distros to package +* Deprecate `asset_dir` variable and remove docs ([#678](https://github.com/poseidon/typhoon/pull/678)) #### DigitalOcean diff --git a/docs/cl/aws.md b/docs/cl/aws.md index 75a0f77b9..8d604de36 100644 --- a/docs/cl/aws.md +++ b/docs/cl/aws.md @@ -204,7 +204,6 @@ Reference the DNS zone id with `aws_route53_zone.zone-for-clusters.zone_id`. | Name | Description | Default | Example | |:-----|:------------|:--------|:--------| -| asset_dir | Absolute path to a directory where generated assets should be placed (contains secrets) | "" (disabled) | "/home/user/.secrets/clusters/tempest" | | controller_count | Number of controllers (i.e. masters) | 1 | 1 | | worker_count | Number of workers | 1 | 3 | | controller_type | EC2 instance type for controllers | "t3.small" | See below | diff --git a/docs/cl/azure.md b/docs/cl/azure.md index 9ca478fb3..e45a4d215 100644 --- a/docs/cl/azure.md +++ b/docs/cl/azure.md @@ -224,7 +224,6 @@ Reference the DNS zone with `azurerm_dns_zone.clusters.name` and its resource gr | Name | Description | Default | Example | |:-----|:------------|:--------|:--------| -| asset_dir | Absolute path to a directory where generated assets should be placed (contains secrets) | "" (disabled) | "/home/user/.secrets/clusters/ramius" | | controller_count | Number of controllers (i.e. masters) | 1 | 1 | | worker_count | Number of workers | 1 | 3 | | controller_type | Machine type for controllers | "Standard_B2s" | See below | diff --git a/docs/cl/bare-metal.md b/docs/cl/bare-metal.md index a1907319b..926f015bd 100644 --- a/docs/cl/bare-metal.md +++ b/docs/cl/bare-metal.md @@ -347,7 +347,6 @@ Check the [variables.tf](https://github.com/poseidon/typhoon/blob/master/bare-me | Name | Description | Default | Example | |:-----|:------------|:--------|:--------| -| asset_dir | Absolute path to a directory where generated assets should be placed (contains secrets) | "" (disabled) | "/home/user/.secrets/clusters/mercury" | | download_protocol | Protocol iPXE uses to download the kernel and initrd. iPXE must be compiled with [crypto](https://ipxe.org/crypto) support for https. Unused if cached_install is true | "https" | "http" | | cached_install | PXE boot and install from the Matchbox `/assets` cache. Admin MUST have downloaded Container Linux or Flatcar images into the cache | false | true | | install_disk | Disk device where Container Linux should be installed | "/dev/sda" | "/dev/sdb" | diff --git a/docs/cl/digital-ocean.md b/docs/cl/digital-ocean.md index 062421e66..bfee6ccaf 100644 --- a/docs/cl/digital-ocean.md +++ b/docs/cl/digital-ocean.md @@ -239,7 +239,6 @@ Digital Ocean requires the SSH public key be uploaded to your account, so you ma | Name | Description | Default | Example | |:-----|:------------|:--------|:--------| -| asset_dir | Absolute path to a directory where generated assets should be placed (contains secrets) | "" (disabled) | "/home/user/.secrets/nemo" | | controller_count | Number of controllers (i.e. masters) | 1 | 1 | | worker_count | Number of workers | 1 | 3 | | controller_type | Droplet type for controllers | "s-2vcpu-2gb" | s-2vcpu-2gb, s-2vcpu-4gb, s-4vcpu-8gb, ... | diff --git a/docs/cl/google-cloud.md b/docs/cl/google-cloud.md index 5b041d1c3..298e16660 100644 --- a/docs/cl/google-cloud.md +++ b/docs/cl/google-cloud.md @@ -231,7 +231,6 @@ resource "google_dns_managed_zone" "zone-for-clusters" { | Name | Description | Default | Example | |:-----|:------------|:--------|:--------| -| asset_dir | Absolute path to a directory where generated assets should be placed (contains secrets) | "" (disabled) | "/home/user/.secrets/clusters/yavin" | | controller_count | Number of controllers (i.e. masters) | 1 | 3 | | worker_count | Number of workers | 1 | 3 | | controller_type | Machine type for controllers | "n1-standard-1" | See below | diff --git a/docs/fedora-coreos/aws.md b/docs/fedora-coreos/aws.md index 01c2d698e..d6ebbaea6 100644 --- a/docs/fedora-coreos/aws.md +++ b/docs/fedora-coreos/aws.md @@ -204,7 +204,6 @@ Reference the DNS zone id with `aws_route53_zone.zone-for-clusters.zone_id`. | Name | Description | Default | Example | |:-----|:------------|:--------|:--------| -| asset_dir | Absolute path to a directory where generated assets should be placed (contains secrets) | "" (disabled) | "/home/user/.secrets/clusters/tempest" | | controller_count | Number of controllers (i.e. masters) | 1 | 1 | | worker_count | Number of workers | 1 | 3 | | controller_type | EC2 instance type for controllers | "t3.small" | See below | diff --git a/docs/fedora-coreos/bare-metal.md b/docs/fedora-coreos/bare-metal.md index e194732f8..55613dde1 100644 --- a/docs/fedora-coreos/bare-metal.md +++ b/docs/fedora-coreos/bare-metal.md @@ -337,7 +337,6 @@ Check the [variables.tf](https://github.com/poseidon/typhoon/blob/master/bare-me | Name | Description | Default | Example | |:-----|:------------|:--------|:--------| -| asset_dir | Absolute path to a directory where generated assets should be placed (contains secrets) | "" (disabled) | "/home/user/.secrets/clusters/mercury" | | cached_install | PXE boot and install from the Matchbox `/assets` cache. Admin MUST have downloaded Fedora CoreOS images into the cache | false | true | | install_disk | Disk device where Fedora CoreOS should be installed | "sda" (not "/dev/sda" like Container Linux) | "sdb" | | networking | Choice of networking provider | "calico" | "calico" or "flannel" | diff --git a/docs/fedora-coreos/google-cloud.md b/docs/fedora-coreos/google-cloud.md index 126c79ed0..4098983ff 100644 --- a/docs/fedora-coreos/google-cloud.md +++ b/docs/fedora-coreos/google-cloud.md @@ -232,7 +232,6 @@ resource "google_dns_managed_zone" "zone-for-clusters" { | Name | Description | Default | Example | |:-----|:------------|:--------|:--------| -| asset_dir | Absolute path to a directory where generated assets should be placed (contains secrets) | "" (disabled) | "/home/user/.secrets/clusters/yavin" | | controller_count | Number of controllers (i.e. masters) | 1 | 3 | | worker_count | Number of workers | 1 | 3 | | controller_type | Machine type for controllers | "n1-standard-1" | See below | From c3bf8bcf96a7f88d9168e174563453d9725b0661 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Wed, 25 Mar 2020 00:31:18 -0700 Subject: [PATCH 367/523] Add Fedora CoreOS to issue template and docs * Update several Container Linux references to start referring to Flatcar Linux * Update docs and mentions of Fedora CoreOS --- .github/ISSUE_TEMPLATE.md | 2 +- README.md | 3 ++- docs/architecture/operating-systems.md | 14 ++++++++------ docs/index.md | 3 ++- mkdocs.yml | 8 ++++---- 5 files changed, 17 insertions(+), 13 deletions(-) diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md index 6796f5b2a..6f9dfaf37 100644 --- a/.github/ISSUE_TEMPLATE.md +++ b/.github/ISSUE_TEMPLATE.md @@ -5,7 +5,7 @@ ### Environment * Platform: aws, azure, bare-metal, google-cloud, digital-ocean -* OS: container-linux, flatcar-linux +* OS: fedora-coreos, flatcar-linux * Release: Typhoon version or Git SHA (reporting latest is **not** helpful) * Terraform: `terraform version` (reporting latest is **not** helpful) * Plugins: Provider plugin versions (reporting latest is **not** helpful) diff --git a/README.md b/README.md index 9cb9f908a..4a7d67d33 100644 --- a/README.md +++ b/README.md @@ -51,7 +51,8 @@ Typhoon is available for [Flatcar Container Linux](https://www.flatcar-linux.org * [Docs](https://typhoon.psdn.io) * Architecture [concepts](https://typhoon.psdn.io/architecture/concepts/) and [operating systems](https://typhoon.psdn.io/architecture/operating-systems/) -* Tutorials for [AWS](docs/cl/aws.md), [Azure](docs/cl/azure.md), [Bare-Metal](docs/cl/bare-metal.md), [Digital Ocean](docs/cl/digital-ocean.md), and [Google-Cloud](docs/cl/google-cloud.md) +* Fedora CoreOS tutorials for [AWS](docs/fedora-coreos/aws.md), [Bare-Metal](docs/fedora-coreos/bare-metal.md), and [Google Cloud](docs/fedora-coreos/google-cloud.md) +* Flatcar Linux tutorials for [AWS](docs/cl/aws.md), [Azure](docs/cl/azure.md), [Bare-Metal](docs/cl/bare-metal.md), [DigitalOcean](docs/cl/digital-ocean.md), and [Google Cloud](docs/cl/google-cloud.md) ## Usage diff --git a/docs/architecture/operating-systems.md b/docs/architecture/operating-systems.md index 50f30a6df..276aff1cb 100644 --- a/docs/architecture/operating-systems.md +++ b/docs/architecture/operating-systems.md @@ -1,6 +1,6 @@ # Operating Systems -Typhoon supports [Container Linux](https://coreos.com/why/), [Flatcar Linux](https://www.flatcar-linux.org/) and [Fedora CoreOS](https://getfedora.org/coreos/) (preview). These operating systems were chosen because they offer: +Typhoon supports [Fedora CoreOS](https://getfedora.org/coreos/), [Flatcar Linux](https://www.flatcar-linux.org/) and Container Linux (EOL in May 2020). These operating systems were chosen because they offer: * Minimalism and focus on clustered operation * Automated and atomic operating system upgrades @@ -9,16 +9,18 @@ Typhoon supports [Container Linux](https://coreos.com/why/), [Flatcar Linux](htt Together, they diversify Typhoon to support a range of container technologies. -* Container Linux: Gentoo core, rkt-fly, docker * Fedora CoreOS: rpm-ostree, podman, moby +* Flatcar Linux: Gentoo core, rkt-fly, docker ## Host Properties -| Property | Container Linux / Flatcar Linux | Fedora CoreOS | +| Property | Flatcar Linux | Fedora CoreOS | |-------------------|---------------------------------|---------------| +| Kernel | ~4.19.x | ~5.5.x | +| systemd | 241 | 243 | | Ignition system | Ignition v2.x spec | Ignition v3.x spec | -| Container Engine | docker | docker | -| storage driver | overlay2 | overlay2 | +| Container Engine | docker 18.06.3-ce | docker 18.09.8 | +| storage driver | overlay2 (extfs) | overlay2 (xfs) | | logging driver | json-file | journald | | cgroup driver | cgroupfs (except Flatcar edge) | systemd | | Networking | systemd-networkd | NetworkManager | @@ -26,7 +28,7 @@ Together, they diversify Typhoon to support a range of container technologies. ## Kubernetes Properties -| Property | Container Linux | Fedora CoreOS | +| Property | Flatcar Linux | Fedora CoreOS | |-------------------|-----------------|---------------| | single-master | all platforms | all platforms | | multi-master | all platforms | all platforms | diff --git a/docs/index.md b/docs/index.md index 3da5d32d2..c59dcf863 100644 --- a/docs/index.md +++ b/docs/index.md @@ -50,7 +50,8 @@ Typhoon is available for [Flatcar Container Linux](https://www.flatcar-linux.org ## Documentation * Architecture [concepts](architecture/concepts.md) and [operating-systems](architecture/operating-systems.md) -* Tutorials for [AWS](cl/aws.md), [Azure](cl/azure.md), [Bare-Metal](cl/bare-metal.md), [Digital Ocean](cl/digital-ocean.md), and [Google-Cloud](cl/google-cloud.md) +* Fedora CoreOS tutorials for [AWS](fedora-coreos/aws.md), [Bare-Metal](fedora-coreos/bare-metal.md), and [Google Cloud](fedora-coreos/google-cloud.md) +* Flatcar Linux tutorials for [AWS](cl/aws.md), [Azure](cl/azure.md), [Bare-Metal](cl/bare-metal.md), [DigitalOcean](cl/digital-ocean.md), and [Google Cloud](cl/google-cloud.md) ## Example diff --git a/mkdocs.yml b/mkdocs.yml index 45d35c0c3..ae9f3fcb7 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -53,16 +53,16 @@ nav: - 'Bare-Metal': 'architecture/bare-metal.md' - 'DigitalOcean': 'architecture/digitalocean.md' - 'Google Cloud': 'architecture/google-cloud.md' + - 'Fedora CoreOS': + - 'AWS': 'fedora-coreos/aws.md' + - 'Bare-Metal': 'fedora-coreos/bare-metal.md' + - 'Google Cloud': 'fedora-coreos/google-cloud.md' - 'Container Linux': - 'AWS': 'cl/aws.md' - 'Azure': 'cl/azure.md' - 'Bare-Metal': 'cl/bare-metal.md' - 'Digital Ocean': 'cl/digital-ocean.md' - 'Google Cloud': 'cl/google-cloud.md' - - 'Fedora CoreOS': - - 'AWS': 'fedora-coreos/aws.md' - - 'Bare-Metal': 'fedora-coreos/bare-metal.md' - - 'Google Cloud': 'fedora-coreos/google-cloud.md' - 'Topics': - 'Maintenance': 'topics/maintenance.md' - 'Hardware': 'topics/hardware.md' From f100a90d2879d240192c540897d1e347c6602bf1 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Sun, 22 Mar 2020 13:47:18 -0700 Subject: [PATCH 368/523] Update Kubernetes from v1.17.4 to v1.18.0 * https://github.com/kubernetes/kubernetes/blob/master/CHANGELOG/CHANGELOG-1.18.md --- CHANGES.md | 2 ++ aws/container-linux/kubernetes/bootstrap.tf | 2 +- aws/container-linux/kubernetes/cl/controller.yaml | 13 +++++++++---- .../kubernetes/workers/cl/worker.yaml | 2 +- aws/fedora-coreos/kubernetes/bootstrap.tf | 2 +- aws/fedora-coreos/kubernetes/fcc/controller.yaml | 13 +++++++++---- .../kubernetes/workers/fcc/worker.yaml | 4 ++-- azure/container-linux/kubernetes/bootstrap.tf | 2 +- azure/container-linux/kubernetes/cl/controller.yaml | 13 +++++++++---- .../kubernetes/workers/cl/worker.yaml | 4 ++-- bare-metal/container-linux/kubernetes/bootstrap.tf | 2 +- .../container-linux/kubernetes/cl/controller.yaml | 13 +++++++++---- .../container-linux/kubernetes/cl/worker.yaml | 2 +- bare-metal/fedora-coreos/kubernetes/bootstrap.tf | 2 +- .../fedora-coreos/kubernetes/fcc/controller.yaml | 13 +++++++++---- bare-metal/fedora-coreos/kubernetes/fcc/worker.yaml | 2 +- .../container-linux/kubernetes/bootstrap.tf | 2 +- .../container-linux/kubernetes/cl/controller.yaml | 13 +++++++++---- .../container-linux/kubernetes/cl/worker.yaml | 4 ++-- .../container-linux/kubernetes/bootstrap.tf | 2 +- .../container-linux/kubernetes/cl/controller.yaml | 13 +++++++++---- .../kubernetes/workers/cl/worker.yaml | 4 ++-- google-cloud/fedora-coreos/kubernetes/bootstrap.tf | 2 +- .../fedora-coreos/kubernetes/fcc/controller.yaml | 13 +++++++++---- .../kubernetes/workers/fcc/worker.yaml | 4 ++-- 25 files changed, 95 insertions(+), 53 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 3b33ac4a0..bcb861c68 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -4,6 +4,7 @@ Notable changes between versions. ## Latest +* Kubernetes [v1.18.0](https://github.com/kubernetes/kubernetes/blob/master/CHANGELOG/CHANGELOG-1.18.md#v1180) * Update etcd from v3.4.4 to [v3.4.5](https://github.com/etcd-io/etcd/releases/tag/v3.4.5) * Switch from upstream hyperkube image to individual images ([#669](https://github.com/poseidon/typhoon/pull/669)) * Use upstream `k8s.gcr.io` `kube-apiserver`, `kube-controller-manager`, `kube-scheduler`, and `kube-proxy` container images @@ -27,6 +28,7 @@ Notable changes between versions. ## v1.17.4 +* Kubernetes [v1.17.4](https://github.com/kubernetes/kubernetes/blob/master/CHANGELOG/CHANGELOG-1.17.md#v1174) * Update etcd from v3.4.3 to [v3.4.4](https://github.com/etcd-io/etcd/releases/tag/v3.4.4) * On Container Linux, fetch using the docker transport format ([#659](https://github.com/poseidon/typhoon/pull/659)) * Update CoreDNS from v1.6.6 to v1.6.7 ([#648](https://github.com/poseidon/typhoon/pull/648)) diff --git a/aws/container-linux/kubernetes/bootstrap.tf b/aws/container-linux/kubernetes/bootstrap.tf index 654312479..a5a100ca3 100644 --- a/aws/container-linux/kubernetes/bootstrap.tf +++ b/aws/container-linux/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=e76f0a09fa9e6421a9cf697ee03714c6224e2581" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=cb170f802d09dcbe88b050257bc676e25d3c4282" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/aws/container-linux/kubernetes/cl/controller.yaml b/aws/container-linux/kubernetes/cl/controller.yaml index 59b634f80..940ec0217 100644 --- a/aws/container-linux/kubernetes/cl/controller.yaml +++ b/aws/container-linux/kubernetes/cl/controller.yaml @@ -91,7 +91,7 @@ systemd: --mount volume=var-log,target=/var/log \ --volume opt-cni-bin,kind=host,source=/opt/cni/bin \ --mount volume=opt-cni-bin,target=/opt/cni/bin \ - docker://quay.io/poseidon/kubelet:v1.17.4 -- \ + docker://quay.io/poseidon/kubelet:v1.18.0 -- \ --anonymous-auth=false \ --authentication-token-webhook \ --authorization-mode=Webhook \ @@ -125,7 +125,6 @@ systemd: Type=oneshot RemainAfterExit=true WorkingDirectory=/opt/bootstrap - ExecStartPre=-/usr/bin/bash -c 'set -x && [ -n "$(ls /opt/bootstrap/assets/manifests-*/* 2>/dev/null)" ] && mv /opt/bootstrap/assets/manifests-*/* /opt/bootstrap/assets/manifests && rm -rf /opt/bootstrap/assets/manifests-*' ExecStart=/usr/bin/rkt run \ --trust-keys-from-https \ --volume config,kind=host,source=/etc/kubernetes/bootstrap-secrets \ @@ -135,7 +134,7 @@ systemd: --volume script,kind=host,source=/opt/bootstrap/apply \ --mount volume=script,target=/apply \ --insecure-options=image \ - docker://quay.io/poseidon/kubelet:v1.17.4 \ + docker://quay.io/poseidon/kubelet:v1.18.0 \ --net=host \ --dns=host \ --exec=/apply @@ -170,7 +169,9 @@ storage: sudo mv static-manifests/* /etc/kubernetes/manifests/ sudo mkdir -p /opt/bootstrap/assets sudo mv manifests /opt/bootstrap/assets/manifests - sudo mv manifests-networking /opt/bootstrap/assets/manifests-networking + sudo mkdir -p /opt/bootstrap/assets/manifests/crds + sudo mv manifests-networking/crd*.yaml /opt/bootstrap/assets/manifests/crds + sudo mv manifests-networking/* /opt/bootstrap/assets/manifests/ rm -rf assets auth static-manifests tls - path: /opt/bootstrap/apply filesystem: root @@ -183,6 +184,10 @@ storage: echo "Waiting for static pod control plane" sleep 5 done + until kubectl apply -f /assets/manifests/crds -R; do + echo "Retry Custom Resource Definition manifests" + sleep 5 + done until kubectl apply -f /assets/manifests -R; do echo "Retry applying manifests" sleep 5 diff --git a/aws/container-linux/kubernetes/workers/cl/worker.yaml b/aws/container-linux/kubernetes/workers/cl/worker.yaml index b2bc6ad65..03780b50b 100644 --- a/aws/container-linux/kubernetes/workers/cl/worker.yaml +++ b/aws/container-linux/kubernetes/workers/cl/worker.yaml @@ -64,7 +64,7 @@ systemd: --mount volume=var-log,target=/var/log \ --volume opt-cni-bin,kind=host,source=/opt/cni/bin \ --mount volume=opt-cni-bin,target=/opt/cni/bin \ - docker://quay.io/poseidon/kubelet:v1.17.4 -- \ + docker://quay.io/poseidon/kubelet:v1.18.0 -- \ --anonymous-auth=false \ --authentication-token-webhook \ --authorization-mode=Webhook \ diff --git a/aws/fedora-coreos/kubernetes/bootstrap.tf b/aws/fedora-coreos/kubernetes/bootstrap.tf index 5036643ef..dcec91e07 100644 --- a/aws/fedora-coreos/kubernetes/bootstrap.tf +++ b/aws/fedora-coreos/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=e76f0a09fa9e6421a9cf697ee03714c6224e2581" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=cb170f802d09dcbe88b050257bc676e25d3c4282" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/aws/fedora-coreos/kubernetes/fcc/controller.yaml b/aws/fedora-coreos/kubernetes/fcc/controller.yaml index b9cac354f..37ea43aac 100644 --- a/aws/fedora-coreos/kubernetes/fcc/controller.yaml +++ b/aws/fedora-coreos/kubernetes/fcc/controller.yaml @@ -79,7 +79,7 @@ systemd: --volume /var/log:/var/log \ --volume /var/run/lock:/var/run/lock:z \ --volume /opt/cni/bin:/opt/cni/bin:z \ - quay.io/poseidon/kubelet:v1.17.4 \ + quay.io/poseidon/kubelet:v1.18.0 \ --anonymous-auth=false \ --authentication-token-webhook \ --authorization-mode=Webhook \ @@ -116,14 +116,13 @@ systemd: Type=oneshot RemainAfterExit=true WorkingDirectory=/opt/bootstrap - ExecStartPre=-/usr/bin/bash -c 'set -x && [ -n "$(ls /opt/bootstrap/assets/manifests-*/* 2>/dev/null)" ] && mv /opt/bootstrap/assets/manifests-*/* /opt/bootstrap/assets/manifests && rm -rf /opt/bootstrap/assets/manifests-*' ExecStart=/usr/bin/podman run --name bootstrap \ --network host \ --volume /etc/kubernetes/bootstrap-secrets:/etc/kubernetes/secrets:ro,Z \ --volume /opt/bootstrap/assets:/assets:ro,Z \ --volume /opt/bootstrap/apply:/apply:ro,Z \ --entrypoint=/apply \ - quay.io/poseidon/kubelet:v1.17.4 + quay.io/poseidon/kubelet:v1.18.0 ExecStartPost=/bin/touch /opt/bootstrap/bootstrap.done ExecStartPost=-/usr/bin/podman stop bootstrap storage: @@ -155,7 +154,9 @@ storage: sudo mv static-manifests/* /etc/kubernetes/manifests/ sudo mkdir -p /opt/bootstrap/assets sudo mv manifests /opt/bootstrap/assets/manifests - sudo mv manifests-networking /opt/bootstrap/assets/manifests-networking + sudo mkdir -p /opt/bootstrap/assets/manifests/crds + sudo mv manifests-networking/crd*.yaml /opt/bootstrap/assets/manifests/crds + sudo mv manifests-networking/* /opt/bootstrap/assets/manifests/ rm -rf assets auth static-manifests tls - path: /opt/bootstrap/apply mode: 0544 @@ -167,6 +168,10 @@ storage: echo "Waiting for static pod control plane" sleep 5 done + until kubectl apply -f /assets/manifests/crds -R; do + echo "Retry Custom Resource Definition manifests" + sleep 5 + done until kubectl apply -f /assets/manifests -R; do echo "Retry applying manifests" sleep 5 diff --git a/aws/fedora-coreos/kubernetes/workers/fcc/worker.yaml b/aws/fedora-coreos/kubernetes/workers/fcc/worker.yaml index c840da1d9..f8b35216a 100644 --- a/aws/fedora-coreos/kubernetes/workers/fcc/worker.yaml +++ b/aws/fedora-coreos/kubernetes/workers/fcc/worker.yaml @@ -49,7 +49,7 @@ systemd: --volume /var/log:/var/log \ --volume /var/run/lock:/var/run/lock:z \ --volume /opt/cni/bin:/opt/cni/bin:z \ - quay.io/poseidon/kubelet:v1.17.4 \ + quay.io/poseidon/kubelet:v1.18.0 \ --anonymous-auth=false \ --authentication-token-webhook \ --authorization-mode=Webhook \ @@ -87,7 +87,7 @@ systemd: Type=oneshot RemainAfterExit=true ExecStart=/bin/true - ExecStop=/bin/bash -c '/usr/bin/podman run --volume /etc/kubernetes:/etc/kubernetes:ro,z --entrypoint /usr/local/bin/kubectl quay.io/poseidon/kubelet:v1.17.4 --kubeconfig=/etc/kubernetes/kubeconfig delete node $HOSTNAME' + ExecStop=/bin/bash -c '/usr/bin/podman run --volume /etc/kubernetes:/etc/kubernetes:ro,z --entrypoint /usr/local/bin/kubectl quay.io/poseidon/kubelet:v1.18.0 --kubeconfig=/etc/kubernetes/kubeconfig delete node $HOSTNAME' [Install] WantedBy=multi-user.target storage: diff --git a/azure/container-linux/kubernetes/bootstrap.tf b/azure/container-linux/kubernetes/bootstrap.tf index 3f8118a67..509ac63b4 100644 --- a/azure/container-linux/kubernetes/bootstrap.tf +++ b/azure/container-linux/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=e76f0a09fa9e6421a9cf697ee03714c6224e2581" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=cb170f802d09dcbe88b050257bc676e25d3c4282" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/azure/container-linux/kubernetes/cl/controller.yaml b/azure/container-linux/kubernetes/cl/controller.yaml index a8dc507d1..f34c46300 100644 --- a/azure/container-linux/kubernetes/cl/controller.yaml +++ b/azure/container-linux/kubernetes/cl/controller.yaml @@ -90,7 +90,7 @@ systemd: --mount volume=var-log,target=/var/log \ --volume opt-cni-bin,kind=host,source=/opt/cni/bin \ --mount volume=opt-cni-bin,target=/opt/cni/bin \ - docker://quay.io/poseidon/kubelet:v1.17.4 -- \ + docker://quay.io/poseidon/kubelet:v1.18.0 -- \ --anonymous-auth=false \ --authentication-token-webhook \ --authorization-mode=Webhook \ @@ -123,7 +123,6 @@ systemd: Type=oneshot RemainAfterExit=true WorkingDirectory=/opt/bootstrap - ExecStartPre=-/usr/bin/bash -c 'set -x && [ -n "$(ls /opt/bootstrap/assets/manifests-*/* 2>/dev/null)" ] && mv /opt/bootstrap/assets/manifests-*/* /opt/bootstrap/assets/manifests && rm -rf /opt/bootstrap/assets/manifests-*' ExecStart=/usr/bin/rkt run \ --trust-keys-from-https \ --volume config,kind=host,source=/etc/kubernetes/bootstrap-secrets \ @@ -133,7 +132,7 @@ systemd: --volume script,kind=host,source=/opt/bootstrap/apply \ --mount volume=script,target=/apply \ --insecure-options=image \ - docker://quay.io/poseidon/kubelet:v1.17.4 \ + docker://quay.io/poseidon/kubelet:v1.18.0 \ --net=host \ --dns=host \ --exec=/apply @@ -168,7 +167,9 @@ storage: sudo mv static-manifests/* /etc/kubernetes/manifests/ sudo mkdir -p /opt/bootstrap/assets sudo mv manifests /opt/bootstrap/assets/manifests - sudo mv manifests-networking /opt/bootstrap/assets/manifests-networking + sudo mkdir -p /opt/bootstrap/assets/manifests/crds + sudo mv manifests-networking/crd*.yaml /opt/bootstrap/assets/manifests/crds + sudo mv manifests-networking/* /opt/bootstrap/assets/manifests/ rm -rf assets auth static-manifests tls - path: /opt/bootstrap/apply filesystem: root @@ -181,6 +182,10 @@ storage: echo "Waiting for static pod control plane" sleep 5 done + until kubectl apply -f /assets/manifests/crds -R; do + echo "Retry Custom Resource Definition manifests" + sleep 5 + done until kubectl apply -f /assets/manifests -R; do echo "Retry applying manifests" sleep 5 diff --git a/azure/container-linux/kubernetes/workers/cl/worker.yaml b/azure/container-linux/kubernetes/workers/cl/worker.yaml index 6ca5794fe..c9fd7a47c 100644 --- a/azure/container-linux/kubernetes/workers/cl/worker.yaml +++ b/azure/container-linux/kubernetes/workers/cl/worker.yaml @@ -63,7 +63,7 @@ systemd: --mount volume=var-log,target=/var/log \ --volume opt-cni-bin,kind=host,source=/opt/cni/bin \ --mount volume=opt-cni-bin,target=/opt/cni/bin \ - docker://quay.io/poseidon/kubelet:v1.17.4 -- \ + docker://quay.io/poseidon/kubelet:v1.18.0 -- \ --anonymous-auth=false \ --authentication-token-webhook \ --authorization-mode=Webhook \ @@ -125,7 +125,7 @@ storage: --volume config,kind=host,source=/etc/kubernetes \ --mount volume=config,target=/etc/kubernetes \ --insecure-options=image \ - docker://quay.io/poseidon/kubelet:v1.17.4 \ + docker://quay.io/poseidon/kubelet:v1.18.0 \ --net=host \ --dns=host \ -- \ diff --git a/bare-metal/container-linux/kubernetes/bootstrap.tf b/bare-metal/container-linux/kubernetes/bootstrap.tf index f60e7b21f..bec22b399 100644 --- a/bare-metal/container-linux/kubernetes/bootstrap.tf +++ b/bare-metal/container-linux/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=e76f0a09fa9e6421a9cf697ee03714c6224e2581" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=cb170f802d09dcbe88b050257bc676e25d3c4282" cluster_name = var.cluster_name api_servers = [var.k8s_domain_name] diff --git a/bare-metal/container-linux/kubernetes/cl/controller.yaml b/bare-metal/container-linux/kubernetes/cl/controller.yaml index db0b430bb..e5c14f304 100644 --- a/bare-metal/container-linux/kubernetes/cl/controller.yaml +++ b/bare-metal/container-linux/kubernetes/cl/controller.yaml @@ -103,7 +103,7 @@ systemd: --mount volume=etc-iscsi,target=/etc/iscsi \ --volume usr-sbin-iscsiadm,kind=host,source=/usr/sbin/iscsiadm \ --mount volume=usr-sbin-iscsiadm,target=/sbin/iscsiadm \ - docker://quay.io/poseidon/kubelet:v1.17.4 -- \ + docker://quay.io/poseidon/kubelet:v1.18.0 -- \ --anonymous-auth=false \ --authentication-token-webhook \ --authorization-mode=Webhook \ @@ -138,7 +138,6 @@ systemd: Type=oneshot RemainAfterExit=true WorkingDirectory=/opt/bootstrap - ExecStartPre=-/usr/bin/bash -c 'set -x && [ -n "$(ls /opt/bootstrap/assets/manifests-*/* 2>/dev/null)" ] && mv /opt/bootstrap/assets/manifests-*/* /opt/bootstrap/assets/manifests && rm -rf /opt/bootstrap/assets/manifests-*' ExecStart=/usr/bin/rkt run \ --trust-keys-from-https \ --volume config,kind=host,source=/etc/kubernetes/bootstrap-secrets \ @@ -148,7 +147,7 @@ systemd: --volume script,kind=host,source=/opt/bootstrap/apply \ --mount volume=script,target=/apply \ --insecure-options=image \ - docker://quay.io/poseidon/kubelet:v1.17.4 \ + docker://quay.io/poseidon/kubelet:v1.18.0 \ --net=host \ --dns=host \ --exec=/apply @@ -186,7 +185,9 @@ storage: sudo mv static-manifests/* /etc/kubernetes/manifests/ sudo mkdir -p /opt/bootstrap/assets sudo mv manifests /opt/bootstrap/assets/manifests - sudo mv manifests-networking /opt/bootstrap/assets/manifests-networking + sudo mkdir -p /opt/bootstrap/assets/manifests/crds + sudo mv manifests-networking/crd*.yaml /opt/bootstrap/assets/manifests/crds + sudo mv manifests-networking/* /opt/bootstrap/assets/manifests/ rm -rf assets auth static-manifests tls - path: /opt/bootstrap/apply filesystem: root @@ -199,6 +200,10 @@ storage: echo "Waiting for static pod control plane" sleep 5 done + until kubectl apply -f /assets/manifests/crds -R; do + echo "Retry Custom Resource Definition manifests" + sleep 5 + done until kubectl apply -f /assets/manifests -R; do echo "Retry applying manifests" sleep 5 diff --git a/bare-metal/container-linux/kubernetes/cl/worker.yaml b/bare-metal/container-linux/kubernetes/cl/worker.yaml index 094504356..794f70ddc 100644 --- a/bare-metal/container-linux/kubernetes/cl/worker.yaml +++ b/bare-metal/container-linux/kubernetes/cl/worker.yaml @@ -76,7 +76,7 @@ systemd: --mount volume=etc-iscsi,target=/etc/iscsi \ --volume usr-sbin-iscsiadm,kind=host,source=/usr/sbin/iscsiadm \ --mount volume=usr-sbin-iscsiadm,target=/sbin/iscsiadm \ - docker://quay.io/poseidon/kubelet:v1.17.4 -- \ + docker://quay.io/poseidon/kubelet:v1.18.0 -- \ --anonymous-auth=false \ --authentication-token-webhook \ --authorization-mode=Webhook \ diff --git a/bare-metal/fedora-coreos/kubernetes/bootstrap.tf b/bare-metal/fedora-coreos/kubernetes/bootstrap.tf index 6941dd269..428a1d664 100644 --- a/bare-metal/fedora-coreos/kubernetes/bootstrap.tf +++ b/bare-metal/fedora-coreos/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=e76f0a09fa9e6421a9cf697ee03714c6224e2581" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=cb170f802d09dcbe88b050257bc676e25d3c4282" cluster_name = var.cluster_name api_servers = [var.k8s_domain_name] diff --git a/bare-metal/fedora-coreos/kubernetes/fcc/controller.yaml b/bare-metal/fedora-coreos/kubernetes/fcc/controller.yaml index 69176ebc2..7a83caef7 100644 --- a/bare-metal/fedora-coreos/kubernetes/fcc/controller.yaml +++ b/bare-metal/fedora-coreos/kubernetes/fcc/controller.yaml @@ -80,7 +80,7 @@ systemd: --volume /opt/cni/bin:/opt/cni/bin:z \ --volume /etc/iscsi:/etc/iscsi \ --volume /sbin/iscsiadm:/sbin/iscsiadm \ - quay.io/poseidon/kubelet:v1.17.4 \ + quay.io/poseidon/kubelet:v1.18.0 \ --anonymous-auth=false \ --authentication-token-webhook \ --authorization-mode=Webhook \ @@ -127,14 +127,13 @@ systemd: Type=oneshot RemainAfterExit=true WorkingDirectory=/opt/bootstrap - ExecStartPre=-/usr/bin/bash -c 'set -x && [ -n "$(ls /opt/bootstrap/assets/manifests-*/* 2>/dev/null)" ] && mv /opt/bootstrap/assets/manifests-*/* /opt/bootstrap/assets/manifests && rm -rf /opt/bootstrap/assets/manifests-*' ExecStart=/usr/bin/podman run --name bootstrap \ --network host \ --volume /etc/kubernetes/bootstrap-secrets:/etc/kubernetes/secrets:ro,Z \ --volume /opt/bootstrap/assets:/assets:ro,Z \ --volume /opt/bootstrap/apply:/apply:ro,Z \ --entrypoint=/apply \ - quay.io/poseidon/kubelet:v1.17.4 + quay.io/poseidon/kubelet:v1.18.0 ExecStartPost=/bin/touch /opt/bootstrap/bootstrap.done ExecStartPost=-/usr/bin/podman stop bootstrap storage: @@ -166,7 +165,9 @@ storage: sudo mv static-manifests/* /etc/kubernetes/manifests/ sudo mkdir -p /opt/bootstrap/assets sudo mv manifests /opt/bootstrap/assets/manifests - sudo mv manifests-networking /opt/bootstrap/assets/manifests-networking + sudo mkdir -p /opt/bootstrap/assets/manifests/crds + sudo mv manifests-networking/crd*.yaml /opt/bootstrap/assets/manifests/crds + sudo mv manifests-networking/* /opt/bootstrap/assets/manifests/ rm -rf assets auth static-manifests tls - path: /opt/bootstrap/apply mode: 0544 @@ -178,6 +179,10 @@ storage: echo "Waiting for static pod control plane" sleep 5 done + until kubectl apply -f /assets/manifests/crds -R; do + echo "Retry Custom Resource Definition manifests" + sleep 5 + done until kubectl apply -f /assets/manifests -R; do echo "Retry applying manifests" sleep 5 diff --git a/bare-metal/fedora-coreos/kubernetes/fcc/worker.yaml b/bare-metal/fedora-coreos/kubernetes/fcc/worker.yaml index e02083d16..730c5c42b 100644 --- a/bare-metal/fedora-coreos/kubernetes/fcc/worker.yaml +++ b/bare-metal/fedora-coreos/kubernetes/fcc/worker.yaml @@ -50,7 +50,7 @@ systemd: --volume /opt/cni/bin:/opt/cni/bin:z \ --volume /etc/iscsi:/etc/iscsi \ --volume /sbin/iscsiadm:/sbin/iscsiadm \ - quay.io/poseidon/kubelet:v1.17.4 \ + quay.io/poseidon/kubelet:v1.18.0 \ --anonymous-auth=false \ --authentication-token-webhook \ --authorization-mode=Webhook \ diff --git a/digital-ocean/container-linux/kubernetes/bootstrap.tf b/digital-ocean/container-linux/kubernetes/bootstrap.tf index e35dd0e6f..0e11eca52 100644 --- a/digital-ocean/container-linux/kubernetes/bootstrap.tf +++ b/digital-ocean/container-linux/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=e76f0a09fa9e6421a9cf697ee03714c6224e2581" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=cb170f802d09dcbe88b050257bc676e25d3c4282" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/digital-ocean/container-linux/kubernetes/cl/controller.yaml b/digital-ocean/container-linux/kubernetes/cl/controller.yaml index cb190e39a..04bb8e7ee 100644 --- a/digital-ocean/container-linux/kubernetes/cl/controller.yaml +++ b/digital-ocean/container-linux/kubernetes/cl/controller.yaml @@ -101,7 +101,7 @@ systemd: --mount volume=var-log,target=/var/log \ --volume opt-cni-bin,kind=host,source=/opt/cni/bin \ --mount volume=opt-cni-bin,target=/opt/cni/bin \ - docker://quay.io/poseidon/kubelet:v1.17.4 -- \ + docker://quay.io/poseidon/kubelet:v1.18.0 -- \ --anonymous-auth=false \ --authentication-token-webhook \ --authorization-mode=Webhook \ @@ -135,7 +135,6 @@ systemd: Type=oneshot RemainAfterExit=true WorkingDirectory=/opt/bootstrap - ExecStartPre=-/usr/bin/bash -c 'set -x && [ -n "$(ls /opt/bootstrap/assets/manifests-*/* 2>/dev/null)" ] && mv /opt/bootstrap/assets/manifests-*/* /opt/bootstrap/assets/manifests && rm -rf /opt/bootstrap/assets/manifests-*' ExecStart=/usr/bin/rkt run \ --trust-keys-from-https \ --volume config,kind=host,source=/etc/kubernetes/bootstrap-secrets \ @@ -145,7 +144,7 @@ systemd: --volume script,kind=host,source=/opt/bootstrap/apply \ --mount volume=script,target=/apply \ --insecure-options=image \ - docker://quay.io/poseidon/kubelet:v1.17.4 \ + docker://quay.io/poseidon/kubelet:v1.18.0 \ --net=host \ --dns=host \ --exec=/apply @@ -177,7 +176,9 @@ storage: sudo mv static-manifests/* /etc/kubernetes/manifests/ sudo mkdir -p /opt/bootstrap/assets sudo mv manifests /opt/bootstrap/assets/manifests - sudo mv manifests-networking /opt/bootstrap/assets/manifests-networking + sudo mkdir -p /opt/bootstrap/assets/manifests/crds + sudo mv manifests-networking/crd*.yaml /opt/bootstrap/assets/manifests/crds + sudo mv manifests-networking/* /opt/bootstrap/assets/manifests/ rm -rf assets auth static-manifests tls - path: /opt/bootstrap/apply filesystem: root @@ -190,6 +191,10 @@ storage: echo "Waiting for static pod control plane" sleep 5 done + until kubectl apply -f /assets/manifests/crds -R; do + echo "Retry Custom Resource Definition manifests" + sleep 5 + done until kubectl apply -f /assets/manifests -R; do echo "Retry applying manifests" sleep 5 diff --git a/digital-ocean/container-linux/kubernetes/cl/worker.yaml b/digital-ocean/container-linux/kubernetes/cl/worker.yaml index 2a36572c3..c3e4f2020 100644 --- a/digital-ocean/container-linux/kubernetes/cl/worker.yaml +++ b/digital-ocean/container-linux/kubernetes/cl/worker.yaml @@ -74,7 +74,7 @@ systemd: --mount volume=var-log,target=/var/log \ --volume opt-cni-bin,kind=host,source=/opt/cni/bin \ --mount volume=opt-cni-bin,target=/opt/cni/bin \ - docker://quay.io/poseidon/kubelet:v1.17.4 -- \ + docker://quay.io/poseidon/kubelet:v1.18.0 -- \ --anonymous-auth=false \ --authentication-token-webhook \ --authorization-mode=Webhook \ @@ -131,7 +131,7 @@ storage: --volume config,kind=host,source=/etc/kubernetes \ --mount volume=config,target=/etc/kubernetes \ --insecure-options=image \ - docker://quay.io/poseidon/kubelet:v1.17.4 \ + docker://quay.io/poseidon/kubelet:v1.18.0 \ --net=host \ --dns=host \ -- \ diff --git a/google-cloud/container-linux/kubernetes/bootstrap.tf b/google-cloud/container-linux/kubernetes/bootstrap.tf index 83cd7ba7a..9f83af17e 100644 --- a/google-cloud/container-linux/kubernetes/bootstrap.tf +++ b/google-cloud/container-linux/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=e76f0a09fa9e6421a9cf697ee03714c6224e2581" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=cb170f802d09dcbe88b050257bc676e25d3c4282" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/google-cloud/container-linux/kubernetes/cl/controller.yaml b/google-cloud/container-linux/kubernetes/cl/controller.yaml index a8d97a54b..424b41aee 100644 --- a/google-cloud/container-linux/kubernetes/cl/controller.yaml +++ b/google-cloud/container-linux/kubernetes/cl/controller.yaml @@ -90,7 +90,7 @@ systemd: --mount volume=var-log,target=/var/log \ --volume opt-cni-bin,kind=host,source=/opt/cni/bin \ --mount volume=opt-cni-bin,target=/opt/cni/bin \ - docker://quay.io/poseidon/kubelet:v1.17.4 -- \ + docker://quay.io/poseidon/kubelet:v1.18.0 -- \ --anonymous-auth=false \ --authentication-token-webhook \ --authorization-mode=Webhook \ @@ -123,7 +123,6 @@ systemd: Type=oneshot RemainAfterExit=true WorkingDirectory=/opt/bootstrap - ExecStartPre=-/usr/bin/bash -c 'set -x && [ -n "$(ls /opt/bootstrap/assets/manifests-*/* 2>/dev/null)" ] && mv /opt/bootstrap/assets/manifests-*/* /opt/bootstrap/assets/manifests && rm -rf /opt/bootstrap/assets/manifests-*' ExecStart=/usr/bin/rkt run \ --trust-keys-from-https \ --volume config,kind=host,source=/etc/kubernetes/bootstrap-secrets \ @@ -133,7 +132,7 @@ systemd: --volume script,kind=host,source=/opt/bootstrap/apply \ --mount volume=script,target=/apply \ --insecure-options=image \ - docker://quay.io/poseidon/kubelet:v1.17.4 \ + docker://quay.io/poseidon/kubelet:v1.18.0 \ --net=host \ --dns=host \ --exec=/apply @@ -168,7 +167,9 @@ storage: sudo mv static-manifests/* /etc/kubernetes/manifests/ sudo mkdir -p /opt/bootstrap/assets sudo mv manifests /opt/bootstrap/assets/manifests - sudo mv manifests-networking /opt/bootstrap/assets/manifests-networking + sudo mkdir -p /opt/bootstrap/assets/manifests/crds + sudo mv manifests-networking/crd*.yaml /opt/bootstrap/assets/manifests/crds + sudo mv manifests-networking/* /opt/bootstrap/assets/manifests/ rm -rf assets auth static-manifests tls - path: /opt/bootstrap/apply filesystem: root @@ -181,6 +182,10 @@ storage: echo "Waiting for static pod control plane" sleep 5 done + until kubectl apply -f /assets/manifests/crds -R; do + echo "Retry Custom Resource Definition manifests" + sleep 5 + done until kubectl apply -f /assets/manifests -R; do echo "Retry applying manifests" sleep 5 diff --git a/google-cloud/container-linux/kubernetes/workers/cl/worker.yaml b/google-cloud/container-linux/kubernetes/workers/cl/worker.yaml index 6ccd52548..ed01bb489 100644 --- a/google-cloud/container-linux/kubernetes/workers/cl/worker.yaml +++ b/google-cloud/container-linux/kubernetes/workers/cl/worker.yaml @@ -63,7 +63,7 @@ systemd: --mount volume=var-log,target=/var/log \ --volume opt-cni-bin,kind=host,source=/opt/cni/bin \ --mount volume=opt-cni-bin,target=/opt/cni/bin \ - docker://quay.io/poseidon/kubelet:v1.17.4 -- \ + docker://quay.io/poseidon/kubelet:v1.18.0 -- \ --anonymous-auth=false \ --authentication-token-webhook \ --authorization-mode=Webhook \ @@ -125,7 +125,7 @@ storage: --volume config,kind=host,source=/etc/kubernetes \ --mount volume=config,target=/etc/kubernetes \ --insecure-options=image \ - docker://quay.io/poseidon/kubelet:v1.17.4 \ + docker://quay.io/poseidon/kubelet:v1.18.0 \ --net=host \ --dns=host \ -- \ diff --git a/google-cloud/fedora-coreos/kubernetes/bootstrap.tf b/google-cloud/fedora-coreos/kubernetes/bootstrap.tf index b8d9c1b5b..113fdc0ff 100644 --- a/google-cloud/fedora-coreos/kubernetes/bootstrap.tf +++ b/google-cloud/fedora-coreos/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=e76f0a09fa9e6421a9cf697ee03714c6224e2581" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=cb170f802d09dcbe88b050257bc676e25d3c4282" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/google-cloud/fedora-coreos/kubernetes/fcc/controller.yaml b/google-cloud/fedora-coreos/kubernetes/fcc/controller.yaml index b9cac354f..37ea43aac 100644 --- a/google-cloud/fedora-coreos/kubernetes/fcc/controller.yaml +++ b/google-cloud/fedora-coreos/kubernetes/fcc/controller.yaml @@ -79,7 +79,7 @@ systemd: --volume /var/log:/var/log \ --volume /var/run/lock:/var/run/lock:z \ --volume /opt/cni/bin:/opt/cni/bin:z \ - quay.io/poseidon/kubelet:v1.17.4 \ + quay.io/poseidon/kubelet:v1.18.0 \ --anonymous-auth=false \ --authentication-token-webhook \ --authorization-mode=Webhook \ @@ -116,14 +116,13 @@ systemd: Type=oneshot RemainAfterExit=true WorkingDirectory=/opt/bootstrap - ExecStartPre=-/usr/bin/bash -c 'set -x && [ -n "$(ls /opt/bootstrap/assets/manifests-*/* 2>/dev/null)" ] && mv /opt/bootstrap/assets/manifests-*/* /opt/bootstrap/assets/manifests && rm -rf /opt/bootstrap/assets/manifests-*' ExecStart=/usr/bin/podman run --name bootstrap \ --network host \ --volume /etc/kubernetes/bootstrap-secrets:/etc/kubernetes/secrets:ro,Z \ --volume /opt/bootstrap/assets:/assets:ro,Z \ --volume /opt/bootstrap/apply:/apply:ro,Z \ --entrypoint=/apply \ - quay.io/poseidon/kubelet:v1.17.4 + quay.io/poseidon/kubelet:v1.18.0 ExecStartPost=/bin/touch /opt/bootstrap/bootstrap.done ExecStartPost=-/usr/bin/podman stop bootstrap storage: @@ -155,7 +154,9 @@ storage: sudo mv static-manifests/* /etc/kubernetes/manifests/ sudo mkdir -p /opt/bootstrap/assets sudo mv manifests /opt/bootstrap/assets/manifests - sudo mv manifests-networking /opt/bootstrap/assets/manifests-networking + sudo mkdir -p /opt/bootstrap/assets/manifests/crds + sudo mv manifests-networking/crd*.yaml /opt/bootstrap/assets/manifests/crds + sudo mv manifests-networking/* /opt/bootstrap/assets/manifests/ rm -rf assets auth static-manifests tls - path: /opt/bootstrap/apply mode: 0544 @@ -167,6 +168,10 @@ storage: echo "Waiting for static pod control plane" sleep 5 done + until kubectl apply -f /assets/manifests/crds -R; do + echo "Retry Custom Resource Definition manifests" + sleep 5 + done until kubectl apply -f /assets/manifests -R; do echo "Retry applying manifests" sleep 5 diff --git a/google-cloud/fedora-coreos/kubernetes/workers/fcc/worker.yaml b/google-cloud/fedora-coreos/kubernetes/workers/fcc/worker.yaml index 17a19c4fe..44ddd2c79 100644 --- a/google-cloud/fedora-coreos/kubernetes/workers/fcc/worker.yaml +++ b/google-cloud/fedora-coreos/kubernetes/workers/fcc/worker.yaml @@ -49,7 +49,7 @@ systemd: --volume /var/log:/var/log \ --volume /var/run/lock:/var/run/lock:z \ --volume /opt/cni/bin:/opt/cni/bin:z \ - quay.io/poseidon/kubelet:v1.17.4 \ + quay.io/poseidon/kubelet:v1.18.0 \ --anonymous-auth=false \ --authentication-token-webhook \ --authorization-mode=Webhook \ @@ -87,7 +87,7 @@ systemd: Type=oneshot RemainAfterExit=true ExecStart=/bin/true - ExecStop=/bin/bash -c '/usr/bin/podman run --volume /etc/kubernetes:/etc/kubernetes:ro,z --entrypoint /usr/local/bin/kubectl quay.io/poseidon/kubelet:v1.17.4 kubectl --kubeconfig=/etc/kubernetes/kubeconfig delete node $HOSTNAME' + ExecStop=/bin/bash -c '/usr/bin/podman run --volume /etc/kubernetes:/etc/kubernetes:ro,z --entrypoint /usr/local/bin/kubectl quay.io/poseidon/kubelet:v1.18.0 kubectl --kubeconfig=/etc/kubernetes/kubeconfig delete node $HOSTNAME' [Install] WantedBy=multi-user.target storage: From d25f23e67569095a7dc268dc0d25fbce102f0d9d Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Wed, 25 Mar 2020 20:28:30 -0700 Subject: [PATCH 369/523] Update docs from Kubernetes v1.17.4 to v1.18.0 --- README.md | 10 +++++----- aws/container-linux/kubernetes/README.md | 2 +- .../kubernetes/workers/cl/worker.yaml | 2 +- aws/fedora-coreos/kubernetes/README.md | 2 +- azure/container-linux/kubernetes/README.md | 2 +- bare-metal/container-linux/kubernetes/README.md | 2 +- bare-metal/fedora-coreos/kubernetes/README.md | 2 +- digital-ocean/container-linux/kubernetes/README.md | 2 +- docs/advanced/worker-pools.md | 14 +++++++------- docs/cl/aws.md | 10 +++++----- docs/cl/azure.md | 10 +++++----- docs/cl/bare-metal.md | 10 +++++----- docs/cl/digital-ocean.md | 10 +++++----- docs/cl/google-cloud.md | 10 +++++----- docs/fedora-coreos/aws.md | 10 +++++----- docs/fedora-coreos/bare-metal.md | 10 +++++----- docs/fedora-coreos/google-cloud.md | 8 ++++---- docs/index.md | 10 +++++----- docs/topics/maintenance.md | 8 ++++---- google-cloud/container-linux/kubernetes/README.md | 2 +- google-cloud/fedora-coreos/kubernetes/README.md | 2 +- 21 files changed, 69 insertions(+), 69 deletions(-) diff --git a/README.md b/README.md index 4a7d67d33..bc755bc74 100644 --- a/README.md +++ b/README.md @@ -11,7 +11,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features -* Kubernetes v1.17.4 (upstream) +* Kubernetes v1.18.0 (upstream) * Single or multi-master, [Calico](https://www.projectcalico.org/) or [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/) * Advanced features like [worker pools](https://typhoon.psdn.io/advanced/worker-pools/), [preemptible](https://typhoon.psdn.io/cl/google-cloud/#preemption) workers, and [snippets](https://typhoon.psdn.io/advanced/customization/#container-linux) customization @@ -60,7 +60,7 @@ Define a Kubernetes cluster by using the Terraform module for your chosen platfo ```tf module "yavin" { - source = "git::https://github.com/poseidon/typhoon//google-cloud/container-linux/kubernetes?ref=v1.17.4" + source = "git::https://github.com/poseidon/typhoon//google-cloud/container-linux/kubernetes?ref=v1.18.0" # Google Cloud cluster_name = "yavin" @@ -99,9 +99,9 @@ In 4-8 minutes (varies by platform), the cluster will be ready. This Google Clou $ export KUBECONFIG=/home/user/.kube/configs/yavin-config $ kubectl get nodes NAME ROLES STATUS AGE VERSION -yavin-controller-0.c.example-com.internal Ready 6m v1.17.4 -yavin-worker-jrbf.c.example-com.internal Ready 5m v1.17.4 -yavin-worker-mzdm.c.example-com.internal Ready 5m v1.17.4 +yavin-controller-0.c.example-com.internal Ready 6m v1.18.0 +yavin-worker-jrbf.c.example-com.internal Ready 5m v1.18.0 +yavin-worker-mzdm.c.example-com.internal Ready 5m v1.18.0 ``` List the pods. diff --git a/aws/container-linux/kubernetes/README.md b/aws/container-linux/kubernetes/README.md index fcd9f1a07..74de8cebb 100644 --- a/aws/container-linux/kubernetes/README.md +++ b/aws/container-linux/kubernetes/README.md @@ -11,7 +11,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features -* Kubernetes v1.17.4 (upstream) +* Kubernetes v1.18.0 (upstream) * Single or multi-master, [Calico](https://www.projectcalico.org/) or [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/) * Advanced features like [worker pools](https://typhoon.psdn.io/advanced/worker-pools/), [spot](https://typhoon.psdn.io/cl/aws/#spot) workers, and [snippets](https://typhoon.psdn.io/advanced/customization/#container-linux) customization diff --git a/aws/container-linux/kubernetes/workers/cl/worker.yaml b/aws/container-linux/kubernetes/workers/cl/worker.yaml index 03780b50b..cc418c0a3 100644 --- a/aws/container-linux/kubernetes/workers/cl/worker.yaml +++ b/aws/container-linux/kubernetes/workers/cl/worker.yaml @@ -127,7 +127,7 @@ storage: --volume config,kind=host,source=/etc/kubernetes \ --mount volume=config,target=/etc/kubernetes \ --insecure-options=image \ - docker://quay.io/poseidon/kubelet:v1.17.4 \ + docker://quay.io/poseidon/kubelet:v1.18.0 \ --net=host \ --dns=host \ -- \ diff --git a/aws/fedora-coreos/kubernetes/README.md b/aws/fedora-coreos/kubernetes/README.md index 5b29531d9..d96e47e33 100644 --- a/aws/fedora-coreos/kubernetes/README.md +++ b/aws/fedora-coreos/kubernetes/README.md @@ -11,7 +11,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features -* Kubernetes v1.17.4 (upstream) +* Kubernetes v1.18.0 (upstream) * Single or multi-master, [Calico](https://www.projectcalico.org/) or [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/) * Advanced features like [worker pools](https://typhoon.psdn.io/advanced/worker-pools/), [spot](https://typhoon.psdn.io/cl/aws/#spot) workers, and [snippets](https://typhoon.psdn.io/advanced/customization/#container-linux) customization diff --git a/azure/container-linux/kubernetes/README.md b/azure/container-linux/kubernetes/README.md index 27d1002eb..8a8127586 100644 --- a/azure/container-linux/kubernetes/README.md +++ b/azure/container-linux/kubernetes/README.md @@ -11,7 +11,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features -* Kubernetes v1.17.4 (upstream) +* Kubernetes v1.18.0 (upstream) * Single or multi-master, [Calico](https://www.projectcalico.org/) or [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/) * Advanced features like [worker pools](https://typhoon.psdn.io/advanced/worker-pools/), [low-priority](https://typhoon.psdn.io/cl/azure/#low-priority) workers, and [snippets](https://typhoon.psdn.io/advanced/customization/#container-linux) customization diff --git a/bare-metal/container-linux/kubernetes/README.md b/bare-metal/container-linux/kubernetes/README.md index 1cc1b3fff..4193ab246 100644 --- a/bare-metal/container-linux/kubernetes/README.md +++ b/bare-metal/container-linux/kubernetes/README.md @@ -11,7 +11,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features -* Kubernetes v1.17.4 (upstream) +* Kubernetes v1.18.0 (upstream) * Single or multi-master, [Calico](https://www.projectcalico.org/) or [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/) * Advanced features like [snippets](https://typhoon.psdn.io/advanced/customization/#container-linux) customization diff --git a/bare-metal/fedora-coreos/kubernetes/README.md b/bare-metal/fedora-coreos/kubernetes/README.md index 088e4e865..1fe93c35b 100644 --- a/bare-metal/fedora-coreos/kubernetes/README.md +++ b/bare-metal/fedora-coreos/kubernetes/README.md @@ -11,7 +11,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features -* Kubernetes v1.17.4 (upstream) +* Kubernetes v1.18.0 (upstream) * Single or multi-master, [Calico](https://www.projectcalico.org/) or [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/) * Advanced features like [snippets](https://typhoon.psdn.io/advanced/customization/#container-linux) customization diff --git a/digital-ocean/container-linux/kubernetes/README.md b/digital-ocean/container-linux/kubernetes/README.md index 5fd66520a..7f134bf81 100644 --- a/digital-ocean/container-linux/kubernetes/README.md +++ b/digital-ocean/container-linux/kubernetes/README.md @@ -11,7 +11,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features -* Kubernetes v1.17.4 (upstream) +* Kubernetes v1.18.0 (upstream) * Single or multi-master, [Calico](https://www.projectcalico.org/) or [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/) * Advanced features like [snippets](https://typhoon.psdn.io/advanced/customization/#container-linux) customization diff --git a/docs/advanced/worker-pools.md b/docs/advanced/worker-pools.md index 2eb295bca..6525b4f1b 100644 --- a/docs/advanced/worker-pools.md +++ b/docs/advanced/worker-pools.md @@ -79,7 +79,7 @@ Create a cluster following the Azure [tutorial](../cl/azure.md#cluster). Define ```tf module "ramius-worker-pool" { - source = "git::https://github.com/poseidon/typhoon//azure/container-linux/kubernetes/workers?ref=v1.17.4" + source = "git::https://github.com/poseidon/typhoon//azure/container-linux/kubernetes/workers?ref=v1.18.0" # Azure region = module.ramius.region @@ -145,7 +145,7 @@ Create a cluster following the Google Cloud [tutorial](../cl/google-cloud.md#clu ```tf module "yavin-worker-pool" { - source = "git::https://github.com/poseidon/typhoon//google-cloud/container-linux/kubernetes/workers?ref=v1.17.4" + source = "git::https://github.com/poseidon/typhoon//google-cloud/container-linux/kubernetes/workers?ref=v1.18.0" # Google Cloud region = "europe-west2" @@ -176,11 +176,11 @@ Verify a managed instance group of workers joins the cluster within a few minute ``` $ kubectl get nodes NAME STATUS AGE VERSION -yavin-controller-0.c.example-com.internal Ready 6m v1.17.4 -yavin-worker-jrbf.c.example-com.internal Ready 5m v1.17.4 -yavin-worker-mzdm.c.example-com.internal Ready 5m v1.17.4 -yavin-16x-worker-jrbf.c.example-com.internal Ready 3m v1.17.4 -yavin-16x-worker-mzdm.c.example-com.internal Ready 3m v1.17.4 +yavin-controller-0.c.example-com.internal Ready 6m v1.18.0 +yavin-worker-jrbf.c.example-com.internal Ready 5m v1.18.0 +yavin-worker-mzdm.c.example-com.internal Ready 5m v1.18.0 +yavin-16x-worker-jrbf.c.example-com.internal Ready 3m v1.18.0 +yavin-16x-worker-mzdm.c.example-com.internal Ready 3m v1.18.0 ``` ### Variables diff --git a/docs/cl/aws.md b/docs/cl/aws.md index 8d604de36..db588b615 100644 --- a/docs/cl/aws.md +++ b/docs/cl/aws.md @@ -1,6 +1,6 @@ # AWS -In this tutorial, we'll create a Kubernetes v1.17.4 cluster on AWS with CoreOS Container Linux or Flatcar Linux. +In this tutorial, we'll create a Kubernetes v1.18.0 cluster on AWS with CoreOS Container Linux or Flatcar Linux. We'll declare a Kubernetes cluster using the Typhoon Terraform module. Then apply the changes to create a VPC, gateway, subnets, security groups, controller instances, worker auto-scaling group, network load balancer, and TLS assets. @@ -70,7 +70,7 @@ Define a Kubernetes cluster using the module `aws/container-linux/kubernetes`. ```tf module "tempest" { - source = "git::https://github.com/poseidon/typhoon//aws/container-linux/kubernetes?ref=v1.17.4" + source = "git::https://github.com/poseidon/typhoon//aws/container-linux/kubernetes?ref=v1.18.0" # AWS cluster_name = "tempest" @@ -143,9 +143,9 @@ List nodes in the cluster. $ export KUBECONFIG=/home/user/.kube/configs/tempest-config $ kubectl get nodes NAME STATUS ROLES AGE VERSION -ip-10-0-3-155 Ready 10m v1.17.4 -ip-10-0-26-65 Ready 10m v1.17.4 -ip-10-0-41-21 Ready 10m v1.17.4 +ip-10-0-3-155 Ready 10m v1.18.0 +ip-10-0-26-65 Ready 10m v1.18.0 +ip-10-0-41-21 Ready 10m v1.18.0 ``` List the pods. diff --git a/docs/cl/azure.md b/docs/cl/azure.md index e45a4d215..e3c9dca48 100644 --- a/docs/cl/azure.md +++ b/docs/cl/azure.md @@ -3,7 +3,7 @@ !!! danger Typhoon for Azure is alpha. For production, use AWS, Google Cloud, or bare-metal. As Azure matures, check [errata](https://github.com/poseidon/typhoon/wiki/Errata) for known shortcomings. -In this tutorial, we'll create a Kubernetes v1.17.4 cluster on Azure with CoreOS Container Linux or Flatcar Linux. +In this tutorial, we'll create a Kubernetes v1.18.0 cluster on Azure with CoreOS Container Linux or Flatcar Linux. We'll declare a Kubernetes cluster using the Typhoon Terraform module. Then apply the changes to create a resource group, virtual network, subnets, security groups, controller availability set, worker scale set, load balancer, and TLS assets. @@ -66,7 +66,7 @@ Define a Kubernetes cluster using the module `azure/container-linux/kubernetes`. ```tf module "ramius" { - source = "git::https://github.com/poseidon/typhoon//azure/container-linux/kubernetes?ref=v1.17.4" + source = "git::https://github.com/poseidon/typhoon//azure/container-linux/kubernetes?ref=v1.18.0" # Azure cluster_name = "ramius" @@ -149,9 +149,9 @@ List nodes in the cluster. $ export KUBECONFIG=/home/user/.kube/configs/ramius-config $ kubectl get nodes NAME STATUS ROLES AGE VERSION -ramius-controller-0 Ready 24m v1.17.4 -ramius-worker-000001 Ready 25m v1.17.4 -ramius-worker-000002 Ready 24m v1.17.4 +ramius-controller-0 Ready 24m v1.18.0 +ramius-worker-000001 Ready 25m v1.18.0 +ramius-worker-000002 Ready 24m v1.18.0 ``` List the pods. diff --git a/docs/cl/bare-metal.md b/docs/cl/bare-metal.md index 926f015bd..177ecd96f 100644 --- a/docs/cl/bare-metal.md +++ b/docs/cl/bare-metal.md @@ -1,6 +1,6 @@ # Bare-Metal -In this tutorial, we'll network boot and provision a Kubernetes v1.17.4 cluster on bare-metal with CoreOS Container Linux or Flatcar Linux. +In this tutorial, we'll network boot and provision a Kubernetes v1.18.0 cluster on bare-metal with CoreOS Container Linux or Flatcar Linux. First, we'll deploy a [Matchbox](https://github.com/poseidon/matchbox) service and setup a network boot environment. Then, we'll declare a Kubernetes cluster using the Typhoon Terraform module and power on machines. On PXE boot, machines will install Container Linux to disk, reboot into the disk install, and provision themselves as Kubernetes controllers or workers via Ignition. @@ -160,7 +160,7 @@ Define a Kubernetes cluster using the module `bare-metal/container-linux/kuberne ```tf module "mercury" { - source = "git::https://github.com/poseidon/typhoon//bare-metal/container-linux/kubernetes?ref=v1.17.4" + source = "git::https://github.com/poseidon/typhoon//bare-metal/container-linux/kubernetes?ref=v1.18.0" # bare-metal cluster_name = "mercury" @@ -299,9 +299,9 @@ List nodes in the cluster. $ export KUBECONFIG=/home/user/.kube/configs/mercury-config $ kubectl get nodes NAME STATUS ROLES AGE VERSION -node1.example.com Ready 10m v1.17.4 -node2.example.com Ready 10m v1.17.4 -node3.example.com Ready 10m v1.17.4 +node1.example.com Ready 10m v1.18.0 +node2.example.com Ready 10m v1.18.0 +node3.example.com Ready 10m v1.18.0 ``` List the pods. diff --git a/docs/cl/digital-ocean.md b/docs/cl/digital-ocean.md index bfee6ccaf..fd4a5bc73 100644 --- a/docs/cl/digital-ocean.md +++ b/docs/cl/digital-ocean.md @@ -1,6 +1,6 @@ # Digital Ocean -In this tutorial, we'll create a Kubernetes v1.17.4 cluster on DigitalOcean with CoreOS Container Linux or Flatcar Linux. +In this tutorial, we'll create a Kubernetes v1.18.0 cluster on DigitalOcean with CoreOS Container Linux or Flatcar Linux. We'll declare a Kubernetes cluster using the Typhoon Terraform module. Then apply the changes to create controller droplets, worker droplets, DNS records, tags, and TLS assets. @@ -65,7 +65,7 @@ Define a Kubernetes cluster using the module `digital-ocean/container-linux/kube ```tf module "nemo" { - source = "git::https://github.com/poseidon/typhoon//digital-ocean/container-linux/kubernetes?ref=v1.17.4" + source = "git::https://github.com/poseidon/typhoon//digital-ocean/container-linux/kubernetes?ref=v1.18.0" # Digital Ocean cluster_name = "nemo" @@ -161,9 +161,9 @@ List nodes in the cluster. $ export KUBECONFIG=/home/user/.kube/configs/nemo-config $ kubectl get nodes NAME STATUS ROLES AGE VERSION -10.132.110.130 Ready 10m v1.17.4 -10.132.115.81 Ready 10m v1.17.4 -10.132.124.107 Ready 10m v1.17.4 +10.132.110.130 Ready 10m v1.18.0 +10.132.115.81 Ready 10m v1.18.0 +10.132.124.107 Ready 10m v1.18.0 ``` List the pods. diff --git a/docs/cl/google-cloud.md b/docs/cl/google-cloud.md index 298e16660..9f2bb03ce 100644 --- a/docs/cl/google-cloud.md +++ b/docs/cl/google-cloud.md @@ -1,6 +1,6 @@ # Google Cloud -In this tutorial, we'll create a Kubernetes v1.17.4 cluster on Google Compute Engine with CoreOS Container Linux or Flatcar Linux. +In this tutorial, we'll create a Kubernetes v1.18.0 cluster on Google Compute Engine with CoreOS Container Linux or Flatcar Linux. We'll declare a Kubernetes cluster using the Typhoon Terraform module. Then apply the changes to create a network, firewall rules, health checks, controller instances, worker managed instance group, load balancers, and TLS assets. @@ -71,7 +71,7 @@ Define a Kubernetes cluster using the module `google-cloud/container-linux/kuber ```tf module "yavin" { - source = "git::https://github.com/poseidon/typhoon//google-cloud/container-linux/kubernetes?ref=v1.17.4" + source = "git::https://github.com/poseidon/typhoon//google-cloud/container-linux/kubernetes?ref=v1.18.0" # Google Cloud cluster_name = "yavin" @@ -167,9 +167,9 @@ List nodes in the cluster. $ export KUBECONFIG=/home/user/.kube/configs/yavin-config $ kubectl get nodes NAME ROLES STATUS AGE VERSION -yavin-controller-0.c.example-com.internal Ready 6m v1.17.4 -yavin-worker-jrbf.c.example-com.internal Ready 5m v1.17.4 -yavin-worker-mzdm.c.example-com.internal Ready 5m v1.17.4 +yavin-controller-0.c.example-com.internal Ready 6m v1.18.0 +yavin-worker-jrbf.c.example-com.internal Ready 5m v1.18.0 +yavin-worker-mzdm.c.example-com.internal Ready 5m v1.18.0 ``` List the pods. diff --git a/docs/fedora-coreos/aws.md b/docs/fedora-coreos/aws.md index d6ebbaea6..da5ce5267 100644 --- a/docs/fedora-coreos/aws.md +++ b/docs/fedora-coreos/aws.md @@ -1,6 +1,6 @@ # AWS -In this tutorial, we'll create a Kubernetes v1.17.4 cluster on AWS with Fedora CoreOS. +In this tutorial, we'll create a Kubernetes v1.18.0 cluster on AWS with Fedora CoreOS. We'll declare a Kubernetes cluster using the Typhoon Terraform module. Then apply the changes to create a VPC, gateway, subnets, security groups, controller instances, worker auto-scaling group, network load balancer, and TLS assets. @@ -70,7 +70,7 @@ Define a Kubernetes cluster using the module `aws/fedora-coreos/kubernetes`. ```tf module "tempest" { - source = "git::https://github.com/poseidon/typhoon//aws/fedora-coreos/kubernetes?ref=v1.17.4" + source = "git::https://github.com/poseidon/typhoon//aws/fedora-coreos/kubernetes?ref=v1.18.0" # AWS cluster_name = "tempest" @@ -143,9 +143,9 @@ List nodes in the cluster. $ export KUBECONFIG=/home/user/.kube/configs/tempest-config $ kubectl get nodes NAME STATUS ROLES AGE VERSION -ip-10-0-3-155 Ready 10m v1.17.4 -ip-10-0-26-65 Ready 10m v1.17.4 -ip-10-0-41-21 Ready 10m v1.17.4 +ip-10-0-3-155 Ready 10m v1.18.0 +ip-10-0-26-65 Ready 10m v1.18.0 +ip-10-0-41-21 Ready 10m v1.18.0 ``` List the pods. diff --git a/docs/fedora-coreos/bare-metal.md b/docs/fedora-coreos/bare-metal.md index 55613dde1..31ae83831 100644 --- a/docs/fedora-coreos/bare-metal.md +++ b/docs/fedora-coreos/bare-metal.md @@ -1,6 +1,6 @@ # Bare-Metal -In this tutorial, we'll network boot and provision a Kubernetes v1.17.4 cluster on bare-metal with Fedora CoreOS. +In this tutorial, we'll network boot and provision a Kubernetes v1.18.0 cluster on bare-metal with Fedora CoreOS. First, we'll deploy a [Matchbox](https://github.com/poseidon/matchbox) service and setup a network boot environment. Then, we'll declare a Kubernetes cluster using the Typhoon Terraform module and power on machines. On PXE boot, machines will install Fedora CoreOS to disk, reboot into the disk install, and provision themselves as Kubernetes controllers or workers via Ignition. @@ -160,7 +160,7 @@ Define a Kubernetes cluster using the module `bare-metal/fedora-coreos/kubernete ```tf module "mercury" { - source = "git::https://github.com/poseidon/typhoon//bare-metal/fedora-coreos/kubernetes?ref=v1.17.4" + source = "git::https://github.com/poseidon/typhoon//bare-metal/fedora-coreos/kubernetes?ref=v1.18.0" # bare-metal cluster_name = "mercury" @@ -289,9 +289,9 @@ List nodes in the cluster. $ export KUBECONFIG=/home/user/.kube/configs/mercury-config $ kubectl get nodes NAME STATUS ROLES AGE VERSION -node1.example.com Ready 10m v1.17.4 -node2.example.com Ready 10m v1.17.4 -node3.example.com Ready 10m v1.17.4 +node1.example.com Ready 10m v1.18.0 +node2.example.com Ready 10m v1.18.0 +node3.example.com Ready 10m v1.18.0 ``` List the pods. diff --git a/docs/fedora-coreos/google-cloud.md b/docs/fedora-coreos/google-cloud.md index 4098983ff..9d84b2313 100644 --- a/docs/fedora-coreos/google-cloud.md +++ b/docs/fedora-coreos/google-cloud.md @@ -3,7 +3,7 @@ !!! danger Typhoon for Fedora CoreOS is an alpha. Please report Fedora CoreOS bugs to [Fedora](https://github.com/coreos/fedora-coreos-tracker/issues) and Typhoon issues to Typhoon. -In this tutorial, we'll create a Kubernetes v1.17.4 cluster on Google Compute Engine with Fedora CoreOS. +In this tutorial, we'll create a Kubernetes v1.18.0 cluster on Google Compute Engine with Fedora CoreOS. We'll declare a Kubernetes cluster using the Typhoon Terraform module. Then apply the changes to create a network, firewall rules, health checks, controller instances, worker managed instance group, load balancers, and TLS assets. @@ -168,9 +168,9 @@ List nodes in the cluster. $ export KUBECONFIG=/home/user/.kube/configs/yavin-config $ kubectl get nodes NAME ROLES STATUS AGE VERSION -yavin-controller-0.c.example-com.internal Ready 6m v1.17.4 -yavin-worker-jrbf.c.example-com.internal Ready 5m v1.17.4 -yavin-worker-mzdm.c.example-com.internal Ready 5m v1.17.4 +yavin-controller-0.c.example-com.internal Ready 6m v1.18.0 +yavin-worker-jrbf.c.example-com.internal Ready 5m v1.18.0 +yavin-worker-mzdm.c.example-com.internal Ready 5m v1.18.0 ``` List the pods. diff --git a/docs/index.md b/docs/index.md index c59dcf863..0dda364b2 100644 --- a/docs/index.md +++ b/docs/index.md @@ -11,7 +11,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features -* Kubernetes v1.17.4 (upstream) +* Kubernetes v1.18.0 (upstream) * Single or multi-master, [Calico](https://www.projectcalico.org/) or [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/) * Advanced features like [worker pools](advanced/worker-pools/), [preemptible](cl/google-cloud/#preemption) workers, and [snippets](advanced/customization/#container-linux) customization @@ -59,7 +59,7 @@ Define a Kubernetes cluster by using the Terraform module for your chosen platfo ```tf module "yavin" { - source = "git::https://github.com/poseidon/typhoon//google-cloud/container-linux/kubernetes?ref=v1.17.4" + source = "git::https://github.com/poseidon/typhoon//google-cloud/container-linux/kubernetes?ref=v1.18.0" # Google Cloud cluster_name = "yavin" @@ -97,9 +97,9 @@ In 4-8 minutes (varies by platform), the cluster will be ready. This Google Clou $ export KUBECONFIG=/home/user/.kube/configs/yavin-config $ kubectl get nodes NAME ROLES STATUS AGE VERSION -yavin-controller-0.c.example-com.internal Ready 6m v1.17.4 -yavin-worker-jrbf.c.example-com.internal Ready 5m v1.17.4 -yavin-worker-mzdm.c.example-com.internal Ready 5m v1.17.4 +yavin-controller-0.c.example-com.internal Ready 6m v1.18.0 +yavin-worker-jrbf.c.example-com.internal Ready 5m v1.18.0 +yavin-worker-mzdm.c.example-com.internal Ready 5m v1.18.0 ``` List the pods. diff --git a/docs/topics/maintenance.md b/docs/topics/maintenance.md index 2b3126a17..de265eea7 100644 --- a/docs/topics/maintenance.md +++ b/docs/topics/maintenance.md @@ -18,7 +18,7 @@ module "yavin" { } module "mercury" { - source = "git::https://github.com/poseidon/typhoon//bare-metal/container-linux/kubernetes?ref=v1.17.4" + source = "git::https://github.com/poseidon/typhoon//bare-metal/container-linux/kubernetes?ref=v1.18.0" ... } ``` @@ -279,15 +279,15 @@ Typhoon modules have been adapted for Terraform v0.12. Provider plugins requirem | Typhoon Release | Terraform version | |-------------------|---------------------| -| v1.17.4 - ? | v0.12.x | -| v1.10.3 - v1.17.4 | v0.11.x | +| v1.18.0 - ? | v0.12.x | +| v1.10.3 - v1.18.0 | v0.11.x | | v1.9.2 - v1.10.2 | v0.10.4+ or v0.11.x | | v1.7.3 - v1.9.1 | v0.10.x | | v1.6.4 - v1.7.2 | v0.9.x | ### New users -New users can start with Terraform v0.12.x and follow the docs for Typhoon v1.17.4+ without issue. +New users can start with Terraform v0.12.x and follow the docs for Typhoon v1.18.0+ without issue. ### Existing users diff --git a/google-cloud/container-linux/kubernetes/README.md b/google-cloud/container-linux/kubernetes/README.md index ff8a08a96..4efb24164 100644 --- a/google-cloud/container-linux/kubernetes/README.md +++ b/google-cloud/container-linux/kubernetes/README.md @@ -11,7 +11,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features -* Kubernetes v1.17.4 (upstream) +* Kubernetes v1.18.0 (upstream) * Single or multi-master, [Calico](https://www.projectcalico.org/) or [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/) * Advanced features like [worker pools](https://typhoon.psdn.io/advanced/worker-pools/), [preemptible](https://typhoon.psdn.io/cl/google-cloud/#preemption) workers, and [snippets](https://typhoon.psdn.io/advanced/customization/#container-linux) customization diff --git a/google-cloud/fedora-coreos/kubernetes/README.md b/google-cloud/fedora-coreos/kubernetes/README.md index ff8a08a96..4efb24164 100644 --- a/google-cloud/fedora-coreos/kubernetes/README.md +++ b/google-cloud/fedora-coreos/kubernetes/README.md @@ -11,7 +11,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features -* Kubernetes v1.17.4 (upstream) +* Kubernetes v1.18.0 (upstream) * Single or multi-master, [Calico](https://www.projectcalico.org/) or [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/) * Advanced features like [worker pools](https://typhoon.psdn.io/advanced/worker-pools/), [preemptible](https://typhoon.psdn.io/cl/google-cloud/#preemption) workers, and [snippets](https://typhoon.psdn.io/advanced/customization/#container-linux) customization From ef5f953e04462f96f2273f8322e92d00469e7219 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Wed, 25 Mar 2020 21:23:18 -0700 Subject: [PATCH 370/523] Set docker log driver to journald on Fedora CoreOS * Before Kubernetes v1.18.0, Kubelet only supported kubectl `--limit-bytes` with the Docker `json-file` log driver so the Fedora CoreOS default was overridden for conformance. See https://github.com/poseidon/typhoon/pull/642 * Kubelet v1.18+ implemented support for other docker log drivers, so the Fedora CoreOS default `journald` can be used again Rel: https://github.com/kubernetes/kubernetes/issues/86367 --- CHANGES.md | 3 +++ aws/fedora-coreos/kubernetes/fcc/controller.yaml | 13 ------------- .../kubernetes/workers/fcc/worker.yaml | 13 ------------- .../fedora-coreos/kubernetes/fcc/controller.yaml | 13 ------------- bare-metal/fedora-coreos/kubernetes/fcc/worker.yaml | 13 ------------- .../fedora-coreos/kubernetes/fcc/controller.yaml | 13 ------------- .../kubernetes/workers/fcc/worker.yaml | 13 ------------- 7 files changed, 3 insertions(+), 78 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index bcb861c68..76d92c43e 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -4,6 +4,8 @@ Notable changes between versions. ## Latest +## v1.18.0 + * Kubernetes [v1.18.0](https://github.com/kubernetes/kubernetes/blob/master/CHANGELOG/CHANGELOG-1.18.md#v1180) * Update etcd from v3.4.4 to [v3.4.5](https://github.com/etcd-io/etcd/releases/tag/v3.4.5) * Switch from upstream hyperkube image to individual images ([#669](https://github.com/poseidon/typhoon/pull/669)) @@ -15,6 +17,7 @@ Notable changes between versions. * Update Typhoon container image security policy to list `quay.io/poseidon/kubelet`as an official distributed artifact * Background: Kubernetes will [stop releasing](https://github.com/kubernetes/kubernetes/pull/88676) the hyperkube container image and provide the Kubelet as a binary for distros to package +* Set Fedora CoreOS log driver back to the default `journald` ([#681](https://github.com/poseidon/typhoon/pull/681)) * Deprecate `asset_dir` variable and remove docs ([#678](https://github.com/poseidon/typhoon/pull/678)) #### DigitalOcean diff --git a/aws/fedora-coreos/kubernetes/fcc/controller.yaml b/aws/fedora-coreos/kubernetes/fcc/controller.yaml index 37ea43aac..4603e4f71 100644 --- a/aws/fedora-coreos/kubernetes/fcc/controller.yaml +++ b/aws/fedora-coreos/kubernetes/fcc/controller.yaml @@ -187,19 +187,6 @@ storage: DefaultCPUAccounting=yes DefaultMemoryAccounting=yes DefaultBlockIOAccounting=yes - - path: /etc/sysconfig/docker - mode: 0644 - overwrite: true - contents: - inline: | - # Modify these options if you want to change the way the docker daemon runs - OPTIONS="--selinux-enabled \ - --log-driver=json-file \ - --live-restore \ - --default-ulimit nofile=1024:1024 \ - --init-path /usr/libexec/docker/docker-init \ - --userland-proxy-path /usr/libexec/docker/docker-proxy \ - " - path: /etc/etcd/etcd.env mode: 0644 contents: diff --git a/aws/fedora-coreos/kubernetes/workers/fcc/worker.yaml b/aws/fedora-coreos/kubernetes/workers/fcc/worker.yaml index f8b35216a..1d5d78aa1 100644 --- a/aws/fedora-coreos/kubernetes/workers/fcc/worker.yaml +++ b/aws/fedora-coreos/kubernetes/workers/fcc/worker.yaml @@ -110,19 +110,6 @@ storage: DefaultCPUAccounting=yes DefaultMemoryAccounting=yes DefaultBlockIOAccounting=yes - - path: /etc/sysconfig/docker - mode: 0644 - overwrite: true - contents: - inline: | - # Modify these options if you want to change the way the docker daemon runs - OPTIONS="--selinux-enabled \ - --log-driver=json-file \ - --live-restore \ - --default-ulimit nofile=1024:1024 \ - --init-path /usr/libexec/docker/docker-init \ - --userland-proxy-path /usr/libexec/docker/docker-proxy \ - " passwd: users: - name: core diff --git a/bare-metal/fedora-coreos/kubernetes/fcc/controller.yaml b/bare-metal/fedora-coreos/kubernetes/fcc/controller.yaml index 7a83caef7..4925d7b54 100644 --- a/bare-metal/fedora-coreos/kubernetes/fcc/controller.yaml +++ b/bare-metal/fedora-coreos/kubernetes/fcc/controller.yaml @@ -198,19 +198,6 @@ storage: DefaultCPUAccounting=yes DefaultMemoryAccounting=yes DefaultBlockIOAccounting=yes - - path: /etc/sysconfig/docker - mode: 0644 - overwrite: true - contents: - inline: | - # Modify these options if you want to change the way the docker daemon runs - OPTIONS="--selinux-enabled \ - --log-driver=json-file \ - --live-restore \ - --default-ulimit nofile=1024:1024 \ - --init-path /usr/libexec/docker/docker-init \ - --userland-proxy-path /usr/libexec/docker/docker-proxy \ - " - path: /etc/etcd/etcd.env mode: 0644 contents: diff --git a/bare-metal/fedora-coreos/kubernetes/fcc/worker.yaml b/bare-metal/fedora-coreos/kubernetes/fcc/worker.yaml index 730c5c42b..c559ae814 100644 --- a/bare-metal/fedora-coreos/kubernetes/fcc/worker.yaml +++ b/bare-metal/fedora-coreos/kubernetes/fcc/worker.yaml @@ -112,19 +112,6 @@ storage: DefaultCPUAccounting=yes DefaultMemoryAccounting=yes DefaultBlockIOAccounting=yes - - path: /etc/sysconfig/docker - mode: 0644 - overwrite: true - contents: - inline: | - # Modify these options if you want to change the way the docker daemon runs - OPTIONS="--selinux-enabled \ - --log-driver=json-file \ - --live-restore \ - --default-ulimit nofile=1024:1024 \ - --init-path /usr/libexec/docker/docker-init \ - --userland-proxy-path /usr/libexec/docker/docker-proxy \ - " passwd: users: - name: core diff --git a/google-cloud/fedora-coreos/kubernetes/fcc/controller.yaml b/google-cloud/fedora-coreos/kubernetes/fcc/controller.yaml index 37ea43aac..4603e4f71 100644 --- a/google-cloud/fedora-coreos/kubernetes/fcc/controller.yaml +++ b/google-cloud/fedora-coreos/kubernetes/fcc/controller.yaml @@ -187,19 +187,6 @@ storage: DefaultCPUAccounting=yes DefaultMemoryAccounting=yes DefaultBlockIOAccounting=yes - - path: /etc/sysconfig/docker - mode: 0644 - overwrite: true - contents: - inline: | - # Modify these options if you want to change the way the docker daemon runs - OPTIONS="--selinux-enabled \ - --log-driver=json-file \ - --live-restore \ - --default-ulimit nofile=1024:1024 \ - --init-path /usr/libexec/docker/docker-init \ - --userland-proxy-path /usr/libexec/docker/docker-proxy \ - " - path: /etc/etcd/etcd.env mode: 0644 contents: diff --git a/google-cloud/fedora-coreos/kubernetes/workers/fcc/worker.yaml b/google-cloud/fedora-coreos/kubernetes/workers/fcc/worker.yaml index 44ddd2c79..5938a0f99 100644 --- a/google-cloud/fedora-coreos/kubernetes/workers/fcc/worker.yaml +++ b/google-cloud/fedora-coreos/kubernetes/workers/fcc/worker.yaml @@ -110,19 +110,6 @@ storage: DefaultCPUAccounting=yes DefaultMemoryAccounting=yes DefaultBlockIOAccounting=yes - - path: /etc/sysconfig/docker - mode: 0644 - overwrite: true - contents: - inline: | - # Modify these options if you want to change the way the docker daemon runs - OPTIONS="--selinux-enabled \ - --log-driver=json-file \ - --live-restore \ - --default-ulimit nofile=1024:1024 \ - --init-path /usr/libexec/docker/docker-init \ - --userland-proxy-path /usr/libexec/docker/docker-proxy \ - " passwd: users: - name: core From 076b8e3c4249da7abbd31f68e0cbfd42b2f9ca1d Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Thu, 26 Mar 2020 22:17:13 -0700 Subject: [PATCH 371/523] Update Prometheus from v2.17.0 to v2.17.1 * https://github.com/prometheus/prometheus/releases/tag/v2.17.1 --- CHANGES.md | 2 +- addons/prometheus/deployment.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 76d92c43e..02da5db1a 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -26,7 +26,7 @@ Notable changes between versions. #### Addons -* Update Prometheus from v2.16.0 to [v2.17.0](https://github.com/prometheus/prometheus/releases/tag/v2.17.0) +* Update Prometheus from v2.16.0 to [v2.17.1](https://github.com/prometheus/prometheus/releases/tag/v2.17.1) * Update Grafana from v6.6.2 to [v6.7.1](https://github.com/grafana/grafana/releases/tag/v6.7.1) ## v1.17.4 diff --git a/addons/prometheus/deployment.yaml b/addons/prometheus/deployment.yaml index 30972f00f..35569e836 100644 --- a/addons/prometheus/deployment.yaml +++ b/addons/prometheus/deployment.yaml @@ -20,7 +20,7 @@ spec: serviceAccountName: prometheus containers: - name: prometheus - image: quay.io/prometheus/prometheus:v2.17.0 + image: quay.io/prometheus/prometheus:v2.17.1 args: - --web.listen-address=0.0.0.0:9090 - --config.file=/etc/prometheus/prometheus.yaml From a1a5da6bc2de6ea1001cccf6c2992bc46e858731 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Thu, 26 Mar 2020 23:37:12 -0700 Subject: [PATCH 372/523] Add CoreOS Container Linux EOL recommendation to CHANGES * Recommend that users who have not yet tried Fedora CoreOS or Flatcar Linux do so. Likely, Container Linux will reach EOL and platform support / stability ratings will be in a mixed state. Nevertheless, folks should migrate by September. --- CHANGES.md | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 02da5db1a..66fa3e2ec 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -9,16 +9,18 @@ Notable changes between versions. * Kubernetes [v1.18.0](https://github.com/kubernetes/kubernetes/blob/master/CHANGELOG/CHANGELOG-1.18.md#v1180) * Update etcd from v3.4.4 to [v3.4.5](https://github.com/etcd-io/etcd/releases/tag/v3.4.5) * Switch from upstream hyperkube image to individual images ([#669](https://github.com/poseidon/typhoon/pull/669)) - * Use upstream `k8s.gcr.io` `kube-apiserver`, `kube-controller-manager`, `kube-scheduler`, and `kube-proxy` container images - * Use [poseidon/kubelet](https://github.com/poseidon/kubelet) to package the upstream Kubelet binary (checksummed) and - its dependencies as an automated build container image [quay.io/poseidon/kubelet](https://quay.io/repository/poseidon/kubelet) - * Update base images for control plane and Kubelet images used by Typhoon from upstream's debian 9 (stretch) to debian 10 - (buster) base - * Update Typhoon container image security policy to list `quay.io/poseidon/kubelet`as an official distributed artifact - * Background: Kubernetes will [stop releasing](https://github.com/kubernetes/kubernetes/pull/88676) the hyperkube container - image and provide the Kubelet as a binary for distros to package + * Use upstream k8s.gcr.io `kube-apiserver`, `kube-controller-manager`, `kube-scheduler`, and `kube-proxy` container images + * Use [poseidon/kubelet](https://github.com/poseidon/kubelet) to package the upstream Kubelet binary and dependencies as a container image (checksummed, automated build) + * Add [quay.io/poseidon/kubelet](https://quay.io/repository/poseidon/kubelet) as a Typhoon distributed artifact in the security policy + * Update base images from debian 9 to debian 10 + * Background: Kubernetes will [stop releasing](https://github.com/kubernetes/kubernetes/pull/88676) the hyperkube container image and provide the Kubelet as a binary for packaging +* Choose Fedora CoreOS or Flatcar Linux (**action recommended**) + * Use a `fedora-coreos` module for Fedora CoreOS + * Use a `container-linux` module with OS set for Flatcar Linux (varies, see docs) + * CoreOS Container Linux [won't receive updates](https://coreos.com/os/eol/) after May 2020 * Set Fedora CoreOS log driver back to the default `journald` ([#681](https://github.com/poseidon/typhoon/pull/681)) * Deprecate `asset_dir` variable and remove docs ([#678](https://github.com/poseidon/typhoon/pull/678)) +* Deprecate support for [gitRepo](https://kubernetes.io/docs/concepts/storage/volumes/#gitrepo) volumes. A future release will drop support. #### DigitalOcean From fc686c8fc729a1481e80172cef0044ca34e7522a Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Sat, 28 Mar 2020 12:31:39 -0700 Subject: [PATCH 373/523] Fix delete-node.service kubectl service exec's * Fix delete-node service that runs on worker (cloud-only) shutdown to delete a Kubernetes node. Regressed in #669 (unreleased) * Use rkt `--exec` to invoke kubectl binary in the kubelet image * Use podman `--entrypoint` to invoke the kubectl binary in the kubelet image --- aws/container-linux/kubernetes/workers/cl/worker.yaml | 3 +-- azure/container-linux/kubernetes/workers/cl/worker.yaml | 3 +-- digital-ocean/container-linux/kubernetes/cl/worker.yaml | 3 +-- google-cloud/container-linux/kubernetes/workers/cl/worker.yaml | 3 +-- google-cloud/fedora-coreos/kubernetes/workers/fcc/worker.yaml | 2 +- 5 files changed, 5 insertions(+), 9 deletions(-) diff --git a/aws/container-linux/kubernetes/workers/cl/worker.yaml b/aws/container-linux/kubernetes/workers/cl/worker.yaml index cc418c0a3..37749b21f 100644 --- a/aws/container-linux/kubernetes/workers/cl/worker.yaml +++ b/aws/container-linux/kubernetes/workers/cl/worker.yaml @@ -130,8 +130,7 @@ storage: docker://quay.io/poseidon/kubelet:v1.18.0 \ --net=host \ --dns=host \ - -- \ - /usr/local/bin/kubectl --kubeconfig=/etc/kubernetes/kubeconfig delete node $(hostname) + --exec=/usr/local/bin/kubectl -- --kubeconfig=/etc/kubernetes/kubeconfig delete node $(hostname) passwd: users: - name: core diff --git a/azure/container-linux/kubernetes/workers/cl/worker.yaml b/azure/container-linux/kubernetes/workers/cl/worker.yaml index c9fd7a47c..9ff0356fb 100644 --- a/azure/container-linux/kubernetes/workers/cl/worker.yaml +++ b/azure/container-linux/kubernetes/workers/cl/worker.yaml @@ -128,8 +128,7 @@ storage: docker://quay.io/poseidon/kubelet:v1.18.0 \ --net=host \ --dns=host \ - -- \ - /usr/local/bin/kubectl -- --kubeconfig=/etc/kubernetes/kubeconfig delete node $(hostname | tr '[:upper:]' '[:lower:]') + --exec=/usr/local/bin/kubectl -- --kubeconfig=/etc/kubernetes/kubeconfig delete node $(hostname | tr '[:upper:]' '[:lower:]') passwd: users: - name: core diff --git a/digital-ocean/container-linux/kubernetes/cl/worker.yaml b/digital-ocean/container-linux/kubernetes/cl/worker.yaml index c3e4f2020..c20464109 100644 --- a/digital-ocean/container-linux/kubernetes/cl/worker.yaml +++ b/digital-ocean/container-linux/kubernetes/cl/worker.yaml @@ -134,5 +134,4 @@ storage: docker://quay.io/poseidon/kubelet:v1.18.0 \ --net=host \ --dns=host \ - -- \ - /usr/local/bin/kubectl -- --kubeconfig=/etc/kubernetes/kubeconfig delete node $(hostname) + --exec=/usr/local/bin/kubectl -- --kubeconfig=/etc/kubernetes/kubeconfig delete node $(hostname) diff --git a/google-cloud/container-linux/kubernetes/workers/cl/worker.yaml b/google-cloud/container-linux/kubernetes/workers/cl/worker.yaml index ed01bb489..d1f08d4ad 100644 --- a/google-cloud/container-linux/kubernetes/workers/cl/worker.yaml +++ b/google-cloud/container-linux/kubernetes/workers/cl/worker.yaml @@ -128,8 +128,7 @@ storage: docker://quay.io/poseidon/kubelet:v1.18.0 \ --net=host \ --dns=host \ - -- \ - /usr/local/bin/kubectl -- --kubeconfig=/etc/kubernetes/kubeconfig delete node $(hostname) + --exec=/usr/local/bin/kubectl -- --kubeconfig=/etc/kubernetes/kubeconfig delete node $(hostname) passwd: users: - name: core diff --git a/google-cloud/fedora-coreos/kubernetes/workers/fcc/worker.yaml b/google-cloud/fedora-coreos/kubernetes/workers/fcc/worker.yaml index 5938a0f99..1d5d78aa1 100644 --- a/google-cloud/fedora-coreos/kubernetes/workers/fcc/worker.yaml +++ b/google-cloud/fedora-coreos/kubernetes/workers/fcc/worker.yaml @@ -87,7 +87,7 @@ systemd: Type=oneshot RemainAfterExit=true ExecStart=/bin/true - ExecStop=/bin/bash -c '/usr/bin/podman run --volume /etc/kubernetes:/etc/kubernetes:ro,z --entrypoint /usr/local/bin/kubectl quay.io/poseidon/kubelet:v1.18.0 kubectl --kubeconfig=/etc/kubernetes/kubeconfig delete node $HOSTNAME' + ExecStop=/bin/bash -c '/usr/bin/podman run --volume /etc/kubernetes:/etc/kubernetes:ro,z --entrypoint /usr/local/bin/kubectl quay.io/poseidon/kubelet:v1.18.0 --kubeconfig=/etc/kubernetes/kubeconfig delete node $HOSTNAME' [Install] WantedBy=multi-user.target storage: From 5fca08064b94a021b8f5ab533b8ee2f118f8a250 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Sat, 28 Mar 2020 12:49:03 -0700 Subject: [PATCH 374/523] Fix Fedora CoreOS AMI to filter for stable images * Fix issue observed in us-east-1 where AMI filters chose the latest testing channel release, rather than the stable chanel * Fedora CoreOS AMI filter selects the latest image with a matching name, x86_64, and hvm, excluding dev images. Add a filter for "Fedora CoreOS stable", which seems to be the only distinguishing metadata indicating the channel --- CHANGES.md | 5 +++++ aws/fedora-coreos/kubernetes/ami.tf | 5 +++++ aws/fedora-coreos/kubernetes/workers/ami.tf | 5 +++++ 3 files changed, 15 insertions(+) diff --git a/CHANGES.md b/CHANGES.md index 66fa3e2ec..cc2e337cf 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -22,6 +22,11 @@ Notable changes between versions. * Deprecate `asset_dir` variable and remove docs ([#678](https://github.com/poseidon/typhoon/pull/678)) * Deprecate support for [gitRepo](https://kubernetes.io/docs/concepts/storage/volumes/#gitrepo) volumes. A future release will drop support. +#### AWS + +* Fix Fedora CoreOS AMI to filter for stable images ([#685](https://github.com/poseidon/typhoon/pull/685)) + * Latest Fedora CoreOS `testing` or `bodhi-update` images could be chosen depending on the region + #### DigitalOcean * Rename `image` variable to `os_image` for consistency ([#677](https://github.com/poseidon/typhoon/pull/677)) (action required) diff --git a/aws/fedora-coreos/kubernetes/ami.tf b/aws/fedora-coreos/kubernetes/ami.tf index c6da3b368..e32ce159f 100644 --- a/aws/fedora-coreos/kubernetes/ami.tf +++ b/aws/fedora-coreos/kubernetes/ami.tf @@ -18,6 +18,11 @@ data "aws_ami" "fedora-coreos" { values = ["fedora-coreos-31.*.*.*-hvm"] } + filter { + name = "description" + values = ["Fedora CoreOS stable*"] + } + # try to filter out dev images (AWS filters can't) name_regex = "^fedora-coreos-31.[0-9]*.[0-9]*.[0-9]*-hvm*" } diff --git a/aws/fedora-coreos/kubernetes/workers/ami.tf b/aws/fedora-coreos/kubernetes/workers/ami.tf index c6da3b368..e32ce159f 100644 --- a/aws/fedora-coreos/kubernetes/workers/ami.tf +++ b/aws/fedora-coreos/kubernetes/workers/ami.tf @@ -18,6 +18,11 @@ data "aws_ami" "fedora-coreos" { values = ["fedora-coreos-31.*.*.*-hvm"] } + filter { + name = "description" + values = ["Fedora CoreOS stable*"] + } + # try to filter out dev images (AWS filters can't) name_regex = "^fedora-coreos-31.[0-9]*.[0-9]*.[0-9]*-hvm*" } From 144bb9403c5b5599f9f556ba1bf27432f446e820 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Sat, 28 Mar 2020 16:11:06 -0700 Subject: [PATCH 375/523] Add support for Fedora CoreOS snippets * Refresh snippets customization docs * Requires terraform-provider-ct v0.5+ --- CHANGES.md | 6 + .../fedora-coreos/kubernetes/profiles.tf | 2 + .../fedora-coreos/kubernetes/variables.tf | 4 +- docs/advanced/customization.md | 129 +++++++++++------- docs/cl/aws.md | 8 +- docs/cl/azure.md | 8 +- docs/cl/bare-metal.md | 8 +- docs/cl/digital-ocean.md | 8 +- docs/cl/google-cloud.md | 8 +- docs/fedora-coreos/aws.md | 12 +- docs/fedora-coreos/bare-metal.md | 10 +- docs/fedora-coreos/google-cloud.md | 12 +- 12 files changed, 124 insertions(+), 91 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index cc2e337cf..98a6df2aa 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -18,6 +18,8 @@ Notable changes between versions. * Use a `fedora-coreos` module for Fedora CoreOS * Use a `container-linux` module with OS set for Flatcar Linux (varies, see docs) * CoreOS Container Linux [won't receive updates](https://coreos.com/os/eol/) after May 2020 +* Add support for Fedora CoreOS snippets (`terraform-provider-ct` v0.5+) ([#686](https://github.com/poseidon/typhoon/pull/686)) +* Recommend updating `terraform-provider-ct` plugin from v0.4.0 to [v0.5.0](https://github.com/poseidon/terraform-provider-ct/releases/tag/v0.5.0) * Set Fedora CoreOS log driver back to the default `journald` ([#681](https://github.com/poseidon/typhoon/pull/681)) * Deprecate `asset_dir` variable and remove docs ([#678](https://github.com/poseidon/typhoon/pull/678)) * Deprecate support for [gitRepo](https://kubernetes.io/docs/concepts/storage/volumes/#gitrepo) volumes. A future release will drop support. @@ -27,6 +29,10 @@ Notable changes between versions. * Fix Fedora CoreOS AMI to filter for stable images ([#685](https://github.com/poseidon/typhoon/pull/685)) * Latest Fedora CoreOS `testing` or `bodhi-update` images could be chosen depending on the region +#### Bare-Metal + +* Update default `os_stream` from testing to stable + #### DigitalOcean * Rename `image` variable to `os_image` for consistency ([#677](https://github.com/poseidon/typhoon/pull/677)) (action required) diff --git a/bare-metal/fedora-coreos/kubernetes/profiles.tf b/bare-metal/fedora-coreos/kubernetes/profiles.tf index 8d75b8307..8a3cfdb16 100644 --- a/bare-metal/fedora-coreos/kubernetes/profiles.tf +++ b/bare-metal/fedora-coreos/kubernetes/profiles.tf @@ -50,6 +50,7 @@ data "ct_config" "controller-ignitions" { content = data.template_file.controller-configs.*.rendered[count.index] strict = true + snippets = lookup(var.snippets, var.controllers.*.name[count.index], []) } data "template_file" "controller-configs" { @@ -85,6 +86,7 @@ data "ct_config" "worker-ignitions" { content = data.template_file.worker-configs.*.rendered[count.index] strict = true + snippets = lookup(var.snippets, var.workers.*.name[count.index], []) } data "template_file" "worker-configs" { diff --git a/bare-metal/fedora-coreos/kubernetes/variables.tf b/bare-metal/fedora-coreos/kubernetes/variables.tf index 45fc0341c..26b1cf655 100644 --- a/bare-metal/fedora-coreos/kubernetes/variables.tf +++ b/bare-metal/fedora-coreos/kubernetes/variables.tf @@ -13,12 +13,12 @@ variable "matchbox_http_endpoint" { variable "os_stream" { type = string description = "Fedora CoreOS release stream (e.g. testing, stable)" - default = "testing" + default = "stable" } variable "os_version" { type = string - description = "Fedora CoreOS version to PXE and install (e.g. 30.20190712.0)" + description = "Fedora CoreOS version to PXE and install (e.g. 31.20200310.3.0)" } # machines diff --git a/docs/advanced/customization.md b/docs/advanced/customization.md index 2fd3373fb..496352c49 100644 --- a/docs/advanced/customization.md +++ b/docs/advanced/customization.md @@ -8,24 +8,88 @@ Typhoon modules accept Terraform input variables for customizing clusters in mer ## Addons -Clusters are kept to a minimal Kubernetes control plane by offering components like Nginx Ingress Controller, Prometheus, Grafana, and Heapster as optional post-install [addons](https://github.com/poseidon/typhoon/tree/master/addons). Customize addons by modifying a copy of our addon manifests. +Clusters are kept to a minimal Kubernetes control plane by offering components like Nginx Ingress Controller, Prometheus, and Grafana as optional post-install [addons](https://github.com/poseidon/typhoon/tree/master/addons). Customize addons by modifying a copy of our addon manifests. ## Hosts -### Container Linux +Typhoon uses the [Ignition](https://github.com/coreos/ignition) system of Fedora CoreOS and Flatcar Linux to immutably declare a system via first-boot disk provisioning. Fedora CoreOS uses a [Fedora CoreOS Config](https://docs.fedoraproject.org/en-US/fedora-coreos/fcct-config/) (FCC) and Flatcar Linux uses a [Container Linux Config](https://github.com/coreos/container-linux-config-transpiler/blob/master/doc/examples.md) (CLC). These define disk partitions, filesystems, systemd units, dropins, config files, mount units, raid arrays, and users. + +Controller and worker instances form a minimal and secure Kubernetes cluster on each platform. Typhoon provides the **snippets** feature to accept Fedora CoreOS Configs or Container Linux Configs to validate and additively merge into instance declarations. This allows advanced host customization and experimentation. + +!!! note + Snippets cannot be used to modify an already existing instance, the antithesis of immutable provisioning. Ignition fully declares a system on first boot only. + +!!! danger + Snippets provide the powerful host customization abilities of Ignition. You are responsible for additional units, configs, files, and conflicts. !!! danger - Container Linux Configs provide powerful host customization abilities. You are responsible for the additional configs defined for hosts. + Edits to snippets for controller instances can (correctly) cause Terraform to observe a diff (if not otherwise suppressed) and propose destroying and recreating controller(s). Recognize that this is destructive since controllers run etcd and are stateful. See [blue/green](/topics/maintenance/#upgrades) clusters. -Container Linux Configs (CLCs) declare how a Container Linux instance's disk should be provisioned on first boot from disk. CLCs define disk partitions, filesystems, files, systemd units, dropins, networkd configs, mount units, raid arrays, and users. Typhoon creates controller and worker instances with base Container Linux Configs to create a minimal, secure Kubernetes cluster on each platform. +### Fedora CoreOS -Typhoon AWS, Azure, bare-metal, DigitalOcean, and Google Cloud support CLC *snippets* - valid Container Linux Configs that are validated and additively merged into the Typhoon base config during `terraform plan`. This allows advanced host customizations and experimentation. +!!! note + Fedora CoreOS snippets require `terraform-provider-ct` v0.5+ -#### Examples +Define a Fedora CoreOS Config (FCC) ([docs](https://docs.fedoraproject.org/en-US/fedora-coreos/fcct-config/), [config](https://github.com/coreos/fcct/blob/master/docs/configuration-v1_0.md), [examples](https://github.com/coreos/fcct/blob/master/docs/examples.md)) in version control near your Terraform workspace directory (e.g. perhaps in a `snippets` subdirectory). You may organize snippets into multiple files, if desired. + +For example, ensure an `/opt/hello` file is created with permissions 0644. + +```yaml +# custom-files +variant: fcos +version: 1.0.0 +storage: + files: + - path: /opt/hello + contents: + inline: | + Hello World + mode: 0644 +``` -Container Linux [docs](https://coreos.com/os/docs/latest/clc-examples.html) show many simple config examples. Ensure a file `/opt/hello` is created with permissions 0644. +Reference the FCC contents by location (e.g. `file("./custom-units.yaml")`). On [AWS](/fedora-coreos/aws/#cluster) or [Google Cloud](/fedora-coreos/google-cloud/#cluster) extend the `controller_snippets` or `worker_snippets` list variables. +```tf +module "nemo" { + ... + + controller_count = 1 + worker_count = 2 + controller_snippets = [ + file("./custom-files"), + file("./custom-units"), + ] + worker_snippets = [ + file("./custom-files"), + file("./custom-units")", + ] + ... +} +``` + +On [Bare-Metal](/fedora-coreos/bare-metal/#cluster), different FCCs may be used for each node (since hardware may be heterogeneous). Extend the `snippets` map variable by mapping a controller or worker name key to a list of snippets. + +```tf +module "mercury" { + ... + snippets = { + "node2" = [file("./units/hello.yaml")] + "node3" = [ + file("./units/world.yaml"), + file("./units/hello.yaml"), + ] + } + ... +} ``` + +### Container Linux + +Define a Container Linux Config (CLC) ([config](https://github.com/coreos/container-linux-config-transpiler/blob/master/doc/configuration.md), [examples](https://github.com/coreos/container-linux-config-transpiler/blob/master/doc/examples.md)) in version control near your Terraform workspace directory (e.g. perhaps in a `snippets` subdirectory). You may organize snippets into multiple files, if desired. + +For example, ensure an `/opt/hello` file is created with permissions 0644. + +```yaml # custom-files storage: files: @@ -37,9 +101,9 @@ storage: mode: 0644 ``` -Ensure a systemd unit `hello.service` is created and a dropin `50-etcd-cluster.conf` is added for `etcd-member.service`. +Or ensure a systemd unit `hello.service` is created and a dropin `50-etcd-cluster.conf` is added for `etcd-member.service`. -``` +```yaml # custom-units systemd: units: @@ -61,17 +125,9 @@ systemd: Environment="ETCD_LOG_PACKAGE_LEVELS=etcdserver=WARNING,security=DEBUG" ``` -#### Specification - -View the Container Linux Config [format](https://coreos.com/os/docs/1576.4.0/configuration.html) to read about each field. +Reference the CLC contents by location (e.g. `file("./custom-units.yaml")`). On [AWS](/cl/aws/#cluster), [Azure](/cl/azure/#cluster), [DigitalOcean](/cl/digital-ocean/#cluster), or [Google Cloud](/cl/google-cloud/#cluster) extend the `controller_clc_snippets` or `worker_clc_snippets` list variables. -#### Usage - -Write Container Linux Configs *snippets* as files in the repository where you keep Terraform configs for clusters (perhaps in a `clc` or `snippets` subdirectory). You may organize snippets in multiple files as desired, provided they are each valid. - -[AWS](/cl/aws/#cluster), [Azure](/cl/azure/#cluster), [DigitalOcean](/cl/digital-ocean/#cluster), and [Google Cloud](/cl/google-cloud/#cluster) clusters allow populating a list of `controller_clc_snippets` or `worker_clc_snippets`. - -``` +```tf module "nemo" { ... @@ -89,16 +145,11 @@ module "nemo" { } ``` -[Bare-Metal](/cl/bare-metal/#cluster) clusters allow different Container Linux snippets to be used for each node (since hardware may be heterogeneous). Populate the optional `clc_snippets` map variable with any controller or worker name keys and lists of snippets. +On [Bare-Metal](/cl/bare-metal/#cluster), different CLCs may be used for each node (since hardware may be heterogeneous). Extend the `clc_snippets` map variable by mapping a controller or worker name key to a list of snippets. -``` +```tf module "mercury" { ... - controller_names = ["node1"] - worker_names = [ - "node2", - "node3", - ] clc_snippets = { "node2" = [file("./units/hello.yaml")] "node3" = [ @@ -110,32 +161,6 @@ module "mercury" { } ``` -Plan the resources to be created. - -``` -$ terraform plan -Plan: 54 to add, 0 to change, 0 to destroy. -``` - -Most syntax errors in CLCs can be caught during planning. For example, mangle the indentation in one of the CLC files: - -``` -$ terraform plan -... -error parsing Container Linux Config: error: yaml: line 3: did not find expected '-' indicator -``` - -Undo the mangle. Apply the changes to create the cluster per the tutorial. - -``` -$ terraform apply -``` - -Container Linux Configs (and the CoreOS Ignition system) create immutable infrastructure. Disk provisioning is performed only on first boot from disk. That means if you change a snippet used by an instance, Terraform will (correctly) try to destroy and recreate that instance. Be careful! - -!!! danger - Destroying and recreating controller instances is destructive! etcd runs on controller instances and stores data there. Do not modify controller snippets. See [blue/green](/topics/maintenance/#upgrades) clusters. - ## Architecture Typhoon chooses variables to expose with purpose. If you must customize clusters in ways that aren't supported by input variables, fork Typhoon and maintain a repository with customizations. Reference the repository by changing the username. diff --git a/docs/cl/aws.md b/docs/cl/aws.md index db588b615..c5f98aaf1 100644 --- a/docs/cl/aws.md +++ b/docs/cl/aws.md @@ -24,9 +24,9 @@ Terraform v0.12.21 Add the [terraform-provider-ct](https://github.com/poseidon/terraform-provider-ct) plugin binary for your system to `~/.terraform.d/plugins/`, noting the final name. ```sh -wget https://github.com/poseidon/terraform-provider-ct/releases/download/v0.4.0/terraform-provider-ct-v0.4.0-linux-amd64.tar.gz -tar xzf terraform-provider-ct-v0.4.0-linux-amd64.tar.gz -mv terraform-provider-ct-v0.4.0-linux-amd64/terraform-provider-ct ~/.terraform.d/plugins/terraform-provider-ct_v0.4.0 +wget https://github.com/poseidon/terraform-provider-ct/releases/download/v0.5.0/terraform-provider-ct-v0.5.0-linux-amd64.tar.gz +tar xzf terraform-provider-ct-v0.5.0-linux-amd64.tar.gz +mv terraform-provider-ct-v0.5.0-linux-amd64/terraform-provider-ct ~/.terraform.d/plugins/terraform-provider-ct_v0.5.0 ``` Read [concepts](/architecture/concepts/) to learn about Terraform, modules, and organizing resources. Change to your infrastructure repository (e.g. `infra`). @@ -55,7 +55,7 @@ provider "aws" { } provider "ct" { - version = "0.4.0" + version = "0.5.0" } ``` diff --git a/docs/cl/azure.md b/docs/cl/azure.md index e3c9dca48..02e70e0b6 100644 --- a/docs/cl/azure.md +++ b/docs/cl/azure.md @@ -27,9 +27,9 @@ Terraform v0.12.21 Add the [terraform-provider-ct](https://github.com/poseidon/terraform-provider-ct) plugin binary for your system to `~/.terraform.d/plugins/`, noting the final name. ```sh -wget https://github.com/poseidon/terraform-provider-ct/releases/download/v0.4.0/terraform-provider-ct-v0.4.0-linux-amd64.tar.gz -tar xzf terraform-provider-ct-v0.4.0-linux-amd64.tar.gz -mv terraform-provider-ct-v0.4.0-linux-amd64/terraform-provider-ct ~/.terraform.d/plugins/terraform-provider-ct_v0.4.0 +wget https://github.com/poseidon/terraform-provider-ct/releases/download/v0.5.0/terraform-provider-ct-v0.5.0-linux-amd64.tar.gz +tar xzf terraform-provider-ct-v0.5.0-linux-amd64.tar.gz +mv terraform-provider-ct-v0.5.0-linux-amd64/terraform-provider-ct ~/.terraform.d/plugins/terraform-provider-ct_v0.5.0 ``` Read [concepts](/architecture/concepts/) to learn about Terraform, modules, and organizing resources. Change to your infrastructure repository (e.g. `infra`). @@ -54,7 +54,7 @@ provider "azurerm" { } provider "ct" { - version = "0.4.0" + version = "0.5.0" } ``` diff --git a/docs/cl/bare-metal.md b/docs/cl/bare-metal.md index 177ecd96f..4f41ba64c 100644 --- a/docs/cl/bare-metal.md +++ b/docs/cl/bare-metal.md @@ -125,9 +125,9 @@ mv terraform-provider-matchbox-v0.3.0-linux-amd64/terraform-provider-matchbox ~/ Add the [terraform-provider-ct](https://github.com/poseidon/terraform-provider-ct) plugin binary for your system to `~/.terraform.d/plugins/`, noting the final name. ```sh -wget https://github.com/poseidon/terraform-provider-ct/releases/download/v0.4.0/terraform-provider-ct-v0.4.0-linux-amd64.tar.gz -tar xzf terraform-provider-ct-v0.4.0-linux-amd64.tar.gz -mv terraform-provider-ct-v0.4.0-linux-amd64/terraform-provider-ct ~/.terraform.d/plugins/terraform-provider-ct_v0.4.0 +wget https://github.com/poseidon/terraform-provider-ct/releases/download/v0.5.0/terraform-provider-ct-v0.5.0-linux-amd64.tar.gz +tar xzf terraform-provider-ct-v0.5.0-linux-amd64.tar.gz +mv terraform-provider-ct-v0.5.0-linux-amd64/terraform-provider-ct ~/.terraform.d/plugins/terraform-provider-ct_v0.5.0 ``` Read [concepts](/architecture/concepts/) to learn about Terraform, modules, and organizing resources. Change to your infrastructure repository (e.g. `infra`). @@ -150,7 +150,7 @@ provider "matchbox" { } provider "ct" { - version = "0.4.0" + version = "0.5.0" } ``` diff --git a/docs/cl/digital-ocean.md b/docs/cl/digital-ocean.md index fd4a5bc73..3104b802a 100644 --- a/docs/cl/digital-ocean.md +++ b/docs/cl/digital-ocean.md @@ -24,9 +24,9 @@ Terraform v0.12.21 Add the [terraform-provider-ct](https://github.com/poseidon/terraform-provider-ct) plugin binary for your system to `~/.terraform.d/plugins/`, noting the final name. ```sh -wget https://github.com/poseidon/terraform-provider-ct/releases/download/v0.4.0/terraform-provider-ct-v0.4.0-linux-amd64.tar.gz -tar xzf terraform-provider-ct-v0.4.0-linux-amd64.tar.gz -mv terraform-provider-ct-v0.4.0-linux-amd64/terraform-provider-ct ~/.terraform.d/plugins/terraform-provider-ct_v0.4.0 +wget https://github.com/poseidon/terraform-provider-ct/releases/download/v0.5.0/terraform-provider-ct-v0.5.0-linux-amd64.tar.gz +tar xzf terraform-provider-ct-v0.5.0-linux-amd64.tar.gz +mv terraform-provider-ct-v0.5.0-linux-amd64/terraform-provider-ct ~/.terraform.d/plugins/terraform-provider-ct_v0.5.0 ``` Read [concepts](/architecture/concepts/) to learn about Terraform, modules, and organizing resources. Change to your infrastructure repository (e.g. `infra`). @@ -55,7 +55,7 @@ provider "digitalocean" { } provider "ct" { - version = "0.4.0" + version = "0.5.0" } ``` diff --git a/docs/cl/google-cloud.md b/docs/cl/google-cloud.md index 9f2bb03ce..aaa9e1dc1 100644 --- a/docs/cl/google-cloud.md +++ b/docs/cl/google-cloud.md @@ -24,9 +24,9 @@ Terraform v0.12.21 Add the [terraform-provider-ct](https://github.com/poseidon/terraform-provider-ct) plugin binary for your system to `~/.terraform.d/plugins/`, noting the final name. ```sh -wget https://github.com/poseidon/terraform-provider-ct/releases/download/v0.4.0/terraform-provider-ct-v0.4.0-linux-amd64.tar.gz -tar xzf terraform-provider-ct-v0.4.0-linux-amd64.tar.gz -mv terraform-provider-ct-v0.4.0-linux-amd64/terraform-provider-ct ~/.terraform.d/plugins/terraform-provider-ct_v0.4.0 +wget https://github.com/poseidon/terraform-provider-ct/releases/download/v0.5.0/terraform-provider-ct-v0.5.0-linux-amd64.tar.gz +tar xzf terraform-provider-ct-v0.5.0-linux-amd64.tar.gz +mv terraform-provider-ct-v0.5.0-linux-amd64/terraform-provider-ct ~/.terraform.d/plugins/terraform-provider-ct_v0.5.0 ``` Read [concepts](/architecture/concepts/) to learn about Terraform, modules, and organizing resources. Change to your infrastructure repository (e.g. `infra`). @@ -56,7 +56,7 @@ provider "google" { } provider "ct" { - version = "0.4.0" + version = "0.5.0" } ``` diff --git a/docs/fedora-coreos/aws.md b/docs/fedora-coreos/aws.md index da5ce5267..825856c67 100644 --- a/docs/fedora-coreos/aws.md +++ b/docs/fedora-coreos/aws.md @@ -24,9 +24,9 @@ Terraform v0.12.21 Add the [terraform-provider-ct](https://github.com/poseidon/terraform-provider-ct) plugin binary for your system to `~/.terraform.d/plugins/`, noting the final name. ```sh -wget https://github.com/poseidon/terraform-provider-ct/releases/download/v0.4.0/terraform-provider-ct-v0.4.0-linux-amd64.tar.gz -tar xzf terraform-provider-ct-v0.4.0-linux-amd64.tar.gz -mv terraform-provider-ct-v0.4.0-linux-amd64/terraform-provider-ct ~/.terraform.d/plugins/terraform-provider-ct_v0.4.0 +wget https://github.com/poseidon/terraform-provider-ct/releases/download/v0.5.0/terraform-provider-ct-v0.5.0-linux-amd64.tar.gz +tar xzf terraform-provider-ct-v0.5.0-linux-amd64.tar.gz +mv terraform-provider-ct-v0.5.0-linux-amd64/terraform-provider-ct ~/.terraform.d/plugins/terraform-provider-ct_v0.5.0 ``` Read [concepts](/architecture/concepts/) to learn about Terraform, modules, and organizing resources. Change to your infrastructure repository (e.g. `infra`). @@ -55,7 +55,7 @@ provider "aws" { } provider "ct" { - version = "0.4.0" + version = "0.5.0" } ``` @@ -214,8 +214,8 @@ Reference the DNS zone id with `aws_route53_zone.zone-for-clusters.zone_id`. | disk_iops | IOPS of the EBS volume | 0 (i.e. auto) | 400 | | worker_target_groups | Target group ARNs to which worker instances should be added | [] | [aws_lb_target_group.app.id] | | worker_price | Spot price in USD for worker instances or 0 to use on-demand instances | 0 | 0.10 | -| controller_snippets | Controller Fedora CoreOS Config snippets | [] | UNSUPPORTED | -| worker_clc_snippets | Worker Fedora CoreOS Config snippets | [] | UNSUPPORTED | +| controller_snippets | Controller Fedora CoreOS Config snippets | [] | [examples](/advanced/customization/) | +| worker_snippets | Worker Fedora CoreOS Config snippets | [] | [examples](/advanced/customization/) | | networking | Choice of networking provider | "calico" | "calico" or "flannel" | | network_mtu | CNI interface MTU (calico only) | 1480 | 8981 | | host_cidr | CIDR IPv4 range to assign to EC2 instances | "10.0.0.0/16" | "10.1.0.0/16" | diff --git a/docs/fedora-coreos/bare-metal.md b/docs/fedora-coreos/bare-metal.md index 31ae83831..05ca82597 100644 --- a/docs/fedora-coreos/bare-metal.md +++ b/docs/fedora-coreos/bare-metal.md @@ -125,9 +125,9 @@ mv terraform-provider-matchbox-v0.3.0-linux-amd64/terraform-provider-matchbox ~/ Add the [terraform-provider-ct](https://github.com/poseidon/terraform-provider-ct) plugin binary for your system to `~/.terraform.d/plugins/`, noting the final name. ```sh -wget https://github.com/poseidon/terraform-provider-ct/releases/download/v0.4.0/terraform-provider-ct-v0.4.0-linux-amd64.tar.gz -tar xzf terraform-provider-ct-v0.4.0-linux-amd64.tar.gz -mv terraform-provider-ct-v0.4.0-linux-amd64/terraform-provider-ct ~/.terraform.d/plugins/terraform-provider-ct_v0.4.0 +wget https://github.com/poseidon/terraform-provider-ct/releases/download/v0.5.0/terraform-provider-ct-v0.5.0-linux-amd64.tar.gz +tar xzf terraform-provider-ct-v0.5.0-linux-amd64.tar.gz +mv terraform-provider-ct-v0.5.0-linux-amd64/terraform-provider-ct ~/.terraform.d/plugins/terraform-provider-ct_v0.5.0 ``` Read [concepts](/architecture/concepts/) to learn about Terraform, modules, and organizing resources. Change to your infrastructure repository (e.g. `infra`). @@ -150,7 +150,7 @@ provider "matchbox" { } provider "ct" { - version = "0.4.0" + version = "0.5.0" } ``` @@ -341,7 +341,7 @@ Check the [variables.tf](https://github.com/poseidon/typhoon/blob/master/bare-me | install_disk | Disk device where Fedora CoreOS should be installed | "sda" (not "/dev/sda" like Container Linux) | "sdb" | | networking | Choice of networking provider | "calico" | "calico" or "flannel" | | network_mtu | CNI interface MTU (calico-only) | 1480 | - | -| snippets | Map from machine names to lists of Fedora CoreOS Config snippets | {} | UNSUPPORTED | +| snippets | Map from machine names to lists of Fedora CoreOS Config snippets | {} | [examples](/advanced/customization/) | | network_ip_autodetection_method | Method to detect host IPv4 address (calico-only) | "first-found" | "can-reach=10.0.0.1" | | pod_cidr | CIDR IPv4 range to assign to Kubernetes pods | "10.2.0.0/16" | "10.22.0.0/16" | | service_cidr | CIDR IPv4 range to assign to Kubernetes services | "10.3.0.0/16" | "10.3.0.0/24" | diff --git a/docs/fedora-coreos/google-cloud.md b/docs/fedora-coreos/google-cloud.md index 9d84b2313..1f0bfd7ad 100644 --- a/docs/fedora-coreos/google-cloud.md +++ b/docs/fedora-coreos/google-cloud.md @@ -27,9 +27,9 @@ Terraform v0.12.21 Add the [terraform-provider-ct](https://github.com/poseidon/terraform-provider-ct) plugin binary for your system to `~/.terraform.d/plugins/`, noting the final name. ```sh -wget https://github.com/poseidon/terraform-provider-ct/releases/download/v0.4.0/terraform-provider-ct-v0.4.0-linux-amd64.tar.gz -tar xzf terraform-provider-ct-v0.4.0-linux-amd64.tar.gz -mv terraform-provider-ct-v0.4.0-linux-amd64/terraform-provider-ct ~/.terraform.d/plugins/terraform-provider-ct_v0.4.0 +wget https://github.com/poseidon/terraform-provider-ct/releases/download/v0.5.0/terraform-provider-ct-v0.5.0-linux-amd64.tar.gz +tar xzf terraform-provider-ct-v0.5.0-linux-amd64.tar.gz +mv terraform-provider-ct-v0.5.0-linux-amd64/terraform-provider-ct ~/.terraform.d/plugins/terraform-provider-ct_v0.5.0 ``` Read [concepts](/architecture/concepts/) to learn about Terraform, modules, and organizing resources. Change to your infrastructure repository (e.g. `infra`). @@ -59,7 +59,7 @@ provider "google" { } provider "ct" { - version = "0.4.0" + version = "0.5.0" } ``` @@ -239,8 +239,8 @@ resource "google_dns_managed_zone" "zone-for-clusters" { | os_image | Fedora CoreOS image for compute instances | "" | "fedora-coreos-31-20200113-3-1" | | disk_size | Size of the disk in GB | 40 | 100 | | worker_preemptible | If enabled, Compute Engine will terminate workers randomly within 24 hours | false | true | -| controller_snippets | Controller Fedora CoreOS Config snippets | [] | UNSUPPORTED | -| worker_snippets | Worker Fedora CoreOS Config snippets | [] | UNSUPPORTED | +| controller_snippets | Controller Fedora CoreOS Config snippets | [] | [examples](/advanced/customization/) | +| worker_snippets | Worker Fedora CoreOS Config snippets | [] | [examples](/advanced/customization/) | | networking | Choice of networking provider | "calico" | "calico" or "flannel" | | pod_cidr | CIDR IPv4 range to assign to Kubernetes pods | "10.2.0.0/16" | "10.22.0.0/16" | | service_cidr | CIDR IPv4 range to assign to Kubernetes services | "10.3.0.0/16" | "10.3.0.0/24" | From 70bdc9ec941a6d5a327c9bef07417fbd6b55319b Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Sat, 28 Mar 2020 17:49:17 -0700 Subject: [PATCH 376/523] Allow bootstrap re-apply for Fedora CoreOS GCP * Problem: Fedora CoreOS images are manually uploaded to GCP. When a cluster is created with a stale image, Zincati immediately checks for the latest stable image, fetches, and reboots. In practice, this can unfortunately occur exactly during the initial cluster bootstrap phase. * Recommended: Upload the latest Fedora CoreOS image regularly * Mitigation: Allow a failed bootstrap.service run (which won't touch the done ConditionalPathExists) to be re-run by running `terraforma apply` again. Add a known issue to CHANGES * Update docs to show the current Fedora CoreOS stable version to reduce likelihood users see this issue Longer term ideas: * Ideal: Fedora CoreOS publishes a stable channel. Instances will always boot with the latest image in a channel. The problem disappears since it works the same way AWS does * Timer: Consider some timer-based approach to have zincati delay any system reboots for the first ~30 min of a machine's life. Possibly just configured on the controller node https://github.com/coreos/zincati/pull/251 * External coordination: For Container Linux, locksmith filled a similar role and was disabled to allow CLUO to coordinate reboots. By running atop Kubernetes, it was not possible for the reboot to occur before cluster bootstrap * Rely on https://github.com/coreos/zincati/issues/115 to delay the reboot since bootstrap involves an SSH session * Use path-based activation of zincati on controllers and set that path at the end of the bootstrap process Rel: https://github.com/coreos/fedora-coreos-tracker/issues/239 --- CHANGES.md | 4 ++++ docs/fedora-coreos/google-cloud.md | 9 +++------ .../fedora-coreos/kubernetes/fcc/controller.yaml | 1 + 3 files changed, 8 insertions(+), 6 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 98a6df2aa..425f2db84 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -33,6 +33,10 @@ Notable changes between versions. * Update default `os_stream` from testing to stable +#### Google Cloud + +* Known: Use of stale Fedora CoreOS image may require terraform re-apply during bootstrap ([#687](https://github.com/poseidon/typhoon/pull/687)) + #### DigitalOcean * Rename `image` variable to `os_image` for consistency ([#677](https://github.com/poseidon/typhoon/pull/677)) (action required) diff --git a/docs/fedora-coreos/google-cloud.md b/docs/fedora-coreos/google-cloud.md index 1f0bfd7ad..b20e6614e 100644 --- a/docs/fedora-coreos/google-cloud.md +++ b/docs/fedora-coreos/google-cloud.md @@ -1,8 +1,5 @@ # Google Cloud -!!! danger - Typhoon for Fedora CoreOS is an alpha. Please report Fedora CoreOS bugs to [Fedora](https://github.com/coreos/fedora-coreos-tracker/issues) and Typhoon issues to Typhoon. - In this tutorial, we'll create a Kubernetes v1.18.0 cluster on Google Compute Engine with Fedora CoreOS. We'll declare a Kubernetes cluster using the Typhoon Terraform module. Then apply the changes to create a network, firewall rules, health checks, controller instances, worker managed instance group, load balancers, and TLS assets. @@ -76,13 +73,13 @@ Fedora CoreOS publishes images for Google Cloud, but does not yet upload them. G ``` gsutil list -gsutil cp fedora-coreos-31.20200113.3.1-gcp.x86_64.tar.gz gs://BUCKET +gsutil cp fedora-coreos-31.20200310.3.0-gcp.x86_64.tar.gz gs://BUCKET ``` Create a Compute Engine image from the file. ``` -gcloud compute images create fedora-coreos-31-20200113-3-1 --source-uri gs://BUCKET/fedora-coreos-31.20200113.3.1-gcp.x86_64.tar.gz +gcloud compute images create fedora-coreos-31-20200310-3-0 --source-uri gs://BUCKET/fedora-coreos-31.20200310.3.0-gcp.x86_64.tar.gz ``` ## Cluster @@ -100,7 +97,7 @@ module "yavin" { dns_zone_name = "example-zone" # custom image name from above - os_image = "fedora-coreos-31-20200113-3-1" + os_image = "fedora-coreos-31-20200310-3-0" # configuration ssh_authorized_key = "ssh-rsa AAAAB3Nz..." diff --git a/google-cloud/fedora-coreos/kubernetes/fcc/controller.yaml b/google-cloud/fedora-coreos/kubernetes/fcc/controller.yaml index 4603e4f71..85a75abda 100644 --- a/google-cloud/fedora-coreos/kubernetes/fcc/controller.yaml +++ b/google-cloud/fedora-coreos/kubernetes/fcc/controller.yaml @@ -116,6 +116,7 @@ systemd: Type=oneshot RemainAfterExit=true WorkingDirectory=/opt/bootstrap + ExecStartPre=-/usr/bin/podman rm bootstrap ExecStart=/usr/bin/podman run --name bootstrap \ --network host \ --volume /etc/kubernetes/bootstrap-secrets:/etc/kubernetes/secrets:ro,Z \ From bac5acb3bd6400d16d47a4a503cfa79cee392d4a Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Thu, 26 Mar 2020 22:18:38 -0700 Subject: [PATCH 377/523] Change default kube-system DaemonSet tolerations * Change kube-proxy, flannel, and calico-node DaemonSet tolerations to tolerate `node.kubernetes.io/not-ready` and `node-role.kubernetes.io/master` (i.e. controllers) explicitly, rather than tolerating all taints * kube-system DaemonSets will no longer tolerate custom node taints by default. Instead, custom node taints must be enumerated to opt-in to scheduling/executing the kube-system DaemonSets * Consider setting the daemonset_tolerations variable of terraform-render-bootstrap at a later date Background: Tolerating all taints ruled out use-cases where certain nodes might legitimately need to keep kube-proxy or CNI networking disabled Related: https://github.com/poseidon/terraform-render-bootstrap/pull/179 --- CHANGES.md | 3 +++ aws/container-linux/kubernetes/bootstrap.tf | 2 +- aws/fedora-coreos/kubernetes/bootstrap.tf | 2 +- azure/container-linux/kubernetes/bootstrap.tf | 2 +- bare-metal/container-linux/kubernetes/bootstrap.tf | 2 +- bare-metal/fedora-coreos/kubernetes/bootstrap.tf | 2 +- digital-ocean/container-linux/kubernetes/bootstrap.tf | 2 +- google-cloud/container-linux/kubernetes/bootstrap.tf | 2 +- google-cloud/fedora-coreos/kubernetes/bootstrap.tf | 2 +- 9 files changed, 11 insertions(+), 8 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 425f2db84..1a9b0f3dc 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -4,6 +4,9 @@ Notable changes between versions. ## Latest +* Change `kube-proxy` and `calico` or `flannel` to tolerate specific taints ([#682](https://github.com/poseidon/typhoon/pull/682)) + * Tolerate master and not-ready taints, rather than tolerating all taints + ## v1.18.0 * Kubernetes [v1.18.0](https://github.com/kubernetes/kubernetes/blob/master/CHANGELOG/CHANGELOG-1.18.md#v1180) diff --git a/aws/container-linux/kubernetes/bootstrap.tf b/aws/container-linux/kubernetes/bootstrap.tf index a5a100ca3..db2f4ac14 100644 --- a/aws/container-linux/kubernetes/bootstrap.tf +++ b/aws/container-linux/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=cb170f802d09dcbe88b050257bc676e25d3c4282" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=42723d13a696ea63af8aacc227169395914ecb18" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/aws/fedora-coreos/kubernetes/bootstrap.tf b/aws/fedora-coreos/kubernetes/bootstrap.tf index dcec91e07..31b18c982 100644 --- a/aws/fedora-coreos/kubernetes/bootstrap.tf +++ b/aws/fedora-coreos/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=cb170f802d09dcbe88b050257bc676e25d3c4282" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=42723d13a696ea63af8aacc227169395914ecb18" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/azure/container-linux/kubernetes/bootstrap.tf b/azure/container-linux/kubernetes/bootstrap.tf index 509ac63b4..803a873ec 100644 --- a/azure/container-linux/kubernetes/bootstrap.tf +++ b/azure/container-linux/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=cb170f802d09dcbe88b050257bc676e25d3c4282" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=42723d13a696ea63af8aacc227169395914ecb18" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/bare-metal/container-linux/kubernetes/bootstrap.tf b/bare-metal/container-linux/kubernetes/bootstrap.tf index bec22b399..e8513925f 100644 --- a/bare-metal/container-linux/kubernetes/bootstrap.tf +++ b/bare-metal/container-linux/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=cb170f802d09dcbe88b050257bc676e25d3c4282" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=42723d13a696ea63af8aacc227169395914ecb18" cluster_name = var.cluster_name api_servers = [var.k8s_domain_name] diff --git a/bare-metal/fedora-coreos/kubernetes/bootstrap.tf b/bare-metal/fedora-coreos/kubernetes/bootstrap.tf index 428a1d664..3359588fe 100644 --- a/bare-metal/fedora-coreos/kubernetes/bootstrap.tf +++ b/bare-metal/fedora-coreos/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=cb170f802d09dcbe88b050257bc676e25d3c4282" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=42723d13a696ea63af8aacc227169395914ecb18" cluster_name = var.cluster_name api_servers = [var.k8s_domain_name] diff --git a/digital-ocean/container-linux/kubernetes/bootstrap.tf b/digital-ocean/container-linux/kubernetes/bootstrap.tf index 0e11eca52..8f603ec51 100644 --- a/digital-ocean/container-linux/kubernetes/bootstrap.tf +++ b/digital-ocean/container-linux/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=cb170f802d09dcbe88b050257bc676e25d3c4282" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=42723d13a696ea63af8aacc227169395914ecb18" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/google-cloud/container-linux/kubernetes/bootstrap.tf b/google-cloud/container-linux/kubernetes/bootstrap.tf index 9f83af17e..d9f3e21f6 100644 --- a/google-cloud/container-linux/kubernetes/bootstrap.tf +++ b/google-cloud/container-linux/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=cb170f802d09dcbe88b050257bc676e25d3c4282" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=42723d13a696ea63af8aacc227169395914ecb18" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/google-cloud/fedora-coreos/kubernetes/bootstrap.tf b/google-cloud/fedora-coreos/kubernetes/bootstrap.tf index 113fdc0ff..e6eee707b 100644 --- a/google-cloud/fedora-coreos/kubernetes/bootstrap.tf +++ b/google-cloud/fedora-coreos/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=cb170f802d09dcbe88b050257bc676e25d3c4282" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=42723d13a696ea63af8aacc227169395914ecb18" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] From 99609727265b3c6fc0a84defecf85bc247d09f6d Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Tue, 31 Mar 2020 18:21:59 -0700 Subject: [PATCH 378/523] Fix bootstrap regression when networking="flannel" * Fix bootstrap error for missing `manifests-networking/crd*yaml` when `networking = "flannel"` * Cleanup manifest-networking directory left during bootstrap * Regressed in v1.18.0 changes for Calico https://github.com/poseidon/typhoon/pull/675 --- CHANGES.md | 2 ++ aws/container-linux/kubernetes/cl/controller.yaml | 4 ++-- aws/fedora-coreos/kubernetes/fcc/controller.yaml | 4 ++-- azure/container-linux/kubernetes/cl/controller.yaml | 4 ++-- bare-metal/container-linux/kubernetes/cl/controller.yaml | 4 ++-- bare-metal/fedora-coreos/kubernetes/fcc/controller.yaml | 4 ++-- digital-ocean/container-linux/kubernetes/cl/controller.yaml | 4 ++-- google-cloud/container-linux/kubernetes/cl/controller.yaml | 4 ++-- google-cloud/fedora-coreos/kubernetes/fcc/controller.yaml | 4 ++-- 9 files changed, 18 insertions(+), 16 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 1a9b0f3dc..9fae1824f 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -6,6 +6,8 @@ Notable changes between versions. * Change `kube-proxy` and `calico` or `flannel` to tolerate specific taints ([#682](https://github.com/poseidon/typhoon/pull/682)) * Tolerate master and not-ready taints, rather than tolerating all taints +* Fix bootstrap when `networking` mode `flannel` (non-default) is chosen + * Regressed in v1.18.0 changes for Calico ([#675](https://github.com/poseidon/typhoon/pull/675)) ## v1.18.0 diff --git a/aws/container-linux/kubernetes/cl/controller.yaml b/aws/container-linux/kubernetes/cl/controller.yaml index 940ec0217..68791791a 100644 --- a/aws/container-linux/kubernetes/cl/controller.yaml +++ b/aws/container-linux/kubernetes/cl/controller.yaml @@ -170,9 +170,9 @@ storage: sudo mkdir -p /opt/bootstrap/assets sudo mv manifests /opt/bootstrap/assets/manifests sudo mkdir -p /opt/bootstrap/assets/manifests/crds - sudo mv manifests-networking/crd*.yaml /opt/bootstrap/assets/manifests/crds + sudo mv manifests-networking/{crd,cluster}*.yaml /opt/bootstrap/assets/manifests/crds 2>/dev/null || true sudo mv manifests-networking/* /opt/bootstrap/assets/manifests/ - rm -rf assets auth static-manifests tls + rm -rf assets auth static-manifests tls manifests-networking - path: /opt/bootstrap/apply filesystem: root mode: 0544 diff --git a/aws/fedora-coreos/kubernetes/fcc/controller.yaml b/aws/fedora-coreos/kubernetes/fcc/controller.yaml index 4603e4f71..439c43015 100644 --- a/aws/fedora-coreos/kubernetes/fcc/controller.yaml +++ b/aws/fedora-coreos/kubernetes/fcc/controller.yaml @@ -155,9 +155,9 @@ storage: sudo mkdir -p /opt/bootstrap/assets sudo mv manifests /opt/bootstrap/assets/manifests sudo mkdir -p /opt/bootstrap/assets/manifests/crds - sudo mv manifests-networking/crd*.yaml /opt/bootstrap/assets/manifests/crds + sudo mv manifests-networking/{crd,cluster}*.yaml /opt/bootstrap/assets/manifests/crds 2>/dev/null || true sudo mv manifests-networking/* /opt/bootstrap/assets/manifests/ - rm -rf assets auth static-manifests tls + rm -rf assets auth static-manifests tls manifests-networking - path: /opt/bootstrap/apply mode: 0544 contents: diff --git a/azure/container-linux/kubernetes/cl/controller.yaml b/azure/container-linux/kubernetes/cl/controller.yaml index f34c46300..91b37e618 100644 --- a/azure/container-linux/kubernetes/cl/controller.yaml +++ b/azure/container-linux/kubernetes/cl/controller.yaml @@ -168,9 +168,9 @@ storage: sudo mkdir -p /opt/bootstrap/assets sudo mv manifests /opt/bootstrap/assets/manifests sudo mkdir -p /opt/bootstrap/assets/manifests/crds - sudo mv manifests-networking/crd*.yaml /opt/bootstrap/assets/manifests/crds + sudo mv manifests-networking/{crd,cluster}*.yaml /opt/bootstrap/assets/manifests/crds 2>/dev/null || true sudo mv manifests-networking/* /opt/bootstrap/assets/manifests/ - rm -rf assets auth static-manifests tls + rm -rf assets auth static-manifests tls manifests-networking - path: /opt/bootstrap/apply filesystem: root mode: 0544 diff --git a/bare-metal/container-linux/kubernetes/cl/controller.yaml b/bare-metal/container-linux/kubernetes/cl/controller.yaml index e5c14f304..532641799 100644 --- a/bare-metal/container-linux/kubernetes/cl/controller.yaml +++ b/bare-metal/container-linux/kubernetes/cl/controller.yaml @@ -186,9 +186,9 @@ storage: sudo mkdir -p /opt/bootstrap/assets sudo mv manifests /opt/bootstrap/assets/manifests sudo mkdir -p /opt/bootstrap/assets/manifests/crds - sudo mv manifests-networking/crd*.yaml /opt/bootstrap/assets/manifests/crds + sudo mv manifests-networking/{crd,cluster}*.yaml /opt/bootstrap/assets/manifests/crds 2>/dev/null || true sudo mv manifests-networking/* /opt/bootstrap/assets/manifests/ - rm -rf assets auth static-manifests tls + rm -rf assets auth static-manifests tls manifests-networking - path: /opt/bootstrap/apply filesystem: root mode: 0544 diff --git a/bare-metal/fedora-coreos/kubernetes/fcc/controller.yaml b/bare-metal/fedora-coreos/kubernetes/fcc/controller.yaml index 4925d7b54..fe774161d 100644 --- a/bare-metal/fedora-coreos/kubernetes/fcc/controller.yaml +++ b/bare-metal/fedora-coreos/kubernetes/fcc/controller.yaml @@ -166,9 +166,9 @@ storage: sudo mkdir -p /opt/bootstrap/assets sudo mv manifests /opt/bootstrap/assets/manifests sudo mkdir -p /opt/bootstrap/assets/manifests/crds - sudo mv manifests-networking/crd*.yaml /opt/bootstrap/assets/manifests/crds + sudo mv manifests-networking/{crd,cluster}*.yaml /opt/bootstrap/assets/manifests/crds 2>/dev/null || true sudo mv manifests-networking/* /opt/bootstrap/assets/manifests/ - rm -rf assets auth static-manifests tls + rm -rf assets auth static-manifests tls manifests-networking - path: /opt/bootstrap/apply mode: 0544 contents: diff --git a/digital-ocean/container-linux/kubernetes/cl/controller.yaml b/digital-ocean/container-linux/kubernetes/cl/controller.yaml index 04bb8e7ee..5419ab452 100644 --- a/digital-ocean/container-linux/kubernetes/cl/controller.yaml +++ b/digital-ocean/container-linux/kubernetes/cl/controller.yaml @@ -177,9 +177,9 @@ storage: sudo mkdir -p /opt/bootstrap/assets sudo mv manifests /opt/bootstrap/assets/manifests sudo mkdir -p /opt/bootstrap/assets/manifests/crds - sudo mv manifests-networking/crd*.yaml /opt/bootstrap/assets/manifests/crds + sudo mv manifests-networking/{crd,cluster}*.yaml /opt/bootstrap/assets/manifests/crds 2>/dev/null || true sudo mv manifests-networking/* /opt/bootstrap/assets/manifests/ - rm -rf assets auth static-manifests tls + rm -rf assets auth static-manifests tls manifests-networking - path: /opt/bootstrap/apply filesystem: root mode: 0544 diff --git a/google-cloud/container-linux/kubernetes/cl/controller.yaml b/google-cloud/container-linux/kubernetes/cl/controller.yaml index 424b41aee..9f2c9f617 100644 --- a/google-cloud/container-linux/kubernetes/cl/controller.yaml +++ b/google-cloud/container-linux/kubernetes/cl/controller.yaml @@ -168,9 +168,9 @@ storage: sudo mkdir -p /opt/bootstrap/assets sudo mv manifests /opt/bootstrap/assets/manifests sudo mkdir -p /opt/bootstrap/assets/manifests/crds - sudo mv manifests-networking/crd*.yaml /opt/bootstrap/assets/manifests/crds + sudo mv manifests-networking/{crd,cluster}*.yaml /opt/bootstrap/assets/manifests/crds 2>/dev/null || true sudo mv manifests-networking/* /opt/bootstrap/assets/manifests/ - rm -rf assets auth static-manifests tls + rm -rf assets auth static-manifests tls manifests-networking - path: /opt/bootstrap/apply filesystem: root mode: 0544 diff --git a/google-cloud/fedora-coreos/kubernetes/fcc/controller.yaml b/google-cloud/fedora-coreos/kubernetes/fcc/controller.yaml index 85a75abda..d254f9e1f 100644 --- a/google-cloud/fedora-coreos/kubernetes/fcc/controller.yaml +++ b/google-cloud/fedora-coreos/kubernetes/fcc/controller.yaml @@ -156,9 +156,9 @@ storage: sudo mkdir -p /opt/bootstrap/assets sudo mv manifests /opt/bootstrap/assets/manifests sudo mkdir -p /opt/bootstrap/assets/manifests/crds - sudo mv manifests-networking/crd*.yaml /opt/bootstrap/assets/manifests/crds + sudo mv manifests-networking/{crd,cluster}*.yaml /opt/bootstrap/assets/manifests/crds 2>/dev/null || true sudo mv manifests-networking/* /opt/bootstrap/assets/manifests/ - rm -rf assets auth static-manifests tls + rm -rf assets auth static-manifests tls manifests-networking - path: /opt/bootstrap/apply mode: 0544 contents: From c53dc66d4ae641f1aa1914073703d531aa7c02c9 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Sun, 29 Mar 2020 11:46:22 -0700 Subject: [PATCH 379/523] Rename Container Linux snippets variable for consistency * Rename controller_clc_snippets to controller_snippets (cloud platforms) * Rename worker_clc_snippets to worker_snippets (cloud platforms) * Rename clc_snippets to snippets (bare-metal) --- CHANGES.md | 5 +++- aws/container-linux/kubernetes/controllers.tf | 2 +- aws/container-linux/kubernetes/variables.tf | 4 +-- aws/container-linux/kubernetes/workers.tf | 2 +- .../kubernetes/workers/variables.tf | 2 +- .../kubernetes/workers/workers.tf | 2 +- .../container-linux/kubernetes/controllers.tf | 2 +- azure/container-linux/kubernetes/variables.tf | 4 +-- azure/container-linux/kubernetes/workers.tf | 2 +- .../kubernetes/workers/variables.tf | 2 +- .../kubernetes/workers/workers.tf | 2 +- .../container-linux/kubernetes/profiles.tf | 25 ++----------------- .../container-linux/kubernetes/variables.tf | 2 +- .../container-linux/kubernetes/controllers.tf | 2 +- .../container-linux/kubernetes/variables.tf | 4 +-- .../container-linux/kubernetes/workers.tf | 2 +- docs/advanced/customization.md | 10 ++++---- docs/advanced/worker-pools.md | 6 ++--- docs/cl/aws.md | 4 +-- docs/cl/azure.md | 4 +-- docs/cl/bare-metal.md | 2 +- docs/cl/digital-ocean.md | 4 +-- docs/cl/google-cloud.md | 4 +-- .../container-linux/kubernetes/controllers.tf | 2 +- .../container-linux/kubernetes/variables.tf | 4 +-- .../container-linux/kubernetes/workers.tf | 2 +- .../kubernetes/workers/variables.tf | 2 +- .../kubernetes/workers/workers.tf | 2 +- 28 files changed, 46 insertions(+), 64 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 9fae1824f..417f95851 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -6,6 +6,9 @@ Notable changes between versions. * Change `kube-proxy` and `calico` or `flannel` to tolerate specific taints ([#682](https://github.com/poseidon/typhoon/pull/682)) * Tolerate master and not-ready taints, rather than tolerating all taints +* Rename Container Linux `controller_clc_snippets` to `controller_snippets` for consistency +* Rename Container Linux `worker_clc_snippets` to `worker_snippets` for consistency +* Rename Container Linux `clc_snippets` (bare-metal) to `snippets` for consistency * Fix bootstrap when `networking` mode `flannel` (non-default) is chosen * Regressed in v1.18.0 changes for Calico ([#675](https://github.com/poseidon/typhoon/pull/675)) @@ -36,7 +39,7 @@ Notable changes between versions. #### Bare-Metal -* Update default `os_stream` from testing to stable +* Update Fedora CoreOS default `os_stream` from testing to stable #### Google Cloud diff --git a/aws/container-linux/kubernetes/controllers.tf b/aws/container-linux/kubernetes/controllers.tf index 76e22b172..cd02024e7 100644 --- a/aws/container-linux/kubernetes/controllers.tf +++ b/aws/container-linux/kubernetes/controllers.tf @@ -52,7 +52,7 @@ data "ct_config" "controller-ignitions" { count = var.controller_count content = data.template_file.controller-configs.*.rendered[count.index] pretty_print = false - snippets = var.controller_clc_snippets + snippets = var.controller_snippets } # Controller Container Linux configs diff --git a/aws/container-linux/kubernetes/variables.tf b/aws/container-linux/kubernetes/variables.tf index 21c41ff1a..bd9dec40c 100644 --- a/aws/container-linux/kubernetes/variables.tf +++ b/aws/container-linux/kubernetes/variables.tf @@ -77,13 +77,13 @@ variable "worker_target_groups" { default = [] } -variable "controller_clc_snippets" { +variable "controller_snippets" { type = list(string) description = "Controller Container Linux Config snippets" default = [] } -variable "worker_clc_snippets" { +variable "worker_snippets" { type = list(string) description = "Worker Container Linux Config snippets" default = [] diff --git a/aws/container-linux/kubernetes/workers.tf b/aws/container-linux/kubernetes/workers.tf index 9bebffdeb..8c831d476 100644 --- a/aws/container-linux/kubernetes/workers.tf +++ b/aws/container-linux/kubernetes/workers.tf @@ -18,7 +18,7 @@ module "workers" { ssh_authorized_key = var.ssh_authorized_key service_cidr = var.service_cidr cluster_domain_suffix = var.cluster_domain_suffix - clc_snippets = var.worker_clc_snippets + snippets = var.worker_snippets node_labels = var.worker_node_labels } diff --git a/aws/container-linux/kubernetes/workers/variables.tf b/aws/container-linux/kubernetes/workers/variables.tf index 94e823d7c..effd650f2 100644 --- a/aws/container-linux/kubernetes/workers/variables.tf +++ b/aws/container-linux/kubernetes/workers/variables.tf @@ -70,7 +70,7 @@ variable "target_groups" { default = [] } -variable "clc_snippets" { +variable "snippets" { type = list(string) description = "Container Linux Config snippets" default = [] diff --git a/aws/container-linux/kubernetes/workers/workers.tf b/aws/container-linux/kubernetes/workers/workers.tf index de0e07aa5..0d6ab035f 100644 --- a/aws/container-linux/kubernetes/workers/workers.tf +++ b/aws/container-linux/kubernetes/workers/workers.tf @@ -73,7 +73,7 @@ resource "aws_launch_configuration" "worker" { data "ct_config" "worker-ignition" { content = data.template_file.worker-config.rendered pretty_print = false - snippets = var.clc_snippets + snippets = var.snippets } # Worker Container Linux config diff --git a/azure/container-linux/kubernetes/controllers.tf b/azure/container-linux/kubernetes/controllers.tf index f535ffcd0..8e9e72f23 100644 --- a/azure/container-linux/kubernetes/controllers.tf +++ b/azure/container-linux/kubernetes/controllers.tf @@ -141,7 +141,7 @@ data "ct_config" "controller-ignitions" { count = var.controller_count content = data.template_file.controller-configs.*.rendered[count.index] pretty_print = false - snippets = var.controller_clc_snippets + snippets = var.controller_snippets } # Controller Container Linux configs diff --git a/azure/container-linux/kubernetes/variables.tf b/azure/container-linux/kubernetes/variables.tf index 7200e82eb..49ff63b0b 100644 --- a/azure/container-linux/kubernetes/variables.tf +++ b/azure/container-linux/kubernetes/variables.tf @@ -64,13 +64,13 @@ variable "worker_priority" { default = "Regular" } -variable "controller_clc_snippets" { +variable "controller_snippets" { type = list(string) description = "Controller Container Linux Config snippets" default = [] } -variable "worker_clc_snippets" { +variable "worker_snippets" { type = list(string) description = "Worker Container Linux Config snippets" default = [] diff --git a/azure/container-linux/kubernetes/workers.tf b/azure/container-linux/kubernetes/workers.tf index a99db98dd..93640dbbf 100644 --- a/azure/container-linux/kubernetes/workers.tf +++ b/azure/container-linux/kubernetes/workers.tf @@ -19,6 +19,6 @@ module "workers" { ssh_authorized_key = var.ssh_authorized_key service_cidr = var.service_cidr cluster_domain_suffix = var.cluster_domain_suffix - clc_snippets = var.worker_clc_snippets + snippets = var.worker_snippets node_labels = var.worker_node_labels } diff --git a/azure/container-linux/kubernetes/workers/variables.tf b/azure/container-linux/kubernetes/workers/variables.tf index 7ccedb94f..3f834dfe2 100644 --- a/azure/container-linux/kubernetes/workers/variables.tf +++ b/azure/container-linux/kubernetes/workers/variables.tf @@ -56,7 +56,7 @@ variable "priority" { default = "Regular" } -variable "clc_snippets" { +variable "snippets" { type = list(string) description = "Container Linux Config snippets" default = [] diff --git a/azure/container-linux/kubernetes/workers/workers.tf b/azure/container-linux/kubernetes/workers/workers.tf index 1a102bd56..66a0af24f 100644 --- a/azure/container-linux/kubernetes/workers/workers.tf +++ b/azure/container-linux/kubernetes/workers/workers.tf @@ -98,7 +98,7 @@ resource "azurerm_monitor_autoscale_setting" "workers" { data "ct_config" "worker-ignition" { content = data.template_file.worker-config.rendered pretty_print = false - snippets = var.clc_snippets + snippets = var.snippets } # Worker Container Linux configs diff --git a/bare-metal/container-linux/kubernetes/profiles.tf b/bare-metal/container-linux/kubernetes/profiles.tf index 98efdcc36..26dca9944 100644 --- a/bare-metal/container-linux/kubernetes/profiles.tf +++ b/bare-metal/container-linux/kubernetes/profiles.tf @@ -144,7 +144,7 @@ data "ct_config" "controller-ignitions" { count = length(var.controllers) content = data.template_file.controller-configs.*.rendered[count.index] pretty_print = false - snippets = local.clc_map[var.controllers.*.name[count.index]] + snippets = lookup(var.snippets, var.controllers.*.name[count.index], []) } data "template_file" "controller-configs" { @@ -174,7 +174,7 @@ data "ct_config" "worker-ignitions" { count = length(var.workers) content = data.template_file.worker-configs.*.rendered[count.index] pretty_print = false - snippets = local.clc_map[var.workers.*.name[count.index]] + snippets = lookup(var.snippets, var.workers.*.name[count.index], []) } data "template_file" "worker-configs" { @@ -192,24 +192,3 @@ data "template_file" "worker-configs" { node_taints = join(",", lookup(var.worker_node_taints, var.workers.*.name[count.index], [])) } } - -locals { - # Hack to workaround https://github.com/hashicorp/terraform/issues/17251 - # Still an issue in Terraform v0.12 https://github.com/hashicorp/terraform/issues/20572 - # Default Container Linux config snippets map every node names to list("\n") so - # all lookups succeed - clc_defaults = zipmap( - concat(var.controllers.*.name, var.workers.*.name), - chunklist(data.template_file.clc-default-snippets.*.rendered, 1), - ) - - # Union of the default and user specific snippets, later overrides prior. - clc_map = merge(local.clc_defaults, var.clc_snippets) -} - -// Horrible hack to generate a Terraform list of node count length -data "template_file" "clc-default-snippets" { - count = length(var.controllers) + length(var.workers) - template = "\n" -} - diff --git a/bare-metal/container-linux/kubernetes/variables.tf b/bare-metal/container-linux/kubernetes/variables.tf index 27b8ac8e7..e8f928b82 100644 --- a/bare-metal/container-linux/kubernetes/variables.tf +++ b/bare-metal/container-linux/kubernetes/variables.tf @@ -49,7 +49,7 @@ List of worker machine details (unique name, identifying MAC address, FQDN) EOD } -variable "clc_snippets" { +variable "snippets" { type = map(list(string)) description = "Map from machine names to lists of Container Linux Config snippets" default = {} diff --git a/digital-ocean/container-linux/kubernetes/controllers.tf b/digital-ocean/container-linux/kubernetes/controllers.tf index dd9562c2b..5d2f34190 100644 --- a/digital-ocean/container-linux/kubernetes/controllers.tf +++ b/digital-ocean/container-linux/kubernetes/controllers.tf @@ -72,7 +72,7 @@ data "ct_config" "controller-ignitions" { count = var.controller_count content = data.template_file.controller-configs.*.rendered[count.index] pretty_print = false - snippets = var.controller_clc_snippets + snippets = var.controller_snippets } # Controller Container Linux configs diff --git a/digital-ocean/container-linux/kubernetes/variables.tf b/digital-ocean/container-linux/kubernetes/variables.tf index 04386941c..c87d99b5f 100644 --- a/digital-ocean/container-linux/kubernetes/variables.tf +++ b/digital-ocean/container-linux/kubernetes/variables.tf @@ -47,13 +47,13 @@ variable "os_image" { default = "coreos-stable" } -variable "controller_clc_snippets" { +variable "controller_snippets" { type = list(string) description = "Controller Container Linux Config snippets" default = [] } -variable "worker_clc_snippets" { +variable "worker_snippets" { type = list(string) description = "Worker Container Linux Config snippets" default = [] diff --git a/digital-ocean/container-linux/kubernetes/workers.tf b/digital-ocean/container-linux/kubernetes/workers.tf index 02858bcb1..fa9b85d61 100644 --- a/digital-ocean/container-linux/kubernetes/workers.tf +++ b/digital-ocean/container-linux/kubernetes/workers.tf @@ -60,7 +60,7 @@ resource "digitalocean_tag" "workers" { data "ct_config" "worker-ignition" { content = data.template_file.worker-config.rendered pretty_print = false - snippets = var.worker_clc_snippets + snippets = var.worker_snippets } # Worker Container Linux config diff --git a/docs/advanced/customization.md b/docs/advanced/customization.md index 496352c49..91552eb40 100644 --- a/docs/advanced/customization.md +++ b/docs/advanced/customization.md @@ -125,7 +125,7 @@ systemd: Environment="ETCD_LOG_PACKAGE_LEVELS=etcdserver=WARNING,security=DEBUG" ``` -Reference the CLC contents by location (e.g. `file("./custom-units.yaml")`). On [AWS](/cl/aws/#cluster), [Azure](/cl/azure/#cluster), [DigitalOcean](/cl/digital-ocean/#cluster), or [Google Cloud](/cl/google-cloud/#cluster) extend the `controller_clc_snippets` or `worker_clc_snippets` list variables. +Reference the CLC contents by location (e.g. `file("./custom-units.yaml")`). On [AWS](/cl/aws/#cluster), [Azure](/cl/azure/#cluster), [DigitalOcean](/cl/digital-ocean/#cluster), or [Google Cloud](/cl/google-cloud/#cluster) extend the `controller_snippets` or `worker_snippets` list variables. ```tf module "nemo" { @@ -133,11 +133,11 @@ module "nemo" { controller_count = 1 worker_count = 2 - controller_clc_snippets = [ + controller_snippets = [ file("./custom-files"), file("./custom-units"), ] - worker_clc_snippets = [ + worker_snippets = [ file("./custom-files"), file("./custom-units")", ] @@ -145,12 +145,12 @@ module "nemo" { } ``` -On [Bare-Metal](/cl/bare-metal/#cluster), different CLCs may be used for each node (since hardware may be heterogeneous). Extend the `clc_snippets` map variable by mapping a controller or worker name key to a list of snippets. +On [Bare-Metal](/cl/bare-metal/#cluster), different CLCs may be used for each node (since hardware may be heterogeneous). Extend the `snippets` map variable by mapping a controller or worker name key to a list of snippets. ```tf module "mercury" { ... - clc_snippets = { + snippets = { "node2" = [file("./units/hello.yaml")] "node3" = [ file("./units/world.yaml"), diff --git a/docs/advanced/worker-pools.md b/docs/advanced/worker-pools.md index 6525b4f1b..04b6def0f 100644 --- a/docs/advanced/worker-pools.md +++ b/docs/advanced/worker-pools.md @@ -67,7 +67,7 @@ The AWS internal `workers` module supports a number of [variables](https://githu | disk_type | Type of the EBS volume | "gp2" | standard, gp2, io1 | | disk_iops | IOPS of the EBS volume | 0 (i.e. auto) | 400 | | spot_price | Spot price in USD for worker instances or 0 to use on-demand instances | 0 | 0.10 | -| clc_snippets | Container Linux Config snippets | [] | [example](/advanced/customization/#usage) | +| snippets | Container Linux Config snippets | [] | [example](/advanced/customization/#usage) | | service_cidr | Must match `service_cidr` of cluster | "10.3.0.0/16" | "10.3.0.0/24" | | node_labels | List of initial node labels | [] | ["worker-pool=foo"] | @@ -133,7 +133,7 @@ The Azure internal `workers` module supports a number of [variables](https://git | vm_type | Machine type for instances | "Standard_DS1_v2" | See below | | os_image | Channel for a Container Linux derivative | "coreos-stable" | coreos-stable, coreos-beta, coreos-alpha | | priority | Set priority to Low to use reduced cost surplus capacity, with the tradeoff that instances can be deallocated at any time | "Regular" | "Low" | -| clc_snippets | Container Linux Config snippets | [] | [example](/advanced/customization/#usage) | +| snippets | Container Linux Config snippets | [] | [example](/advanced/customization/#usage) | | service_cidr | CIDR IPv4 range to assign to Kubernetes services | "10.3.0.0/16" | "10.3.0.0/24" | | node_labels | List of initial node labels | [] | ["worker-pool=foo"] | @@ -209,7 +209,7 @@ Check the list of regions [docs](https://cloud.google.com/compute/docs/regions-z | os_image | Container Linux image for compute instances | "coreos-stable" | "coreos-alpha", "coreos-beta" | | disk_size | Size of the disk in GB | 40 | 100 | | preemptible | If true, Compute Engine will terminate instances randomly within 24 hours | false | true | -| clc_snippets | Container Linux Config snippets | [] | [example](/advanced/customization/#usage) | +| snippets | Container Linux Config snippets | [] | [example](/advanced/customization/#usage) | | service_cidr | Must match `service_cidr` of cluster | "10.3.0.0/16" | "10.3.0.0/24" | | node_labels | List of initial node labels | [] | ["worker-pool=foo"] | diff --git a/docs/cl/aws.md b/docs/cl/aws.md index c5f98aaf1..e4e67346c 100644 --- a/docs/cl/aws.md +++ b/docs/cl/aws.md @@ -214,8 +214,8 @@ Reference the DNS zone id with `aws_route53_zone.zone-for-clusters.zone_id`. | disk_iops | IOPS of the EBS volume | 0 (i.e. auto) | 400 | | worker_target_groups | Target group ARNs to which worker instances should be added | [] | [aws_lb_target_group.app.id] | | worker_price | Spot price in USD for worker instances or 0 to use on-demand instances | 0/null | 0.10 | -| controller_clc_snippets | Controller Container Linux Config snippets | [] | [example](/advanced/customization/) | -| worker_clc_snippets | Worker Container Linux Config snippets | [] | [example](/advanced/customization/) | +| controller_snippets | Controller Container Linux Config snippets | [] | [example](/advanced/customization/) | +| worker_snippets | Worker Container Linux Config snippets | [] | [example](/advanced/customization/) | | networking | Choice of networking provider | "calico" | "calico" or "flannel" | | network_mtu | CNI interface MTU (calico only) | 1480 | 8981 | | host_cidr | CIDR IPv4 range to assign to EC2 instances | "10.0.0.0/16" | "10.1.0.0/16" | diff --git a/docs/cl/azure.md b/docs/cl/azure.md index 02e70e0b6..b5ca47c34 100644 --- a/docs/cl/azure.md +++ b/docs/cl/azure.md @@ -231,8 +231,8 @@ Reference the DNS zone with `azurerm_dns_zone.clusters.name` and its resource gr | os_image | Channel for a Container Linux derivative | "coreos-stable" | coreos-stable, coreos-beta, coreos-alpha, flatcar-stable, flatcar-beta | | disk_size | Size of the disk in GB | 40 | 100 | | worker_priority | Set priority to Spot to use reduced cost surplus capacity, with the tradeoff that instances can be deallocated at any time | Regular | Spot | -| controller_clc_snippets | Controller Container Linux Config snippets | [] | [example](/advanced/customization/#usage) | -| worker_clc_snippets | Worker Container Linux Config snippets | [] | [example](/advanced/customization/#usage) | +| controller_snippets | Controller Container Linux Config snippets | [] | [example](/advanced/customization/#usage) | +| worker_snippets | Worker Container Linux Config snippets | [] | [example](/advanced/customization/#usage) | | networking | Choice of networking provider | "calico" | "flannel" or "calico" | | host_cidr | CIDR IPv4 range to assign to instances | "10.0.0.0/16" | "10.0.0.0/20" | | pod_cidr | CIDR IPv4 range to assign to Kubernetes pods | "10.2.0.0/16" | "10.22.0.0/16" | diff --git a/docs/cl/bare-metal.md b/docs/cl/bare-metal.md index 4f41ba64c..b84a700e3 100644 --- a/docs/cl/bare-metal.md +++ b/docs/cl/bare-metal.md @@ -352,7 +352,7 @@ Check the [variables.tf](https://github.com/poseidon/typhoon/blob/master/bare-me | install_disk | Disk device where Container Linux should be installed | "/dev/sda" | "/dev/sdb" | | networking | Choice of networking provider | "calico" | "calico" or "flannel" | | network_mtu | CNI interface MTU (calico-only) | 1480 | - | -| clc_snippets | Map from machine names to lists of Container Linux Config snippets | {} | [example](/advanced/customization/#usage) | +| snippets | Map from machine names to lists of Container Linux Config snippets | {} | [examples](/advanced/customization/) | | network_ip_autodetection_method | Method to detect host IPv4 address (calico-only) | "first-found" | "can-reach=10.0.0.1" | | pod_cidr | CIDR IPv4 range to assign to Kubernetes pods | "10.2.0.0/16" | "10.22.0.0/16" | | service_cidr | CIDR IPv4 range to assign to Kubernetes services | "10.3.0.0/16" | "10.3.0.0/24" | diff --git a/docs/cl/digital-ocean.md b/docs/cl/digital-ocean.md index 3104b802a..9295e72f7 100644 --- a/docs/cl/digital-ocean.md +++ b/docs/cl/digital-ocean.md @@ -244,8 +244,8 @@ Digital Ocean requires the SSH public key be uploaded to your account, so you ma | controller_type | Droplet type for controllers | "s-2vcpu-2gb" | s-2vcpu-2gb, s-2vcpu-4gb, s-4vcpu-8gb, ... | | worker_type | Droplet type for workers | "s-1vcpu-2gb" | s-1vcpu-2gb, s-2vcpu-2gb, ... | | os_image | Container Linux image for instances | "coreos-stable" | coreos-stable, coreos-beta, coreos-alpha, "custom-image-id" | -| controller_clc_snippets | Controller Container Linux Config snippets | [] | [example](/advanced/customization/) | -| worker_clc_snippets | Worker Container Linux Config snippets | [] | [example](/advanced/customization/) | +| controller_snippets | Controller Container Linux Config snippets | [] | [example](/advanced/customization/) | +| worker_snippets | Worker Container Linux Config snippets | [] | [example](/advanced/customization/) | | networking | Choice of networking provider | "calico" | "flannel" or "calico" | | pod_cidr | CIDR IPv4 range to assign to Kubernetes pods | "10.2.0.0/16" | "10.22.0.0/16" | | service_cidr | CIDR IPv4 range to assign to Kubernetes services | "10.3.0.0/16" | "10.3.0.0/24" | diff --git a/docs/cl/google-cloud.md b/docs/cl/google-cloud.md index aaa9e1dc1..c4fab5070 100644 --- a/docs/cl/google-cloud.md +++ b/docs/cl/google-cloud.md @@ -238,8 +238,8 @@ resource "google_dns_managed_zone" "zone-for-clusters" { | os_image | Container Linux image for compute instances | "coreos-stable" | "flatcar-linux-2303-4-0" | | disk_size | Size of the disk in GB | 40 | 100 | | worker_preemptible | If enabled, Compute Engine will terminate workers randomly within 24 hours | false | true | -| controller_clc_snippets | Controller Container Linux Config snippets | [] | [example](/advanced/customization/) | -| worker_clc_snippets | Worker Container Linux Config snippets | [] | [example](/advanced/customization/) | +| controller_snippets | Controller Container Linux Config snippets | [] | [example](/advanced/customization/) | +| worker_snippets | Worker Container Linux Config snippets | [] | [example](/advanced/customization/) | | networking | Choice of networking provider | "calico" | "calico" or "flannel" | | pod_cidr | CIDR IPv4 range to assign to Kubernetes pods | "10.2.0.0/16" | "10.22.0.0/16" | | service_cidr | CIDR IPv4 range to assign to Kubernetes services | "10.3.0.0/16" | "10.3.0.0/24" | diff --git a/google-cloud/container-linux/kubernetes/controllers.tf b/google-cloud/container-linux/kubernetes/controllers.tf index 3669b1732..6ab09219d 100644 --- a/google-cloud/container-linux/kubernetes/controllers.tf +++ b/google-cloud/container-linux/kubernetes/controllers.tf @@ -68,7 +68,7 @@ data "ct_config" "controller-ignitions" { count = var.controller_count content = data.template_file.controller-configs.*.rendered[count.index] pretty_print = false - snippets = var.controller_clc_snippets + snippets = var.controller_snippets } # Controller Container Linux configs diff --git a/google-cloud/container-linux/kubernetes/variables.tf b/google-cloud/container-linux/kubernetes/variables.tf index d8ed8e303..fc5e89c1a 100644 --- a/google-cloud/container-linux/kubernetes/variables.tf +++ b/google-cloud/container-linux/kubernetes/variables.tf @@ -64,13 +64,13 @@ variable "worker_preemptible" { default = false } -variable "controller_clc_snippets" { +variable "controller_snippets" { type = list(string) description = "Controller Container Linux Config snippets" default = [] } -variable "worker_clc_snippets" { +variable "worker_snippets" { type = list(string) description = "Worker Container Linux Config snippets" default = [] diff --git a/google-cloud/container-linux/kubernetes/workers.tf b/google-cloud/container-linux/kubernetes/workers.tf index 00e34c5f1..d58a119fb 100644 --- a/google-cloud/container-linux/kubernetes/workers.tf +++ b/google-cloud/container-linux/kubernetes/workers.tf @@ -17,7 +17,7 @@ module "workers" { ssh_authorized_key = var.ssh_authorized_key service_cidr = var.service_cidr cluster_domain_suffix = var.cluster_domain_suffix - clc_snippets = var.worker_clc_snippets + snippets = var.worker_snippets node_labels = var.worker_node_labels } diff --git a/google-cloud/container-linux/kubernetes/workers/variables.tf b/google-cloud/container-linux/kubernetes/workers/variables.tf index 025b68e9d..ec6983582 100644 --- a/google-cloud/container-linux/kubernetes/workers/variables.tf +++ b/google-cloud/container-linux/kubernetes/workers/variables.tf @@ -52,7 +52,7 @@ variable "preemptible" { default = false } -variable "clc_snippets" { +variable "snippets" { type = list(string) description = "Container Linux Config snippets" default = [] diff --git a/google-cloud/container-linux/kubernetes/workers/workers.tf b/google-cloud/container-linux/kubernetes/workers/workers.tf index c9f109484..7c130ed20 100644 --- a/google-cloud/container-linux/kubernetes/workers/workers.tf +++ b/google-cloud/container-linux/kubernetes/workers/workers.tf @@ -73,7 +73,7 @@ resource "google_compute_instance_template" "worker" { data "ct_config" "worker-ignition" { content = data.template_file.worker-config.rendered pretty_print = false - snippets = var.clc_snippets + snippets = var.snippets } # Worker Container Linux config From 135c6182b84d6601c8964e88385d35ee0c9e5f36 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Tue, 31 Mar 2020 18:30:10 -0700 Subject: [PATCH 380/523] Update flannel from v0.11.0 to v0.12.0 * https://github.com/coreos/flannel/releases/tag/v0.12.0 --- CHANGES.md | 5 +++-- aws/container-linux/kubernetes/bootstrap.tf | 2 +- aws/fedora-coreos/kubernetes/bootstrap.tf | 2 +- azure/container-linux/kubernetes/bootstrap.tf | 2 +- bare-metal/container-linux/kubernetes/bootstrap.tf | 2 +- bare-metal/fedora-coreos/kubernetes/bootstrap.tf | 2 +- digital-ocean/container-linux/kubernetes/bootstrap.tf | 2 +- google-cloud/container-linux/kubernetes/bootstrap.tf | 2 +- google-cloud/fedora-coreos/kubernetes/bootstrap.tf | 2 +- 9 files changed, 11 insertions(+), 10 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 417f95851..0658ba994 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -6,10 +6,11 @@ Notable changes between versions. * Change `kube-proxy` and `calico` or `flannel` to tolerate specific taints ([#682](https://github.com/poseidon/typhoon/pull/682)) * Tolerate master and not-ready taints, rather than tolerating all taints -* Rename Container Linux `controller_clc_snippets` to `controller_snippets` for consistency +* Update flannel from v0.11.0 to v0.12.0 ([#690](https://github.com/poseidon/typhoon/pull/690)) +* Rename Container Linux `controller_clc_snippets` to `controller_snippets` for consistency ([#688](https://github.com/poseidon/typhoon/pull/688)) * Rename Container Linux `worker_clc_snippets` to `worker_snippets` for consistency * Rename Container Linux `clc_snippets` (bare-metal) to `snippets` for consistency -* Fix bootstrap when `networking` mode `flannel` (non-default) is chosen +* Fix bootstrap when `networking` mode `flannel` (non-default) is chosen ([#689](https://github.com/poseidon/typhoon/pull/689)) * Regressed in v1.18.0 changes for Calico ([#675](https://github.com/poseidon/typhoon/pull/675)) ## v1.18.0 diff --git a/aws/container-linux/kubernetes/bootstrap.tf b/aws/container-linux/kubernetes/bootstrap.tf index db2f4ac14..20663f7e4 100644 --- a/aws/container-linux/kubernetes/bootstrap.tf +++ b/aws/container-linux/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=42723d13a696ea63af8aacc227169395914ecb18" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/aws/fedora-coreos/kubernetes/bootstrap.tf b/aws/fedora-coreos/kubernetes/bootstrap.tf index 31b18c982..59639ac92 100644 --- a/aws/fedora-coreos/kubernetes/bootstrap.tf +++ b/aws/fedora-coreos/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=42723d13a696ea63af8aacc227169395914ecb18" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/azure/container-linux/kubernetes/bootstrap.tf b/azure/container-linux/kubernetes/bootstrap.tf index 803a873ec..60da3ca36 100644 --- a/azure/container-linux/kubernetes/bootstrap.tf +++ b/azure/container-linux/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=42723d13a696ea63af8aacc227169395914ecb18" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/bare-metal/container-linux/kubernetes/bootstrap.tf b/bare-metal/container-linux/kubernetes/bootstrap.tf index e8513925f..d9898814f 100644 --- a/bare-metal/container-linux/kubernetes/bootstrap.tf +++ b/bare-metal/container-linux/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=42723d13a696ea63af8aacc227169395914ecb18" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=" cluster_name = var.cluster_name api_servers = [var.k8s_domain_name] diff --git a/bare-metal/fedora-coreos/kubernetes/bootstrap.tf b/bare-metal/fedora-coreos/kubernetes/bootstrap.tf index 3359588fe..587d155ca 100644 --- a/bare-metal/fedora-coreos/kubernetes/bootstrap.tf +++ b/bare-metal/fedora-coreos/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=42723d13a696ea63af8aacc227169395914ecb18" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=" cluster_name = var.cluster_name api_servers = [var.k8s_domain_name] diff --git a/digital-ocean/container-linux/kubernetes/bootstrap.tf b/digital-ocean/container-linux/kubernetes/bootstrap.tf index 8f603ec51..339dde6ca 100644 --- a/digital-ocean/container-linux/kubernetes/bootstrap.tf +++ b/digital-ocean/container-linux/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=42723d13a696ea63af8aacc227169395914ecb18" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/google-cloud/container-linux/kubernetes/bootstrap.tf b/google-cloud/container-linux/kubernetes/bootstrap.tf index d9f3e21f6..79ec3db70 100644 --- a/google-cloud/container-linux/kubernetes/bootstrap.tf +++ b/google-cloud/container-linux/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=42723d13a696ea63af8aacc227169395914ecb18" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/google-cloud/fedora-coreos/kubernetes/bootstrap.tf b/google-cloud/fedora-coreos/kubernetes/bootstrap.tf index e6eee707b..cbdb3d239 100644 --- a/google-cloud/fedora-coreos/kubernetes/bootstrap.tf +++ b/google-cloud/fedora-coreos/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=42723d13a696ea63af8aacc227169395914ecb18" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] From bbbaf949f98a33fdd72f72ee924ab4ae5a6b12d3 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Tue, 31 Mar 2020 20:28:27 -0700 Subject: [PATCH 381/523] Fix UDP outbound and clock sync timeouts on Azure workers * Add "lb" outbound rule for worker TCP _and_ UDP traffic * Fix Azure worker nodes clock synchronization being inactive due to timeouts reaching the CoreOS / Flatcar NTP pool * Fix Azure worker nodes not providing outbount UDP connectivity Background: Azure provides VMs outbound connectivity either by having a public IP or via an SNAT masquerade feature bundled with their virtual load balancing abstraction (in contrast with, say, a NAT gateway). Azure worker nodes have only a private IP, but are associated with the cluster load balancer's backend pool and ingress frontend IP. Outbound traffic uses SNAT with this frontend IP. A subtle detail with Azure SNAT seems to be that since both inbound lb_rule's are TCP only, outbound UDP traffic isn't SNAT'd (highlights the reasons Azure shouldn't have conflated inbound load balancing with outbound SNAT concepts). However, adding a separate outbound rule and disabling outbound SNAT on our ingress lb_rule's we can tell Azure to continue load balancing as before, and support outbound SNAT for worker traffic of both the TCP and UDP protocol. Fixes clock synchronization timeouts: ``` systemd-timesyncd[786]: Timed out waiting for reply from 45.79.36.123:123 (3.flatcar.pool.ntp.org) ``` Azure controller nodes have their own public IP, so controllers (and etcd) nodes have not had clock synchronization or outbound UDP issues --- CHANGES.md | 2 ++ azure/container-linux/kubernetes/lb.tf | 16 ++++++++++++++++ 2 files changed, 18 insertions(+) diff --git a/CHANGES.md b/CHANGES.md index 0658ba994..255f91b9a 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -12,6 +12,8 @@ Notable changes between versions. * Rename Container Linux `clc_snippets` (bare-metal) to `snippets` for consistency * Fix bootstrap when `networking` mode `flannel` (non-default) is chosen ([#689](https://github.com/poseidon/typhoon/pull/689)) * Regressed in v1.18.0 changes for Calico ([#675](https://github.com/poseidon/typhoon/pull/675)) +* Fix Azure worker UDP outbound connections ([#691](https://github.com/poseidon/typhoon/pull/691)) + * Fix Azure worker clock sync timeouts ## v1.18.0 diff --git a/azure/container-linux/kubernetes/lb.tf b/azure/container-linux/kubernetes/lb.tf index 6de81b22f..ef9247598 100644 --- a/azure/container-linux/kubernetes/lb.tf +++ b/azure/container-linux/kubernetes/lb.tf @@ -72,6 +72,7 @@ resource "azurerm_lb_rule" "ingress-http" { name = "ingress-http" loadbalancer_id = azurerm_lb.cluster.id frontend_ip_configuration_name = "ingress" + disable_outbound_snat = true protocol = "Tcp" frontend_port = 80 @@ -86,6 +87,7 @@ resource "azurerm_lb_rule" "ingress-https" { name = "ingress-https" loadbalancer_id = azurerm_lb.cluster.id frontend_ip_configuration_name = "ingress" + disable_outbound_snat = true protocol = "Tcp" frontend_port = 443 @@ -94,6 +96,20 @@ resource "azurerm_lb_rule" "ingress-https" { probe_id = azurerm_lb_probe.ingress.id } +# Worker outbound TCP/UDP SNAT +resource "azurerm_lb_outbound_rule" "worker-outbound" { + resource_group_name = azurerm_resource_group.cluster.name + + name = "worker" + loadbalancer_id = azurerm_lb.cluster.id + frontend_ip_configuration { + name = "ingress" + } + + protocol = "All" + backend_address_pool_id = azurerm_lb_backend_address_pool.worker.id +} + # Address pool of controllers resource "azurerm_lb_backend_address_pool" "controller" { resource_group_name = azurerm_resource_group.cluster.name From 3c1be7b0e0b9df1ce493e000e61a5c85a06e9078 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Tue, 31 Mar 2020 21:42:51 -0700 Subject: [PATCH 382/523] Fix terraform fmt --- aws/container-linux/kubernetes/network.tf | 8 ++++---- aws/container-linux/kubernetes/workers.tf | 2 +- aws/fedora-coreos/kubernetes/network.tf | 8 ++++---- .../container-linux/kubernetes/controllers.tf | 4 ++-- azure/container-linux/kubernetes/workers.tf | 2 +- .../kubernetes/workers/workers.tf | 18 +++++++++--------- .../container-linux/kubernetes/profiles.tf | 4 ++-- .../container-linux/kubernetes/variables.tf | 8 ++++---- .../fedora-coreos/kubernetes/profiles.tf | 8 ++++---- .../fedora-coreos/kubernetes/variables.tf | 8 ++++---- .../container-linux/kubernetes/controllers.tf | 2 +- .../container-linux/kubernetes/workers.tf | 2 +- .../kubernetes/workers/workers.tf | 6 +++--- 13 files changed, 40 insertions(+), 40 deletions(-) diff --git a/aws/container-linux/kubernetes/network.tf b/aws/container-linux/kubernetes/network.tf index f8ea0cec8..bdb4bff1e 100644 --- a/aws/container-linux/kubernetes/network.tf +++ b/aws/container-linux/kubernetes/network.tf @@ -31,15 +31,15 @@ resource "aws_route_table" "default" { } resource "aws_route" "egress-ipv4" { - route_table_id = aws_route_table.default.id + route_table_id = aws_route_table.default.id destination_cidr_block = "0.0.0.0/0" - gateway_id = aws_internet_gateway.gateway.id + gateway_id = aws_internet_gateway.gateway.id } resource "aws_route" "egress-ipv6" { - route_table_id = aws_route_table.default.id + route_table_id = aws_route_table.default.id destination_ipv6_cidr_block = "::/0" - gateway_id = aws_internet_gateway.gateway.id + gateway_id = aws_internet_gateway.gateway.id } # Subnets (one per availability zone) diff --git a/aws/container-linux/kubernetes/workers.tf b/aws/container-linux/kubernetes/workers.tf index 8c831d476..e8b57e620 100644 --- a/aws/container-linux/kubernetes/workers.tf +++ b/aws/container-linux/kubernetes/workers.tf @@ -18,7 +18,7 @@ module "workers" { ssh_authorized_key = var.ssh_authorized_key service_cidr = var.service_cidr cluster_domain_suffix = var.cluster_domain_suffix - snippets = var.worker_snippets + snippets = var.worker_snippets node_labels = var.worker_node_labels } diff --git a/aws/fedora-coreos/kubernetes/network.tf b/aws/fedora-coreos/kubernetes/network.tf index f8ea0cec8..bdb4bff1e 100644 --- a/aws/fedora-coreos/kubernetes/network.tf +++ b/aws/fedora-coreos/kubernetes/network.tf @@ -31,15 +31,15 @@ resource "aws_route_table" "default" { } resource "aws_route" "egress-ipv4" { - route_table_id = aws_route_table.default.id + route_table_id = aws_route_table.default.id destination_cidr_block = "0.0.0.0/0" - gateway_id = aws_internet_gateway.gateway.id + gateway_id = aws_internet_gateway.gateway.id } resource "aws_route" "egress-ipv6" { - route_table_id = aws_route_table.default.id + route_table_id = aws_route_table.default.id destination_ipv6_cidr_block = "::/0" - gateway_id = aws_internet_gateway.gateway.id + gateway_id = aws_internet_gateway.gateway.id } # Subnets (one per availability zone) diff --git a/azure/container-linux/kubernetes/controllers.tf b/azure/container-linux/kubernetes/controllers.tf index 8e9e72f23..671e7fb8f 100644 --- a/azure/container-linux/kubernetes/controllers.tf +++ b/azure/container-linux/kubernetes/controllers.tf @@ -65,9 +65,9 @@ resource "azurerm_linux_virtual_machine" "controllers" { for_each = local.flavor == "flatcar" ? [1] : [] content { - name = local.channel + name = local.channel publisher = "kinvolk" - product = "flatcar-container-linux" + product = "flatcar-container-linux" } } diff --git a/azure/container-linux/kubernetes/workers.tf b/azure/container-linux/kubernetes/workers.tf index 93640dbbf..808c473fc 100644 --- a/azure/container-linux/kubernetes/workers.tf +++ b/azure/container-linux/kubernetes/workers.tf @@ -19,6 +19,6 @@ module "workers" { ssh_authorized_key = var.ssh_authorized_key service_cidr = var.service_cidr cluster_domain_suffix = var.cluster_domain_suffix - snippets = var.worker_snippets + snippets = var.worker_snippets node_labels = var.worker_node_labels } diff --git a/azure/container-linux/kubernetes/workers/workers.tf b/azure/container-linux/kubernetes/workers/workers.tf index 66a0af24f..3e151e6e5 100644 --- a/azure/container-linux/kubernetes/workers/workers.tf +++ b/azure/container-linux/kubernetes/workers/workers.tf @@ -9,19 +9,19 @@ locals { resource "azurerm_linux_virtual_machine_scale_set" "workers" { resource_group_name = var.resource_group_name - name = "${var.name}-worker" - location = var.region - sku = var.vm_type + name = "${var.name}-worker" + location = var.region + sku = var.vm_type instances = var.worker_count # instance name prefix for instances in the set - computer_name_prefix = "${var.name}-worker" + computer_name_prefix = "${var.name}-worker" single_placement_group = false - custom_data = base64encode(data.ct_config.worker-ignition.rendered) + custom_data = base64encode(data.ct_config.worker-ignition.rendered) # storage os_disk { storage_account_type = "Standard_LRS" - caching = "ReadWrite" + caching = "ReadWrite" } source_image_reference { @@ -36,16 +36,16 @@ resource "azurerm_linux_virtual_machine_scale_set" "workers" { for_each = local.flavor == "flatcar" ? [1] : [] content { - name = local.channel + name = local.channel publisher = "kinvolk" - product = "flatcar-container-linux" + product = "flatcar-container-linux" } } # Azure requires setting admin_ssh_key, though Ignition custom_data handles it too admin_username = "core" admin_ssh_key { - username = "core" + username = "core" public_key = var.ssh_authorized_key } diff --git a/bare-metal/container-linux/kubernetes/profiles.tf b/bare-metal/container-linux/kubernetes/profiles.tf index 26dca9944..7ec6cc3a7 100644 --- a/bare-metal/container-linux/kubernetes/profiles.tf +++ b/bare-metal/container-linux/kubernetes/profiles.tf @@ -144,7 +144,7 @@ data "ct_config" "controller-ignitions" { count = length(var.controllers) content = data.template_file.controller-configs.*.rendered[count.index] pretty_print = false - snippets = lookup(var.snippets, var.controllers.*.name[count.index], []) + snippets = lookup(var.snippets, var.controllers.*.name[count.index], []) } data "template_file" "controller-configs" { @@ -174,7 +174,7 @@ data "ct_config" "worker-ignitions" { count = length(var.workers) content = data.template_file.worker-configs.*.rendered[count.index] pretty_print = false - snippets = lookup(var.snippets, var.workers.*.name[count.index], []) + snippets = lookup(var.snippets, var.workers.*.name[count.index], []) } data "template_file" "worker-configs" { diff --git a/bare-metal/container-linux/kubernetes/variables.tf b/bare-metal/container-linux/kubernetes/variables.tf index e8f928b82..62cf47c29 100644 --- a/bare-metal/container-linux/kubernetes/variables.tf +++ b/bare-metal/container-linux/kubernetes/variables.tf @@ -56,15 +56,15 @@ variable "snippets" { } variable "worker_node_labels" { - type = map(list(string)) + type = map(list(string)) description = "Map from worker names to lists of initial node labels" - default = {} + default = {} } variable "worker_node_taints" { - type = map(list(string)) + type = map(list(string)) description = "Map from worker names to lists of initial node taints" - default = {} + default = {} } # configuration diff --git a/bare-metal/fedora-coreos/kubernetes/profiles.tf b/bare-metal/fedora-coreos/kubernetes/profiles.tf index 8a3cfdb16..0ad5bd86a 100644 --- a/bare-metal/fedora-coreos/kubernetes/profiles.tf +++ b/bare-metal/fedora-coreos/kubernetes/profiles.tf @@ -48,8 +48,8 @@ resource "matchbox_profile" "controllers" { data "ct_config" "controller-ignitions" { count = length(var.controllers) - content = data.template_file.controller-configs.*.rendered[count.index] - strict = true + content = data.template_file.controller-configs.*.rendered[count.index] + strict = true snippets = lookup(var.snippets, var.controllers.*.name[count.index], []) } @@ -84,8 +84,8 @@ resource "matchbox_profile" "workers" { data "ct_config" "worker-ignitions" { count = length(var.workers) - content = data.template_file.worker-configs.*.rendered[count.index] - strict = true + content = data.template_file.worker-configs.*.rendered[count.index] + strict = true snippets = lookup(var.snippets, var.workers.*.name[count.index], []) } diff --git a/bare-metal/fedora-coreos/kubernetes/variables.tf b/bare-metal/fedora-coreos/kubernetes/variables.tf index 26b1cf655..eefa0a4b4 100644 --- a/bare-metal/fedora-coreos/kubernetes/variables.tf +++ b/bare-metal/fedora-coreos/kubernetes/variables.tf @@ -57,15 +57,15 @@ variable "snippets" { } variable "worker_node_labels" { - type = map(list(string)) + type = map(list(string)) description = "Map from worker names to lists of initial node labels" - default = {} + default = {} } variable "worker_node_taints" { - type = map(list(string)) + type = map(list(string)) description = "Map from worker names to lists of initial node taints" - default = {} + default = {} } # configuration diff --git a/digital-ocean/container-linux/kubernetes/controllers.tf b/digital-ocean/container-linux/kubernetes/controllers.tf index 5d2f34190..6d183b952 100644 --- a/digital-ocean/container-linux/kubernetes/controllers.tf +++ b/digital-ocean/container-linux/kubernetes/controllers.tf @@ -1,5 +1,5 @@ locals { - official_images = ["coreos-stable", "coreos-beta", "coreos-alpha"] + official_images = ["coreos-stable", "coreos-beta", "coreos-alpha"] is_official_image = contains(local.official_images, var.os_image) } diff --git a/google-cloud/container-linux/kubernetes/workers.tf b/google-cloud/container-linux/kubernetes/workers.tf index d58a119fb..91a32bd0c 100644 --- a/google-cloud/container-linux/kubernetes/workers.tf +++ b/google-cloud/container-linux/kubernetes/workers.tf @@ -17,7 +17,7 @@ module "workers" { ssh_authorized_key = var.ssh_authorized_key service_cidr = var.service_cidr cluster_domain_suffix = var.cluster_domain_suffix - snippets = var.worker_snippets + snippets = var.worker_snippets node_labels = var.worker_node_labels } diff --git a/google-cloud/fedora-coreos/kubernetes/workers/workers.tf b/google-cloud/fedora-coreos/kubernetes/workers/workers.tf index 767732126..1d3d65894 100644 --- a/google-cloud/fedora-coreos/kubernetes/workers/workers.tf +++ b/google-cloud/fedora-coreos/kubernetes/workers/workers.tf @@ -71,9 +71,9 @@ resource "google_compute_instance_template" "worker" { # Worker Ignition config data "ct_config" "worker-ignition" { - content = data.template_file.worker-config.rendered - strict = true - snippets = var.snippets + content = data.template_file.worker-config.rendered + strict = true + snippets = var.snippets } # Worker Fedora CoreOS config From d47d40b51789166b64e57791ede306ebc2fae0c7 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Tue, 31 Mar 2020 00:50:16 -0700 Subject: [PATCH 383/523] Refresh Prometheus rules/alerts and Grafana dashboards * Refresh upstream Prometheus rules and alerts and Grafana dashboards * All Loki recording rules for convenience --- CHANGES.md | 4 + .../grafana/dashboards-k8s-resources-1.yaml | 159 ++---------- .../grafana/dashboards-k8s-resources-2.yaml | 175 +++---------- addons/prometheus/rules.yaml | 231 +++++++++++++++--- 4 files changed, 267 insertions(+), 302 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 255f91b9a..a96e2dede 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -15,6 +15,10 @@ Notable changes between versions. * Fix Azure worker UDP outbound connections ([#691](https://github.com/poseidon/typhoon/pull/691)) * Fix Azure worker clock sync timeouts +#### Addons + +* Refresh Prometheus rules and alerts + ## v1.18.0 * Kubernetes [v1.18.0](https://github.com/kubernetes/kubernetes/blob/master/CHANGELOG/CHANGELOG-1.18.md#v1180) diff --git a/addons/grafana/dashboards-k8s-resources-1.yaml b/addons/grafana/dashboards-k8s-resources-1.yaml index b59a48be2..70f207244 100644 --- a/addons/grafana/dashboards-k8s-resources-1.yaml +++ b/addons/grafana/dashboards-k8s-resources-1.yaml @@ -59,7 +59,7 @@ data: "steppedLine": false, "targets": [ { - "expr": "1 - avg(rate(node_cpu_seconds_total{mode=\"idle\", cluster=\"$cluster\"}[1m]))", + "expr": "1 - avg(rate(node_cpu_seconds_total{mode=\"idle\", cluster=\"$cluster\"}[$__interval]))", "format": "time_series", "instant": true, "intervalFactor": 2, @@ -1561,7 +1561,7 @@ data: ], "targets": [ { - "expr": "sum(irate(container_network_receive_bytes_total{cluster=\"$cluster\", namespace=~\".+\"}[$interval])) by (namespace)", + "expr": "sum(irate(container_network_receive_bytes_total{cluster=\"$cluster\", namespace=~\".+\"}[$__interval])) by (namespace)", "format": "table", "instant": true, "intervalFactor": 2, @@ -1570,7 +1570,7 @@ data: "step": 10 }, { - "expr": "sum(irate(container_network_transmit_bytes_total{cluster=\"$cluster\", namespace=~\".+\"}[$interval])) by (namespace)", + "expr": "sum(irate(container_network_transmit_bytes_total{cluster=\"$cluster\", namespace=~\".+\"}[$__interval])) by (namespace)", "format": "table", "instant": true, "intervalFactor": 2, @@ -1579,7 +1579,7 @@ data: "step": 10 }, { - "expr": "sum(irate(container_network_receive_packets_total{cluster=\"$cluster\", namespace=~\".+\"}[$interval])) by (namespace)", + "expr": "sum(irate(container_network_receive_packets_total{cluster=\"$cluster\", namespace=~\".+\"}[$__interval])) by (namespace)", "format": "table", "instant": true, "intervalFactor": 2, @@ -1588,7 +1588,7 @@ data: "step": 10 }, { - "expr": "sum(irate(container_network_transmit_packets_total{cluster=\"$cluster\", namespace=~\".+\"}[$interval])) by (namespace)", + "expr": "sum(irate(container_network_transmit_packets_total{cluster=\"$cluster\", namespace=~\".+\"}[$__interval])) by (namespace)", "format": "table", "instant": true, "intervalFactor": 2, @@ -1597,7 +1597,7 @@ data: "step": 10 }, { - "expr": "sum(irate(container_network_receive_packets_dropped_total{cluster=\"$cluster\", namespace=~\".+\"}[$interval])) by (namespace)", + "expr": "sum(irate(container_network_receive_packets_dropped_total{cluster=\"$cluster\", namespace=~\".+\"}[$__interval])) by (namespace)", "format": "table", "instant": true, "intervalFactor": 2, @@ -1606,7 +1606,7 @@ data: "step": 10 }, { - "expr": "sum(irate(container_network_transmit_packets_dropped_total{cluster=\"$cluster\", namespace=~\".+\"}[$interval])) by (namespace)", + "expr": "sum(irate(container_network_transmit_packets_dropped_total{cluster=\"$cluster\", namespace=~\".+\"}[$__interval])) by (namespace)", "format": "table", "instant": true, "intervalFactor": 2, @@ -1706,7 +1706,7 @@ data: "steppedLine": false, "targets": [ { - "expr": "sum(irate(container_network_receive_bytes_total{cluster=\"$cluster\", namespace=~\".+\"}[$interval])) by (namespace)", + "expr": "sum(irate(container_network_receive_bytes_total{cluster=\"$cluster\", namespace=~\".+\"}[$__interval])) by (namespace)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{namespace}}", @@ -1804,7 +1804,7 @@ data: "steppedLine": false, "targets": [ { - "expr": "sum(irate(container_network_transmit_bytes_total{cluster=\"$cluster\", namespace=~\".+\"}[$interval])) by (namespace)", + "expr": "sum(irate(container_network_transmit_bytes_total{cluster=\"$cluster\", namespace=~\".+\"}[$__interval])) by (namespace)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{namespace}}", @@ -1902,7 +1902,7 @@ data: "steppedLine": false, "targets": [ { - "expr": "avg(irate(container_network_receive_bytes_total{cluster=\"$cluster\", namespace=~\".+\"}[$interval])) by (namespace)", + "expr": "avg(irate(container_network_receive_bytes_total{cluster=\"$cluster\", namespace=~\".+\"}[$__interval])) by (namespace)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{namespace}}", @@ -2000,7 +2000,7 @@ data: "steppedLine": false, "targets": [ { - "expr": "avg(irate(container_network_transmit_bytes_total{cluster=\"$cluster\", namespace=~\".+\"}[$interval])) by (namespace)", + "expr": "avg(irate(container_network_transmit_bytes_total{cluster=\"$cluster\", namespace=~\".+\"}[$__interval])) by (namespace)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{namespace}}", @@ -2098,7 +2098,7 @@ data: "steppedLine": false, "targets": [ { - "expr": "sum(irate(container_network_receive_packets_total{cluster=\"$cluster\", namespace=~\".+\"}[$interval])) by (namespace)", + "expr": "sum(irate(container_network_receive_packets_total{cluster=\"$cluster\", namespace=~\".+\"}[$__interval])) by (namespace)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{namespace}}", @@ -2196,7 +2196,7 @@ data: "steppedLine": false, "targets": [ { - "expr": "sum(irate(container_network_receive_packets_total{cluster=\"$cluster\", namespace=~\".+\"}[$interval])) by (namespace)", + "expr": "sum(irate(container_network_receive_packets_total{cluster=\"$cluster\", namespace=~\".+\"}[$__interval])) by (namespace)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{namespace}}", @@ -2294,7 +2294,7 @@ data: "steppedLine": false, "targets": [ { - "expr": "sum(irate(container_network_receive_packets_dropped_total{cluster=\"$cluster\", namespace=~\".+\"}[$interval])) by (namespace)", + "expr": "sum(irate(container_network_receive_packets_dropped_total{cluster=\"$cluster\", namespace=~\".+\"}[$__interval])) by (namespace)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{namespace}}", @@ -2392,7 +2392,7 @@ data: "steppedLine": false, "targets": [ { - "expr": "sum(irate(container_network_transmit_packets_dropped_total{cluster=\"$cluster\", namespace=~\".+\"}[$interval])) by (namespace)", + "expr": "sum(irate(container_network_transmit_packets_dropped_total{cluster=\"$cluster\", namespace=~\".+\"}[$__interval])) by (namespace)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{namespace}}", @@ -2499,41 +2499,6 @@ data: "type": "query", "useTags": false }, - { - "allValue": null, - "auto": false, - "auto_count": 30, - "auto_min": "10s", - "current": { - "text": "5m", - "value": "5m" - }, - "datasource": "$datasource", - "hide": 2, - "includeAll": false, - "label": null, - "multi": false, - "name": "interval", - "options": [ - { - "selected": true, - "text": "4h", - "value": "4h" - } - ], - "query": "4h", - "refresh": 2, - "regex": "", - "skipUrlSync": false, - "sort": 1, - "tagValuesQuery": "", - "tags": [ - - ], - "tagsQuery": "", - "type": "interval", - "useTags": false - }, { "allValue": null, "current": { @@ -4033,7 +3998,7 @@ data: ], "targets": [ { - "expr": "sum(irate(container_network_receive_bytes_total{cluster=\"$cluster\", namespace=~\"$namespace\"}[$interval])) by (pod)", + "expr": "sum(irate(container_network_receive_bytes_total{cluster=\"$cluster\", namespace=~\"$namespace\"}[$__interval])) by (pod)", "format": "table", "instant": true, "intervalFactor": 2, @@ -4042,7 +4007,7 @@ data: "step": 10 }, { - "expr": "sum(irate(container_network_transmit_bytes_total{cluster=\"$cluster\", namespace=~\"$namespace\"}[$interval])) by (pod)", + "expr": "sum(irate(container_network_transmit_bytes_total{cluster=\"$cluster\", namespace=~\"$namespace\"}[$__interval])) by (pod)", "format": "table", "instant": true, "intervalFactor": 2, @@ -4051,7 +4016,7 @@ data: "step": 10 }, { - "expr": "sum(irate(container_network_receive_packets_total{cluster=\"$cluster\", namespace=~\"$namespace\"}[$interval])) by (pod)", + "expr": "sum(irate(container_network_receive_packets_total{cluster=\"$cluster\", namespace=~\"$namespace\"}[$__interval])) by (pod)", "format": "table", "instant": true, "intervalFactor": 2, @@ -4060,7 +4025,7 @@ data: "step": 10 }, { - "expr": "sum(irate(container_network_transmit_packets_total{cluster=\"$cluster\", namespace=~\"$namespace\"}[$interval])) by (pod)", + "expr": "sum(irate(container_network_transmit_packets_total{cluster=\"$cluster\", namespace=~\"$namespace\"}[$__interval])) by (pod)", "format": "table", "instant": true, "intervalFactor": 2, @@ -4069,7 +4034,7 @@ data: "step": 10 }, { - "expr": "sum(irate(container_network_receive_packets_dropped_total{cluster=\"$cluster\", namespace=~\"$namespace\"}[$interval])) by (pod)", + "expr": "sum(irate(container_network_receive_packets_dropped_total{cluster=\"$cluster\", namespace=~\"$namespace\"}[$__interval])) by (pod)", "format": "table", "instant": true, "intervalFactor": 2, @@ -4078,7 +4043,7 @@ data: "step": 10 }, { - "expr": "sum(irate(container_network_transmit_packets_dropped_total{cluster=\"$cluster\", namespace=~\"$namespace\"}[$interval])) by (pod)", + "expr": "sum(irate(container_network_transmit_packets_dropped_total{cluster=\"$cluster\", namespace=~\"$namespace\"}[$__interval])) by (pod)", "format": "table", "instant": true, "intervalFactor": 2, @@ -4178,7 +4143,7 @@ data: "steppedLine": false, "targets": [ { - "expr": "sum(irate(container_network_receive_bytes_total{cluster=\"$cluster\", namespace=~\"$namespace\"}[$interval])) by (pod)", + "expr": "sum(irate(container_network_receive_bytes_total{cluster=\"$cluster\", namespace=~\"$namespace\"}[$__interval])) by (pod)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{pod}}", @@ -4276,7 +4241,7 @@ data: "steppedLine": false, "targets": [ { - "expr": "sum(irate(container_network_transmit_bytes_total{cluster=\"$cluster\", namespace=~\"$namespace\"}[$interval])) by (pod)", + "expr": "sum(irate(container_network_transmit_bytes_total{cluster=\"$cluster\", namespace=~\"$namespace\"}[$__interval])) by (pod)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{pod}}", @@ -4374,7 +4339,7 @@ data: "steppedLine": false, "targets": [ { - "expr": "sum(irate(container_network_receive_packets_total{cluster=\"$cluster\", namespace=~\"$namespace\"}[$interval])) by (pod)", + "expr": "sum(irate(container_network_receive_packets_total{cluster=\"$cluster\", namespace=~\"$namespace\"}[$__interval])) by (pod)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{pod}}", @@ -4472,7 +4437,7 @@ data: "steppedLine": false, "targets": [ { - "expr": "sum(irate(container_network_receive_packets_total{cluster=\"$cluster\", namespace=~\"$namespace\"}[$interval])) by (pod)", + "expr": "sum(irate(container_network_receive_packets_total{cluster=\"$cluster\", namespace=~\"$namespace\"}[$__interval])) by (pod)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{pod}}", @@ -4570,7 +4535,7 @@ data: "steppedLine": false, "targets": [ { - "expr": "sum(irate(container_network_receive_packets_dropped_total{cluster=\"$cluster\", namespace=~\"$namespace\"}[$interval])) by (pod)", + "expr": "sum(irate(container_network_receive_packets_dropped_total{cluster=\"$cluster\", namespace=~\"$namespace\"}[$__interval])) by (pod)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{pod}}", @@ -4668,7 +4633,7 @@ data: "steppedLine": false, "targets": [ { - "expr": "sum(irate(container_network_transmit_packets_dropped_total{cluster=\"$cluster\", namespace=~\"$namespace\"}[$interval])) by (pod)", + "expr": "sum(irate(container_network_transmit_packets_dropped_total{cluster=\"$cluster\", namespace=~\"$namespace\"}[$__interval])) by (pod)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{pod}}", @@ -4748,41 +4713,6 @@ data: "regex": "", "type": "datasource" }, - { - "allValue": null, - "auto": false, - "auto_count": 30, - "auto_min": "10s", - "current": { - "text": "5m", - "value": "5m" - }, - "datasource": "$datasource", - "hide": 2, - "includeAll": false, - "label": null, - "multi": false, - "name": "interval", - "options": [ - { - "selected": true, - "text": "4h", - "value": "4h" - } - ], - "query": "4h", - "refresh": 2, - "regex": "", - "skipUrlSync": false, - "sort": 1, - "tagValuesQuery": "", - "tags": [ - - ], - "tagsQuery": "", - "type": "interval", - "useTags": false - }, { "allValue": null, "current": { @@ -5724,41 +5654,6 @@ data: "regex": "", "type": "datasource" }, - { - "allValue": null, - "auto": false, - "auto_count": 30, - "auto_min": "10s", - "current": { - "text": "5m", - "value": "5m" - }, - "datasource": "$datasource", - "hide": 2, - "includeAll": false, - "label": null, - "multi": false, - "name": "interval", - "options": [ - { - "selected": true, - "text": "4h", - "value": "4h" - } - ], - "query": "4h", - "refresh": 2, - "regex": "", - "skipUrlSync": false, - "sort": 1, - "tagValuesQuery": "", - "tags": [ - - ], - "tagsQuery": "", - "type": "interval", - "useTags": false - }, { "allValue": null, "current": { diff --git a/addons/grafana/dashboards-k8s-resources-2.yaml b/addons/grafana/dashboards-k8s-resources-2.yaml index 2475dbac3..efe2c00b9 100644 --- a/addons/grafana/dashboards-k8s-resources-2.yaml +++ b/addons/grafana/dashboards-k8s-resources-2.yaml @@ -1042,7 +1042,7 @@ data: "steppedLine": false, "targets": [ { - "expr": "sum(irate(container_network_receive_bytes_total{namespace=~\"$namespace\", pod=~\"$pod\"}[$interval])) by (pod)", + "expr": "sum(irate(container_network_receive_bytes_total{namespace=~\"$namespace\", pod=~\"$pod\"}[$__interval])) by (pod)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{pod}}", @@ -1140,7 +1140,7 @@ data: "steppedLine": false, "targets": [ { - "expr": "sum(irate(container_network_transmit_bytes_total{namespace=~\"$namespace\", pod=~\"$pod\"}[$interval])) by (pod)", + "expr": "sum(irate(container_network_transmit_bytes_total{namespace=~\"$namespace\", pod=~\"$pod\"}[$__interval])) by (pod)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{pod}}", @@ -1238,7 +1238,7 @@ data: "steppedLine": false, "targets": [ { - "expr": "sum(irate(container_network_receive_packets_total{namespace=~\"$namespace\", pod=~\"$pod\"}[$interval])) by (pod)", + "expr": "sum(irate(container_network_receive_packets_total{namespace=~\"$namespace\", pod=~\"$pod\"}[$__interval])) by (pod)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{pod}}", @@ -1336,7 +1336,7 @@ data: "steppedLine": false, "targets": [ { - "expr": "sum(irate(container_network_transmit_packets_total{namespace=~\"$namespace\", pod=~\"$pod\"}[$interval])) by (pod)", + "expr": "sum(irate(container_network_transmit_packets_total{namespace=~\"$namespace\", pod=~\"$pod\"}[$__interval])) by (pod)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{pod}}", @@ -1434,7 +1434,7 @@ data: "steppedLine": false, "targets": [ { - "expr": "sum(irate(container_network_receive_packets_dropped_total{namespace=~\"$namespace\", pod=~\"$pod\"}[$interval])) by (pod)", + "expr": "sum(irate(container_network_receive_packets_dropped_total{namespace=~\"$namespace\", pod=~\"$pod\"}[$__interval])) by (pod)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{pod}}", @@ -1532,7 +1532,7 @@ data: "steppedLine": false, "targets": [ { - "expr": "sum(irate(container_network_transmit_packets_dropped_total{namespace=~\"$namespace\", pod=~\"$pod\"}[$interval])) by (pod)", + "expr": "sum(irate(container_network_transmit_packets_dropped_total{namespace=~\"$namespace\", pod=~\"$pod\"}[$__interval])) by (pod)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{pod}}", @@ -1612,41 +1612,6 @@ data: "regex": "", "type": "datasource" }, - { - "allValue": null, - "auto": false, - "auto_count": 30, - "auto_min": "10s", - "current": { - "text": "5m", - "value": "5m" - }, - "datasource": "$datasource", - "hide": 2, - "includeAll": false, - "label": null, - "multi": false, - "name": "interval", - "options": [ - { - "selected": true, - "text": "4h", - "value": "4h" - } - ], - "query": "4h", - "refresh": 2, - "regex": "", - "skipUrlSync": false, - "sort": 1, - "tagValuesQuery": "", - "tags": [ - - ], - "tagsQuery": "", - "type": "interval", - "useTags": false - }, { "allValue": null, "current": { @@ -2701,7 +2666,7 @@ data: ], "targets": [ { - "expr": "(sum(irate(container_network_receive_bytes_total{cluster=\"$cluster\", namespace=~\"$namespace\"}[$interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=~\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "expr": "(sum(irate(container_network_receive_bytes_total{cluster=\"$cluster\", namespace=~\"$namespace\"}[$__interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=~\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", "format": "table", "instant": true, "intervalFactor": 2, @@ -2710,7 +2675,7 @@ data: "step": 10 }, { - "expr": "(sum(irate(container_network_transmit_bytes_total{cluster=\"$cluster\", namespace=~\"$namespace\"}[$interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=~\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "expr": "(sum(irate(container_network_transmit_bytes_total{cluster=\"$cluster\", namespace=~\"$namespace\"}[$__interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=~\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", "format": "table", "instant": true, "intervalFactor": 2, @@ -2719,7 +2684,7 @@ data: "step": 10 }, { - "expr": "(sum(irate(container_network_receive_packets_total{cluster=\"$cluster\", namespace=~\"$namespace\"}[$interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=~\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "expr": "(sum(irate(container_network_receive_packets_total{cluster=\"$cluster\", namespace=~\"$namespace\"}[$__interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=~\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", "format": "table", "instant": true, "intervalFactor": 2, @@ -2728,7 +2693,7 @@ data: "step": 10 }, { - "expr": "(sum(irate(container_network_transmit_packets_total{cluster=\"$cluster\", namespace=~\"$namespace\"}[$interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=~\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "expr": "(sum(irate(container_network_transmit_packets_total{cluster=\"$cluster\", namespace=~\"$namespace\"}[$__interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=~\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", "format": "table", "instant": true, "intervalFactor": 2, @@ -2737,7 +2702,7 @@ data: "step": 10 }, { - "expr": "(sum(irate(container_network_receive_packets_dropped_total{cluster=\"$cluster\", namespace=~\"$namespace\"}[$interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=~\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "expr": "(sum(irate(container_network_receive_packets_dropped_total{cluster=\"$cluster\", namespace=~\"$namespace\"}[$__interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=~\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", "format": "table", "instant": true, "intervalFactor": 2, @@ -2746,7 +2711,7 @@ data: "step": 10 }, { - "expr": "(sum(irate(container_network_transmit_packets_dropped_total{cluster=\"$cluster\", namespace=~\"$namespace\"}[$interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=~\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "expr": "(sum(irate(container_network_transmit_packets_dropped_total{cluster=\"$cluster\", namespace=~\"$namespace\"}[$__interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=~\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", "format": "table", "instant": true, "intervalFactor": 2, @@ -2846,7 +2811,7 @@ data: "steppedLine": false, "targets": [ { - "expr": "(sum(irate(container_network_receive_bytes_total{cluster=\"$cluster\", namespace=~\"$namespace\"}[$interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=~\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "expr": "(sum(irate(container_network_receive_bytes_total{cluster=\"$cluster\", namespace=~\"$namespace\"}[$__interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=~\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{pod}}", @@ -2944,7 +2909,7 @@ data: "steppedLine": false, "targets": [ { - "expr": "(sum(irate(container_network_transmit_bytes_total{cluster=\"$cluster\", namespace=~\"$namespace\"}[$interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=~\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "expr": "(sum(irate(container_network_transmit_bytes_total{cluster=\"$cluster\", namespace=~\"$namespace\"}[$__interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=~\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{pod}}", @@ -3042,7 +3007,7 @@ data: "steppedLine": false, "targets": [ { - "expr": "(avg(irate(container_network_receive_bytes_total{cluster=\"$cluster\", namespace=~\"$namespace\"}[$interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=~\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "expr": "(avg(irate(container_network_receive_bytes_total{cluster=\"$cluster\", namespace=~\"$namespace\"}[$__interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=~\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{pod}}", @@ -3140,7 +3105,7 @@ data: "steppedLine": false, "targets": [ { - "expr": "(avg(irate(container_network_transmit_bytes_total{cluster=\"$cluster\", namespace=~\"$namespace\"}[$interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=~\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "expr": "(avg(irate(container_network_transmit_bytes_total{cluster=\"$cluster\", namespace=~\"$namespace\"}[$__interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=~\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{pod}}", @@ -3238,7 +3203,7 @@ data: "steppedLine": false, "targets": [ { - "expr": "(sum(irate(container_network_receive_packets_total{cluster=\"$cluster\", namespace=~\"$namespace\"}[$interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=~\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "expr": "(sum(irate(container_network_receive_packets_total{cluster=\"$cluster\", namespace=~\"$namespace\"}[$__interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=~\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{pod}}", @@ -3336,7 +3301,7 @@ data: "steppedLine": false, "targets": [ { - "expr": "(sum(irate(container_network_transmit_packets_total{cluster=\"$cluster\", namespace=~\"$namespace\"}[$interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=~\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "expr": "(sum(irate(container_network_transmit_packets_total{cluster=\"$cluster\", namespace=~\"$namespace\"}[$__interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=~\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{pod}}", @@ -3434,7 +3399,7 @@ data: "steppedLine": false, "targets": [ { - "expr": "(sum(irate(container_network_receive_packets_dropped_total{cluster=\"$cluster\", namespace=~\"$namespace\"}[$interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=~\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "expr": "(sum(irate(container_network_receive_packets_dropped_total{cluster=\"$cluster\", namespace=~\"$namespace\"}[$__interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=~\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{pod}}", @@ -3532,7 +3497,7 @@ data: "steppedLine": false, "targets": [ { - "expr": "(sum(irate(container_network_transmit_packets_dropped_total{cluster=\"$cluster\", namespace=~\"$namespace\"}[$interval])\n* on (namespace,pod) \ngroup_left(workload,workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=~\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "expr": "(sum(irate(container_network_transmit_packets_dropped_total{cluster=\"$cluster\", namespace=~\"$namespace\"}[$__interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=~\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{pod}}", @@ -3612,41 +3577,6 @@ data: "regex": "", "type": "datasource" }, - { - "allValue": null, - "auto": false, - "auto_count": 30, - "auto_min": "10s", - "current": { - "text": "5m", - "value": "5m" - }, - "datasource": "$datasource", - "hide": 2, - "includeAll": false, - "label": null, - "multi": false, - "name": "interval", - "options": [ - { - "selected": true, - "text": "4h", - "value": "4h" - } - ], - "query": "4h", - "refresh": 2, - "regex": "", - "skipUrlSync": false, - "sort": 1, - "tagValuesQuery": "", - "tags": [ - - ], - "tagsQuery": "", - "type": "interval", - "useTags": false - }, { "allValue": null, "current": { @@ -3708,7 +3638,7 @@ data: "value": "" }, "datasource": "$datasource", - "hide": 2, + "hide": 0, "includeAll": false, "label": null, "multi": false, @@ -4906,7 +4836,7 @@ data: ], "targets": [ { - "expr": "(sum(irate(container_network_receive_bytes_total{cluster=\"$cluster\", namespace=~\"$namespace\"}[$interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=~\"$namespace\", workload_type=\"$type\"}) by (workload))\n", + "expr": "(sum(irate(container_network_receive_bytes_total{cluster=\"$cluster\", namespace=~\"$namespace\"}[$__interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=~\"$namespace\", workload_type=\"$type\"}) by (workload))\n", "format": "table", "instant": true, "intervalFactor": 2, @@ -4915,7 +4845,7 @@ data: "step": 10 }, { - "expr": "(sum(irate(container_network_transmit_bytes_total{cluster=\"$cluster\", namespace=~\"$namespace\"}[$interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=~\"$namespace\", workload_type=\"$type\"}) by (workload))\n", + "expr": "(sum(irate(container_network_transmit_bytes_total{cluster=\"$cluster\", namespace=~\"$namespace\"}[$__interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=~\"$namespace\", workload_type=\"$type\"}) by (workload))\n", "format": "table", "instant": true, "intervalFactor": 2, @@ -4924,7 +4854,7 @@ data: "step": 10 }, { - "expr": "(sum(irate(container_network_receive_packets_total{cluster=\"$cluster\", namespace=~\"$namespace\"}[$interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=~\"$namespace\", workload_type=\"$type\"}) by (workload))\n", + "expr": "(sum(irate(container_network_receive_packets_total{cluster=\"$cluster\", namespace=~\"$namespace\"}[$__interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=~\"$namespace\", workload_type=\"$type\"}) by (workload))\n", "format": "table", "instant": true, "intervalFactor": 2, @@ -4933,7 +4863,7 @@ data: "step": 10 }, { - "expr": "(sum(irate(container_network_transmit_packets_total{cluster=\"$cluster\", namespace=~\"$namespace\"}[$interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=~\"$namespace\", workload_type=\"$type\"}) by (workload))\n", + "expr": "(sum(irate(container_network_transmit_packets_total{cluster=\"$cluster\", namespace=~\"$namespace\"}[$__interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=~\"$namespace\", workload_type=\"$type\"}) by (workload))\n", "format": "table", "instant": true, "intervalFactor": 2, @@ -4942,7 +4872,7 @@ data: "step": 10 }, { - "expr": "(sum(irate(container_network_receive_packets_dropped_total{cluster=\"$cluster\", namespace=~\"$namespace\"}[$interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=~\"$namespace\", workload_type=\"$type\"}) by (workload))\n", + "expr": "(sum(irate(container_network_receive_packets_dropped_total{cluster=\"$cluster\", namespace=~\"$namespace\"}[$__interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=~\"$namespace\", workload_type=\"$type\"}) by (workload))\n", "format": "table", "instant": true, "intervalFactor": 2, @@ -4951,7 +4881,7 @@ data: "step": 10 }, { - "expr": "(sum(irate(container_network_transmit_packets_dropped_total{cluster=\"$cluster\", namespace=~\"$namespace\"}[$interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=~\"$namespace\", workload_type=\"$type\"}) by (workload))\n", + "expr": "(sum(irate(container_network_transmit_packets_dropped_total{cluster=\"$cluster\", namespace=~\"$namespace\"}[$__interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=~\"$namespace\", workload_type=\"$type\"}) by (workload))\n", "format": "table", "instant": true, "intervalFactor": 2, @@ -5051,7 +4981,7 @@ data: "steppedLine": false, "targets": [ { - "expr": "(sum(irate(container_network_receive_bytes_total{cluster=\"$cluster\", namespace=~\"$namespace\"}[$interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=~\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "expr": "(sum(irate(container_network_receive_bytes_total{cluster=\"$cluster\", namespace=~\"$namespace\"}[$__interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=~\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{workload}}", @@ -5149,7 +5079,7 @@ data: "steppedLine": false, "targets": [ { - "expr": "(sum(irate(container_network_transmit_bytes_total{cluster=\"$cluster\", namespace=~\"$namespace\"}[$interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=~\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "expr": "(sum(irate(container_network_transmit_bytes_total{cluster=\"$cluster\", namespace=~\"$namespace\"}[$__interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=~\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{workload}}", @@ -5247,7 +5177,7 @@ data: "steppedLine": false, "targets": [ { - "expr": "(avg(irate(container_network_receive_bytes_total{cluster=\"$cluster\", namespace=~\"$namespace\"}[$interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=~\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "expr": "(avg(irate(container_network_receive_bytes_total{cluster=\"$cluster\", namespace=~\"$namespace\"}[$__interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=~\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{workload}}", @@ -5345,7 +5275,7 @@ data: "steppedLine": false, "targets": [ { - "expr": "(avg(irate(container_network_transmit_bytes_total{cluster=\"$cluster\", namespace=~\"$namespace\"}[$interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=~\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "expr": "(avg(irate(container_network_transmit_bytes_total{cluster=\"$cluster\", namespace=~\"$namespace\"}[$__interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=~\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{workload}}", @@ -5443,7 +5373,7 @@ data: "steppedLine": false, "targets": [ { - "expr": "(sum(irate(container_network_receive_packets_total{cluster=\"$cluster\", namespace=~\"$namespace\"}[$interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=~\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "expr": "(sum(irate(container_network_receive_packets_total{cluster=\"$cluster\", namespace=~\"$namespace\"}[$__interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=~\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{workload}}", @@ -5541,7 +5471,7 @@ data: "steppedLine": false, "targets": [ { - "expr": "(sum(irate(container_network_transmit_packets_total{cluster=\"$cluster\", namespace=~\"$namespace\"}[$interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=~\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "expr": "(sum(irate(container_network_transmit_packets_total{cluster=\"$cluster\", namespace=~\"$namespace\"}[$__interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=~\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{workload}}", @@ -5639,7 +5569,7 @@ data: "steppedLine": false, "targets": [ { - "expr": "(sum(irate(container_network_receive_packets_dropped_total{cluster=\"$cluster\", namespace=~\"$namespace\"}[$interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=~\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "expr": "(sum(irate(container_network_receive_packets_dropped_total{cluster=\"$cluster\", namespace=~\"$namespace\"}[$__interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=~\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{workload}}", @@ -5737,7 +5667,7 @@ data: "steppedLine": false, "targets": [ { - "expr": "(sum(irate(container_network_transmit_packets_dropped_total{cluster=\"$cluster\", namespace=~\"$namespace\"}[$interval])\n* on (namespace,pod) \ngroup_left(workload,workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=~\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "expr": "(sum(irate(container_network_transmit_packets_dropped_total{cluster=\"$cluster\", namespace=~\"$namespace\"}[$__interval])\n* on (namespace,pod) \ngroup_left(workload,workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=~\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{workload}}", @@ -5817,41 +5747,6 @@ data: "regex": "", "type": "datasource" }, - { - "allValue": null, - "auto": false, - "auto_count": 30, - "auto_min": "10s", - "current": { - "text": "5m", - "value": "5m" - }, - "datasource": "$datasource", - "hide": 2, - "includeAll": false, - "label": null, - "multi": false, - "name": "interval", - "options": [ - { - "selected": true, - "text": "4h", - "value": "4h" - } - ], - "query": "4h", - "refresh": 2, - "regex": "", - "skipUrlSync": false, - "sort": 1, - "tagValuesQuery": "", - "tags": [ - - ], - "tagsQuery": "", - "type": "interval", - "useTags": false - }, { "allValue": null, "auto": false, diff --git a/addons/prometheus/rules.yaml b/addons/prometheus/rules.yaml index e1ffbc6f0..69026c1ef 100644 --- a/addons/prometheus/rules.yaml +++ b/addons/prometheus/rules.yaml @@ -252,25 +252,25 @@ data: "name": "kube-apiserver.rules", "rules": [ { - "expr": "sum(rate(apiserver_request_duration_seconds_sum{subresource!=\"log\",verb!~\"LIST|WATCH|WATCHLIST|PROXY|CONNECT\"}[5m])) without(instance, pod)\n/\nsum(rate(apiserver_request_duration_seconds_count{subresource!=\"log\",verb!~\"LIST|WATCH|WATCHLIST|PROXY|CONNECT\"}[5m])) without(instance, pod)\n", + "expr": "sum(rate(apiserver_request_duration_seconds_sum{subresource!=\"log\",verb!~\"LIST|WATCH|WATCHLIST|DELETECOLLECTION|PROXY|CONNECT\"}[5m])) without(instance, pod)\n/\nsum(rate(apiserver_request_duration_seconds_count{subresource!=\"log\",verb!~\"LIST|WATCH|WATCHLIST|DELETECOLLECTION|PROXY|CONNECT\"}[5m])) without(instance, pod)\n", "record": "cluster:apiserver_request_duration_seconds:mean5m" }, { - "expr": "histogram_quantile(0.99, sum(rate(apiserver_request_duration_seconds_bucket{job=\"apiserver\",subresource!=\"log\",verb!~\"LIST|WATCH|WATCHLIST|PROXY|CONNECT\"}[5m])) without(instance, pod))\n", + "expr": "histogram_quantile(0.99, sum(rate(apiserver_request_duration_seconds_bucket{job=\"apiserver\",subresource!=\"log\",verb!~\"LIST|WATCH|WATCHLIST|DELETECOLLECTION|PROXY|CONNECT\"}[5m])) without(instance, pod))\n", "labels": { "quantile": "0.99" }, "record": "cluster_quantile:apiserver_request_duration_seconds:histogram_quantile" }, { - "expr": "histogram_quantile(0.9, sum(rate(apiserver_request_duration_seconds_bucket{job=\"apiserver\",subresource!=\"log\",verb!~\"LIST|WATCH|WATCHLIST|PROXY|CONNECT\"}[5m])) without(instance, pod))\n", + "expr": "histogram_quantile(0.9, sum(rate(apiserver_request_duration_seconds_bucket{job=\"apiserver\",subresource!=\"log\",verb!~\"LIST|WATCH|WATCHLIST|DELETECOLLECTION|PROXY|CONNECT\"}[5m])) without(instance, pod))\n", "labels": { "quantile": "0.9" }, "record": "cluster_quantile:apiserver_request_duration_seconds:histogram_quantile" }, { - "expr": "histogram_quantile(0.5, sum(rate(apiserver_request_duration_seconds_bucket{job=\"apiserver\",subresource!=\"log\",verb!~\"LIST|WATCH|WATCHLIST|PROXY|CONNECT\"}[5m])) without(instance, pod))\n", + "expr": "histogram_quantile(0.5, sum(rate(apiserver_request_duration_seconds_bucket{job=\"apiserver\",subresource!=\"log\",verb!~\"LIST|WATCH|WATCHLIST|DELETECOLLECTION|PROXY|CONNECT\"}[5m])) without(instance, pod))\n", "labels": { "quantile": "0.5" }, @@ -805,6 +805,7 @@ data: { "alert": "ErrorBudgetBurn", "annotations": { + "message": "High requests error budget burn for job=apiserver (current value: {{ $value }})", "runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-errorbudgetburn" }, "expr": "(\n status_class_5xx:apiserver_request_total:ratio_rate1h{job=\"apiserver\"} > (14.4*0.010000)\n and\n status_class_5xx:apiserver_request_total:ratio_rate5m{job=\"apiserver\"} > (14.4*0.010000)\n)\nor\n(\n status_class_5xx:apiserver_request_total:ratio_rate6h{job=\"apiserver\"} > (6*0.010000)\n and\n status_class_5xx:apiserver_request_total:ratio_rate30m{job=\"apiserver\"} > (6*0.010000)\n)\n", @@ -816,6 +817,7 @@ data: { "alert": "ErrorBudgetBurn", "annotations": { + "message": "High requests error budget burn for job=apiserver (current value: {{ $value }})", "runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-errorbudgetburn" }, "expr": "(\n status_class_5xx:apiserver_request_total:ratio_rate1d{job=\"apiserver\"} > (3*0.010000)\n and\n status_class_5xx:apiserver_request_total:ratio_rate2h{job=\"apiserver\"} > (3*0.010000)\n)\nor\n(\n status_class_5xx:apiserver_request_total:ratio_rate3d{job=\"apiserver\"} > (0.010000)\n and\n status_class_5xx:apiserver_request_total:ratio_rate6h{job=\"apiserver\"} > (0.010000)\n)\n", @@ -853,30 +855,6 @@ data: "severity": "critical" } }, - { - "alert": "KubeAPIErrorsHigh", - "annotations": { - "message": "API server is returning errors for {{ $value | humanizePercentage }} of requests.", - "runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubeapierrorshigh" - }, - "expr": "sum(rate(apiserver_request_total{job=\"apiserver\",code=~\"5..\"}[5m]))\n /\nsum(rate(apiserver_request_total{job=\"apiserver\"}[5m])) > 0.03\n", - "for": "10m", - "labels": { - "severity": "critical" - } - }, - { - "alert": "KubeAPIErrorsHigh", - "annotations": { - "message": "API server is returning errors for {{ $value | humanizePercentage }} of requests.", - "runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubeapierrorshigh" - }, - "expr": "sum(rate(apiserver_request_total{job=\"apiserver\",code=~\"5..\"}[5m]))\n /\nsum(rate(apiserver_request_total{job=\"apiserver\"}[5m])) > 0.01\n", - "for": "10m", - "labels": { - "severity": "warning" - } - }, { "alert": "KubeAPIErrorsHigh", "annotations": { @@ -993,7 +971,7 @@ data: "message": "Kubelet '{{ $labels.node }}' is running at {{ $value | humanizePercentage }} of its Pod capacity.", "runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubelettoomanypods" }, - "expr": "max(max(kubelet_running_pod_count{job=\"kubelet\"}) by(instance) * on(instance) group_left(node) kubelet_node_name{job=\"kubelet\"}) by(node) / max(kube_node_status_capacity_pods{job=\"kube-state-metrics\"}) by(node) > 0.95\n", + "expr": "max(max(kubelet_running_pod_count{job=\"kubelet\"}) by(instance) * on(instance) group_left(node) kubelet_node_name{job=\"kubelet\"}) by(node) / max(kube_node_status_capacity_pods{job=\"kube-state-metrics\"} != 1) by(node) > 0.95\n", "for": "15m", "labels": { "severity": "warning" @@ -1029,7 +1007,7 @@ data: "message": "Kubelet Pod startup 99th percentile latency is {{ $value }} seconds on node {{ $labels.node }}.", "runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubeletpodstartuplatencyhigh" }, - "expr": "histogram_quantile(0.99, sum(rate(kubelet_pod_worker_duration_seconds_bucket{job=\"kubelet\"}[5m])) by (instance, le)) * on(instance) group_left(node) kubelet_node_name > 5\n", + "expr": "histogram_quantile(0.99, sum(rate(kubelet_pod_worker_duration_seconds_bucket{job=\"kubelet\"}[5m])) by (instance, le)) * on(instance) group_left(node) kubelet_node_name > 60\n", "for": "15m", "labels": { "severity": "warning" @@ -1085,9 +1063,167 @@ data: } ] } + loki.yaml: |- + { + "groups": [ + { + "name": "loki_rules", + "rules": [ + { + "expr": "histogram_quantile(0.99, sum(rate(loki_request_duration_seconds_bucket[1m])) by (le, job))", + "record": "job:loki_request_duration_seconds:99quantile" + }, + { + "expr": "histogram_quantile(0.50, sum(rate(loki_request_duration_seconds_bucket[1m])) by (le, job))", + "record": "job:loki_request_duration_seconds:50quantile" + }, + { + "expr": "sum(rate(loki_request_duration_seconds_sum[1m])) by (job) / sum(rate(loki_request_duration_seconds_count[1m])) by (job)", + "record": "job:loki_request_duration_seconds:avg" + }, + { + "expr": "sum(rate(loki_request_duration_seconds_bucket[1m])) by (le, job)", + "record": "job:loki_request_duration_seconds_bucket:sum_rate" + }, + { + "expr": "sum(rate(loki_request_duration_seconds_sum[1m])) by (job)", + "record": "job:loki_request_duration_seconds_sum:sum_rate" + }, + { + "expr": "sum(rate(loki_request_duration_seconds_count[1m])) by (job)", + "record": "job:loki_request_duration_seconds_count:sum_rate" + }, + { + "expr": "histogram_quantile(0.99, sum(rate(loki_request_duration_seconds_bucket[1m])) by (le, job, route))", + "record": "job_route:loki_request_duration_seconds:99quantile" + }, + { + "expr": "histogram_quantile(0.50, sum(rate(loki_request_duration_seconds_bucket[1m])) by (le, job, route))", + "record": "job_route:loki_request_duration_seconds:50quantile" + }, + { + "expr": "sum(rate(loki_request_duration_seconds_sum[1m])) by (job, route) / sum(rate(loki_request_duration_seconds_count[1m])) by (job, route)", + "record": "job_route:loki_request_duration_seconds:avg" + }, + { + "expr": "sum(rate(loki_request_duration_seconds_bucket[1m])) by (le, job, route)", + "record": "job_route:loki_request_duration_seconds_bucket:sum_rate" + }, + { + "expr": "sum(rate(loki_request_duration_seconds_sum[1m])) by (job, route)", + "record": "job_route:loki_request_duration_seconds_sum:sum_rate" + }, + { + "expr": "sum(rate(loki_request_duration_seconds_count[1m])) by (job, route)", + "record": "job_route:loki_request_duration_seconds_count:sum_rate" + }, + { + "expr": "histogram_quantile(0.99, sum(rate(loki_request_duration_seconds_bucket[1m])) by (le, namespace, job, route))", + "record": "namespace_job_route:loki_request_duration_seconds:99quantile" + }, + { + "expr": "histogram_quantile(0.50, sum(rate(loki_request_duration_seconds_bucket[1m])) by (le, namespace, job, route))", + "record": "namespace_job_route:loki_request_duration_seconds:50quantile" + }, + { + "expr": "sum(rate(loki_request_duration_seconds_sum[1m])) by (namespace, job, route) / sum(rate(loki_request_duration_seconds_count[1m])) by (namespace, job, route)", + "record": "namespace_job_route:loki_request_duration_seconds:avg" + }, + { + "expr": "sum(rate(loki_request_duration_seconds_bucket[1m])) by (le, namespace, job, route)", + "record": "namespace_job_route:loki_request_duration_seconds_bucket:sum_rate" + }, + { + "expr": "sum(rate(loki_request_duration_seconds_sum[1m])) by (namespace, job, route)", + "record": "namespace_job_route:loki_request_duration_seconds_sum:sum_rate" + }, + { + "expr": "sum(rate(loki_request_duration_seconds_count[1m])) by (namespace, job, route)", + "record": "namespace_job_route:loki_request_duration_seconds_count:sum_rate" + } + ] + }, + { + "name": "loki_alerts", + "rules": [ + { + "alert": "LokiRequestErrors", + "annotations": { + "message": "{{ $labels.job }} {{ $labels.route }} is experiencing {{ printf \"%.2f\" $value }}% errors.\n" + }, + "expr": "100 * sum(rate(loki_request_duration_seconds_count{status_code=~\"5..\"}[1m])) by (namespace, job, route)\n /\nsum(rate(loki_request_duration_seconds_count[1m])) by (namespace, job, route)\n > 10\n", + "for": "15m", + "labels": { + "severity": "critical" + } + }, + { + "alert": "LokiRequestLatency", + "annotations": { + "message": "{{ $labels.job }} {{ $labels.route }} is experiencing {{ printf \"%.2f\" $value }}s 99th percentile latency.\n" + }, + "expr": "namespace_job_route:loki_request_duration_seconds:99quantile{route!~\"(?i).*tail.*\"} > 1\n", + "for": "15m", + "labels": { + "severity": "critical" + } + } + ] + } + ] + } node-exporter.yaml: |- { "groups": [ + { + "name": "node-exporter.rules", + "rules": [ + { + "expr": "count without (cpu) (\n count without (mode) (\n node_cpu_seconds_total{job=\"node-exporter\"}\n )\n)\n", + "record": "instance:node_num_cpu:sum" + }, + { + "expr": "1 - avg without (cpu, mode) (\n rate(node_cpu_seconds_total{job=\"node-exporter\", mode=\"idle\"}[1m])\n)\n", + "record": "instance:node_cpu_utilisation:rate1m" + }, + { + "expr": "(\n node_load1{job=\"node-exporter\"}\n/\n instance:node_num_cpu:sum{job=\"node-exporter\"}\n)\n", + "record": "instance:node_load1_per_cpu:ratio" + }, + { + "expr": "1 - (\n node_memory_MemAvailable_bytes{job=\"node-exporter\"}\n/\n node_memory_MemTotal_bytes{job=\"node-exporter\"}\n)\n", + "record": "instance:node_memory_utilisation:ratio" + }, + { + "expr": "rate(node_vmstat_pgmajfault{job=\"node-exporter\"}[1m])\n", + "record": "instance:node_vmstat_pgmajfault:rate1m" + }, + { + "expr": "rate(node_disk_io_time_seconds_total{job=\"node-exporter\", device!~\"dm.*\"}[1m])\n", + "record": "instance_device:node_disk_io_time_seconds:rate1m" + }, + { + "expr": "rate(node_disk_io_time_weighted_seconds_total{job=\"node-exporter\", device!~\"dm.*\"}[1m])\n", + "record": "instance_device:node_disk_io_time_weighted_seconds:rate1m" + }, + { + "expr": "sum without (device) (\n rate(node_network_receive_bytes_total{job=\"node-exporter\", device!=\"lo\"}[1m])\n)\n", + "record": "instance:node_network_receive_bytes_excluding_lo:rate1m" + }, + { + "expr": "sum without (device) (\n rate(node_network_transmit_bytes_total{job=\"node-exporter\", device!=\"lo\"}[1m])\n)\n", + "record": "instance:node_network_transmit_bytes_excluding_lo:rate1m" + }, + { + "expr": "sum without (device) (\n rate(node_network_receive_drop_total{job=\"node-exporter\", device!=\"lo\"}[1m])\n)\n", + "record": "instance:node_network_receive_drop_excluding_lo:rate1m" + }, + { + "expr": "sum without (device) (\n rate(node_network_transmit_drop_total{job=\"node-exporter\", device!=\"lo\"}[1m])\n)\n", + "record": "instance:node_network_transmit_drop_excluding_lo:rate1m" + } + ] + }, { "name": "node-exporter", "rules": [ @@ -1210,6 +1346,41 @@ data: "labels": { "severity": "warning" } + }, + { + "alert": "NodeHighNumberConntrackEntriesUsed", + "annotations": { + "description": "{{ $value | humanizePercentage }} of conntrack entries are used", + "summary": "Number of conntrack are getting close to the limit" + }, + "expr": "(node_nf_conntrack_entries / node_nf_conntrack_entries_limit) > 0.75\n", + "labels": { + "severity": "warning" + } + }, + { + "alert": "NodeClockSkewDetected", + "annotations": { + "message": "Clock on {{ $labels.instance }} is out of sync by more than 300s. Ensure NTP is configured correctly on this host.", + "summary": "Clock skew detected." + }, + "expr": "(\n node_timex_offset_seconds > 0.05\nand\n deriv(node_timex_offset_seconds[5m]) >= 0\n)\nor\n(\n node_timex_offset_seconds < -0.05\nand\n deriv(node_timex_offset_seconds[5m]) <= 0\n)\n", + "for": "10m", + "labels": { + "severity": "warning" + } + }, + { + "alert": "NodeClockNotSynchronising", + "annotations": { + "message": "Clock on {{ $labels.instance }} is not synchronising. Ensure NTP is configured on this host.", + "summary": "Clock not synchronising." + }, + "expr": "min_over_time(node_timex_sync_status[5m]) == 0\n", + "for": "10m", + "labels": { + "severity": "warning" + } } ] } From 2b5dfece933d448fa0d834ff547024d6076d339e Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Sat, 4 Apr 2020 13:13:19 -0700 Subject: [PATCH 384/523] Update Grafana from v6.7.1 to v6.7.2 * https://github.com/grafana/grafana/releases/tag/v6.7.2 --- CHANGES.md | 3 ++- addons/grafana/deployment.yaml | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index a96e2dede..9ae9c6f5b 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -17,7 +17,8 @@ Notable changes between versions. #### Addons -* Refresh Prometheus rules and alerts +* Refresh Prometheus rules/alerts and Grafana dashboards ([#692](https://github.com/poseidon/typhoon/pull/692)) +* Update Grafana from v6.7.1 to v6.7.2 ## v1.18.0 diff --git a/addons/grafana/deployment.yaml b/addons/grafana/deployment.yaml index 369bc4941..57a0a2476 100644 --- a/addons/grafana/deployment.yaml +++ b/addons/grafana/deployment.yaml @@ -23,7 +23,7 @@ spec: spec: containers: - name: grafana - image: docker.io/grafana/grafana:6.7.1 + image: docker.io/grafana/grafana:6.7.2 env: - name: GF_PATHS_CONFIG value: "/etc/grafana/custom.ini" From 17ea5477231e48f8683bc7024736d45f51d7ec2b Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Mon, 6 Apr 2020 21:09:25 -0700 Subject: [PATCH 385/523] Update etcd from v3.4.5 to v3.4.7 * https://github.com/etcd-io/etcd/releases/tag/v3.4.7 * https://github.com/etcd-io/etcd/releases/tag/v3.4.6 --- CHANGES.md | 1 + aws/container-linux/kubernetes/cl/controller.yaml | 2 +- aws/fedora-coreos/kubernetes/fcc/controller.yaml | 2 +- azure/container-linux/kubernetes/cl/controller.yaml | 2 +- bare-metal/container-linux/kubernetes/cl/controller.yaml | 2 +- bare-metal/fedora-coreos/kubernetes/fcc/controller.yaml | 2 +- digital-ocean/container-linux/kubernetes/cl/controller.yaml | 2 +- google-cloud/container-linux/kubernetes/cl/controller.yaml | 2 +- google-cloud/fedora-coreos/kubernetes/fcc/controller.yaml | 2 +- 9 files changed, 9 insertions(+), 8 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 9ae9c6f5b..6ba8e1122 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -4,6 +4,7 @@ Notable changes between versions. ## Latest +* Update etcd from v3.4.5 to [v3.4.7](https://github.com/etcd-io/etcd/releases/tag/v3.4.7) * Change `kube-proxy` and `calico` or `flannel` to tolerate specific taints ([#682](https://github.com/poseidon/typhoon/pull/682)) * Tolerate master and not-ready taints, rather than tolerating all taints * Update flannel from v0.11.0 to v0.12.0 ([#690](https://github.com/poseidon/typhoon/pull/690)) diff --git a/aws/container-linux/kubernetes/cl/controller.yaml b/aws/container-linux/kubernetes/cl/controller.yaml index 68791791a..f2295528c 100644 --- a/aws/container-linux/kubernetes/cl/controller.yaml +++ b/aws/container-linux/kubernetes/cl/controller.yaml @@ -7,7 +7,7 @@ systemd: - name: 40-etcd-cluster.conf contents: | [Service] - Environment="ETCD_IMAGE_TAG=v3.4.5" + Environment="ETCD_IMAGE_TAG=v3.4.7" Environment="ETCD_IMAGE_URL=docker://quay.io/coreos/etcd" Environment="RKT_RUN_ARGS=--insecure-options=image" Environment="ETCD_NAME=${etcd_name}" diff --git a/aws/fedora-coreos/kubernetes/fcc/controller.yaml b/aws/fedora-coreos/kubernetes/fcc/controller.yaml index 439c43015..d0c404ff2 100644 --- a/aws/fedora-coreos/kubernetes/fcc/controller.yaml +++ b/aws/fedora-coreos/kubernetes/fcc/controller.yaml @@ -28,7 +28,7 @@ systemd: --network host \ --volume /var/lib/etcd:/var/lib/etcd:rw,Z \ --volume /etc/ssl/etcd:/etc/ssl/certs:ro,Z \ - quay.io/coreos/etcd:v3.4.5 + quay.io/coreos/etcd:v3.4.7 ExecStop=/usr/bin/podman stop etcd [Install] WantedBy=multi-user.target diff --git a/azure/container-linux/kubernetes/cl/controller.yaml b/azure/container-linux/kubernetes/cl/controller.yaml index 91b37e618..979e858cc 100644 --- a/azure/container-linux/kubernetes/cl/controller.yaml +++ b/azure/container-linux/kubernetes/cl/controller.yaml @@ -7,7 +7,7 @@ systemd: - name: 40-etcd-cluster.conf contents: | [Service] - Environment="ETCD_IMAGE_TAG=v3.4.5" + Environment="ETCD_IMAGE_TAG=v3.4.7" Environment="ETCD_IMAGE_URL=docker://quay.io/coreos/etcd" Environment="RKT_RUN_ARGS=--insecure-options=image" Environment="ETCD_NAME=${etcd_name}" diff --git a/bare-metal/container-linux/kubernetes/cl/controller.yaml b/bare-metal/container-linux/kubernetes/cl/controller.yaml index 532641799..2aaaac5be 100644 --- a/bare-metal/container-linux/kubernetes/cl/controller.yaml +++ b/bare-metal/container-linux/kubernetes/cl/controller.yaml @@ -7,7 +7,7 @@ systemd: - name: 40-etcd-cluster.conf contents: | [Service] - Environment="ETCD_IMAGE_TAG=v3.4.5" + Environment="ETCD_IMAGE_TAG=v3.4.7" Environment="ETCD_IMAGE_URL=docker://quay.io/coreos/etcd" Environment="RKT_RUN_ARGS=--insecure-options=image" Environment="ETCD_NAME=${etcd_name}" diff --git a/bare-metal/fedora-coreos/kubernetes/fcc/controller.yaml b/bare-metal/fedora-coreos/kubernetes/fcc/controller.yaml index fe774161d..fcc8584d7 100644 --- a/bare-metal/fedora-coreos/kubernetes/fcc/controller.yaml +++ b/bare-metal/fedora-coreos/kubernetes/fcc/controller.yaml @@ -28,7 +28,7 @@ systemd: --network host \ --volume /var/lib/etcd:/var/lib/etcd:rw,Z \ --volume /etc/ssl/etcd:/etc/ssl/certs:ro,Z \ - quay.io/coreos/etcd:v3.4.5 + quay.io/coreos/etcd:v3.4.7 ExecStop=/usr/bin/podman stop etcd [Install] WantedBy=multi-user.target diff --git a/digital-ocean/container-linux/kubernetes/cl/controller.yaml b/digital-ocean/container-linux/kubernetes/cl/controller.yaml index 5419ab452..ce8d207d0 100644 --- a/digital-ocean/container-linux/kubernetes/cl/controller.yaml +++ b/digital-ocean/container-linux/kubernetes/cl/controller.yaml @@ -7,7 +7,7 @@ systemd: - name: 40-etcd-cluster.conf contents: | [Service] - Environment="ETCD_IMAGE_TAG=v3.4.5" + Environment="ETCD_IMAGE_TAG=v3.4.7" Environment="ETCD_IMAGE_URL=docker://quay.io/coreos/etcd" Environment="RKT_RUN_ARGS=--insecure-options=image" Environment="ETCD_NAME=${etcd_name}" diff --git a/google-cloud/container-linux/kubernetes/cl/controller.yaml b/google-cloud/container-linux/kubernetes/cl/controller.yaml index 9f2c9f617..d49f75d19 100644 --- a/google-cloud/container-linux/kubernetes/cl/controller.yaml +++ b/google-cloud/container-linux/kubernetes/cl/controller.yaml @@ -7,7 +7,7 @@ systemd: - name: 40-etcd-cluster.conf contents: | [Service] - Environment="ETCD_IMAGE_TAG=v3.4.5" + Environment="ETCD_IMAGE_TAG=v3.4.7" Environment="ETCD_IMAGE_URL=docker://quay.io/coreos/etcd" Environment="RKT_RUN_ARGS=--insecure-options=image" Environment="ETCD_NAME=${etcd_name}" diff --git a/google-cloud/fedora-coreos/kubernetes/fcc/controller.yaml b/google-cloud/fedora-coreos/kubernetes/fcc/controller.yaml index d254f9e1f..d441371c8 100644 --- a/google-cloud/fedora-coreos/kubernetes/fcc/controller.yaml +++ b/google-cloud/fedora-coreos/kubernetes/fcc/controller.yaml @@ -28,7 +28,7 @@ systemd: --network host \ --volume /var/lib/etcd:/var/lib/etcd:rw,Z \ --volume /etc/ssl/etcd:/etc/ssl/certs:ro,Z \ - quay.io/coreos/etcd:v3.4.5 + quay.io/coreos/etcd:v3.4.7 ExecStop=/usr/bin/podman stop etcd [Install] WantedBy=multi-user.target From 73af2f3b7c40bbf8bcf786c6eae2096096f36cd2 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Wed, 8 Apr 2020 19:38:46 -0700 Subject: [PATCH 386/523] Update Kubernetes from v1.18.0 to v1.18.1 * https://github.com/kubernetes/kubernetes/blob/master/CHANGELOG/CHANGELOG-1.18.md#v1181 --- CHANGES.md | 3 +++ README.md | 10 +++++----- aws/container-linux/kubernetes/README.md | 2 +- aws/container-linux/kubernetes/bootstrap.tf | 2 +- aws/container-linux/kubernetes/cl/controller.yaml | 4 ++-- .../kubernetes/workers/cl/worker.yaml | 4 ++-- aws/fedora-coreos/kubernetes/README.md | 2 +- aws/fedora-coreos/kubernetes/bootstrap.tf | 2 +- aws/fedora-coreos/kubernetes/fcc/controller.yaml | 4 ++-- .../kubernetes/workers/fcc/worker.yaml | 4 ++-- azure/container-linux/kubernetes/README.md | 2 +- azure/container-linux/kubernetes/bootstrap.tf | 2 +- .../container-linux/kubernetes/cl/controller.yaml | 4 ++-- .../kubernetes/workers/cl/worker.yaml | 4 ++-- bare-metal/container-linux/kubernetes/README.md | 2 +- bare-metal/container-linux/kubernetes/bootstrap.tf | 2 +- .../container-linux/kubernetes/cl/controller.yaml | 4 ++-- .../container-linux/kubernetes/cl/worker.yaml | 2 +- bare-metal/fedora-coreos/kubernetes/README.md | 2 +- bare-metal/fedora-coreos/kubernetes/bootstrap.tf | 2 +- .../fedora-coreos/kubernetes/fcc/controller.yaml | 4 ++-- .../fedora-coreos/kubernetes/fcc/worker.yaml | 2 +- digital-ocean/container-linux/kubernetes/README.md | 2 +- .../container-linux/kubernetes/bootstrap.tf | 2 +- .../container-linux/kubernetes/cl/controller.yaml | 4 ++-- .../container-linux/kubernetes/cl/worker.yaml | 4 ++-- docs/advanced/worker-pools.md | 14 +++++++------- docs/cl/aws.md | 10 +++++----- docs/cl/azure.md | 10 +++++----- docs/cl/bare-metal.md | 10 +++++----- docs/cl/digital-ocean.md | 10 +++++----- docs/cl/google-cloud.md | 10 +++++----- docs/fedora-coreos/aws.md | 10 +++++----- docs/fedora-coreos/bare-metal.md | 10 +++++----- docs/fedora-coreos/google-cloud.md | 8 ++++---- docs/index.md | 10 +++++----- docs/topics/maintenance.md | 8 ++++---- google-cloud/container-linux/kubernetes/README.md | 2 +- .../container-linux/kubernetes/bootstrap.tf | 2 +- .../container-linux/kubernetes/cl/controller.yaml | 4 ++-- .../kubernetes/workers/cl/worker.yaml | 4 ++-- google-cloud/fedora-coreos/kubernetes/README.md | 2 +- google-cloud/fedora-coreos/kubernetes/bootstrap.tf | 2 +- .../fedora-coreos/kubernetes/fcc/controller.yaml | 4 ++-- .../kubernetes/workers/fcc/worker.yaml | 4 ++-- 45 files changed, 109 insertions(+), 106 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 6ba8e1122..d11990ba1 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -4,6 +4,9 @@ Notable changes between versions. ## Latest +## v1.18.1 + +* Kubernetes [v1.18.1](https://github.com/kubernetes/kubernetes/blob/master/CHANGELOG/CHANGELOG-1.18.md#v1181) * Update etcd from v3.4.5 to [v3.4.7](https://github.com/etcd-io/etcd/releases/tag/v3.4.7) * Change `kube-proxy` and `calico` or `flannel` to tolerate specific taints ([#682](https://github.com/poseidon/typhoon/pull/682)) * Tolerate master and not-ready taints, rather than tolerating all taints diff --git a/README.md b/README.md index bc755bc74..bb053dda6 100644 --- a/README.md +++ b/README.md @@ -11,7 +11,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features -* Kubernetes v1.18.0 (upstream) +* Kubernetes v1.18.1 (upstream) * Single or multi-master, [Calico](https://www.projectcalico.org/) or [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/) * Advanced features like [worker pools](https://typhoon.psdn.io/advanced/worker-pools/), [preemptible](https://typhoon.psdn.io/cl/google-cloud/#preemption) workers, and [snippets](https://typhoon.psdn.io/advanced/customization/#container-linux) customization @@ -60,7 +60,7 @@ Define a Kubernetes cluster by using the Terraform module for your chosen platfo ```tf module "yavin" { - source = "git::https://github.com/poseidon/typhoon//google-cloud/container-linux/kubernetes?ref=v1.18.0" + source = "git::https://github.com/poseidon/typhoon//google-cloud/container-linux/kubernetes?ref=v1.18.1" # Google Cloud cluster_name = "yavin" @@ -99,9 +99,9 @@ In 4-8 minutes (varies by platform), the cluster will be ready. This Google Clou $ export KUBECONFIG=/home/user/.kube/configs/yavin-config $ kubectl get nodes NAME ROLES STATUS AGE VERSION -yavin-controller-0.c.example-com.internal Ready 6m v1.18.0 -yavin-worker-jrbf.c.example-com.internal Ready 5m v1.18.0 -yavin-worker-mzdm.c.example-com.internal Ready 5m v1.18.0 +yavin-controller-0.c.example-com.internal Ready 6m v1.18.1 +yavin-worker-jrbf.c.example-com.internal Ready 5m v1.18.1 +yavin-worker-mzdm.c.example-com.internal Ready 5m v1.18.1 ``` List the pods. diff --git a/aws/container-linux/kubernetes/README.md b/aws/container-linux/kubernetes/README.md index 74de8cebb..ad9dd1222 100644 --- a/aws/container-linux/kubernetes/README.md +++ b/aws/container-linux/kubernetes/README.md @@ -11,7 +11,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features -* Kubernetes v1.18.0 (upstream) +* Kubernetes v1.18.1 (upstream) * Single or multi-master, [Calico](https://www.projectcalico.org/) or [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/) * Advanced features like [worker pools](https://typhoon.psdn.io/advanced/worker-pools/), [spot](https://typhoon.psdn.io/cl/aws/#spot) workers, and [snippets](https://typhoon.psdn.io/advanced/customization/#container-linux) customization diff --git a/aws/container-linux/kubernetes/bootstrap.tf b/aws/container-linux/kubernetes/bootstrap.tf index 20663f7e4..a990373ef 100644 --- a/aws/container-linux/kubernetes/bootstrap.tf +++ b/aws/container-linux/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=1ad53d3b1c1ad75a4ed27f124f772fc5dc025245" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/aws/container-linux/kubernetes/cl/controller.yaml b/aws/container-linux/kubernetes/cl/controller.yaml index f2295528c..ef6ceab5a 100644 --- a/aws/container-linux/kubernetes/cl/controller.yaml +++ b/aws/container-linux/kubernetes/cl/controller.yaml @@ -91,7 +91,7 @@ systemd: --mount volume=var-log,target=/var/log \ --volume opt-cni-bin,kind=host,source=/opt/cni/bin \ --mount volume=opt-cni-bin,target=/opt/cni/bin \ - docker://quay.io/poseidon/kubelet:v1.18.0 -- \ + docker://quay.io/poseidon/kubelet:v1.18.1 -- \ --anonymous-auth=false \ --authentication-token-webhook \ --authorization-mode=Webhook \ @@ -134,7 +134,7 @@ systemd: --volume script,kind=host,source=/opt/bootstrap/apply \ --mount volume=script,target=/apply \ --insecure-options=image \ - docker://quay.io/poseidon/kubelet:v1.18.0 \ + docker://quay.io/poseidon/kubelet:v1.18.1 \ --net=host \ --dns=host \ --exec=/apply diff --git a/aws/container-linux/kubernetes/workers/cl/worker.yaml b/aws/container-linux/kubernetes/workers/cl/worker.yaml index 37749b21f..fd0382567 100644 --- a/aws/container-linux/kubernetes/workers/cl/worker.yaml +++ b/aws/container-linux/kubernetes/workers/cl/worker.yaml @@ -64,7 +64,7 @@ systemd: --mount volume=var-log,target=/var/log \ --volume opt-cni-bin,kind=host,source=/opt/cni/bin \ --mount volume=opt-cni-bin,target=/opt/cni/bin \ - docker://quay.io/poseidon/kubelet:v1.18.0 -- \ + docker://quay.io/poseidon/kubelet:v1.18.1 -- \ --anonymous-auth=false \ --authentication-token-webhook \ --authorization-mode=Webhook \ @@ -127,7 +127,7 @@ storage: --volume config,kind=host,source=/etc/kubernetes \ --mount volume=config,target=/etc/kubernetes \ --insecure-options=image \ - docker://quay.io/poseidon/kubelet:v1.18.0 \ + docker://quay.io/poseidon/kubelet:v1.18.1 \ --net=host \ --dns=host \ --exec=/usr/local/bin/kubectl -- --kubeconfig=/etc/kubernetes/kubeconfig delete node $(hostname) diff --git a/aws/fedora-coreos/kubernetes/README.md b/aws/fedora-coreos/kubernetes/README.md index d96e47e33..d57d0ce9e 100644 --- a/aws/fedora-coreos/kubernetes/README.md +++ b/aws/fedora-coreos/kubernetes/README.md @@ -11,7 +11,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features -* Kubernetes v1.18.0 (upstream) +* Kubernetes v1.18.1 (upstream) * Single or multi-master, [Calico](https://www.projectcalico.org/) or [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/) * Advanced features like [worker pools](https://typhoon.psdn.io/advanced/worker-pools/), [spot](https://typhoon.psdn.io/cl/aws/#spot) workers, and [snippets](https://typhoon.psdn.io/advanced/customization/#container-linux) customization diff --git a/aws/fedora-coreos/kubernetes/bootstrap.tf b/aws/fedora-coreos/kubernetes/bootstrap.tf index 59639ac92..f406896c3 100644 --- a/aws/fedora-coreos/kubernetes/bootstrap.tf +++ b/aws/fedora-coreos/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=1ad53d3b1c1ad75a4ed27f124f772fc5dc025245" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/aws/fedora-coreos/kubernetes/fcc/controller.yaml b/aws/fedora-coreos/kubernetes/fcc/controller.yaml index d0c404ff2..e5948f262 100644 --- a/aws/fedora-coreos/kubernetes/fcc/controller.yaml +++ b/aws/fedora-coreos/kubernetes/fcc/controller.yaml @@ -79,7 +79,7 @@ systemd: --volume /var/log:/var/log \ --volume /var/run/lock:/var/run/lock:z \ --volume /opt/cni/bin:/opt/cni/bin:z \ - quay.io/poseidon/kubelet:v1.18.0 \ + quay.io/poseidon/kubelet:v1.18.1 \ --anonymous-auth=false \ --authentication-token-webhook \ --authorization-mode=Webhook \ @@ -122,7 +122,7 @@ systemd: --volume /opt/bootstrap/assets:/assets:ro,Z \ --volume /opt/bootstrap/apply:/apply:ro,Z \ --entrypoint=/apply \ - quay.io/poseidon/kubelet:v1.18.0 + quay.io/poseidon/kubelet:v1.18.1 ExecStartPost=/bin/touch /opt/bootstrap/bootstrap.done ExecStartPost=-/usr/bin/podman stop bootstrap storage: diff --git a/aws/fedora-coreos/kubernetes/workers/fcc/worker.yaml b/aws/fedora-coreos/kubernetes/workers/fcc/worker.yaml index 1d5d78aa1..7c41b484c 100644 --- a/aws/fedora-coreos/kubernetes/workers/fcc/worker.yaml +++ b/aws/fedora-coreos/kubernetes/workers/fcc/worker.yaml @@ -49,7 +49,7 @@ systemd: --volume /var/log:/var/log \ --volume /var/run/lock:/var/run/lock:z \ --volume /opt/cni/bin:/opt/cni/bin:z \ - quay.io/poseidon/kubelet:v1.18.0 \ + quay.io/poseidon/kubelet:v1.18.1 \ --anonymous-auth=false \ --authentication-token-webhook \ --authorization-mode=Webhook \ @@ -87,7 +87,7 @@ systemd: Type=oneshot RemainAfterExit=true ExecStart=/bin/true - ExecStop=/bin/bash -c '/usr/bin/podman run --volume /etc/kubernetes:/etc/kubernetes:ro,z --entrypoint /usr/local/bin/kubectl quay.io/poseidon/kubelet:v1.18.0 --kubeconfig=/etc/kubernetes/kubeconfig delete node $HOSTNAME' + ExecStop=/bin/bash -c '/usr/bin/podman run --volume /etc/kubernetes:/etc/kubernetes:ro,z --entrypoint /usr/local/bin/kubectl quay.io/poseidon/kubelet:v1.18.1 --kubeconfig=/etc/kubernetes/kubeconfig delete node $HOSTNAME' [Install] WantedBy=multi-user.target storage: diff --git a/azure/container-linux/kubernetes/README.md b/azure/container-linux/kubernetes/README.md index 8a8127586..3f2c74adb 100644 --- a/azure/container-linux/kubernetes/README.md +++ b/azure/container-linux/kubernetes/README.md @@ -11,7 +11,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features -* Kubernetes v1.18.0 (upstream) +* Kubernetes v1.18.1 (upstream) * Single or multi-master, [Calico](https://www.projectcalico.org/) or [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/) * Advanced features like [worker pools](https://typhoon.psdn.io/advanced/worker-pools/), [low-priority](https://typhoon.psdn.io/cl/azure/#low-priority) workers, and [snippets](https://typhoon.psdn.io/advanced/customization/#container-linux) customization diff --git a/azure/container-linux/kubernetes/bootstrap.tf b/azure/container-linux/kubernetes/bootstrap.tf index 60da3ca36..309c1620c 100644 --- a/azure/container-linux/kubernetes/bootstrap.tf +++ b/azure/container-linux/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=1ad53d3b1c1ad75a4ed27f124f772fc5dc025245" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/azure/container-linux/kubernetes/cl/controller.yaml b/azure/container-linux/kubernetes/cl/controller.yaml index 979e858cc..e80272d74 100644 --- a/azure/container-linux/kubernetes/cl/controller.yaml +++ b/azure/container-linux/kubernetes/cl/controller.yaml @@ -90,7 +90,7 @@ systemd: --mount volume=var-log,target=/var/log \ --volume opt-cni-bin,kind=host,source=/opt/cni/bin \ --mount volume=opt-cni-bin,target=/opt/cni/bin \ - docker://quay.io/poseidon/kubelet:v1.18.0 -- \ + docker://quay.io/poseidon/kubelet:v1.18.1 -- \ --anonymous-auth=false \ --authentication-token-webhook \ --authorization-mode=Webhook \ @@ -132,7 +132,7 @@ systemd: --volume script,kind=host,source=/opt/bootstrap/apply \ --mount volume=script,target=/apply \ --insecure-options=image \ - docker://quay.io/poseidon/kubelet:v1.18.0 \ + docker://quay.io/poseidon/kubelet:v1.18.1 \ --net=host \ --dns=host \ --exec=/apply diff --git a/azure/container-linux/kubernetes/workers/cl/worker.yaml b/azure/container-linux/kubernetes/workers/cl/worker.yaml index 9ff0356fb..d2d76636e 100644 --- a/azure/container-linux/kubernetes/workers/cl/worker.yaml +++ b/azure/container-linux/kubernetes/workers/cl/worker.yaml @@ -63,7 +63,7 @@ systemd: --mount volume=var-log,target=/var/log \ --volume opt-cni-bin,kind=host,source=/opt/cni/bin \ --mount volume=opt-cni-bin,target=/opt/cni/bin \ - docker://quay.io/poseidon/kubelet:v1.18.0 -- \ + docker://quay.io/poseidon/kubelet:v1.18.1 -- \ --anonymous-auth=false \ --authentication-token-webhook \ --authorization-mode=Webhook \ @@ -125,7 +125,7 @@ storage: --volume config,kind=host,source=/etc/kubernetes \ --mount volume=config,target=/etc/kubernetes \ --insecure-options=image \ - docker://quay.io/poseidon/kubelet:v1.18.0 \ + docker://quay.io/poseidon/kubelet:v1.18.1 \ --net=host \ --dns=host \ --exec=/usr/local/bin/kubectl -- --kubeconfig=/etc/kubernetes/kubeconfig delete node $(hostname | tr '[:upper:]' '[:lower:]') diff --git a/bare-metal/container-linux/kubernetes/README.md b/bare-metal/container-linux/kubernetes/README.md index 4193ab246..d9e47720b 100644 --- a/bare-metal/container-linux/kubernetes/README.md +++ b/bare-metal/container-linux/kubernetes/README.md @@ -11,7 +11,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features -* Kubernetes v1.18.0 (upstream) +* Kubernetes v1.18.1 (upstream) * Single or multi-master, [Calico](https://www.projectcalico.org/) or [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/) * Advanced features like [snippets](https://typhoon.psdn.io/advanced/customization/#container-linux) customization diff --git a/bare-metal/container-linux/kubernetes/bootstrap.tf b/bare-metal/container-linux/kubernetes/bootstrap.tf index d9898814f..1f29c12fb 100644 --- a/bare-metal/container-linux/kubernetes/bootstrap.tf +++ b/bare-metal/container-linux/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=1ad53d3b1c1ad75a4ed27f124f772fc5dc025245" cluster_name = var.cluster_name api_servers = [var.k8s_domain_name] diff --git a/bare-metal/container-linux/kubernetes/cl/controller.yaml b/bare-metal/container-linux/kubernetes/cl/controller.yaml index 2aaaac5be..ea5c7bb09 100644 --- a/bare-metal/container-linux/kubernetes/cl/controller.yaml +++ b/bare-metal/container-linux/kubernetes/cl/controller.yaml @@ -103,7 +103,7 @@ systemd: --mount volume=etc-iscsi,target=/etc/iscsi \ --volume usr-sbin-iscsiadm,kind=host,source=/usr/sbin/iscsiadm \ --mount volume=usr-sbin-iscsiadm,target=/sbin/iscsiadm \ - docker://quay.io/poseidon/kubelet:v1.18.0 -- \ + docker://quay.io/poseidon/kubelet:v1.18.1 -- \ --anonymous-auth=false \ --authentication-token-webhook \ --authorization-mode=Webhook \ @@ -147,7 +147,7 @@ systemd: --volume script,kind=host,source=/opt/bootstrap/apply \ --mount volume=script,target=/apply \ --insecure-options=image \ - docker://quay.io/poseidon/kubelet:v1.18.0 \ + docker://quay.io/poseidon/kubelet:v1.18.1 \ --net=host \ --dns=host \ --exec=/apply diff --git a/bare-metal/container-linux/kubernetes/cl/worker.yaml b/bare-metal/container-linux/kubernetes/cl/worker.yaml index 794f70ddc..6f411b1cc 100644 --- a/bare-metal/container-linux/kubernetes/cl/worker.yaml +++ b/bare-metal/container-linux/kubernetes/cl/worker.yaml @@ -76,7 +76,7 @@ systemd: --mount volume=etc-iscsi,target=/etc/iscsi \ --volume usr-sbin-iscsiadm,kind=host,source=/usr/sbin/iscsiadm \ --mount volume=usr-sbin-iscsiadm,target=/sbin/iscsiadm \ - docker://quay.io/poseidon/kubelet:v1.18.0 -- \ + docker://quay.io/poseidon/kubelet:v1.18.1 -- \ --anonymous-auth=false \ --authentication-token-webhook \ --authorization-mode=Webhook \ diff --git a/bare-metal/fedora-coreos/kubernetes/README.md b/bare-metal/fedora-coreos/kubernetes/README.md index 1fe93c35b..3fbc067bc 100644 --- a/bare-metal/fedora-coreos/kubernetes/README.md +++ b/bare-metal/fedora-coreos/kubernetes/README.md @@ -11,7 +11,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features -* Kubernetes v1.18.0 (upstream) +* Kubernetes v1.18.1 (upstream) * Single or multi-master, [Calico](https://www.projectcalico.org/) or [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/) * Advanced features like [snippets](https://typhoon.psdn.io/advanced/customization/#container-linux) customization diff --git a/bare-metal/fedora-coreos/kubernetes/bootstrap.tf b/bare-metal/fedora-coreos/kubernetes/bootstrap.tf index 587d155ca..47ea0d306 100644 --- a/bare-metal/fedora-coreos/kubernetes/bootstrap.tf +++ b/bare-metal/fedora-coreos/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=1ad53d3b1c1ad75a4ed27f124f772fc5dc025245" cluster_name = var.cluster_name api_servers = [var.k8s_domain_name] diff --git a/bare-metal/fedora-coreos/kubernetes/fcc/controller.yaml b/bare-metal/fedora-coreos/kubernetes/fcc/controller.yaml index fcc8584d7..691cb534e 100644 --- a/bare-metal/fedora-coreos/kubernetes/fcc/controller.yaml +++ b/bare-metal/fedora-coreos/kubernetes/fcc/controller.yaml @@ -80,7 +80,7 @@ systemd: --volume /opt/cni/bin:/opt/cni/bin:z \ --volume /etc/iscsi:/etc/iscsi \ --volume /sbin/iscsiadm:/sbin/iscsiadm \ - quay.io/poseidon/kubelet:v1.18.0 \ + quay.io/poseidon/kubelet:v1.18.1 \ --anonymous-auth=false \ --authentication-token-webhook \ --authorization-mode=Webhook \ @@ -133,7 +133,7 @@ systemd: --volume /opt/bootstrap/assets:/assets:ro,Z \ --volume /opt/bootstrap/apply:/apply:ro,Z \ --entrypoint=/apply \ - quay.io/poseidon/kubelet:v1.18.0 + quay.io/poseidon/kubelet:v1.18.1 ExecStartPost=/bin/touch /opt/bootstrap/bootstrap.done ExecStartPost=-/usr/bin/podman stop bootstrap storage: diff --git a/bare-metal/fedora-coreos/kubernetes/fcc/worker.yaml b/bare-metal/fedora-coreos/kubernetes/fcc/worker.yaml index c559ae814..c20dcc9c0 100644 --- a/bare-metal/fedora-coreos/kubernetes/fcc/worker.yaml +++ b/bare-metal/fedora-coreos/kubernetes/fcc/worker.yaml @@ -50,7 +50,7 @@ systemd: --volume /opt/cni/bin:/opt/cni/bin:z \ --volume /etc/iscsi:/etc/iscsi \ --volume /sbin/iscsiadm:/sbin/iscsiadm \ - quay.io/poseidon/kubelet:v1.18.0 \ + quay.io/poseidon/kubelet:v1.18.1 \ --anonymous-auth=false \ --authentication-token-webhook \ --authorization-mode=Webhook \ diff --git a/digital-ocean/container-linux/kubernetes/README.md b/digital-ocean/container-linux/kubernetes/README.md index 7f134bf81..d9d022638 100644 --- a/digital-ocean/container-linux/kubernetes/README.md +++ b/digital-ocean/container-linux/kubernetes/README.md @@ -11,7 +11,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features -* Kubernetes v1.18.0 (upstream) +* Kubernetes v1.18.1 (upstream) * Single or multi-master, [Calico](https://www.projectcalico.org/) or [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/) * Advanced features like [snippets](https://typhoon.psdn.io/advanced/customization/#container-linux) customization diff --git a/digital-ocean/container-linux/kubernetes/bootstrap.tf b/digital-ocean/container-linux/kubernetes/bootstrap.tf index 339dde6ca..5ee3d54d7 100644 --- a/digital-ocean/container-linux/kubernetes/bootstrap.tf +++ b/digital-ocean/container-linux/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=1ad53d3b1c1ad75a4ed27f124f772fc5dc025245" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/digital-ocean/container-linux/kubernetes/cl/controller.yaml b/digital-ocean/container-linux/kubernetes/cl/controller.yaml index ce8d207d0..7dcb5e7a0 100644 --- a/digital-ocean/container-linux/kubernetes/cl/controller.yaml +++ b/digital-ocean/container-linux/kubernetes/cl/controller.yaml @@ -101,7 +101,7 @@ systemd: --mount volume=var-log,target=/var/log \ --volume opt-cni-bin,kind=host,source=/opt/cni/bin \ --mount volume=opt-cni-bin,target=/opt/cni/bin \ - docker://quay.io/poseidon/kubelet:v1.18.0 -- \ + docker://quay.io/poseidon/kubelet:v1.18.1 -- \ --anonymous-auth=false \ --authentication-token-webhook \ --authorization-mode=Webhook \ @@ -144,7 +144,7 @@ systemd: --volume script,kind=host,source=/opt/bootstrap/apply \ --mount volume=script,target=/apply \ --insecure-options=image \ - docker://quay.io/poseidon/kubelet:v1.18.0 \ + docker://quay.io/poseidon/kubelet:v1.18.1 \ --net=host \ --dns=host \ --exec=/apply diff --git a/digital-ocean/container-linux/kubernetes/cl/worker.yaml b/digital-ocean/container-linux/kubernetes/cl/worker.yaml index c20464109..e54a83cc8 100644 --- a/digital-ocean/container-linux/kubernetes/cl/worker.yaml +++ b/digital-ocean/container-linux/kubernetes/cl/worker.yaml @@ -74,7 +74,7 @@ systemd: --mount volume=var-log,target=/var/log \ --volume opt-cni-bin,kind=host,source=/opt/cni/bin \ --mount volume=opt-cni-bin,target=/opt/cni/bin \ - docker://quay.io/poseidon/kubelet:v1.18.0 -- \ + docker://quay.io/poseidon/kubelet:v1.18.1 -- \ --anonymous-auth=false \ --authentication-token-webhook \ --authorization-mode=Webhook \ @@ -131,7 +131,7 @@ storage: --volume config,kind=host,source=/etc/kubernetes \ --mount volume=config,target=/etc/kubernetes \ --insecure-options=image \ - docker://quay.io/poseidon/kubelet:v1.18.0 \ + docker://quay.io/poseidon/kubelet:v1.18.1 \ --net=host \ --dns=host \ --exec=/usr/local/bin/kubectl -- --kubeconfig=/etc/kubernetes/kubeconfig delete node $(hostname) diff --git a/docs/advanced/worker-pools.md b/docs/advanced/worker-pools.md index 04b6def0f..2392a4a73 100644 --- a/docs/advanced/worker-pools.md +++ b/docs/advanced/worker-pools.md @@ -79,7 +79,7 @@ Create a cluster following the Azure [tutorial](../cl/azure.md#cluster). Define ```tf module "ramius-worker-pool" { - source = "git::https://github.com/poseidon/typhoon//azure/container-linux/kubernetes/workers?ref=v1.18.0" + source = "git::https://github.com/poseidon/typhoon//azure/container-linux/kubernetes/workers?ref=v1.18.1" # Azure region = module.ramius.region @@ -145,7 +145,7 @@ Create a cluster following the Google Cloud [tutorial](../cl/google-cloud.md#clu ```tf module "yavin-worker-pool" { - source = "git::https://github.com/poseidon/typhoon//google-cloud/container-linux/kubernetes/workers?ref=v1.18.0" + source = "git::https://github.com/poseidon/typhoon//google-cloud/container-linux/kubernetes/workers?ref=v1.18.1" # Google Cloud region = "europe-west2" @@ -176,11 +176,11 @@ Verify a managed instance group of workers joins the cluster within a few minute ``` $ kubectl get nodes NAME STATUS AGE VERSION -yavin-controller-0.c.example-com.internal Ready 6m v1.18.0 -yavin-worker-jrbf.c.example-com.internal Ready 5m v1.18.0 -yavin-worker-mzdm.c.example-com.internal Ready 5m v1.18.0 -yavin-16x-worker-jrbf.c.example-com.internal Ready 3m v1.18.0 -yavin-16x-worker-mzdm.c.example-com.internal Ready 3m v1.18.0 +yavin-controller-0.c.example-com.internal Ready 6m v1.18.1 +yavin-worker-jrbf.c.example-com.internal Ready 5m v1.18.1 +yavin-worker-mzdm.c.example-com.internal Ready 5m v1.18.1 +yavin-16x-worker-jrbf.c.example-com.internal Ready 3m v1.18.1 +yavin-16x-worker-mzdm.c.example-com.internal Ready 3m v1.18.1 ``` ### Variables diff --git a/docs/cl/aws.md b/docs/cl/aws.md index e4e67346c..e87c97bd8 100644 --- a/docs/cl/aws.md +++ b/docs/cl/aws.md @@ -1,6 +1,6 @@ # AWS -In this tutorial, we'll create a Kubernetes v1.18.0 cluster on AWS with CoreOS Container Linux or Flatcar Linux. +In this tutorial, we'll create a Kubernetes v1.18.1 cluster on AWS with CoreOS Container Linux or Flatcar Linux. We'll declare a Kubernetes cluster using the Typhoon Terraform module. Then apply the changes to create a VPC, gateway, subnets, security groups, controller instances, worker auto-scaling group, network load balancer, and TLS assets. @@ -70,7 +70,7 @@ Define a Kubernetes cluster using the module `aws/container-linux/kubernetes`. ```tf module "tempest" { - source = "git::https://github.com/poseidon/typhoon//aws/container-linux/kubernetes?ref=v1.18.0" + source = "git::https://github.com/poseidon/typhoon//aws/container-linux/kubernetes?ref=v1.18.1" # AWS cluster_name = "tempest" @@ -143,9 +143,9 @@ List nodes in the cluster. $ export KUBECONFIG=/home/user/.kube/configs/tempest-config $ kubectl get nodes NAME STATUS ROLES AGE VERSION -ip-10-0-3-155 Ready 10m v1.18.0 -ip-10-0-26-65 Ready 10m v1.18.0 -ip-10-0-41-21 Ready 10m v1.18.0 +ip-10-0-3-155 Ready 10m v1.18.1 +ip-10-0-26-65 Ready 10m v1.18.1 +ip-10-0-41-21 Ready 10m v1.18.1 ``` List the pods. diff --git a/docs/cl/azure.md b/docs/cl/azure.md index b5ca47c34..6b2dc4a24 100644 --- a/docs/cl/azure.md +++ b/docs/cl/azure.md @@ -3,7 +3,7 @@ !!! danger Typhoon for Azure is alpha. For production, use AWS, Google Cloud, or bare-metal. As Azure matures, check [errata](https://github.com/poseidon/typhoon/wiki/Errata) for known shortcomings. -In this tutorial, we'll create a Kubernetes v1.18.0 cluster on Azure with CoreOS Container Linux or Flatcar Linux. +In this tutorial, we'll create a Kubernetes v1.18.1 cluster on Azure with CoreOS Container Linux or Flatcar Linux. We'll declare a Kubernetes cluster using the Typhoon Terraform module. Then apply the changes to create a resource group, virtual network, subnets, security groups, controller availability set, worker scale set, load balancer, and TLS assets. @@ -66,7 +66,7 @@ Define a Kubernetes cluster using the module `azure/container-linux/kubernetes`. ```tf module "ramius" { - source = "git::https://github.com/poseidon/typhoon//azure/container-linux/kubernetes?ref=v1.18.0" + source = "git::https://github.com/poseidon/typhoon//azure/container-linux/kubernetes?ref=v1.18.1" # Azure cluster_name = "ramius" @@ -149,9 +149,9 @@ List nodes in the cluster. $ export KUBECONFIG=/home/user/.kube/configs/ramius-config $ kubectl get nodes NAME STATUS ROLES AGE VERSION -ramius-controller-0 Ready 24m v1.18.0 -ramius-worker-000001 Ready 25m v1.18.0 -ramius-worker-000002 Ready 24m v1.18.0 +ramius-controller-0 Ready 24m v1.18.1 +ramius-worker-000001 Ready 25m v1.18.1 +ramius-worker-000002 Ready 24m v1.18.1 ``` List the pods. diff --git a/docs/cl/bare-metal.md b/docs/cl/bare-metal.md index b84a700e3..013e7dfbd 100644 --- a/docs/cl/bare-metal.md +++ b/docs/cl/bare-metal.md @@ -1,6 +1,6 @@ # Bare-Metal -In this tutorial, we'll network boot and provision a Kubernetes v1.18.0 cluster on bare-metal with CoreOS Container Linux or Flatcar Linux. +In this tutorial, we'll network boot and provision a Kubernetes v1.18.1 cluster on bare-metal with CoreOS Container Linux or Flatcar Linux. First, we'll deploy a [Matchbox](https://github.com/poseidon/matchbox) service and setup a network boot environment. Then, we'll declare a Kubernetes cluster using the Typhoon Terraform module and power on machines. On PXE boot, machines will install Container Linux to disk, reboot into the disk install, and provision themselves as Kubernetes controllers or workers via Ignition. @@ -160,7 +160,7 @@ Define a Kubernetes cluster using the module `bare-metal/container-linux/kuberne ```tf module "mercury" { - source = "git::https://github.com/poseidon/typhoon//bare-metal/container-linux/kubernetes?ref=v1.18.0" + source = "git::https://github.com/poseidon/typhoon//bare-metal/container-linux/kubernetes?ref=v1.18.1" # bare-metal cluster_name = "mercury" @@ -299,9 +299,9 @@ List nodes in the cluster. $ export KUBECONFIG=/home/user/.kube/configs/mercury-config $ kubectl get nodes NAME STATUS ROLES AGE VERSION -node1.example.com Ready 10m v1.18.0 -node2.example.com Ready 10m v1.18.0 -node3.example.com Ready 10m v1.18.0 +node1.example.com Ready 10m v1.18.1 +node2.example.com Ready 10m v1.18.1 +node3.example.com Ready 10m v1.18.1 ``` List the pods. diff --git a/docs/cl/digital-ocean.md b/docs/cl/digital-ocean.md index 9295e72f7..2af0d0f8c 100644 --- a/docs/cl/digital-ocean.md +++ b/docs/cl/digital-ocean.md @@ -1,6 +1,6 @@ # Digital Ocean -In this tutorial, we'll create a Kubernetes v1.18.0 cluster on DigitalOcean with CoreOS Container Linux or Flatcar Linux. +In this tutorial, we'll create a Kubernetes v1.18.1 cluster on DigitalOcean with CoreOS Container Linux or Flatcar Linux. We'll declare a Kubernetes cluster using the Typhoon Terraform module. Then apply the changes to create controller droplets, worker droplets, DNS records, tags, and TLS assets. @@ -65,7 +65,7 @@ Define a Kubernetes cluster using the module `digital-ocean/container-linux/kube ```tf module "nemo" { - source = "git::https://github.com/poseidon/typhoon//digital-ocean/container-linux/kubernetes?ref=v1.18.0" + source = "git::https://github.com/poseidon/typhoon//digital-ocean/container-linux/kubernetes?ref=v1.18.1" # Digital Ocean cluster_name = "nemo" @@ -161,9 +161,9 @@ List nodes in the cluster. $ export KUBECONFIG=/home/user/.kube/configs/nemo-config $ kubectl get nodes NAME STATUS ROLES AGE VERSION -10.132.110.130 Ready 10m v1.18.0 -10.132.115.81 Ready 10m v1.18.0 -10.132.124.107 Ready 10m v1.18.0 +10.132.110.130 Ready 10m v1.18.1 +10.132.115.81 Ready 10m v1.18.1 +10.132.124.107 Ready 10m v1.18.1 ``` List the pods. diff --git a/docs/cl/google-cloud.md b/docs/cl/google-cloud.md index c4fab5070..5bbe024c6 100644 --- a/docs/cl/google-cloud.md +++ b/docs/cl/google-cloud.md @@ -1,6 +1,6 @@ # Google Cloud -In this tutorial, we'll create a Kubernetes v1.18.0 cluster on Google Compute Engine with CoreOS Container Linux or Flatcar Linux. +In this tutorial, we'll create a Kubernetes v1.18.1 cluster on Google Compute Engine with CoreOS Container Linux or Flatcar Linux. We'll declare a Kubernetes cluster using the Typhoon Terraform module. Then apply the changes to create a network, firewall rules, health checks, controller instances, worker managed instance group, load balancers, and TLS assets. @@ -71,7 +71,7 @@ Define a Kubernetes cluster using the module `google-cloud/container-linux/kuber ```tf module "yavin" { - source = "git::https://github.com/poseidon/typhoon//google-cloud/container-linux/kubernetes?ref=v1.18.0" + source = "git::https://github.com/poseidon/typhoon//google-cloud/container-linux/kubernetes?ref=v1.18.1" # Google Cloud cluster_name = "yavin" @@ -167,9 +167,9 @@ List nodes in the cluster. $ export KUBECONFIG=/home/user/.kube/configs/yavin-config $ kubectl get nodes NAME ROLES STATUS AGE VERSION -yavin-controller-0.c.example-com.internal Ready 6m v1.18.0 -yavin-worker-jrbf.c.example-com.internal Ready 5m v1.18.0 -yavin-worker-mzdm.c.example-com.internal Ready 5m v1.18.0 +yavin-controller-0.c.example-com.internal Ready 6m v1.18.1 +yavin-worker-jrbf.c.example-com.internal Ready 5m v1.18.1 +yavin-worker-mzdm.c.example-com.internal Ready 5m v1.18.1 ``` List the pods. diff --git a/docs/fedora-coreos/aws.md b/docs/fedora-coreos/aws.md index 825856c67..6102f6a70 100644 --- a/docs/fedora-coreos/aws.md +++ b/docs/fedora-coreos/aws.md @@ -1,6 +1,6 @@ # AWS -In this tutorial, we'll create a Kubernetes v1.18.0 cluster on AWS with Fedora CoreOS. +In this tutorial, we'll create a Kubernetes v1.18.1 cluster on AWS with Fedora CoreOS. We'll declare a Kubernetes cluster using the Typhoon Terraform module. Then apply the changes to create a VPC, gateway, subnets, security groups, controller instances, worker auto-scaling group, network load balancer, and TLS assets. @@ -70,7 +70,7 @@ Define a Kubernetes cluster using the module `aws/fedora-coreos/kubernetes`. ```tf module "tempest" { - source = "git::https://github.com/poseidon/typhoon//aws/fedora-coreos/kubernetes?ref=v1.18.0" + source = "git::https://github.com/poseidon/typhoon//aws/fedora-coreos/kubernetes?ref=v1.18.1" # AWS cluster_name = "tempest" @@ -143,9 +143,9 @@ List nodes in the cluster. $ export KUBECONFIG=/home/user/.kube/configs/tempest-config $ kubectl get nodes NAME STATUS ROLES AGE VERSION -ip-10-0-3-155 Ready 10m v1.18.0 -ip-10-0-26-65 Ready 10m v1.18.0 -ip-10-0-41-21 Ready 10m v1.18.0 +ip-10-0-3-155 Ready 10m v1.18.1 +ip-10-0-26-65 Ready 10m v1.18.1 +ip-10-0-41-21 Ready 10m v1.18.1 ``` List the pods. diff --git a/docs/fedora-coreos/bare-metal.md b/docs/fedora-coreos/bare-metal.md index 05ca82597..a8e0a7bac 100644 --- a/docs/fedora-coreos/bare-metal.md +++ b/docs/fedora-coreos/bare-metal.md @@ -1,6 +1,6 @@ # Bare-Metal -In this tutorial, we'll network boot and provision a Kubernetes v1.18.0 cluster on bare-metal with Fedora CoreOS. +In this tutorial, we'll network boot and provision a Kubernetes v1.18.1 cluster on bare-metal with Fedora CoreOS. First, we'll deploy a [Matchbox](https://github.com/poseidon/matchbox) service and setup a network boot environment. Then, we'll declare a Kubernetes cluster using the Typhoon Terraform module and power on machines. On PXE boot, machines will install Fedora CoreOS to disk, reboot into the disk install, and provision themselves as Kubernetes controllers or workers via Ignition. @@ -160,7 +160,7 @@ Define a Kubernetes cluster using the module `bare-metal/fedora-coreos/kubernete ```tf module "mercury" { - source = "git::https://github.com/poseidon/typhoon//bare-metal/fedora-coreos/kubernetes?ref=v1.18.0" + source = "git::https://github.com/poseidon/typhoon//bare-metal/fedora-coreos/kubernetes?ref=v1.18.1" # bare-metal cluster_name = "mercury" @@ -289,9 +289,9 @@ List nodes in the cluster. $ export KUBECONFIG=/home/user/.kube/configs/mercury-config $ kubectl get nodes NAME STATUS ROLES AGE VERSION -node1.example.com Ready 10m v1.18.0 -node2.example.com Ready 10m v1.18.0 -node3.example.com Ready 10m v1.18.0 +node1.example.com Ready 10m v1.18.1 +node2.example.com Ready 10m v1.18.1 +node3.example.com Ready 10m v1.18.1 ``` List the pods. diff --git a/docs/fedora-coreos/google-cloud.md b/docs/fedora-coreos/google-cloud.md index b20e6614e..295f1a197 100644 --- a/docs/fedora-coreos/google-cloud.md +++ b/docs/fedora-coreos/google-cloud.md @@ -1,6 +1,6 @@ # Google Cloud -In this tutorial, we'll create a Kubernetes v1.18.0 cluster on Google Compute Engine with Fedora CoreOS. +In this tutorial, we'll create a Kubernetes v1.18.1 cluster on Google Compute Engine with Fedora CoreOS. We'll declare a Kubernetes cluster using the Typhoon Terraform module. Then apply the changes to create a network, firewall rules, health checks, controller instances, worker managed instance group, load balancers, and TLS assets. @@ -165,9 +165,9 @@ List nodes in the cluster. $ export KUBECONFIG=/home/user/.kube/configs/yavin-config $ kubectl get nodes NAME ROLES STATUS AGE VERSION -yavin-controller-0.c.example-com.internal Ready 6m v1.18.0 -yavin-worker-jrbf.c.example-com.internal Ready 5m v1.18.0 -yavin-worker-mzdm.c.example-com.internal Ready 5m v1.18.0 +yavin-controller-0.c.example-com.internal Ready 6m v1.18.1 +yavin-worker-jrbf.c.example-com.internal Ready 5m v1.18.1 +yavin-worker-mzdm.c.example-com.internal Ready 5m v1.18.1 ``` List the pods. diff --git a/docs/index.md b/docs/index.md index 0dda364b2..279d60f22 100644 --- a/docs/index.md +++ b/docs/index.md @@ -11,7 +11,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features -* Kubernetes v1.18.0 (upstream) +* Kubernetes v1.18.1 (upstream) * Single or multi-master, [Calico](https://www.projectcalico.org/) or [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/) * Advanced features like [worker pools](advanced/worker-pools/), [preemptible](cl/google-cloud/#preemption) workers, and [snippets](advanced/customization/#container-linux) customization @@ -59,7 +59,7 @@ Define a Kubernetes cluster by using the Terraform module for your chosen platfo ```tf module "yavin" { - source = "git::https://github.com/poseidon/typhoon//google-cloud/container-linux/kubernetes?ref=v1.18.0" + source = "git::https://github.com/poseidon/typhoon//google-cloud/container-linux/kubernetes?ref=v1.18.1" # Google Cloud cluster_name = "yavin" @@ -97,9 +97,9 @@ In 4-8 minutes (varies by platform), the cluster will be ready. This Google Clou $ export KUBECONFIG=/home/user/.kube/configs/yavin-config $ kubectl get nodes NAME ROLES STATUS AGE VERSION -yavin-controller-0.c.example-com.internal Ready 6m v1.18.0 -yavin-worker-jrbf.c.example-com.internal Ready 5m v1.18.0 -yavin-worker-mzdm.c.example-com.internal Ready 5m v1.18.0 +yavin-controller-0.c.example-com.internal Ready 6m v1.18.1 +yavin-worker-jrbf.c.example-com.internal Ready 5m v1.18.1 +yavin-worker-mzdm.c.example-com.internal Ready 5m v1.18.1 ``` List the pods. diff --git a/docs/topics/maintenance.md b/docs/topics/maintenance.md index de265eea7..59978802f 100644 --- a/docs/topics/maintenance.md +++ b/docs/topics/maintenance.md @@ -18,7 +18,7 @@ module "yavin" { } module "mercury" { - source = "git::https://github.com/poseidon/typhoon//bare-metal/container-linux/kubernetes?ref=v1.18.0" + source = "git::https://github.com/poseidon/typhoon//bare-metal/container-linux/kubernetes?ref=v1.18.1" ... } ``` @@ -279,15 +279,15 @@ Typhoon modules have been adapted for Terraform v0.12. Provider plugins requirem | Typhoon Release | Terraform version | |-------------------|---------------------| -| v1.18.0 - ? | v0.12.x | -| v1.10.3 - v1.18.0 | v0.11.x | +| v1.18.1 - ? | v0.12.x | +| v1.10.3 - v1.18.1 | v0.11.x | | v1.9.2 - v1.10.2 | v0.10.4+ or v0.11.x | | v1.7.3 - v1.9.1 | v0.10.x | | v1.6.4 - v1.7.2 | v0.9.x | ### New users -New users can start with Terraform v0.12.x and follow the docs for Typhoon v1.18.0+ without issue. +New users can start with Terraform v0.12.x and follow the docs for Typhoon v1.18.1+ without issue. ### Existing users diff --git a/google-cloud/container-linux/kubernetes/README.md b/google-cloud/container-linux/kubernetes/README.md index 4efb24164..94cd7d8aa 100644 --- a/google-cloud/container-linux/kubernetes/README.md +++ b/google-cloud/container-linux/kubernetes/README.md @@ -11,7 +11,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features -* Kubernetes v1.18.0 (upstream) +* Kubernetes v1.18.1 (upstream) * Single or multi-master, [Calico](https://www.projectcalico.org/) or [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/) * Advanced features like [worker pools](https://typhoon.psdn.io/advanced/worker-pools/), [preemptible](https://typhoon.psdn.io/cl/google-cloud/#preemption) workers, and [snippets](https://typhoon.psdn.io/advanced/customization/#container-linux) customization diff --git a/google-cloud/container-linux/kubernetes/bootstrap.tf b/google-cloud/container-linux/kubernetes/bootstrap.tf index 79ec3db70..6db9f8013 100644 --- a/google-cloud/container-linux/kubernetes/bootstrap.tf +++ b/google-cloud/container-linux/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=1ad53d3b1c1ad75a4ed27f124f772fc5dc025245" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/google-cloud/container-linux/kubernetes/cl/controller.yaml b/google-cloud/container-linux/kubernetes/cl/controller.yaml index d49f75d19..8464e7bae 100644 --- a/google-cloud/container-linux/kubernetes/cl/controller.yaml +++ b/google-cloud/container-linux/kubernetes/cl/controller.yaml @@ -90,7 +90,7 @@ systemd: --mount volume=var-log,target=/var/log \ --volume opt-cni-bin,kind=host,source=/opt/cni/bin \ --mount volume=opt-cni-bin,target=/opt/cni/bin \ - docker://quay.io/poseidon/kubelet:v1.18.0 -- \ + docker://quay.io/poseidon/kubelet:v1.18.1 -- \ --anonymous-auth=false \ --authentication-token-webhook \ --authorization-mode=Webhook \ @@ -132,7 +132,7 @@ systemd: --volume script,kind=host,source=/opt/bootstrap/apply \ --mount volume=script,target=/apply \ --insecure-options=image \ - docker://quay.io/poseidon/kubelet:v1.18.0 \ + docker://quay.io/poseidon/kubelet:v1.18.1 \ --net=host \ --dns=host \ --exec=/apply diff --git a/google-cloud/container-linux/kubernetes/workers/cl/worker.yaml b/google-cloud/container-linux/kubernetes/workers/cl/worker.yaml index d1f08d4ad..01d69d983 100644 --- a/google-cloud/container-linux/kubernetes/workers/cl/worker.yaml +++ b/google-cloud/container-linux/kubernetes/workers/cl/worker.yaml @@ -63,7 +63,7 @@ systemd: --mount volume=var-log,target=/var/log \ --volume opt-cni-bin,kind=host,source=/opt/cni/bin \ --mount volume=opt-cni-bin,target=/opt/cni/bin \ - docker://quay.io/poseidon/kubelet:v1.18.0 -- \ + docker://quay.io/poseidon/kubelet:v1.18.1 -- \ --anonymous-auth=false \ --authentication-token-webhook \ --authorization-mode=Webhook \ @@ -125,7 +125,7 @@ storage: --volume config,kind=host,source=/etc/kubernetes \ --mount volume=config,target=/etc/kubernetes \ --insecure-options=image \ - docker://quay.io/poseidon/kubelet:v1.18.0 \ + docker://quay.io/poseidon/kubelet:v1.18.1 \ --net=host \ --dns=host \ --exec=/usr/local/bin/kubectl -- --kubeconfig=/etc/kubernetes/kubeconfig delete node $(hostname) diff --git a/google-cloud/fedora-coreos/kubernetes/README.md b/google-cloud/fedora-coreos/kubernetes/README.md index 4efb24164..94cd7d8aa 100644 --- a/google-cloud/fedora-coreos/kubernetes/README.md +++ b/google-cloud/fedora-coreos/kubernetes/README.md @@ -11,7 +11,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features -* Kubernetes v1.18.0 (upstream) +* Kubernetes v1.18.1 (upstream) * Single or multi-master, [Calico](https://www.projectcalico.org/) or [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/) * Advanced features like [worker pools](https://typhoon.psdn.io/advanced/worker-pools/), [preemptible](https://typhoon.psdn.io/cl/google-cloud/#preemption) workers, and [snippets](https://typhoon.psdn.io/advanced/customization/#container-linux) customization diff --git a/google-cloud/fedora-coreos/kubernetes/bootstrap.tf b/google-cloud/fedora-coreos/kubernetes/bootstrap.tf index cbdb3d239..31693c2bb 100644 --- a/google-cloud/fedora-coreos/kubernetes/bootstrap.tf +++ b/google-cloud/fedora-coreos/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=1ad53d3b1c1ad75a4ed27f124f772fc5dc025245" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/google-cloud/fedora-coreos/kubernetes/fcc/controller.yaml b/google-cloud/fedora-coreos/kubernetes/fcc/controller.yaml index d441371c8..3bbcbda32 100644 --- a/google-cloud/fedora-coreos/kubernetes/fcc/controller.yaml +++ b/google-cloud/fedora-coreos/kubernetes/fcc/controller.yaml @@ -79,7 +79,7 @@ systemd: --volume /var/log:/var/log \ --volume /var/run/lock:/var/run/lock:z \ --volume /opt/cni/bin:/opt/cni/bin:z \ - quay.io/poseidon/kubelet:v1.18.0 \ + quay.io/poseidon/kubelet:v1.18.1 \ --anonymous-auth=false \ --authentication-token-webhook \ --authorization-mode=Webhook \ @@ -123,7 +123,7 @@ systemd: --volume /opt/bootstrap/assets:/assets:ro,Z \ --volume /opt/bootstrap/apply:/apply:ro,Z \ --entrypoint=/apply \ - quay.io/poseidon/kubelet:v1.18.0 + quay.io/poseidon/kubelet:v1.18.1 ExecStartPost=/bin/touch /opt/bootstrap/bootstrap.done ExecStartPost=-/usr/bin/podman stop bootstrap storage: diff --git a/google-cloud/fedora-coreos/kubernetes/workers/fcc/worker.yaml b/google-cloud/fedora-coreos/kubernetes/workers/fcc/worker.yaml index 1d5d78aa1..7c41b484c 100644 --- a/google-cloud/fedora-coreos/kubernetes/workers/fcc/worker.yaml +++ b/google-cloud/fedora-coreos/kubernetes/workers/fcc/worker.yaml @@ -49,7 +49,7 @@ systemd: --volume /var/log:/var/log \ --volume /var/run/lock:/var/run/lock:z \ --volume /opt/cni/bin:/opt/cni/bin:z \ - quay.io/poseidon/kubelet:v1.18.0 \ + quay.io/poseidon/kubelet:v1.18.1 \ --anonymous-auth=false \ --authentication-token-webhook \ --authorization-mode=Webhook \ @@ -87,7 +87,7 @@ systemd: Type=oneshot RemainAfterExit=true ExecStart=/bin/true - ExecStop=/bin/bash -c '/usr/bin/podman run --volume /etc/kubernetes:/etc/kubernetes:ro,z --entrypoint /usr/local/bin/kubectl quay.io/poseidon/kubelet:v1.18.0 --kubeconfig=/etc/kubernetes/kubeconfig delete node $HOSTNAME' + ExecStop=/bin/bash -c '/usr/bin/podman run --volume /etc/kubernetes:/etc/kubernetes:ro,z --entrypoint /usr/local/bin/kubectl quay.io/poseidon/kubelet:v1.18.1 --kubeconfig=/etc/kubernetes/kubeconfig delete node $HOSTNAME' [Install] WantedBy=multi-user.target storage: From 80538e295361f818acbed6b60aa5cd6829a06b32 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Thu, 9 Apr 2020 23:13:08 -0700 Subject: [PATCH 387/523] Add support for Fedora CoreOS on DigitalOcean * Add `digital-ocean/fedora-coreos/kubernetes` module * DigitalOcean custom uploaded images do not permit droplet IPv6 networking --- CHANGES.md | 11 +- README.md | 5 +- .../fedora-coreos/kubernetes/LICENSE | 23 ++ .../fedora-coreos/kubernetes/README.md | 23 ++ .../fedora-coreos/kubernetes/bootstrap.tf | 25 ++ .../fedora-coreos/kubernetes/controllers.tf | 100 +++++++ .../kubernetes/fcc/controller.yaml | 214 +++++++++++++++ .../fedora-coreos/kubernetes/fcc/worker.yaml | 117 +++++++++ .../fedora-coreos/kubernetes/network.tf | 117 +++++++++ .../fedora-coreos/kubernetes/outputs.tf | 47 ++++ digital-ocean/fedora-coreos/kubernetes/ssh.tf | 87 ++++++ .../fedora-coreos/kubernetes/variables.tf | 114 ++++++++ .../fedora-coreos/kubernetes/versions.tf | 12 + .../fedora-coreos/kubernetes/workers.tf | 77 ++++++ digital-ocean/ignore/.gitkeep | 0 docs/cl/digital-ocean.md | 4 +- docs/fedora-coreos/digitalocean.md | 247 ++++++++++++++++++ docs/fedora-coreos/google-cloud.md | 12 +- docs/index.md | 5 +- 19 files changed, 1226 insertions(+), 14 deletions(-) create mode 100644 digital-ocean/fedora-coreos/kubernetes/LICENSE create mode 100644 digital-ocean/fedora-coreos/kubernetes/README.md create mode 100644 digital-ocean/fedora-coreos/kubernetes/bootstrap.tf create mode 100644 digital-ocean/fedora-coreos/kubernetes/controllers.tf create mode 100644 digital-ocean/fedora-coreos/kubernetes/fcc/controller.yaml create mode 100644 digital-ocean/fedora-coreos/kubernetes/fcc/worker.yaml create mode 100644 digital-ocean/fedora-coreos/kubernetes/network.tf create mode 100644 digital-ocean/fedora-coreos/kubernetes/outputs.tf create mode 100644 digital-ocean/fedora-coreos/kubernetes/ssh.tf create mode 100644 digital-ocean/fedora-coreos/kubernetes/variables.tf create mode 100644 digital-ocean/fedora-coreos/kubernetes/versions.tf create mode 100644 digital-ocean/fedora-coreos/kubernetes/workers.tf delete mode 100644 digital-ocean/ignore/.gitkeep create mode 100644 docs/fedora-coreos/digitalocean.md diff --git a/CHANGES.md b/CHANGES.md index d11990ba1..6957ee58d 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -11,14 +11,21 @@ Notable changes between versions. * Change `kube-proxy` and `calico` or `flannel` to tolerate specific taints ([#682](https://github.com/poseidon/typhoon/pull/682)) * Tolerate master and not-ready taints, rather than tolerating all taints * Update flannel from v0.11.0 to v0.12.0 ([#690](https://github.com/poseidon/typhoon/pull/690)) +* Fix bootstrap when `networking` mode `flannel` (non-default) is chosen ([#689](https://github.com/poseidon/typhoon/pull/689)) + * Regressed in v1.18.0 changes for Calico ([#675](https://github.com/poseidon/typhoon/pull/675)) * Rename Container Linux `controller_clc_snippets` to `controller_snippets` for consistency ([#688](https://github.com/poseidon/typhoon/pull/688)) * Rename Container Linux `worker_clc_snippets` to `worker_snippets` for consistency * Rename Container Linux `clc_snippets` (bare-metal) to `snippets` for consistency -* Fix bootstrap when `networking` mode `flannel` (non-default) is chosen ([#689](https://github.com/poseidon/typhoon/pull/689)) - * Regressed in v1.18.0 changes for Calico ([#675](https://github.com/poseidon/typhoon/pull/675)) + +#### Azure + * Fix Azure worker UDP outbound connections ([#691](https://github.com/poseidon/typhoon/pull/691)) * Fix Azure worker clock sync timeouts +#### DigitalOcean + +* Add support for Fedora CoreOS ([#699](https://github.com/poseidon/typhoon/pull/699)) + #### Addons * Refresh Prometheus rules/alerts and Grafana dashboards ([#692](https://github.com/poseidon/typhoon/pull/692)) diff --git a/README.md b/README.md index bb053dda6..079f5f250 100644 --- a/README.md +++ b/README.md @@ -35,6 +35,7 @@ Typhoon is available for [Fedora CoreOS](https://getfedora.org/coreos/). |---------------|------------------|------------------|--------| | AWS | Fedora CoreOS | [aws/fedora-coreos/kubernetes](aws/fedora-coreos/kubernetes) | stable | | Bare-Metal | Fedora CoreOS | [bare-metal/fedora-coreos/kubernetes](bare-metal/fedora-coreos/kubernetes) | beta | +| DigitalOcean | Fedora CoreOS | [digital-ocean/fedora-coreos/kubernetes](digital-ocean/fedora-coreos/kubernetes) | alpha | | Google Cloud | Fedora CoreOS | [google-cloud/fedora-coreos/kubernetes](google-cloud/fedora-coreos/kubernetes) | beta | Typhoon is available for [Flatcar Container Linux](https://www.flatcar-linux.org/releases/). @@ -44,14 +45,14 @@ Typhoon is available for [Flatcar Container Linux](https://www.flatcar-linux.org | AWS | Flatcar Linux | [aws/container-linux/kubernetes](aws/container-linux/kubernetes) | stable | | Azure | Flatcar Linux | [azure/container-linux/kubernetes](azure/container-linux/kubernetes) | alpha | | Bare-Metal | Flatcar Linux | [bare-metal/container-linux/kubernetes](bare-metal/container-linux/kubernetes) | stable | +| DigitalOcean | Flatcar Linux | [digital-ocean/container-linux/kubernetes](digital-ocean/container-linux/kubernetes) | alpha | | Google Cloud | Flatcar Linux | [google-cloud/container-linux/kubernetes](google-cloud/container-linux/kubernetes) | alpha | -| Digital Ocean | Flatcar Linux | [digital-ocean/container-linux/kubernetes](digital-ocean/container-linux/kubernetes) | alpha | ## Documentation * [Docs](https://typhoon.psdn.io) * Architecture [concepts](https://typhoon.psdn.io/architecture/concepts/) and [operating systems](https://typhoon.psdn.io/architecture/operating-systems/) -* Fedora CoreOS tutorials for [AWS](docs/fedora-coreos/aws.md), [Bare-Metal](docs/fedora-coreos/bare-metal.md), and [Google Cloud](docs/fedora-coreos/google-cloud.md) +* Fedora CoreOS tutorials for [AWS](docs/fedora-coreos/aws.md), [Bare-Metal](docs/fedora-coreos/bare-metal.md), [DigitalOcean](docs/fedora-coreos/digitalocean.md), and [Google Cloud](docs/fedora-coreos/google-cloud.md) * Flatcar Linux tutorials for [AWS](docs/cl/aws.md), [Azure](docs/cl/azure.md), [Bare-Metal](docs/cl/bare-metal.md), [DigitalOcean](docs/cl/digital-ocean.md), and [Google Cloud](docs/cl/google-cloud.md) ## Usage diff --git a/digital-ocean/fedora-coreos/kubernetes/LICENSE b/digital-ocean/fedora-coreos/kubernetes/LICENSE new file mode 100644 index 000000000..658b1c469 --- /dev/null +++ b/digital-ocean/fedora-coreos/kubernetes/LICENSE @@ -0,0 +1,23 @@ +The MIT License (MIT) + +Copyright (c) 2020 Typhoon Authors +Copyright (c) 2020 Dalton Hubble + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. + diff --git a/digital-ocean/fedora-coreos/kubernetes/README.md b/digital-ocean/fedora-coreos/kubernetes/README.md new file mode 100644 index 000000000..18cd0c6bc --- /dev/null +++ b/digital-ocean/fedora-coreos/kubernetes/README.md @@ -0,0 +1,23 @@ +# Typhoon + +Typhoon is a minimal and free Kubernetes distribution. + +* Minimal, stable base Kubernetes distribution +* Declarative infrastructure and configuration +* Free (freedom and cost) and privacy-respecting +* Practical for labs, datacenters, and clouds + +Typhoon distributes upstream Kubernetes, architectural conventions, and cluster addons, much like a GNU/Linux distribution provides the Linux kernel and userspace components. + +## Features + +* Kubernetes v1.18.1 (upstream) +* Single or multi-master, [Calico](https://www.projectcalico.org/) or [flannel](https://github.com/coreos/flannel) networking +* On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/) +* Advanced features like [snippets](https://typhoon.psdn.io/advanced/customization/) customization +* Ready for Ingress, Prometheus, Grafana, CSI, and other [addons](https://typhoon.psdn.io/addons/overview/) + +## Docs + +Please see the [official docs](https://typhoon.psdn.io) and the Digital Ocean [tutorial](https://typhoon.psdn.io/fedora-coreos/digitalocean/). + diff --git a/digital-ocean/fedora-coreos/kubernetes/bootstrap.tf b/digital-ocean/fedora-coreos/kubernetes/bootstrap.tf new file mode 100644 index 000000000..17847ce77 --- /dev/null +++ b/digital-ocean/fedora-coreos/kubernetes/bootstrap.tf @@ -0,0 +1,25 @@ +# Kubernetes assets (kubeconfig, manifests) +module "bootstrap" { + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=1ad53d3b1c1ad75a4ed27f124f772fc5dc025245" + + cluster_name = var.cluster_name + api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] + etcd_servers = digitalocean_record.etcds.*.fqdn + asset_dir = var.asset_dir + + networking = var.networking + + # only effective with Calico networking + network_encapsulation = "vxlan" + network_mtu = "1450" + + pod_cidr = var.pod_cidr + service_cidr = var.service_cidr + cluster_domain_suffix = var.cluster_domain_suffix + enable_reporting = var.enable_reporting + enable_aggregation = var.enable_aggregation + + # Fedora CoreOS + trusted_certs_dir = "/etc/pki/tls/certs" +} + diff --git a/digital-ocean/fedora-coreos/kubernetes/controllers.tf b/digital-ocean/fedora-coreos/kubernetes/controllers.tf new file mode 100644 index 000000000..5bf2c18a8 --- /dev/null +++ b/digital-ocean/fedora-coreos/kubernetes/controllers.tf @@ -0,0 +1,100 @@ +# Controller Instance DNS records +resource "digitalocean_record" "controllers" { + count = var.controller_count + + # DNS zone where record should be created + domain = var.dns_zone + + # DNS record (will be prepended to domain) + name = var.cluster_name + type = "A" + ttl = 300 + + # IPv4 addresses of controllers + value = digitalocean_droplet.controllers.*.ipv4_address[count.index] +} + +# Discrete DNS records for each controller's private IPv4 for etcd usage +resource "digitalocean_record" "etcds" { + count = var.controller_count + + # DNS zone where record should be created + domain = var.dns_zone + + # DNS record (will be prepended to domain) + name = "${var.cluster_name}-etcd${count.index}" + type = "A" + ttl = 300 + + # private IPv4 address for etcd + value = digitalocean_droplet.controllers.*.ipv4_address_private[count.index] +} + +# Controller droplet instances +resource "digitalocean_droplet" "controllers" { + count = var.controller_count + + name = "${var.cluster_name}-controller-${count.index}" + region = var.region + + image = var.os_image + size = var.controller_type + + # network + # TODO: Only official DigitalOcean images support IPv6 + ipv6 = false + private_networking = true + + user_data = data.ct_config.controller-ignitions.*.rendered[count.index] + ssh_keys = var.ssh_fingerprints + + tags = [ + digitalocean_tag.controllers.id, + ] + + lifecycle { + ignore_changes = [user_data] + } +} + +# Tag to label controllers +resource "digitalocean_tag" "controllers" { + name = "${var.cluster_name}-controller" +} + +# Controller Ignition configs +data "ct_config" "controller-ignitions" { + count = var.controller_count + content = data.template_file.controller-configs.*.rendered[count.index] + strict = true + snippets = var.controller_snippets +} + +# Controller Fedora CoreOS configs +data "template_file" "controller-configs" { + count = var.controller_count + + template = file("${path.module}/fcc/controller.yaml") + + vars = { + # Cannot use cyclic dependencies on controllers or their DNS records + etcd_name = "etcd${count.index}" + etcd_domain = "${var.cluster_name}-etcd${count.index}.${var.dns_zone}" + # etcd0=https://cluster-etcd0.example.com,etcd1=https://cluster-etcd1.example.com,... + etcd_initial_cluster = join(",", data.template_file.etcds.*.rendered) + cluster_dns_service_ip = cidrhost(var.service_cidr, 10) + cluster_domain_suffix = var.cluster_domain_suffix + } +} + +data "template_file" "etcds" { + count = var.controller_count + template = "etcd$${index}=https://$${cluster_name}-etcd$${index}.$${dns_zone}:2380" + + vars = { + index = count.index + cluster_name = var.cluster_name + dns_zone = var.dns_zone + } +} + diff --git a/digital-ocean/fedora-coreos/kubernetes/fcc/controller.yaml b/digital-ocean/fedora-coreos/kubernetes/fcc/controller.yaml new file mode 100644 index 000000000..916ed266d --- /dev/null +++ b/digital-ocean/fedora-coreos/kubernetes/fcc/controller.yaml @@ -0,0 +1,214 @@ +--- +variant: fcos +version: 1.0.0 +systemd: + units: + - name: etcd-member.service + enabled: true + contents: | + [Unit] + Description=etcd (System Container) + Documentation=https://github.com/coreos/etcd + Wants=network-online.target network.target + After=network-online.target + [Service] + # https://github.com/opencontainers/runc/pull/1807 + # Type=notify + # NotifyAccess=exec + Type=exec + Restart=on-failure + RestartSec=10s + TimeoutStartSec=0 + LimitNOFILE=40000 + ExecStartPre=/bin/mkdir -p /var/lib/etcd + ExecStartPre=-/usr/bin/podman rm etcd + #--volume $${NOTIFY_SOCKET}:/run/systemd/notify \ + ExecStart=/usr/bin/podman run --name etcd \ + --env-file /etc/etcd/etcd.env \ + --network host \ + --volume /var/lib/etcd:/var/lib/etcd:rw,Z \ + --volume /etc/ssl/etcd:/etc/ssl/certs:ro,Z \ + quay.io/coreos/etcd:v3.4.7 + ExecStop=/usr/bin/podman stop etcd + [Install] + WantedBy=multi-user.target + - name: docker.service + enabled: true + - name: wait-for-dns.service + enabled: true + contents: | + [Unit] + Description=Wait for DNS entries + Before=kubelet.service + [Service] + Type=oneshot + RemainAfterExit=true + ExecStart=/bin/sh -c 'while ! /usr/bin/grep '^[^#[:space:]]' /etc/resolv.conf > /dev/null; do sleep 1; done' + [Install] + RequiredBy=kubelet.service + RequiredBy=etcd-member.service + - name: kubelet.service + contents: | + [Unit] + Description=Kubelet via Hyperkube (System Container) + Requires=afterburn.service + After=afterburn.service + Wants=rpc-statd.service + [Service] + EnvironmentFile=/run/metadata/afterburn + ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d + ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests + ExecStartPre=/bin/mkdir -p /opt/cni/bin + ExecStartPre=/bin/mkdir -p /var/lib/calico + ExecStartPre=/bin/mkdir -p /var/lib/kubelet/volumeplugins + ExecStartPre=/usr/bin/bash -c "grep 'certificate-authority-data' /etc/kubernetes/kubeconfig | awk '{print $2}' | base64 -d > /etc/kubernetes/ca.crt" + ExecStartPre=-/usr/bin/podman rm kubelet + ExecStart=/usr/bin/podman run --name kubelet \ + --privileged \ + --pid host \ + --network host \ + --volume /etc/kubernetes:/etc/kubernetes:ro,z \ + --volume /usr/lib/os-release:/etc/os-release:ro \ + --volume /etc/ssl/certs:/etc/ssl/certs:ro \ + --volume /lib/modules:/lib/modules:ro \ + --volume /run:/run \ + --volume /sys/fs/cgroup:/sys/fs/cgroup:ro \ + --volume /sys/fs/cgroup/systemd:/sys/fs/cgroup/systemd \ + --volume /etc/pki/tls/certs:/usr/share/ca-certificates:ro \ + --volume /var/lib/calico:/var/lib/calico:ro \ + --volume /var/lib/docker:/var/lib/docker \ + --volume /var/lib/kubelet:/var/lib/kubelet:rshared,z \ + --volume /var/log:/var/log \ + --volume /var/run/lock:/var/run/lock:z \ + --volume /opt/cni/bin:/opt/cni/bin:z \ + quay.io/poseidon/kubelet:v1.18.1 \ + --anonymous-auth=false \ + --authentication-token-webhook \ + --authorization-mode=Webhook \ + --cgroup-driver=systemd \ + --cgroups-per-qos=true \ + --enforce-node-allocatable=pods \ + --client-ca-file=/etc/kubernetes/ca.crt \ + --cluster_dns=${cluster_dns_service_ip} \ + --cluster_domain=${cluster_domain_suffix} \ + --cni-conf-dir=/etc/kubernetes/cni/net.d \ + --exit-on-lock-contention \ + --healthz-port=0 \ + --hostname-override=$${AFTERBURN_DIGITALOCEAN_IPV4_PRIVATE_0} \ + --kubeconfig=/etc/kubernetes/kubeconfig \ + --lock-file=/var/run/lock/kubelet.lock \ + --network-plugin=cni \ + --node-labels=node.kubernetes.io/master \ + --node-labels=node.kubernetes.io/controller="true" \ + --pod-manifest-path=/etc/kubernetes/manifests \ + --read-only-port=0 \ + --register-with-taints=node-role.kubernetes.io/master=:NoSchedule \ + --volume-plugin-dir=/var/lib/kubelet/volumeplugins + ExecStop=-/usr/bin/podman stop kubelet + Delegate=yes + Restart=always + RestartSec=10 + [Install] + WantedBy=multi-user.target + - name: kubelet.path + enabled: true + contents: | + [Unit] + Description=Watch for kubeconfig + [Path] + PathExists=/etc/kubernetes/kubeconfig + [Install] + WantedBy=multi-user.target + - name: bootstrap.service + contents: | + [Unit] + Description=Kubernetes control plane + ConditionPathExists=!/opt/bootstrap/bootstrap.done + [Service] + Type=oneshot + RemainAfterExit=true + WorkingDirectory=/opt/bootstrap + ExecStartPre=-/usr/bin/podman rm bootstrap + ExecStart=/usr/bin/podman run --name bootstrap \ + --network host \ + --volume /etc/kubernetes/bootstrap-secrets:/etc/kubernetes/secrets:ro,Z \ + --volume /opt/bootstrap/assets:/assets:ro,Z \ + --volume /opt/bootstrap/apply:/apply:ro,Z \ + --entrypoint=/apply \ + quay.io/poseidon/kubelet:v1.18.1 + ExecStartPost=/bin/touch /opt/bootstrap/bootstrap.done + ExecStartPost=-/usr/bin/podman stop bootstrap +storage: + directories: + - path: /etc/kubernetes + - path: /opt/bootstrap + files: + - path: /opt/bootstrap/layout + mode: 0544 + contents: + inline: | + #!/bin/bash -e + mkdir -p -- auth tls/etcd tls/k8s static-manifests manifests/coredns manifests-networking + awk '/#####/ {filename=$2; next} {print > filename}' assets + mkdir -p /etc/ssl/etcd/etcd + mkdir -p /etc/kubernetes/bootstrap-secrets + mv tls/etcd/{peer*,server*} /etc/ssl/etcd/etcd/ + mv tls/etcd/etcd-client* /etc/kubernetes/bootstrap-secrets/ + chown -R etcd:etcd /etc/ssl/etcd + chmod -R 500 /etc/ssl/etcd + mv auth/kubeconfig /etc/kubernetes/bootstrap-secrets/ + mv tls/k8s/* /etc/kubernetes/bootstrap-secrets/ + sudo mkdir -p /etc/kubernetes/manifests + sudo mv static-manifests/* /etc/kubernetes/manifests/ + sudo mkdir -p /opt/bootstrap/assets + sudo mv manifests /opt/bootstrap/assets/manifests + sudo mv manifests-networking/* /opt/bootstrap/assets/manifests/ + rm -rf assets auth static-manifests tls manifests-networking + - path: /opt/bootstrap/apply + mode: 0544 + contents: + inline: | + #!/bin/bash -e + export KUBECONFIG=/etc/kubernetes/secrets/kubeconfig + until kubectl version; do + echo "Waiting for static pod control plane" + sleep 5 + done + until kubectl apply -f /assets/manifests -R; do + echo "Retry applying manifests" + sleep 5 + done + - path: /etc/sysctl.d/max-user-watches.conf + contents: + inline: | + fs.inotify.max_user_watches=16184 + - path: /etc/systemd/system.conf.d/accounting.conf + contents: + inline: | + [Manager] + DefaultCPUAccounting=yes + DefaultMemoryAccounting=yes + DefaultBlockIOAccounting=yes + - path: /etc/etcd/etcd.env + mode: 0644 + contents: + inline: | + # TODO: Use a systemd dropin once podman v1.4.5 is avail. + NOTIFY_SOCKET=/run/systemd/notify + ETCD_NAME=${etcd_name} + ETCD_DATA_DIR=/var/lib/etcd + ETCD_ADVERTISE_CLIENT_URLS=https://${etcd_domain}:2379 + ETCD_INITIAL_ADVERTISE_PEER_URLS=https://${etcd_domain}:2380 + ETCD_LISTEN_CLIENT_URLS=https://0.0.0.0:2379 + ETCD_LISTEN_PEER_URLS=https://0.0.0.0:2380 + ETCD_LISTEN_METRICS_URLS=http://0.0.0.0:2381 + ETCD_INITIAL_CLUSTER=${etcd_initial_cluster} + ETCD_STRICT_RECONFIG_CHECK=true + ETCD_TRUSTED_CA_FILE=/etc/ssl/certs/etcd/server-ca.crt + ETCD_CERT_FILE=/etc/ssl/certs/etcd/server.crt + ETCD_KEY_FILE=/etc/ssl/certs/etcd/server.key + ETCD_CLIENT_CERT_AUTH=true + ETCD_PEER_TRUSTED_CA_FILE=/etc/ssl/certs/etcd/peer-ca.crt + ETCD_PEER_CERT_FILE=/etc/ssl/certs/etcd/peer.crt + ETCD_PEER_KEY_FILE=/etc/ssl/certs/etcd/peer.key + ETCD_PEER_CLIENT_CERT_AUTH=true diff --git a/digital-ocean/fedora-coreos/kubernetes/fcc/worker.yaml b/digital-ocean/fedora-coreos/kubernetes/fcc/worker.yaml new file mode 100644 index 000000000..3d2f48e34 --- /dev/null +++ b/digital-ocean/fedora-coreos/kubernetes/fcc/worker.yaml @@ -0,0 +1,117 @@ +--- +variant: fcos +version: 1.0.0 +systemd: + units: + - name: docker.service + enabled: true + - name: wait-for-dns.service + enabled: true + contents: | + [Unit] + Description=Wait for DNS entries + Before=kubelet.service + [Service] + Type=oneshot + RemainAfterExit=true + ExecStart=/bin/sh -c 'while ! /usr/bin/grep '^[^#[:space:]]' /etc/resolv.conf > /dev/null; do sleep 1; done' + [Install] + RequiredBy=kubelet.service + - name: kubelet.service + enabled: true + contents: | + [Unit] + Description=Kubelet via Hyperkube (System Container) + Requires=afterburn.service + After=afterburn.service + Wants=rpc-statd.service + [Service] + EnvironmentFile=/run/metadata/afterburn + ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d + ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests + ExecStartPre=/bin/mkdir -p /opt/cni/bin + ExecStartPre=/bin/mkdir -p /var/lib/calico + ExecStartPre=/bin/mkdir -p /var/lib/kubelet/volumeplugins + ExecStartPre=/usr/bin/bash -c "grep 'certificate-authority-data' /etc/kubernetes/kubeconfig | awk '{print $2}' | base64 -d > /etc/kubernetes/ca.crt" + ExecStartPre=-/usr/bin/podman rm kubelet + ExecStart=/usr/bin/podman run --name kubelet \ + --privileged \ + --pid host \ + --network host \ + --volume /etc/kubernetes:/etc/kubernetes:ro,z \ + --volume /usr/lib/os-release:/etc/os-release:ro \ + --volume /etc/ssl/certs:/etc/ssl/certs:ro \ + --volume /lib/modules:/lib/modules:ro \ + --volume /run:/run \ + --volume /sys/fs/cgroup:/sys/fs/cgroup:ro \ + --volume /sys/fs/cgroup/systemd:/sys/fs/cgroup/systemd \ + --volume /etc/pki/tls/certs:/usr/share/ca-certificates:ro \ + --volume /var/lib/calico:/var/lib/calico:ro \ + --volume /var/lib/docker:/var/lib/docker \ + --volume /var/lib/kubelet:/var/lib/kubelet:rshared,z \ + --volume /var/log:/var/log \ + --volume /var/run/lock:/var/run/lock:z \ + --volume /opt/cni/bin:/opt/cni/bin:z \ + quay.io/poseidon/kubelet:v1.18.1 \ + --anonymous-auth=false \ + --authentication-token-webhook \ + --authorization-mode=Webhook \ + --cgroup-driver=systemd \ + --cgroups-per-qos=true \ + --enforce-node-allocatable=pods \ + --client-ca-file=/etc/kubernetes/ca.crt \ + --cluster_dns=${cluster_dns_service_ip} \ + --cluster_domain=${cluster_domain_suffix} \ + --cni-conf-dir=/etc/kubernetes/cni/net.d \ + --exit-on-lock-contention \ + --healthz-port=0 \ + --hostname-override=$${AFTERBURN_DIGITALOCEAN_IPV4_PRIVATE_0} \ + --kubeconfig=/etc/kubernetes/kubeconfig \ + --lock-file=/var/run/lock/kubelet.lock \ + --network-plugin=cni \ + --node-labels=node.kubernetes.io/node \ + --pod-manifest-path=/etc/kubernetes/manifests \ + --read-only-port=0 \ + --volume-plugin-dir=/var/lib/kubelet/volumeplugins + ExecStop=-/usr/bin/podman stop kubelet + Delegate=yes + Restart=always + RestartSec=10 + [Install] + WantedBy=multi-user.target + - name: kubelet.path + enabled: true + contents: | + [Unit] + Description=Watch for kubeconfig + [Path] + PathExists=/etc/kubernetes/kubeconfig + [Install] + WantedBy=multi-user.target + - name: delete-node.service + enabled: true + contents: | + [Unit] + Description=Delete Kubernetes node on shutdown + [Service] + Type=oneshot + RemainAfterExit=true + ExecStart=/bin/true + ExecStop=/bin/bash -c '/usr/bin/podman run --volume /etc/kubernetes:/etc/kubernetes:ro,z --entrypoint /usr/local/bin/kubectl quay.io/poseidon/kubelet:v1.18.1 --kubeconfig=/etc/kubernetes/kubeconfig delete node $HOSTNAME' + [Install] + WantedBy=multi-user.target +storage: + directories: + - path: /etc/kubernetes + files: + - path: /etc/sysctl.d/max-user-watches.conf + contents: + inline: | + fs.inotify.max_user_watches=16184 + - path: /etc/systemd/system.conf.d/accounting.conf + contents: + inline: | + [Manager] + DefaultCPUAccounting=yes + DefaultMemoryAccounting=yes + DefaultBlockIOAccounting=yes diff --git a/digital-ocean/fedora-coreos/kubernetes/network.tf b/digital-ocean/fedora-coreos/kubernetes/network.tf new file mode 100644 index 000000000..bc5434852 --- /dev/null +++ b/digital-ocean/fedora-coreos/kubernetes/network.tf @@ -0,0 +1,117 @@ +resource "digitalocean_firewall" "rules" { + name = var.cluster_name + + tags = ["${var.cluster_name}-controller", "${var.cluster_name}-worker"] + + # allow ssh, internal flannel, internal node-exporter, internal kubelet + inbound_rule { + protocol = "tcp" + port_range = "22" + source_addresses = ["0.0.0.0/0", "::/0"] + } + + inbound_rule { + protocol = "udp" + port_range = "4789" + source_tags = [digitalocean_tag.controllers.name, digitalocean_tag.workers.name] + } + + # Allow Prometheus to scrape node-exporter + inbound_rule { + protocol = "tcp" + port_range = "9100" + source_tags = [digitalocean_tag.workers.name] + } + + # Allow Prometheus to scrape kube-proxy + inbound_rule { + protocol = "tcp" + port_range = "10249" + source_tags = [digitalocean_tag.workers.name] + } + + inbound_rule { + protocol = "tcp" + port_range = "10250" + source_tags = [digitalocean_tag.controllers.name, digitalocean_tag.workers.name] + } + + # allow all outbound traffic + outbound_rule { + protocol = "tcp" + port_range = "1-65535" + destination_addresses = ["0.0.0.0/0", "::/0"] + } + + outbound_rule { + protocol = "udp" + port_range = "1-65535" + destination_addresses = ["0.0.0.0/0", "::/0"] + } + + outbound_rule { + protocol = "icmp" + port_range = "1-65535" + destination_addresses = ["0.0.0.0/0", "::/0"] + } +} + +resource "digitalocean_firewall" "controllers" { + name = "${var.cluster_name}-controllers" + + tags = ["${var.cluster_name}-controller"] + + # etcd + inbound_rule { + protocol = "tcp" + port_range = "2379-2380" + source_tags = [digitalocean_tag.controllers.name] + } + + # etcd metrics + inbound_rule { + protocol = "tcp" + port_range = "2381" + source_tags = [digitalocean_tag.workers.name] + } + + # kube-apiserver + inbound_rule { + protocol = "tcp" + port_range = "6443" + source_addresses = ["0.0.0.0/0", "::/0"] + } + + # kube-scheduler metrics, kube-controller-manager metrics + inbound_rule { + protocol = "tcp" + port_range = "10251-10252" + source_tags = [digitalocean_tag.workers.name] + } +} + +resource "digitalocean_firewall" "workers" { + name = "${var.cluster_name}-workers" + + tags = ["${var.cluster_name}-worker"] + + # allow HTTP/HTTPS ingress + inbound_rule { + protocol = "tcp" + port_range = "80" + source_addresses = ["0.0.0.0/0", "::/0"] + } + + inbound_rule { + protocol = "tcp" + port_range = "443" + source_addresses = ["0.0.0.0/0", "::/0"] + } + + inbound_rule { + protocol = "tcp" + port_range = "10254" + source_addresses = ["0.0.0.0/0"] + } +} + diff --git a/digital-ocean/fedora-coreos/kubernetes/outputs.tf b/digital-ocean/fedora-coreos/kubernetes/outputs.tf new file mode 100644 index 000000000..429893c58 --- /dev/null +++ b/digital-ocean/fedora-coreos/kubernetes/outputs.tf @@ -0,0 +1,47 @@ +output "kubeconfig-admin" { + value = module.bootstrap.kubeconfig-admin +} + +output "controllers_dns" { + value = digitalocean_record.controllers[0].fqdn +} + +output "workers_dns" { + # Multiple A and AAAA records with the same FQDN + value = digitalocean_record.workers-record-a[0].fqdn +} + +output "controllers_ipv4" { + value = digitalocean_droplet.controllers.*.ipv4_address +} + +output "controllers_ipv6" { + value = digitalocean_droplet.controllers.*.ipv6_address +} + +output "workers_ipv4" { + value = digitalocean_droplet.workers.*.ipv4_address +} + +output "workers_ipv6" { + value = digitalocean_droplet.workers.*.ipv6_address +} + +# Outputs for worker pools + +output "kubeconfig" { + value = module.bootstrap.kubeconfig-kubelet +} + +# Outputs for custom firewalls + +output "controller_tag" { + description = "Tag applied to controller droplets" + value = digitalocean_tag.controllers.name +} + +output "worker_tag" { + description = "Tag applied to worker droplets" + value = digitalocean_tag.workers.name +} + diff --git a/digital-ocean/fedora-coreos/kubernetes/ssh.tf b/digital-ocean/fedora-coreos/kubernetes/ssh.tf new file mode 100644 index 000000000..f4888fe02 --- /dev/null +++ b/digital-ocean/fedora-coreos/kubernetes/ssh.tf @@ -0,0 +1,87 @@ +locals { + # format assets for distribution + assets_bundle = [ + # header with the unpack location + for key, value in module.bootstrap.assets_dist : + format("##### %s\n%s", key, value) + ] +} + +# Secure copy assets to controllers. Activates kubelet.service +resource "null_resource" "copy-controller-secrets" { + count = var.controller_count + + depends_on = [ + module.bootstrap, + digitalocean_firewall.rules + ] + + connection { + type = "ssh" + host = digitalocean_droplet.controllers.*.ipv4_address[count.index] + user = "core" + timeout = "15m" + } + + provisioner "file" { + content = module.bootstrap.kubeconfig-kubelet + destination = "$HOME/kubeconfig" + } + + provisioner "file" { + content = join("\n", local.assets_bundle) + destination = "$HOME/assets" + } + + provisioner "remote-exec" { + inline = [ + "sudo mv $HOME/kubeconfig /etc/kubernetes/kubeconfig", + "sudo /opt/bootstrap/layout", + ] + } +} + +# Secure copy kubeconfig to all workers. Activates kubelet.service. +resource "null_resource" "copy-worker-secrets" { + count = var.worker_count + + connection { + type = "ssh" + host = digitalocean_droplet.workers.*.ipv4_address[count.index] + user = "core" + timeout = "15m" + } + + provisioner "file" { + content = module.bootstrap.kubeconfig-kubelet + destination = "$HOME/kubeconfig" + } + + provisioner "remote-exec" { + inline = [ + "sudo mv $HOME/kubeconfig /etc/kubernetes/kubeconfig", + ] + } +} + +# Connect to a controller to perform one-time cluster bootstrap. +resource "null_resource" "bootstrap" { + depends_on = [ + null_resource.copy-controller-secrets, + null_resource.copy-worker-secrets, + ] + + connection { + type = "ssh" + host = digitalocean_droplet.controllers[0].ipv4_address + user = "core" + timeout = "15m" + } + + provisioner "remote-exec" { + inline = [ + "sudo systemctl start bootstrap", + ] + } +} + diff --git a/digital-ocean/fedora-coreos/kubernetes/variables.tf b/digital-ocean/fedora-coreos/kubernetes/variables.tf new file mode 100644 index 000000000..a27192338 --- /dev/null +++ b/digital-ocean/fedora-coreos/kubernetes/variables.tf @@ -0,0 +1,114 @@ +variable "cluster_name" { + type = string + description = "Unique cluster name (prepended to dns_zone)" +} + +# Digital Ocean + +variable "region" { + type = string + description = "Digital Ocean region (e.g. nyc1, sfo2, fra1, tor1)" +} + +variable "dns_zone" { + type = string + description = "Digital Ocean domain (i.e. DNS zone) (e.g. do.example.com)" +} + +# instances + +variable "controller_count" { + type = number + description = "Number of controllers (i.e. masters)" + default = 1 +} + +variable "worker_count" { + type = number + description = "Number of workers" + default = 1 +} + +variable "controller_type" { + type = string + description = "Droplet type for controllers (e.g. s-2vcpu-2gb, s-2vcpu-4gb, s-4vcpu-8gb)." + default = "s-2vcpu-2gb" +} + +variable "worker_type" { + type = string + description = "Droplet type for workers (e.g. s-1vcpu-2gb, s-2vcpu-2gb)" + default = "s-1vcpu-2gb" +} + +variable "os_image" { + type = string + description = "Fedora CoreOS image for instances" +} + +variable "controller_snippets" { + type = list(string) + description = "Controller Fedora CoreOS Config snippets" + default = [] +} + +variable "worker_snippets" { + type = list(string) + description = "Worker Fedora CoreOS Config snippets" + default = [] +} + +# configuration + +variable "ssh_fingerprints" { + type = list(string) + description = "SSH public key fingerprints. (e.g. see `ssh-add -l -E md5`)" +} + +variable "networking" { + type = string + description = "Choice of networking provider (flannel or calico)" + default = "calico" +} + +variable "pod_cidr" { + type = string + description = "CIDR IPv4 range to assign Kubernetes pods" + default = "10.2.0.0/16" +} + +variable "service_cidr" { + type = string + description = < ~/.config/digital-ocean/token +``` + +Configure the DigitalOcean provider to use your token in a `providers.tf` file. + +```tf +provider "digitalocean" { + version = "1.15.1" + token = "${chomp(file("~/.config/digital-ocean/token"))}" +} + +provider "ct" { + version = "0.5.0" +} +``` + +## Fedora CoreOS Images + +Fedora CoreOS publishes images for DigitalOcean, but does not yet upload them. DigitalOcean allows [custom images](https://blog.digitalocean.com/custom-images/) to be uploaded via URL or file. + +Import a [Fedora CoreOS](https://getfedora.org/en/coreos/download?tab=cloud_operators&stream=stable) image via URL to desired a region(s). Reference the DigitalOcean image and set the `os_image` in the next step. + +```tf +data "digitalocean_image" "fedora-coreos-31-20200323-3-2" { + name = "fedora-coreos-31.20200323.3.2-digitalocean.x86_64.qcow2.gz" +} +``` + +## Cluster + +Define a Kubernetes cluster using the module `digital-ocean/fedora-coreos/kubernetes`. + +```tf +module "nemo" { + source = "git::https://github.com/poseidon/typhoon//digital-ocean/fedora-coreos/kubernetes?ref=v1.18.1" + + # Digital Ocean + cluster_name = "nemo" + region = "nyc3" + dns_zone = "digital-ocean.example.com" + os_image = data.digitalocean_image.fedora-coreos-31-20200323-3-2.id + + # configuration + ssh_fingerprints = ["d7:9d:79:ae:56:32:73:79:95:88:e3:a2:ab:5d:45:e7"] + + # optional + worker_count = 2 +} +``` + +Reference the [variables docs](#variables) or the [variables.tf](https://github.com/poseidon/typhoon/blob/master/digital-ocean/fedora-coreos/kubernetes/variables.tf) source. + +## ssh-agent + +Initial bootstrapping requires `bootstrap.service` be started on one controller node. Terraform uses `ssh-agent` to automate this step. Add your SSH private key to `ssh-agent`. + +```sh +ssh-add ~/.ssh/id_rsa +ssh-add -L +``` + +## Apply + +Initialize the config directory if this is the first use with Terraform. + +```sh +terraform init +``` + +Plan the resources to be created. + +```sh +$ terraform plan +Plan: 54 to add, 0 to change, 0 to destroy. +``` + +Apply the changes to create the cluster. + +```sh +$ terraform apply +module.nemo.null_resource.bootstrap: Still creating... (30s elapsed) +module.nemo.null_resource.bootstrap: Provisioning with 'remote-exec'... +... +module.nemo.null_resource.bootstrap: Still creating... (6m20s elapsed) +module.nemo.null_resource.bootstrap: Creation complete (ID: 7599298447329218468) + +Apply complete! Resources: 42 added, 0 changed, 0 destroyed. +``` + +In 3-6 minutes, the Kubernetes cluster will be ready. + +## Verify + +[Install kubectl](https://kubernetes.io/docs/tasks/tools/install-kubectl/) on your system. Obtain the generated cluster `kubeconfig` from module outputs (e.g. write to a local file). + +``` +resource "local_file" "kubeconfig-nemo" { + content = module.nemo.kubeconfig-admin + filename = "/home/user/.kube/configs/nemo-config" +} +``` + +List nodes in the cluster. + +``` +$ export KUBECONFIG=/home/user/.kube/configs/nemo-config +$ kubectl get nodes +NAME STATUS ROLES AGE VERSION +10.132.110.130 Ready 10m v1.18.1 +10.132.115.81 Ready 10m v1.18.1 +10.132.124.107 Ready 10m v1.18.1 +``` + +List the pods. + +``` +NAMESPACE NAME READY STATUS RESTARTS AGE +kube-system coredns-1187388186-ld1j7 1/1 Running 0 11m +kube-system coredns-1187388186-rdhf7 1/1 Running 0 11m +kube-system calico-node-1m5bf 2/2 Running 0 11m +kube-system calico-node-7jmr1 2/2 Running 0 11m +kube-system calico-node-bknc8 2/2 Running 0 11m +kube-system kube-apiserver-ip-10.132.115.81 1/1 Running 0 11m +kube-system kube-controller-manager-ip-10.132.115.81 1/1 Running 0 11m +kube-system kube-proxy-6kxjf 1/1 Running 0 11m +kube-system kube-proxy-fh3td 1/1 Running 0 11m +kube-system kube-proxy-k35rc 1/1 Running 0 11m +kube-system kube-scheduler-ip-10.132.115.81 1/1 Running 0 11m +``` + +## Going Further + +Learn about [maintenance](/topics/maintenance/) and [addons](/addons/overview/). + +## Variables + +Check the [variables.tf](https://github.com/poseidon/typhoon/blob/master/digital-ocean/fedora-coreos/kubernetes/variables.tf) source. + +### Required + +| Name | Description | Example | +|:-----|:------------|:--------| +| cluster_name | Unique cluster name (prepended to dns_zone) | "nemo" | +| region | Digital Ocean region | "nyc1", "sfo2", "fra1", tor1" | +| dns_zone | Digital Ocean domain (i.e. DNS zone) | "do.example.com" | +| os_image | Fedora CoreOS image for instances | "custom-image-id" | +| ssh_fingerprints | SSH public key fingerprints | ["d7:9d..."] | + +#### DNS Zone + +Clusters create DNS A records `${cluster_name}.${dns_zone}` to resolve to controller droplets (round robin). This FQDN is used by workers and `kubectl` to access the apiserver(s). In this example, the cluster's apiserver would be accessible at `nemo.do.example.com`. + +You'll need a registered domain name or delegated subdomain in DigitalOcean Domains (i.e. DNS zones). You can set this up once and create many clusters with unique names. + +```tf +# Declare a DigitalOcean record to also create a zone file +resource "digitalocean_domain" "zone-for-clusters" { + name = "do.example.com" + ip_address = "8.8.8.8" +} +``` + +!!! tip "" + If you have an existing domain name with a zone file elsewhere, just delegate a subdomain that can be managed on DigitalOcean (e.g. do.mydomain.com) and [update nameservers](https://www.digitalocean.com/community/tutorials/how-to-set-up-a-host-name-with-digitalocean). + +#### SSH Fingerprints + +DigitalOcean droplets are created with your SSH public key "fingerprint" (i.e. MD5 hash) to allow access. If your SSH public key is at `~/.ssh/id_rsa`, find the fingerprint with, + +```bash +ssh-keygen -E md5 -lf ~/.ssh/id_rsa.pub | awk '{print $2}' +MD5:d7:9d:79:ae:56:32:73:79:95:88:e3:a2:ab:5d:45:e7 +``` + +If you use `ssh-agent` (e.g. Yubikey for SSH), find the fingerprint with, + +``` +ssh-add -l -E md5 +2048 MD5:d7:9d:79:ae:56:32:73:79:95:88:e3:a2:ab:5d:45:e7 cardno:000603633110 (RSA) +``` + +Digital Ocean requires the SSH public key be uploaded to your account, so you may also find the fingerprint under Settings -> Security. Finally, if you don't have an SSH key, [create one now](https://help.github.com/articles/generating-a-new-ssh-key-and-adding-it-to-the-ssh-agent/). + +### Optional + +| Name | Description | Default | Example | +|:-----|:------------|:--------|:--------| +| controller_count | Number of controllers (i.e. masters) | 1 | 1 | +| worker_count | Number of workers | 1 | 3 | +| controller_type | Droplet type for controllers | "s-2vcpu-2gb" | s-2vcpu-2gb, s-2vcpu-4gb, s-4vcpu-8gb, ... | +| worker_type | Droplet type for workers | "s-1vcpu-2gb" | s-1vcpu-2gb, s-2vcpu-2gb, ... | +| controller_snippets | Controller Fedora CoreOS Config snippets | [] | [example](/advanced/customization/) | +| worker_snippets | Worker Fedora CoreOS Config snippets | [] | [example](/advanced/customization/) | +| networking | Choice of networking provider | "calico" | "flannel" or "calico" | +| pod_cidr | CIDR IPv4 range to assign to Kubernetes pods | "10.2.0.0/16" | "10.22.0.0/16" | +| service_cidr | CIDR IPv4 range to assign to Kubernetes services | "10.3.0.0/16" | "10.3.0.0/24" | + +Check the list of valid [droplet types](https://developers.digitalocean.com/documentation/changelog/api-v2/new-size-slugs-for-droplet-plan-changes/) or use `doctl compute size list`. + +!!! warning + Do not choose a `controller_type` smaller than 2GB. Smaller droplets are not sufficient for running a controller and bootstrapping will fail. + diff --git a/docs/fedora-coreos/google-cloud.md b/docs/fedora-coreos/google-cloud.md index 295f1a197..7ebcd2e18 100644 --- a/docs/fedora-coreos/google-cloud.md +++ b/docs/fedora-coreos/google-cloud.md @@ -73,13 +73,13 @@ Fedora CoreOS publishes images for Google Cloud, but does not yet upload them. G ``` gsutil list -gsutil cp fedora-coreos-31.20200310.3.0-gcp.x86_64.tar.gz gs://BUCKET +gsutil cp fedora-coreos-31.20200323.3.2-gcp.x86_64.tar.gz gs://BUCKET ``` Create a Compute Engine image from the file. ``` -gcloud compute images create fedora-coreos-31-20200310-3-0 --source-uri gs://BUCKET/fedora-coreos-31.20200310.3.0-gcp.x86_64.tar.gz +gcloud compute images create fedora-coreos-31-20200323-3-2 --source-uri gs://BUCKET/fedora-coreos-31.20200323.3.2-gcp.x86_64.tar.gz ``` ## Cluster @@ -97,7 +97,7 @@ module "yavin" { dns_zone_name = "example-zone" # custom image name from above - os_image = "fedora-coreos-31-20200310-3-0" + os_image = "fedora-coreos-31-20200323-3-2" # configuration ssh_authorized_key = "ssh-rsa AAAAB3Nz..." @@ -107,7 +107,7 @@ module "yavin" { } ``` -Reference the [variables docs](#variables) or the [variables.tf](https://github.com/poseidon/typhoon/blob/master/google-cloud/container-linux/kubernetes/variables.tf) source. +Reference the [variables docs](#variables) or the [variables.tf](https://github.com/poseidon/typhoon/blob/master/google-cloud/fedora-coreos/kubernetes/variables.tf) source. ## ssh-agent @@ -194,7 +194,7 @@ Learn about [maintenance](/topics/maintenance/) and [addons](/addons/overview/). ## Variables -Check the [variables.tf](https://github.com/poseidon/typhoon/blob/master/google-cloud/container-linux/kubernetes/variables.tf) source. +Check the [variables.tf](https://github.com/poseidon/typhoon/blob/master/google-cloud/fedora-coreos/kubernetes/variables.tf) source. ### Required @@ -204,6 +204,7 @@ Check the [variables.tf](https://github.com/poseidon/typhoon/blob/master/google- | region | Google Cloud region | "us-central1" | | dns_zone | Google Cloud DNS zone | "google-cloud.example.com" | | dns_zone_name | Google Cloud DNS zone name | "example-zone" | +| os_image | Fedora CoreOS image for compute instances | "fedora-coreos-31-20200323-3-2" | | ssh_authorized_key | SSH public key for user 'core' | "ssh-rsa AAAAB3NZ..." | Check the list of valid [regions](https://cloud.google.com/compute/docs/regions-zones/regions-zones) and list Fedora CoreOS [images](https://cloud.google.com/compute/docs/images) with `gcloud compute images list | grep fedora-coreos`. @@ -233,7 +234,6 @@ resource "google_dns_managed_zone" "zone-for-clusters" { | worker_count | Number of workers | 1 | 3 | | controller_type | Machine type for controllers | "n1-standard-1" | See below | | worker_type | Machine type for workers | "n1-standard-1" | See below | -| os_image | Fedora CoreOS image for compute instances | "" | "fedora-coreos-31-20200113-3-1" | | disk_size | Size of the disk in GB | 40 | 100 | | worker_preemptible | If enabled, Compute Engine will terminate workers randomly within 24 hours | false | true | | controller_snippets | Controller Fedora CoreOS Config snippets | [] | [examples](/advanced/customization/) | diff --git a/docs/index.md b/docs/index.md index 279d60f22..16a307299 100644 --- a/docs/index.md +++ b/docs/index.md @@ -35,6 +35,7 @@ Typhoon is available for [Fedora CoreOS](https://getfedora.org/coreos/). |---------------|------------------|------------------|--------| | AWS | Fedora CoreOS | [aws/fedora-coreos/kubernetes](fedora-coreos/aws.md) | stable | | Bare-Metal | Fedora CoreOS | [bare-metal/fedora-coreos/kubernetes](fedora-coreos/bare-metal.md) | beta | +| DigitalOcean | Fedora CoreOS | [digital-ocean/fedora-coreos/kubernetes](fedora-coreos/digitalocean.md) | alpha | | Google Cloud | Fedora CoreOS | [google-cloud/fedora-coreos/kubernetes](google-cloud/fedora-coreos/kubernetes) | beta | Typhoon is available for [Flatcar Container Linux](https://www.flatcar-linux.org/releases/). @@ -44,13 +45,13 @@ Typhoon is available for [Flatcar Container Linux](https://www.flatcar-linux.org | AWS | Flatcar Linux | [aws/container-linux/kubernetes](cl/aws.md) | stable | | Azure | Flatcar Linux | [azure/container-linux/kubernetes](cl/azure.md) | alpha | | Bare-Metal | Flatcar Linux | [bare-metal/container-linux/kubernetes](cl/bare-metal.md) | stable | +| DigitalOcean | Flatcar Linux | [digital-ocean/container-linux/kubernetes](cl/digital-ocean.md) | alpha | | Google Cloud | Flatcar Linux | [google-cloud/container-linux/kubernetes](cl/google-cloud.md) | alpha | -| Digital Ocean | Flatcar Linux | [digital-ocean/container-linux/kubernetes](cl/digital-ocean.md) | alpha | ## Documentation * Architecture [concepts](architecture/concepts.md) and [operating-systems](architecture/operating-systems.md) -* Fedora CoreOS tutorials for [AWS](fedora-coreos/aws.md), [Bare-Metal](fedora-coreos/bare-metal.md), and [Google Cloud](fedora-coreos/google-cloud.md) +* Fedora CoreOS tutorials for [AWS](fedora-coreos/aws.md), [Bare-Metal](fedora-coreos/bare-metal.md), [DigitalOcean](fedora-coreos/digitalocean.md), and [Google Cloud](fedora-coreos/google-cloud.md) * Flatcar Linux tutorials for [AWS](cl/aws.md), [Azure](cl/azure.md), [Bare-Metal](cl/bare-metal.md), [DigitalOcean](cl/digital-ocean.md), and [Google Cloud](cl/google-cloud.md) ## Example From 1420700bc0a46b674533f6cd92138b9cf18cf2de Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Sat, 11 Apr 2020 13:18:26 -0700 Subject: [PATCH 388/523] Update CHANGES for v1.18.1 release * Change order of modules in the README --- CHANGES.md | 4 ++++ README.md | 18 ++++++++++-------- docs/index.md | 19 +++++++++++-------- 3 files changed, 25 insertions(+), 16 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 6957ee58d..3a5a2a8c4 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -7,6 +7,9 @@ Notable changes between versions. ## v1.18.1 * Kubernetes [v1.18.1](https://github.com/kubernetes/kubernetes/blob/master/CHANGELOG/CHANGELOG-1.18.md#v1181) +* Choose Fedora CoreOS or Flatcar Linux (**action recommended**) + * Use a `fedora-coreos` module for Fedora CoreOS + * Use a `container-linux` module with OS set to Flatcar Linux * Update etcd from v3.4.5 to [v3.4.7](https://github.com/etcd-io/etcd/releases/tag/v3.4.7) * Change `kube-proxy` and `calico` or `flannel` to tolerate specific taints ([#682](https://github.com/poseidon/typhoon/pull/682)) * Tolerate master and not-ready taints, rather than tolerating all taints @@ -16,6 +19,7 @@ Notable changes between versions. * Rename Container Linux `controller_clc_snippets` to `controller_snippets` for consistency ([#688](https://github.com/poseidon/typhoon/pull/688)) * Rename Container Linux `worker_clc_snippets` to `worker_snippets` for consistency * Rename Container Linux `clc_snippets` (bare-metal) to `snippets` for consistency +* Drop support for [gitRepo](https://kubernetes.io/docs/concepts/storage/volumes/#gitrepo) volumes #### Azure diff --git a/README.md b/README.md index 079f5f250..93520d596 100644 --- a/README.md +++ b/README.md @@ -21,14 +21,6 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster Typhoon provides a Terraform Module for each supported operating system and platform. -| Platform | Operating System | Terraform Module | Status | -|---------------|------------------|------------------|--------| -| AWS | Container Linux | [aws/container-linux/kubernetes](aws/container-linux/kubernetes) | stable | -| Azure | Container Linux | [azure/container-linux/kubernetes](azure/container-linux/kubernetes) | alpha | -| Bare-Metal | Container Linux | [bare-metal/container-linux/kubernetes](bare-metal/container-linux/kubernetes) | stable | -| Digital Ocean | Container Linux | [digital-ocean/container-linux/kubernetes](digital-ocean/container-linux/kubernetes) | beta | -| Google Cloud | Container Linux | [google-cloud/container-linux/kubernetes](google-cloud/container-linux/kubernetes) | stable | - Typhoon is available for [Fedora CoreOS](https://getfedora.org/coreos/). | Platform | Operating System | Terraform Module | Status | @@ -48,6 +40,16 @@ Typhoon is available for [Flatcar Container Linux](https://www.flatcar-linux.org | DigitalOcean | Flatcar Linux | [digital-ocean/container-linux/kubernetes](digital-ocean/container-linux/kubernetes) | alpha | | Google Cloud | Flatcar Linux | [google-cloud/container-linux/kubernetes](google-cloud/container-linux/kubernetes) | alpha | +Typhoon is available for CoreOS Container Linux ([no updates](https://coreos.com/os/eol/) after May 2020). + +| Platform | Operating System | Terraform Module | Status | +|---------------|------------------|------------------|--------| +| AWS | Container Linux | [aws/container-linux/kubernetes](aws/container-linux/kubernetes) | stable | +| Azure | Container Linux | [azure/container-linux/kubernetes](azure/container-linux/kubernetes) | alpha | +| Bare-Metal | Container Linux | [bare-metal/container-linux/kubernetes](bare-metal/container-linux/kubernetes) | stable | +| Digital Ocean | Container Linux | [digital-ocean/container-linux/kubernetes](digital-ocean/container-linux/kubernetes) | beta | +| Google Cloud | Container Linux | [google-cloud/container-linux/kubernetes](google-cloud/container-linux/kubernetes) | stable | + ## Documentation * [Docs](https://typhoon.psdn.io) diff --git a/docs/index.md b/docs/index.md index 16a307299..f3407483a 100644 --- a/docs/index.md +++ b/docs/index.md @@ -21,14 +21,6 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster Typhoon provides a Terraform Module for each supported operating system and platform. -| Platform | Operating System | Terraform Module | Status | -|---------------|------------------|------------------|--------| -| AWS | Container Linux | [aws/container-linux/kubernetes](cl/aws.md) | stable | -| Azure | Container Linux | [azure/container-linux/kubernetes](cl/azure.md) | alpha | -| Bare-Metal | Container Linux | [bare-metal/container-linux/kubernetes](cl/bare-metal.md) | stable | -| Digital Ocean | Container Linux | [digital-ocean/container-linux/kubernetes](cl/digital-ocean.md) | beta | -| Google Cloud | Container Linux | [google-cloud/container-linux/kubernetes](cl/google-cloud.md) | stable | - Typhoon is available for [Fedora CoreOS](https://getfedora.org/coreos/). | Platform | Operating System | Terraform Module | Status | @@ -48,6 +40,17 @@ Typhoon is available for [Flatcar Container Linux](https://www.flatcar-linux.org | DigitalOcean | Flatcar Linux | [digital-ocean/container-linux/kubernetes](cl/digital-ocean.md) | alpha | | Google Cloud | Flatcar Linux | [google-cloud/container-linux/kubernetes](cl/google-cloud.md) | alpha | +Typhoon is available for CoreOS Container Linux ([no updates](https://coreos.com/os/eol/) after May 2020). + +| Platform | Operating System | Terraform Module | Status | +|---------------|------------------|------------------|--------| +| AWS | Container Linux | [aws/container-linux/kubernetes](cl/aws.md) | stable | +| Azure | Container Linux | [azure/container-linux/kubernetes](cl/azure.md) | alpha | +| Bare-Metal | Container Linux | [bare-metal/container-linux/kubernetes](cl/bare-metal.md) | stable | +| Digital Ocean | Container Linux | [digital-ocean/container-linux/kubernetes](cl/digital-ocean.md) | beta | +| Google Cloud | Container Linux | [google-cloud/container-linux/kubernetes](cl/google-cloud.md) | stable | + + ## Documentation * Architecture [concepts](architecture/concepts.md) and [operating-systems](architecture/operating-systems.md) From 1627ecaf273cc10f9fcc38104987a66bc3878d2b Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Sat, 11 Apr 2020 14:07:09 -0700 Subject: [PATCH 389/523] Fix docs TOC to include Fedora CoreOS DigitalOcean --- mkdocs.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/mkdocs.yml b/mkdocs.yml index ae9f3fcb7..afca2372e 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -56,6 +56,7 @@ nav: - 'Fedora CoreOS': - 'AWS': 'fedora-coreos/aws.md' - 'Bare-Metal': 'fedora-coreos/bare-metal.md' + - 'Digital Ocean': 'fedora-coreos/digitalocean.md' - 'Google Cloud': 'fedora-coreos/google-cloud.md' - 'Container Linux': - 'AWS': 'cl/aws.md' From 76ab4c4c2a9c9727b45b7d1c657c8da55e0afeb9 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Sat, 11 Apr 2020 14:52:30 -0700 Subject: [PATCH 390/523] Change `container-linux` module preference to Flatcar Linux * No change to Fedora CoreOS modules * For Container Linx AWS and Azure, change the `os_image` default from coreos-stable to flatcar-stable * For Container Linux GCP and DigitalOcean, change `os_image` to be required since users should upload a Flatcar Linux image and set the variable * For Container Linux bare-metal, recommend users change the `os_channel` to Flatcar Linux. No actual module change. --- CHANGES.md | 25 +++++++++++ aws/container-linux/kubernetes/variables.tf | 14 +++--- .../kubernetes/workers/variables.tf | 2 +- aws/fedora-coreos/kubernetes/variables.tf | 2 +- .../kubernetes/workers/variables.tf | 2 +- azure/container-linux/kubernetes/variables.tf | 14 +++--- .../kubernetes/workers/variables.tf | 4 +- .../container-linux/kubernetes/variables.tf | 12 ++--- .../container-linux/kubernetes/variables.tf | 15 +++---- docs/advanced/worker-pools.md | 24 +++++----- docs/cl/aws.md | 2 +- docs/cl/azure.md | 2 +- docs/cl/bare-metal.md | 6 +-- docs/cl/digital-ocean.md | 40 +++++++---------- docs/cl/google-cloud.md | 44 +++++++++---------- .../container-linux/kubernetes/variables.tf | 15 +++---- .../kubernetes/workers/variables.tf | 1 - 17 files changed, 119 insertions(+), 105 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 3a5a2a8c4..548dfae91 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -4,6 +4,31 @@ Notable changes between versions. ## Latest +* Choose Fedora CoreOS or Flatcar Linux (**action required**) + * Use a `fedora-coreos` module for Fedora CoreOS + * Use a `container-linux` module for Flatcar Linux + * CoreOS Container Linux [won't receive updates](https://coreos.com/os/eol/) after May 2020 + +#### AWS + +* Change Container Linux `os_image` default from `coreos-stable` to `flatcar-stable` + +#### Azure + +* Change Container Linux `os_image` default from `coreos-stable` to `flatcar-stable` + +#### Bare-Metal + +* Container Linux users should change [os_channel](https://typhoon.psdn.io/cl/bare-metal/#required) from a CoreOS channel to a Flatcar channel + +#### Google + +* Change Container Linux `os_image` to be required. Container Linux users should upload a Flatcar Linux image and set it (**action required**) + +#### DigitalOcean + +* Change Container Linux `os_image` to be required. Container Linux users should upload a Flatcar Linux image and set it (**action required**) + ## v1.18.1 * Kubernetes [v1.18.1](https://github.com/kubernetes/kubernetes/blob/master/CHANGELOG/CHANGELOG-1.18.md#v1181) diff --git a/aws/container-linux/kubernetes/variables.tf b/aws/container-linux/kubernetes/variables.tf index bd9dec40c..7cc63f2bb 100644 --- a/aws/container-linux/kubernetes/variables.tf +++ b/aws/container-linux/kubernetes/variables.tf @@ -44,7 +44,7 @@ variable "worker_type" { variable "os_image" { type = string description = "AMI channel for a Container Linux derivative (coreos-stable, coreos-beta, coreos-alpha, flatcar-stable, flatcar-beta, flatcar-alpha, flatcar-edge)" - default = "coreos-stable" + default = "flatcar-stable" } variable "disk_size" { @@ -96,12 +96,6 @@ variable "ssh_authorized_key" { description = "SSH public key for user 'core'" } -variable "asset_dir" { - type = string - description = "Absolute path to a directory where generated assets should be placed (contains secrets)" - default = "" -} - variable "networking" { type = string description = "Choice of networking provider (calico or flannel)" @@ -155,6 +149,12 @@ variable "worker_node_labels" { # unofficial, undocumented, unsupported +variable "asset_dir" { + type = string + description = "Absolute path to a directory where generated assets should be placed (contains secrets)" + default = "" +} + variable "cluster_domain_suffix" { type = string description = "Queries for domains with the suffix will be answered by CoreDNS. Default is cluster.local (e.g. foo.default.svc.cluster.local)" diff --git a/aws/container-linux/kubernetes/workers/variables.tf b/aws/container-linux/kubernetes/workers/variables.tf index effd650f2..6f0d3a0a2 100644 --- a/aws/container-linux/kubernetes/workers/variables.tf +++ b/aws/container-linux/kubernetes/workers/variables.tf @@ -37,7 +37,7 @@ variable "instance_type" { variable "os_image" { type = string description = "AMI channel for a Container Linux derivative (coreos-stable, coreos-beta, coreos-alpha, flatcar-stable, flatcar-beta, flatcar-alpha, flatcar-edge)" - default = "coreos-stable" + default = "flatcar-stable" } variable "disk_size" { diff --git a/aws/fedora-coreos/kubernetes/variables.tf b/aws/fedora-coreos/kubernetes/variables.tf index 890ad2887..13284f634 100644 --- a/aws/fedora-coreos/kubernetes/variables.tf +++ b/aws/fedora-coreos/kubernetes/variables.tf @@ -44,7 +44,7 @@ variable "worker_type" { variable "os_image" { type = string description = "AMI channel for Fedora CoreOS (not yet used)" - default = "coreos-stable" + default = "stable" } variable "disk_size" { diff --git a/aws/fedora-coreos/kubernetes/workers/variables.tf b/aws/fedora-coreos/kubernetes/workers/variables.tf index 4128ee675..6c21d0a05 100644 --- a/aws/fedora-coreos/kubernetes/workers/variables.tf +++ b/aws/fedora-coreos/kubernetes/workers/variables.tf @@ -37,7 +37,7 @@ variable "instance_type" { variable "os_image" { type = string description = "AMI channel for Fedora CoreOS (not yet used)" - default = "coreos-stable" + default = "stable" } variable "disk_size" { diff --git a/azure/container-linux/kubernetes/variables.tf b/azure/container-linux/kubernetes/variables.tf index 49ff63b0b..96eae93ec 100644 --- a/azure/container-linux/kubernetes/variables.tf +++ b/azure/container-linux/kubernetes/variables.tf @@ -48,8 +48,8 @@ variable "worker_type" { variable "os_image" { type = string - default = "coreos-stable" description = "Channel for a Container Linux derivative (coreos-stable, coreos-beta, coreos-alpha, flatcar-stable, flatcar-beta)" + default = "flatcar-stable" } variable "disk_size" { @@ -83,12 +83,6 @@ variable "ssh_authorized_key" { description = "SSH public key for user 'core'" } -variable "asset_dir" { - type = string - description = "Absolute path to a directory where generated assets should be placed (contains secrets)" - default = "" -} - variable "networking" { type = string description = "Choice of networking provider (flannel or calico)" @@ -136,6 +130,12 @@ variable "worker_node_labels" { # unofficial, undocumented, unsupported +variable "asset_dir" { + type = string + description = "Absolute path to a directory where generated assets should be placed (contains secrets)" + default = "" +} + variable "cluster_domain_suffix" { type = string description = "Queries for domains with the suffix will be answered by coredns. Default is cluster.local (e.g. foo.default.svc.cluster.local) " diff --git a/azure/container-linux/kubernetes/workers/variables.tf b/azure/container-linux/kubernetes/workers/variables.tf index 3f834dfe2..06ab97011 100644 --- a/azure/container-linux/kubernetes/workers/variables.tf +++ b/azure/container-linux/kubernetes/workers/variables.tf @@ -46,8 +46,8 @@ variable "vm_type" { variable "os_image" { type = string - description = "Channel for a Container Linux derivative (coreos-stable, coreos-beta, coreos-alpha)" - default = "coreos-stable" + description = "Channel for a Container Linux derivative (flatcar-stable, flatcar-beta, coreos-stable, coreos-beta, coreos-alpha)" + default = "flatcar-stable" } variable "priority" { diff --git a/bare-metal/container-linux/kubernetes/variables.tf b/bare-metal/container-linux/kubernetes/variables.tf index 62cf47c29..9905229ae 100644 --- a/bare-metal/container-linux/kubernetes/variables.tf +++ b/bare-metal/container-linux/kubernetes/variables.tf @@ -79,12 +79,6 @@ variable "ssh_authorized_key" { description = "SSH public key for user 'core'" } -variable "asset_dir" { - type = string - description = "Absolute path to a directory where generated assets should be placed (contains secrets)" - default = "" -} - variable "networking" { type = string description = "Choice of networking provider (flannel or calico)" @@ -158,6 +152,12 @@ variable "enable_aggregation" { # unofficial, undocumented, unsupported +variable "asset_dir" { + type = string + description = "Absolute path to a directory where generated assets should be placed (contains secrets)" + default = "" +} + variable "cluster_domain_suffix" { type = string description = "Queries for domains with the suffix will be answered by coredns. Default is cluster.local (e.g. foo.default.svc.cluster.local) " diff --git a/digital-ocean/container-linux/kubernetes/variables.tf b/digital-ocean/container-linux/kubernetes/variables.tf index c87d99b5f..67ba99ffa 100644 --- a/digital-ocean/container-linux/kubernetes/variables.tf +++ b/digital-ocean/container-linux/kubernetes/variables.tf @@ -43,8 +43,7 @@ variable "worker_type" { variable "os_image" { type = string - description = "Container Linux image for instances (e.g. coreos-stable)" - default = "coreos-stable" + description = "Container Linux image for instances (e.g. coreos-stable, custom-image-id)" } variable "controller_snippets" { @@ -66,12 +65,6 @@ variable "ssh_fingerprints" { description = "SSH public key fingerprints. (e.g. see `ssh-add -l -E md5`)" } -variable "asset_dir" { - type = string - description = "Absolute path to a directory where generated assets should be placed (contains secrets)" - default = "" -} - variable "networking" { type = string description = "Choice of networking provider (flannel or calico)" @@ -107,6 +100,12 @@ variable "enable_aggregation" { # unofficial, undocumented, unsupported +variable "asset_dir" { + type = string + description = "Absolute path to a directory where generated assets should be placed (contains secrets)" + default = "" +} + variable "cluster_domain_suffix" { type = string description = "Queries for domains with the suffix will be answered by coredns. Default is cluster.local (e.g. foo.default.svc.cluster.local) " diff --git a/docs/advanced/worker-pools.md b/docs/advanced/worker-pools.md index 2392a4a73..7e6b0b259 100644 --- a/docs/advanced/worker-pools.md +++ b/docs/advanced/worker-pools.md @@ -5,8 +5,10 @@ Typhoon AWS, Azure, and Google Cloud allow additional groups of workers to be de Internal Terraform Modules: * `aws/container-linux/kubernetes/workers` +* `aws/fedora-coreos/kubernetes/workers` * `azure/container-linux/kubernetes/workers` * `google-cloud/container-linux/kubernetes/workers` +* `google-cloud/fedora-coreos/kubernetes/workers` ## AWS @@ -20,7 +22,7 @@ module "tempest-worker-pool" { vpc_id = module.tempest.vpc_id subnet_ids = module.tempest.subnet_ids security_groups = module.tempest.worker_security_groups - + # configuration name = "tempest-pool" kubeconfig = module.tempest.kubeconfig @@ -29,7 +31,7 @@ module "tempest-worker-pool" { # optional worker_count = 2 instance_type = "m5.large" - os_image = "coreos-beta" + os_image = "flatcar-beta" } ``` @@ -62,12 +64,12 @@ The AWS internal `workers` module supports a number of [variables](https://githu |:-----|:------------|:--------|:--------| | worker_count | Number of instances | 1 | 3 | | instance_type | EC2 instance type | "t3.small" | "t3.medium" | -| os_image | AMI channel for a Container Linux derivative | "coreos-stable" | coreos-stable, coreos-beta, coreos-alpha, flatcar-stable, flatcar-beta, flatcar-alpha | +| os_image | AMI channel for a Container Linux derivative | "flatcar-stable" | flatcar-stable, flatcar-beta, flatcar-alph, coreos-stable, coreos-beta, coreos-alpha | | disk_size | Size of the EBS volume in GB | 40 | 100 | | disk_type | Type of the EBS volume | "gp2" | standard, gp2, io1 | | disk_iops | IOPS of the EBS volume | 0 (i.e. auto) | 400 | | spot_price | Spot price in USD for worker instances or 0 to use on-demand instances | 0 | 0.10 | -| snippets | Container Linux Config snippets | [] | [example](/advanced/customization/#usage) | +| snippets | Container Linux Config snippets | [] | [examples](/advanced/customization/) | | service_cidr | Must match `service_cidr` of cluster | "10.3.0.0/16" | "10.3.0.0/24" | | node_labels | List of initial node labels | [] | ["worker-pool=foo"] | @@ -80,7 +82,7 @@ Create a cluster following the Azure [tutorial](../cl/azure.md#cluster). Define ```tf module "ramius-worker-pool" { source = "git::https://github.com/poseidon/typhoon//azure/container-linux/kubernetes/workers?ref=v1.18.1" - + # Azure region = module.ramius.region resource_group_name = module.ramius.resource_group_name @@ -131,9 +133,9 @@ The Azure internal `workers` module supports a number of [variables](https://git |:-----|:------------|:--------|:--------| | worker_count | Number of instances | 1 | 3 | | vm_type | Machine type for instances | "Standard_DS1_v2" | See below | -| os_image | Channel for a Container Linux derivative | "coreos-stable" | coreos-stable, coreos-beta, coreos-alpha | -| priority | Set priority to Low to use reduced cost surplus capacity, with the tradeoff that instances can be deallocated at any time | "Regular" | "Low" | -| snippets | Container Linux Config snippets | [] | [example](/advanced/customization/#usage) | +| os_image | Channel for a Container Linux derivative | "flatcar-stable" | flatcar-stable, flatcar-beta, coreos-stable, coreos-beta, coreos-alpha | +| priority | Set priority to Spot to use reduced cost surplus capacity, with the tradeoff that instances can be deallocated at any time | "Regular" | "Spot" | +| snippets | Container Linux Config snippets | [] | [examples](/advanced/customization/) | | service_cidr | CIDR IPv4 range to assign to Kubernetes services | "10.3.0.0/16" | "10.3.0.0/24" | | node_labels | List of initial node labels | [] | ["worker-pool=foo"] | @@ -156,7 +158,7 @@ module "yavin-worker-pool" { name = "yavin-16x" kubeconfig = module.yavin.kubeconfig ssh_authorized_key = var.ssh_authorized_key - + # optional worker_count = 2 machine_type = "n1-standard-16" @@ -196,6 +198,7 @@ The Google Cloud internal `workers` module supports a number of [variables](http | region | Region for the worker pool instances. May differ from the cluster's region | "europe-west2" | | network | Must be set to `network_name` output by cluster | module.cluster.network_name | | kubeconfig | Must be set to `kubeconfig` output by cluster | module.cluster.kubeconfig | +| os_image | Container Linux image for compute instances | "fedora-coreos-or-flatcar-image", coreos-stable, coreos-beta, coreos-alpha | | ssh_authorized_key | SSH public key for user 'core' | "ssh-rsa AAAAB3NZ..." | Check the list of regions [docs](https://cloud.google.com/compute/docs/regions-zones/regions-zones) or with `gcloud compute regions list`. @@ -206,10 +209,9 @@ Check the list of regions [docs](https://cloud.google.com/compute/docs/regions-z |:-----|:------------|:--------|:--------| | worker_count | Number of instances | 1 | 3 | | machine_type | Compute instance machine type | "n1-standard-1" | See below | -| os_image | Container Linux image for compute instances | "coreos-stable" | "coreos-alpha", "coreos-beta" | | disk_size | Size of the disk in GB | 40 | 100 | | preemptible | If true, Compute Engine will terminate instances randomly within 24 hours | false | true | -| snippets | Container Linux Config snippets | [] | [example](/advanced/customization/#usage) | +| snippets | Container Linux Config snippets | [] | [examples](/advanced/customization/) | | service_cidr | Must match `service_cidr` of cluster | "10.3.0.0/16" | "10.3.0.0/24" | | node_labels | List of initial node labels | [] | ["worker-pool=foo"] | diff --git a/docs/cl/aws.md b/docs/cl/aws.md index e87c97bd8..4ab9e11a9 100644 --- a/docs/cl/aws.md +++ b/docs/cl/aws.md @@ -208,7 +208,7 @@ Reference the DNS zone id with `aws_route53_zone.zone-for-clusters.zone_id`. | worker_count | Number of workers | 1 | 3 | | controller_type | EC2 instance type for controllers | "t3.small" | See below | | worker_type | EC2 instance type for workers | "t3.small" | See below | -| os_image | AMI channel for a Container Linux derivative | coreos-stable | coreos-stable, coreos-beta, coreos-alpha, flatcar-stable, flatcar-beta, flatcar-alpha, flatcar-edge | +| os_image | AMI channel for a Container Linux derivative | "flatcar-stable" | coreos-stable, coreos-beta, coreos-alpha, flatcar-stable, flatcar-beta, flatcar-alpha, flatcar-edge | | disk_size | Size of the EBS volume in GB | 40 | 100 | | disk_type | Type of the EBS volume | "gp2" | standard, gp2, io1 | | disk_iops | IOPS of the EBS volume | 0 (i.e. auto) | 400 | diff --git a/docs/cl/azure.md b/docs/cl/azure.md index 6b2dc4a24..530be53bb 100644 --- a/docs/cl/azure.md +++ b/docs/cl/azure.md @@ -228,7 +228,7 @@ Reference the DNS zone with `azurerm_dns_zone.clusters.name` and its resource gr | worker_count | Number of workers | 1 | 3 | | controller_type | Machine type for controllers | "Standard_B2s" | See below | | worker_type | Machine type for workers | "Standard_DS1_v2" | See below | -| os_image | Channel for a Container Linux derivative | "coreos-stable" | coreos-stable, coreos-beta, coreos-alpha, flatcar-stable, flatcar-beta | +| os_image | Channel for a Container Linux derivative | "flatcar-stable" | coreos-stable, coreos-beta, coreos-alpha, flatcar-stable, flatcar-beta | | disk_size | Size of the disk in GB | 40 | 100 | | worker_priority | Set priority to Spot to use reduced cost surplus capacity, with the tradeoff that instances can be deallocated at any time | Regular | Spot | | controller_snippets | Controller Container Linux Config snippets | [] | [example](/advanced/customization/#usage) | diff --git a/docs/cl/bare-metal.md b/docs/cl/bare-metal.md index 013e7dfbd..2a2f2274a 100644 --- a/docs/cl/bare-metal.md +++ b/docs/cl/bare-metal.md @@ -165,8 +165,8 @@ module "mercury" { # bare-metal cluster_name = "mercury" matchbox_http_endpoint = "http://matchbox.example.com" - os_channel = "coreos-stable" - os_version = "2191.5.0" + os_channel = "flatcar-stable" + os_version = "2345.3.1" # configuration k8s_domain_name = "node1.example.com" @@ -337,7 +337,7 @@ Check the [variables.tf](https://github.com/poseidon/typhoon/blob/master/bare-me | cluster_name | Unique cluster name | "mercury" | | matchbox_http_endpoint | Matchbox HTTP read-only endpoint | "http://matchbox.example.com:port" | | os_channel | Channel for a Container Linux derivative | coreos-stable, coreos-beta, coreos-alpha, flatcar-stable, flatcar-beta, flatcar-alpha, flatcar-edge | -| os_version | Version for a Container Linux derivative to PXE and install | "1632.3.0" | +| os_version | Version for a Container Linux derivative to PXE and install | "2345.3.1" | | k8s_domain_name | FQDN resolving to the controller(s) nodes. Workers and kubectl will communicate with this endpoint | "myk8s.example.com" | | ssh_authorized_key | SSH public key for user 'core' | "ssh-rsa AAAAB3Nz..." | | controllers | List of controller machine detail objects (unique name, identifying MAC address, FQDN) | `[{name="node1", mac="52:54:00:a1:9c:ae", domain="node1.example.com"}]` | diff --git a/docs/cl/digital-ocean.md b/docs/cl/digital-ocean.md index e0e128068..ed32f1ed5 100644 --- a/docs/cl/digital-ocean.md +++ b/docs/cl/digital-ocean.md @@ -59,6 +59,20 @@ provider "ct" { } ``` +### Flatcar Linux Images + +Flatcar Linux publishes DigitalOcean images, but does not yet upload them. DigitalOcean allows [custom images](https://blog.digitalocean.com/custom-images/) to be uploaded via URLor file. + +[Download](https://www.flatcar-linux.org/releases/) the Flatcar Linux DigitalOcean bin image. Rename the image with the channel and version (to refer to these images over time) and [upload](https://cloud.digitalocean.com/images/custom_images) it as a custom image. + +```tf +data "digitalocean_image" "flatcar-stable-2303-4-0" { + name = "flatcar-stable-2303.4.0.bin.bz2" +} +``` + +Set the [os_image](#variables) in the next step. + ## Cluster Define a Kubernetes cluster using the module `digital-ocean/container-linux/kubernetes`. @@ -71,9 +85,9 @@ module "nemo" { cluster_name = "nemo" region = "nyc3" dns_zone = "digital-ocean.example.com" - os_image = "coreos-stable" # configuration + os_image = data.digitalocean_image.flatcar-stable-2303-4-0.id ssh_fingerprints = ["d7:9d:79:ae:56:32:73:79:95:88:e3:a2:ab:5d:45:e7"] # optional @@ -83,28 +97,6 @@ module "nemo" { Reference the [variables docs](#variables) or the [variables.tf](https://github.com/poseidon/typhoon/blob/master/digital-ocean/container-linux/kubernetes/variables.tf) source. -### Flatcar Linux Only - -!!! warning - Typhoon for Flatcar Linux on DigitalOcean is alpha. Also IPv6 is unsupported with DigitalOcean custom images. - -Flatcar Linux publishes DigitalOcean images, but does not upload them. DigitalOcean allows [custom boot images](https://blog.digitalocean.com/custom-images/) by file or URL. - -[Download](https://www.flatcar-linux.org/releases/) the Flatcar Linux DigitalOcean bin image (or copy the URL) and [upload](https://cloud.digitalocean.com/images/custom_images) it as a custom image. Rename the image with the channel and version to refer to these images over time. - -```tf -module "nemo" { - ... - os_image = data.digitalocean_image.flatcar-stable.id -} - -data "digitalocean_image" "flatcar-stable" { - name = "flatcar-stable-2303.4.0.bin.bz2" -} -``` - -Set the [os_image](#variables) to the custom image id. - ## ssh-agent Initial bootstrapping requires `bootstrap.service` be started on one controller node. Terraform uses `ssh-agent` to automate this step. Add your SSH private key to `ssh-agent`. @@ -198,6 +190,7 @@ Check the [variables.tf](https://github.com/poseidon/typhoon/blob/master/digital | cluster_name | Unique cluster name (prepended to dns_zone) | "nemo" | | region | Digital Ocean region | "nyc1", "sfo2", "fra1", tor1" | | dns_zone | Digital Ocean domain (i.e. DNS zone) | "do.example.com" | +| os_image | Container Linux image for instances | "custom-image-id", coreos-stable, coreos-beta, coreos-alpha | | ssh_fingerprints | SSH public key fingerprints | ["d7:9d..."] | #### DNS Zone @@ -243,7 +236,6 @@ Digital Ocean requires the SSH public key be uploaded to your account, so you ma | worker_count | Number of workers | 1 | 3 | | controller_type | Droplet type for controllers | "s-2vcpu-2gb" | s-2vcpu-2gb, s-2vcpu-4gb, s-4vcpu-8gb, ... | | worker_type | Droplet type for workers | "s-1vcpu-2gb" | s-1vcpu-2gb, s-2vcpu-2gb, ... | -| os_image | Container Linux image for instances | "coreos-stable" | coreos-stable, coreos-beta, coreos-alpha, "custom-image-id" | | controller_snippets | Controller Container Linux Config snippets | [] | [example](/advanced/customization/) | | worker_snippets | Worker Container Linux Config snippets | [] | [example](/advanced/customization/) | | networking | Choice of networking provider | "calico" | "flannel" or "calico" | diff --git a/docs/cl/google-cloud.md b/docs/cl/google-cloud.md index 5bbe024c6..479690f5d 100644 --- a/docs/cl/google-cloud.md +++ b/docs/cl/google-cloud.md @@ -65,6 +65,25 @@ Additional configuration options are described in the `google` provider [docs](h !!! tip Regions are listed in [docs](https://cloud.google.com/compute/docs/regions-zones/regions-zones) or with `gcloud compute regions list`. A project may contain multiple clusters across different regions. +### Flatcar Linux Images + +Flatcar Linux publishes Google Cloud images, but does not yet upload them. Google Cloud allows [custom boot images](https://cloud.google.com/compute/docs/images/import-existing-image) to be uploaded to a bucket and imported into your project. + +[Download](https://www.flatcar-linux.org/releases/) the Flatcar Linux GCE gzipped tarball and upload it to a Google Cloud storage bucket. + +``` +gsutil list +gsutil cp flatcar_production_gce.tar.gz gs://BUCKET +``` + +Create a Compute Engine image from the file. + +``` +gcloud compute images create flatcar-linux-2303-4-0 --source-uri gs://BUCKET_NAME/flatcar_production_gce.tar.gz +``` + +Set the [os_image](#variables) in the next step. + ## Cluster Define a Kubernetes cluster using the module `google-cloud/container-linux/kubernetes`. @@ -80,6 +99,7 @@ module "yavin" { dns_zone_name = "example-zone" # configuration + os_image = "flatcar-linux-2303-4-0" ssh_authorized_key = "ssh-rsa AAAAB3Nz..." # optional @@ -89,28 +109,6 @@ module "yavin" { Reference the [variables docs](#variables) or the [variables.tf](https://github.com/poseidon/typhoon/blob/master/google-cloud/container-linux/kubernetes/variables.tf) source. -### Flatcar Linux Only - -!!! warning - Typhoon for Flatcar Linux on Google Cloud is alpha. - -Flatcar Linux publishes Google Cloud images, but does not upload them. Google Cloud allows [custom boot images](https://cloud.google.com/compute/docs/images/import-existing-image) to be uploaded to a bucket and imported into a project. - -[Download](https://www.flatcar-linux.org/releases/) the Flatcar Linux GCE gzipped tarball and upload it to a Google Cloud storage bucket. - -``` -gsutil list -gsutil cp flatcar_production_gce.tar.gz gs://BUCKET -``` - -Create a Compute Engine image from the file. - -``` -gcloud compute images create flatcar-linux-2303-4-0 --source-uri gs://BUCKET_NAME/flatcar_production_gce.tar.gz -``` - -Set the [os_image](#variables) to the image name (e.g. `flatcar-linux-2303-4-0`) - ## ssh-agent Initial bootstrapping requires `bootstrap.service` be started on one controller node. Terraform uses `ssh-agent` to automate this step. Add your SSH private key to `ssh-agent`. @@ -206,6 +204,7 @@ Check the [variables.tf](https://github.com/poseidon/typhoon/blob/master/google- | region | Google Cloud region | "us-central1" | | dns_zone | Google Cloud DNS zone | "google-cloud.example.com" | | dns_zone_name | Google Cloud DNS zone name | "example-zone" | +| os_image | Container Linux image for compute instances | "flatcar-linux-2303-4-0", coreos-stable, coreos-beta, coreos-alpha | | ssh_authorized_key | SSH public key for user 'core' | "ssh-rsa AAAAB3NZ..." | Check the list of valid [regions](https://cloud.google.com/compute/docs/regions-zones/regions-zones) and list Container Linux [images](https://cloud.google.com/compute/docs/images) with `gcloud compute images list | grep coreos`. @@ -235,7 +234,6 @@ resource "google_dns_managed_zone" "zone-for-clusters" { | worker_count | Number of workers | 1 | 3 | | controller_type | Machine type for controllers | "n1-standard-1" | See below | | worker_type | Machine type for workers | "n1-standard-1" | See below | -| os_image | Container Linux image for compute instances | "coreos-stable" | "flatcar-linux-2303-4-0" | | disk_size | Size of the disk in GB | 40 | 100 | | worker_preemptible | If enabled, Compute Engine will terminate workers randomly within 24 hours | false | true | | controller_snippets | Controller Container Linux Config snippets | [] | [example](/advanced/customization/) | diff --git a/google-cloud/container-linux/kubernetes/variables.tf b/google-cloud/container-linux/kubernetes/variables.tf index fc5e89c1a..5adb48b9a 100644 --- a/google-cloud/container-linux/kubernetes/variables.tf +++ b/google-cloud/container-linux/kubernetes/variables.tf @@ -48,8 +48,7 @@ variable "worker_type" { variable "os_image" { type = string - description = "Container Linux image for compute instances (e.g. coreos-stable)" - default = "coreos-stable" + description = "Container Linux image for compute instances (e.g. coreos-stable, custom-image)" } variable "disk_size" { @@ -83,12 +82,6 @@ variable "ssh_authorized_key" { description = "SSH public key for user 'core'" } -variable "asset_dir" { - type = string - description = "Absolute path to a directory where generated assets should be placed (contains secrets)" - default = "" -} - variable "networking" { type = string description = "Choice of networking provider (flannel or calico)" @@ -131,6 +124,12 @@ variable "worker_node_labels" { # unofficial, undocumented, unsupported +variable "asset_dir" { + type = string + description = "Absolute path to a directory where generated assets should be placed (contains secrets)" + default = "" +} + variable "cluster_domain_suffix" { type = string description = "Queries for domains with the suffix will be answered by coredns. Default is cluster.local (e.g. foo.default.svc.cluster.local) " diff --git a/google-cloud/container-linux/kubernetes/workers/variables.tf b/google-cloud/container-linux/kubernetes/workers/variables.tf index ec6983582..049e7a8b5 100644 --- a/google-cloud/container-linux/kubernetes/workers/variables.tf +++ b/google-cloud/container-linux/kubernetes/workers/variables.tf @@ -37,7 +37,6 @@ variable "machine_type" { variable "os_image" { type = string description = "Container Linux image for compute instanges (e.g. gcloud compute images list)" - default = "coreos-stable" } variable "disk_size" { From 5c4a3f73d5cfe4d304541634f6889c5a897f6c89 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Sun, 12 Apr 2020 15:49:48 -0700 Subject: [PATCH 391/523] Add support for Fedora CoreOS on Azure * Add `azure/fedora-coreos/kubernetes` module --- CHANGES.md | 11 +- README.md | 3 +- azure/container-linux/kubernetes/variables.tf | 2 +- .../kubernetes/workers/variables.tf | 2 +- azure/fedora-coreos/kubernetes/LICENSE | 23 ++ azure/fedora-coreos/kubernetes/README.md | 23 ++ azure/fedora-coreos/kubernetes/bootstrap.tf | 26 ++ azure/fedora-coreos/kubernetes/controllers.tf | 151 ++++++++ .../kubernetes/fcc/controller.yaml | 213 +++++++++++ azure/fedora-coreos/kubernetes/lb.tf | 161 +++++++++ azure/fedora-coreos/kubernetes/network.tf | 44 +++ azure/fedora-coreos/kubernetes/outputs.tf | 59 +++ azure/fedora-coreos/kubernetes/security.tf | 336 ++++++++++++++++++ azure/fedora-coreos/kubernetes/ssh.tf | 59 +++ azure/fedora-coreos/kubernetes/variables.tf | 143 ++++++++ azure/fedora-coreos/kubernetes/versions.tf | 12 + azure/fedora-coreos/kubernetes/workers.tf | 24 ++ .../kubernetes/workers/fcc/worker.yaml | 119 +++++++ .../kubernetes/workers/variables.tf | 98 +++++ .../kubernetes/workers/versions.tf | 4 + .../kubernetes/workers/workers.tf | 92 +++++ azure/ignore/.gitkeep | 0 docs/advanced/worker-pools.md | 5 +- docs/cl/azure.md | 5 +- docs/fedora-coreos/azure.md | 261 ++++++++++++++ docs/fedora-coreos/digitalocean.md | 6 +- docs/fedora-coreos/google-cloud.md | 6 +- docs/index.md | 3 +- .../fedora-coreos/kubernetes/README.md | 4 +- 29 files changed, 1873 insertions(+), 22 deletions(-) create mode 100644 azure/fedora-coreos/kubernetes/LICENSE create mode 100644 azure/fedora-coreos/kubernetes/README.md create mode 100644 azure/fedora-coreos/kubernetes/bootstrap.tf create mode 100644 azure/fedora-coreos/kubernetes/controllers.tf create mode 100644 azure/fedora-coreos/kubernetes/fcc/controller.yaml create mode 100644 azure/fedora-coreos/kubernetes/lb.tf create mode 100644 azure/fedora-coreos/kubernetes/network.tf create mode 100644 azure/fedora-coreos/kubernetes/outputs.tf create mode 100644 azure/fedora-coreos/kubernetes/security.tf create mode 100644 azure/fedora-coreos/kubernetes/ssh.tf create mode 100644 azure/fedora-coreos/kubernetes/variables.tf create mode 100644 azure/fedora-coreos/kubernetes/versions.tf create mode 100644 azure/fedora-coreos/kubernetes/workers.tf create mode 100644 azure/fedora-coreos/kubernetes/workers/fcc/worker.yaml create mode 100644 azure/fedora-coreos/kubernetes/workers/variables.tf create mode 100644 azure/fedora-coreos/kubernetes/workers/versions.tf create mode 100644 azure/fedora-coreos/kubernetes/workers/workers.tf delete mode 100644 azure/ignore/.gitkeep create mode 100644 docs/fedora-coreos/azure.md diff --git a/CHANGES.md b/CHANGES.md index 548dfae91..251887930 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -11,11 +11,12 @@ Notable changes between versions. #### AWS -* Change Container Linux `os_image` default from `coreos-stable` to `flatcar-stable` +* Change Container Linux `os_image` default from `coreos-stable` to `flatcar-stable` ([#702](https://github.com/poseidon/typhoon/pull/702)) #### Azure -* Change Container Linux `os_image` default from `coreos-stable` to `flatcar-stable` +* Add support for Fedora CoreOS ([#704](https://github.com/poseidon/typhoon/pull/704)) +* Change Container Linux `os_image` default from `coreos-stable` to `flatcar-stable` ([#702](https://github.com/poseidon/typhoon/pull/702)) #### Bare-Metal @@ -23,11 +24,11 @@ Notable changes between versions. #### Google -* Change Container Linux `os_image` to be required. Container Linux users should upload a Flatcar Linux image and set it (**action required**) +* Change Container Linux `os_image` to be required. Container Linux users should upload a Flatcar Linux image and set it (**action required**) ([#702](https://github.com/poseidon/typhoon/pull/702)) #### DigitalOcean -* Change Container Linux `os_image` to be required. Container Linux users should upload a Flatcar Linux image and set it (**action required**) +* Change Container Linux `os_image` to be required. Container Linux users should upload a Flatcar Linux image and set it (**action required**) ([#702](https://github.com/poseidon/typhoon/pull/702)) ## v1.18.1 @@ -44,7 +45,7 @@ Notable changes between versions. * Rename Container Linux `controller_clc_snippets` to `controller_snippets` for consistency ([#688](https://github.com/poseidon/typhoon/pull/688)) * Rename Container Linux `worker_clc_snippets` to `worker_snippets` for consistency * Rename Container Linux `clc_snippets` (bare-metal) to `snippets` for consistency -* Drop support for [gitRepo](https://kubernetes.io/docs/concepts/storage/volumes/#gitrepo) volumes +* Drop support for [gitRepo](https://kubernetes.io/docs/concepts/storage/volumes/#gitrepo) volumes ([kubelet#3](https://github.com/poseidon/kubelet/pull/3)) #### Azure diff --git a/README.md b/README.md index 93520d596..863f5e04b 100644 --- a/README.md +++ b/README.md @@ -26,6 +26,7 @@ Typhoon is available for [Fedora CoreOS](https://getfedora.org/coreos/). | Platform | Operating System | Terraform Module | Status | |---------------|------------------|------------------|--------| | AWS | Fedora CoreOS | [aws/fedora-coreos/kubernetes](aws/fedora-coreos/kubernetes) | stable | +| Azure | Fedora CoreOS | [azure/fedora-coreos/kubernetes](azure/fedora-coreos/kubernetes) | alpha | | Bare-Metal | Fedora CoreOS | [bare-metal/fedora-coreos/kubernetes](bare-metal/fedora-coreos/kubernetes) | beta | | DigitalOcean | Fedora CoreOS | [digital-ocean/fedora-coreos/kubernetes](digital-ocean/fedora-coreos/kubernetes) | alpha | | Google Cloud | Fedora CoreOS | [google-cloud/fedora-coreos/kubernetes](google-cloud/fedora-coreos/kubernetes) | beta | @@ -54,7 +55,7 @@ Typhoon is available for CoreOS Container Linux ([no updates](https://coreos.com * [Docs](https://typhoon.psdn.io) * Architecture [concepts](https://typhoon.psdn.io/architecture/concepts/) and [operating systems](https://typhoon.psdn.io/architecture/operating-systems/) -* Fedora CoreOS tutorials for [AWS](docs/fedora-coreos/aws.md), [Bare-Metal](docs/fedora-coreos/bare-metal.md), [DigitalOcean](docs/fedora-coreos/digitalocean.md), and [Google Cloud](docs/fedora-coreos/google-cloud.md) +* Fedora CoreOS tutorials for [AWS](docs/fedora-coreos/aws.md), [Azure](docs/fedora-coreos/azure.md), [Bare-Metal](docs/fedora-coreos/bare-metal.md), [DigitalOcean](docs/fedora-coreos/digitalocean.md), and [Google Cloud](docs/fedora-coreos/google-cloud.md) * Flatcar Linux tutorials for [AWS](docs/cl/aws.md), [Azure](docs/cl/azure.md), [Bare-Metal](docs/cl/bare-metal.md), [DigitalOcean](docs/cl/digital-ocean.md), and [Google Cloud](docs/cl/google-cloud.md) ## Usage diff --git a/azure/container-linux/kubernetes/variables.tf b/azure/container-linux/kubernetes/variables.tf index 96eae93ec..44827db49 100644 --- a/azure/container-linux/kubernetes/variables.tf +++ b/azure/container-linux/kubernetes/variables.tf @@ -60,7 +60,7 @@ variable "disk_size" { variable "worker_priority" { type = string - description = "Set worker priority to Low to use reduced cost surplus capacity, with the tradeoff that instances can be deallocated at any time." + description = "Set worker priority to Spot to use reduced cost surplus capacity, with the tradeoff that instances can be deallocated at any time." default = "Regular" } diff --git a/azure/container-linux/kubernetes/workers/variables.tf b/azure/container-linux/kubernetes/workers/variables.tf index 06ab97011..0ebd606fe 100644 --- a/azure/container-linux/kubernetes/workers/variables.tf +++ b/azure/container-linux/kubernetes/workers/variables.tf @@ -52,7 +52,7 @@ variable "os_image" { variable "priority" { type = string - description = "Set priority to Low to use reduced cost surplus capacity, with the tradeoff that instances can be evicted at any time." + description = "Set priority to Spot to use reduced cost surplus capacity, with the tradeoff that instances can be evicted at any time." default = "Regular" } diff --git a/azure/fedora-coreos/kubernetes/LICENSE b/azure/fedora-coreos/kubernetes/LICENSE new file mode 100644 index 000000000..658b1c469 --- /dev/null +++ b/azure/fedora-coreos/kubernetes/LICENSE @@ -0,0 +1,23 @@ +The MIT License (MIT) + +Copyright (c) 2020 Typhoon Authors +Copyright (c) 2020 Dalton Hubble + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. + diff --git a/azure/fedora-coreos/kubernetes/README.md b/azure/fedora-coreos/kubernetes/README.md new file mode 100644 index 000000000..e5ee37870 --- /dev/null +++ b/azure/fedora-coreos/kubernetes/README.md @@ -0,0 +1,23 @@ +# Typhoon + +Typhoon is a minimal and free Kubernetes distribution. + +* Minimal, stable base Kubernetes distribution +* Declarative infrastructure and configuration +* Free (freedom and cost) and privacy-respecting +* Practical for labs, datacenters, and clouds + +Typhoon distributes upstream Kubernetes, architectural conventions, and cluster addons, much like a GNU/Linux distribution provides the Linux kernel and userspace components. + +## Features + +* Kubernetes v1.18.1 (upstream) +* Single or multi-master, [Calico](https://www.projectcalico.org/) or [flannel](https://github.com/coreos/flannel) networking +* On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/) +* Advanced features like [worker pools](https://typhoon.psdn.io/advanced/worker-pools/), [spot priority](https://typhoon.psdn.io/fedora-coreos/azure/#low-priority) workers, and [snippets](https://typhoon.psdn.io/advanced/customization/) customization +* Ready for Ingress, Prometheus, Grafana, and other optional [addons](https://typhoon.psdn.io/addons/overview/) + +## Docs + +Please see the [official docs](https://typhoon.psdn.io) and the Azure [tutorial](https://typhoon.psdn.io/fedora-coreos/azure/). + diff --git a/azure/fedora-coreos/kubernetes/bootstrap.tf b/azure/fedora-coreos/kubernetes/bootstrap.tf new file mode 100644 index 000000000..e22a3401e --- /dev/null +++ b/azure/fedora-coreos/kubernetes/bootstrap.tf @@ -0,0 +1,26 @@ +# Kubernetes assets (kubeconfig, manifests) +module "bootstrap" { + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=1ad53d3b1c1ad75a4ed27f124f772fc5dc025245" + + cluster_name = var.cluster_name + api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] + etcd_servers = formatlist("%s.%s", azurerm_dns_a_record.etcds.*.name, var.dns_zone) + asset_dir = var.asset_dir + + networking = var.networking + + # only effective with Calico networking + # we should be able to use 1450 MTU, but in practice, 1410 was needed + network_encapsulation = "vxlan" + network_mtu = "1410" + + pod_cidr = var.pod_cidr + service_cidr = var.service_cidr + cluster_domain_suffix = var.cluster_domain_suffix + enable_reporting = var.enable_reporting + enable_aggregation = var.enable_aggregation + + # Fedora CoreOS + trusted_certs_dir = "/etc/pki/tls/certs" +} + diff --git a/azure/fedora-coreos/kubernetes/controllers.tf b/azure/fedora-coreos/kubernetes/controllers.tf new file mode 100644 index 000000000..b82992315 --- /dev/null +++ b/azure/fedora-coreos/kubernetes/controllers.tf @@ -0,0 +1,151 @@ +# Discrete DNS records for each controller's private IPv4 for etcd usage +resource "azurerm_dns_a_record" "etcds" { + count = var.controller_count + resource_group_name = var.dns_zone_group + + # DNS Zone name where record should be created + zone_name = var.dns_zone + + # DNS record + name = format("%s-etcd%d", var.cluster_name, count.index) + ttl = 300 + + # private IPv4 address for etcd + records = [azurerm_network_interface.controllers.*.private_ip_address[count.index]] +} + +# Controller availability set to spread controllers +resource "azurerm_availability_set" "controllers" { + resource_group_name = azurerm_resource_group.cluster.name + + name = "${var.cluster_name}-controllers" + location = var.region + platform_fault_domain_count = 2 + platform_update_domain_count = 4 + managed = true +} + +# Controller instances +resource "azurerm_linux_virtual_machine" "controllers" { + count = var.controller_count + resource_group_name = azurerm_resource_group.cluster.name + + name = "${var.cluster_name}-controller-${count.index}" + location = var.region + availability_set_id = azurerm_availability_set.controllers.id + + size = var.controller_type + custom_data = base64encode(data.ct_config.controller-ignitions.*.rendered[count.index]) + + # storage + source_image_id = var.os_image + os_disk { + name = "${var.cluster_name}-controller-${count.index}" + caching = "None" + disk_size_gb = var.disk_size + storage_account_type = "Premium_LRS" + } + + # network + network_interface_ids = [ + azurerm_network_interface.controllers.*.id[count.index] + ] + + # Azure requires setting admin_ssh_key, though Ignition custom_data handles it too + admin_username = "core" + admin_ssh_key { + username = "core" + public_key = var.ssh_authorized_key + } + + lifecycle { + ignore_changes = [ + os_disk, + custom_data, + ] + } +} + +# Controller public IPv4 addresses +resource "azurerm_public_ip" "controllers" { + count = var.controller_count + resource_group_name = azurerm_resource_group.cluster.name + + name = "${var.cluster_name}-controller-${count.index}" + location = azurerm_resource_group.cluster.location + sku = "Standard" + allocation_method = "Static" +} + +# Controller NICs with public and private IPv4 +resource "azurerm_network_interface" "controllers" { + count = var.controller_count + resource_group_name = azurerm_resource_group.cluster.name + + name = "${var.cluster_name}-controller-${count.index}" + location = azurerm_resource_group.cluster.location + + ip_configuration { + name = "ip0" + subnet_id = azurerm_subnet.controller.id + private_ip_address_allocation = "Dynamic" + # instance public IPv4 + public_ip_address_id = azurerm_public_ip.controllers.*.id[count.index] + } +} + +# Associate controller network interface with controller security group +resource "azurerm_network_interface_security_group_association" "controllers" { + count = var.controller_count + + network_interface_id = azurerm_network_interface.controllers[count.index].id + network_security_group_id = azurerm_network_security_group.controller.id +} + +# Associate controller network interface with controller backend address pool +resource "azurerm_network_interface_backend_address_pool_association" "controllers" { + count = var.controller_count + + network_interface_id = azurerm_network_interface.controllers[count.index].id + ip_configuration_name = "ip0" + backend_address_pool_id = azurerm_lb_backend_address_pool.controller.id +} + +# Controller Ignition configs +data "ct_config" "controller-ignitions" { + count = var.controller_count + content = data.template_file.controller-configs.*.rendered[count.index] + pretty_print = false + snippets = var.controller_snippets +} + +# Controller Fedora CoreOS configs +data "template_file" "controller-configs" { + count = var.controller_count + + template = file("${path.module}/fcc/controller.yaml") + + vars = { + # Cannot use cyclic dependencies on controllers or their DNS records + etcd_name = "etcd${count.index}" + etcd_domain = "${var.cluster_name}-etcd${count.index}.${var.dns_zone}" + # etcd0=https://cluster-etcd0.example.com,etcd1=https://cluster-etcd1.example.com,... + etcd_initial_cluster = join(",", data.template_file.etcds.*.rendered) + kubeconfig = indent(10, module.bootstrap.kubeconfig-kubelet) + ssh_authorized_key = var.ssh_authorized_key + cluster_dns_service_ip = cidrhost(var.service_cidr, 10) + cluster_domain_suffix = var.cluster_domain_suffix + } +} + +data "template_file" "etcds" { + count = var.controller_count + template = "etcd$${index}=https://$${cluster_name}-etcd$${index}.$${dns_zone}:2380" + + vars = { + index = count.index + cluster_name = var.cluster_name + dns_zone = var.dns_zone + } +} + diff --git a/azure/fedora-coreos/kubernetes/fcc/controller.yaml b/azure/fedora-coreos/kubernetes/fcc/controller.yaml new file mode 100644 index 000000000..06c8d1d27 --- /dev/null +++ b/azure/fedora-coreos/kubernetes/fcc/controller.yaml @@ -0,0 +1,213 @@ +--- +variant: fcos +version: 1.0.0 +systemd: + units: + - name: etcd-member.service + enabled: true + contents: | + [Unit] + Description=etcd (System Container) + Documentation=https://github.com/coreos/etcd + Wants=network-online.target network.target + After=network-online.target + [Service] + # https://github.com/opencontainers/runc/pull/1807 + # Type=notify + # NotifyAccess=exec + Type=exec + Restart=on-failure + RestartSec=10s + TimeoutStartSec=0 + LimitNOFILE=40000 + ExecStartPre=/bin/mkdir -p /var/lib/etcd + ExecStartPre=-/usr/bin/podman rm etcd + #--volume $${NOTIFY_SOCKET}:/run/systemd/notify \ + ExecStart=/usr/bin/podman run --name etcd \ + --env-file /etc/etcd/etcd.env \ + --network host \ + --volume /var/lib/etcd:/var/lib/etcd:rw,Z \ + --volume /etc/ssl/etcd:/etc/ssl/certs:ro,Z \ + quay.io/coreos/etcd:v3.4.7 + ExecStop=/usr/bin/podman stop etcd + [Install] + WantedBy=multi-user.target + - name: docker.service + enabled: true + - name: wait-for-dns.service + enabled: true + contents: | + [Unit] + Description=Wait for DNS entries + Before=kubelet.service + [Service] + Type=oneshot + RemainAfterExit=true + ExecStart=/bin/sh -c 'while ! /usr/bin/grep '^[^#[:space:]]' /etc/resolv.conf > /dev/null; do sleep 1; done' + [Install] + RequiredBy=kubelet.service + RequiredBy=etcd-member.service + - name: kubelet.service + enabled: true + contents: | + [Unit] + Description=Kubelet via Hyperkube (System Container) + Wants=rpc-statd.service + [Service] + ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d + ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests + ExecStartPre=/bin/mkdir -p /opt/cni/bin + ExecStartPre=/bin/mkdir -p /var/lib/calico + ExecStartPre=/bin/mkdir -p /var/lib/kubelet/volumeplugins + ExecStartPre=/usr/bin/bash -c "grep 'certificate-authority-data' /etc/kubernetes/kubeconfig | awk '{print $2}' | base64 -d > /etc/kubernetes/ca.crt" + ExecStartPre=-/usr/bin/podman rm kubelet + ExecStart=/usr/bin/podman run --name kubelet \ + --privileged \ + --pid host \ + --network host \ + --volume /etc/kubernetes:/etc/kubernetes:ro,z \ + --volume /usr/lib/os-release:/etc/os-release:ro \ + --volume /etc/ssl/certs:/etc/ssl/certs:ro \ + --volume /lib/modules:/lib/modules:ro \ + --volume /run:/run \ + --volume /sys/fs/cgroup:/sys/fs/cgroup:ro \ + --volume /sys/fs/cgroup/systemd:/sys/fs/cgroup/systemd \ + --volume /etc/pki/tls/certs:/usr/share/ca-certificates:ro \ + --volume /var/lib/calico:/var/lib/calico:ro \ + --volume /var/lib/docker:/var/lib/docker \ + --volume /var/lib/kubelet:/var/lib/kubelet:rshared,z \ + --volume /var/log:/var/log \ + --volume /var/run/lock:/var/run/lock:z \ + --volume /opt/cni/bin:/opt/cni/bin:z \ + quay.io/poseidon/kubelet:v1.18.1 \ + --anonymous-auth=false \ + --authentication-token-webhook \ + --authorization-mode=Webhook \ + --cgroup-driver=systemd \ + --cgroups-per-qos=true \ + --enforce-node-allocatable=pods \ + --client-ca-file=/etc/kubernetes/ca.crt \ + --cluster_dns=${cluster_dns_service_ip} \ + --cluster_domain=${cluster_domain_suffix} \ + --cni-conf-dir=/etc/kubernetes/cni/net.d \ + --exit-on-lock-contention \ + --healthz-port=0 \ + --kubeconfig=/etc/kubernetes/kubeconfig \ + --lock-file=/var/run/lock/kubelet.lock \ + --network-plugin=cni \ + --node-labels=node.kubernetes.io/master \ + --node-labels=node.kubernetes.io/controller="true" \ + --pod-manifest-path=/etc/kubernetes/manifests \ + --read-only-port=0 \ + --register-with-taints=node-role.kubernetes.io/master=:NoSchedule \ + --volume-plugin-dir=/var/lib/kubelet/volumeplugins + ExecStop=-/usr/bin/podman stop kubelet + Delegate=yes + Restart=always + RestartSec=10 + [Install] + WantedBy=multi-user.target + - name: bootstrap.service + contents: | + [Unit] + Description=Kubernetes control plane + ConditionPathExists=!/opt/bootstrap/bootstrap.done + [Service] + Type=oneshot + RemainAfterExit=true + WorkingDirectory=/opt/bootstrap + ExecStartPre=-/usr/bin/podman rm bootstrap + ExecStart=/usr/bin/podman run --name bootstrap \ + --network host \ + --volume /etc/kubernetes/bootstrap-secrets:/etc/kubernetes/secrets:ro,Z \ + --volume /opt/bootstrap/assets:/assets:ro,Z \ + --volume /opt/bootstrap/apply:/apply:ro,Z \ + --entrypoint=/apply \ + quay.io/poseidon/kubelet:v1.18.1 + ExecStartPost=/bin/touch /opt/bootstrap/bootstrap.done + ExecStartPost=-/usr/bin/podman stop bootstrap +storage: + directories: + - path: /etc/kubernetes + - path: /opt/bootstrap + files: + - path: /etc/kubernetes/kubeconfig + mode: 0644 + contents: + inline: | + ${kubeconfig} + - path: /opt/bootstrap/layout + mode: 0544 + contents: + inline: | + #!/bin/bash -e + mkdir -p -- auth tls/etcd tls/k8s static-manifests manifests/coredns manifests-networking + awk '/#####/ {filename=$2; next} {print > filename}' assets + mkdir -p /etc/ssl/etcd/etcd + mkdir -p /etc/kubernetes/bootstrap-secrets + mv tls/etcd/{peer*,server*} /etc/ssl/etcd/etcd/ + mv tls/etcd/etcd-client* /etc/kubernetes/bootstrap-secrets/ + chown -R etcd:etcd /etc/ssl/etcd + chmod -R 500 /etc/ssl/etcd + mv auth/kubeconfig /etc/kubernetes/bootstrap-secrets/ + mv tls/k8s/* /etc/kubernetes/bootstrap-secrets/ + sudo mkdir -p /etc/kubernetes/manifests + sudo mv static-manifests/* /etc/kubernetes/manifests/ + sudo mkdir -p /opt/bootstrap/assets + sudo mv manifests /opt/bootstrap/assets/manifests + sudo mv manifests-networking/* /opt/bootstrap/assets/manifests/ + rm -rf assets auth static-manifests tls manifests-networking + - path: /opt/bootstrap/apply + mode: 0544 + contents: + inline: | + #!/bin/bash -e + export KUBECONFIG=/etc/kubernetes/secrets/kubeconfig + until kubectl version; do + echo "Waiting for static pod control plane" + sleep 5 + done + until kubectl apply -f /assets/manifests -R; do + echo "Retry applying manifests" + sleep 5 + done + - path: /etc/sysctl.d/max-user-watches.conf + contents: + inline: | + fs.inotify.max_user_watches=16184 + - path: /etc/systemd/system.conf.d/accounting.conf + contents: + inline: | + [Manager] + DefaultCPUAccounting=yes + DefaultMemoryAccounting=yes + DefaultBlockIOAccounting=yes + - path: /etc/etcd/etcd.env + mode: 0644 + contents: + inline: | + # TODO: Use a systemd dropin once podman v1.4.5 is avail. + NOTIFY_SOCKET=/run/systemd/notify + ETCD_NAME=${etcd_name} + ETCD_DATA_DIR=/var/lib/etcd + ETCD_ADVERTISE_CLIENT_URLS=https://${etcd_domain}:2379 + ETCD_INITIAL_ADVERTISE_PEER_URLS=https://${etcd_domain}:2380 + ETCD_LISTEN_CLIENT_URLS=https://0.0.0.0:2379 + ETCD_LISTEN_PEER_URLS=https://0.0.0.0:2380 + ETCD_LISTEN_METRICS_URLS=http://0.0.0.0:2381 + ETCD_INITIAL_CLUSTER=${etcd_initial_cluster} + ETCD_STRICT_RECONFIG_CHECK=true + ETCD_TRUSTED_CA_FILE=/etc/ssl/certs/etcd/server-ca.crt + ETCD_CERT_FILE=/etc/ssl/certs/etcd/server.crt + ETCD_KEY_FILE=/etc/ssl/certs/etcd/server.key + ETCD_CLIENT_CERT_AUTH=true + ETCD_PEER_TRUSTED_CA_FILE=/etc/ssl/certs/etcd/peer-ca.crt + ETCD_PEER_CERT_FILE=/etc/ssl/certs/etcd/peer.crt + ETCD_PEER_KEY_FILE=/etc/ssl/certs/etcd/peer.key + ETCD_PEER_CLIENT_CERT_AUTH=true +passwd: + users: + - name: core + ssh_authorized_keys: + - ${ssh_authorized_key} + diff --git a/azure/fedora-coreos/kubernetes/lb.tf b/azure/fedora-coreos/kubernetes/lb.tf new file mode 100644 index 000000000..ef9247598 --- /dev/null +++ b/azure/fedora-coreos/kubernetes/lb.tf @@ -0,0 +1,161 @@ +# DNS record for the apiserver load balancer +resource "azurerm_dns_a_record" "apiserver" { + resource_group_name = var.dns_zone_group + + # DNS Zone name where record should be created + zone_name = var.dns_zone + + # DNS record + name = var.cluster_name + ttl = 300 + + # IPv4 address of apiserver load balancer + records = [azurerm_public_ip.apiserver-ipv4.ip_address] +} + +# Static IPv4 address for the apiserver frontend +resource "azurerm_public_ip" "apiserver-ipv4" { + resource_group_name = azurerm_resource_group.cluster.name + + name = "${var.cluster_name}-apiserver-ipv4" + location = var.region + sku = "Standard" + allocation_method = "Static" +} + +# Static IPv4 address for the ingress frontend +resource "azurerm_public_ip" "ingress-ipv4" { + resource_group_name = azurerm_resource_group.cluster.name + + name = "${var.cluster_name}-ingress-ipv4" + location = var.region + sku = "Standard" + allocation_method = "Static" +} + +# Network Load Balancer for apiservers and ingress +resource "azurerm_lb" "cluster" { + resource_group_name = azurerm_resource_group.cluster.name + + name = var.cluster_name + location = var.region + sku = "Standard" + + frontend_ip_configuration { + name = "apiserver" + public_ip_address_id = azurerm_public_ip.apiserver-ipv4.id + } + + frontend_ip_configuration { + name = "ingress" + public_ip_address_id = azurerm_public_ip.ingress-ipv4.id + } +} + +resource "azurerm_lb_rule" "apiserver" { + resource_group_name = azurerm_resource_group.cluster.name + + name = "apiserver" + loadbalancer_id = azurerm_lb.cluster.id + frontend_ip_configuration_name = "apiserver" + + protocol = "Tcp" + frontend_port = 6443 + backend_port = 6443 + backend_address_pool_id = azurerm_lb_backend_address_pool.controller.id + probe_id = azurerm_lb_probe.apiserver.id +} + +resource "azurerm_lb_rule" "ingress-http" { + resource_group_name = azurerm_resource_group.cluster.name + + name = "ingress-http" + loadbalancer_id = azurerm_lb.cluster.id + frontend_ip_configuration_name = "ingress" + disable_outbound_snat = true + + protocol = "Tcp" + frontend_port = 80 + backend_port = 80 + backend_address_pool_id = azurerm_lb_backend_address_pool.worker.id + probe_id = azurerm_lb_probe.ingress.id +} + +resource "azurerm_lb_rule" "ingress-https" { + resource_group_name = azurerm_resource_group.cluster.name + + name = "ingress-https" + loadbalancer_id = azurerm_lb.cluster.id + frontend_ip_configuration_name = "ingress" + disable_outbound_snat = true + + protocol = "Tcp" + frontend_port = 443 + backend_port = 443 + backend_address_pool_id = azurerm_lb_backend_address_pool.worker.id + probe_id = azurerm_lb_probe.ingress.id +} + +# Worker outbound TCP/UDP SNAT +resource "azurerm_lb_outbound_rule" "worker-outbound" { + resource_group_name = azurerm_resource_group.cluster.name + + name = "worker" + loadbalancer_id = azurerm_lb.cluster.id + frontend_ip_configuration { + name = "ingress" + } + + protocol = "All" + backend_address_pool_id = azurerm_lb_backend_address_pool.worker.id +} + +# Address pool of controllers +resource "azurerm_lb_backend_address_pool" "controller" { + resource_group_name = azurerm_resource_group.cluster.name + + name = "controller" + loadbalancer_id = azurerm_lb.cluster.id +} + +# Address pool of workers +resource "azurerm_lb_backend_address_pool" "worker" { + resource_group_name = azurerm_resource_group.cluster.name + + name = "worker" + loadbalancer_id = azurerm_lb.cluster.id +} + +# Health checks / probes + +# TCP health check for apiserver +resource "azurerm_lb_probe" "apiserver" { + resource_group_name = azurerm_resource_group.cluster.name + + name = "apiserver" + loadbalancer_id = azurerm_lb.cluster.id + protocol = "Tcp" + port = 6443 + + # unhealthy threshold + number_of_probes = 3 + + interval_in_seconds = 5 +} + +# HTTP health check for ingress +resource "azurerm_lb_probe" "ingress" { + resource_group_name = azurerm_resource_group.cluster.name + + name = "ingress" + loadbalancer_id = azurerm_lb.cluster.id + protocol = "Http" + port = 10254 + request_path = "/healthz" + + # unhealthy threshold + number_of_probes = 3 + + interval_in_seconds = 5 +} + diff --git a/azure/fedora-coreos/kubernetes/network.tf b/azure/fedora-coreos/kubernetes/network.tf new file mode 100644 index 000000000..ea92a5a7d --- /dev/null +++ b/azure/fedora-coreos/kubernetes/network.tf @@ -0,0 +1,44 @@ +# Organize cluster into a resource group +resource "azurerm_resource_group" "cluster" { + name = var.cluster_name + location = var.region +} + +resource "azurerm_virtual_network" "network" { + resource_group_name = azurerm_resource_group.cluster.name + + name = var.cluster_name + location = azurerm_resource_group.cluster.location + address_space = [var.host_cidr] +} + +# Subnets - separate subnets for controller and workers because Azure +# network security groups are based on IPv4 CIDR rather than instance +# tags like GCP or security group membership like AWS + +resource "azurerm_subnet" "controller" { + resource_group_name = azurerm_resource_group.cluster.name + + name = "controller" + virtual_network_name = azurerm_virtual_network.network.name + address_prefix = cidrsubnet(var.host_cidr, 1, 0) +} + +resource "azurerm_subnet_network_security_group_association" "controller" { + subnet_id = azurerm_subnet.controller.id + network_security_group_id = azurerm_network_security_group.controller.id +} + +resource "azurerm_subnet" "worker" { + resource_group_name = azurerm_resource_group.cluster.name + + name = "worker" + virtual_network_name = azurerm_virtual_network.network.name + address_prefix = cidrsubnet(var.host_cidr, 1, 1) +} + +resource "azurerm_subnet_network_security_group_association" "worker" { + subnet_id = azurerm_subnet.worker.id + network_security_group_id = azurerm_network_security_group.worker.id +} + diff --git a/azure/fedora-coreos/kubernetes/outputs.tf b/azure/fedora-coreos/kubernetes/outputs.tf new file mode 100644 index 000000000..fe5eaffb0 --- /dev/null +++ b/azure/fedora-coreos/kubernetes/outputs.tf @@ -0,0 +1,59 @@ +output "kubeconfig-admin" { + value = module.bootstrap.kubeconfig-admin +} + +# Outputs for Kubernetes Ingress + +output "ingress_static_ipv4" { + value = azurerm_public_ip.ingress-ipv4.ip_address + description = "IPv4 address of the load balancer for distributing traffic to Ingress controllers" +} + +# Outputs for worker pools + +output "region" { + value = azurerm_resource_group.cluster.location +} + +output "resource_group_name" { + value = azurerm_resource_group.cluster.name +} + +output "resource_group_id" { + value = azurerm_resource_group.cluster.id +} + +output "subnet_id" { + value = azurerm_subnet.worker.id +} + +output "security_group_id" { + value = azurerm_network_security_group.worker.id +} + +output "kubeconfig" { + value = module.bootstrap.kubeconfig-kubelet +} + +# Outputs for custom firewalling + +output "worker_security_group_name" { + value = azurerm_network_security_group.worker.name +} + +output "worker_address_prefix" { + description = "Worker network subnet CIDR address (for source/destination)" + value = azurerm_subnet.worker.address_prefix +} + +# Outputs for custom load balancing + +output "loadbalancer_id" { + description = "ID of the cluster load balancer" + value = azurerm_lb.cluster.id +} + +output "backend_address_pool_id" { + description = "ID of the worker backend address pool" + value = azurerm_lb_backend_address_pool.worker.id +} diff --git a/azure/fedora-coreos/kubernetes/security.tf b/azure/fedora-coreos/kubernetes/security.tf new file mode 100644 index 000000000..feb6fef54 --- /dev/null +++ b/azure/fedora-coreos/kubernetes/security.tf @@ -0,0 +1,336 @@ +# Controller security group + +resource "azurerm_network_security_group" "controller" { + resource_group_name = azurerm_resource_group.cluster.name + + name = "${var.cluster_name}-controller" + location = azurerm_resource_group.cluster.location +} + +resource "azurerm_network_security_rule" "controller-ssh" { + resource_group_name = azurerm_resource_group.cluster.name + + name = "allow-ssh" + network_security_group_name = azurerm_network_security_group.controller.name + priority = "2000" + access = "Allow" + direction = "Inbound" + protocol = "Tcp" + source_port_range = "*" + destination_port_range = "22" + source_address_prefix = "*" + destination_address_prefix = azurerm_subnet.controller.address_prefix +} + +resource "azurerm_network_security_rule" "controller-etcd" { + resource_group_name = azurerm_resource_group.cluster.name + + name = "allow-etcd" + network_security_group_name = azurerm_network_security_group.controller.name + priority = "2005" + access = "Allow" + direction = "Inbound" + protocol = "Tcp" + source_port_range = "*" + destination_port_range = "2379-2380" + source_address_prefix = azurerm_subnet.controller.address_prefix + destination_address_prefix = azurerm_subnet.controller.address_prefix +} + +# Allow Prometheus to scrape etcd metrics +resource "azurerm_network_security_rule" "controller-etcd-metrics" { + resource_group_name = azurerm_resource_group.cluster.name + + name = "allow-etcd-metrics" + network_security_group_name = azurerm_network_security_group.controller.name + priority = "2010" + access = "Allow" + direction = "Inbound" + protocol = "Tcp" + source_port_range = "*" + destination_port_range = "2381" + source_address_prefix = azurerm_subnet.worker.address_prefix + destination_address_prefix = azurerm_subnet.controller.address_prefix +} + +# Allow Prometheus to scrape kube-proxy metrics +resource "azurerm_network_security_rule" "controller-kube-proxy" { + resource_group_name = azurerm_resource_group.cluster.name + + name = "allow-kube-proxy-metrics" + network_security_group_name = azurerm_network_security_group.controller.name + priority = "2011" + access = "Allow" + direction = "Inbound" + protocol = "Tcp" + source_port_range = "*" + destination_port_range = "10249" + source_address_prefix = azurerm_subnet.worker.address_prefix + destination_address_prefix = azurerm_subnet.controller.address_prefix +} + +# Allow Prometheus to scrape kube-scheduler and kube-controller-manager metrics +resource "azurerm_network_security_rule" "controller-kube-metrics" { + resource_group_name = azurerm_resource_group.cluster.name + + name = "allow-kube-metrics" + network_security_group_name = azurerm_network_security_group.controller.name + priority = "2012" + access = "Allow" + direction = "Inbound" + protocol = "Tcp" + source_port_range = "*" + destination_port_range = "10251-10252" + source_address_prefix = azurerm_subnet.worker.address_prefix + destination_address_prefix = azurerm_subnet.controller.address_prefix +} + +resource "azurerm_network_security_rule" "controller-apiserver" { + resource_group_name = azurerm_resource_group.cluster.name + + name = "allow-apiserver" + network_security_group_name = azurerm_network_security_group.controller.name + priority = "2015" + access = "Allow" + direction = "Inbound" + protocol = "Tcp" + source_port_range = "*" + destination_port_range = "6443" + source_address_prefix = "*" + destination_address_prefix = azurerm_subnet.controller.address_prefix +} + +resource "azurerm_network_security_rule" "controller-vxlan" { + resource_group_name = azurerm_resource_group.cluster.name + + name = "allow-vxlan" + network_security_group_name = azurerm_network_security_group.controller.name + priority = "2020" + access = "Allow" + direction = "Inbound" + protocol = "Udp" + source_port_range = "*" + destination_port_range = "4789" + source_address_prefixes = [azurerm_subnet.controller.address_prefix, azurerm_subnet.worker.address_prefix] + destination_address_prefix = azurerm_subnet.controller.address_prefix +} + +# Allow Prometheus to scrape node-exporter daemonset +resource "azurerm_network_security_rule" "controller-node-exporter" { + resource_group_name = azurerm_resource_group.cluster.name + + name = "allow-node-exporter" + network_security_group_name = azurerm_network_security_group.controller.name + priority = "2025" + access = "Allow" + direction = "Inbound" + protocol = "Tcp" + source_port_range = "*" + destination_port_range = "9100" + source_address_prefix = azurerm_subnet.worker.address_prefix + destination_address_prefix = azurerm_subnet.controller.address_prefix +} + +# Allow apiserver to access kubelet's for exec, log, port-forward +resource "azurerm_network_security_rule" "controller-kubelet" { + resource_group_name = azurerm_resource_group.cluster.name + + name = "allow-kubelet" + network_security_group_name = azurerm_network_security_group.controller.name + priority = "2030" + access = "Allow" + direction = "Inbound" + protocol = "Tcp" + source_port_range = "*" + destination_port_range = "10250" + + # allow Prometheus to scrape kubelet metrics too + source_address_prefixes = [azurerm_subnet.controller.address_prefix, azurerm_subnet.worker.address_prefix] + destination_address_prefix = azurerm_subnet.controller.address_prefix +} + +# Override Azure AllowVNetInBound and AllowAzureLoadBalancerInBound +# https://docs.microsoft.com/en-us/azure/virtual-network/security-overview#default-security-rules + +resource "azurerm_network_security_rule" "controller-allow-loadblancer" { + resource_group_name = azurerm_resource_group.cluster.name + + name = "allow-loadbalancer" + network_security_group_name = azurerm_network_security_group.controller.name + priority = "3000" + access = "Allow" + direction = "Inbound" + protocol = "*" + source_port_range = "*" + destination_port_range = "*" + source_address_prefix = "AzureLoadBalancer" + destination_address_prefix = "*" +} + +resource "azurerm_network_security_rule" "controller-deny-all" { + resource_group_name = azurerm_resource_group.cluster.name + + name = "deny-all" + network_security_group_name = azurerm_network_security_group.controller.name + priority = "3005" + access = "Deny" + direction = "Inbound" + protocol = "*" + source_port_range = "*" + destination_port_range = "*" + source_address_prefix = "*" + destination_address_prefix = "*" +} + +# Worker security group + +resource "azurerm_network_security_group" "worker" { + resource_group_name = azurerm_resource_group.cluster.name + + name = "${var.cluster_name}-worker" + location = azurerm_resource_group.cluster.location +} + +resource "azurerm_network_security_rule" "worker-ssh" { + resource_group_name = azurerm_resource_group.cluster.name + + name = "allow-ssh" + network_security_group_name = azurerm_network_security_group.worker.name + priority = "2000" + access = "Allow" + direction = "Inbound" + protocol = "Tcp" + source_port_range = "*" + destination_port_range = "22" + source_address_prefix = azurerm_subnet.controller.address_prefix + destination_address_prefix = azurerm_subnet.worker.address_prefix +} + +resource "azurerm_network_security_rule" "worker-http" { + resource_group_name = azurerm_resource_group.cluster.name + + name = "allow-http" + network_security_group_name = azurerm_network_security_group.worker.name + priority = "2005" + access = "Allow" + direction = "Inbound" + protocol = "Tcp" + source_port_range = "*" + destination_port_range = "80" + source_address_prefix = "*" + destination_address_prefix = azurerm_subnet.worker.address_prefix +} + +resource "azurerm_network_security_rule" "worker-https" { + resource_group_name = azurerm_resource_group.cluster.name + + name = "allow-https" + network_security_group_name = azurerm_network_security_group.worker.name + priority = "2010" + access = "Allow" + direction = "Inbound" + protocol = "Tcp" + source_port_range = "*" + destination_port_range = "443" + source_address_prefix = "*" + destination_address_prefix = azurerm_subnet.worker.address_prefix +} + +resource "azurerm_network_security_rule" "worker-vxlan" { + resource_group_name = azurerm_resource_group.cluster.name + + name = "allow-vxlan" + network_security_group_name = azurerm_network_security_group.worker.name + priority = "2015" + access = "Allow" + direction = "Inbound" + protocol = "Udp" + source_port_range = "*" + destination_port_range = "4789" + source_address_prefixes = [azurerm_subnet.controller.address_prefix, azurerm_subnet.worker.address_prefix] + destination_address_prefix = azurerm_subnet.worker.address_prefix +} + +# Allow Prometheus to scrape node-exporter daemonset +resource "azurerm_network_security_rule" "worker-node-exporter" { + resource_group_name = azurerm_resource_group.cluster.name + + name = "allow-node-exporter" + network_security_group_name = azurerm_network_security_group.worker.name + priority = "2020" + access = "Allow" + direction = "Inbound" + protocol = "Tcp" + source_port_range = "*" + destination_port_range = "9100" + source_address_prefix = azurerm_subnet.worker.address_prefix + destination_address_prefix = azurerm_subnet.worker.address_prefix +} + +# Allow Prometheus to scrape kube-proxy +resource "azurerm_network_security_rule" "worker-kube-proxy" { + resource_group_name = azurerm_resource_group.cluster.name + + name = "allow-kube-proxy" + network_security_group_name = azurerm_network_security_group.worker.name + priority = "2024" + access = "Allow" + direction = "Inbound" + protocol = "Tcp" + source_port_range = "*" + destination_port_range = "10249" + source_address_prefix = azurerm_subnet.worker.address_prefix + destination_address_prefix = azurerm_subnet.worker.address_prefix +} + +# Allow apiserver to access kubelet's for exec, log, port-forward +resource "azurerm_network_security_rule" "worker-kubelet" { + resource_group_name = azurerm_resource_group.cluster.name + + name = "allow-kubelet" + network_security_group_name = azurerm_network_security_group.worker.name + priority = "2025" + access = "Allow" + direction = "Inbound" + protocol = "Tcp" + source_port_range = "*" + destination_port_range = "10250" + + # allow Prometheus to scrape kubelet metrics too + source_address_prefixes = [azurerm_subnet.controller.address_prefix, azurerm_subnet.worker.address_prefix] + destination_address_prefix = azurerm_subnet.worker.address_prefix +} + +# Override Azure AllowVNetInBound and AllowAzureLoadBalancerInBound +# https://docs.microsoft.com/en-us/azure/virtual-network/security-overview#default-security-rules + +resource "azurerm_network_security_rule" "worker-allow-loadblancer" { + resource_group_name = azurerm_resource_group.cluster.name + + name = "allow-loadbalancer" + network_security_group_name = azurerm_network_security_group.worker.name + priority = "3000" + access = "Allow" + direction = "Inbound" + protocol = "*" + source_port_range = "*" + destination_port_range = "*" + source_address_prefix = "AzureLoadBalancer" + destination_address_prefix = "*" +} + +resource "azurerm_network_security_rule" "worker-deny-all" { + resource_group_name = azurerm_resource_group.cluster.name + + name = "deny-all" + network_security_group_name = azurerm_network_security_group.worker.name + priority = "3005" + access = "Deny" + direction = "Inbound" + protocol = "*" + source_port_range = "*" + destination_port_range = "*" + source_address_prefix = "*" + destination_address_prefix = "*" +} + diff --git a/azure/fedora-coreos/kubernetes/ssh.tf b/azure/fedora-coreos/kubernetes/ssh.tf new file mode 100644 index 000000000..04f54393e --- /dev/null +++ b/azure/fedora-coreos/kubernetes/ssh.tf @@ -0,0 +1,59 @@ +locals { + # format assets for distribution + assets_bundle = [ + # header with the unpack location + for key, value in module.bootstrap.assets_dist : + format("##### %s\n%s", key, value) + ] +} + +# Secure copy assets to controllers. +resource "null_resource" "copy-controller-secrets" { + count = var.controller_count + + depends_on = [ + module.bootstrap, + azurerm_linux_virtual_machine.controllers + ] + + connection { + type = "ssh" + host = azurerm_public_ip.controllers.*.ip_address[count.index] + user = "core" + timeout = "15m" + } + + provisioner "file" { + content = join("\n", local.assets_bundle) + destination = "$HOME/assets" + } + + provisioner "remote-exec" { + inline = [ + "sudo /opt/bootstrap/layout", + ] + } +} + +# Connect to a controller to perform one-time cluster bootstrap. +resource "null_resource" "bootstrap" { + depends_on = [ + null_resource.copy-controller-secrets, + module.workers, + azurerm_dns_a_record.apiserver, + ] + + connection { + type = "ssh" + host = azurerm_public_ip.controllers.*.ip_address[0] + user = "core" + timeout = "15m" + } + + provisioner "remote-exec" { + inline = [ + "sudo systemctl start bootstrap", + ] + } +} + diff --git a/azure/fedora-coreos/kubernetes/variables.tf b/azure/fedora-coreos/kubernetes/variables.tf new file mode 100644 index 000000000..1bff1e7e2 --- /dev/null +++ b/azure/fedora-coreos/kubernetes/variables.tf @@ -0,0 +1,143 @@ +variable "cluster_name" { + type = string + description = "Unique cluster name (prepended to dns_zone)" +} + +# Azure + +variable "region" { + type = string + description = "Azure Region (e.g. centralus , see `az account list-locations --output table`)" +} + +variable "dns_zone" { + type = string + description = "Azure DNS Zone (e.g. azure.example.com)" +} + +variable "dns_zone_group" { + type = string + description = "Resource group where the Azure DNS Zone resides (e.g. global)" +} + +# instances + +variable "controller_count" { + type = number + description = "Number of controllers (i.e. masters)" + default = 1 +} + +variable "worker_count" { + type = number + description = "Number of workers" + default = 1 +} + +variable "controller_type" { + type = string + description = "Machine type for controllers (see `az vm list-skus --location centralus`)" + default = "Standard_B2s" +} + +variable "worker_type" { + type = string + description = "Machine type for workers (see `az vm list-skus --location centralus`)" + default = "Standard_DS1_v2" +} + +variable "os_image" { + type = string + description = "Fedora CoreOS image for instances" +} + +variable "disk_size" { + type = number + description = "Size of the disk in GB" + default = 40 +} + +variable "worker_priority" { + type = string + description = "Set worker priority to Spot to use reduced cost surplus capacity, with the tradeoff that instances can be deallocated at any time." + default = "Regular" +} + +variable "controller_snippets" { + type = list(string) + description = "Controller Fedora CoreOS Config snippets" + default = [] +} + +variable "worker_snippets" { + type = list(string) + description = "Worker Fedora CoreOS Config snippets" + default = [] +} + +# configuration + +variable "ssh_authorized_key" { + type = string + description = "SSH public key for user 'core'" +} + +variable "networking" { + type = string + description = "Choice of networking provider (flannel or calico)" + default = "calico" +} + +variable "host_cidr" { + type = string + description = "CIDR IPv4 range to assign to instances" + default = "10.0.0.0/16" +} + +variable "pod_cidr" { + type = string + description = "CIDR IPv4 range to assign Kubernetes pods" + default = "10.2.0.0/16" +} + +variable "service_cidr" { + type = string + description = < /dev/null; do sleep 1; done' + [Install] + RequiredBy=kubelet.service + - name: kubelet.service + enabled: true + contents: | + [Unit] + Description=Kubelet via Hyperkube (System Container) + Wants=rpc-statd.service + [Service] + ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d + ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests + ExecStartPre=/bin/mkdir -p /opt/cni/bin + ExecStartPre=/bin/mkdir -p /var/lib/calico + ExecStartPre=/bin/mkdir -p /var/lib/kubelet/volumeplugins + ExecStartPre=/usr/bin/bash -c "grep 'certificate-authority-data' /etc/kubernetes/kubeconfig | awk '{print $2}' | base64 -d > /etc/kubernetes/ca.crt" + ExecStartPre=-/usr/bin/podman rm kubelet + ExecStart=/usr/bin/podman run --name kubelet \ + --privileged \ + --pid host \ + --network host \ + --volume /etc/kubernetes:/etc/kubernetes:ro,z \ + --volume /usr/lib/os-release:/etc/os-release:ro \ + --volume /etc/ssl/certs:/etc/ssl/certs:ro \ + --volume /lib/modules:/lib/modules:ro \ + --volume /run:/run \ + --volume /sys/fs/cgroup:/sys/fs/cgroup:ro \ + --volume /sys/fs/cgroup/systemd:/sys/fs/cgroup/systemd \ + --volume /etc/pki/tls/certs:/usr/share/ca-certificates:ro \ + --volume /var/lib/calico:/var/lib/calico:ro \ + --volume /var/lib/docker:/var/lib/docker \ + --volume /var/lib/kubelet:/var/lib/kubelet:rshared,z \ + --volume /var/log:/var/log \ + --volume /var/run/lock:/var/run/lock:z \ + --volume /opt/cni/bin:/opt/cni/bin:z \ + quay.io/poseidon/kubelet:v1.18.1 \ + --anonymous-auth=false \ + --authentication-token-webhook \ + --authorization-mode=Webhook \ + --cgroup-driver=systemd \ + --cgroups-per-qos=true \ + --enforce-node-allocatable=pods \ + --client-ca-file=/etc/kubernetes/ca.crt \ + --cluster_dns=${cluster_dns_service_ip} \ + --cluster_domain=${cluster_domain_suffix} \ + --cni-conf-dir=/etc/kubernetes/cni/net.d \ + --exit-on-lock-contention \ + --healthz-port=0 \ + --kubeconfig=/etc/kubernetes/kubeconfig \ + --lock-file=/var/run/lock/kubelet.lock \ + --network-plugin=cni \ + --node-labels=node.kubernetes.io/node \ + %{~ for label in split(",", node_labels) ~} + --node-labels=${label} \ + %{~ endfor ~} + --pod-manifest-path=/etc/kubernetes/manifests \ + --read-only-port=0 \ + --volume-plugin-dir=/var/lib/kubelet/volumeplugins + ExecStop=-/usr/bin/podman stop kubelet + Delegate=yes + Restart=always + RestartSec=10 + [Install] + WantedBy=multi-user.target + - name: delete-node.service + enabled: true + contents: | + [Unit] + Description=Delete Kubernetes node on shutdown + [Service] + Type=oneshot + RemainAfterExit=true + ExecStart=/bin/true + ExecStop=/bin/bash -c '/usr/bin/podman run --volume /etc/kubernetes:/etc/kubernetes:ro,z --entrypoint /usr/local/bin/kubectl quay.io/poseidon/kubelet:v1.18.1 --kubeconfig=/etc/kubernetes/kubeconfig delete node $HOSTNAME' + [Install] + WantedBy=multi-user.target +storage: + directories: + - path: /etc/kubernetes + files: + - path: /etc/kubernetes/kubeconfig + mode: 0644 + contents: + inline: | + ${kubeconfig} + - path: /etc/sysctl.d/max-user-watches.conf + contents: + inline: | + fs.inotify.max_user_watches=16184 + - path: /etc/systemd/system.conf.d/accounting.conf + contents: + inline: | + [Manager] + DefaultCPUAccounting=yes + DefaultMemoryAccounting=yes + DefaultBlockIOAccounting=yes +passwd: + users: + - name: core + ssh_authorized_keys: + - ${ssh_authorized_key} + + diff --git a/azure/fedora-coreos/kubernetes/workers/variables.tf b/azure/fedora-coreos/kubernetes/workers/variables.tf new file mode 100644 index 000000000..0e12fd08c --- /dev/null +++ b/azure/fedora-coreos/kubernetes/workers/variables.tf @@ -0,0 +1,98 @@ +variable "name" { + type = string + description = "Unique name for the worker pool" +} + +# Azure + +variable "region" { + type = string + description = "Must be set to the Azure Region of cluster" +} + +variable "resource_group_name" { + type = string + description = "Must be set to the resource group name of cluster" +} + +variable "subnet_id" { + type = string + description = "Must be set to the `worker_subnet_id` output by cluster" +} + +variable "security_group_id" { + type = string + description = "Must be set to the `worker_security_group_id` output by cluster" +} + +variable "backend_address_pool_id" { + type = string + description = "Must be set to the `worker_backend_address_pool_id` output by cluster" +} + +# instances + +variable "worker_count" { + type = number + description = "Number of instances" + default = 1 +} + +variable "vm_type" { + type = string + description = "Machine type for instances (see `az vm list-skus --location centralus`)" + default = "Standard_DS1_v2" +} + +variable "os_image" { + type = string + description = "Fedora CoreOS image for instances" +} + +variable "priority" { + type = string + description = "Set priority to Spot to use reduced cost surplus capacity, with the tradeoff that instances can be evicted at any time." + default = "Regular" +} + +variable "snippets" { + type = list(string) + description = "Fedora CoreOS Config snippets" + default = [] +} + +# configuration + +variable "kubeconfig" { + type = string + description = "Must be set to `kubeconfig` output by cluster" +} + +variable "ssh_authorized_key" { + type = string + description = "SSH public key for user 'core'" +} + +variable "service_cidr" { + type = string + description = < 24m v1.18.1 +ramius-worker-000001 Ready 25m v1.18.1 +ramius-worker-000002 Ready 24m v1.18.1 +``` + +List the pods. + +``` +$ kubectl get pods --all-namespaces +NAMESPACE NAME READY STATUS RESTARTS AGE +kube-system coredns-7c6fbb4f4b-b6qzx 1/1 Running 0 26m +kube-system coredns-7c6fbb4f4b-j2k3d 1/1 Running 0 26m +kube-system calico-node-1m5bf 2/2 Running 0 26m +kube-system calico-node-7jmr1 2/2 Running 0 26m +kube-system calico-node-bknc8 2/2 Running 0 26m +kube-system kube-apiserver-ramius-controller-0 1/1 Running 0 26m +kube-system kube-controller-manager-ramius-controller-0 1/1 Running 0 26m +kube-system kube-proxy-j4vpq 1/1 Running 0 26m +kube-system kube-proxy-jxr5d 1/1 Running 0 26m +kube-system kube-proxy-lbdw5 1/1 Running 0 26m +kube-system kube-scheduler-ramius-controller-0 1/1 Running 0 26m +``` + +## Going Further + +Learn about [maintenance](/topics/maintenance/) and [addons](/addons/overview/). + +## Variables + +Check the [variables.tf](https://github.com/poseidon/typhoon/blob/master/azure/fedora-coreos/kubernetes/variables.tf) source. + +### Required + +| Name | Description | Example | +|:-----|:------------|:--------| +| cluster_name | Unique cluster name (prepended to dns_zone) | "ramius" | +| region | Azure region | "centralus" | +| dns_zone | Azure DNS zone | "azure.example.com" | +| dns_zone_group | Resource group where the Azure DNS zone resides | "global" | +| os_image | Fedora CoreOS image for instances | "/subscriptions/..../custom-image" | +| ssh_authorized_key | SSH public key for user 'core' | "ssh-rsa AAAAB3NZ..." | + +!!! tip + Regions are shown in [docs](https://azure.microsoft.com/en-us/global-infrastructure/regions/) or with `az account list-locations --output table`. + +#### DNS Zone + +Clusters create a DNS A record `${cluster_name}.${dns_zone}` to resolve a load balancer backed by controller instances. This FQDN is used by workers and `kubectl` to access the apiserver(s). In this example, the cluster's apiserver would be accessible at `ramius.azure.example.com`. + +You'll need a registered domain name or delegated subdomain on Azure DNS. You can set this up once and create many clusters with unique names. + +```tf +# Azure resource group for DNS zone +resource "azurerm_resource_group" "global" { + name = "global" + location = "centralus" +} + +# DNS zone for clusters +resource "azurerm_dns_zone" "clusters" { + resource_group_name = azurerm_resource_group.global.name + + name = "azure.example.com" + zone_type = "Public" +} +``` + +Reference the DNS zone with `azurerm_dns_zone.clusters.name` and its resource group with `"azurerm_resource_group.global.name`. + +!!! tip "" + If you have an existing domain name with a zone file elsewhere, just delegate a subdomain that can be managed on Azure DNS (e.g. azure.mydomain.com) and [update nameservers](https://docs.microsoft.com/en-us/azure/dns/dns-delegate-domain-azure-dns). + +### Optional + +| Name | Description | Default | Example | +|:-----|:------------|:--------|:--------| +| controller_count | Number of controllers (i.e. masters) | 1 | 1 | +| worker_count | Number of workers | 1 | 3 | +| controller_type | Machine type for controllers | "Standard_B2s" | See below | +| worker_type | Machine type for workers | "Standard_DS1_v2" | See below | +| disk_size | Size of the disk in GB | 40 | 100 | +| worker_priority | Set priority to Spot to use reduced cost surplus capacity, with the tradeoff that instances can be deallocated at any time | Regular | Spot | +| controller_snippets | Controller Fedora CoreOS Config snippets | [] | [example](/advanced/customization/#usage) | +| worker_snippets | Worker Fedora CoreOS Config snippets | [] | [example](/advanced/customization/#usage) | +| networking | Choice of networking provider | "calico" | "flannel" or "calico" | +| host_cidr | CIDR IPv4 range to assign to instances | "10.0.0.0/16" | "10.0.0.0/20" | +| pod_cidr | CIDR IPv4 range to assign to Kubernetes pods | "10.2.0.0/16" | "10.22.0.0/16" | +| service_cidr | CIDR IPv4 range to assign to Kubernetes services | "10.3.0.0/16" | "10.3.0.0/24" | +| worker_node_labels | List of initial worker node labels | [] | ["worker-pool=default"] | + +Check the list of valid [machine types](https://azure.microsoft.com/en-us/pricing/details/virtual-machines/linux/) and their [specs](https://docs.microsoft.com/en-us/azure/virtual-machines/linux/sizes-general). Use `az vm list-skus` to get the identifier. + +!!! warning + Unlike AWS and GCP, Azure requires its *virtual* networks to have non-overlapping IPv4 CIDRs (yeah, go figure). Instead of each cluster just using `10.0.0.0/16` for instances, each Azure cluster's `host_cidr` must be non-overlapping (e.g. 10.0.0.0/20 for the 1st cluster, 10.0.16.0/20 for the 2nd cluster, etc). + +!!! warning + Do not choose a `controller_type` smaller than `Standard_B2s`. Smaller instances are not sufficient for running a controller. + +#### Spot Priority + +Add `worker_priority=Spot` to use [Spot Priority](https://docs.microsoft.com/en-us/azure/virtual-machines/linux/spot-vms) workers that run on Azure's surplus capacity at lower cost, but with the tradeoff that they can be deallocated at random. Spot priority VMs are Azure's analog to AWS spot instances or GCP premptible instances. diff --git a/docs/fedora-coreos/digitalocean.md b/docs/fedora-coreos/digitalocean.md index 3af5f97e1..b0c209a28 100644 --- a/docs/fedora-coreos/digitalocean.md +++ b/docs/fedora-coreos/digitalocean.md @@ -63,7 +63,7 @@ provider "ct" { Fedora CoreOS publishes images for DigitalOcean, but does not yet upload them. DigitalOcean allows [custom images](https://blog.digitalocean.com/custom-images/) to be uploaded via URL or file. -Import a [Fedora CoreOS](https://getfedora.org/en/coreos/download?tab=cloud_operators&stream=stable) image via URL to desired a region(s). Reference the DigitalOcean image and set the `os_image` in the next step. +Import a [Fedora CoreOS](https://getfedora.org/en/coreos/download?tab=cloud_operators&stream=stable) image via URL to desired a region(s). ```tf data "digitalocean_image" "fedora-coreos-31-20200323-3-2" { @@ -71,6 +71,8 @@ data "digitalocean_image" "fedora-coreos-31-20200323-3-2" { } ``` +Set the [os_image](#variables) in the next step. + ## Cluster Define a Kubernetes cluster using the module `digital-ocean/fedora-coreos/kubernetes`. @@ -83,9 +85,9 @@ module "nemo" { cluster_name = "nemo" region = "nyc3" dns_zone = "digital-ocean.example.com" - os_image = data.digitalocean_image.fedora-coreos-31-20200323-3-2.id # configuration + os_image = data.digitalocean_image.fedora-coreos-31-20200323-3-2.id ssh_fingerprints = ["d7:9d:79:ae:56:32:73:79:95:88:e3:a2:ab:5d:45:e7"] # optional diff --git a/docs/fedora-coreos/google-cloud.md b/docs/fedora-coreos/google-cloud.md index 7ebcd2e18..482900b3f 100644 --- a/docs/fedora-coreos/google-cloud.md +++ b/docs/fedora-coreos/google-cloud.md @@ -82,6 +82,8 @@ Create a Compute Engine image from the file. gcloud compute images create fedora-coreos-31-20200323-3-2 --source-uri gs://BUCKET/fedora-coreos-31.20200323.3.2-gcp.x86_64.tar.gz ``` +Set the [os_image](#variables) in the next step. + ## Cluster Define a Kubernetes cluster using the module `google-cloud/fedora-coreos/kubernetes`. @@ -96,10 +98,8 @@ module "yavin" { dns_zone = "example.com" dns_zone_name = "example-zone" - # custom image name from above - os_image = "fedora-coreos-31-20200323-3-2" - # configuration + os_image = "fedora-coreos-31-20200323-3-2" ssh_authorized_key = "ssh-rsa AAAAB3Nz..." # optional diff --git a/docs/index.md b/docs/index.md index f3407483a..b27c4017b 100644 --- a/docs/index.md +++ b/docs/index.md @@ -26,6 +26,7 @@ Typhoon is available for [Fedora CoreOS](https://getfedora.org/coreos/). | Platform | Operating System | Terraform Module | Status | |---------------|------------------|------------------|--------| | AWS | Fedora CoreOS | [aws/fedora-coreos/kubernetes](fedora-coreos/aws.md) | stable | +| Azure | Fedora CoreOS | [azure/fedora-coreos/kubernetes](fedora-coreos/azure.md) | alpha | | Bare-Metal | Fedora CoreOS | [bare-metal/fedora-coreos/kubernetes](fedora-coreos/bare-metal.md) | beta | | DigitalOcean | Fedora CoreOS | [digital-ocean/fedora-coreos/kubernetes](fedora-coreos/digitalocean.md) | alpha | | Google Cloud | Fedora CoreOS | [google-cloud/fedora-coreos/kubernetes](google-cloud/fedora-coreos/kubernetes) | beta | @@ -54,7 +55,7 @@ Typhoon is available for CoreOS Container Linux ([no updates](https://coreos.com ## Documentation * Architecture [concepts](architecture/concepts.md) and [operating-systems](architecture/operating-systems.md) -* Fedora CoreOS tutorials for [AWS](fedora-coreos/aws.md), [Bare-Metal](fedora-coreos/bare-metal.md), [DigitalOcean](fedora-coreos/digitalocean.md), and [Google Cloud](fedora-coreos/google-cloud.md) +* Fedora CoreOS tutorials for [AWS](fedora-coreos/aws.md), [Azure](fedora-coreos/azure.md), [Bare-Metal](fedora-coreos/bare-metal.md), [DigitalOcean](fedora-coreos/digitalocean.md), and [Google Cloud](fedora-coreos/google-cloud.md) * Flatcar Linux tutorials for [AWS](cl/aws.md), [Azure](cl/azure.md), [Bare-Metal](cl/bare-metal.md), [DigitalOcean](cl/digital-ocean.md), and [Google Cloud](cl/google-cloud.md) ## Example diff --git a/google-cloud/fedora-coreos/kubernetes/README.md b/google-cloud/fedora-coreos/kubernetes/README.md index 94cd7d8aa..1eb918360 100644 --- a/google-cloud/fedora-coreos/kubernetes/README.md +++ b/google-cloud/fedora-coreos/kubernetes/README.md @@ -14,10 +14,10 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster * Kubernetes v1.18.1 (upstream) * Single or multi-master, [Calico](https://www.projectcalico.org/) or [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/) -* Advanced features like [worker pools](https://typhoon.psdn.io/advanced/worker-pools/), [preemptible](https://typhoon.psdn.io/cl/google-cloud/#preemption) workers, and [snippets](https://typhoon.psdn.io/advanced/customization/#container-linux) customization +* Advanced features like [worker pools](https://typhoon.psdn.io/advanced/worker-pools/), [preemptible](https://typhoon.psdn.io/fedora-coreos/google-cloud/#preemption) workers, and [snippets](https://typhoon.psdn.io/advanced/customization/) customization * Ready for Ingress, Prometheus, Grafana, and other optional [addons](https://typhoon.psdn.io/addons/overview/) ## Docs -Please see the [official docs](https://typhoon.psdn.io) and the Google Cloud [tutorial](https://typhoon.psdn.io/cl/google-cloud/). +Please see the [official docs](https://typhoon.psdn.io) and the Google Cloud [tutorial](https://typhoon.psdn.io/fedora-coreos/google-cloud/). From e2d4af43be42d6d7ebcca6c3116faa2faf5730a8 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Sun, 12 Apr 2020 23:20:04 -0700 Subject: [PATCH 392/523] Fix Fedora CoreOS Azure MTU with Calico * With Calico VXLAN on Fedora CoreOS the 1450 MTU should be used --- azure/fedora-coreos/kubernetes/bootstrap.tf | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/azure/fedora-coreos/kubernetes/bootstrap.tf b/azure/fedora-coreos/kubernetes/bootstrap.tf index e22a3401e..4955db1b6 100644 --- a/azure/fedora-coreos/kubernetes/bootstrap.tf +++ b/azure/fedora-coreos/kubernetes/bootstrap.tf @@ -10,9 +10,8 @@ module "bootstrap" { networking = var.networking # only effective with Calico networking - # we should be able to use 1450 MTU, but in practice, 1410 was needed network_encapsulation = "vxlan" - network_mtu = "1410" + network_mtu = "1450" pod_cidr = var.pod_cidr service_cidr = var.service_cidr From 671eacb86edfab131ae4dc5713c30dbc52e32aab Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Thu, 16 Apr 2020 23:40:52 -0700 Subject: [PATCH 393/523] Update Kubernetes from v1.18.1 to v1.18.2 * https://github.com/kubernetes/kubernetes/blob/master/CHANGELOG/CHANGELOG-1.18.md#changelog-since-v1181 --- CHANGES.md | 10 +++++++++- README.md | 10 +++++----- aws/container-linux/kubernetes/README.md | 2 +- aws/container-linux/kubernetes/bootstrap.tf | 2 +- aws/container-linux/kubernetes/cl/controller.yaml | 4 ++-- .../kubernetes/workers/cl/worker.yaml | 4 ++-- aws/fedora-coreos/kubernetes/README.md | 2 +- aws/fedora-coreos/kubernetes/bootstrap.tf | 2 +- aws/fedora-coreos/kubernetes/fcc/controller.yaml | 4 ++-- .../kubernetes/workers/fcc/worker.yaml | 4 ++-- azure/container-linux/kubernetes/README.md | 2 +- azure/container-linux/kubernetes/bootstrap.tf | 2 +- .../container-linux/kubernetes/cl/controller.yaml | 4 ++-- .../kubernetes/workers/cl/worker.yaml | 4 ++-- azure/fedora-coreos/kubernetes/README.md | 2 +- azure/fedora-coreos/kubernetes/bootstrap.tf | 2 +- azure/fedora-coreos/kubernetes/fcc/controller.yaml | 4 ++-- .../kubernetes/workers/fcc/worker.yaml | 4 ++-- bare-metal/container-linux/kubernetes/README.md | 2 +- bare-metal/container-linux/kubernetes/bootstrap.tf | 2 +- .../container-linux/kubernetes/cl/controller.yaml | 4 ++-- .../container-linux/kubernetes/cl/worker.yaml | 2 +- bare-metal/fedora-coreos/kubernetes/README.md | 2 +- bare-metal/fedora-coreos/kubernetes/bootstrap.tf | 2 +- .../fedora-coreos/kubernetes/fcc/controller.yaml | 4 ++-- .../fedora-coreos/kubernetes/fcc/worker.yaml | 2 +- digital-ocean/container-linux/kubernetes/README.md | 2 +- .../container-linux/kubernetes/bootstrap.tf | 2 +- .../container-linux/kubernetes/cl/controller.yaml | 4 ++-- .../container-linux/kubernetes/cl/worker.yaml | 4 ++-- digital-ocean/fedora-coreos/kubernetes/README.md | 2 +- .../fedora-coreos/kubernetes/bootstrap.tf | 2 +- .../fedora-coreos/kubernetes/fcc/controller.yaml | 4 ++-- .../fedora-coreos/kubernetes/fcc/worker.yaml | 4 ++-- docs/advanced/worker-pools.md | 14 +++++++------- docs/cl/aws.md | 10 +++++----- docs/cl/azure.md | 10 +++++----- docs/cl/bare-metal.md | 10 +++++----- docs/cl/digital-ocean.md | 10 +++++----- docs/cl/google-cloud.md | 10 +++++----- docs/fedora-coreos/aws.md | 10 +++++----- docs/fedora-coreos/azure.md | 10 +++++----- docs/fedora-coreos/bare-metal.md | 10 +++++----- docs/fedora-coreos/digitalocean.md | 10 +++++----- docs/fedora-coreos/google-cloud.md | 8 ++++---- docs/index.md | 10 +++++----- docs/topics/maintenance.md | 8 ++++---- google-cloud/container-linux/kubernetes/README.md | 2 +- .../container-linux/kubernetes/bootstrap.tf | 2 +- .../container-linux/kubernetes/cl/controller.yaml | 4 ++-- .../kubernetes/workers/cl/worker.yaml | 4 ++-- google-cloud/fedora-coreos/kubernetes/README.md | 2 +- google-cloud/fedora-coreos/kubernetes/bootstrap.tf | 2 +- .../fedora-coreos/kubernetes/fcc/controller.yaml | 4 ++-- .../kubernetes/workers/fcc/worker.yaml | 4 ++-- 55 files changed, 137 insertions(+), 129 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 251887930..26a194eca 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -4,18 +4,26 @@ Notable changes between versions. ## Latest +* Kubernetes [v1.18.2](https://github.com/kubernetes/kubernetes/blob/master/CHANGELOG/CHANGELOG-1.18.md#v1182) * Choose Fedora CoreOS or Flatcar Linux (**action required**) * Use a `fedora-coreos` module for Fedora CoreOS * Use a `container-linux` module for Flatcar Linux * CoreOS Container Linux [won't receive updates](https://coreos.com/os/eol/) after May 2020 +### Fedora CoreOS + +#### Azure + +* Add support for Fedora CoreOS ([#704](https://github.com/poseidon/typhoon/pull/704)) + +### Flatcar Linux / Container Linux + #### AWS * Change Container Linux `os_image` default from `coreos-stable` to `flatcar-stable` ([#702](https://github.com/poseidon/typhoon/pull/702)) #### Azure -* Add support for Fedora CoreOS ([#704](https://github.com/poseidon/typhoon/pull/704)) * Change Container Linux `os_image` default from `coreos-stable` to `flatcar-stable` ([#702](https://github.com/poseidon/typhoon/pull/702)) #### Bare-Metal diff --git a/README.md b/README.md index 863f5e04b..bc327d909 100644 --- a/README.md +++ b/README.md @@ -11,7 +11,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features -* Kubernetes v1.18.1 (upstream) +* Kubernetes v1.18.2 (upstream) * Single or multi-master, [Calico](https://www.projectcalico.org/) or [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/) * Advanced features like [worker pools](https://typhoon.psdn.io/advanced/worker-pools/), [preemptible](https://typhoon.psdn.io/cl/google-cloud/#preemption) workers, and [snippets](https://typhoon.psdn.io/advanced/customization/#container-linux) customization @@ -64,7 +64,7 @@ Define a Kubernetes cluster by using the Terraform module for your chosen platfo ```tf module "yavin" { - source = "git::https://github.com/poseidon/typhoon//google-cloud/container-linux/kubernetes?ref=v1.18.1" + source = "git::https://github.com/poseidon/typhoon//google-cloud/container-linux/kubernetes?ref=v1.18.2" # Google Cloud cluster_name = "yavin" @@ -103,9 +103,9 @@ In 4-8 minutes (varies by platform), the cluster will be ready. This Google Clou $ export KUBECONFIG=/home/user/.kube/configs/yavin-config $ kubectl get nodes NAME ROLES STATUS AGE VERSION -yavin-controller-0.c.example-com.internal Ready 6m v1.18.1 -yavin-worker-jrbf.c.example-com.internal Ready 5m v1.18.1 -yavin-worker-mzdm.c.example-com.internal Ready 5m v1.18.1 +yavin-controller-0.c.example-com.internal Ready 6m v1.18.2 +yavin-worker-jrbf.c.example-com.internal Ready 5m v1.18.2 +yavin-worker-mzdm.c.example-com.internal Ready 5m v1.18.2 ``` List the pods. diff --git a/aws/container-linux/kubernetes/README.md b/aws/container-linux/kubernetes/README.md index ad9dd1222..b45143847 100644 --- a/aws/container-linux/kubernetes/README.md +++ b/aws/container-linux/kubernetes/README.md @@ -11,7 +11,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features -* Kubernetes v1.18.1 (upstream) +* Kubernetes v1.18.2 (upstream) * Single or multi-master, [Calico](https://www.projectcalico.org/) or [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/) * Advanced features like [worker pools](https://typhoon.psdn.io/advanced/worker-pools/), [spot](https://typhoon.psdn.io/cl/aws/#spot) workers, and [snippets](https://typhoon.psdn.io/advanced/customization/#container-linux) customization diff --git a/aws/container-linux/kubernetes/bootstrap.tf b/aws/container-linux/kubernetes/bootstrap.tf index a990373ef..1652b95dc 100644 --- a/aws/container-linux/kubernetes/bootstrap.tf +++ b/aws/container-linux/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=1ad53d3b1c1ad75a4ed27f124f772fc5dc025245" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=14d0b2087962a0f2557c184f3f523548ce19bbdc" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/aws/container-linux/kubernetes/cl/controller.yaml b/aws/container-linux/kubernetes/cl/controller.yaml index ef6ceab5a..c9648120b 100644 --- a/aws/container-linux/kubernetes/cl/controller.yaml +++ b/aws/container-linux/kubernetes/cl/controller.yaml @@ -91,7 +91,7 @@ systemd: --mount volume=var-log,target=/var/log \ --volume opt-cni-bin,kind=host,source=/opt/cni/bin \ --mount volume=opt-cni-bin,target=/opt/cni/bin \ - docker://quay.io/poseidon/kubelet:v1.18.1 -- \ + docker://quay.io/poseidon/kubelet:v1.18.2 -- \ --anonymous-auth=false \ --authentication-token-webhook \ --authorization-mode=Webhook \ @@ -134,7 +134,7 @@ systemd: --volume script,kind=host,source=/opt/bootstrap/apply \ --mount volume=script,target=/apply \ --insecure-options=image \ - docker://quay.io/poseidon/kubelet:v1.18.1 \ + docker://quay.io/poseidon/kubelet:v1.18.2 \ --net=host \ --dns=host \ --exec=/apply diff --git a/aws/container-linux/kubernetes/workers/cl/worker.yaml b/aws/container-linux/kubernetes/workers/cl/worker.yaml index fd0382567..63a1e3b6d 100644 --- a/aws/container-linux/kubernetes/workers/cl/worker.yaml +++ b/aws/container-linux/kubernetes/workers/cl/worker.yaml @@ -64,7 +64,7 @@ systemd: --mount volume=var-log,target=/var/log \ --volume opt-cni-bin,kind=host,source=/opt/cni/bin \ --mount volume=opt-cni-bin,target=/opt/cni/bin \ - docker://quay.io/poseidon/kubelet:v1.18.1 -- \ + docker://quay.io/poseidon/kubelet:v1.18.2 -- \ --anonymous-auth=false \ --authentication-token-webhook \ --authorization-mode=Webhook \ @@ -127,7 +127,7 @@ storage: --volume config,kind=host,source=/etc/kubernetes \ --mount volume=config,target=/etc/kubernetes \ --insecure-options=image \ - docker://quay.io/poseidon/kubelet:v1.18.1 \ + docker://quay.io/poseidon/kubelet:v1.18.2 \ --net=host \ --dns=host \ --exec=/usr/local/bin/kubectl -- --kubeconfig=/etc/kubernetes/kubeconfig delete node $(hostname) diff --git a/aws/fedora-coreos/kubernetes/README.md b/aws/fedora-coreos/kubernetes/README.md index d57d0ce9e..3acb9ecd8 100644 --- a/aws/fedora-coreos/kubernetes/README.md +++ b/aws/fedora-coreos/kubernetes/README.md @@ -11,7 +11,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features -* Kubernetes v1.18.1 (upstream) +* Kubernetes v1.18.2 (upstream) * Single or multi-master, [Calico](https://www.projectcalico.org/) or [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/) * Advanced features like [worker pools](https://typhoon.psdn.io/advanced/worker-pools/), [spot](https://typhoon.psdn.io/cl/aws/#spot) workers, and [snippets](https://typhoon.psdn.io/advanced/customization/#container-linux) customization diff --git a/aws/fedora-coreos/kubernetes/bootstrap.tf b/aws/fedora-coreos/kubernetes/bootstrap.tf index f406896c3..d35fd11ef 100644 --- a/aws/fedora-coreos/kubernetes/bootstrap.tf +++ b/aws/fedora-coreos/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=1ad53d3b1c1ad75a4ed27f124f772fc5dc025245" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=14d0b2087962a0f2557c184f3f523548ce19bbdc" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/aws/fedora-coreos/kubernetes/fcc/controller.yaml b/aws/fedora-coreos/kubernetes/fcc/controller.yaml index e5948f262..67a4ba4e7 100644 --- a/aws/fedora-coreos/kubernetes/fcc/controller.yaml +++ b/aws/fedora-coreos/kubernetes/fcc/controller.yaml @@ -79,7 +79,7 @@ systemd: --volume /var/log:/var/log \ --volume /var/run/lock:/var/run/lock:z \ --volume /opt/cni/bin:/opt/cni/bin:z \ - quay.io/poseidon/kubelet:v1.18.1 \ + quay.io/poseidon/kubelet:v1.18.2 \ --anonymous-auth=false \ --authentication-token-webhook \ --authorization-mode=Webhook \ @@ -122,7 +122,7 @@ systemd: --volume /opt/bootstrap/assets:/assets:ro,Z \ --volume /opt/bootstrap/apply:/apply:ro,Z \ --entrypoint=/apply \ - quay.io/poseidon/kubelet:v1.18.1 + quay.io/poseidon/kubelet:v1.18.2 ExecStartPost=/bin/touch /opt/bootstrap/bootstrap.done ExecStartPost=-/usr/bin/podman stop bootstrap storage: diff --git a/aws/fedora-coreos/kubernetes/workers/fcc/worker.yaml b/aws/fedora-coreos/kubernetes/workers/fcc/worker.yaml index 7c41b484c..0501b8de9 100644 --- a/aws/fedora-coreos/kubernetes/workers/fcc/worker.yaml +++ b/aws/fedora-coreos/kubernetes/workers/fcc/worker.yaml @@ -49,7 +49,7 @@ systemd: --volume /var/log:/var/log \ --volume /var/run/lock:/var/run/lock:z \ --volume /opt/cni/bin:/opt/cni/bin:z \ - quay.io/poseidon/kubelet:v1.18.1 \ + quay.io/poseidon/kubelet:v1.18.2 \ --anonymous-auth=false \ --authentication-token-webhook \ --authorization-mode=Webhook \ @@ -87,7 +87,7 @@ systemd: Type=oneshot RemainAfterExit=true ExecStart=/bin/true - ExecStop=/bin/bash -c '/usr/bin/podman run --volume /etc/kubernetes:/etc/kubernetes:ro,z --entrypoint /usr/local/bin/kubectl quay.io/poseidon/kubelet:v1.18.1 --kubeconfig=/etc/kubernetes/kubeconfig delete node $HOSTNAME' + ExecStop=/bin/bash -c '/usr/bin/podman run --volume /etc/kubernetes:/etc/kubernetes:ro,z --entrypoint /usr/local/bin/kubectl quay.io/poseidon/kubelet:v1.18.2 --kubeconfig=/etc/kubernetes/kubeconfig delete node $HOSTNAME' [Install] WantedBy=multi-user.target storage: diff --git a/azure/container-linux/kubernetes/README.md b/azure/container-linux/kubernetes/README.md index 3f2c74adb..29bff5080 100644 --- a/azure/container-linux/kubernetes/README.md +++ b/azure/container-linux/kubernetes/README.md @@ -11,7 +11,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features -* Kubernetes v1.18.1 (upstream) +* Kubernetes v1.18.2 (upstream) * Single or multi-master, [Calico](https://www.projectcalico.org/) or [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/) * Advanced features like [worker pools](https://typhoon.psdn.io/advanced/worker-pools/), [low-priority](https://typhoon.psdn.io/cl/azure/#low-priority) workers, and [snippets](https://typhoon.psdn.io/advanced/customization/#container-linux) customization diff --git a/azure/container-linux/kubernetes/bootstrap.tf b/azure/container-linux/kubernetes/bootstrap.tf index 309c1620c..ef9aea940 100644 --- a/azure/container-linux/kubernetes/bootstrap.tf +++ b/azure/container-linux/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=1ad53d3b1c1ad75a4ed27f124f772fc5dc025245" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=14d0b2087962a0f2557c184f3f523548ce19bbdc" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/azure/container-linux/kubernetes/cl/controller.yaml b/azure/container-linux/kubernetes/cl/controller.yaml index e80272d74..9102a6923 100644 --- a/azure/container-linux/kubernetes/cl/controller.yaml +++ b/azure/container-linux/kubernetes/cl/controller.yaml @@ -90,7 +90,7 @@ systemd: --mount volume=var-log,target=/var/log \ --volume opt-cni-bin,kind=host,source=/opt/cni/bin \ --mount volume=opt-cni-bin,target=/opt/cni/bin \ - docker://quay.io/poseidon/kubelet:v1.18.1 -- \ + docker://quay.io/poseidon/kubelet:v1.18.2 -- \ --anonymous-auth=false \ --authentication-token-webhook \ --authorization-mode=Webhook \ @@ -132,7 +132,7 @@ systemd: --volume script,kind=host,source=/opt/bootstrap/apply \ --mount volume=script,target=/apply \ --insecure-options=image \ - docker://quay.io/poseidon/kubelet:v1.18.1 \ + docker://quay.io/poseidon/kubelet:v1.18.2 \ --net=host \ --dns=host \ --exec=/apply diff --git a/azure/container-linux/kubernetes/workers/cl/worker.yaml b/azure/container-linux/kubernetes/workers/cl/worker.yaml index d2d76636e..e070c975c 100644 --- a/azure/container-linux/kubernetes/workers/cl/worker.yaml +++ b/azure/container-linux/kubernetes/workers/cl/worker.yaml @@ -63,7 +63,7 @@ systemd: --mount volume=var-log,target=/var/log \ --volume opt-cni-bin,kind=host,source=/opt/cni/bin \ --mount volume=opt-cni-bin,target=/opt/cni/bin \ - docker://quay.io/poseidon/kubelet:v1.18.1 -- \ + docker://quay.io/poseidon/kubelet:v1.18.2 -- \ --anonymous-auth=false \ --authentication-token-webhook \ --authorization-mode=Webhook \ @@ -125,7 +125,7 @@ storage: --volume config,kind=host,source=/etc/kubernetes \ --mount volume=config,target=/etc/kubernetes \ --insecure-options=image \ - docker://quay.io/poseidon/kubelet:v1.18.1 \ + docker://quay.io/poseidon/kubelet:v1.18.2 \ --net=host \ --dns=host \ --exec=/usr/local/bin/kubectl -- --kubeconfig=/etc/kubernetes/kubeconfig delete node $(hostname | tr '[:upper:]' '[:lower:]') diff --git a/azure/fedora-coreos/kubernetes/README.md b/azure/fedora-coreos/kubernetes/README.md index e5ee37870..17b80679d 100644 --- a/azure/fedora-coreos/kubernetes/README.md +++ b/azure/fedora-coreos/kubernetes/README.md @@ -11,7 +11,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features -* Kubernetes v1.18.1 (upstream) +* Kubernetes v1.18.2 (upstream) * Single or multi-master, [Calico](https://www.projectcalico.org/) or [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/) * Advanced features like [worker pools](https://typhoon.psdn.io/advanced/worker-pools/), [spot priority](https://typhoon.psdn.io/fedora-coreos/azure/#low-priority) workers, and [snippets](https://typhoon.psdn.io/advanced/customization/) customization diff --git a/azure/fedora-coreos/kubernetes/bootstrap.tf b/azure/fedora-coreos/kubernetes/bootstrap.tf index 4955db1b6..152f4adcb 100644 --- a/azure/fedora-coreos/kubernetes/bootstrap.tf +++ b/azure/fedora-coreos/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=1ad53d3b1c1ad75a4ed27f124f772fc5dc025245" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=14d0b2087962a0f2557c184f3f523548ce19bbdc" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/azure/fedora-coreos/kubernetes/fcc/controller.yaml b/azure/fedora-coreos/kubernetes/fcc/controller.yaml index 06c8d1d27..95af2d12c 100644 --- a/azure/fedora-coreos/kubernetes/fcc/controller.yaml +++ b/azure/fedora-coreos/kubernetes/fcc/controller.yaml @@ -79,7 +79,7 @@ systemd: --volume /var/log:/var/log \ --volume /var/run/lock:/var/run/lock:z \ --volume /opt/cni/bin:/opt/cni/bin:z \ - quay.io/poseidon/kubelet:v1.18.1 \ + quay.io/poseidon/kubelet:v1.18.2 \ --anonymous-auth=false \ --authentication-token-webhook \ --authorization-mode=Webhook \ @@ -123,7 +123,7 @@ systemd: --volume /opt/bootstrap/assets:/assets:ro,Z \ --volume /opt/bootstrap/apply:/apply:ro,Z \ --entrypoint=/apply \ - quay.io/poseidon/kubelet:v1.18.1 + quay.io/poseidon/kubelet:v1.18.2 ExecStartPost=/bin/touch /opt/bootstrap/bootstrap.done ExecStartPost=-/usr/bin/podman stop bootstrap storage: diff --git a/azure/fedora-coreos/kubernetes/workers/fcc/worker.yaml b/azure/fedora-coreos/kubernetes/workers/fcc/worker.yaml index 70f643d86..664f0c6a2 100644 --- a/azure/fedora-coreos/kubernetes/workers/fcc/worker.yaml +++ b/azure/fedora-coreos/kubernetes/workers/fcc/worker.yaml @@ -49,7 +49,7 @@ systemd: --volume /var/log:/var/log \ --volume /var/run/lock:/var/run/lock:z \ --volume /opt/cni/bin:/opt/cni/bin:z \ - quay.io/poseidon/kubelet:v1.18.1 \ + quay.io/poseidon/kubelet:v1.18.2 \ --anonymous-auth=false \ --authentication-token-webhook \ --authorization-mode=Webhook \ @@ -87,7 +87,7 @@ systemd: Type=oneshot RemainAfterExit=true ExecStart=/bin/true - ExecStop=/bin/bash -c '/usr/bin/podman run --volume /etc/kubernetes:/etc/kubernetes:ro,z --entrypoint /usr/local/bin/kubectl quay.io/poseidon/kubelet:v1.18.1 --kubeconfig=/etc/kubernetes/kubeconfig delete node $HOSTNAME' + ExecStop=/bin/bash -c '/usr/bin/podman run --volume /etc/kubernetes:/etc/kubernetes:ro,z --entrypoint /usr/local/bin/kubectl quay.io/poseidon/kubelet:v1.18.2 --kubeconfig=/etc/kubernetes/kubeconfig delete node $HOSTNAME' [Install] WantedBy=multi-user.target storage: diff --git a/bare-metal/container-linux/kubernetes/README.md b/bare-metal/container-linux/kubernetes/README.md index d9e47720b..5c52402e8 100644 --- a/bare-metal/container-linux/kubernetes/README.md +++ b/bare-metal/container-linux/kubernetes/README.md @@ -11,7 +11,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features -* Kubernetes v1.18.1 (upstream) +* Kubernetes v1.18.2 (upstream) * Single or multi-master, [Calico](https://www.projectcalico.org/) or [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/) * Advanced features like [snippets](https://typhoon.psdn.io/advanced/customization/#container-linux) customization diff --git a/bare-metal/container-linux/kubernetes/bootstrap.tf b/bare-metal/container-linux/kubernetes/bootstrap.tf index 1f29c12fb..6d577ae0c 100644 --- a/bare-metal/container-linux/kubernetes/bootstrap.tf +++ b/bare-metal/container-linux/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=1ad53d3b1c1ad75a4ed27f124f772fc5dc025245" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=14d0b2087962a0f2557c184f3f523548ce19bbdc" cluster_name = var.cluster_name api_servers = [var.k8s_domain_name] diff --git a/bare-metal/container-linux/kubernetes/cl/controller.yaml b/bare-metal/container-linux/kubernetes/cl/controller.yaml index ea5c7bb09..e6026b8ef 100644 --- a/bare-metal/container-linux/kubernetes/cl/controller.yaml +++ b/bare-metal/container-linux/kubernetes/cl/controller.yaml @@ -103,7 +103,7 @@ systemd: --mount volume=etc-iscsi,target=/etc/iscsi \ --volume usr-sbin-iscsiadm,kind=host,source=/usr/sbin/iscsiadm \ --mount volume=usr-sbin-iscsiadm,target=/sbin/iscsiadm \ - docker://quay.io/poseidon/kubelet:v1.18.1 -- \ + docker://quay.io/poseidon/kubelet:v1.18.2 -- \ --anonymous-auth=false \ --authentication-token-webhook \ --authorization-mode=Webhook \ @@ -147,7 +147,7 @@ systemd: --volume script,kind=host,source=/opt/bootstrap/apply \ --mount volume=script,target=/apply \ --insecure-options=image \ - docker://quay.io/poseidon/kubelet:v1.18.1 \ + docker://quay.io/poseidon/kubelet:v1.18.2 \ --net=host \ --dns=host \ --exec=/apply diff --git a/bare-metal/container-linux/kubernetes/cl/worker.yaml b/bare-metal/container-linux/kubernetes/cl/worker.yaml index 6f411b1cc..164323cf5 100644 --- a/bare-metal/container-linux/kubernetes/cl/worker.yaml +++ b/bare-metal/container-linux/kubernetes/cl/worker.yaml @@ -76,7 +76,7 @@ systemd: --mount volume=etc-iscsi,target=/etc/iscsi \ --volume usr-sbin-iscsiadm,kind=host,source=/usr/sbin/iscsiadm \ --mount volume=usr-sbin-iscsiadm,target=/sbin/iscsiadm \ - docker://quay.io/poseidon/kubelet:v1.18.1 -- \ + docker://quay.io/poseidon/kubelet:v1.18.2 -- \ --anonymous-auth=false \ --authentication-token-webhook \ --authorization-mode=Webhook \ diff --git a/bare-metal/fedora-coreos/kubernetes/README.md b/bare-metal/fedora-coreos/kubernetes/README.md index 3fbc067bc..d9da6a1c2 100644 --- a/bare-metal/fedora-coreos/kubernetes/README.md +++ b/bare-metal/fedora-coreos/kubernetes/README.md @@ -11,7 +11,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features -* Kubernetes v1.18.1 (upstream) +* Kubernetes v1.18.2 (upstream) * Single or multi-master, [Calico](https://www.projectcalico.org/) or [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/) * Advanced features like [snippets](https://typhoon.psdn.io/advanced/customization/#container-linux) customization diff --git a/bare-metal/fedora-coreos/kubernetes/bootstrap.tf b/bare-metal/fedora-coreos/kubernetes/bootstrap.tf index 47ea0d306..daa929a7c 100644 --- a/bare-metal/fedora-coreos/kubernetes/bootstrap.tf +++ b/bare-metal/fedora-coreos/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=1ad53d3b1c1ad75a4ed27f124f772fc5dc025245" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=14d0b2087962a0f2557c184f3f523548ce19bbdc" cluster_name = var.cluster_name api_servers = [var.k8s_domain_name] diff --git a/bare-metal/fedora-coreos/kubernetes/fcc/controller.yaml b/bare-metal/fedora-coreos/kubernetes/fcc/controller.yaml index 691cb534e..0689e7599 100644 --- a/bare-metal/fedora-coreos/kubernetes/fcc/controller.yaml +++ b/bare-metal/fedora-coreos/kubernetes/fcc/controller.yaml @@ -80,7 +80,7 @@ systemd: --volume /opt/cni/bin:/opt/cni/bin:z \ --volume /etc/iscsi:/etc/iscsi \ --volume /sbin/iscsiadm:/sbin/iscsiadm \ - quay.io/poseidon/kubelet:v1.18.1 \ + quay.io/poseidon/kubelet:v1.18.2 \ --anonymous-auth=false \ --authentication-token-webhook \ --authorization-mode=Webhook \ @@ -133,7 +133,7 @@ systemd: --volume /opt/bootstrap/assets:/assets:ro,Z \ --volume /opt/bootstrap/apply:/apply:ro,Z \ --entrypoint=/apply \ - quay.io/poseidon/kubelet:v1.18.1 + quay.io/poseidon/kubelet:v1.18.2 ExecStartPost=/bin/touch /opt/bootstrap/bootstrap.done ExecStartPost=-/usr/bin/podman stop bootstrap storage: diff --git a/bare-metal/fedora-coreos/kubernetes/fcc/worker.yaml b/bare-metal/fedora-coreos/kubernetes/fcc/worker.yaml index c20dcc9c0..5133d36e6 100644 --- a/bare-metal/fedora-coreos/kubernetes/fcc/worker.yaml +++ b/bare-metal/fedora-coreos/kubernetes/fcc/worker.yaml @@ -50,7 +50,7 @@ systemd: --volume /opt/cni/bin:/opt/cni/bin:z \ --volume /etc/iscsi:/etc/iscsi \ --volume /sbin/iscsiadm:/sbin/iscsiadm \ - quay.io/poseidon/kubelet:v1.18.1 \ + quay.io/poseidon/kubelet:v1.18.2 \ --anonymous-auth=false \ --authentication-token-webhook \ --authorization-mode=Webhook \ diff --git a/digital-ocean/container-linux/kubernetes/README.md b/digital-ocean/container-linux/kubernetes/README.md index d9d022638..73b5fad91 100644 --- a/digital-ocean/container-linux/kubernetes/README.md +++ b/digital-ocean/container-linux/kubernetes/README.md @@ -11,7 +11,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features -* Kubernetes v1.18.1 (upstream) +* Kubernetes v1.18.2 (upstream) * Single or multi-master, [Calico](https://www.projectcalico.org/) or [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/) * Advanced features like [snippets](https://typhoon.psdn.io/advanced/customization/#container-linux) customization diff --git a/digital-ocean/container-linux/kubernetes/bootstrap.tf b/digital-ocean/container-linux/kubernetes/bootstrap.tf index 5ee3d54d7..8fe3c1b49 100644 --- a/digital-ocean/container-linux/kubernetes/bootstrap.tf +++ b/digital-ocean/container-linux/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=1ad53d3b1c1ad75a4ed27f124f772fc5dc025245" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=14d0b2087962a0f2557c184f3f523548ce19bbdc" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/digital-ocean/container-linux/kubernetes/cl/controller.yaml b/digital-ocean/container-linux/kubernetes/cl/controller.yaml index 7dcb5e7a0..88f8d93fc 100644 --- a/digital-ocean/container-linux/kubernetes/cl/controller.yaml +++ b/digital-ocean/container-linux/kubernetes/cl/controller.yaml @@ -101,7 +101,7 @@ systemd: --mount volume=var-log,target=/var/log \ --volume opt-cni-bin,kind=host,source=/opt/cni/bin \ --mount volume=opt-cni-bin,target=/opt/cni/bin \ - docker://quay.io/poseidon/kubelet:v1.18.1 -- \ + docker://quay.io/poseidon/kubelet:v1.18.2 -- \ --anonymous-auth=false \ --authentication-token-webhook \ --authorization-mode=Webhook \ @@ -144,7 +144,7 @@ systemd: --volume script,kind=host,source=/opt/bootstrap/apply \ --mount volume=script,target=/apply \ --insecure-options=image \ - docker://quay.io/poseidon/kubelet:v1.18.1 \ + docker://quay.io/poseidon/kubelet:v1.18.2 \ --net=host \ --dns=host \ --exec=/apply diff --git a/digital-ocean/container-linux/kubernetes/cl/worker.yaml b/digital-ocean/container-linux/kubernetes/cl/worker.yaml index e54a83cc8..2248e0770 100644 --- a/digital-ocean/container-linux/kubernetes/cl/worker.yaml +++ b/digital-ocean/container-linux/kubernetes/cl/worker.yaml @@ -74,7 +74,7 @@ systemd: --mount volume=var-log,target=/var/log \ --volume opt-cni-bin,kind=host,source=/opt/cni/bin \ --mount volume=opt-cni-bin,target=/opt/cni/bin \ - docker://quay.io/poseidon/kubelet:v1.18.1 -- \ + docker://quay.io/poseidon/kubelet:v1.18.2 -- \ --anonymous-auth=false \ --authentication-token-webhook \ --authorization-mode=Webhook \ @@ -131,7 +131,7 @@ storage: --volume config,kind=host,source=/etc/kubernetes \ --mount volume=config,target=/etc/kubernetes \ --insecure-options=image \ - docker://quay.io/poseidon/kubelet:v1.18.1 \ + docker://quay.io/poseidon/kubelet:v1.18.2 \ --net=host \ --dns=host \ --exec=/usr/local/bin/kubectl -- --kubeconfig=/etc/kubernetes/kubeconfig delete node $(hostname) diff --git a/digital-ocean/fedora-coreos/kubernetes/README.md b/digital-ocean/fedora-coreos/kubernetes/README.md index 18cd0c6bc..94ce6985f 100644 --- a/digital-ocean/fedora-coreos/kubernetes/README.md +++ b/digital-ocean/fedora-coreos/kubernetes/README.md @@ -11,7 +11,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features -* Kubernetes v1.18.1 (upstream) +* Kubernetes v1.18.2 (upstream) * Single or multi-master, [Calico](https://www.projectcalico.org/) or [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/) * Advanced features like [snippets](https://typhoon.psdn.io/advanced/customization/) customization diff --git a/digital-ocean/fedora-coreos/kubernetes/bootstrap.tf b/digital-ocean/fedora-coreos/kubernetes/bootstrap.tf index 17847ce77..2f95cc53f 100644 --- a/digital-ocean/fedora-coreos/kubernetes/bootstrap.tf +++ b/digital-ocean/fedora-coreos/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=1ad53d3b1c1ad75a4ed27f124f772fc5dc025245" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=14d0b2087962a0f2557c184f3f523548ce19bbdc" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/digital-ocean/fedora-coreos/kubernetes/fcc/controller.yaml b/digital-ocean/fedora-coreos/kubernetes/fcc/controller.yaml index 916ed266d..6ccfb5ae0 100644 --- a/digital-ocean/fedora-coreos/kubernetes/fcc/controller.yaml +++ b/digital-ocean/fedora-coreos/kubernetes/fcc/controller.yaml @@ -81,7 +81,7 @@ systemd: --volume /var/log:/var/log \ --volume /var/run/lock:/var/run/lock:z \ --volume /opt/cni/bin:/opt/cni/bin:z \ - quay.io/poseidon/kubelet:v1.18.1 \ + quay.io/poseidon/kubelet:v1.18.2 \ --anonymous-auth=false \ --authentication-token-webhook \ --authorization-mode=Webhook \ @@ -135,7 +135,7 @@ systemd: --volume /opt/bootstrap/assets:/assets:ro,Z \ --volume /opt/bootstrap/apply:/apply:ro,Z \ --entrypoint=/apply \ - quay.io/poseidon/kubelet:v1.18.1 + quay.io/poseidon/kubelet:v1.18.2 ExecStartPost=/bin/touch /opt/bootstrap/bootstrap.done ExecStartPost=-/usr/bin/podman stop bootstrap storage: diff --git a/digital-ocean/fedora-coreos/kubernetes/fcc/worker.yaml b/digital-ocean/fedora-coreos/kubernetes/fcc/worker.yaml index 3d2f48e34..8209e4e0b 100644 --- a/digital-ocean/fedora-coreos/kubernetes/fcc/worker.yaml +++ b/digital-ocean/fedora-coreos/kubernetes/fcc/worker.yaml @@ -52,7 +52,7 @@ systemd: --volume /var/log:/var/log \ --volume /var/run/lock:/var/run/lock:z \ --volume /opt/cni/bin:/opt/cni/bin:z \ - quay.io/poseidon/kubelet:v1.18.1 \ + quay.io/poseidon/kubelet:v1.18.2 \ --anonymous-auth=false \ --authentication-token-webhook \ --authorization-mode=Webhook \ @@ -97,7 +97,7 @@ systemd: Type=oneshot RemainAfterExit=true ExecStart=/bin/true - ExecStop=/bin/bash -c '/usr/bin/podman run --volume /etc/kubernetes:/etc/kubernetes:ro,z --entrypoint /usr/local/bin/kubectl quay.io/poseidon/kubelet:v1.18.1 --kubeconfig=/etc/kubernetes/kubeconfig delete node $HOSTNAME' + ExecStop=/bin/bash -c '/usr/bin/podman run --volume /etc/kubernetes:/etc/kubernetes:ro,z --entrypoint /usr/local/bin/kubectl quay.io/poseidon/kubelet:v1.18.2 --kubeconfig=/etc/kubernetes/kubeconfig delete node $HOSTNAME' [Install] WantedBy=multi-user.target storage: diff --git a/docs/advanced/worker-pools.md b/docs/advanced/worker-pools.md index 9d920f249..6f92646f1 100644 --- a/docs/advanced/worker-pools.md +++ b/docs/advanced/worker-pools.md @@ -82,7 +82,7 @@ Create a cluster following the Azure [tutorial](../cl/azure.md#cluster). Define ```tf module "ramius-worker-pool" { - source = "git::https://github.com/poseidon/typhoon//azure/container-linux/kubernetes/workers?ref=v1.18.1" + source = "git::https://github.com/poseidon/typhoon//azure/container-linux/kubernetes/workers?ref=v1.18.2" # Azure region = module.ramius.region @@ -148,7 +148,7 @@ Create a cluster following the Google Cloud [tutorial](../cl/google-cloud.md#clu ```tf module "yavin-worker-pool" { - source = "git::https://github.com/poseidon/typhoon//google-cloud/container-linux/kubernetes/workers?ref=v1.18.1" + source = "git::https://github.com/poseidon/typhoon//google-cloud/container-linux/kubernetes/workers?ref=v1.18.2" # Google Cloud region = "europe-west2" @@ -179,11 +179,11 @@ Verify a managed instance group of workers joins the cluster within a few minute ``` $ kubectl get nodes NAME STATUS AGE VERSION -yavin-controller-0.c.example-com.internal Ready 6m v1.18.1 -yavin-worker-jrbf.c.example-com.internal Ready 5m v1.18.1 -yavin-worker-mzdm.c.example-com.internal Ready 5m v1.18.1 -yavin-16x-worker-jrbf.c.example-com.internal Ready 3m v1.18.1 -yavin-16x-worker-mzdm.c.example-com.internal Ready 3m v1.18.1 +yavin-controller-0.c.example-com.internal Ready 6m v1.18.2 +yavin-worker-jrbf.c.example-com.internal Ready 5m v1.18.2 +yavin-worker-mzdm.c.example-com.internal Ready 5m v1.18.2 +yavin-16x-worker-jrbf.c.example-com.internal Ready 3m v1.18.2 +yavin-16x-worker-mzdm.c.example-com.internal Ready 3m v1.18.2 ``` ### Variables diff --git a/docs/cl/aws.md b/docs/cl/aws.md index 4ab9e11a9..afc469cea 100644 --- a/docs/cl/aws.md +++ b/docs/cl/aws.md @@ -1,6 +1,6 @@ # AWS -In this tutorial, we'll create a Kubernetes v1.18.1 cluster on AWS with CoreOS Container Linux or Flatcar Linux. +In this tutorial, we'll create a Kubernetes v1.18.2 cluster on AWS with CoreOS Container Linux or Flatcar Linux. We'll declare a Kubernetes cluster using the Typhoon Terraform module. Then apply the changes to create a VPC, gateway, subnets, security groups, controller instances, worker auto-scaling group, network load balancer, and TLS assets. @@ -70,7 +70,7 @@ Define a Kubernetes cluster using the module `aws/container-linux/kubernetes`. ```tf module "tempest" { - source = "git::https://github.com/poseidon/typhoon//aws/container-linux/kubernetes?ref=v1.18.1" + source = "git::https://github.com/poseidon/typhoon//aws/container-linux/kubernetes?ref=v1.18.2" # AWS cluster_name = "tempest" @@ -143,9 +143,9 @@ List nodes in the cluster. $ export KUBECONFIG=/home/user/.kube/configs/tempest-config $ kubectl get nodes NAME STATUS ROLES AGE VERSION -ip-10-0-3-155 Ready 10m v1.18.1 -ip-10-0-26-65 Ready 10m v1.18.1 -ip-10-0-41-21 Ready 10m v1.18.1 +ip-10-0-3-155 Ready 10m v1.18.2 +ip-10-0-26-65 Ready 10m v1.18.2 +ip-10-0-41-21 Ready 10m v1.18.2 ``` List the pods. diff --git a/docs/cl/azure.md b/docs/cl/azure.md index 2a519403d..a721a06ba 100644 --- a/docs/cl/azure.md +++ b/docs/cl/azure.md @@ -1,6 +1,6 @@ # Azure -In this tutorial, we'll create a Kubernetes v1.18.1 cluster on Azure with CoreOS Container Linux or Flatcar Linux. +In this tutorial, we'll create a Kubernetes v1.18.2 cluster on Azure with CoreOS Container Linux or Flatcar Linux. We'll declare a Kubernetes cluster using the Typhoon Terraform module. Then apply the changes to create a resource group, virtual network, subnets, security groups, controller availability set, worker scale set, load balancer, and TLS assets. @@ -63,7 +63,7 @@ Define a Kubernetes cluster using the module `azure/container-linux/kubernetes`. ```tf module "ramius" { - source = "git::https://github.com/poseidon/typhoon//azure/container-linux/kubernetes?ref=v1.18.1" + source = "git::https://github.com/poseidon/typhoon//azure/container-linux/kubernetes?ref=v1.18.2" # Azure cluster_name = "ramius" @@ -146,9 +146,9 @@ List nodes in the cluster. $ export KUBECONFIG=/home/user/.kube/configs/ramius-config $ kubectl get nodes NAME STATUS ROLES AGE VERSION -ramius-controller-0 Ready 24m v1.18.1 -ramius-worker-000001 Ready 25m v1.18.1 -ramius-worker-000002 Ready 24m v1.18.1 +ramius-controller-0 Ready 24m v1.18.2 +ramius-worker-000001 Ready 25m v1.18.2 +ramius-worker-000002 Ready 24m v1.18.2 ``` List the pods. diff --git a/docs/cl/bare-metal.md b/docs/cl/bare-metal.md index 2a2f2274a..e9b8ab1ab 100644 --- a/docs/cl/bare-metal.md +++ b/docs/cl/bare-metal.md @@ -1,6 +1,6 @@ # Bare-Metal -In this tutorial, we'll network boot and provision a Kubernetes v1.18.1 cluster on bare-metal with CoreOS Container Linux or Flatcar Linux. +In this tutorial, we'll network boot and provision a Kubernetes v1.18.2 cluster on bare-metal with CoreOS Container Linux or Flatcar Linux. First, we'll deploy a [Matchbox](https://github.com/poseidon/matchbox) service and setup a network boot environment. Then, we'll declare a Kubernetes cluster using the Typhoon Terraform module and power on machines. On PXE boot, machines will install Container Linux to disk, reboot into the disk install, and provision themselves as Kubernetes controllers or workers via Ignition. @@ -160,7 +160,7 @@ Define a Kubernetes cluster using the module `bare-metal/container-linux/kuberne ```tf module "mercury" { - source = "git::https://github.com/poseidon/typhoon//bare-metal/container-linux/kubernetes?ref=v1.18.1" + source = "git::https://github.com/poseidon/typhoon//bare-metal/container-linux/kubernetes?ref=v1.18.2" # bare-metal cluster_name = "mercury" @@ -299,9 +299,9 @@ List nodes in the cluster. $ export KUBECONFIG=/home/user/.kube/configs/mercury-config $ kubectl get nodes NAME STATUS ROLES AGE VERSION -node1.example.com Ready 10m v1.18.1 -node2.example.com Ready 10m v1.18.1 -node3.example.com Ready 10m v1.18.1 +node1.example.com Ready 10m v1.18.2 +node2.example.com Ready 10m v1.18.2 +node3.example.com Ready 10m v1.18.2 ``` List the pods. diff --git a/docs/cl/digital-ocean.md b/docs/cl/digital-ocean.md index ed32f1ed5..164ba9ed7 100644 --- a/docs/cl/digital-ocean.md +++ b/docs/cl/digital-ocean.md @@ -1,6 +1,6 @@ # Digital Ocean -In this tutorial, we'll create a Kubernetes v1.18.1 cluster on DigitalOcean with CoreOS Container Linux or Flatcar Linux. +In this tutorial, we'll create a Kubernetes v1.18.2 cluster on DigitalOcean with CoreOS Container Linux or Flatcar Linux. We'll declare a Kubernetes cluster using the Typhoon Terraform module. Then apply the changes to create controller droplets, worker droplets, DNS records, tags, and TLS assets. @@ -79,7 +79,7 @@ Define a Kubernetes cluster using the module `digital-ocean/container-linux/kube ```tf module "nemo" { - source = "git::https://github.com/poseidon/typhoon//digital-ocean/container-linux/kubernetes?ref=v1.18.1" + source = "git::https://github.com/poseidon/typhoon//digital-ocean/container-linux/kubernetes?ref=v1.18.2" # Digital Ocean cluster_name = "nemo" @@ -153,9 +153,9 @@ List nodes in the cluster. $ export KUBECONFIG=/home/user/.kube/configs/nemo-config $ kubectl get nodes NAME STATUS ROLES AGE VERSION -10.132.110.130 Ready 10m v1.18.1 -10.132.115.81 Ready 10m v1.18.1 -10.132.124.107 Ready 10m v1.18.1 +10.132.110.130 Ready 10m v1.18.2 +10.132.115.81 Ready 10m v1.18.2 +10.132.124.107 Ready 10m v1.18.2 ``` List the pods. diff --git a/docs/cl/google-cloud.md b/docs/cl/google-cloud.md index 479690f5d..511c09c56 100644 --- a/docs/cl/google-cloud.md +++ b/docs/cl/google-cloud.md @@ -1,6 +1,6 @@ # Google Cloud -In this tutorial, we'll create a Kubernetes v1.18.1 cluster on Google Compute Engine with CoreOS Container Linux or Flatcar Linux. +In this tutorial, we'll create a Kubernetes v1.18.2 cluster on Google Compute Engine with CoreOS Container Linux or Flatcar Linux. We'll declare a Kubernetes cluster using the Typhoon Terraform module. Then apply the changes to create a network, firewall rules, health checks, controller instances, worker managed instance group, load balancers, and TLS assets. @@ -90,7 +90,7 @@ Define a Kubernetes cluster using the module `google-cloud/container-linux/kuber ```tf module "yavin" { - source = "git::https://github.com/poseidon/typhoon//google-cloud/container-linux/kubernetes?ref=v1.18.1" + source = "git::https://github.com/poseidon/typhoon//google-cloud/container-linux/kubernetes?ref=v1.18.2" # Google Cloud cluster_name = "yavin" @@ -165,9 +165,9 @@ List nodes in the cluster. $ export KUBECONFIG=/home/user/.kube/configs/yavin-config $ kubectl get nodes NAME ROLES STATUS AGE VERSION -yavin-controller-0.c.example-com.internal Ready 6m v1.18.1 -yavin-worker-jrbf.c.example-com.internal Ready 5m v1.18.1 -yavin-worker-mzdm.c.example-com.internal Ready 5m v1.18.1 +yavin-controller-0.c.example-com.internal Ready 6m v1.18.2 +yavin-worker-jrbf.c.example-com.internal Ready 5m v1.18.2 +yavin-worker-mzdm.c.example-com.internal Ready 5m v1.18.2 ``` List the pods. diff --git a/docs/fedora-coreos/aws.md b/docs/fedora-coreos/aws.md index 6102f6a70..cdcef4d01 100644 --- a/docs/fedora-coreos/aws.md +++ b/docs/fedora-coreos/aws.md @@ -1,6 +1,6 @@ # AWS -In this tutorial, we'll create a Kubernetes v1.18.1 cluster on AWS with Fedora CoreOS. +In this tutorial, we'll create a Kubernetes v1.18.2 cluster on AWS with Fedora CoreOS. We'll declare a Kubernetes cluster using the Typhoon Terraform module. Then apply the changes to create a VPC, gateway, subnets, security groups, controller instances, worker auto-scaling group, network load balancer, and TLS assets. @@ -70,7 +70,7 @@ Define a Kubernetes cluster using the module `aws/fedora-coreos/kubernetes`. ```tf module "tempest" { - source = "git::https://github.com/poseidon/typhoon//aws/fedora-coreos/kubernetes?ref=v1.18.1" + source = "git::https://github.com/poseidon/typhoon//aws/fedora-coreos/kubernetes?ref=v1.18.2" # AWS cluster_name = "tempest" @@ -143,9 +143,9 @@ List nodes in the cluster. $ export KUBECONFIG=/home/user/.kube/configs/tempest-config $ kubectl get nodes NAME STATUS ROLES AGE VERSION -ip-10-0-3-155 Ready 10m v1.18.1 -ip-10-0-26-65 Ready 10m v1.18.1 -ip-10-0-41-21 Ready 10m v1.18.1 +ip-10-0-3-155 Ready 10m v1.18.2 +ip-10-0-26-65 Ready 10m v1.18.2 +ip-10-0-41-21 Ready 10m v1.18.2 ``` List the pods. diff --git a/docs/fedora-coreos/azure.md b/docs/fedora-coreos/azure.md index 518670df3..719e3030f 100644 --- a/docs/fedora-coreos/azure.md +++ b/docs/fedora-coreos/azure.md @@ -1,6 +1,6 @@ # Azure -In this tutorial, we'll create a Kubernetes v1.18.1 cluster on Azure with Fedora CoreOS. +In this tutorial, we'll create a Kubernetes v1.18.2 cluster on Azure with Fedora CoreOS. We'll declare a Kubernetes cluster using the Typhoon Terraform module. Then apply the changes to create a resource group, virtual network, subnets, security groups, controller availability set, worker scale set, load balancer, and TLS assets. @@ -83,7 +83,7 @@ Define a Kubernetes cluster using the module `azure/fedora-coreos/kubernetes`. ```tf module "ramius" { - source = "git::https://github.com/poseidon/typhoon//azure/fedora-coreos/kubernetes?ref=v1.18.1" + source = "git::https://github.com/poseidon/typhoon//azure/fedora-coreos/kubernetes?ref=v1.18.2" # Azure cluster_name = "ramius" @@ -158,9 +158,9 @@ List nodes in the cluster. $ export KUBECONFIG=/home/user/.kube/configs/ramius-config $ kubectl get nodes NAME STATUS ROLES AGE VERSION -ramius-controller-0 Ready 24m v1.18.1 -ramius-worker-000001 Ready 25m v1.18.1 -ramius-worker-000002 Ready 24m v1.18.1 +ramius-controller-0 Ready 24m v1.18.2 +ramius-worker-000001 Ready 25m v1.18.2 +ramius-worker-000002 Ready 24m v1.18.2 ``` List the pods. diff --git a/docs/fedora-coreos/bare-metal.md b/docs/fedora-coreos/bare-metal.md index a8e0a7bac..f789db64a 100644 --- a/docs/fedora-coreos/bare-metal.md +++ b/docs/fedora-coreos/bare-metal.md @@ -1,6 +1,6 @@ # Bare-Metal -In this tutorial, we'll network boot and provision a Kubernetes v1.18.1 cluster on bare-metal with Fedora CoreOS. +In this tutorial, we'll network boot and provision a Kubernetes v1.18.2 cluster on bare-metal with Fedora CoreOS. First, we'll deploy a [Matchbox](https://github.com/poseidon/matchbox) service and setup a network boot environment. Then, we'll declare a Kubernetes cluster using the Typhoon Terraform module and power on machines. On PXE boot, machines will install Fedora CoreOS to disk, reboot into the disk install, and provision themselves as Kubernetes controllers or workers via Ignition. @@ -160,7 +160,7 @@ Define a Kubernetes cluster using the module `bare-metal/fedora-coreos/kubernete ```tf module "mercury" { - source = "git::https://github.com/poseidon/typhoon//bare-metal/fedora-coreos/kubernetes?ref=v1.18.1" + source = "git::https://github.com/poseidon/typhoon//bare-metal/fedora-coreos/kubernetes?ref=v1.18.2" # bare-metal cluster_name = "mercury" @@ -289,9 +289,9 @@ List nodes in the cluster. $ export KUBECONFIG=/home/user/.kube/configs/mercury-config $ kubectl get nodes NAME STATUS ROLES AGE VERSION -node1.example.com Ready 10m v1.18.1 -node2.example.com Ready 10m v1.18.1 -node3.example.com Ready 10m v1.18.1 +node1.example.com Ready 10m v1.18.2 +node2.example.com Ready 10m v1.18.2 +node3.example.com Ready 10m v1.18.2 ``` List the pods. diff --git a/docs/fedora-coreos/digitalocean.md b/docs/fedora-coreos/digitalocean.md index b0c209a28..81d4287c5 100644 --- a/docs/fedora-coreos/digitalocean.md +++ b/docs/fedora-coreos/digitalocean.md @@ -1,6 +1,6 @@ # Digital Ocean -In this tutorial, we'll create a Kubernetes v1.18.1 cluster on DigitalOcean with Fedora CoreOS. +In this tutorial, we'll create a Kubernetes v1.18.2 cluster on DigitalOcean with Fedora CoreOS. We'll declare a Kubernetes cluster using the Typhoon Terraform module. Then apply the changes to create controller droplets, worker droplets, DNS records, tags, and TLS assets. @@ -79,7 +79,7 @@ Define a Kubernetes cluster using the module `digital-ocean/fedora-coreos/kubern ```tf module "nemo" { - source = "git::https://github.com/poseidon/typhoon//digital-ocean/fedora-coreos/kubernetes?ref=v1.18.1" + source = "git::https://github.com/poseidon/typhoon//digital-ocean/fedora-coreos/kubernetes?ref=v1.18.2" # Digital Ocean cluster_name = "nemo" @@ -153,9 +153,9 @@ List nodes in the cluster. $ export KUBECONFIG=/home/user/.kube/configs/nemo-config $ kubectl get nodes NAME STATUS ROLES AGE VERSION -10.132.110.130 Ready 10m v1.18.1 -10.132.115.81 Ready 10m v1.18.1 -10.132.124.107 Ready 10m v1.18.1 +10.132.110.130 Ready 10m v1.18.2 +10.132.115.81 Ready 10m v1.18.2 +10.132.124.107 Ready 10m v1.18.2 ``` List the pods. diff --git a/docs/fedora-coreos/google-cloud.md b/docs/fedora-coreos/google-cloud.md index 482900b3f..a635c6bf8 100644 --- a/docs/fedora-coreos/google-cloud.md +++ b/docs/fedora-coreos/google-cloud.md @@ -1,6 +1,6 @@ # Google Cloud -In this tutorial, we'll create a Kubernetes v1.18.1 cluster on Google Compute Engine with Fedora CoreOS. +In this tutorial, we'll create a Kubernetes v1.18.2 cluster on Google Compute Engine with Fedora CoreOS. We'll declare a Kubernetes cluster using the Typhoon Terraform module. Then apply the changes to create a network, firewall rules, health checks, controller instances, worker managed instance group, load balancers, and TLS assets. @@ -165,9 +165,9 @@ List nodes in the cluster. $ export KUBECONFIG=/home/user/.kube/configs/yavin-config $ kubectl get nodes NAME ROLES STATUS AGE VERSION -yavin-controller-0.c.example-com.internal Ready 6m v1.18.1 -yavin-worker-jrbf.c.example-com.internal Ready 5m v1.18.1 -yavin-worker-mzdm.c.example-com.internal Ready 5m v1.18.1 +yavin-controller-0.c.example-com.internal Ready 6m v1.18.2 +yavin-worker-jrbf.c.example-com.internal Ready 5m v1.18.2 +yavin-worker-mzdm.c.example-com.internal Ready 5m v1.18.2 ``` List the pods. diff --git a/docs/index.md b/docs/index.md index b27c4017b..9d9fedc51 100644 --- a/docs/index.md +++ b/docs/index.md @@ -11,7 +11,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features -* Kubernetes v1.18.1 (upstream) +* Kubernetes v1.18.2 (upstream) * Single or multi-master, [Calico](https://www.projectcalico.org/) or [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/) * Advanced features like [worker pools](advanced/worker-pools/), [preemptible](cl/google-cloud/#preemption) workers, and [snippets](advanced/customization/#container-linux) customization @@ -64,7 +64,7 @@ Define a Kubernetes cluster by using the Terraform module for your chosen platfo ```tf module "yavin" { - source = "git::https://github.com/poseidon/typhoon//google-cloud/container-linux/kubernetes?ref=v1.18.1" + source = "git::https://github.com/poseidon/typhoon//google-cloud/container-linux/kubernetes?ref=v1.18.2" # Google Cloud cluster_name = "yavin" @@ -102,9 +102,9 @@ In 4-8 minutes (varies by platform), the cluster will be ready. This Google Clou $ export KUBECONFIG=/home/user/.kube/configs/yavin-config $ kubectl get nodes NAME ROLES STATUS AGE VERSION -yavin-controller-0.c.example-com.internal Ready 6m v1.18.1 -yavin-worker-jrbf.c.example-com.internal Ready 5m v1.18.1 -yavin-worker-mzdm.c.example-com.internal Ready 5m v1.18.1 +yavin-controller-0.c.example-com.internal Ready 6m v1.18.2 +yavin-worker-jrbf.c.example-com.internal Ready 5m v1.18.2 +yavin-worker-mzdm.c.example-com.internal Ready 5m v1.18.2 ``` List the pods. diff --git a/docs/topics/maintenance.md b/docs/topics/maintenance.md index 59978802f..b01a610f8 100644 --- a/docs/topics/maintenance.md +++ b/docs/topics/maintenance.md @@ -18,7 +18,7 @@ module "yavin" { } module "mercury" { - source = "git::https://github.com/poseidon/typhoon//bare-metal/container-linux/kubernetes?ref=v1.18.1" + source = "git::https://github.com/poseidon/typhoon//bare-metal/container-linux/kubernetes?ref=v1.18.2" ... } ``` @@ -279,15 +279,15 @@ Typhoon modules have been adapted for Terraform v0.12. Provider plugins requirem | Typhoon Release | Terraform version | |-------------------|---------------------| -| v1.18.1 - ? | v0.12.x | -| v1.10.3 - v1.18.1 | v0.11.x | +| v1.18.2 - ? | v0.12.x | +| v1.10.3 - v1.18.2 | v0.11.x | | v1.9.2 - v1.10.2 | v0.10.4+ or v0.11.x | | v1.7.3 - v1.9.1 | v0.10.x | | v1.6.4 - v1.7.2 | v0.9.x | ### New users -New users can start with Terraform v0.12.x and follow the docs for Typhoon v1.18.1+ without issue. +New users can start with Terraform v0.12.x and follow the docs for Typhoon v1.18.2+ without issue. ### Existing users diff --git a/google-cloud/container-linux/kubernetes/README.md b/google-cloud/container-linux/kubernetes/README.md index 94cd7d8aa..1d46bb9f3 100644 --- a/google-cloud/container-linux/kubernetes/README.md +++ b/google-cloud/container-linux/kubernetes/README.md @@ -11,7 +11,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features -* Kubernetes v1.18.1 (upstream) +* Kubernetes v1.18.2 (upstream) * Single or multi-master, [Calico](https://www.projectcalico.org/) or [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/) * Advanced features like [worker pools](https://typhoon.psdn.io/advanced/worker-pools/), [preemptible](https://typhoon.psdn.io/cl/google-cloud/#preemption) workers, and [snippets](https://typhoon.psdn.io/advanced/customization/#container-linux) customization diff --git a/google-cloud/container-linux/kubernetes/bootstrap.tf b/google-cloud/container-linux/kubernetes/bootstrap.tf index 6db9f8013..a8729f733 100644 --- a/google-cloud/container-linux/kubernetes/bootstrap.tf +++ b/google-cloud/container-linux/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=1ad53d3b1c1ad75a4ed27f124f772fc5dc025245" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=14d0b2087962a0f2557c184f3f523548ce19bbdc" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/google-cloud/container-linux/kubernetes/cl/controller.yaml b/google-cloud/container-linux/kubernetes/cl/controller.yaml index 8464e7bae..fa1ba8ead 100644 --- a/google-cloud/container-linux/kubernetes/cl/controller.yaml +++ b/google-cloud/container-linux/kubernetes/cl/controller.yaml @@ -90,7 +90,7 @@ systemd: --mount volume=var-log,target=/var/log \ --volume opt-cni-bin,kind=host,source=/opt/cni/bin \ --mount volume=opt-cni-bin,target=/opt/cni/bin \ - docker://quay.io/poseidon/kubelet:v1.18.1 -- \ + docker://quay.io/poseidon/kubelet:v1.18.2 -- \ --anonymous-auth=false \ --authentication-token-webhook \ --authorization-mode=Webhook \ @@ -132,7 +132,7 @@ systemd: --volume script,kind=host,source=/opt/bootstrap/apply \ --mount volume=script,target=/apply \ --insecure-options=image \ - docker://quay.io/poseidon/kubelet:v1.18.1 \ + docker://quay.io/poseidon/kubelet:v1.18.2 \ --net=host \ --dns=host \ --exec=/apply diff --git a/google-cloud/container-linux/kubernetes/workers/cl/worker.yaml b/google-cloud/container-linux/kubernetes/workers/cl/worker.yaml index 01d69d983..52a0cfae8 100644 --- a/google-cloud/container-linux/kubernetes/workers/cl/worker.yaml +++ b/google-cloud/container-linux/kubernetes/workers/cl/worker.yaml @@ -63,7 +63,7 @@ systemd: --mount volume=var-log,target=/var/log \ --volume opt-cni-bin,kind=host,source=/opt/cni/bin \ --mount volume=opt-cni-bin,target=/opt/cni/bin \ - docker://quay.io/poseidon/kubelet:v1.18.1 -- \ + docker://quay.io/poseidon/kubelet:v1.18.2 -- \ --anonymous-auth=false \ --authentication-token-webhook \ --authorization-mode=Webhook \ @@ -125,7 +125,7 @@ storage: --volume config,kind=host,source=/etc/kubernetes \ --mount volume=config,target=/etc/kubernetes \ --insecure-options=image \ - docker://quay.io/poseidon/kubelet:v1.18.1 \ + docker://quay.io/poseidon/kubelet:v1.18.2 \ --net=host \ --dns=host \ --exec=/usr/local/bin/kubectl -- --kubeconfig=/etc/kubernetes/kubeconfig delete node $(hostname) diff --git a/google-cloud/fedora-coreos/kubernetes/README.md b/google-cloud/fedora-coreos/kubernetes/README.md index 1eb918360..6c319b0b2 100644 --- a/google-cloud/fedora-coreos/kubernetes/README.md +++ b/google-cloud/fedora-coreos/kubernetes/README.md @@ -11,7 +11,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features -* Kubernetes v1.18.1 (upstream) +* Kubernetes v1.18.2 (upstream) * Single or multi-master, [Calico](https://www.projectcalico.org/) or [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/) * Advanced features like [worker pools](https://typhoon.psdn.io/advanced/worker-pools/), [preemptible](https://typhoon.psdn.io/fedora-coreos/google-cloud/#preemption) workers, and [snippets](https://typhoon.psdn.io/advanced/customization/) customization diff --git a/google-cloud/fedora-coreos/kubernetes/bootstrap.tf b/google-cloud/fedora-coreos/kubernetes/bootstrap.tf index 31693c2bb..168e55564 100644 --- a/google-cloud/fedora-coreos/kubernetes/bootstrap.tf +++ b/google-cloud/fedora-coreos/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=1ad53d3b1c1ad75a4ed27f124f772fc5dc025245" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=14d0b2087962a0f2557c184f3f523548ce19bbdc" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/google-cloud/fedora-coreos/kubernetes/fcc/controller.yaml b/google-cloud/fedora-coreos/kubernetes/fcc/controller.yaml index 3bbcbda32..8d51d374c 100644 --- a/google-cloud/fedora-coreos/kubernetes/fcc/controller.yaml +++ b/google-cloud/fedora-coreos/kubernetes/fcc/controller.yaml @@ -79,7 +79,7 @@ systemd: --volume /var/log:/var/log \ --volume /var/run/lock:/var/run/lock:z \ --volume /opt/cni/bin:/opt/cni/bin:z \ - quay.io/poseidon/kubelet:v1.18.1 \ + quay.io/poseidon/kubelet:v1.18.2 \ --anonymous-auth=false \ --authentication-token-webhook \ --authorization-mode=Webhook \ @@ -123,7 +123,7 @@ systemd: --volume /opt/bootstrap/assets:/assets:ro,Z \ --volume /opt/bootstrap/apply:/apply:ro,Z \ --entrypoint=/apply \ - quay.io/poseidon/kubelet:v1.18.1 + quay.io/poseidon/kubelet:v1.18.2 ExecStartPost=/bin/touch /opt/bootstrap/bootstrap.done ExecStartPost=-/usr/bin/podman stop bootstrap storage: diff --git a/google-cloud/fedora-coreos/kubernetes/workers/fcc/worker.yaml b/google-cloud/fedora-coreos/kubernetes/workers/fcc/worker.yaml index 7c41b484c..0501b8de9 100644 --- a/google-cloud/fedora-coreos/kubernetes/workers/fcc/worker.yaml +++ b/google-cloud/fedora-coreos/kubernetes/workers/fcc/worker.yaml @@ -49,7 +49,7 @@ systemd: --volume /var/log:/var/log \ --volume /var/run/lock:/var/run/lock:z \ --volume /opt/cni/bin:/opt/cni/bin:z \ - quay.io/poseidon/kubelet:v1.18.1 \ + quay.io/poseidon/kubelet:v1.18.2 \ --anonymous-auth=false \ --authentication-token-webhook \ --authorization-mode=Webhook \ @@ -87,7 +87,7 @@ systemd: Type=oneshot RemainAfterExit=true ExecStart=/bin/true - ExecStop=/bin/bash -c '/usr/bin/podman run --volume /etc/kubernetes:/etc/kubernetes:ro,z --entrypoint /usr/local/bin/kubectl quay.io/poseidon/kubelet:v1.18.1 --kubeconfig=/etc/kubernetes/kubeconfig delete node $HOSTNAME' + ExecStop=/bin/bash -c '/usr/bin/podman run --volume /etc/kubernetes:/etc/kubernetes:ro,z --entrypoint /usr/local/bin/kubectl quay.io/poseidon/kubelet:v1.18.2 --kubeconfig=/etc/kubernetes/kubeconfig delete node $HOSTNAME' [Install] WantedBy=multi-user.target storage: From bf22222f7dbf929f8650698df6e9b845906fab22 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Thu, 16 Apr 2020 23:49:55 -0700 Subject: [PATCH 394/523] Remove temporary workaround for v1.18.0 apply issue * In v1.18.0, kubectl apply would fail to apply manifests if any single manifest was unable to validate. For example, if a CRD and CR were defined in the same directory, apply would fail since the CR would be invalid as the CRD wouldn't exist * Typhoon temporary workaround was to separate CNI CRD manifests and explicitly apply them first. No longer needed in v1.18.1+ * Kubernetes v1.18.1 restored the prior behavior where kubectl apply applies as many valid manifests as it can. In the example above, the CRD would be applied and the CR could be applied if the kubectl apply was re-run (allowing for apply loops). * Upstream fix: https://github.com/kubernetes/kubernetes/pull/89864 --- aws/container-linux/kubernetes/cl/controller.yaml | 6 ------ aws/fedora-coreos/kubernetes/fcc/controller.yaml | 6 ------ azure/container-linux/kubernetes/cl/controller.yaml | 6 ------ bare-metal/container-linux/kubernetes/cl/controller.yaml | 6 ------ bare-metal/fedora-coreos/kubernetes/fcc/controller.yaml | 6 ------ digital-ocean/container-linux/kubernetes/cl/controller.yaml | 6 ------ google-cloud/container-linux/kubernetes/cl/controller.yaml | 6 ------ google-cloud/fedora-coreos/kubernetes/fcc/controller.yaml | 6 ------ 8 files changed, 48 deletions(-) diff --git a/aws/container-linux/kubernetes/cl/controller.yaml b/aws/container-linux/kubernetes/cl/controller.yaml index c9648120b..c1a131c40 100644 --- a/aws/container-linux/kubernetes/cl/controller.yaml +++ b/aws/container-linux/kubernetes/cl/controller.yaml @@ -169,8 +169,6 @@ storage: sudo mv static-manifests/* /etc/kubernetes/manifests/ sudo mkdir -p /opt/bootstrap/assets sudo mv manifests /opt/bootstrap/assets/manifests - sudo mkdir -p /opt/bootstrap/assets/manifests/crds - sudo mv manifests-networking/{crd,cluster}*.yaml /opt/bootstrap/assets/manifests/crds 2>/dev/null || true sudo mv manifests-networking/* /opt/bootstrap/assets/manifests/ rm -rf assets auth static-manifests tls manifests-networking - path: /opt/bootstrap/apply @@ -184,10 +182,6 @@ storage: echo "Waiting for static pod control plane" sleep 5 done - until kubectl apply -f /assets/manifests/crds -R; do - echo "Retry Custom Resource Definition manifests" - sleep 5 - done until kubectl apply -f /assets/manifests -R; do echo "Retry applying manifests" sleep 5 diff --git a/aws/fedora-coreos/kubernetes/fcc/controller.yaml b/aws/fedora-coreos/kubernetes/fcc/controller.yaml index 67a4ba4e7..1bcb4b5fa 100644 --- a/aws/fedora-coreos/kubernetes/fcc/controller.yaml +++ b/aws/fedora-coreos/kubernetes/fcc/controller.yaml @@ -154,8 +154,6 @@ storage: sudo mv static-manifests/* /etc/kubernetes/manifests/ sudo mkdir -p /opt/bootstrap/assets sudo mv manifests /opt/bootstrap/assets/manifests - sudo mkdir -p /opt/bootstrap/assets/manifests/crds - sudo mv manifests-networking/{crd,cluster}*.yaml /opt/bootstrap/assets/manifests/crds 2>/dev/null || true sudo mv manifests-networking/* /opt/bootstrap/assets/manifests/ rm -rf assets auth static-manifests tls manifests-networking - path: /opt/bootstrap/apply @@ -168,10 +166,6 @@ storage: echo "Waiting for static pod control plane" sleep 5 done - until kubectl apply -f /assets/manifests/crds -R; do - echo "Retry Custom Resource Definition manifests" - sleep 5 - done until kubectl apply -f /assets/manifests -R; do echo "Retry applying manifests" sleep 5 diff --git a/azure/container-linux/kubernetes/cl/controller.yaml b/azure/container-linux/kubernetes/cl/controller.yaml index 9102a6923..3e036c7e4 100644 --- a/azure/container-linux/kubernetes/cl/controller.yaml +++ b/azure/container-linux/kubernetes/cl/controller.yaml @@ -167,8 +167,6 @@ storage: sudo mv static-manifests/* /etc/kubernetes/manifests/ sudo mkdir -p /opt/bootstrap/assets sudo mv manifests /opt/bootstrap/assets/manifests - sudo mkdir -p /opt/bootstrap/assets/manifests/crds - sudo mv manifests-networking/{crd,cluster}*.yaml /opt/bootstrap/assets/manifests/crds 2>/dev/null || true sudo mv manifests-networking/* /opt/bootstrap/assets/manifests/ rm -rf assets auth static-manifests tls manifests-networking - path: /opt/bootstrap/apply @@ -182,10 +180,6 @@ storage: echo "Waiting for static pod control plane" sleep 5 done - until kubectl apply -f /assets/manifests/crds -R; do - echo "Retry Custom Resource Definition manifests" - sleep 5 - done until kubectl apply -f /assets/manifests -R; do echo "Retry applying manifests" sleep 5 diff --git a/bare-metal/container-linux/kubernetes/cl/controller.yaml b/bare-metal/container-linux/kubernetes/cl/controller.yaml index e6026b8ef..ea9c2d9d1 100644 --- a/bare-metal/container-linux/kubernetes/cl/controller.yaml +++ b/bare-metal/container-linux/kubernetes/cl/controller.yaml @@ -185,8 +185,6 @@ storage: sudo mv static-manifests/* /etc/kubernetes/manifests/ sudo mkdir -p /opt/bootstrap/assets sudo mv manifests /opt/bootstrap/assets/manifests - sudo mkdir -p /opt/bootstrap/assets/manifests/crds - sudo mv manifests-networking/{crd,cluster}*.yaml /opt/bootstrap/assets/manifests/crds 2>/dev/null || true sudo mv manifests-networking/* /opt/bootstrap/assets/manifests/ rm -rf assets auth static-manifests tls manifests-networking - path: /opt/bootstrap/apply @@ -200,10 +198,6 @@ storage: echo "Waiting for static pod control plane" sleep 5 done - until kubectl apply -f /assets/manifests/crds -R; do - echo "Retry Custom Resource Definition manifests" - sleep 5 - done until kubectl apply -f /assets/manifests -R; do echo "Retry applying manifests" sleep 5 diff --git a/bare-metal/fedora-coreos/kubernetes/fcc/controller.yaml b/bare-metal/fedora-coreos/kubernetes/fcc/controller.yaml index 0689e7599..c16cc4afd 100644 --- a/bare-metal/fedora-coreos/kubernetes/fcc/controller.yaml +++ b/bare-metal/fedora-coreos/kubernetes/fcc/controller.yaml @@ -165,8 +165,6 @@ storage: sudo mv static-manifests/* /etc/kubernetes/manifests/ sudo mkdir -p /opt/bootstrap/assets sudo mv manifests /opt/bootstrap/assets/manifests - sudo mkdir -p /opt/bootstrap/assets/manifests/crds - sudo mv manifests-networking/{crd,cluster}*.yaml /opt/bootstrap/assets/manifests/crds 2>/dev/null || true sudo mv manifests-networking/* /opt/bootstrap/assets/manifests/ rm -rf assets auth static-manifests tls manifests-networking - path: /opt/bootstrap/apply @@ -179,10 +177,6 @@ storage: echo "Waiting for static pod control plane" sleep 5 done - until kubectl apply -f /assets/manifests/crds -R; do - echo "Retry Custom Resource Definition manifests" - sleep 5 - done until kubectl apply -f /assets/manifests -R; do echo "Retry applying manifests" sleep 5 diff --git a/digital-ocean/container-linux/kubernetes/cl/controller.yaml b/digital-ocean/container-linux/kubernetes/cl/controller.yaml index 88f8d93fc..0816ca90a 100644 --- a/digital-ocean/container-linux/kubernetes/cl/controller.yaml +++ b/digital-ocean/container-linux/kubernetes/cl/controller.yaml @@ -176,8 +176,6 @@ storage: sudo mv static-manifests/* /etc/kubernetes/manifests/ sudo mkdir -p /opt/bootstrap/assets sudo mv manifests /opt/bootstrap/assets/manifests - sudo mkdir -p /opt/bootstrap/assets/manifests/crds - sudo mv manifests-networking/{crd,cluster}*.yaml /opt/bootstrap/assets/manifests/crds 2>/dev/null || true sudo mv manifests-networking/* /opt/bootstrap/assets/manifests/ rm -rf assets auth static-manifests tls manifests-networking - path: /opt/bootstrap/apply @@ -191,10 +189,6 @@ storage: echo "Waiting for static pod control plane" sleep 5 done - until kubectl apply -f /assets/manifests/crds -R; do - echo "Retry Custom Resource Definition manifests" - sleep 5 - done until kubectl apply -f /assets/manifests -R; do echo "Retry applying manifests" sleep 5 diff --git a/google-cloud/container-linux/kubernetes/cl/controller.yaml b/google-cloud/container-linux/kubernetes/cl/controller.yaml index fa1ba8ead..a07c23bb6 100644 --- a/google-cloud/container-linux/kubernetes/cl/controller.yaml +++ b/google-cloud/container-linux/kubernetes/cl/controller.yaml @@ -167,8 +167,6 @@ storage: sudo mv static-manifests/* /etc/kubernetes/manifests/ sudo mkdir -p /opt/bootstrap/assets sudo mv manifests /opt/bootstrap/assets/manifests - sudo mkdir -p /opt/bootstrap/assets/manifests/crds - sudo mv manifests-networking/{crd,cluster}*.yaml /opt/bootstrap/assets/manifests/crds 2>/dev/null || true sudo mv manifests-networking/* /opt/bootstrap/assets/manifests/ rm -rf assets auth static-manifests tls manifests-networking - path: /opt/bootstrap/apply @@ -182,10 +180,6 @@ storage: echo "Waiting for static pod control plane" sleep 5 done - until kubectl apply -f /assets/manifests/crds -R; do - echo "Retry Custom Resource Definition manifests" - sleep 5 - done until kubectl apply -f /assets/manifests -R; do echo "Retry applying manifests" sleep 5 diff --git a/google-cloud/fedora-coreos/kubernetes/fcc/controller.yaml b/google-cloud/fedora-coreos/kubernetes/fcc/controller.yaml index 8d51d374c..33416d2c3 100644 --- a/google-cloud/fedora-coreos/kubernetes/fcc/controller.yaml +++ b/google-cloud/fedora-coreos/kubernetes/fcc/controller.yaml @@ -155,8 +155,6 @@ storage: sudo mv static-manifests/* /etc/kubernetes/manifests/ sudo mkdir -p /opt/bootstrap/assets sudo mv manifests /opt/bootstrap/assets/manifests - sudo mkdir -p /opt/bootstrap/assets/manifests/crds - sudo mv manifests-networking/{crd,cluster}*.yaml /opt/bootstrap/assets/manifests/crds 2>/dev/null || true sudo mv manifests-networking/* /opt/bootstrap/assets/manifests/ rm -rf assets auth static-manifests tls manifests-networking - path: /opt/bootstrap/apply @@ -169,10 +167,6 @@ storage: echo "Waiting for static pod control plane" sleep 5 done - until kubectl apply -f /assets/manifests/crds -R; do - echo "Retry Custom Resource Definition manifests" - sleep 5 - done until kubectl apply -f /assets/manifests -R; do echo "Retry applying manifests" sleep 5 From 2b1b918b43fce625edf52e72c03e9724f44c83c9 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Sat, 18 Apr 2020 15:30:28 -0700 Subject: [PATCH 395/523] Revert Flatcar Linux Azure to manual upload images * Initial support for Flatcar Linux on Azure used the Flatcar Linux Azure Marketplace images (e.g. `flatcar-stable`) in https://github.com/poseidon/typhoon/pull/664 * Flatcar Linux Azure Marketplace images have some unresolved items https://github.com/poseidon/typhoon/issues/703 * Until the Marketplace items are resolved, revert to requiring Flatcar Linux's images be manually uploaded (like GCP and DigitalOcean) --- CHANGES.md | 15 ++++---- .../container-linux/kubernetes/controllers.tf | 21 +++++------ azure/container-linux/kubernetes/variables.tf | 3 +- .../kubernetes/workers/workers.tf | 21 +++++------ docs/cl/azure.md | 35 ++++++++++++++++++- docs/fedora-coreos/azure.md | 2 +- 6 files changed, 61 insertions(+), 36 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 26a194eca..bad4fd6ed 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -8,6 +8,7 @@ Notable changes between versions. * Choose Fedora CoreOS or Flatcar Linux (**action required**) * Use a `fedora-coreos` module for Fedora CoreOS * Use a `container-linux` module for Flatcar Linux +* Change Container Linux modules' defaults from CoreOS Container Linux to [Flatcar Container Linux](https://typhoon.psdn.io/architecture/operating-systems/) ([#702](https://github.com/poseidon/typhoon/pull/702)) * CoreOS Container Linux [won't receive updates](https://coreos.com/os/eol/) after May 2020 ### Fedora CoreOS @@ -16,27 +17,29 @@ Notable changes between versions. * Add support for Fedora CoreOS ([#704](https://github.com/poseidon/typhoon/pull/704)) -### Flatcar Linux / Container Linux +### Container Linux #### AWS -* Change Container Linux `os_image` default from `coreos-stable` to `flatcar-stable` ([#702](https://github.com/poseidon/typhoon/pull/702)) +* Change `os_image` default from `coreos-stable` to `flatcar-stable` ([#702](https://github.com/poseidon/typhoon/pull/702)) #### Azure -* Change Container Linux `os_image` default from `coreos-stable` to `flatcar-stable` ([#702](https://github.com/poseidon/typhoon/pull/702)) +* Change `os_image` to be required. Recommend uploading a Flatcar Linux image (**action required**) ([#702](https://github.com/poseidon/typhoon/pull/702)) +* Disable Flatcar Linux Azure Marketplace image [support](https://github.com/poseidon/typhoon/pull/664) (**breaking**, [#707](https://github.com/poseidon/typhoon/pull/707)) + * Revert to manual uploading until marketplace issue is closed ([#703](https://github.com/poseidon/typhoon/issues/703)) #### Bare-Metal -* Container Linux users should change [os_channel](https://typhoon.psdn.io/cl/bare-metal/#required) from a CoreOS channel to a Flatcar channel +* Recommend changing [os_channel](https://typhoon.psdn.io/cl/bare-metal/#required) from `coreos-stable` to `flatcar-stable` #### Google -* Change Container Linux `os_image` to be required. Container Linux users should upload a Flatcar Linux image and set it (**action required**) ([#702](https://github.com/poseidon/typhoon/pull/702)) +* Change `os_image` to be required. Recommend uploading a Flatcar Linux image (**action required**) ([#702](https://github.com/poseidon/typhoon/pull/702)) #### DigitalOcean -* Change Container Linux `os_image` to be required. Container Linux users should upload a Flatcar Linux image and set it (**action required**) ([#702](https://github.com/poseidon/typhoon/pull/702)) +* Change `os_image` to be required. Recommend uploading a Flatcar Linux image (**action required**) ([#702](https://github.com/poseidon/typhoon/pull/702)) ## v1.18.1 diff --git a/azure/container-linux/kubernetes/controllers.tf b/azure/container-linux/kubernetes/controllers.tf index 671e7fb8f..28adcbc5a 100644 --- a/azure/container-linux/kubernetes/controllers.tf +++ b/azure/container-linux/kubernetes/controllers.tf @@ -53,23 +53,18 @@ resource "azurerm_linux_virtual_machine" "controllers" { storage_account_type = "Premium_LRS" } - source_image_reference { - publisher = local.flavor == "flatcar" ? "Kinvolk" : "CoreOS" - offer = local.flavor == "flatcar" ? "flatcar-container-linux" : "CoreOS" - sku = local.channel - version = "latest" - } - - # Gross hack just for Flatcar Linux - dynamic "plan" { - for_each = local.flavor == "flatcar" ? [1] : [] + // CoreOS Container Linux or Flatcar Container Linux (manual upload) + dynamic "source_image_reference" { + for_each = local.flavor == "coreos" ? [1] : [] content { - name = local.channel - publisher = "kinvolk" - product = "flatcar-container-linux" + publisher = "CoreOS" + offer = "CoreOS" + sku = local.channel + version = "latest" } } + source_image_id = local.flavor == "coreos" ? null : var.os_image # network network_interface_ids = [ diff --git a/azure/container-linux/kubernetes/variables.tf b/azure/container-linux/kubernetes/variables.tf index 44827db49..a31e20063 100644 --- a/azure/container-linux/kubernetes/variables.tf +++ b/azure/container-linux/kubernetes/variables.tf @@ -48,8 +48,7 @@ variable "worker_type" { variable "os_image" { type = string - description = "Channel for a Container Linux derivative (coreos-stable, coreos-beta, coreos-alpha, flatcar-stable, flatcar-beta)" - default = "flatcar-stable" + description = "Channel for a Container Linux derivative (/subscriptions/some-flatcar-upload, coreos-stable, coreos-beta, coreos-alpha)" } variable "disk_size" { diff --git a/azure/container-linux/kubernetes/workers/workers.tf b/azure/container-linux/kubernetes/workers/workers.tf index 3e151e6e5..6b626eddb 100644 --- a/azure/container-linux/kubernetes/workers/workers.tf +++ b/azure/container-linux/kubernetes/workers/workers.tf @@ -24,23 +24,18 @@ resource "azurerm_linux_virtual_machine_scale_set" "workers" { caching = "ReadWrite" } - source_image_reference { - publisher = local.flavor == "flatcar" ? "Kinvolk" : "CoreOS" - offer = local.flavor == "flatcar" ? "flatcar-container-linux" : "CoreOS" - sku = local.channel - version = "latest" - } - - # Gross hack just for Flatcar Linux - dynamic "plan" { - for_each = local.flavor == "flatcar" ? [1] : [] + // CoreOS Container Linux or Flatcar Container Linux (manual upload) + dynamic "source_image_reference" { + for_each = local.flavor == "coreos" ? [1] : [] content { - name = local.channel - publisher = "kinvolk" - product = "flatcar-container-linux" + publisher = "CoreOS" + offer = "CoreOS" + sku = local.channel + version = "latest" } } + source_image_id = local.flavor == "coreos" ? null : var.os_image # Azure requires setting admin_ssh_key, though Ignition custom_data handles it too admin_username = "core" diff --git a/docs/cl/azure.md b/docs/cl/azure.md index a721a06ba..fc0855a11 100644 --- a/docs/cl/azure.md +++ b/docs/cl/azure.md @@ -57,6 +57,38 @@ provider "ct" { Additional configuration options are described in the `azurerm` provider [docs](https://www.terraform.io/docs/providers/azurerm/). +### Flatcar Linux Images + +Flatcar Linux publishes images for Azure. Azure allows custom images to be uploaded to a storage account bucket and imported. + +[Download](https://www.flatcar-linux.org/releases/) a Flatcar Linux Azure VHD image and upload it to an Azure storage account container (i.e. bucket). + +Azure requires fixed VHDs and Flatcar Linux provides dynamic VHDs, so uploads require Azure tools and cannot be done through the UI. Azure's tool compilation requires old versions, so Flatcar Linux has packaged a container image you may choose to use. See their [docs](https://docs.flatcar-linux.org/os/booting-on-azure/#uploading-your-own-image). + +``` +bzip2 -d flatcar_production_azure_image.vhd.bz2 +``` + +``` +podman run -it --entrypoint=/bin/bash quay.io/kinvolk/azure-flatcar-image-upload +... + +# az login +# az storage account keys list --resource-group GROUP --account-name BUCKET | jq -r '.[0].value' +# azure-vhd-utils upload --localvhdpath /data/flatcar_production_azure_image.vhd --stgaccountname BUCKET --containername flatcar-linux --blobname flatcar-stable-2345.3.1 --stgaccountkey "KEYFROMABOVE" +# exit +``` + +Create an Azure disk (note disk ID) and create an Azure image from it (note image ID). + +``` +az disk create --name flatcar-stable-2345.3.1 -g GROUP --source https://BUCKET.blob.core.windows.net/flatcar-linux/flatcar_production_azure_image.vhd + +az image create --name flatcar-stable-2345.3.1 -g GROUP --os-type=linux --source /subscriptions/some/path/providers/Microsoft.Compute/disks/flatcar-stable-2345.3.1 +``` + +Set the [os_image](#variables) in the next step. + ## Cluster Define a Kubernetes cluster using the module `azure/container-linux/kubernetes`. @@ -72,6 +104,7 @@ module "ramius" { dns_zone_group = "example-group" # configuration + os_image = "/subscriptions/some/path/Microsoft.Compute/images/flatcar-stable-2345.3.1" ssh_authorized_key = "ssh-rsa AAAAB3Nz..." # optional @@ -185,6 +218,7 @@ Check the [variables.tf](https://github.com/poseidon/typhoon/blob/master/azure/c | region | Azure region | "centralus" | | dns_zone | Azure DNS zone | "azure.example.com" | | dns_zone_group | Resource group where the Azure DNS zone resides | "global" | +| os_image | Container Linux image for instances | "/subscriptions/..../some-flatcar-image", coreos-stable, coreos-beta, coreos-alpha | | ssh_authorized_key | SSH public key for user 'core' | "ssh-rsa AAAAB3NZ..." | !!! tip @@ -225,7 +259,6 @@ Reference the DNS zone with `azurerm_dns_zone.clusters.name` and its resource gr | worker_count | Number of workers | 1 | 3 | | controller_type | Machine type for controllers | "Standard_B2s" | See below | | worker_type | Machine type for workers | "Standard_DS1_v2" | See below | -| os_image | Channel for a Container Linux derivative | "flatcar-stable" | coreos-stable, coreos-beta, coreos-alpha, flatcar-stable, flatcar-beta | | disk_size | Size of the disk in GB | 40 | 100 | | worker_priority | Set priority to Spot to use reduced cost surplus capacity, with the tradeoff that instances can be deallocated at any time | Regular | Spot | | controller_snippets | Controller Container Linux Config snippets | [] | [example](/advanced/customization/#usage) | diff --git a/docs/fedora-coreos/azure.md b/docs/fedora-coreos/azure.md index 719e3030f..e7e1278e0 100644 --- a/docs/fedora-coreos/azure.md +++ b/docs/fedora-coreos/azure.md @@ -67,7 +67,7 @@ Fedora CoreOS publishes images for Azure, but does not yet upload them. Azure al xz -d fedora-coreos-31.20200323.3.2-azure.x86_64.vhd.xz ``` -Create an Azure disk (note its ID) and create an Azure image from it (note its ID). +Create an Azure disk (note disk ID) and create an Azure image from it (note image ID). ``` az disk create --name fedora-coreos-31.20200323.3.2 -g GROUP --source https://BUCKET.blob.core.windows.net/fedora-coreos/fedora-coreos-31.20200323.3.2-azure.x86_64.vhd From feac94605ad4fcd395661e1b8ebbd5b0804bca16 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Sun, 19 Apr 2020 16:14:16 -0700 Subject: [PATCH 396/523] Fix bootstrap mount to use shared volume SELinux label * Race: During initial bootstrap, static control plane pods could hang with Permission denied to bootstrap secrets. A manual fix involved restarting Kubelet, which relabeled mounts The race had no effect on subsequent reboots. * bootstrap.service runs podman with a private unshared mount of /etc/kubernetes/bootstrap-secrets which uses an SELinux MCS label with a category pair. However, bootstrap-secrets should be shared as its mounted by Docker pods kube-apiserver, kube-scheduler, and kube-controller-manager. Restarting Kubelet was a manual fix because Kubelet relabels all /etc/kubernetes * Fix bootstrap Pod to use the shared volume label, which leaves bootstrap-secrets files with SELinux level s0 without MCS * Also allow failed bootstrap.service to be re-applied. This was missing on bare-metal and AWS --- CHANGES.md | 2 ++ aws/fedora-coreos/kubernetes/fcc/controller.yaml | 3 ++- azure/fedora-coreos/kubernetes/fcc/controller.yaml | 2 +- bare-metal/fedora-coreos/kubernetes/fcc/controller.yaml | 3 ++- digital-ocean/fedora-coreos/kubernetes/fcc/controller.yaml | 2 +- google-cloud/fedora-coreos/kubernetes/fcc/controller.yaml | 2 +- 6 files changed, 9 insertions(+), 5 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index bad4fd6ed..d70e1b465 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -13,6 +13,8 @@ Notable changes between versions. ### Fedora CoreOS +* Fix race condition during bootstrap related to SELinux shared content label ([#708](https://github.com/poseidon/typhoon/pull/708)) + #### Azure * Add support for Fedora CoreOS ([#704](https://github.com/poseidon/typhoon/pull/704)) diff --git a/aws/fedora-coreos/kubernetes/fcc/controller.yaml b/aws/fedora-coreos/kubernetes/fcc/controller.yaml index 1bcb4b5fa..656e5ca0a 100644 --- a/aws/fedora-coreos/kubernetes/fcc/controller.yaml +++ b/aws/fedora-coreos/kubernetes/fcc/controller.yaml @@ -116,9 +116,10 @@ systemd: Type=oneshot RemainAfterExit=true WorkingDirectory=/opt/bootstrap + ExecStartPre=-/usr/bin/podman rm bootstrap ExecStart=/usr/bin/podman run --name bootstrap \ --network host \ - --volume /etc/kubernetes/bootstrap-secrets:/etc/kubernetes/secrets:ro,Z \ + --volume /etc/kubernetes/bootstrap-secrets:/etc/kubernetes/secrets:ro,z \ --volume /opt/bootstrap/assets:/assets:ro,Z \ --volume /opt/bootstrap/apply:/apply:ro,Z \ --entrypoint=/apply \ diff --git a/azure/fedora-coreos/kubernetes/fcc/controller.yaml b/azure/fedora-coreos/kubernetes/fcc/controller.yaml index 95af2d12c..aeae2f883 100644 --- a/azure/fedora-coreos/kubernetes/fcc/controller.yaml +++ b/azure/fedora-coreos/kubernetes/fcc/controller.yaml @@ -119,7 +119,7 @@ systemd: ExecStartPre=-/usr/bin/podman rm bootstrap ExecStart=/usr/bin/podman run --name bootstrap \ --network host \ - --volume /etc/kubernetes/bootstrap-secrets:/etc/kubernetes/secrets:ro,Z \ + --volume /etc/kubernetes/bootstrap-secrets:/etc/kubernetes/secrets:ro,z \ --volume /opt/bootstrap/assets:/assets:ro,Z \ --volume /opt/bootstrap/apply:/apply:ro,Z \ --entrypoint=/apply \ diff --git a/bare-metal/fedora-coreos/kubernetes/fcc/controller.yaml b/bare-metal/fedora-coreos/kubernetes/fcc/controller.yaml index c16cc4afd..444d56ad9 100644 --- a/bare-metal/fedora-coreos/kubernetes/fcc/controller.yaml +++ b/bare-metal/fedora-coreos/kubernetes/fcc/controller.yaml @@ -127,9 +127,10 @@ systemd: Type=oneshot RemainAfterExit=true WorkingDirectory=/opt/bootstrap + ExecStartPre=-/usr/bin/podman rm bootstrap ExecStart=/usr/bin/podman run --name bootstrap \ --network host \ - --volume /etc/kubernetes/bootstrap-secrets:/etc/kubernetes/secrets:ro,Z \ + --volume /etc/kubernetes/bootstrap-secrets:/etc/kubernetes/secrets:ro,z \ --volume /opt/bootstrap/assets:/assets:ro,Z \ --volume /opt/bootstrap/apply:/apply:ro,Z \ --entrypoint=/apply \ diff --git a/digital-ocean/fedora-coreos/kubernetes/fcc/controller.yaml b/digital-ocean/fedora-coreos/kubernetes/fcc/controller.yaml index 6ccfb5ae0..24ad42e1d 100644 --- a/digital-ocean/fedora-coreos/kubernetes/fcc/controller.yaml +++ b/digital-ocean/fedora-coreos/kubernetes/fcc/controller.yaml @@ -131,7 +131,7 @@ systemd: ExecStartPre=-/usr/bin/podman rm bootstrap ExecStart=/usr/bin/podman run --name bootstrap \ --network host \ - --volume /etc/kubernetes/bootstrap-secrets:/etc/kubernetes/secrets:ro,Z \ + --volume /etc/kubernetes/bootstrap-secrets:/etc/kubernetes/secrets:ro,z \ --volume /opt/bootstrap/assets:/assets:ro,Z \ --volume /opt/bootstrap/apply:/apply:ro,Z \ --entrypoint=/apply \ diff --git a/google-cloud/fedora-coreos/kubernetes/fcc/controller.yaml b/google-cloud/fedora-coreos/kubernetes/fcc/controller.yaml index 33416d2c3..656e5ca0a 100644 --- a/google-cloud/fedora-coreos/kubernetes/fcc/controller.yaml +++ b/google-cloud/fedora-coreos/kubernetes/fcc/controller.yaml @@ -119,7 +119,7 @@ systemd: ExecStartPre=-/usr/bin/podman rm bootstrap ExecStart=/usr/bin/podman run --name bootstrap \ --network host \ - --volume /etc/kubernetes/bootstrap-secrets:/etc/kubernetes/secrets:ro,Z \ + --volume /etc/kubernetes/bootstrap-secrets:/etc/kubernetes/secrets:ro,z \ --volume /opt/bootstrap/assets:/assets:ro,Z \ --volume /opt/bootstrap/apply:/apply:ro,Z \ --entrypoint=/apply \ From fcbee123342edbe210296b9846fbb03dbf1f5310 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Sun, 19 Apr 2020 16:44:26 -0700 Subject: [PATCH 397/523] Fix race condition creating DigitalOcean firewall rules * DigitalOcean firewall rules should reference Terraform tag resources rather than using tag strings. Otherwise, terraform apply can fail (neeeds rerun) if a tag has not yet been created --- CHANGES.md | 11 +++++++++-- digital-ocean/container-linux/kubernetes/network.tf | 9 ++++++--- digital-ocean/fedora-coreos/kubernetes/network.tf | 9 ++++++--- 3 files changed, 21 insertions(+), 8 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index d70e1b465..e7684f8af 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -4,6 +4,8 @@ Notable changes between versions. ## Latest +## v1.18.2 + * Kubernetes [v1.18.2](https://github.com/kubernetes/kubernetes/blob/master/CHANGELOG/CHANGELOG-1.18.md#v1182) * Choose Fedora CoreOS or Flatcar Linux (**action required**) * Use a `fedora-coreos` module for Fedora CoreOS @@ -13,13 +15,17 @@ Notable changes between versions. ### Fedora CoreOS -* Fix race condition during bootstrap related to SELinux shared content label ([#708](https://github.com/poseidon/typhoon/pull/708)) +* Fix bootstrap race condition from SELinux unshared content label ([#708](https://github.com/poseidon/typhoon/pull/708)) #### Azure * Add support for Fedora CoreOS ([#704](https://github.com/poseidon/typhoon/pull/704)) -### Container Linux +#### DigitalOcean + +* Fix race condition creating firewall allow rules ([#709](https://github.com/poseidon/typhoon/pull/709)) + +### Flatcar Linux #### AWS @@ -42,6 +48,7 @@ Notable changes between versions. #### DigitalOcean * Change `os_image` to be required. Recommend uploading a Flatcar Linux image (**action required**) ([#702](https://github.com/poseidon/typhoon/pull/702)) +* Fix race condition creating firewall allow rules ([#709](https://github.com/poseidon/typhoon/pull/709)) ## v1.18.1 diff --git a/digital-ocean/container-linux/kubernetes/network.tf b/digital-ocean/container-linux/kubernetes/network.tf index bc5434852..e8b0564db 100644 --- a/digital-ocean/container-linux/kubernetes/network.tf +++ b/digital-ocean/container-linux/kubernetes/network.tf @@ -1,7 +1,10 @@ resource "digitalocean_firewall" "rules" { name = var.cluster_name - tags = ["${var.cluster_name}-controller", "${var.cluster_name}-worker"] + tags = [ + digitalocean_tag.controllers.name, + digitalocean_tag.workers.name + ] # allow ssh, internal flannel, internal node-exporter, internal kubelet inbound_rule { @@ -59,7 +62,7 @@ resource "digitalocean_firewall" "rules" { resource "digitalocean_firewall" "controllers" { name = "${var.cluster_name}-controllers" - tags = ["${var.cluster_name}-controller"] + tags = [digitalocean_tag.controllers.name] # etcd inbound_rule { @@ -93,7 +96,7 @@ resource "digitalocean_firewall" "controllers" { resource "digitalocean_firewall" "workers" { name = "${var.cluster_name}-workers" - tags = ["${var.cluster_name}-worker"] + tags = [digitalocean_tag.workers.name] # allow HTTP/HTTPS ingress inbound_rule { diff --git a/digital-ocean/fedora-coreos/kubernetes/network.tf b/digital-ocean/fedora-coreos/kubernetes/network.tf index bc5434852..e8b0564db 100644 --- a/digital-ocean/fedora-coreos/kubernetes/network.tf +++ b/digital-ocean/fedora-coreos/kubernetes/network.tf @@ -1,7 +1,10 @@ resource "digitalocean_firewall" "rules" { name = var.cluster_name - tags = ["${var.cluster_name}-controller", "${var.cluster_name}-worker"] + tags = [ + digitalocean_tag.controllers.name, + digitalocean_tag.workers.name + ] # allow ssh, internal flannel, internal node-exporter, internal kubelet inbound_rule { @@ -59,7 +62,7 @@ resource "digitalocean_firewall" "rules" { resource "digitalocean_firewall" "controllers" { name = "${var.cluster_name}-controllers" - tags = ["${var.cluster_name}-controller"] + tags = [digitalocean_tag.controllers.name] # etcd inbound_rule { @@ -93,7 +96,7 @@ resource "digitalocean_firewall" "controllers" { resource "digitalocean_firewall" "workers" { name = "${var.cluster_name}-workers" - tags = ["${var.cluster_name}-worker"] + tags = [digitalocean_tag.workers.name] # allow HTTP/HTTPS ingress inbound_rule { From 84ed0a31c30b0cd544b2d8a68ad5a4a5b992f030 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Mon, 20 Apr 2020 18:09:24 -0700 Subject: [PATCH 398/523] Update Prometheus from v2.17.1 to v2.17.2 * https://github.com/prometheus/prometheus/releases/tag/v2.17.2 --- CHANGES.md | 4 ++++ addons/prometheus/deployment.yaml | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index e7684f8af..e27a8bf96 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -4,6 +4,10 @@ Notable changes between versions. ## Latest +#### Addons + +* Update Prometheus from v2.17.1 to v2.17.2 + ## v1.18.2 * Kubernetes [v1.18.2](https://github.com/kubernetes/kubernetes/blob/master/CHANGELOG/CHANGELOG-1.18.md#v1182) diff --git a/addons/prometheus/deployment.yaml b/addons/prometheus/deployment.yaml index 35569e836..6f008437f 100644 --- a/addons/prometheus/deployment.yaml +++ b/addons/prometheus/deployment.yaml @@ -20,7 +20,7 @@ spec: serviceAccountName: prometheus containers: - name: prometheus - image: quay.io/prometheus/prometheus:v2.17.1 + image: quay.io/prometheus/prometheus:v2.17.2 args: - --web.listen-address=0.0.0.0:9090 - --config.file=/etc/prometheus/prometheus.yaml From d8966afddaddf60515e3887bb235350d9744aaf7 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Wed, 22 Apr 2020 20:27:08 -0700 Subject: [PATCH 399/523] Remove extraneous sudo from layout asset unpacking --- aws/container-linux/kubernetes/cl/controller.yaml | 10 +++++----- aws/fedora-coreos/kubernetes/fcc/controller.yaml | 10 +++++----- azure/container-linux/kubernetes/cl/controller.yaml | 10 +++++----- azure/fedora-coreos/kubernetes/fcc/controller.yaml | 10 +++++----- .../container-linux/kubernetes/cl/controller.yaml | 10 +++++----- .../fedora-coreos/kubernetes/fcc/controller.yaml | 10 +++++----- .../container-linux/kubernetes/cl/controller.yaml | 10 +++++----- .../fedora-coreos/kubernetes/fcc/controller.yaml | 10 +++++----- .../container-linux/kubernetes/cl/controller.yaml | 10 +++++----- .../fedora-coreos/kubernetes/fcc/controller.yaml | 10 +++++----- 10 files changed, 50 insertions(+), 50 deletions(-) diff --git a/aws/container-linux/kubernetes/cl/controller.yaml b/aws/container-linux/kubernetes/cl/controller.yaml index c1a131c40..9beb4a8b9 100644 --- a/aws/container-linux/kubernetes/cl/controller.yaml +++ b/aws/container-linux/kubernetes/cl/controller.yaml @@ -165,11 +165,11 @@ storage: chmod -R 500 /etc/ssl/etcd mv auth/kubeconfig /etc/kubernetes/bootstrap-secrets/ mv tls/k8s/* /etc/kubernetes/bootstrap-secrets/ - sudo mkdir -p /etc/kubernetes/manifests - sudo mv static-manifests/* /etc/kubernetes/manifests/ - sudo mkdir -p /opt/bootstrap/assets - sudo mv manifests /opt/bootstrap/assets/manifests - sudo mv manifests-networking/* /opt/bootstrap/assets/manifests/ + mkdir -p /etc/kubernetes/manifests + mv static-manifests/* /etc/kubernetes/manifests/ + mkdir -p /opt/bootstrap/assets + mv manifests /opt/bootstrap/assets/manifests + mv manifests-networking/* /opt/bootstrap/assets/manifests/ rm -rf assets auth static-manifests tls manifests-networking - path: /opt/bootstrap/apply filesystem: root diff --git a/aws/fedora-coreos/kubernetes/fcc/controller.yaml b/aws/fedora-coreos/kubernetes/fcc/controller.yaml index 656e5ca0a..727960d71 100644 --- a/aws/fedora-coreos/kubernetes/fcc/controller.yaml +++ b/aws/fedora-coreos/kubernetes/fcc/controller.yaml @@ -151,11 +151,11 @@ storage: chmod -R 500 /etc/ssl/etcd mv auth/kubeconfig /etc/kubernetes/bootstrap-secrets/ mv tls/k8s/* /etc/kubernetes/bootstrap-secrets/ - sudo mkdir -p /etc/kubernetes/manifests - sudo mv static-manifests/* /etc/kubernetes/manifests/ - sudo mkdir -p /opt/bootstrap/assets - sudo mv manifests /opt/bootstrap/assets/manifests - sudo mv manifests-networking/* /opt/bootstrap/assets/manifests/ + mkdir -p /etc/kubernetes/manifests + mv static-manifests/* /etc/kubernetes/manifests/ + mkdir -p /opt/bootstrap/assets + mv manifests /opt/bootstrap/assets/manifests + mv manifests-networking/* /opt/bootstrap/assets/manifests/ rm -rf assets auth static-manifests tls manifests-networking - path: /opt/bootstrap/apply mode: 0544 diff --git a/azure/container-linux/kubernetes/cl/controller.yaml b/azure/container-linux/kubernetes/cl/controller.yaml index 3e036c7e4..293e3b457 100644 --- a/azure/container-linux/kubernetes/cl/controller.yaml +++ b/azure/container-linux/kubernetes/cl/controller.yaml @@ -163,11 +163,11 @@ storage: chmod -R 500 /etc/ssl/etcd mv auth/kubeconfig /etc/kubernetes/bootstrap-secrets/ mv tls/k8s/* /etc/kubernetes/bootstrap-secrets/ - sudo mkdir -p /etc/kubernetes/manifests - sudo mv static-manifests/* /etc/kubernetes/manifests/ - sudo mkdir -p /opt/bootstrap/assets - sudo mv manifests /opt/bootstrap/assets/manifests - sudo mv manifests-networking/* /opt/bootstrap/assets/manifests/ + mkdir -p /etc/kubernetes/manifests + mv static-manifests/* /etc/kubernetes/manifests/ + mkdir -p /opt/bootstrap/assets + mv manifests /opt/bootstrap/assets/manifests + mv manifests-networking/* /opt/bootstrap/assets/manifests/ rm -rf assets auth static-manifests tls manifests-networking - path: /opt/bootstrap/apply filesystem: root diff --git a/azure/fedora-coreos/kubernetes/fcc/controller.yaml b/azure/fedora-coreos/kubernetes/fcc/controller.yaml index aeae2f883..c74ecf585 100644 --- a/azure/fedora-coreos/kubernetes/fcc/controller.yaml +++ b/azure/fedora-coreos/kubernetes/fcc/controller.yaml @@ -151,11 +151,11 @@ storage: chmod -R 500 /etc/ssl/etcd mv auth/kubeconfig /etc/kubernetes/bootstrap-secrets/ mv tls/k8s/* /etc/kubernetes/bootstrap-secrets/ - sudo mkdir -p /etc/kubernetes/manifests - sudo mv static-manifests/* /etc/kubernetes/manifests/ - sudo mkdir -p /opt/bootstrap/assets - sudo mv manifests /opt/bootstrap/assets/manifests - sudo mv manifests-networking/* /opt/bootstrap/assets/manifests/ + mkdir -p /etc/kubernetes/manifests + mv static-manifests/* /etc/kubernetes/manifests/ + mkdir -p /opt/bootstrap/assets + mv manifests /opt/bootstrap/assets/manifests + mv manifests-networking/* /opt/bootstrap/assets/manifests/ rm -rf assets auth static-manifests tls manifests-networking - path: /opt/bootstrap/apply mode: 0544 diff --git a/bare-metal/container-linux/kubernetes/cl/controller.yaml b/bare-metal/container-linux/kubernetes/cl/controller.yaml index ea9c2d9d1..9cde76e89 100644 --- a/bare-metal/container-linux/kubernetes/cl/controller.yaml +++ b/bare-metal/container-linux/kubernetes/cl/controller.yaml @@ -181,11 +181,11 @@ storage: chmod -R 500 /etc/ssl/etcd mv auth/kubeconfig /etc/kubernetes/bootstrap-secrets/ mv tls/k8s/* /etc/kubernetes/bootstrap-secrets/ - sudo mkdir -p /etc/kubernetes/manifests - sudo mv static-manifests/* /etc/kubernetes/manifests/ - sudo mkdir -p /opt/bootstrap/assets - sudo mv manifests /opt/bootstrap/assets/manifests - sudo mv manifests-networking/* /opt/bootstrap/assets/manifests/ + mkdir -p /etc/kubernetes/manifests + mv static-manifests/* /etc/kubernetes/manifests/ + mkdir -p /opt/bootstrap/assets + mv manifests /opt/bootstrap/assets/manifests + mv manifests-networking/* /opt/bootstrap/assets/manifests/ rm -rf assets auth static-manifests tls manifests-networking - path: /opt/bootstrap/apply filesystem: root diff --git a/bare-metal/fedora-coreos/kubernetes/fcc/controller.yaml b/bare-metal/fedora-coreos/kubernetes/fcc/controller.yaml index 444d56ad9..630213168 100644 --- a/bare-metal/fedora-coreos/kubernetes/fcc/controller.yaml +++ b/bare-metal/fedora-coreos/kubernetes/fcc/controller.yaml @@ -162,11 +162,11 @@ storage: chmod -R 500 /etc/ssl/etcd mv auth/kubeconfig /etc/kubernetes/bootstrap-secrets/ mv tls/k8s/* /etc/kubernetes/bootstrap-secrets/ - sudo mkdir -p /etc/kubernetes/manifests - sudo mv static-manifests/* /etc/kubernetes/manifests/ - sudo mkdir -p /opt/bootstrap/assets - sudo mv manifests /opt/bootstrap/assets/manifests - sudo mv manifests-networking/* /opt/bootstrap/assets/manifests/ + mkdir -p /etc/kubernetes/manifests + mv static-manifests/* /etc/kubernetes/manifests/ + mkdir -p /opt/bootstrap/assets + mv manifests /opt/bootstrap/assets/manifests + mv manifests-networking/* /opt/bootstrap/assets/manifests/ rm -rf assets auth static-manifests tls manifests-networking - path: /opt/bootstrap/apply mode: 0544 diff --git a/digital-ocean/container-linux/kubernetes/cl/controller.yaml b/digital-ocean/container-linux/kubernetes/cl/controller.yaml index 0816ca90a..c7a297518 100644 --- a/digital-ocean/container-linux/kubernetes/cl/controller.yaml +++ b/digital-ocean/container-linux/kubernetes/cl/controller.yaml @@ -172,11 +172,11 @@ storage: chmod -R 500 /etc/ssl/etcd mv auth/kubeconfig /etc/kubernetes/bootstrap-secrets/ mv tls/k8s/* /etc/kubernetes/bootstrap-secrets/ - sudo mkdir -p /etc/kubernetes/manifests - sudo mv static-manifests/* /etc/kubernetes/manifests/ - sudo mkdir -p /opt/bootstrap/assets - sudo mv manifests /opt/bootstrap/assets/manifests - sudo mv manifests-networking/* /opt/bootstrap/assets/manifests/ + mkdir -p /etc/kubernetes/manifests + mv static-manifests/* /etc/kubernetes/manifests/ + mkdir -p /opt/bootstrap/assets + mv manifests /opt/bootstrap/assets/manifests + mv manifests-networking/* /opt/bootstrap/assets/manifests/ rm -rf assets auth static-manifests tls manifests-networking - path: /opt/bootstrap/apply filesystem: root diff --git a/digital-ocean/fedora-coreos/kubernetes/fcc/controller.yaml b/digital-ocean/fedora-coreos/kubernetes/fcc/controller.yaml index 24ad42e1d..999340575 100644 --- a/digital-ocean/fedora-coreos/kubernetes/fcc/controller.yaml +++ b/digital-ocean/fedora-coreos/kubernetes/fcc/controller.yaml @@ -158,11 +158,11 @@ storage: chmod -R 500 /etc/ssl/etcd mv auth/kubeconfig /etc/kubernetes/bootstrap-secrets/ mv tls/k8s/* /etc/kubernetes/bootstrap-secrets/ - sudo mkdir -p /etc/kubernetes/manifests - sudo mv static-manifests/* /etc/kubernetes/manifests/ - sudo mkdir -p /opt/bootstrap/assets - sudo mv manifests /opt/bootstrap/assets/manifests - sudo mv manifests-networking/* /opt/bootstrap/assets/manifests/ + mkdir -p /etc/kubernetes/manifests + mv static-manifests/* /etc/kubernetes/manifests/ + mkdir -p /opt/bootstrap/assets + mv manifests /opt/bootstrap/assets/manifests + mv manifests-networking/* /opt/bootstrap/assets/manifests/ rm -rf assets auth static-manifests tls manifests-networking - path: /opt/bootstrap/apply mode: 0544 diff --git a/google-cloud/container-linux/kubernetes/cl/controller.yaml b/google-cloud/container-linux/kubernetes/cl/controller.yaml index a07c23bb6..1c88d409c 100644 --- a/google-cloud/container-linux/kubernetes/cl/controller.yaml +++ b/google-cloud/container-linux/kubernetes/cl/controller.yaml @@ -163,11 +163,11 @@ storage: chmod -R 500 /etc/ssl/etcd mv auth/kubeconfig /etc/kubernetes/bootstrap-secrets/ mv tls/k8s/* /etc/kubernetes/bootstrap-secrets/ - sudo mkdir -p /etc/kubernetes/manifests - sudo mv static-manifests/* /etc/kubernetes/manifests/ - sudo mkdir -p /opt/bootstrap/assets - sudo mv manifests /opt/bootstrap/assets/manifests - sudo mv manifests-networking/* /opt/bootstrap/assets/manifests/ + mkdir -p /etc/kubernetes/manifests + mv static-manifests/* /etc/kubernetes/manifests/ + mkdir -p /opt/bootstrap/assets + mv manifests /opt/bootstrap/assets/manifests + mv manifests-networking/* /opt/bootstrap/assets/manifests/ rm -rf assets auth static-manifests tls manifests-networking - path: /opt/bootstrap/apply filesystem: root diff --git a/google-cloud/fedora-coreos/kubernetes/fcc/controller.yaml b/google-cloud/fedora-coreos/kubernetes/fcc/controller.yaml index 656e5ca0a..727960d71 100644 --- a/google-cloud/fedora-coreos/kubernetes/fcc/controller.yaml +++ b/google-cloud/fedora-coreos/kubernetes/fcc/controller.yaml @@ -151,11 +151,11 @@ storage: chmod -R 500 /etc/ssl/etcd mv auth/kubeconfig /etc/kubernetes/bootstrap-secrets/ mv tls/k8s/* /etc/kubernetes/bootstrap-secrets/ - sudo mkdir -p /etc/kubernetes/manifests - sudo mv static-manifests/* /etc/kubernetes/manifests/ - sudo mkdir -p /opt/bootstrap/assets - sudo mv manifests /opt/bootstrap/assets/manifests - sudo mv manifests-networking/* /opt/bootstrap/assets/manifests/ + mkdir -p /etc/kubernetes/manifests + mv static-manifests/* /etc/kubernetes/manifests/ + mkdir -p /opt/bootstrap/assets + mv manifests /opt/bootstrap/assets/manifests + mv manifests-networking/* /opt/bootstrap/assets/manifests/ rm -rf assets auth static-manifests tls manifests-networking - path: /opt/bootstrap/apply mode: 0544 From 38a6bddd06bba710aab44926e6e92375e0fd36f3 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Wed, 22 Apr 2020 20:28:55 -0700 Subject: [PATCH 400/523] Update Calico from v3.13.1 to v3.13.3 * https://docs.projectcalico.org/v3.13/release-notes/ --- CHANGES.md | 2 ++ aws/container-linux/kubernetes/bootstrap.tf | 2 +- aws/fedora-coreos/kubernetes/bootstrap.tf | 2 +- azure/container-linux/kubernetes/bootstrap.tf | 2 +- azure/fedora-coreos/kubernetes/bootstrap.tf | 2 +- bare-metal/container-linux/kubernetes/bootstrap.tf | 2 +- bare-metal/fedora-coreos/kubernetes/bootstrap.tf | 2 +- digital-ocean/container-linux/kubernetes/bootstrap.tf | 2 +- digital-ocean/fedora-coreos/kubernetes/bootstrap.tf | 2 +- google-cloud/container-linux/kubernetes/bootstrap.tf | 2 +- google-cloud/fedora-coreos/kubernetes/bootstrap.tf | 2 +- 11 files changed, 12 insertions(+), 10 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index e27a8bf96..6a28683d6 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -4,6 +4,8 @@ Notable changes between versions. ## Latest +* Update Calico from v3.13.1 to [v3.13.3](https://docs.projectcalico.org/v3.13/release-notes/) + #### Addons * Update Prometheus from v2.17.1 to v2.17.2 diff --git a/aws/container-linux/kubernetes/bootstrap.tf b/aws/container-linux/kubernetes/bootstrap.tf index 1652b95dc..860821aee 100644 --- a/aws/container-linux/kubernetes/bootstrap.tf +++ b/aws/container-linux/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=14d0b2087962a0f2557c184f3f523548ce19bbdc" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=c62c7f5a1a3a3f9cebe7c1382077ad2dbf3727e6" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/aws/fedora-coreos/kubernetes/bootstrap.tf b/aws/fedora-coreos/kubernetes/bootstrap.tf index d35fd11ef..1991cb13c 100644 --- a/aws/fedora-coreos/kubernetes/bootstrap.tf +++ b/aws/fedora-coreos/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=14d0b2087962a0f2557c184f3f523548ce19bbdc" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=c62c7f5a1a3a3f9cebe7c1382077ad2dbf3727e6" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/azure/container-linux/kubernetes/bootstrap.tf b/azure/container-linux/kubernetes/bootstrap.tf index ef9aea940..4495381db 100644 --- a/azure/container-linux/kubernetes/bootstrap.tf +++ b/azure/container-linux/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=14d0b2087962a0f2557c184f3f523548ce19bbdc" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=c62c7f5a1a3a3f9cebe7c1382077ad2dbf3727e6" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/azure/fedora-coreos/kubernetes/bootstrap.tf b/azure/fedora-coreos/kubernetes/bootstrap.tf index 152f4adcb..6b4c90ff2 100644 --- a/azure/fedora-coreos/kubernetes/bootstrap.tf +++ b/azure/fedora-coreos/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=14d0b2087962a0f2557c184f3f523548ce19bbdc" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=c62c7f5a1a3a3f9cebe7c1382077ad2dbf3727e6" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/bare-metal/container-linux/kubernetes/bootstrap.tf b/bare-metal/container-linux/kubernetes/bootstrap.tf index 6d577ae0c..45179f959 100644 --- a/bare-metal/container-linux/kubernetes/bootstrap.tf +++ b/bare-metal/container-linux/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=14d0b2087962a0f2557c184f3f523548ce19bbdc" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=c62c7f5a1a3a3f9cebe7c1382077ad2dbf3727e6" cluster_name = var.cluster_name api_servers = [var.k8s_domain_name] diff --git a/bare-metal/fedora-coreos/kubernetes/bootstrap.tf b/bare-metal/fedora-coreos/kubernetes/bootstrap.tf index daa929a7c..e3851ddf6 100644 --- a/bare-metal/fedora-coreos/kubernetes/bootstrap.tf +++ b/bare-metal/fedora-coreos/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=14d0b2087962a0f2557c184f3f523548ce19bbdc" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=c62c7f5a1a3a3f9cebe7c1382077ad2dbf3727e6" cluster_name = var.cluster_name api_servers = [var.k8s_domain_name] diff --git a/digital-ocean/container-linux/kubernetes/bootstrap.tf b/digital-ocean/container-linux/kubernetes/bootstrap.tf index 8fe3c1b49..b256588ab 100644 --- a/digital-ocean/container-linux/kubernetes/bootstrap.tf +++ b/digital-ocean/container-linux/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=14d0b2087962a0f2557c184f3f523548ce19bbdc" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=c62c7f5a1a3a3f9cebe7c1382077ad2dbf3727e6" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/digital-ocean/fedora-coreos/kubernetes/bootstrap.tf b/digital-ocean/fedora-coreos/kubernetes/bootstrap.tf index 2f95cc53f..002360c3a 100644 --- a/digital-ocean/fedora-coreos/kubernetes/bootstrap.tf +++ b/digital-ocean/fedora-coreos/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=14d0b2087962a0f2557c184f3f523548ce19bbdc" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=c62c7f5a1a3a3f9cebe7c1382077ad2dbf3727e6" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/google-cloud/container-linux/kubernetes/bootstrap.tf b/google-cloud/container-linux/kubernetes/bootstrap.tf index a8729f733..f95498fb1 100644 --- a/google-cloud/container-linux/kubernetes/bootstrap.tf +++ b/google-cloud/container-linux/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=14d0b2087962a0f2557c184f3f523548ce19bbdc" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=c62c7f5a1a3a3f9cebe7c1382077ad2dbf3727e6" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/google-cloud/fedora-coreos/kubernetes/bootstrap.tf b/google-cloud/fedora-coreos/kubernetes/bootstrap.tf index 168e55564..ef6937a21 100644 --- a/google-cloud/fedora-coreos/kubernetes/bootstrap.tf +++ b/google-cloud/fedora-coreos/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=14d0b2087962a0f2557c184f3f523548ce19bbdc" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=c62c7f5a1a3a3f9cebe7c1382077ad2dbf3727e6" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] From fd044ee1174b3e9a79582381ac59660b98825628 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Sat, 25 Apr 2020 16:50:51 -0700 Subject: [PATCH 401/523] Enable Kubelet TLS bootstrap and NodeRestriction * Enable bootstrap token authentication on kube-apiserver * Generate the bootstrap.kubernetes.io/token Secret that may be used as a bootstrap token * Generate a bootstrap kubeconfig (with a bootstrap token) to be securely distributed to nodes. Each Kubelet will use the bootstrap kubeconfig to authenticate to kube-apiserver as `system:bootstrappers` and send a node-unique CSR for kube-controller-manager to automatically approve to issue a Kubelet certificate and kubeconfig (expires in 72 hours) * Add ClusterRoleBinding for bootstrap token subjects (`system:bootstrappers`) to have the `system:node-bootstrapper` ClusterRole * Add ClusterRoleBinding for bootstrap token subjects (`system:bootstrappers`) to have the csr nodeclient ClusterRole * Add ClusterRoleBinding for bootstrap token subjects (`system:bootstrappers`) to have the csr selfnodeclient ClusterRole * Enable NodeRestriction admission controller to limit the scope of Node or Pod objects a Kubelet can modify to those of the node itself * Ability for a Kubelet to delete its Node object is retained as preemptible nodes or those in auto-scaling instance groups need to be able to remove themselves on shutdown. This need continues to have precedence over any risk of a node deleting itself maliciously Security notes: 1. Issued Kubelet certificates authenticate as user `system:node:NAME` and group `system:nodes` and are limited in their authorization to perform API operations by Node authorization and NodeRestriction admission. Previously, a Kubelet's authorization was broader. This is the primary security motivation. 2. The bootstrap kubeconfig credential has the same sensitivity as the previous generated TLS client-certificate kubeconfig. It must be distributed securely to nodes. Its compromise still allows an attacker to obtain a Kubelet kubeconfig 3. Bootstrapping Kubelet kubeconfig's with a limited lifetime offers a slight security improvement. * An attacker who obtains the kubeconfig can likely obtain the bootstrap kubeconfig as well, to obtain the ability to renew their access * A compromised bootstrap kubeconfig could plausibly be handled by replacing the bootstrap token Secret, distributing the token to new nodes, and expiration. Whereas a compromised TLS-client certificate kubeconfig can't be revoked (no CRL). However, replacing a bootstrap token can be impractical in real cluster environments, so the limited lifetime is mostly a theoretical benefit. * Cluster CSR objects are visible via kubectl which is nice 4. Bootstrapping node-unique Kubelet kubeconfigs means Kubelet clients have more identity information, which can improve the utility of audits and future features Rel: https://kubernetes.io/docs/reference/command-line-tools-reference/kubelet-tls-bootstrapping/ Rel: https://github.com/poseidon/terraform-render-bootstrap/pull/185 --- CHANGES.md | 3 +++ addons/prometheus/rules.yaml | 8 ++++---- aws/container-linux/kubernetes/bootstrap.tf | 2 +- aws/container-linux/kubernetes/cl/controller.yaml | 6 ++++-- aws/container-linux/kubernetes/workers/cl/worker.yaml | 6 ++++-- aws/fedora-coreos/kubernetes/bootstrap.tf | 2 +- aws/fedora-coreos/kubernetes/fcc/controller.yaml | 6 ++++-- aws/fedora-coreos/kubernetes/workers/fcc/worker.yaml | 6 ++++-- azure/container-linux/kubernetes/bootstrap.tf | 2 +- azure/container-linux/kubernetes/cl/controller.yaml | 6 ++++-- azure/container-linux/kubernetes/workers/cl/worker.yaml | 6 ++++-- azure/fedora-coreos/kubernetes/bootstrap.tf | 2 +- azure/fedora-coreos/kubernetes/fcc/controller.yaml | 6 ++++-- azure/fedora-coreos/kubernetes/workers/fcc/worker.yaml | 6 ++++-- bare-metal/container-linux/kubernetes/bootstrap.tf | 2 +- bare-metal/container-linux/kubernetes/cl/controller.yaml | 6 ++++-- bare-metal/container-linux/kubernetes/cl/worker.yaml | 6 ++++-- bare-metal/fedora-coreos/kubernetes/bootstrap.tf | 2 +- bare-metal/fedora-coreos/kubernetes/fcc/controller.yaml | 6 ++++-- bare-metal/fedora-coreos/kubernetes/fcc/worker.yaml | 6 ++++-- digital-ocean/container-linux/kubernetes/bootstrap.tf | 2 +- .../container-linux/kubernetes/cl/controller.yaml | 6 ++++-- digital-ocean/container-linux/kubernetes/cl/worker.yaml | 6 ++++-- digital-ocean/fedora-coreos/kubernetes/bootstrap.tf | 2 +- .../fedora-coreos/kubernetes/fcc/controller.yaml | 6 ++++-- digital-ocean/fedora-coreos/kubernetes/fcc/worker.yaml | 6 ++++-- docs/topics/security.md | 6 ++++-- google-cloud/container-linux/kubernetes/bootstrap.tf | 2 +- .../container-linux/kubernetes/cl/controller.yaml | 6 ++++-- .../container-linux/kubernetes/workers/cl/worker.yaml | 6 ++++-- google-cloud/fedora-coreos/kubernetes/bootstrap.tf | 2 +- google-cloud/fedora-coreos/kubernetes/fcc/controller.yaml | 6 ++++-- .../fedora-coreos/kubernetes/workers/fcc/worker.yaml | 6 ++++-- 33 files changed, 101 insertions(+), 56 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 6a28683d6..96ef54535 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -4,6 +4,9 @@ Notable changes between versions. ## Latest +* Use Kubelet [TLS bootstrap](https://kubernetes.io/docs/reference/command-line-tools-reference/kubelet-tls-bootstrapping/) with bootstrap token authentication ([#713](https://github.com/poseidon/typhoon/pull/713)) + * Enable Node [Authorization](https://kubernetes.io/docs/reference/access-authn-authz/node/) and [NodeRestriction](https://kubernetes.io/docs/reference/access-authn-authz/admission-controllers/#noderestriction) to reduce authorization scope + * Renew Kubelet certificates every 72 hours * Update Calico from v3.13.1 to [v3.13.3](https://docs.projectcalico.org/v3.13/release-notes/) #### Addons diff --git a/addons/prometheus/rules.yaml b/addons/prometheus/rules.yaml index 69026c1ef..359cad7e6 100644 --- a/addons/prometheus/rules.yaml +++ b/addons/prometheus/rules.yaml @@ -882,10 +882,10 @@ data: { "alert": "KubeClientCertificateExpiration", "annotations": { - "message": "A client certificate used to authenticate to the apiserver is expiring in less than 7.0 days.", + "message": "A client certificate used to authenticate to the apiserver is expiring in less than 1.0 hours.", "runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubeclientcertificateexpiration" }, - "expr": "apiserver_client_certificate_expiration_seconds_count{job=\"apiserver\"} > 0 and on(job) histogram_quantile(0.01, sum by (job, le) (rate(apiserver_client_certificate_expiration_seconds_bucket{job=\"apiserver\"}[5m]))) < 604800\n", + "expr": "apiserver_client_certificate_expiration_seconds_count{job=\"apiserver\"} > 0 and on(job) histogram_quantile(0.01, sum by (job, le) (rate(apiserver_client_certificate_expiration_seconds_bucket{job=\"apiserver\"}[5m]))) < 3600\n", "labels": { "severity": "warning" } @@ -893,10 +893,10 @@ data: { "alert": "KubeClientCertificateExpiration", "annotations": { - "message": "A client certificate used to authenticate to the apiserver is expiring in less than 24.0 hours.", + "message": "A client certificate used to authenticate to the apiserver is expiring in less than 0.1 hours.", "runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubeclientcertificateexpiration" }, - "expr": "apiserver_client_certificate_expiration_seconds_count{job=\"apiserver\"} > 0 and on(job) histogram_quantile(0.01, sum by (job, le) (rate(apiserver_client_certificate_expiration_seconds_bucket{job=\"apiserver\"}[5m]))) < 86400\n", + "expr": "apiserver_client_certificate_expiration_seconds_count{job=\"apiserver\"} > 0 and on(job) histogram_quantile(0.01, sum by (job, le) (rate(apiserver_client_certificate_expiration_seconds_bucket{job=\"apiserver\"}[5m]))) < 300\n", "labels": { "severity": "critical" } diff --git a/aws/container-linux/kubernetes/bootstrap.tf b/aws/container-linux/kubernetes/bootstrap.tf index 860821aee..9a371c89d 100644 --- a/aws/container-linux/kubernetes/bootstrap.tf +++ b/aws/container-linux/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=c62c7f5a1a3a3f9cebe7c1382077ad2dbf3727e6" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=924beb4b0cb3ca076c29c85983070d0f66dddc5c" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/aws/container-linux/kubernetes/cl/controller.yaml b/aws/container-linux/kubernetes/cl/controller.yaml index 9beb4a8b9..2506dd466 100644 --- a/aws/container-linux/kubernetes/cl/controller.yaml +++ b/aws/container-linux/kubernetes/cl/controller.yaml @@ -49,7 +49,7 @@ systemd: enable: true contents: | [Unit] - Description=Kubelet via Hyperkube + Description=Kubelet Wants=rpc-statd.service [Service] Environment=KUBELET_CGROUP_DRIVER=${cgroup_driver} @@ -95,6 +95,7 @@ systemd: --anonymous-auth=false \ --authentication-token-webhook \ --authorization-mode=Webhook \ + --bootstrap-kubeconfig=/etc/kubernetes/kubeconfig \ --cgroup-driver=$${KUBELET_CGROUP_DRIVER} \ --client-ca-file=/etc/kubernetes/ca.crt \ --cluster_dns=${cluster_dns_service_ip} \ @@ -102,7 +103,7 @@ systemd: --cni-conf-dir=/etc/kubernetes/cni/net.d \ --exit-on-lock-contention \ --healthz-port=0 \ - --kubeconfig=/etc/kubernetes/kubeconfig \ + --kubeconfig=/var/lib/kubelet/kubeconfig \ --lock-file=/var/run/lock/kubelet.lock \ --network-plugin=cni \ --node-labels=node.kubernetes.io/master \ @@ -110,6 +111,7 @@ systemd: --pod-manifest-path=/etc/kubernetes/manifests \ --read-only-port=0 \ --register-with-taints=node-role.kubernetes.io/master=:NoSchedule \ + --rotate-certificates \ --volume-plugin-dir=/var/lib/kubelet/volumeplugins ExecStop=-/usr/bin/rkt stop --uuid-file=/var/cache/kubelet-pod.uuid Restart=always diff --git a/aws/container-linux/kubernetes/workers/cl/worker.yaml b/aws/container-linux/kubernetes/workers/cl/worker.yaml index 63a1e3b6d..0924c634f 100644 --- a/aws/container-linux/kubernetes/workers/cl/worker.yaml +++ b/aws/container-linux/kubernetes/workers/cl/worker.yaml @@ -22,7 +22,7 @@ systemd: enable: true contents: | [Unit] - Description=Kubelet via Hyperkube + Description=Kubelet Wants=rpc-statd.service [Service] Environment=KUBELET_CGROUP_DRIVER=${cgroup_driver} @@ -68,6 +68,7 @@ systemd: --anonymous-auth=false \ --authentication-token-webhook \ --authorization-mode=Webhook \ + --bootstrap-kubeconfig=/etc/kubernetes/kubeconfig \ --cgroup-driver=$${KUBELET_CGROUP_DRIVER} \ --client-ca-file=/etc/kubernetes/ca.crt \ --cluster_dns=${cluster_dns_service_ip} \ @@ -75,7 +76,7 @@ systemd: --cni-conf-dir=/etc/kubernetes/cni/net.d \ --exit-on-lock-contention \ --healthz-port=0 \ - --kubeconfig=/etc/kubernetes/kubeconfig \ + --kubeconfig=/var/lib/kubelet/kubeconfig \ --lock-file=/var/run/lock/kubelet.lock \ --network-plugin=cni \ --node-labels=node.kubernetes.io/node \ @@ -84,6 +85,7 @@ systemd: %{~ endfor ~} --pod-manifest-path=/etc/kubernetes/manifests \ --read-only-port=0 \ + --rotate-certificates \ --volume-plugin-dir=/var/lib/kubelet/volumeplugins ExecStop=-/usr/bin/rkt stop --uuid-file=/var/cache/kubelet-pod.uuid Restart=always diff --git a/aws/fedora-coreos/kubernetes/bootstrap.tf b/aws/fedora-coreos/kubernetes/bootstrap.tf index 1991cb13c..64ffb08d0 100644 --- a/aws/fedora-coreos/kubernetes/bootstrap.tf +++ b/aws/fedora-coreos/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=c62c7f5a1a3a3f9cebe7c1382077ad2dbf3727e6" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=924beb4b0cb3ca076c29c85983070d0f66dddc5c" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/aws/fedora-coreos/kubernetes/fcc/controller.yaml b/aws/fedora-coreos/kubernetes/fcc/controller.yaml index 727960d71..c7474e10d 100644 --- a/aws/fedora-coreos/kubernetes/fcc/controller.yaml +++ b/aws/fedora-coreos/kubernetes/fcc/controller.yaml @@ -51,7 +51,7 @@ systemd: enabled: true contents: | [Unit] - Description=Kubelet via Hyperkube (System Container) + Description=Kubelet (System Container) Wants=rpc-statd.service [Service] ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d @@ -83,6 +83,7 @@ systemd: --anonymous-auth=false \ --authentication-token-webhook \ --authorization-mode=Webhook \ + --bootstrap-kubeconfig=/etc/kubernetes/kubeconfig \ --cgroup-driver=systemd \ --cgroups-per-qos=true \ --enforce-node-allocatable=pods \ @@ -92,7 +93,7 @@ systemd: --cni-conf-dir=/etc/kubernetes/cni/net.d \ --exit-on-lock-contention \ --healthz-port=0 \ - --kubeconfig=/etc/kubernetes/kubeconfig \ + --kubeconfig=/var/lib/kubelet/kubeconfig \ --lock-file=/var/run/lock/kubelet.lock \ --network-plugin=cni \ --node-labels=node.kubernetes.io/master \ @@ -100,6 +101,7 @@ systemd: --pod-manifest-path=/etc/kubernetes/manifests \ --read-only-port=0 \ --register-with-taints=node-role.kubernetes.io/master=:NoSchedule \ + --rotate-certificates \ --volume-plugin-dir=/var/lib/kubelet/volumeplugins ExecStop=-/usr/bin/podman stop kubelet Delegate=yes diff --git a/aws/fedora-coreos/kubernetes/workers/fcc/worker.yaml b/aws/fedora-coreos/kubernetes/workers/fcc/worker.yaml index 0501b8de9..865fb9d8e 100644 --- a/aws/fedora-coreos/kubernetes/workers/fcc/worker.yaml +++ b/aws/fedora-coreos/kubernetes/workers/fcc/worker.yaml @@ -21,7 +21,7 @@ systemd: enabled: true contents: | [Unit] - Description=Kubelet via Hyperkube (System Container) + Description=Kubelet (System Container) Wants=rpc-statd.service [Service] ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d @@ -53,6 +53,7 @@ systemd: --anonymous-auth=false \ --authentication-token-webhook \ --authorization-mode=Webhook \ + --bootstrap-kubeconfig=/etc/kubernetes/kubeconfig \ --cgroup-driver=systemd \ --cgroups-per-qos=true \ --enforce-node-allocatable=pods \ @@ -62,7 +63,7 @@ systemd: --cni-conf-dir=/etc/kubernetes/cni/net.d \ --exit-on-lock-contention \ --healthz-port=0 \ - --kubeconfig=/etc/kubernetes/kubeconfig \ + --kubeconfig=/var/lib/kubelet/kubeconfig \ --lock-file=/var/run/lock/kubelet.lock \ --network-plugin=cni \ --node-labels=node.kubernetes.io/node \ @@ -71,6 +72,7 @@ systemd: %{~ endfor ~} --pod-manifest-path=/etc/kubernetes/manifests \ --read-only-port=0 \ + --rotate-certificates \ --volume-plugin-dir=/var/lib/kubelet/volumeplugins ExecStop=-/usr/bin/podman stop kubelet Delegate=yes diff --git a/azure/container-linux/kubernetes/bootstrap.tf b/azure/container-linux/kubernetes/bootstrap.tf index 4495381db..1ab4a4768 100644 --- a/azure/container-linux/kubernetes/bootstrap.tf +++ b/azure/container-linux/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=c62c7f5a1a3a3f9cebe7c1382077ad2dbf3727e6" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=924beb4b0cb3ca076c29c85983070d0f66dddc5c" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/azure/container-linux/kubernetes/cl/controller.yaml b/azure/container-linux/kubernetes/cl/controller.yaml index 293e3b457..aa0e6941e 100644 --- a/azure/container-linux/kubernetes/cl/controller.yaml +++ b/azure/container-linux/kubernetes/cl/controller.yaml @@ -49,7 +49,7 @@ systemd: enable: true contents: | [Unit] - Description=Kubelet via Hyperkube + Description=Kubelet Wants=rpc-statd.service [Service] ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d @@ -94,13 +94,14 @@ systemd: --anonymous-auth=false \ --authentication-token-webhook \ --authorization-mode=Webhook \ + --bootstrap-kubeconfig=/etc/kubernetes/kubeconfig \ --client-ca-file=/etc/kubernetes/ca.crt \ --cluster_dns=${cluster_dns_service_ip} \ --cluster_domain=${cluster_domain_suffix} \ --cni-conf-dir=/etc/kubernetes/cni/net.d \ --exit-on-lock-contention \ --healthz-port=0 \ - --kubeconfig=/etc/kubernetes/kubeconfig \ + --kubeconfig=/var/lib/kubelet/kubeconfig \ --lock-file=/var/run/lock/kubelet.lock \ --network-plugin=cni \ --node-labels=node.kubernetes.io/master \ @@ -108,6 +109,7 @@ systemd: --pod-manifest-path=/etc/kubernetes/manifests \ --read-only-port=0 \ --register-with-taints=node-role.kubernetes.io/master=:NoSchedule \ + --rotate-certificates \ --volume-plugin-dir=/var/lib/kubelet/volumeplugins ExecStop=-/usr/bin/rkt stop --uuid-file=/var/cache/kubelet-pod.uuid Restart=always diff --git a/azure/container-linux/kubernetes/workers/cl/worker.yaml b/azure/container-linux/kubernetes/workers/cl/worker.yaml index e070c975c..2e1d74b2c 100644 --- a/azure/container-linux/kubernetes/workers/cl/worker.yaml +++ b/azure/container-linux/kubernetes/workers/cl/worker.yaml @@ -22,7 +22,7 @@ systemd: enable: true contents: | [Unit] - Description=Kubelet via Hyperkube + Description=Kubelet Wants=rpc-statd.service [Service] ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d @@ -67,13 +67,14 @@ systemd: --anonymous-auth=false \ --authentication-token-webhook \ --authorization-mode=Webhook \ + --bootstrap-kubeconfig=/etc/kubernetes/kubeconfig \ --client-ca-file=/etc/kubernetes/ca.crt \ --cluster_dns=${cluster_dns_service_ip} \ --cluster_domain=${cluster_domain_suffix} \ --cni-conf-dir=/etc/kubernetes/cni/net.d \ --exit-on-lock-contention \ --healthz-port=0 \ - --kubeconfig=/etc/kubernetes/kubeconfig \ + --kubeconfig=/var/lib/kubelet/kubeconfig \ --lock-file=/var/run/lock/kubelet.lock \ --network-plugin=cni \ --node-labels=node.kubernetes.io/node \ @@ -82,6 +83,7 @@ systemd: %{~ endfor ~} --pod-manifest-path=/etc/kubernetes/manifests \ --read-only-port=0 \ + --rotate-certificates \ --volume-plugin-dir=/var/lib/kubelet/volumeplugins ExecStop=-/usr/bin/rkt stop --uuid-file=/var/cache/kubelet-pod.uuid Restart=always diff --git a/azure/fedora-coreos/kubernetes/bootstrap.tf b/azure/fedora-coreos/kubernetes/bootstrap.tf index 6b4c90ff2..01c74b1f0 100644 --- a/azure/fedora-coreos/kubernetes/bootstrap.tf +++ b/azure/fedora-coreos/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=c62c7f5a1a3a3f9cebe7c1382077ad2dbf3727e6" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=924beb4b0cb3ca076c29c85983070d0f66dddc5c" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/azure/fedora-coreos/kubernetes/fcc/controller.yaml b/azure/fedora-coreos/kubernetes/fcc/controller.yaml index c74ecf585..035667c04 100644 --- a/azure/fedora-coreos/kubernetes/fcc/controller.yaml +++ b/azure/fedora-coreos/kubernetes/fcc/controller.yaml @@ -51,7 +51,7 @@ systemd: enabled: true contents: | [Unit] - Description=Kubelet via Hyperkube (System Container) + Description=Kubelet (System Container) Wants=rpc-statd.service [Service] ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d @@ -83,6 +83,7 @@ systemd: --anonymous-auth=false \ --authentication-token-webhook \ --authorization-mode=Webhook \ + --bootstrap-kubeconfig=/etc/kubernetes/kubeconfig \ --cgroup-driver=systemd \ --cgroups-per-qos=true \ --enforce-node-allocatable=pods \ @@ -92,7 +93,7 @@ systemd: --cni-conf-dir=/etc/kubernetes/cni/net.d \ --exit-on-lock-contention \ --healthz-port=0 \ - --kubeconfig=/etc/kubernetes/kubeconfig \ + --kubeconfig=/var/lib/kubelet/kubeconfig \ --lock-file=/var/run/lock/kubelet.lock \ --network-plugin=cni \ --node-labels=node.kubernetes.io/master \ @@ -100,6 +101,7 @@ systemd: --pod-manifest-path=/etc/kubernetes/manifests \ --read-only-port=0 \ --register-with-taints=node-role.kubernetes.io/master=:NoSchedule \ + --rotate-certificates \ --volume-plugin-dir=/var/lib/kubelet/volumeplugins ExecStop=-/usr/bin/podman stop kubelet Delegate=yes diff --git a/azure/fedora-coreos/kubernetes/workers/fcc/worker.yaml b/azure/fedora-coreos/kubernetes/workers/fcc/worker.yaml index 664f0c6a2..056cbafdb 100644 --- a/azure/fedora-coreos/kubernetes/workers/fcc/worker.yaml +++ b/azure/fedora-coreos/kubernetes/workers/fcc/worker.yaml @@ -21,7 +21,7 @@ systemd: enabled: true contents: | [Unit] - Description=Kubelet via Hyperkube (System Container) + Description=Kubelet (System Container) Wants=rpc-statd.service [Service] ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d @@ -53,6 +53,7 @@ systemd: --anonymous-auth=false \ --authentication-token-webhook \ --authorization-mode=Webhook \ + --bootstrap-kubeconfig=/etc/kubernetes/kubeconfig \ --cgroup-driver=systemd \ --cgroups-per-qos=true \ --enforce-node-allocatable=pods \ @@ -62,7 +63,7 @@ systemd: --cni-conf-dir=/etc/kubernetes/cni/net.d \ --exit-on-lock-contention \ --healthz-port=0 \ - --kubeconfig=/etc/kubernetes/kubeconfig \ + --kubeconfig=/var/lib/kubelet/kubeconfig \ --lock-file=/var/run/lock/kubelet.lock \ --network-plugin=cni \ --node-labels=node.kubernetes.io/node \ @@ -71,6 +72,7 @@ systemd: %{~ endfor ~} --pod-manifest-path=/etc/kubernetes/manifests \ --read-only-port=0 \ + --rotate-certificates \ --volume-plugin-dir=/var/lib/kubelet/volumeplugins ExecStop=-/usr/bin/podman stop kubelet Delegate=yes diff --git a/bare-metal/container-linux/kubernetes/bootstrap.tf b/bare-metal/container-linux/kubernetes/bootstrap.tf index 45179f959..6cae75043 100644 --- a/bare-metal/container-linux/kubernetes/bootstrap.tf +++ b/bare-metal/container-linux/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=c62c7f5a1a3a3f9cebe7c1382077ad2dbf3727e6" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=924beb4b0cb3ca076c29c85983070d0f66dddc5c" cluster_name = var.cluster_name api_servers = [var.k8s_domain_name] diff --git a/bare-metal/container-linux/kubernetes/cl/controller.yaml b/bare-metal/container-linux/kubernetes/cl/controller.yaml index 9cde76e89..9f5c5f562 100644 --- a/bare-metal/container-linux/kubernetes/cl/controller.yaml +++ b/bare-metal/container-linux/kubernetes/cl/controller.yaml @@ -57,7 +57,7 @@ systemd: - name: kubelet.service contents: | [Unit] - Description=Kubelet via Hyperkube + Description=Kubelet Wants=rpc-statd.service [Service] Environment=KUBELET_CGROUP_DRIVER=${cgroup_driver} @@ -107,6 +107,7 @@ systemd: --anonymous-auth=false \ --authentication-token-webhook \ --authorization-mode=Webhook \ + --bootstrap-kubeconfig=/etc/kubernetes/kubeconfig \ --cgroup-driver=$${KUBELET_CGROUP_DRIVER} \ --client-ca-file=/etc/kubernetes/ca.crt \ --cluster_dns=${cluster_dns_service_ip} \ @@ -115,7 +116,7 @@ systemd: --exit-on-lock-contention \ --healthz-port=0 \ --hostname-override=${domain_name} \ - --kubeconfig=/etc/kubernetes/kubeconfig \ + --kubeconfig=/var/lib/kubelet/kubeconfig \ --lock-file=/var/run/lock/kubelet.lock \ --network-plugin=cni \ --node-labels=node.kubernetes.io/master \ @@ -123,6 +124,7 @@ systemd: --pod-manifest-path=/etc/kubernetes/manifests \ --read-only-port=0 \ --register-with-taints=node-role.kubernetes.io/master=:NoSchedule \ + --rotate-certificates \ --volume-plugin-dir=/var/lib/kubelet/volumeplugins ExecStop=-/usr/bin/rkt stop --uuid-file=/var/cache/kubelet-pod.uuid Restart=always diff --git a/bare-metal/container-linux/kubernetes/cl/worker.yaml b/bare-metal/container-linux/kubernetes/cl/worker.yaml index 164323cf5..dde9d7586 100644 --- a/bare-metal/container-linux/kubernetes/cl/worker.yaml +++ b/bare-metal/container-linux/kubernetes/cl/worker.yaml @@ -30,7 +30,7 @@ systemd: - name: kubelet.service contents: | [Unit] - Description=Kubelet via Hyperkube + Description=Kubelet Wants=rpc-statd.service [Service] Environment=KUBELET_CGROUP_DRIVER=${cgroup_driver} @@ -80,6 +80,7 @@ systemd: --anonymous-auth=false \ --authentication-token-webhook \ --authorization-mode=Webhook \ + --bootstrap-kubeconfig=/etc/kubernetes/kubeconfig \ --cgroup-driver=$${KUBELET_CGROUP_DRIVER} \ --client-ca-file=/etc/kubernetes/ca.crt \ --cluster_dns=${cluster_dns_service_ip} \ @@ -88,7 +89,7 @@ systemd: --exit-on-lock-contention \ --healthz-port=0 \ --hostname-override=${domain_name} \ - --kubeconfig=/etc/kubernetes/kubeconfig \ + --kubeconfig=/var/lib/kubelet/kubeconfig \ --lock-file=/var/run/lock/kubelet.lock \ --network-plugin=cni \ --node-labels=node.kubernetes.io/node \ @@ -100,6 +101,7 @@ systemd: %{~ endfor ~} --pod-manifest-path=/etc/kubernetes/manifests \ --read-only-port=0 \ + --rotate-certificates \ --volume-plugin-dir=/var/lib/kubelet/volumeplugins ExecStop=-/usr/bin/rkt stop --uuid-file=/var/cache/kubelet-pod.uuid Restart=always diff --git a/bare-metal/fedora-coreos/kubernetes/bootstrap.tf b/bare-metal/fedora-coreos/kubernetes/bootstrap.tf index e3851ddf6..587d4f1ec 100644 --- a/bare-metal/fedora-coreos/kubernetes/bootstrap.tf +++ b/bare-metal/fedora-coreos/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=c62c7f5a1a3a3f9cebe7c1382077ad2dbf3727e6" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=924beb4b0cb3ca076c29c85983070d0f66dddc5c" cluster_name = var.cluster_name api_servers = [var.k8s_domain_name] diff --git a/bare-metal/fedora-coreos/kubernetes/fcc/controller.yaml b/bare-metal/fedora-coreos/kubernetes/fcc/controller.yaml index 630213168..14cbfdb8b 100644 --- a/bare-metal/fedora-coreos/kubernetes/fcc/controller.yaml +++ b/bare-metal/fedora-coreos/kubernetes/fcc/controller.yaml @@ -50,7 +50,7 @@ systemd: - name: kubelet.service contents: | [Unit] - Description=Kubelet via Hyperkube (System Container) + Description=Kubelet (System Container) Wants=rpc-statd.service [Service] ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d @@ -84,6 +84,7 @@ systemd: --anonymous-auth=false \ --authentication-token-webhook \ --authorization-mode=Webhook \ + --bootstrap-kubeconfig=/etc/kubernetes/kubeconfig \ --cgroup-driver=systemd \ --cgroups-per-qos=true \ --enforce-node-allocatable=pods \ @@ -94,7 +95,7 @@ systemd: --exit-on-lock-contention \ --healthz-port=0 \ --hostname-override=${domain_name} \ - --kubeconfig=/etc/kubernetes/kubeconfig \ + --kubeconfig=/var/lib/kubelet/kubeconfig \ --lock-file=/var/run/lock/kubelet.lock \ --network-plugin=cni \ --node-labels=node.kubernetes.io/master \ @@ -102,6 +103,7 @@ systemd: --pod-manifest-path=/etc/kubernetes/manifests \ --read-only-port=0 \ --register-with-taints=node-role.kubernetes.io/master=:NoSchedule \ + --rotate-certificates \ --volume-plugin-dir=/var/lib/kubelet/volumeplugins ExecStop=-/usr/bin/podman stop kubelet Delegate=yes diff --git a/bare-metal/fedora-coreos/kubernetes/fcc/worker.yaml b/bare-metal/fedora-coreos/kubernetes/fcc/worker.yaml index 5133d36e6..f61fe8d69 100644 --- a/bare-metal/fedora-coreos/kubernetes/fcc/worker.yaml +++ b/bare-metal/fedora-coreos/kubernetes/fcc/worker.yaml @@ -20,7 +20,7 @@ systemd: - name: kubelet.service contents: | [Unit] - Description=Kubelet via Hyperkube (System Container) + Description=Kubelet (System Container) Wants=rpc-statd.service [Service] ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d @@ -54,6 +54,7 @@ systemd: --anonymous-auth=false \ --authentication-token-webhook \ --authorization-mode=Webhook \ + --bootstrap-kubeconfig=/etc/kubernetes/kubeconfig \ --cgroup-driver=systemd \ --cgroups-per-qos=true \ --enforce-node-allocatable=pods \ @@ -64,7 +65,7 @@ systemd: --exit-on-lock-contention \ --healthz-port=0 \ --hostname-override=${domain_name} \ - --kubeconfig=/etc/kubernetes/kubeconfig \ + --kubeconfig=/var/lib/kubelet/kubeconfig \ --lock-file=/var/run/lock/kubelet.lock \ --network-plugin=cni \ --node-labels=node.kubernetes.io/node \ @@ -76,6 +77,7 @@ systemd: %{~ endfor ~} --pod-manifest-path=/etc/kubernetes/manifests \ --read-only-port=0 \ + --rotate-certificates \ --volume-plugin-dir=/var/lib/kubelet/volumeplugins ExecStop=-/usr/bin/podman stop kubelet Delegate=yes diff --git a/digital-ocean/container-linux/kubernetes/bootstrap.tf b/digital-ocean/container-linux/kubernetes/bootstrap.tf index b256588ab..af5e3e469 100644 --- a/digital-ocean/container-linux/kubernetes/bootstrap.tf +++ b/digital-ocean/container-linux/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=c62c7f5a1a3a3f9cebe7c1382077ad2dbf3727e6" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=924beb4b0cb3ca076c29c85983070d0f66dddc5c" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/digital-ocean/container-linux/kubernetes/cl/controller.yaml b/digital-ocean/container-linux/kubernetes/cl/controller.yaml index c7a297518..08f653723 100644 --- a/digital-ocean/container-linux/kubernetes/cl/controller.yaml +++ b/digital-ocean/container-linux/kubernetes/cl/controller.yaml @@ -57,7 +57,7 @@ systemd: - name: kubelet.service contents: | [Unit] - Description=Kubelet via Hyperkube + Description=Kubelet Requires=coreos-metadata.service After=coreos-metadata.service Wants=rpc-statd.service @@ -105,6 +105,7 @@ systemd: --anonymous-auth=false \ --authentication-token-webhook \ --authorization-mode=Webhook \ + --bootstrap-kubeconfig=/etc/kubernetes/kubeconfig \ --client-ca-file=/etc/kubernetes/ca.crt \ --cluster_dns=${cluster_dns_service_ip} \ --cluster_domain=${cluster_domain_suffix} \ @@ -112,7 +113,7 @@ systemd: --exit-on-lock-contention \ --healthz-port=0 \ --hostname-override=$${COREOS_DIGITALOCEAN_IPV4_PRIVATE_0} \ - --kubeconfig=/etc/kubernetes/kubeconfig \ + --kubeconfig=/var/lib/kubelet/kubeconfig \ --lock-file=/var/run/lock/kubelet.lock \ --network-plugin=cni \ --node-labels=node.kubernetes.io/master \ @@ -120,6 +121,7 @@ systemd: --pod-manifest-path=/etc/kubernetes/manifests \ --read-only-port=0 \ --register-with-taints=node-role.kubernetes.io/master=:NoSchedule \ + --rotate-certificates \ --volume-plugin-dir=/var/lib/kubelet/volumeplugins ExecStop=-/usr/bin/rkt stop --uuid-file=/var/cache/kubelet-pod.uuid Restart=always diff --git a/digital-ocean/container-linux/kubernetes/cl/worker.yaml b/digital-ocean/container-linux/kubernetes/cl/worker.yaml index 2248e0770..c2f9419df 100644 --- a/digital-ocean/container-linux/kubernetes/cl/worker.yaml +++ b/digital-ocean/container-linux/kubernetes/cl/worker.yaml @@ -30,7 +30,7 @@ systemd: - name: kubelet.service contents: | [Unit] - Description=Kubelet via Hyperkube + Description=Kubelet Requires=coreos-metadata.service After=coreos-metadata.service Wants=rpc-statd.service @@ -78,6 +78,7 @@ systemd: --anonymous-auth=false \ --authentication-token-webhook \ --authorization-mode=Webhook \ + --bootstrap-kubeconfig=/etc/kubernetes/kubeconfig \ --client-ca-file=/etc/kubernetes/ca.crt \ --cluster_dns=${cluster_dns_service_ip} \ --cluster_domain=${cluster_domain_suffix} \ @@ -85,12 +86,13 @@ systemd: --exit-on-lock-contention \ --healthz-port=0 \ --hostname-override=$${COREOS_DIGITALOCEAN_IPV4_PRIVATE_0} \ - --kubeconfig=/etc/kubernetes/kubeconfig \ + --kubeconfig=/var/lib/kubelet/kubeconfig \ --lock-file=/var/run/lock/kubelet.lock \ --network-plugin=cni \ --node-labels=node.kubernetes.io/node \ --pod-manifest-path=/etc/kubernetes/manifests \ --read-only-port=0 \ + --rotate-certificates \ --volume-plugin-dir=/var/lib/kubelet/volumeplugins ExecStop=-/usr/bin/rkt stop --uuid-file=/var/cache/kubelet-pod.uuid Restart=always diff --git a/digital-ocean/fedora-coreos/kubernetes/bootstrap.tf b/digital-ocean/fedora-coreos/kubernetes/bootstrap.tf index 002360c3a..3bd95bbe4 100644 --- a/digital-ocean/fedora-coreos/kubernetes/bootstrap.tf +++ b/digital-ocean/fedora-coreos/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=c62c7f5a1a3a3f9cebe7c1382077ad2dbf3727e6" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=924beb4b0cb3ca076c29c85983070d0f66dddc5c" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/digital-ocean/fedora-coreos/kubernetes/fcc/controller.yaml b/digital-ocean/fedora-coreos/kubernetes/fcc/controller.yaml index 999340575..67106af2d 100644 --- a/digital-ocean/fedora-coreos/kubernetes/fcc/controller.yaml +++ b/digital-ocean/fedora-coreos/kubernetes/fcc/controller.yaml @@ -50,7 +50,7 @@ systemd: - name: kubelet.service contents: | [Unit] - Description=Kubelet via Hyperkube (System Container) + Description=Kubelet (System Container) Requires=afterburn.service After=afterburn.service Wants=rpc-statd.service @@ -85,6 +85,7 @@ systemd: --anonymous-auth=false \ --authentication-token-webhook \ --authorization-mode=Webhook \ + --bootstrap-kubeconfig=/etc/kubernetes/kubeconfig \ --cgroup-driver=systemd \ --cgroups-per-qos=true \ --enforce-node-allocatable=pods \ @@ -95,7 +96,7 @@ systemd: --exit-on-lock-contention \ --healthz-port=0 \ --hostname-override=$${AFTERBURN_DIGITALOCEAN_IPV4_PRIVATE_0} \ - --kubeconfig=/etc/kubernetes/kubeconfig \ + --kubeconfig=/var/lib/kubelet/kubeconfig \ --lock-file=/var/run/lock/kubelet.lock \ --network-plugin=cni \ --node-labels=node.kubernetes.io/master \ @@ -103,6 +104,7 @@ systemd: --pod-manifest-path=/etc/kubernetes/manifests \ --read-only-port=0 \ --register-with-taints=node-role.kubernetes.io/master=:NoSchedule \ + --rotate-certificates \ --volume-plugin-dir=/var/lib/kubelet/volumeplugins ExecStop=-/usr/bin/podman stop kubelet Delegate=yes diff --git a/digital-ocean/fedora-coreos/kubernetes/fcc/worker.yaml b/digital-ocean/fedora-coreos/kubernetes/fcc/worker.yaml index 8209e4e0b..a18b5052a 100644 --- a/digital-ocean/fedora-coreos/kubernetes/fcc/worker.yaml +++ b/digital-ocean/fedora-coreos/kubernetes/fcc/worker.yaml @@ -21,7 +21,7 @@ systemd: enabled: true contents: | [Unit] - Description=Kubelet via Hyperkube (System Container) + Description=Kubelet (System Container) Requires=afterburn.service After=afterburn.service Wants=rpc-statd.service @@ -56,6 +56,7 @@ systemd: --anonymous-auth=false \ --authentication-token-webhook \ --authorization-mode=Webhook \ + --bootstrap-kubeconfig=/etc/kubernetes/kubeconfig \ --cgroup-driver=systemd \ --cgroups-per-qos=true \ --enforce-node-allocatable=pods \ @@ -66,12 +67,13 @@ systemd: --exit-on-lock-contention \ --healthz-port=0 \ --hostname-override=$${AFTERBURN_DIGITALOCEAN_IPV4_PRIVATE_0} \ - --kubeconfig=/etc/kubernetes/kubeconfig \ + --kubeconfig=/var/lib/kubelet/kubeconfig \ --lock-file=/var/run/lock/kubelet.lock \ --network-plugin=cni \ --node-labels=node.kubernetes.io/node \ --pod-manifest-path=/etc/kubernetes/manifests \ --read-only-port=0 \ + --rotate-certificates \ --volume-plugin-dir=/var/lib/kubelet/volumeplugins ExecStop=-/usr/bin/podman stop kubelet Delegate=yes diff --git a/docs/topics/security.md b/docs/topics/security.md index fc4cbb420..e8949c83d 100644 --- a/docs/topics/security.md +++ b/docs/topics/security.md @@ -7,8 +7,10 @@ Typhoon aims to be minimal and secure. We're running it ourselves after all. **Kubernetes** * etcd with peer-to-peer and client-auth TLS -* Generated kubelet TLS certificates and `kubeconfig` (365 days) -* [Role-Based Access Control](https://kubernetes.io/docs/admin/authorization/rbac/) is enabled. Apps must define RBAC policies +* Kubelets TLS bootstrap certificates (72 hours) +* Generated TLS certificate (365 days) for admin `kubeconfig` +* [NodeRestriction](https://kubernetes.io/docs/reference/access-authn-authz/node/) is enabled to limit Kubelet authorization +* [Role-Based Access Control](https://kubernetes.io/docs/admin/authorization/rbac/) is enabled. Apps must define RBAC policies for API access * Workloads run on worker nodes only, unless they tolerate the master taint * Kubernetes [Network Policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/) and Calico [NetworkPolicy](https://docs.projectcalico.org/latest/reference/calicoctl/resources/networkpolicy) support [^1] diff --git a/google-cloud/container-linux/kubernetes/bootstrap.tf b/google-cloud/container-linux/kubernetes/bootstrap.tf index f95498fb1..49af2d10d 100644 --- a/google-cloud/container-linux/kubernetes/bootstrap.tf +++ b/google-cloud/container-linux/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=c62c7f5a1a3a3f9cebe7c1382077ad2dbf3727e6" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=924beb4b0cb3ca076c29c85983070d0f66dddc5c" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/google-cloud/container-linux/kubernetes/cl/controller.yaml b/google-cloud/container-linux/kubernetes/cl/controller.yaml index 1c88d409c..30fbee4d9 100644 --- a/google-cloud/container-linux/kubernetes/cl/controller.yaml +++ b/google-cloud/container-linux/kubernetes/cl/controller.yaml @@ -49,7 +49,7 @@ systemd: enable: true contents: | [Unit] - Description=Kubelet via Hyperkube + Description=Kubelet Wants=rpc-statd.service [Service] ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d @@ -94,13 +94,14 @@ systemd: --anonymous-auth=false \ --authentication-token-webhook \ --authorization-mode=Webhook \ + --bootstrap-kubeconfig=/etc/kubernetes/kubeconfig \ --client-ca-file=/etc/kubernetes/ca.crt \ --cluster_dns=${cluster_dns_service_ip} \ --cluster_domain=${cluster_domain_suffix} \ --cni-conf-dir=/etc/kubernetes/cni/net.d \ --exit-on-lock-contention \ --healthz-port=0 \ - --kubeconfig=/etc/kubernetes/kubeconfig \ + --kubeconfig=/var/lib/kubelet/kubeconfig \ --lock-file=/var/run/lock/kubelet.lock \ --network-plugin=cni \ --node-labels=node.kubernetes.io/master \ @@ -108,6 +109,7 @@ systemd: --pod-manifest-path=/etc/kubernetes/manifests \ --register-with-taints=node-role.kubernetes.io/master=:NoSchedule \ --read-only-port=0 \ + --rotate-certificates \ --volume-plugin-dir=/var/lib/kubelet/volumeplugins ExecStop=-/usr/bin/rkt stop --uuid-file=/var/cache/kubelet-pod.uuid Restart=always diff --git a/google-cloud/container-linux/kubernetes/workers/cl/worker.yaml b/google-cloud/container-linux/kubernetes/workers/cl/worker.yaml index 52a0cfae8..0bb704e2c 100644 --- a/google-cloud/container-linux/kubernetes/workers/cl/worker.yaml +++ b/google-cloud/container-linux/kubernetes/workers/cl/worker.yaml @@ -22,7 +22,7 @@ systemd: enable: true contents: | [Unit] - Description=Kubelet via Hyperkube + Description=Kubelet Wants=rpc-statd.service [Service] ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d @@ -67,13 +67,14 @@ systemd: --anonymous-auth=false \ --authentication-token-webhook \ --authorization-mode=Webhook \ + --bootstrap-kubeconfig=/etc/kubernetes/kubeconfig \ --client-ca-file=/etc/kubernetes/ca.crt \ --cluster_dns=${cluster_dns_service_ip} \ --cluster_domain=${cluster_domain_suffix} \ --cni-conf-dir=/etc/kubernetes/cni/net.d \ --exit-on-lock-contention \ --healthz-port=0 \ - --kubeconfig=/etc/kubernetes/kubeconfig \ + --kubeconfig=/var/lib/kubelet/kubeconfig \ --lock-file=/var/run/lock/kubelet.lock \ --network-plugin=cni \ --node-labels=node.kubernetes.io/node \ @@ -82,6 +83,7 @@ systemd: %{~ endfor ~} --pod-manifest-path=/etc/kubernetes/manifests \ --read-only-port=0 \ + --rotate-certificates \ --volume-plugin-dir=/var/lib/kubelet/volumeplugins ExecStop=-/usr/bin/rkt stop --uuid-file=/var/cache/kubelet-pod.uuid Restart=always diff --git a/google-cloud/fedora-coreos/kubernetes/bootstrap.tf b/google-cloud/fedora-coreos/kubernetes/bootstrap.tf index ef6937a21..0b3b41d72 100644 --- a/google-cloud/fedora-coreos/kubernetes/bootstrap.tf +++ b/google-cloud/fedora-coreos/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=c62c7f5a1a3a3f9cebe7c1382077ad2dbf3727e6" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=924beb4b0cb3ca076c29c85983070d0f66dddc5c" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/google-cloud/fedora-coreos/kubernetes/fcc/controller.yaml b/google-cloud/fedora-coreos/kubernetes/fcc/controller.yaml index 727960d71..c7474e10d 100644 --- a/google-cloud/fedora-coreos/kubernetes/fcc/controller.yaml +++ b/google-cloud/fedora-coreos/kubernetes/fcc/controller.yaml @@ -51,7 +51,7 @@ systemd: enabled: true contents: | [Unit] - Description=Kubelet via Hyperkube (System Container) + Description=Kubelet (System Container) Wants=rpc-statd.service [Service] ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d @@ -83,6 +83,7 @@ systemd: --anonymous-auth=false \ --authentication-token-webhook \ --authorization-mode=Webhook \ + --bootstrap-kubeconfig=/etc/kubernetes/kubeconfig \ --cgroup-driver=systemd \ --cgroups-per-qos=true \ --enforce-node-allocatable=pods \ @@ -92,7 +93,7 @@ systemd: --cni-conf-dir=/etc/kubernetes/cni/net.d \ --exit-on-lock-contention \ --healthz-port=0 \ - --kubeconfig=/etc/kubernetes/kubeconfig \ + --kubeconfig=/var/lib/kubelet/kubeconfig \ --lock-file=/var/run/lock/kubelet.lock \ --network-plugin=cni \ --node-labels=node.kubernetes.io/master \ @@ -100,6 +101,7 @@ systemd: --pod-manifest-path=/etc/kubernetes/manifests \ --read-only-port=0 \ --register-with-taints=node-role.kubernetes.io/master=:NoSchedule \ + --rotate-certificates \ --volume-plugin-dir=/var/lib/kubelet/volumeplugins ExecStop=-/usr/bin/podman stop kubelet Delegate=yes diff --git a/google-cloud/fedora-coreos/kubernetes/workers/fcc/worker.yaml b/google-cloud/fedora-coreos/kubernetes/workers/fcc/worker.yaml index 0501b8de9..865fb9d8e 100644 --- a/google-cloud/fedora-coreos/kubernetes/workers/fcc/worker.yaml +++ b/google-cloud/fedora-coreos/kubernetes/workers/fcc/worker.yaml @@ -21,7 +21,7 @@ systemd: enabled: true contents: | [Unit] - Description=Kubelet via Hyperkube (System Container) + Description=Kubelet (System Container) Wants=rpc-statd.service [Service] ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d @@ -53,6 +53,7 @@ systemd: --anonymous-auth=false \ --authentication-token-webhook \ --authorization-mode=Webhook \ + --bootstrap-kubeconfig=/etc/kubernetes/kubeconfig \ --cgroup-driver=systemd \ --cgroups-per-qos=true \ --enforce-node-allocatable=pods \ @@ -62,7 +63,7 @@ systemd: --cni-conf-dir=/etc/kubernetes/cni/net.d \ --exit-on-lock-contention \ --healthz-port=0 \ - --kubeconfig=/etc/kubernetes/kubeconfig \ + --kubeconfig=/var/lib/kubelet/kubeconfig \ --lock-file=/var/run/lock/kubelet.lock \ --network-plugin=cni \ --node-labels=node.kubernetes.io/node \ @@ -71,6 +72,7 @@ systemd: %{~ endfor ~} --pod-manifest-path=/etc/kubernetes/manifests \ --read-only-port=0 \ + --rotate-certificates \ --volume-plugin-dir=/var/lib/kubelet/volumeplugins ExecStop=-/usr/bin/podman stop kubelet Delegate=yes From 4ac2d949993e3b6e968ba9a1f8e42577e48d223b Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Tue, 28 Apr 2020 19:54:37 -0700 Subject: [PATCH 402/523] Add Fedora CoreOS Azure docs to site navigation * Fix missing Fedora CoreOS Azure docs --- mkdocs.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/mkdocs.yml b/mkdocs.yml index afca2372e..49d470db5 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -55,6 +55,7 @@ nav: - 'Google Cloud': 'architecture/google-cloud.md' - 'Fedora CoreOS': - 'AWS': 'fedora-coreos/aws.md' + - 'Azure': 'fedora-coreos/azure.md' - 'Bare-Metal': 'fedora-coreos/bare-metal.md' - 'Digital Ocean': 'fedora-coreos/digitalocean.md' - 'Google Cloud': 'fedora-coreos/google-cloud.md' From 2c1af917ece69cbd6be0a15c9c4c6a0bca88836e Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Tue, 28 Apr 2020 19:57:13 -0700 Subject: [PATCH 403/523] Update recommended Terraform provider versions * Sync the Terraform provider plugin versions to those actively used and tested by the author * Fix terraform fmt --- digital-ocean/fedora-coreos/kubernetes/controllers.tf | 8 ++++---- digital-ocean/fedora-coreos/kubernetes/workers.tf | 6 +++--- docs/cl/aws.md | 2 +- docs/cl/azure.md | 2 +- docs/cl/digital-ocean.md | 2 +- docs/cl/google-cloud.md | 2 +- docs/fedora-coreos/aws.md | 2 +- docs/fedora-coreos/azure.md | 2 +- docs/fedora-coreos/digitalocean.md | 2 +- docs/fedora-coreos/google-cloud.md | 2 +- 10 files changed, 15 insertions(+), 15 deletions(-) diff --git a/digital-ocean/fedora-coreos/kubernetes/controllers.tf b/digital-ocean/fedora-coreos/kubernetes/controllers.tf index 5bf2c18a8..2c0964c93 100644 --- a/digital-ocean/fedora-coreos/kubernetes/controllers.tf +++ b/digital-ocean/fedora-coreos/kubernetes/controllers.tf @@ -64,10 +64,10 @@ resource "digitalocean_tag" "controllers" { # Controller Ignition configs data "ct_config" "controller-ignitions" { - count = var.controller_count - content = data.template_file.controller-configs.*.rendered[count.index] - strict = true - snippets = var.controller_snippets + count = var.controller_count + content = data.template_file.controller-configs.*.rendered[count.index] + strict = true + snippets = var.controller_snippets } # Controller Fedora CoreOS configs diff --git a/digital-ocean/fedora-coreos/kubernetes/workers.tf b/digital-ocean/fedora-coreos/kubernetes/workers.tf index f00e5510e..c1a087ae6 100644 --- a/digital-ocean/fedora-coreos/kubernetes/workers.tf +++ b/digital-ocean/fedora-coreos/kubernetes/workers.tf @@ -60,9 +60,9 @@ resource "digitalocean_tag" "workers" { # Worker Ignition config data "ct_config" "worker-ignition" { - content = data.template_file.worker-config.rendered - strict = true - snippets = var.worker_snippets + content = data.template_file.worker-config.rendered + strict = true + snippets = var.worker_snippets } # Worker Fedora CoreOS config diff --git a/docs/cl/aws.md b/docs/cl/aws.md index afc469cea..0e028ac67 100644 --- a/docs/cl/aws.md +++ b/docs/cl/aws.md @@ -49,7 +49,7 @@ Configure the AWS provider to use your access key credentials in a `providers.tf ```tf provider "aws" { - version = "2.53.0" + version = "2.59.0" region = "eu-central-1" shared_credentials_file = "/home/user/.config/aws/credentials" } diff --git a/docs/cl/azure.md b/docs/cl/azure.md index fc0855a11..6c67c209d 100644 --- a/docs/cl/azure.md +++ b/docs/cl/azure.md @@ -47,7 +47,7 @@ Configure the Azure provider in a `providers.tf` file. ```tf provider "azurerm" { - version = "2.5.0" + version = "2.7.0" } provider "ct" { diff --git a/docs/cl/digital-ocean.md b/docs/cl/digital-ocean.md index 164ba9ed7..929e63f4c 100644 --- a/docs/cl/digital-ocean.md +++ b/docs/cl/digital-ocean.md @@ -50,7 +50,7 @@ Configure the DigitalOcean provider to use your token in a `providers.tf` file. ```tf provider "digitalocean" { - version = "1.15.1" + version = "1.17.0" token = "${chomp(file("~/.config/digital-ocean/token"))}" } diff --git a/docs/cl/google-cloud.md b/docs/cl/google-cloud.md index 511c09c56..a1cbb7b43 100644 --- a/docs/cl/google-cloud.md +++ b/docs/cl/google-cloud.md @@ -49,7 +49,7 @@ Configure the Google Cloud provider to use your service account key, project-id, ```tf provider "google" { - version = "3.12.0" + version = "3.19.0" project = "project-id" region = "us-central1" credentials = file("~/.config/google-cloud/terraform.json") diff --git a/docs/fedora-coreos/aws.md b/docs/fedora-coreos/aws.md index cdcef4d01..5df5b1f17 100644 --- a/docs/fedora-coreos/aws.md +++ b/docs/fedora-coreos/aws.md @@ -49,7 +49,7 @@ Configure the AWS provider to use your access key credentials in a `providers.tf ```tf provider "aws" { - version = "2.53.0" + version = "2.59.0" region = "eu-central-1" shared_credentials_file = "/home/user/.config/aws/credentials" } diff --git a/docs/fedora-coreos/azure.md b/docs/fedora-coreos/azure.md index e7e1278e0..3a49620d2 100644 --- a/docs/fedora-coreos/azure.md +++ b/docs/fedora-coreos/azure.md @@ -47,7 +47,7 @@ Configure the Azure provider in a `providers.tf` file. ```tf provider "azurerm" { - version = "2.5.0" + version = "2.7.0" } provider "ct" { diff --git a/docs/fedora-coreos/digitalocean.md b/docs/fedora-coreos/digitalocean.md index 81d4287c5..a13db6616 100644 --- a/docs/fedora-coreos/digitalocean.md +++ b/docs/fedora-coreos/digitalocean.md @@ -50,7 +50,7 @@ Configure the DigitalOcean provider to use your token in a `providers.tf` file. ```tf provider "digitalocean" { - version = "1.15.1" + version = "1.17.0" token = "${chomp(file("~/.config/digital-ocean/token"))}" } diff --git a/docs/fedora-coreos/google-cloud.md b/docs/fedora-coreos/google-cloud.md index a635c6bf8..0e80cd26c 100644 --- a/docs/fedora-coreos/google-cloud.md +++ b/docs/fedora-coreos/google-cloud.md @@ -49,7 +49,7 @@ Configure the Google Cloud provider to use your service account key, project-id, ```tf provider "google" { - version = "3.12.0" + version = "3.19.0" project = "project-id" region = "us-central1" credentials = file("~/.config/google-cloud/terraform.json") From 317416b31637dcf6cd9186d7bfa381f17e524dbf Mon Sep 17 00:00:00 2001 From: Ben Drucker Date: Wed, 29 Apr 2020 20:41:08 -0700 Subject: [PATCH 404/523] Use Terraform element wrap-around for AWS controllers subnet_id (#714) * Fix Terraform plan error when controller_count exceeds available AWS zones (e.g. 5 controllers) --- aws/container-linux/kubernetes/controllers.tf | 2 +- aws/fedora-coreos/kubernetes/controllers.tf | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/aws/container-linux/kubernetes/controllers.tf b/aws/container-linux/kubernetes/controllers.tf index cd02024e7..66a0a9bc2 100644 --- a/aws/container-linux/kubernetes/controllers.tf +++ b/aws/container-linux/kubernetes/controllers.tf @@ -36,7 +36,7 @@ resource "aws_instance" "controllers" { # network associate_public_ip_address = true - subnet_id = aws_subnet.public.*.id[count.index] + subnet_id = element(aws_subnet.public.*.id, count.index) vpc_security_group_ids = [aws_security_group.controller.id] lifecycle { diff --git a/aws/fedora-coreos/kubernetes/controllers.tf b/aws/fedora-coreos/kubernetes/controllers.tf index e4bd9ddba..2fd253db7 100644 --- a/aws/fedora-coreos/kubernetes/controllers.tf +++ b/aws/fedora-coreos/kubernetes/controllers.tf @@ -36,7 +36,7 @@ resource "aws_instance" "controllers" { # network associate_public_ip_address = true - subnet_id = aws_subnet.public.*.id[count.index] + subnet_id = element(aws_subnet.public.*.id, count.index) vpc_security_group_ids = [aws_security_group.controller.id] lifecycle { From 64035005d429051afa79968b0016462dc67a07ec Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Wed, 29 Apr 2020 20:51:45 -0700 Subject: [PATCH 405/523] Update Grafana from v6.7.2 to v7.0.0-beta1 * https://github.com/grafana/grafana/releases/tag/v7.0.0-beta1 --- CHANGES.md | 6 ++++++ addons/grafana/deployment.yaml | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index 96ef54535..f90b70dda 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -9,9 +9,15 @@ Notable changes between versions. * Renew Kubelet certificates every 72 hours * Update Calico from v3.13.1 to [v3.13.3](https://docs.projectcalico.org/v3.13/release-notes/) +#### AWS + +* Fix Terraform plan error when `controller_count` exceeds AWS zones (e.g. 5 controllers) ([#714](https://github.com/poseidon/typhoon/pull/714)) + * Regressed in v1.17.1 ([#605](https://github.com/poseidon/typhoon/pull/605)) + #### Addons * Update Prometheus from v2.17.1 to v2.17.2 +* Update Grafana from v6.7.2 to v7.0.0-beta1 ## v1.18.2 diff --git a/addons/grafana/deployment.yaml b/addons/grafana/deployment.yaml index 57a0a2476..1cd246c9d 100644 --- a/addons/grafana/deployment.yaml +++ b/addons/grafana/deployment.yaml @@ -23,7 +23,7 @@ spec: spec: containers: - name: grafana - image: docker.io/grafana/grafana:6.7.2 + image: docker.io/grafana/grafana:7.0.0-beta1 env: - name: GF_PATHS_CONFIG value: "/etc/grafana/custom.ini" From e71e27e7696903eb3b10cb30ec267233ae6d771a Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Tue, 28 Apr 2020 20:28:52 -0700 Subject: [PATCH 406/523] Update Prometheus from v2.17.2 to v2.18.0-rc.1 * https://github.com/prometheus/prometheus/releases/tag/v2.18.0-rc.1 --- CHANGES.md | 2 +- addons/prometheus/deployment.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index f90b70dda..dd6f52d12 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -16,7 +16,7 @@ Notable changes between versions. #### Addons -* Update Prometheus from v2.17.1 to v2.17.2 +* Update Prometheus from v2.17.1 to v2.18.0-rc.1 * Update Grafana from v6.7.2 to v7.0.0-beta1 ## v1.18.2 diff --git a/addons/prometheus/deployment.yaml b/addons/prometheus/deployment.yaml index 6f008437f..6537b0785 100644 --- a/addons/prometheus/deployment.yaml +++ b/addons/prometheus/deployment.yaml @@ -20,7 +20,7 @@ spec: serviceAccountName: prometheus containers: - name: prometheus - image: quay.io/prometheus/prometheus:v2.17.2 + image: quay.io/prometheus/prometheus:v2.18.0-rc.1 args: - --web.listen-address=0.0.0.0:9090 - --config.file=/etc/prometheus/prometheus.yaml From 6afc1643d97edb369efe819e6d7da8a08f89164a Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Sun, 3 May 2020 23:18:47 -0700 Subject: [PATCH 407/523] Update nginx-ingress from v0.30.0 to v0.32.0 * Add support for IngressClass and RBAC authorization * Since our nginx ingress controller example uses the flag `--ingress-class=public`, add an IngressClass to go along with it Rel: https://kubernetes.io/docs/concepts/services-networking/ingress/#ingress-class --- CHANGES.md | 2 ++ addons/nginx-ingress/aws/class.yaml | 6 ++++++ addons/nginx-ingress/aws/deployment.yaml | 2 +- addons/nginx-ingress/aws/rbac/cluster-role.yaml | 9 +++++++++ addons/nginx-ingress/azure/class.yaml | 6 ++++++ addons/nginx-ingress/azure/deployment.yaml | 2 +- addons/nginx-ingress/azure/rbac/cluster-role.yaml | 9 +++++++++ addons/nginx-ingress/bare-metal/class.yaml | 6 ++++++ addons/nginx-ingress/bare-metal/deployment.yaml | 2 +- addons/nginx-ingress/bare-metal/rbac/cluster-role.yaml | 9 +++++++++ addons/nginx-ingress/digital-ocean/class.yaml | 6 ++++++ addons/nginx-ingress/digital-ocean/daemonset.yaml | 2 +- .../nginx-ingress/digital-ocean/rbac/cluster-role.yaml | 9 +++++++++ addons/nginx-ingress/google-cloud/class.yaml | 6 ++++++ addons/nginx-ingress/google-cloud/deployment.yaml | 2 +- addons/nginx-ingress/google-cloud/rbac/cluster-role.yaml | 9 +++++++++ 16 files changed, 82 insertions(+), 5 deletions(-) create mode 100644 addons/nginx-ingress/aws/class.yaml create mode 100644 addons/nginx-ingress/azure/class.yaml create mode 100644 addons/nginx-ingress/bare-metal/class.yaml create mode 100644 addons/nginx-ingress/digital-ocean/class.yaml create mode 100644 addons/nginx-ingress/google-cloud/class.yaml diff --git a/CHANGES.md b/CHANGES.md index dd6f52d12..175a0fe54 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -16,6 +16,8 @@ Notable changes between versions. #### Addons +* Update nginx-ingress from v0.30.0 to [v0.32.0](https://github.com/kubernetes/ingress-nginx/releases/tag/nginx-0.32.0) + * Add support for [IngressClass](https://kubernetes.io/docs/concepts/services-networking/ingress/#ingress-class) * Update Prometheus from v2.17.1 to v2.18.0-rc.1 * Update Grafana from v6.7.2 to v7.0.0-beta1 diff --git a/addons/nginx-ingress/aws/class.yaml b/addons/nginx-ingress/aws/class.yaml new file mode 100644 index 000000000..bbc8015c3 --- /dev/null +++ b/addons/nginx-ingress/aws/class.yaml @@ -0,0 +1,6 @@ +apiVersion: networking.k8s.io/v1beta1 +kind: IngressClass +metadata: + name: public +spec: + controller: k8s.io/ingress-nginx diff --git a/addons/nginx-ingress/aws/deployment.yaml b/addons/nginx-ingress/aws/deployment.yaml index 56b74e882..3f3b2fe4d 100644 --- a/addons/nginx-ingress/aws/deployment.yaml +++ b/addons/nginx-ingress/aws/deployment.yaml @@ -22,7 +22,7 @@ spec: spec: containers: - name: nginx-ingress-controller - image: quay.io/kubernetes-ingress-controller/nginx-ingress-controller:0.30.0 + image: quay.io/kubernetes-ingress-controller/nginx-ingress-controller:0.32.0 args: - /nginx-ingress-controller - --ingress-class=public diff --git a/addons/nginx-ingress/aws/rbac/cluster-role.yaml b/addons/nginx-ingress/aws/rbac/cluster-role.yaml index 5682d3974..90edbeb17 100644 --- a/addons/nginx-ingress/aws/rbac/cluster-role.yaml +++ b/addons/nginx-ingress/aws/rbac/cluster-role.yaml @@ -51,3 +51,12 @@ rules: - ingresses/status verbs: - update + - apiGroups: + - "networking.k8s.io" + resources: + - ingressclasses + verbs: + - get + - list + - watch + diff --git a/addons/nginx-ingress/azure/class.yaml b/addons/nginx-ingress/azure/class.yaml new file mode 100644 index 000000000..bbc8015c3 --- /dev/null +++ b/addons/nginx-ingress/azure/class.yaml @@ -0,0 +1,6 @@ +apiVersion: networking.k8s.io/v1beta1 +kind: IngressClass +metadata: + name: public +spec: + controller: k8s.io/ingress-nginx diff --git a/addons/nginx-ingress/azure/deployment.yaml b/addons/nginx-ingress/azure/deployment.yaml index 56b74e882..3f3b2fe4d 100644 --- a/addons/nginx-ingress/azure/deployment.yaml +++ b/addons/nginx-ingress/azure/deployment.yaml @@ -22,7 +22,7 @@ spec: spec: containers: - name: nginx-ingress-controller - image: quay.io/kubernetes-ingress-controller/nginx-ingress-controller:0.30.0 + image: quay.io/kubernetes-ingress-controller/nginx-ingress-controller:0.32.0 args: - /nginx-ingress-controller - --ingress-class=public diff --git a/addons/nginx-ingress/azure/rbac/cluster-role.yaml b/addons/nginx-ingress/azure/rbac/cluster-role.yaml index 5682d3974..90edbeb17 100644 --- a/addons/nginx-ingress/azure/rbac/cluster-role.yaml +++ b/addons/nginx-ingress/azure/rbac/cluster-role.yaml @@ -51,3 +51,12 @@ rules: - ingresses/status verbs: - update + - apiGroups: + - "networking.k8s.io" + resources: + - ingressclasses + verbs: + - get + - list + - watch + diff --git a/addons/nginx-ingress/bare-metal/class.yaml b/addons/nginx-ingress/bare-metal/class.yaml new file mode 100644 index 000000000..bbc8015c3 --- /dev/null +++ b/addons/nginx-ingress/bare-metal/class.yaml @@ -0,0 +1,6 @@ +apiVersion: networking.k8s.io/v1beta1 +kind: IngressClass +metadata: + name: public +spec: + controller: k8s.io/ingress-nginx diff --git a/addons/nginx-ingress/bare-metal/deployment.yaml b/addons/nginx-ingress/bare-metal/deployment.yaml index ac86bd5fe..5cabf818a 100644 --- a/addons/nginx-ingress/bare-metal/deployment.yaml +++ b/addons/nginx-ingress/bare-metal/deployment.yaml @@ -22,7 +22,7 @@ spec: spec: containers: - name: nginx-ingress-controller - image: quay.io/kubernetes-ingress-controller/nginx-ingress-controller:0.30.0 + image: quay.io/kubernetes-ingress-controller/nginx-ingress-controller:0.32.0 args: - /nginx-ingress-controller - --ingress-class=public diff --git a/addons/nginx-ingress/bare-metal/rbac/cluster-role.yaml b/addons/nginx-ingress/bare-metal/rbac/cluster-role.yaml index 5682d3974..90edbeb17 100644 --- a/addons/nginx-ingress/bare-metal/rbac/cluster-role.yaml +++ b/addons/nginx-ingress/bare-metal/rbac/cluster-role.yaml @@ -51,3 +51,12 @@ rules: - ingresses/status verbs: - update + - apiGroups: + - "networking.k8s.io" + resources: + - ingressclasses + verbs: + - get + - list + - watch + diff --git a/addons/nginx-ingress/digital-ocean/class.yaml b/addons/nginx-ingress/digital-ocean/class.yaml new file mode 100644 index 000000000..bbc8015c3 --- /dev/null +++ b/addons/nginx-ingress/digital-ocean/class.yaml @@ -0,0 +1,6 @@ +apiVersion: networking.k8s.io/v1beta1 +kind: IngressClass +metadata: + name: public +spec: + controller: k8s.io/ingress-nginx diff --git a/addons/nginx-ingress/digital-ocean/daemonset.yaml b/addons/nginx-ingress/digital-ocean/daemonset.yaml index 1bf474d66..4edd51b07 100644 --- a/addons/nginx-ingress/digital-ocean/daemonset.yaml +++ b/addons/nginx-ingress/digital-ocean/daemonset.yaml @@ -22,7 +22,7 @@ spec: spec: containers: - name: nginx-ingress-controller - image: quay.io/kubernetes-ingress-controller/nginx-ingress-controller:0.30.0 + image: quay.io/kubernetes-ingress-controller/nginx-ingress-controller:0.32.0 args: - /nginx-ingress-controller - --ingress-class=public diff --git a/addons/nginx-ingress/digital-ocean/rbac/cluster-role.yaml b/addons/nginx-ingress/digital-ocean/rbac/cluster-role.yaml index 5682d3974..90edbeb17 100644 --- a/addons/nginx-ingress/digital-ocean/rbac/cluster-role.yaml +++ b/addons/nginx-ingress/digital-ocean/rbac/cluster-role.yaml @@ -51,3 +51,12 @@ rules: - ingresses/status verbs: - update + - apiGroups: + - "networking.k8s.io" + resources: + - ingressclasses + verbs: + - get + - list + - watch + diff --git a/addons/nginx-ingress/google-cloud/class.yaml b/addons/nginx-ingress/google-cloud/class.yaml new file mode 100644 index 000000000..bbc8015c3 --- /dev/null +++ b/addons/nginx-ingress/google-cloud/class.yaml @@ -0,0 +1,6 @@ +apiVersion: networking.k8s.io/v1beta1 +kind: IngressClass +metadata: + name: public +spec: + controller: k8s.io/ingress-nginx diff --git a/addons/nginx-ingress/google-cloud/deployment.yaml b/addons/nginx-ingress/google-cloud/deployment.yaml index 56b74e882..3f3b2fe4d 100644 --- a/addons/nginx-ingress/google-cloud/deployment.yaml +++ b/addons/nginx-ingress/google-cloud/deployment.yaml @@ -22,7 +22,7 @@ spec: spec: containers: - name: nginx-ingress-controller - image: quay.io/kubernetes-ingress-controller/nginx-ingress-controller:0.30.0 + image: quay.io/kubernetes-ingress-controller/nginx-ingress-controller:0.32.0 args: - /nginx-ingress-controller - --ingress-class=public diff --git a/addons/nginx-ingress/google-cloud/rbac/cluster-role.yaml b/addons/nginx-ingress/google-cloud/rbac/cluster-role.yaml index 5682d3974..90edbeb17 100644 --- a/addons/nginx-ingress/google-cloud/rbac/cluster-role.yaml +++ b/addons/nginx-ingress/google-cloud/rbac/cluster-role.yaml @@ -51,3 +51,12 @@ rules: - ingresses/status verbs: - update + - apiGroups: + - "networking.k8s.io" + resources: + - ingressclasses + verbs: + - get + - list + - watch + From 70f30d9c07351c4230a17069d8f904fa784fbf31 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Tue, 5 May 2020 22:31:11 -0700 Subject: [PATCH 408/523] Update Prometheus from v2.18.0-rc.1 to v2.18.0 * https://github.com/prometheus/prometheus/releases/tag/v2.18.0 --- CHANGES.md | 2 +- addons/prometheus/deployment.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 175a0fe54..486be9d7a 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -18,7 +18,7 @@ Notable changes between versions. * Update nginx-ingress from v0.30.0 to [v0.32.0](https://github.com/kubernetes/ingress-nginx/releases/tag/nginx-0.32.0) * Add support for [IngressClass](https://kubernetes.io/docs/concepts/services-networking/ingress/#ingress-class) -* Update Prometheus from v2.17.1 to v2.18.0-rc.1 +* Update Prometheus from v2.17.1 to v2.18.0 * Update Grafana from v6.7.2 to v7.0.0-beta1 ## v1.18.2 diff --git a/addons/prometheus/deployment.yaml b/addons/prometheus/deployment.yaml index 6537b0785..03108312f 100644 --- a/addons/prometheus/deployment.yaml +++ b/addons/prometheus/deployment.yaml @@ -20,7 +20,7 @@ spec: serviceAccountName: prometheus containers: - name: prometheus - image: quay.io/prometheus/prometheus:v2.18.0-rc.1 + image: quay.io/prometheus/prometheus:v2.18.0 args: - --web.listen-address=0.0.0.0:9090 - --config.file=/etc/prometheus/prometheus.yaml From 33173c0206494eae379aa65958aaa2772b541d39 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Thu, 7 May 2020 22:59:11 -0700 Subject: [PATCH 409/523] Update Prometheus from v2.18.0 to v2.18.1 * https://github.com/prometheus/prometheus/releases/tag/v2.18.1 --- CHANGES.md | 2 +- addons/prometheus/deployment.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 486be9d7a..4b41701b1 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -18,7 +18,7 @@ Notable changes between versions. * Update nginx-ingress from v0.30.0 to [v0.32.0](https://github.com/kubernetes/ingress-nginx/releases/tag/nginx-0.32.0) * Add support for [IngressClass](https://kubernetes.io/docs/concepts/services-networking/ingress/#ingress-class) -* Update Prometheus from v2.17.1 to v2.18.0 +* Update Prometheus from v2.17.1 to v2.18.1 * Update Grafana from v6.7.2 to v7.0.0-beta1 ## v1.18.2 diff --git a/addons/prometheus/deployment.yaml b/addons/prometheus/deployment.yaml index 03108312f..59fd27698 100644 --- a/addons/prometheus/deployment.yaml +++ b/addons/prometheus/deployment.yaml @@ -20,7 +20,7 @@ spec: serviceAccountName: prometheus containers: - name: prometheus - image: quay.io/prometheus/prometheus:v2.18.0 + image: quay.io/prometheus/prometheus:v2.18.1 args: - --web.listen-address=0.0.0.0:9090 - --config.file=/etc/prometheus/prometheus.yaml From 3f0a5d2715df51789ff2dc3fc90bfadb9b151c99 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Thu, 7 May 2020 23:04:44 -0700 Subject: [PATCH 410/523] Update Grafana from v7.0.0-beta1 to v7.0.0-beta2 * https://github.com/grafana/grafana/releases/tag/v7.0.0-beta2 --- CHANGES.md | 2 +- addons/grafana/deployment.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 4b41701b1..ec18df3d4 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -19,7 +19,7 @@ Notable changes between versions. * Update nginx-ingress from v0.30.0 to [v0.32.0](https://github.com/kubernetes/ingress-nginx/releases/tag/nginx-0.32.0) * Add support for [IngressClass](https://kubernetes.io/docs/concepts/services-networking/ingress/#ingress-class) * Update Prometheus from v2.17.1 to v2.18.1 -* Update Grafana from v6.7.2 to v7.0.0-beta1 +* Update Grafana from v6.7.2 to v7.0.0-beta2 ## v1.18.2 diff --git a/addons/grafana/deployment.yaml b/addons/grafana/deployment.yaml index 1cd246c9d..566cd8ccb 100644 --- a/addons/grafana/deployment.yaml +++ b/addons/grafana/deployment.yaml @@ -23,7 +23,7 @@ spec: spec: containers: - name: grafana - image: docker.io/grafana/grafana:7.0.0-beta1 + image: docker.io/grafana/grafana:7.0.0-beta2 env: - name: GF_PATHS_CONFIG value: "/etc/grafana/custom.ini" From b5dabcea319b4b2cd844745307ef9394d6a3e464 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Fri, 8 May 2020 01:18:34 -0700 Subject: [PATCH 411/523] Use Fedora CoreOS image streams on Google Cloud * Add `os_stream` variable to set a Fedora CoreOS stream to `stable` (default), `testing`, or `next` * Deprecate `os_image` variable. Remove docs about uploading Fedora CoreOS images manually, this is no longer needed * https://docs.fedoraproject.org/en-US/fedora-coreos/update-streams/ Rel: https://github.com/coreos/fedora-coreos-docs/pull/70 --- CHANGES.md | 6 +++++ docs/advanced/worker-pools.md | 1 + docs/fedora-coreos/google-cloud.md | 22 +------------------ .../fedora-coreos/kubernetes/controllers.tf | 2 +- .../fedora-coreos/kubernetes/image.tf | 6 +++++ .../fedora-coreos/kubernetes/variables.tf | 8 +++++++ .../fedora-coreos/kubernetes/workers.tf | 1 + .../fedora-coreos/kubernetes/workers/image.tf | 6 +++++ .../kubernetes/workers/variables.tf | 10 ++++++++- .../kubernetes/workers/workers.tf | 2 +- 10 files changed, 40 insertions(+), 24 deletions(-) create mode 100644 google-cloud/fedora-coreos/kubernetes/image.tf create mode 100644 google-cloud/fedora-coreos/kubernetes/workers/image.tf diff --git a/CHANGES.md b/CHANGES.md index ec18df3d4..b38e9a89c 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -14,6 +14,12 @@ Notable changes between versions. * Fix Terraform plan error when `controller_count` exceeds AWS zones (e.g. 5 controllers) ([#714](https://github.com/poseidon/typhoon/pull/714)) * Regressed in v1.17.1 ([#605](https://github.com/poseidon/typhoon/pull/605)) +#### Google + +* Use new Fedora CoreOS official [image streams](https://docs.fedoraproject.org/en-US/fedora-coreos/update-streams/) ([#723](https://github.com/poseidon/typhoon/pull/722)) + * Add `os_stream` variable to set the stream to `stable` (default), `testing`, or `next` + * Deprecate `os_image` variable. Manual image uploads are no longer needed + #### Addons * Update nginx-ingress from v0.30.0 to [v0.32.0](https://github.com/kubernetes/ingress-nginx/releases/tag/nginx-0.32.0) diff --git a/docs/advanced/worker-pools.md b/docs/advanced/worker-pools.md index 6f92646f1..4162ca812 100644 --- a/docs/advanced/worker-pools.md +++ b/docs/advanced/worker-pools.md @@ -210,6 +210,7 @@ Check the list of regions [docs](https://cloud.google.com/compute/docs/regions-z |:-----|:------------|:--------|:--------| | worker_count | Number of instances | 1 | 3 | | machine_type | Compute instance machine type | "n1-standard-1" | See below | +| os_stream | Fedora CoreOS stream for compute instances | "stable" | "testing", "next" | | disk_size | Size of the disk in GB | 40 | 100 | | preemptible | If true, Compute Engine will terminate instances randomly within 24 hours | false | true | | snippets | Container Linux Config snippets | [] | [examples](/advanced/customization/) | diff --git a/docs/fedora-coreos/google-cloud.md b/docs/fedora-coreos/google-cloud.md index 0e80cd26c..6dba7819b 100644 --- a/docs/fedora-coreos/google-cloud.md +++ b/docs/fedora-coreos/google-cloud.md @@ -65,25 +65,6 @@ Additional configuration options are described in the `google` provider [docs](h !!! tip Regions are listed in [docs](https://cloud.google.com/compute/docs/regions-zones/regions-zones) or with `gcloud compute regions list`. A project may contain multiple clusters across different regions. -## Fedora CoreOS Images - -Fedora CoreOS publishes images for Google Cloud, but does not yet upload them. Google Cloud allows [custom boot images](https://cloud.google.com/compute/docs/images/import-existing-image) to be uploaded to a bucket and imported into your project. - -[Download](https://getfedora.org/coreos/download/) a Fedora CoreOS GCP gzipped tarball and upload it to a Google Cloud storage bucket. - -``` -gsutil list -gsutil cp fedora-coreos-31.20200323.3.2-gcp.x86_64.tar.gz gs://BUCKET -``` - -Create a Compute Engine image from the file. - -``` -gcloud compute images create fedora-coreos-31-20200323-3-2 --source-uri gs://BUCKET/fedora-coreos-31.20200323.3.2-gcp.x86_64.tar.gz -``` - -Set the [os_image](#variables) in the next step. - ## Cluster Define a Kubernetes cluster using the module `google-cloud/fedora-coreos/kubernetes`. @@ -99,7 +80,6 @@ module "yavin" { dns_zone_name = "example-zone" # configuration - os_image = "fedora-coreos-31-20200323-3-2" ssh_authorized_key = "ssh-rsa AAAAB3Nz..." # optional @@ -204,7 +184,6 @@ Check the [variables.tf](https://github.com/poseidon/typhoon/blob/master/google- | region | Google Cloud region | "us-central1" | | dns_zone | Google Cloud DNS zone | "google-cloud.example.com" | | dns_zone_name | Google Cloud DNS zone name | "example-zone" | -| os_image | Fedora CoreOS image for compute instances | "fedora-coreos-31-20200323-3-2" | | ssh_authorized_key | SSH public key for user 'core' | "ssh-rsa AAAAB3NZ..." | Check the list of valid [regions](https://cloud.google.com/compute/docs/regions-zones/regions-zones) and list Fedora CoreOS [images](https://cloud.google.com/compute/docs/images) with `gcloud compute images list | grep fedora-coreos`. @@ -234,6 +213,7 @@ resource "google_dns_managed_zone" "zone-for-clusters" { | worker_count | Number of workers | 1 | 3 | | controller_type | Machine type for controllers | "n1-standard-1" | See below | | worker_type | Machine type for workers | "n1-standard-1" | See below | +| os_stream | Fedora CoreOS stream for compute instances | "stable" | "testing", "next" | | disk_size | Size of the disk in GB | 40 | 100 | | worker_preemptible | If enabled, Compute Engine will terminate workers randomly within 24 hours | false | true | | controller_snippets | Controller Fedora CoreOS Config snippets | [] | [examples](/advanced/customization/) | diff --git a/google-cloud/fedora-coreos/kubernetes/controllers.tf b/google-cloud/fedora-coreos/kubernetes/controllers.tf index b2cde4341..81e20ed0b 100644 --- a/google-cloud/fedora-coreos/kubernetes/controllers.tf +++ b/google-cloud/fedora-coreos/kubernetes/controllers.tf @@ -42,7 +42,7 @@ resource "google_compute_instance" "controllers" { auto_delete = true initialize_params { - image = var.os_image + image = var.os_image == "" ? data.google_compute_image.fedora-coreos.self_link : var.os_image size = var.disk_size } } diff --git a/google-cloud/fedora-coreos/kubernetes/image.tf b/google-cloud/fedora-coreos/kubernetes/image.tf new file mode 100644 index 000000000..e35e274c4 --- /dev/null +++ b/google-cloud/fedora-coreos/kubernetes/image.tf @@ -0,0 +1,6 @@ + +# Fedora CoreOS most recent image from stream +data "google_compute_image" "fedora-coreos" { + project = "fedora-coreos-cloud" + family = "fedora-coreos-${var.os_stream}" +} diff --git a/google-cloud/fedora-coreos/kubernetes/variables.tf b/google-cloud/fedora-coreos/kubernetes/variables.tf index c7be56d0a..59626a88c 100644 --- a/google-cloud/fedora-coreos/kubernetes/variables.tf +++ b/google-cloud/fedora-coreos/kubernetes/variables.tf @@ -46,9 +46,17 @@ variable "worker_type" { default = "n1-standard-1" } +variable "os_stream" { + type = string + description = "Fedora CoreOS stream for compute instances (e.g. stable, testing, next)" + default = "stable" +} + +# Deprecated variable "os_image" { type = string description = "Fedora CoreOS image for compute instances (e.g. fedora-coreos)" + default = "" } variable "disk_size" { diff --git a/google-cloud/fedora-coreos/kubernetes/workers.tf b/google-cloud/fedora-coreos/kubernetes/workers.tf index 91a32bd0c..5afec0826 100644 --- a/google-cloud/fedora-coreos/kubernetes/workers.tf +++ b/google-cloud/fedora-coreos/kubernetes/workers.tf @@ -8,6 +8,7 @@ module "workers" { network = google_compute_network.network.name worker_count = var.worker_count machine_type = var.worker_type + os_stream = var.os_stream os_image = var.os_image disk_size = var.disk_size preemptible = var.worker_preemptible diff --git a/google-cloud/fedora-coreos/kubernetes/workers/image.tf b/google-cloud/fedora-coreos/kubernetes/workers/image.tf new file mode 100644 index 000000000..e35e274c4 --- /dev/null +++ b/google-cloud/fedora-coreos/kubernetes/workers/image.tf @@ -0,0 +1,6 @@ + +# Fedora CoreOS most recent image from stream +data "google_compute_image" "fedora-coreos" { + project = "fedora-coreos-cloud" + family = "fedora-coreos-${var.os_stream}" +} diff --git a/google-cloud/fedora-coreos/kubernetes/workers/variables.tf b/google-cloud/fedora-coreos/kubernetes/workers/variables.tf index 8f1ef933b..1a47e9547 100644 --- a/google-cloud/fedora-coreos/kubernetes/workers/variables.tf +++ b/google-cloud/fedora-coreos/kubernetes/workers/variables.tf @@ -34,9 +34,17 @@ variable "machine_type" { default = "n1-standard-1" } +variable "os_stream" { + type = string + description = "Fedora CoreOS stream for compute instances (e.g. stable, testing, next)" + default = "stable" +} + +# Deprecated variable "os_image" { type = string - description = "Fedora CoreOS image for compute instanges (e.g. gcloud compute images list)" + description = "Fedora CoreOS image for compute instances (e.g. fedora-coreos)" + default = "" } variable "disk_size" { diff --git a/google-cloud/fedora-coreos/kubernetes/workers/workers.tf b/google-cloud/fedora-coreos/kubernetes/workers/workers.tf index 1d3d65894..c6620afe5 100644 --- a/google-cloud/fedora-coreos/kubernetes/workers/workers.tf +++ b/google-cloud/fedora-coreos/kubernetes/workers/workers.tf @@ -43,7 +43,7 @@ resource "google_compute_instance_template" "worker" { disk { auto_delete = true boot = true - source_image = var.os_image + source_image = var.os_image == "" ? data.google_compute_image.fedora-coreos.self_link : var.os_image disk_size_gb = var.disk_size } From 358854e712ad0e94f5a057107815833ee7c60e16 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Sat, 9 May 2020 15:58:45 -0700 Subject: [PATCH 412/523] Fix Calico install-cni crash loop on Pod restarts * Set a consistent MCS level/range for Calico install-cni * Note: Rebooting a node was a workaround, because Kubelet relabels /etc/kubernetes(/cni/net.d) Background: * On SELinux enforcing systems, the Calico CNI install-cni container ran with default SELinux context and a random MCS pair. install-cni places CNI configs by first creating a temporary file and then moving them into place, which means the file MCS categories depend on the containers SELinux context. * calico-node Pod restarts creates a new install-cni container with a different MCS pair that cannot access the earlier written file (it places configs every time), causing the init container to error and calico-node to crash loop * https://github.com/projectcalico/cni-plugin/issues/874 ``` mv: inter-device move failed: '/calico.conf.tmp' to '/host/etc/cni/net.d/10-calico.conflist'; unable to remove target: Permission denied Failed to mv files. This may be caused by selinux configuration on the host, or something else. ``` Note, this isn't a host SELinux configuration issue. Related: * https://github.com/poseidon/terraform-render-bootstrap/pull/186 --- CHANGES.md | 8 +++++++- aws/container-linux/kubernetes/bootstrap.tf | 2 +- aws/fedora-coreos/kubernetes/bootstrap.tf | 2 +- azure/container-linux/kubernetes/bootstrap.tf | 2 +- azure/fedora-coreos/kubernetes/bootstrap.tf | 2 +- bare-metal/container-linux/kubernetes/bootstrap.tf | 2 +- bare-metal/fedora-coreos/kubernetes/bootstrap.tf | 2 +- digital-ocean/container-linux/kubernetes/bootstrap.tf | 2 +- digital-ocean/fedora-coreos/kubernetes/bootstrap.tf | 2 +- google-cloud/container-linux/kubernetes/bootstrap.tf | 2 +- google-cloud/fedora-coreos/kubernetes/bootstrap.tf | 2 +- 11 files changed, 17 insertions(+), 11 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index b38e9a89c..4e6dbcbe9 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -9,11 +9,17 @@ Notable changes between versions. * Renew Kubelet certificates every 72 hours * Update Calico from v3.13.1 to [v3.13.3](https://docs.projectcalico.org/v3.13/release-notes/) -#### AWS +### AWS * Fix Terraform plan error when `controller_count` exceeds AWS zones (e.g. 5 controllers) ([#714](https://github.com/poseidon/typhoon/pull/714)) * Regressed in v1.17.1 ([#605](https://github.com/poseidon/typhoon/pull/605)) +### Fedora CoreOS + +* Fix Calico `install-cni` crashloop on Pod restarts ([#724](https://github.com/poseidon/typhoon/pull/724)) + * SELinux enforcement requires consistent file context MCS level + * Restarting a node resolved the issue as a previous workaround + #### Google * Use new Fedora CoreOS official [image streams](https://docs.fedoraproject.org/en-US/fedora-coreos/update-streams/) ([#723](https://github.com/poseidon/typhoon/pull/722)) diff --git a/aws/container-linux/kubernetes/bootstrap.tf b/aws/container-linux/kubernetes/bootstrap.tf index 9a371c89d..92423b986 100644 --- a/aws/container-linux/kubernetes/bootstrap.tf +++ b/aws/container-linux/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=924beb4b0cb3ca076c29c85983070d0f66dddc5c" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=1dc36b58b83b68b09d072ee7548e006c74e56bb1" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/aws/fedora-coreos/kubernetes/bootstrap.tf b/aws/fedora-coreos/kubernetes/bootstrap.tf index 64ffb08d0..27f7bc36b 100644 --- a/aws/fedora-coreos/kubernetes/bootstrap.tf +++ b/aws/fedora-coreos/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=924beb4b0cb3ca076c29c85983070d0f66dddc5c" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=1dc36b58b83b68b09d072ee7548e006c74e56bb1" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/azure/container-linux/kubernetes/bootstrap.tf b/azure/container-linux/kubernetes/bootstrap.tf index 1ab4a4768..a07e21873 100644 --- a/azure/container-linux/kubernetes/bootstrap.tf +++ b/azure/container-linux/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=924beb4b0cb3ca076c29c85983070d0f66dddc5c" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=1dc36b58b83b68b09d072ee7548e006c74e56bb1" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/azure/fedora-coreos/kubernetes/bootstrap.tf b/azure/fedora-coreos/kubernetes/bootstrap.tf index 01c74b1f0..5b842fe34 100644 --- a/azure/fedora-coreos/kubernetes/bootstrap.tf +++ b/azure/fedora-coreos/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=924beb4b0cb3ca076c29c85983070d0f66dddc5c" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=1dc36b58b83b68b09d072ee7548e006c74e56bb1" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/bare-metal/container-linux/kubernetes/bootstrap.tf b/bare-metal/container-linux/kubernetes/bootstrap.tf index 6cae75043..06d9cfbc4 100644 --- a/bare-metal/container-linux/kubernetes/bootstrap.tf +++ b/bare-metal/container-linux/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=924beb4b0cb3ca076c29c85983070d0f66dddc5c" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=1dc36b58b83b68b09d072ee7548e006c74e56bb1" cluster_name = var.cluster_name api_servers = [var.k8s_domain_name] diff --git a/bare-metal/fedora-coreos/kubernetes/bootstrap.tf b/bare-metal/fedora-coreos/kubernetes/bootstrap.tf index 587d4f1ec..4c3ea223c 100644 --- a/bare-metal/fedora-coreos/kubernetes/bootstrap.tf +++ b/bare-metal/fedora-coreos/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=924beb4b0cb3ca076c29c85983070d0f66dddc5c" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=1dc36b58b83b68b09d072ee7548e006c74e56bb1" cluster_name = var.cluster_name api_servers = [var.k8s_domain_name] diff --git a/digital-ocean/container-linux/kubernetes/bootstrap.tf b/digital-ocean/container-linux/kubernetes/bootstrap.tf index af5e3e469..102f664bc 100644 --- a/digital-ocean/container-linux/kubernetes/bootstrap.tf +++ b/digital-ocean/container-linux/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=924beb4b0cb3ca076c29c85983070d0f66dddc5c" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=1dc36b58b83b68b09d072ee7548e006c74e56bb1" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/digital-ocean/fedora-coreos/kubernetes/bootstrap.tf b/digital-ocean/fedora-coreos/kubernetes/bootstrap.tf index 3bd95bbe4..21e726b2b 100644 --- a/digital-ocean/fedora-coreos/kubernetes/bootstrap.tf +++ b/digital-ocean/fedora-coreos/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=924beb4b0cb3ca076c29c85983070d0f66dddc5c" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=1dc36b58b83b68b09d072ee7548e006c74e56bb1" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/google-cloud/container-linux/kubernetes/bootstrap.tf b/google-cloud/container-linux/kubernetes/bootstrap.tf index 49af2d10d..b7486a534 100644 --- a/google-cloud/container-linux/kubernetes/bootstrap.tf +++ b/google-cloud/container-linux/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=924beb4b0cb3ca076c29c85983070d0f66dddc5c" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=1dc36b58b83b68b09d072ee7548e006c74e56bb1" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/google-cloud/fedora-coreos/kubernetes/bootstrap.tf b/google-cloud/fedora-coreos/kubernetes/bootstrap.tf index 0b3b41d72..69c9724b7 100644 --- a/google-cloud/fedora-coreos/kubernetes/bootstrap.tf +++ b/google-cloud/fedora-coreos/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=924beb4b0cb3ca076c29c85983070d0f66dddc5c" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=1dc36b58b83b68b09d072ee7548e006c74e56bb1" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] From a2db4fa8c4cb22136af728fec03686108140f792 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Sat, 9 May 2020 16:05:30 -0700 Subject: [PATCH 413/523] Update Calico from v3.13.3 to v3.14.0 * https://docs.projectcalico.org/v3.14/release-notes/ --- CHANGES.md | 2 +- aws/container-linux/kubernetes/bootstrap.tf | 2 +- aws/fedora-coreos/kubernetes/bootstrap.tf | 2 +- azure/container-linux/kubernetes/bootstrap.tf | 2 +- azure/fedora-coreos/kubernetes/bootstrap.tf | 2 +- bare-metal/container-linux/kubernetes/bootstrap.tf | 2 +- bare-metal/fedora-coreos/kubernetes/bootstrap.tf | 2 +- digital-ocean/container-linux/kubernetes/bootstrap.tf | 2 +- digital-ocean/fedora-coreos/kubernetes/bootstrap.tf | 2 +- google-cloud/container-linux/kubernetes/bootstrap.tf | 2 +- google-cloud/fedora-coreos/kubernetes/bootstrap.tf | 2 +- 11 files changed, 11 insertions(+), 11 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 4e6dbcbe9..bff8f70ab 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -7,7 +7,7 @@ Notable changes between versions. * Use Kubelet [TLS bootstrap](https://kubernetes.io/docs/reference/command-line-tools-reference/kubelet-tls-bootstrapping/) with bootstrap token authentication ([#713](https://github.com/poseidon/typhoon/pull/713)) * Enable Node [Authorization](https://kubernetes.io/docs/reference/access-authn-authz/node/) and [NodeRestriction](https://kubernetes.io/docs/reference/access-authn-authz/admission-controllers/#noderestriction) to reduce authorization scope * Renew Kubelet certificates every 72 hours -* Update Calico from v3.13.1 to [v3.13.3](https://docs.projectcalico.org/v3.13/release-notes/) +* Update Calico from v3.13.1 to [v3.14.0](https://docs.projectcalico.org/v3.14/release-notes/) ### AWS diff --git a/aws/container-linux/kubernetes/bootstrap.tf b/aws/container-linux/kubernetes/bootstrap.tf index 92423b986..ae27f5b91 100644 --- a/aws/container-linux/kubernetes/bootstrap.tf +++ b/aws/container-linux/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=1dc36b58b83b68b09d072ee7548e006c74e56bb1" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=157336db92daf1f1b7f09ab2b3ccbe16a4b66c57" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/aws/fedora-coreos/kubernetes/bootstrap.tf b/aws/fedora-coreos/kubernetes/bootstrap.tf index 27f7bc36b..bde0ea923 100644 --- a/aws/fedora-coreos/kubernetes/bootstrap.tf +++ b/aws/fedora-coreos/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=1dc36b58b83b68b09d072ee7548e006c74e56bb1" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=157336db92daf1f1b7f09ab2b3ccbe16a4b66c57" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/azure/container-linux/kubernetes/bootstrap.tf b/azure/container-linux/kubernetes/bootstrap.tf index a07e21873..7a8252786 100644 --- a/azure/container-linux/kubernetes/bootstrap.tf +++ b/azure/container-linux/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=1dc36b58b83b68b09d072ee7548e006c74e56bb1" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=157336db92daf1f1b7f09ab2b3ccbe16a4b66c57" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/azure/fedora-coreos/kubernetes/bootstrap.tf b/azure/fedora-coreos/kubernetes/bootstrap.tf index 5b842fe34..a95d3ebfc 100644 --- a/azure/fedora-coreos/kubernetes/bootstrap.tf +++ b/azure/fedora-coreos/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=1dc36b58b83b68b09d072ee7548e006c74e56bb1" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=157336db92daf1f1b7f09ab2b3ccbe16a4b66c57" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/bare-metal/container-linux/kubernetes/bootstrap.tf b/bare-metal/container-linux/kubernetes/bootstrap.tf index 06d9cfbc4..45097295a 100644 --- a/bare-metal/container-linux/kubernetes/bootstrap.tf +++ b/bare-metal/container-linux/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=1dc36b58b83b68b09d072ee7548e006c74e56bb1" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=157336db92daf1f1b7f09ab2b3ccbe16a4b66c57" cluster_name = var.cluster_name api_servers = [var.k8s_domain_name] diff --git a/bare-metal/fedora-coreos/kubernetes/bootstrap.tf b/bare-metal/fedora-coreos/kubernetes/bootstrap.tf index 4c3ea223c..ae8084b80 100644 --- a/bare-metal/fedora-coreos/kubernetes/bootstrap.tf +++ b/bare-metal/fedora-coreos/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=1dc36b58b83b68b09d072ee7548e006c74e56bb1" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=157336db92daf1f1b7f09ab2b3ccbe16a4b66c57" cluster_name = var.cluster_name api_servers = [var.k8s_domain_name] diff --git a/digital-ocean/container-linux/kubernetes/bootstrap.tf b/digital-ocean/container-linux/kubernetes/bootstrap.tf index 102f664bc..0a636748c 100644 --- a/digital-ocean/container-linux/kubernetes/bootstrap.tf +++ b/digital-ocean/container-linux/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=1dc36b58b83b68b09d072ee7548e006c74e56bb1" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=157336db92daf1f1b7f09ab2b3ccbe16a4b66c57" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/digital-ocean/fedora-coreos/kubernetes/bootstrap.tf b/digital-ocean/fedora-coreos/kubernetes/bootstrap.tf index 21e726b2b..c39cdeea4 100644 --- a/digital-ocean/fedora-coreos/kubernetes/bootstrap.tf +++ b/digital-ocean/fedora-coreos/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=1dc36b58b83b68b09d072ee7548e006c74e56bb1" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=157336db92daf1f1b7f09ab2b3ccbe16a4b66c57" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/google-cloud/container-linux/kubernetes/bootstrap.tf b/google-cloud/container-linux/kubernetes/bootstrap.tf index b7486a534..452c96748 100644 --- a/google-cloud/container-linux/kubernetes/bootstrap.tf +++ b/google-cloud/container-linux/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=1dc36b58b83b68b09d072ee7548e006c74e56bb1" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=157336db92daf1f1b7f09ab2b3ccbe16a4b66c57" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/google-cloud/fedora-coreos/kubernetes/bootstrap.tf b/google-cloud/fedora-coreos/kubernetes/bootstrap.tf index 69c9724b7..2358bd744 100644 --- a/google-cloud/fedora-coreos/kubernetes/bootstrap.tf +++ b/google-cloud/fedora-coreos/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=1dc36b58b83b68b09d072ee7548e006c74e56bb1" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=157336db92daf1f1b7f09ab2b3ccbe16a4b66c57" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] From f4194cd57a757b732d131686114ad137971b169f Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Sat, 9 May 2020 17:50:40 -0700 Subject: [PATCH 414/523] Update Grafana from v7.0.0-beta2 to v7.0.0-beta.3 * https://github.com/grafana/grafana/releases/tag/v7.0.0-beta3 --- CHANGES.md | 2 +- addons/grafana/deployment.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index bff8f70ab..49134b394 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -31,7 +31,7 @@ Notable changes between versions. * Update nginx-ingress from v0.30.0 to [v0.32.0](https://github.com/kubernetes/ingress-nginx/releases/tag/nginx-0.32.0) * Add support for [IngressClass](https://kubernetes.io/docs/concepts/services-networking/ingress/#ingress-class) * Update Prometheus from v2.17.1 to v2.18.1 -* Update Grafana from v6.7.2 to v7.0.0-beta2 +* Update Grafana from v6.7.2 to v7.0.0-beta3 ## v1.18.2 diff --git a/addons/grafana/deployment.yaml b/addons/grafana/deployment.yaml index 566cd8ccb..50024813f 100644 --- a/addons/grafana/deployment.yaml +++ b/addons/grafana/deployment.yaml @@ -23,7 +23,7 @@ spec: spec: containers: - name: grafana - image: docker.io/grafana/grafana:7.0.0-beta2 + image: docker.io/grafana/grafana:7.0.0-beta3 env: - name: GF_PATHS_CONFIG value: "/etc/grafana/custom.ini" From 01905b00bc11521e2044dfd38dea6e1db69dfb73 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Wed, 13 May 2020 21:37:18 -0700 Subject: [PATCH 415/523] Support Fedora CoreOS OS image streams on AWS * Add `os_stream` variable to set the stream to stable (default), testing, or next * Remove unused os_image variable on Fedora CoreOS AWS --- CHANGES.md | 8 +++++++- aws/fedora-coreos/kubernetes/ami.tf | 10 +--------- aws/fedora-coreos/kubernetes/variables.tf | 4 ++-- aws/fedora-coreos/kubernetes/workers.tf | 2 +- aws/fedora-coreos/kubernetes/workers/ami.tf | 10 +--------- aws/fedora-coreos/kubernetes/workers/variables.tf | 4 ++-- 6 files changed, 14 insertions(+), 24 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 49134b394..f860e298e 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -20,9 +20,15 @@ Notable changes between versions. * SELinux enforcement requires consistent file context MCS level * Restarting a node resolved the issue as a previous workaround +#### AWS + +* Support Fedora CoreOS official [image streams](https://docs.fedoraproject.org/en-US/fedora-coreos/update-streams/) ([#727](https://github.com/poseidon/typhoon/pull/727)) + * Add `os_stream` variable to set the stream to `stable` (default), `testing`, or `next` + * Remove unused `os_image` variable + #### Google -* Use new Fedora CoreOS official [image streams](https://docs.fedoraproject.org/en-US/fedora-coreos/update-streams/) ([#723](https://github.com/poseidon/typhoon/pull/722)) +* Support Fedora CoreOS official [image streams](https://docs.fedoraproject.org/en-US/fedora-coreos/update-streams/) ([#723](https://github.com/poseidon/typhoon/pull/722)) * Add `os_stream` variable to set the stream to `stable` (default), `testing`, or `next` * Deprecate `os_image` variable. Manual image uploads are no longer needed diff --git a/aws/fedora-coreos/kubernetes/ami.tf b/aws/fedora-coreos/kubernetes/ami.tf index e32ce159f..a7ab184bd 100644 --- a/aws/fedora-coreos/kubernetes/ami.tf +++ b/aws/fedora-coreos/kubernetes/ami.tf @@ -13,16 +13,8 @@ data "aws_ami" "fedora-coreos" { values = ["hvm"] } - filter { - name = "name" - values = ["fedora-coreos-31.*.*.*-hvm"] - } - filter { name = "description" - values = ["Fedora CoreOS stable*"] + values = ["Fedora CoreOS ${var.os_stream} *"] } - - # try to filter out dev images (AWS filters can't) - name_regex = "^fedora-coreos-31.[0-9]*.[0-9]*.[0-9]*-hvm*" } diff --git a/aws/fedora-coreos/kubernetes/variables.tf b/aws/fedora-coreos/kubernetes/variables.tf index 13284f634..a34493ec9 100644 --- a/aws/fedora-coreos/kubernetes/variables.tf +++ b/aws/fedora-coreos/kubernetes/variables.tf @@ -41,9 +41,9 @@ variable "worker_type" { default = "t3.small" } -variable "os_image" { +variable "os_stream" { type = string - description = "AMI channel for Fedora CoreOS (not yet used)" + description = "Fedora CoreOs image stream for instances (e.g. stable, testing, next)" default = "stable" } diff --git a/aws/fedora-coreos/kubernetes/workers.tf b/aws/fedora-coreos/kubernetes/workers.tf index e8b57e620..d02e5288c 100644 --- a/aws/fedora-coreos/kubernetes/workers.tf +++ b/aws/fedora-coreos/kubernetes/workers.tf @@ -8,7 +8,7 @@ module "workers" { security_groups = [aws_security_group.worker.id] worker_count = var.worker_count instance_type = var.worker_type - os_image = var.os_image + os_stream = var.os_stream disk_size = var.disk_size spot_price = var.worker_price target_groups = var.worker_target_groups diff --git a/aws/fedora-coreos/kubernetes/workers/ami.tf b/aws/fedora-coreos/kubernetes/workers/ami.tf index e32ce159f..a7ab184bd 100644 --- a/aws/fedora-coreos/kubernetes/workers/ami.tf +++ b/aws/fedora-coreos/kubernetes/workers/ami.tf @@ -13,16 +13,8 @@ data "aws_ami" "fedora-coreos" { values = ["hvm"] } - filter { - name = "name" - values = ["fedora-coreos-31.*.*.*-hvm"] - } - filter { name = "description" - values = ["Fedora CoreOS stable*"] + values = ["Fedora CoreOS ${var.os_stream} *"] } - - # try to filter out dev images (AWS filters can't) - name_regex = "^fedora-coreos-31.[0-9]*.[0-9]*.[0-9]*-hvm*" } diff --git a/aws/fedora-coreos/kubernetes/workers/variables.tf b/aws/fedora-coreos/kubernetes/workers/variables.tf index 6c21d0a05..76b33cbf9 100644 --- a/aws/fedora-coreos/kubernetes/workers/variables.tf +++ b/aws/fedora-coreos/kubernetes/workers/variables.tf @@ -34,9 +34,9 @@ variable "instance_type" { default = "t3.small" } -variable "os_image" { +variable "os_stream" { type = string - description = "AMI channel for Fedora CoreOS (not yet used)" + description = "Fedora CoreOs image stream for instances (e.g. stable, testing, next)" default = "stable" } From a18bd0a7078bfb4d8bc343843e03114ab15121a8 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Wed, 13 May 2020 21:57:09 -0700 Subject: [PATCH 416/523] Highlight SELinux enforcing mode in features --- README.md | 4 ++-- aws/container-linux/kubernetes/README.md | 2 +- aws/fedora-coreos/kubernetes/README.md | 4 ++-- azure/fedora-coreos/kubernetes/README.md | 2 +- bare-metal/fedora-coreos/kubernetes/README.md | 2 +- digital-ocean/fedora-coreos/kubernetes/README.md | 2 +- docs/index.md | 4 ++-- docs/topics/security.md | 3 +++ google-cloud/container-linux/kubernetes/README.md | 2 +- google-cloud/fedora-coreos/kubernetes/README.md | 4 ++-- 10 files changed, 16 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index bc327d909..79c40e4f7 100644 --- a/README.md +++ b/README.md @@ -13,7 +13,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster * Kubernetes v1.18.2 (upstream) * Single or multi-master, [Calico](https://www.projectcalico.org/) or [flannel](https://github.com/coreos/flannel) networking -* On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/) +* On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/), SELinux enforcing * Advanced features like [worker pools](https://typhoon.psdn.io/advanced/worker-pools/), [preemptible](https://typhoon.psdn.io/cl/google-cloud/#preemption) workers, and [snippets](https://typhoon.psdn.io/advanced/customization/#container-linux) customization * Ready for Ingress, Prometheus, Grafana, CSI, or other [addons](https://typhoon.psdn.io/addons/overview/) @@ -64,7 +64,7 @@ Define a Kubernetes cluster by using the Terraform module for your chosen platfo ```tf module "yavin" { - source = "git::https://github.com/poseidon/typhoon//google-cloud/container-linux/kubernetes?ref=v1.18.2" + source = "git::https://github.com/poseidon/typhoon//google-cloud/fedora-coreos/kubernetes?ref=v1.18.2" # Google Cloud cluster_name = "yavin" diff --git a/aws/container-linux/kubernetes/README.md b/aws/container-linux/kubernetes/README.md index b45143847..053433256 100644 --- a/aws/container-linux/kubernetes/README.md +++ b/aws/container-linux/kubernetes/README.md @@ -15,7 +15,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster * Single or multi-master, [Calico](https://www.projectcalico.org/) or [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/) * Advanced features like [worker pools](https://typhoon.psdn.io/advanced/worker-pools/), [spot](https://typhoon.psdn.io/cl/aws/#spot) workers, and [snippets](https://typhoon.psdn.io/advanced/customization/#container-linux) customization -* Ready for Ingress, Prometheus, Grafana, and other optional [addons](https://typhoon.psdn.io/addons/overview/) +* Ready for Ingress, Prometheus, Grafana, CSI, and other optional [addons](https://typhoon.psdn.io/addons/overview/) ## Docs diff --git a/aws/fedora-coreos/kubernetes/README.md b/aws/fedora-coreos/kubernetes/README.md index 3acb9ecd8..a10826b46 100644 --- a/aws/fedora-coreos/kubernetes/README.md +++ b/aws/fedora-coreos/kubernetes/README.md @@ -13,9 +13,9 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster * Kubernetes v1.18.2 (upstream) * Single or multi-master, [Calico](https://www.projectcalico.org/) or [flannel](https://github.com/coreos/flannel) networking -* On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/) +* On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/), SELinux enforcing * Advanced features like [worker pools](https://typhoon.psdn.io/advanced/worker-pools/), [spot](https://typhoon.psdn.io/cl/aws/#spot) workers, and [snippets](https://typhoon.psdn.io/advanced/customization/#container-linux) customization -* Ready for Ingress, Prometheus, Grafana, and other optional [addons](https://typhoon.psdn.io/addons/overview/) +* Ready for Ingress, Prometheus, Grafana, CSI, and other optional [addons](https://typhoon.psdn.io/addons/overview/) ## Docs diff --git a/azure/fedora-coreos/kubernetes/README.md b/azure/fedora-coreos/kubernetes/README.md index 17b80679d..2b493533e 100644 --- a/azure/fedora-coreos/kubernetes/README.md +++ b/azure/fedora-coreos/kubernetes/README.md @@ -13,7 +13,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster * Kubernetes v1.18.2 (upstream) * Single or multi-master, [Calico](https://www.projectcalico.org/) or [flannel](https://github.com/coreos/flannel) networking -* On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/) +* On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/), SELinux enforcing * Advanced features like [worker pools](https://typhoon.psdn.io/advanced/worker-pools/), [spot priority](https://typhoon.psdn.io/fedora-coreos/azure/#low-priority) workers, and [snippets](https://typhoon.psdn.io/advanced/customization/) customization * Ready for Ingress, Prometheus, Grafana, and other optional [addons](https://typhoon.psdn.io/addons/overview/) diff --git a/bare-metal/fedora-coreos/kubernetes/README.md b/bare-metal/fedora-coreos/kubernetes/README.md index d9da6a1c2..70d6e5b6e 100644 --- a/bare-metal/fedora-coreos/kubernetes/README.md +++ b/bare-metal/fedora-coreos/kubernetes/README.md @@ -13,7 +13,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster * Kubernetes v1.18.2 (upstream) * Single or multi-master, [Calico](https://www.projectcalico.org/) or [flannel](https://github.com/coreos/flannel) networking -* On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/) +* On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/), SELinux enforcing * Advanced features like [snippets](https://typhoon.psdn.io/advanced/customization/#container-linux) customization * Ready for Ingress, Prometheus, Grafana, and other optional [addons](https://typhoon.psdn.io/addons/overview/) diff --git a/digital-ocean/fedora-coreos/kubernetes/README.md b/digital-ocean/fedora-coreos/kubernetes/README.md index 94ce6985f..3945c7c8c 100644 --- a/digital-ocean/fedora-coreos/kubernetes/README.md +++ b/digital-ocean/fedora-coreos/kubernetes/README.md @@ -13,7 +13,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster * Kubernetes v1.18.2 (upstream) * Single or multi-master, [Calico](https://www.projectcalico.org/) or [flannel](https://github.com/coreos/flannel) networking -* On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/) +* On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/), SELinux enforcing * Advanced features like [snippets](https://typhoon.psdn.io/advanced/customization/) customization * Ready for Ingress, Prometheus, Grafana, CSI, and other [addons](https://typhoon.psdn.io/addons/overview/) diff --git a/docs/index.md b/docs/index.md index 9d9fedc51..dc0d3fdda 100644 --- a/docs/index.md +++ b/docs/index.md @@ -13,7 +13,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster * Kubernetes v1.18.2 (upstream) * Single or multi-master, [Calico](https://www.projectcalico.org/) or [flannel](https://github.com/coreos/flannel) networking -* On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/) +* On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/), SELinux enforcing * Advanced features like [worker pools](advanced/worker-pools/), [preemptible](cl/google-cloud/#preemption) workers, and [snippets](advanced/customization/#container-linux) customization * Ready for Ingress, Prometheus, Grafana, CSI, or other [addons](addons/overview/) @@ -64,7 +64,7 @@ Define a Kubernetes cluster by using the Terraform module for your chosen platfo ```tf module "yavin" { - source = "git::https://github.com/poseidon/typhoon//google-cloud/container-linux/kubernetes?ref=v1.18.2" + source = "git::https://github.com/poseidon/typhoon//google-cloud/fedora-coreos/kubernetes?ref=v1.18.2" # Google Cloud cluster_name = "yavin" diff --git a/docs/topics/security.md b/docs/topics/security.md index e8949c83d..5542a98fc 100644 --- a/docs/topics/security.md +++ b/docs/topics/security.md @@ -20,6 +20,9 @@ Typhoon aims to be minimal and secure. We're running it ourselves after all. * Container Linux auto-updates are enabled * Hosts limit logins to SSH key-based auth (user "core") +* SELinux enforcing mode [^2] + +[^2]: SELinux is enforcing on Fedora CoreOS, permissive on Flatcar Linux. **Platform** diff --git a/google-cloud/container-linux/kubernetes/README.md b/google-cloud/container-linux/kubernetes/README.md index 1d46bb9f3..1035adfae 100644 --- a/google-cloud/container-linux/kubernetes/README.md +++ b/google-cloud/container-linux/kubernetes/README.md @@ -15,7 +15,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster * Single or multi-master, [Calico](https://www.projectcalico.org/) or [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/) * Advanced features like [worker pools](https://typhoon.psdn.io/advanced/worker-pools/), [preemptible](https://typhoon.psdn.io/cl/google-cloud/#preemption) workers, and [snippets](https://typhoon.psdn.io/advanced/customization/#container-linux) customization -* Ready for Ingress, Prometheus, Grafana, and other optional [addons](https://typhoon.psdn.io/addons/overview/) +* Ready for Ingress, Prometheus, Grafana, CSI, and other optional [addons](https://typhoon.psdn.io/addons/overview/) ## Docs diff --git a/google-cloud/fedora-coreos/kubernetes/README.md b/google-cloud/fedora-coreos/kubernetes/README.md index 6c319b0b2..69c5e585a 100644 --- a/google-cloud/fedora-coreos/kubernetes/README.md +++ b/google-cloud/fedora-coreos/kubernetes/README.md @@ -13,9 +13,9 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster * Kubernetes v1.18.2 (upstream) * Single or multi-master, [Calico](https://www.projectcalico.org/) or [flannel](https://github.com/coreos/flannel) networking -* On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/) +* On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/), SELinux enforcing * Advanced features like [worker pools](https://typhoon.psdn.io/advanced/worker-pools/), [preemptible](https://typhoon.psdn.io/fedora-coreos/google-cloud/#preemption) workers, and [snippets](https://typhoon.psdn.io/advanced/customization/) customization -* Ready for Ingress, Prometheus, Grafana, and other optional [addons](https://typhoon.psdn.io/addons/overview/) +* Ready for Ingress, Prometheus, Grafana, CSI, and other optional [addons](https://typhoon.psdn.io/addons/overview/) ## Docs From 70e389f37fe6c7446cc09ce9368eb02e977f4c96 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Sat, 9 May 2020 17:37:35 -0700 Subject: [PATCH 417/523] Restore use of Flatcar Linux Azure Marketplace image * Switch Flatcar Linux Azure to use the Marketplace image from Kinvolk (offer `flatcar-container-linux-free`) * Accepting Azure Marketplace terms is still neccessary, update docs to show accepting the free offer rather than BYOL * Upstream Flatcar: https://github.com/flatcar-linux/Flatcar/issues/82 * Typhoon: https://github.com/poseidon/typhoon/issues/703 --- CHANGES.md | 9 ++++ .../container-linux/kubernetes/controllers.tf | 22 ++++++---- azure/container-linux/kubernetes/variables.tf | 3 +- .../kubernetes/workers/variables.tf | 2 +- .../kubernetes/workers/workers.tf | 22 ++++++---- docs/advanced/worker-pools.md | 2 +- docs/cl/azure.md | 43 +++---------------- 7 files changed, 46 insertions(+), 57 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index f860e298e..7f1fb62bd 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -32,6 +32,15 @@ Notable changes between versions. * Add `os_stream` variable to set the stream to `stable` (default), `testing`, or `next` * Deprecate `os_image` variable. Manual image uploads are no longer needed + +### Flatcar Linux + +#### Azure + +* Use the Flatcar Linux Azure Marketplace image + * Restore [#664](https://github.com/poseidon/typhoon/pull/664) (reverted in [#707](https://github.com/poseidon/typhoon/pull/707)) but use Flatcar Linux new free offer (not byol) +* Change `os_image` to use a `flatcar-stable` default + #### Addons * Update nginx-ingress from v0.30.0 to [v0.32.0](https://github.com/kubernetes/ingress-nginx/releases/tag/nginx-0.32.0) diff --git a/azure/container-linux/kubernetes/controllers.tf b/azure/container-linux/kubernetes/controllers.tf index 28adcbc5a..cec707158 100644 --- a/azure/container-linux/kubernetes/controllers.tf +++ b/azure/container-linux/kubernetes/controllers.tf @@ -53,18 +53,24 @@ resource "azurerm_linux_virtual_machine" "controllers" { storage_account_type = "Premium_LRS" } - // CoreOS Container Linux or Flatcar Container Linux (manual upload) - dynamic "source_image_reference" { - for_each = local.flavor == "coreos" ? [1] : [] + # CoreOS Container Linux or Flatcar Container Linux + source_image_reference { + publisher = local.flavor == "flatcar" ? "Kinvolk" : "CoreOS" + offer = local.flavor == "flatcar" ? "flatcar-container-linux-free" : "CoreOS" + sku = local.channel + version = "latest" + } + + # Gross hack for Flatcar Linux + dynamic "plan" { + for_each = local.flavor == "flatcar" ? [1] : [] content { - publisher = "CoreOS" - offer = "CoreOS" - sku = local.channel - version = "latest" + name = local.channel + publisher = "kinvolk" + product = "flatcar-container-linux-free" } } - source_image_id = local.flavor == "coreos" ? null : var.os_image # network network_interface_ids = [ diff --git a/azure/container-linux/kubernetes/variables.tf b/azure/container-linux/kubernetes/variables.tf index a31e20063..50b57aed3 100644 --- a/azure/container-linux/kubernetes/variables.tf +++ b/azure/container-linux/kubernetes/variables.tf @@ -48,7 +48,8 @@ variable "worker_type" { variable "os_image" { type = string - description = "Channel for a Container Linux derivative (/subscriptions/some-flatcar-upload, coreos-stable, coreos-beta, coreos-alpha)" + description = "Channel for a Container Linux derivative (flatcar-stable, flatcar-beta, flatcar-alpha, flatcar-edge, coreos-stable, coreos-beta, coreos-alpha)" + default = "flatcar-stable" } variable "disk_size" { diff --git a/azure/container-linux/kubernetes/workers/variables.tf b/azure/container-linux/kubernetes/workers/variables.tf index 0ebd606fe..48197d3ef 100644 --- a/azure/container-linux/kubernetes/workers/variables.tf +++ b/azure/container-linux/kubernetes/workers/variables.tf @@ -46,7 +46,7 @@ variable "vm_type" { variable "os_image" { type = string - description = "Channel for a Container Linux derivative (flatcar-stable, flatcar-beta, coreos-stable, coreos-beta, coreos-alpha)" + description = "Channel for a Container Linux derivative (flatcar-stable, flatcar-beta, flatcar-alpha, flatcar-edge, coreos-stable, coreos-beta, coreos-alpha)" default = "flatcar-stable" } diff --git a/azure/container-linux/kubernetes/workers/workers.tf b/azure/container-linux/kubernetes/workers/workers.tf index 6b626eddb..f8eaa8084 100644 --- a/azure/container-linux/kubernetes/workers/workers.tf +++ b/azure/container-linux/kubernetes/workers/workers.tf @@ -24,18 +24,24 @@ resource "azurerm_linux_virtual_machine_scale_set" "workers" { caching = "ReadWrite" } - // CoreOS Container Linux or Flatcar Container Linux (manual upload) - dynamic "source_image_reference" { - for_each = local.flavor == "coreos" ? [1] : [] + # CoreOS Container Linux or Flatcar Container Linux + source_image_reference { + publisher = local.flavor == "flatcar" ? "Kinvolk" : "CoreOS" + offer = local.flavor == "flatcar" ? "flatcar-container-linux-free" : "CoreOS" + sku = local.channel + version = "latest" + } + + # Gross hack for Flatcar Linux + dynamic "plan" { + for_each = local.flavor == "flatcar" ? [1] : [] content { - publisher = "CoreOS" - offer = "CoreOS" - sku = local.channel - version = "latest" + name = local.channel + publisher = "kinvolk" + product = "flatcar-container-linux-free" } } - source_image_id = local.flavor == "coreos" ? null : var.os_image # Azure requires setting admin_ssh_key, though Ignition custom_data handles it too admin_username = "core" diff --git a/docs/advanced/worker-pools.md b/docs/advanced/worker-pools.md index 4162ca812..6face16f0 100644 --- a/docs/advanced/worker-pools.md +++ b/docs/advanced/worker-pools.md @@ -134,7 +134,7 @@ The Azure internal `workers` module supports a number of [variables](https://git |:-----|:------------|:--------|:--------| | worker_count | Number of instances | 1 | 3 | | vm_type | Machine type for instances | "Standard_DS1_v2" | See below | -| os_image | Channel for a Container Linux derivative | "flatcar-stable" | flatcar-stable, flatcar-beta, coreos-stable, coreos-beta, coreos-alpha | +| os_image | Channel for a Container Linux derivative | "flatcar-stable" | flatcar-stable, flatcar-beta, flatcar-alpha, flatcar-edge, coreos-stable, coreos-beta, coreos-alpha | | priority | Set priority to Spot to use reduced cost surplus capacity, with the tradeoff that instances can be deallocated at any time | "Regular" | "Spot" | | snippets | Container Linux Config snippets | [] | [examples](/advanced/customization/) | | service_cidr | CIDR IPv4 range to assign to Kubernetes services | "10.3.0.0/16" | "10.3.0.0/24" | diff --git a/docs/cl/azure.md b/docs/cl/azure.md index 6c67c209d..ec69ec9e8 100644 --- a/docs/cl/azure.md +++ b/docs/cl/azure.md @@ -57,38 +57,15 @@ provider "ct" { Additional configuration options are described in the `azurerm` provider [docs](https://www.terraform.io/docs/providers/azurerm/). -### Flatcar Linux Images +## Flatcar Linux Images -Flatcar Linux publishes images for Azure. Azure allows custom images to be uploaded to a storage account bucket and imported. - -[Download](https://www.flatcar-linux.org/releases/) a Flatcar Linux Azure VHD image and upload it to an Azure storage account container (i.e. bucket). - -Azure requires fixed VHDs and Flatcar Linux provides dynamic VHDs, so uploads require Azure tools and cannot be done through the UI. Azure's tool compilation requires old versions, so Flatcar Linux has packaged a container image you may choose to use. See their [docs](https://docs.flatcar-linux.org/os/booting-on-azure/#uploading-your-own-image). - -``` -bzip2 -d flatcar_production_azure_image.vhd.bz2 -``` - -``` -podman run -it --entrypoint=/bin/bash quay.io/kinvolk/azure-flatcar-image-upload -... - -# az login -# az storage account keys list --resource-group GROUP --account-name BUCKET | jq -r '.[0].value' -# azure-vhd-utils upload --localvhdpath /data/flatcar_production_azure_image.vhd --stgaccountname BUCKET --containername flatcar-linux --blobname flatcar-stable-2345.3.1 --stgaccountkey "KEYFROMABOVE" -# exit -``` - -Create an Azure disk (note disk ID) and create an Azure image from it (note image ID). +Flatcar Linux publishes images to the Azure Marketplace and requires accepting terms. ``` -az disk create --name flatcar-stable-2345.3.1 -g GROUP --source https://BUCKET.blob.core.windows.net/flatcar-linux/flatcar_production_azure_image.vhd - -az image create --name flatcar-stable-2345.3.1 -g GROUP --os-type=linux --source /subscriptions/some/path/providers/Microsoft.Compute/disks/flatcar-stable-2345.3.1 +az vm image terms show --publish kinvolk --offer flatcar-container-linux-free --plan stable +az vm image terms accept --publish kinvolk --offer flatcar-container-linux-free --plan stable ``` -Set the [os_image](#variables) in the next step. - ## Cluster Define a Kubernetes cluster using the module `azure/container-linux/kubernetes`. @@ -104,7 +81,6 @@ module "ramius" { dns_zone_group = "example-group" # configuration - os_image = "/subscriptions/some/path/Microsoft.Compute/images/flatcar-stable-2345.3.1" ssh_authorized_key = "ssh-rsa AAAAB3Nz..." # optional @@ -115,15 +91,6 @@ module "ramius" { Reference the [variables docs](#variables) or the [variables.tf](https://github.com/poseidon/typhoon/blob/master/azure/container-linux/kubernetes/variables.tf) source. -### Flatcar Linux Only - -Flatcar Linux publishes images to the Azure Marketplace and requires accepting their legal terms. - -``` -az vm image terms show --publish kinvolk --offer flatcar-container-linux --plan stable -az vm image terms accept --publish kinvolk --offer flatcar-container-linux --plan stable -``` - ## ssh-agent Initial bootstrapping requires `bootstrap.service` be started on one controller node. Terraform uses `ssh-agent` to automate this step. Add your SSH private key to `ssh-agent`. @@ -218,7 +185,6 @@ Check the [variables.tf](https://github.com/poseidon/typhoon/blob/master/azure/c | region | Azure region | "centralus" | | dns_zone | Azure DNS zone | "azure.example.com" | | dns_zone_group | Resource group where the Azure DNS zone resides | "global" | -| os_image | Container Linux image for instances | "/subscriptions/..../some-flatcar-image", coreos-stable, coreos-beta, coreos-alpha | | ssh_authorized_key | SSH public key for user 'core' | "ssh-rsa AAAAB3NZ..." | !!! tip @@ -259,6 +225,7 @@ Reference the DNS zone with `azurerm_dns_zone.clusters.name` and its resource gr | worker_count | Number of workers | 1 | 3 | | controller_type | Machine type for controllers | "Standard_B2s" | See below | | worker_type | Machine type for workers | "Standard_DS1_v2" | See below | +| os_image | Channel for a Container Linux derivative | "flatcar-stable" | flatcar-stable, flatcar-beta, flatcar-alpha, flatcar-edge, coreos-stable, coreos-beta, coreos-alpha | | disk_size | Size of the disk in GB | 40 | 100 | | worker_priority | Set priority to Spot to use reduced cost surplus capacity, with the tradeoff that instances can be deallocated at any time | Regular | Spot | | controller_snippets | Controller Container Linux Config snippets | [] | [example](/advanced/customization/#usage) | From d952576d2f3125705ebc8e6c17f8d3b0fe6f13a9 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Fri, 15 May 2020 17:38:59 -0700 Subject: [PATCH 418/523] Update Grafana from v7.0.0-beta3 to v7.0.0 * https://github.com/grafana/grafana/releases/tag/7.0.0 --- CHANGES.md | 5 ++--- addons/grafana/deployment.yaml | 2 +- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 7f1fb62bd..1c0f91ce1 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -32,7 +32,6 @@ Notable changes between versions. * Add `os_stream` variable to set the stream to `stable` (default), `testing`, or `next` * Deprecate `os_image` variable. Manual image uploads are no longer needed - ### Flatcar Linux #### Azure @@ -41,12 +40,12 @@ Notable changes between versions. * Restore [#664](https://github.com/poseidon/typhoon/pull/664) (reverted in [#707](https://github.com/poseidon/typhoon/pull/707)) but use Flatcar Linux new free offer (not byol) * Change `os_image` to use a `flatcar-stable` default -#### Addons +### Addons * Update nginx-ingress from v0.30.0 to [v0.32.0](https://github.com/kubernetes/ingress-nginx/releases/tag/nginx-0.32.0) * Add support for [IngressClass](https://kubernetes.io/docs/concepts/services-networking/ingress/#ingress-class) * Update Prometheus from v2.17.1 to v2.18.1 -* Update Grafana from v6.7.2 to v7.0.0-beta3 +* Update Grafana from v6.7.2 to v7.0.0 ## v1.18.2 diff --git a/addons/grafana/deployment.yaml b/addons/grafana/deployment.yaml index 50024813f..47262b0f1 100644 --- a/addons/grafana/deployment.yaml +++ b/addons/grafana/deployment.yaml @@ -23,7 +23,7 @@ spec: spec: containers: - name: grafana - image: docker.io/grafana/grafana:7.0.0-beta3 + image: docker.io/grafana/grafana:7.0.0 env: - name: GF_PATHS_CONFIG value: "/etc/grafana/custom.ini" From a927c7c7908080be288120d417d02ae784b1e140 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Fri, 15 May 2020 17:42:24 -0700 Subject: [PATCH 419/523] Update kube-state-metrics from v1.9.5 to v1.9.6 * https://github.com/kubernetes/kube-state-metrics/releases/tag/v1.9.6 --- CHANGES.md | 1 + addons/prometheus/exporters/kube-state-metrics/deployment.yaml | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index 1c0f91ce1..795ad63b5 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -45,6 +45,7 @@ Notable changes between versions. * Update nginx-ingress from v0.30.0 to [v0.32.0](https://github.com/kubernetes/ingress-nginx/releases/tag/nginx-0.32.0) * Add support for [IngressClass](https://kubernetes.io/docs/concepts/services-networking/ingress/#ingress-class) * Update Prometheus from v2.17.1 to v2.18.1 + * Update kube-state-metrics from v1.9.5 to [v1.9.6](https://github.com/kubernetes/kube-state-metrics/releases/tag/v1.9.6) * Update Grafana from v6.7.2 to v7.0.0 ## v1.18.2 diff --git a/addons/prometheus/exporters/kube-state-metrics/deployment.yaml b/addons/prometheus/exporters/kube-state-metrics/deployment.yaml index bf9274fe9..d05f811a8 100644 --- a/addons/prometheus/exporters/kube-state-metrics/deployment.yaml +++ b/addons/prometheus/exporters/kube-state-metrics/deployment.yaml @@ -24,7 +24,7 @@ spec: serviceAccountName: kube-state-metrics containers: - name: kube-state-metrics - image: quay.io/coreos/kube-state-metrics:v1.9.5 + image: quay.io/coreos/kube-state-metrics:v1.9.6 ports: - name: metrics containerPort: 8080 From 90edcd3d777913b227a45467f0f697a0dc876572 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Fri, 15 May 2020 18:03:19 -0700 Subject: [PATCH 420/523] Update node-exporter from v1.0.0-rc.0 to v1.0.0-rc.1 * https://github.com/prometheus/node_exporter/releases/tag/v1.0.0-rc.1 --- CHANGES.md | 1 + addons/prometheus/exporters/node-exporter/daemonset.yaml | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index 795ad63b5..309ef7a45 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -46,6 +46,7 @@ Notable changes between versions. * Add support for [IngressClass](https://kubernetes.io/docs/concepts/services-networking/ingress/#ingress-class) * Update Prometheus from v2.17.1 to v2.18.1 * Update kube-state-metrics from v1.9.5 to [v1.9.6](https://github.com/kubernetes/kube-state-metrics/releases/tag/v1.9.6) + * Update node-exporter from v1.0.0-rc.0 to [v1.0.0-rc.1](https://github.com/prometheus/node_exporter/releases/tag/v1.0.0-rc.1) * Update Grafana from v6.7.2 to v7.0.0 ## v1.18.2 diff --git a/addons/prometheus/exporters/node-exporter/daemonset.yaml b/addons/prometheus/exporters/node-exporter/daemonset.yaml index da3f723af..3b4199f0a 100644 --- a/addons/prometheus/exporters/node-exporter/daemonset.yaml +++ b/addons/prometheus/exporters/node-exporter/daemonset.yaml @@ -28,7 +28,7 @@ spec: hostPID: true containers: - name: node-exporter - image: quay.io/prometheus/node-exporter:v1.0.0-rc.0 + image: quay.io/prometheus/node-exporter:v1.0.0-rc.1 args: - --path.procfs=/host/proc - --path.sysfs=/host/sys From 2578be1f96aa6be3aa4a612afe60a3539a583c04 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Sat, 16 May 2020 12:32:10 -0700 Subject: [PATCH 421/523] Rollback Grafana to v7.0.0-beta3, v7.0.0 image is missing * Grafana hasn't published the v7.0.0 image yet --- addons/grafana/deployment.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/addons/grafana/deployment.yaml b/addons/grafana/deployment.yaml index 47262b0f1..50024813f 100644 --- a/addons/grafana/deployment.yaml +++ b/addons/grafana/deployment.yaml @@ -23,7 +23,7 @@ spec: spec: containers: - name: grafana - image: docker.io/grafana/grafana:7.0.0 + image: docker.io/grafana/grafana:7.0.0-beta3 env: - name: GF_PATHS_CONFIG value: "/etc/grafana/custom.ini" From ff4187a1fbb766f961901ef1c4fe2d91da5895cf Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Mon, 18 May 2020 23:25:30 -0700 Subject: [PATCH 422/523] Use new Azure subnet to set address_prefixes list * Update Azure subnet `address_prefix` to `azure_prefixes` list * Fix warning that `address_prefix` is deprecated * Require `terraform-provider-azurerm` v2.8.0+ (action required) Rel: https://github.com/terraform-providers/terraform-provider-azurerm/pull/6493 --- CHANGES.md | 6 ++++++ azure/container-linux/kubernetes/controllers.tf | 4 ++-- azure/container-linux/kubernetes/network.tf | 4 ++-- azure/container-linux/kubernetes/versions.tf | 2 +- azure/container-linux/kubernetes/workers/workers.tf | 4 ++-- azure/fedora-coreos/kubernetes/network.tf | 4 ++-- azure/fedora-coreos/kubernetes/versions.tf | 2 +- 7 files changed, 16 insertions(+), 10 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 309ef7a45..e33644cc2 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -14,6 +14,12 @@ Notable changes between versions. * Fix Terraform plan error when `controller_count` exceeds AWS zones (e.g. 5 controllers) ([#714](https://github.com/poseidon/typhoon/pull/714)) * Regressed in v1.17.1 ([#605](https://github.com/poseidon/typhoon/pull/605)) +### Azure + +* Update Azure subnets to set `address_prefixes` list ([#730](https://github.com/poseidon/typhoon/pull/730)) + * Fix warning that `address_prefix` is deprecated + * Require `terraform-provider-azurerm` v2.8.0+ (action required) + ### Fedora CoreOS * Fix Calico `install-cni` crashloop on Pod restarts ([#724](https://github.com/poseidon/typhoon/pull/724)) diff --git a/azure/container-linux/kubernetes/controllers.tf b/azure/container-linux/kubernetes/controllers.tf index cec707158..ad329b0f7 100644 --- a/azure/container-linux/kubernetes/controllers.tf +++ b/azure/container-linux/kubernetes/controllers.tf @@ -66,9 +66,9 @@ resource "azurerm_linux_virtual_machine" "controllers" { for_each = local.flavor == "flatcar" ? [1] : [] content { - name = local.channel + name = local.channel publisher = "kinvolk" - product = "flatcar-container-linux-free" + product = "flatcar-container-linux-free" } } diff --git a/azure/container-linux/kubernetes/network.tf b/azure/container-linux/kubernetes/network.tf index ea92a5a7d..562156117 100644 --- a/azure/container-linux/kubernetes/network.tf +++ b/azure/container-linux/kubernetes/network.tf @@ -21,7 +21,7 @@ resource "azurerm_subnet" "controller" { name = "controller" virtual_network_name = azurerm_virtual_network.network.name - address_prefix = cidrsubnet(var.host_cidr, 1, 0) + address_prefixes = [cidrsubnet(var.host_cidr, 1, 0)] } resource "azurerm_subnet_network_security_group_association" "controller" { @@ -34,7 +34,7 @@ resource "azurerm_subnet" "worker" { name = "worker" virtual_network_name = azurerm_virtual_network.network.name - address_prefix = cidrsubnet(var.host_cidr, 1, 1) + address_prefixes = [cidrsubnet(var.host_cidr, 1, 1)] } resource "azurerm_subnet_network_security_group_association" "worker" { diff --git a/azure/container-linux/kubernetes/versions.tf b/azure/container-linux/kubernetes/versions.tf index f9653cab3..09f4f3e2d 100644 --- a/azure/container-linux/kubernetes/versions.tf +++ b/azure/container-linux/kubernetes/versions.tf @@ -3,7 +3,7 @@ terraform { required_version = "~> 0.12.6" required_providers { - azurerm = "~> 2.0" + azurerm = "~> 2.8" ct = "~> 0.3" template = "~> 2.1" null = "~> 2.1" diff --git a/azure/container-linux/kubernetes/workers/workers.tf b/azure/container-linux/kubernetes/workers/workers.tf index f8eaa8084..025045541 100644 --- a/azure/container-linux/kubernetes/workers/workers.tf +++ b/azure/container-linux/kubernetes/workers/workers.tf @@ -37,9 +37,9 @@ resource "azurerm_linux_virtual_machine_scale_set" "workers" { for_each = local.flavor == "flatcar" ? [1] : [] content { - name = local.channel + name = local.channel publisher = "kinvolk" - product = "flatcar-container-linux-free" + product = "flatcar-container-linux-free" } } diff --git a/azure/fedora-coreos/kubernetes/network.tf b/azure/fedora-coreos/kubernetes/network.tf index ea92a5a7d..562156117 100644 --- a/azure/fedora-coreos/kubernetes/network.tf +++ b/azure/fedora-coreos/kubernetes/network.tf @@ -21,7 +21,7 @@ resource "azurerm_subnet" "controller" { name = "controller" virtual_network_name = azurerm_virtual_network.network.name - address_prefix = cidrsubnet(var.host_cidr, 1, 0) + address_prefixes = [cidrsubnet(var.host_cidr, 1, 0)] } resource "azurerm_subnet_network_security_group_association" "controller" { @@ -34,7 +34,7 @@ resource "azurerm_subnet" "worker" { name = "worker" virtual_network_name = azurerm_virtual_network.network.name - address_prefix = cidrsubnet(var.host_cidr, 1, 1) + address_prefixes = [cidrsubnet(var.host_cidr, 1, 1)] } resource "azurerm_subnet_network_security_group_association" "worker" { diff --git a/azure/fedora-coreos/kubernetes/versions.tf b/azure/fedora-coreos/kubernetes/versions.tf index f9653cab3..09f4f3e2d 100644 --- a/azure/fedora-coreos/kubernetes/versions.tf +++ b/azure/fedora-coreos/kubernetes/versions.tf @@ -3,7 +3,7 @@ terraform { required_version = "~> 0.12.6" required_providers { - azurerm = "~> 2.0" + azurerm = "~> 2.8" ct = "~> 0.3" template = "~> 2.1" null = "~> 2.1" From 3bdddc452c0832e86d9320d61409f48a4cb8ed00 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Mon, 18 May 2020 23:42:32 -0700 Subject: [PATCH 423/523] Update Grafana from v7.0.0-beta2 to v7.0.0 * https://grafana.com/docs/grafana/latest/guides/whats-new-in-v7-0/ --- CHANGES.md | 2 +- addons/grafana/deployment.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index e33644cc2..5b7d9b878 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -53,7 +53,7 @@ Notable changes between versions. * Update Prometheus from v2.17.1 to v2.18.1 * Update kube-state-metrics from v1.9.5 to [v1.9.6](https://github.com/kubernetes/kube-state-metrics/releases/tag/v1.9.6) * Update node-exporter from v1.0.0-rc.0 to [v1.0.0-rc.1](https://github.com/prometheus/node_exporter/releases/tag/v1.0.0-rc.1) -* Update Grafana from v6.7.2 to v7.0.0 +* Update Grafana from v6.7.2 to [v7.0.0](https://grafana.com/docs/grafana/latest/guides/whats-new-in-v7-0/) ## v1.18.2 diff --git a/addons/grafana/deployment.yaml b/addons/grafana/deployment.yaml index 50024813f..47262b0f1 100644 --- a/addons/grafana/deployment.yaml +++ b/addons/grafana/deployment.yaml @@ -23,7 +23,7 @@ spec: spec: containers: - name: grafana - image: docker.io/grafana/grafana:7.0.0-beta3 + image: docker.io/grafana/grafana:7.0.0 env: - name: GF_PATHS_CONFIG value: "/etc/grafana/custom.ini" From 8d024d22ad400fd6e8137cd542ed0232a2b633d1 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Mon, 18 May 2020 23:50:46 -0700 Subject: [PATCH 424/523] Update etcd from v3.4.7 to v3.4.8 * https://github.com/etcd-io/etcd/blob/master/CHANGELOG-3.4.md#v348-2020-05-18 --- aws/container-linux/kubernetes/cl/controller.yaml | 2 +- aws/fedora-coreos/kubernetes/fcc/controller.yaml | 2 +- azure/container-linux/kubernetes/cl/controller.yaml | 2 +- azure/fedora-coreos/kubernetes/fcc/controller.yaml | 2 +- bare-metal/container-linux/kubernetes/cl/controller.yaml | 2 +- bare-metal/fedora-coreos/kubernetes/fcc/controller.yaml | 2 +- digital-ocean/container-linux/kubernetes/cl/controller.yaml | 2 +- digital-ocean/fedora-coreos/kubernetes/fcc/controller.yaml | 2 +- google-cloud/container-linux/kubernetes/cl/controller.yaml | 2 +- google-cloud/fedora-coreos/kubernetes/fcc/controller.yaml | 2 +- 10 files changed, 10 insertions(+), 10 deletions(-) diff --git a/aws/container-linux/kubernetes/cl/controller.yaml b/aws/container-linux/kubernetes/cl/controller.yaml index 2506dd466..b08297c8b 100644 --- a/aws/container-linux/kubernetes/cl/controller.yaml +++ b/aws/container-linux/kubernetes/cl/controller.yaml @@ -7,7 +7,7 @@ systemd: - name: 40-etcd-cluster.conf contents: | [Service] - Environment="ETCD_IMAGE_TAG=v3.4.7" + Environment="ETCD_IMAGE_TAG=v3.4.8" Environment="ETCD_IMAGE_URL=docker://quay.io/coreos/etcd" Environment="RKT_RUN_ARGS=--insecure-options=image" Environment="ETCD_NAME=${etcd_name}" diff --git a/aws/fedora-coreos/kubernetes/fcc/controller.yaml b/aws/fedora-coreos/kubernetes/fcc/controller.yaml index c7474e10d..d06663239 100644 --- a/aws/fedora-coreos/kubernetes/fcc/controller.yaml +++ b/aws/fedora-coreos/kubernetes/fcc/controller.yaml @@ -28,7 +28,7 @@ systemd: --network host \ --volume /var/lib/etcd:/var/lib/etcd:rw,Z \ --volume /etc/ssl/etcd:/etc/ssl/certs:ro,Z \ - quay.io/coreos/etcd:v3.4.7 + quay.io/coreos/etcd:v3.4.8 ExecStop=/usr/bin/podman stop etcd [Install] WantedBy=multi-user.target diff --git a/azure/container-linux/kubernetes/cl/controller.yaml b/azure/container-linux/kubernetes/cl/controller.yaml index aa0e6941e..17357966a 100644 --- a/azure/container-linux/kubernetes/cl/controller.yaml +++ b/azure/container-linux/kubernetes/cl/controller.yaml @@ -7,7 +7,7 @@ systemd: - name: 40-etcd-cluster.conf contents: | [Service] - Environment="ETCD_IMAGE_TAG=v3.4.7" + Environment="ETCD_IMAGE_TAG=v3.4.8" Environment="ETCD_IMAGE_URL=docker://quay.io/coreos/etcd" Environment="RKT_RUN_ARGS=--insecure-options=image" Environment="ETCD_NAME=${etcd_name}" diff --git a/azure/fedora-coreos/kubernetes/fcc/controller.yaml b/azure/fedora-coreos/kubernetes/fcc/controller.yaml index 035667c04..aaedb1611 100644 --- a/azure/fedora-coreos/kubernetes/fcc/controller.yaml +++ b/azure/fedora-coreos/kubernetes/fcc/controller.yaml @@ -28,7 +28,7 @@ systemd: --network host \ --volume /var/lib/etcd:/var/lib/etcd:rw,Z \ --volume /etc/ssl/etcd:/etc/ssl/certs:ro,Z \ - quay.io/coreos/etcd:v3.4.7 + quay.io/coreos/etcd:v3.4.8 ExecStop=/usr/bin/podman stop etcd [Install] WantedBy=multi-user.target diff --git a/bare-metal/container-linux/kubernetes/cl/controller.yaml b/bare-metal/container-linux/kubernetes/cl/controller.yaml index 9f5c5f562..f4f0e4617 100644 --- a/bare-metal/container-linux/kubernetes/cl/controller.yaml +++ b/bare-metal/container-linux/kubernetes/cl/controller.yaml @@ -7,7 +7,7 @@ systemd: - name: 40-etcd-cluster.conf contents: | [Service] - Environment="ETCD_IMAGE_TAG=v3.4.7" + Environment="ETCD_IMAGE_TAG=v3.4.8" Environment="ETCD_IMAGE_URL=docker://quay.io/coreos/etcd" Environment="RKT_RUN_ARGS=--insecure-options=image" Environment="ETCD_NAME=${etcd_name}" diff --git a/bare-metal/fedora-coreos/kubernetes/fcc/controller.yaml b/bare-metal/fedora-coreos/kubernetes/fcc/controller.yaml index 14cbfdb8b..b8b068113 100644 --- a/bare-metal/fedora-coreos/kubernetes/fcc/controller.yaml +++ b/bare-metal/fedora-coreos/kubernetes/fcc/controller.yaml @@ -28,7 +28,7 @@ systemd: --network host \ --volume /var/lib/etcd:/var/lib/etcd:rw,Z \ --volume /etc/ssl/etcd:/etc/ssl/certs:ro,Z \ - quay.io/coreos/etcd:v3.4.7 + quay.io/coreos/etcd:v3.4.8 ExecStop=/usr/bin/podman stop etcd [Install] WantedBy=multi-user.target diff --git a/digital-ocean/container-linux/kubernetes/cl/controller.yaml b/digital-ocean/container-linux/kubernetes/cl/controller.yaml index 08f653723..254e22de7 100644 --- a/digital-ocean/container-linux/kubernetes/cl/controller.yaml +++ b/digital-ocean/container-linux/kubernetes/cl/controller.yaml @@ -7,7 +7,7 @@ systemd: - name: 40-etcd-cluster.conf contents: | [Service] - Environment="ETCD_IMAGE_TAG=v3.4.7" + Environment="ETCD_IMAGE_TAG=v3.4.8" Environment="ETCD_IMAGE_URL=docker://quay.io/coreos/etcd" Environment="RKT_RUN_ARGS=--insecure-options=image" Environment="ETCD_NAME=${etcd_name}" diff --git a/digital-ocean/fedora-coreos/kubernetes/fcc/controller.yaml b/digital-ocean/fedora-coreos/kubernetes/fcc/controller.yaml index 67106af2d..9fde71b6a 100644 --- a/digital-ocean/fedora-coreos/kubernetes/fcc/controller.yaml +++ b/digital-ocean/fedora-coreos/kubernetes/fcc/controller.yaml @@ -28,7 +28,7 @@ systemd: --network host \ --volume /var/lib/etcd:/var/lib/etcd:rw,Z \ --volume /etc/ssl/etcd:/etc/ssl/certs:ro,Z \ - quay.io/coreos/etcd:v3.4.7 + quay.io/coreos/etcd:v3.4.8 ExecStop=/usr/bin/podman stop etcd [Install] WantedBy=multi-user.target diff --git a/google-cloud/container-linux/kubernetes/cl/controller.yaml b/google-cloud/container-linux/kubernetes/cl/controller.yaml index 30fbee4d9..2c8f5d5f0 100644 --- a/google-cloud/container-linux/kubernetes/cl/controller.yaml +++ b/google-cloud/container-linux/kubernetes/cl/controller.yaml @@ -7,7 +7,7 @@ systemd: - name: 40-etcd-cluster.conf contents: | [Service] - Environment="ETCD_IMAGE_TAG=v3.4.7" + Environment="ETCD_IMAGE_TAG=v3.4.8" Environment="ETCD_IMAGE_URL=docker://quay.io/coreos/etcd" Environment="RKT_RUN_ARGS=--insecure-options=image" Environment="ETCD_NAME=${etcd_name}" diff --git a/google-cloud/fedora-coreos/kubernetes/fcc/controller.yaml b/google-cloud/fedora-coreos/kubernetes/fcc/controller.yaml index c7474e10d..d06663239 100644 --- a/google-cloud/fedora-coreos/kubernetes/fcc/controller.yaml +++ b/google-cloud/fedora-coreos/kubernetes/fcc/controller.yaml @@ -28,7 +28,7 @@ systemd: --network host \ --volume /var/lib/etcd:/var/lib/etcd:rw,Z \ --volume /etc/ssl/etcd:/etc/ssl/certs:ro,Z \ - quay.io/coreos/etcd:v3.4.7 + quay.io/coreos/etcd:v3.4.8 ExecStop=/usr/bin/podman stop etcd [Install] WantedBy=multi-user.target From 09eb208b4e9858827b130880ed64d0da295379a8 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Tue, 19 May 2020 21:41:51 -0700 Subject: [PATCH 425/523] Fix Fedora CoreOS on GCP proposing controller recreate * With Fedora CoreOS image stream support (#727), the latest resolved image will change over the lifecycle of a cluster. * Fix issue where an image diff proposed replacing a Fedora CoreOS controller on GCP, introduced in #727 (unreleased) * Also ignore image diffs to the GCP managed instance group of workers. This aligns with worker AMI diffs being ignored on AWS and similar on Azure, since workers update themselves. Background: * Controller nodes should strictly not be recreated by Terraform, they are stateful (etcd) and should not be replaced * Across cloud platforms, OS image diffs are ignored since both Flatcar Linux and Fedora CoreOS nodes update themselves. For workers, user-data or disk size diffs (where relevant) are allowed to recreate workers templates/configs since these are considered to be user-initiated declarations that a reprovision should be done --- google-cloud/fedora-coreos/kubernetes/controllers.tf | 5 ++++- google-cloud/fedora-coreos/kubernetes/workers/workers.tf | 3 +++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/google-cloud/fedora-coreos/kubernetes/controllers.tf b/google-cloud/fedora-coreos/kubernetes/controllers.tf index 81e20ed0b..3dc96a3e9 100644 --- a/google-cloud/fedora-coreos/kubernetes/controllers.tf +++ b/google-cloud/fedora-coreos/kubernetes/controllers.tf @@ -59,7 +59,10 @@ resource "google_compute_instance" "controllers" { tags = ["${var.cluster_name}-controller"] lifecycle { - ignore_changes = [metadata] + ignore_changes = [ + metadata, + boot_disk[0].initialize_params + ] } } diff --git a/google-cloud/fedora-coreos/kubernetes/workers/workers.tf b/google-cloud/fedora-coreos/kubernetes/workers/workers.tf index c6620afe5..78367c673 100644 --- a/google-cloud/fedora-coreos/kubernetes/workers/workers.tf +++ b/google-cloud/fedora-coreos/kubernetes/workers/workers.tf @@ -64,6 +64,9 @@ resource "google_compute_instance_template" "worker" { } lifecycle { + ignore_changes = [ + disk[0].source_image + ] # To update an Instance Template, Terraform should replace the existing resource create_before_destroy = true } From 47605433564a2b586f443ef2a6e308490861657b Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Tue, 19 May 2020 22:39:53 -0700 Subject: [PATCH 426/523] Set Kubelet image via kubelet.service KUBELET_IMAGE * Write the systemd kubelet.service to use `KUBELET_IMAGE` as the Kubelet. This provides a nice way to use systemd dropins to temporarily override the image (e.g. during a registry outage) Note: Only Typhoon Kubelet images and registries are supported. --- aws/container-linux/kubernetes/cl/controller.yaml | 3 ++- aws/container-linux/kubernetes/workers/cl/worker.yaml | 3 ++- aws/fedora-coreos/kubernetes/fcc/controller.yaml | 3 ++- aws/fedora-coreos/kubernetes/workers/fcc/worker.yaml | 3 ++- azure/container-linux/kubernetes/cl/controller.yaml | 3 ++- azure/container-linux/kubernetes/workers/cl/worker.yaml | 3 ++- azure/fedora-coreos/kubernetes/fcc/controller.yaml | 3 ++- azure/fedora-coreos/kubernetes/workers/fcc/worker.yaml | 3 ++- bare-metal/container-linux/kubernetes/cl/controller.yaml | 3 ++- bare-metal/container-linux/kubernetes/cl/worker.yaml | 3 ++- bare-metal/fedora-coreos/kubernetes/fcc/controller.yaml | 3 ++- bare-metal/fedora-coreos/kubernetes/fcc/worker.yaml | 3 ++- digital-ocean/container-linux/kubernetes/cl/controller.yaml | 3 ++- digital-ocean/container-linux/kubernetes/cl/worker.yaml | 3 ++- digital-ocean/fedora-coreos/kubernetes/fcc/controller.yaml | 3 ++- digital-ocean/fedora-coreos/kubernetes/fcc/worker.yaml | 3 ++- google-cloud/container-linux/kubernetes/cl/controller.yaml | 3 ++- google-cloud/container-linux/kubernetes/workers/cl/worker.yaml | 3 ++- google-cloud/fedora-coreos/kubernetes/fcc/controller.yaml | 3 ++- google-cloud/fedora-coreos/kubernetes/workers/fcc/worker.yaml | 3 ++- 20 files changed, 40 insertions(+), 20 deletions(-) diff --git a/aws/container-linux/kubernetes/cl/controller.yaml b/aws/container-linux/kubernetes/cl/controller.yaml index b08297c8b..6c70396fa 100644 --- a/aws/container-linux/kubernetes/cl/controller.yaml +++ b/aws/container-linux/kubernetes/cl/controller.yaml @@ -52,6 +52,7 @@ systemd: Description=Kubelet Wants=rpc-statd.service [Service] + Environment=KUBELET_IMAGE=docker://quay.io/poseidon/kubelet:v1.18.2 Environment=KUBELET_CGROUP_DRIVER=${cgroup_driver} ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests @@ -91,7 +92,7 @@ systemd: --mount volume=var-log,target=/var/log \ --volume opt-cni-bin,kind=host,source=/opt/cni/bin \ --mount volume=opt-cni-bin,target=/opt/cni/bin \ - docker://quay.io/poseidon/kubelet:v1.18.2 -- \ + $${KUBELET_IMAGE} -- \ --anonymous-auth=false \ --authentication-token-webhook \ --authorization-mode=Webhook \ diff --git a/aws/container-linux/kubernetes/workers/cl/worker.yaml b/aws/container-linux/kubernetes/workers/cl/worker.yaml index 0924c634f..30bfc70eb 100644 --- a/aws/container-linux/kubernetes/workers/cl/worker.yaml +++ b/aws/container-linux/kubernetes/workers/cl/worker.yaml @@ -25,6 +25,7 @@ systemd: Description=Kubelet Wants=rpc-statd.service [Service] + Environment=KUBELET_IMAGE=docker://quay.io/poseidon/kubelet:v1.18.2 Environment=KUBELET_CGROUP_DRIVER=${cgroup_driver} ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests @@ -64,7 +65,7 @@ systemd: --mount volume=var-log,target=/var/log \ --volume opt-cni-bin,kind=host,source=/opt/cni/bin \ --mount volume=opt-cni-bin,target=/opt/cni/bin \ - docker://quay.io/poseidon/kubelet:v1.18.2 -- \ + $${KUBELET_IMAGE} -- \ --anonymous-auth=false \ --authentication-token-webhook \ --authorization-mode=Webhook \ diff --git a/aws/fedora-coreos/kubernetes/fcc/controller.yaml b/aws/fedora-coreos/kubernetes/fcc/controller.yaml index d06663239..f5c80984f 100644 --- a/aws/fedora-coreos/kubernetes/fcc/controller.yaml +++ b/aws/fedora-coreos/kubernetes/fcc/controller.yaml @@ -54,6 +54,7 @@ systemd: Description=Kubelet (System Container) Wants=rpc-statd.service [Service] + Environment=KUBELET_IMAGE=quay.io/poseidon/kubelet:v1.18.2 ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests ExecStartPre=/bin/mkdir -p /opt/cni/bin @@ -79,7 +80,7 @@ systemd: --volume /var/log:/var/log \ --volume /var/run/lock:/var/run/lock:z \ --volume /opt/cni/bin:/opt/cni/bin:z \ - quay.io/poseidon/kubelet:v1.18.2 \ + $${KUBELET_IMAGE} \ --anonymous-auth=false \ --authentication-token-webhook \ --authorization-mode=Webhook \ diff --git a/aws/fedora-coreos/kubernetes/workers/fcc/worker.yaml b/aws/fedora-coreos/kubernetes/workers/fcc/worker.yaml index 865fb9d8e..2fbebc733 100644 --- a/aws/fedora-coreos/kubernetes/workers/fcc/worker.yaml +++ b/aws/fedora-coreos/kubernetes/workers/fcc/worker.yaml @@ -24,6 +24,7 @@ systemd: Description=Kubelet (System Container) Wants=rpc-statd.service [Service] + Environment=KUBELET_IMAGE=quay.io/poseidon/kubelet:v1.18.2 ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests ExecStartPre=/bin/mkdir -p /opt/cni/bin @@ -49,7 +50,7 @@ systemd: --volume /var/log:/var/log \ --volume /var/run/lock:/var/run/lock:z \ --volume /opt/cni/bin:/opt/cni/bin:z \ - quay.io/poseidon/kubelet:v1.18.2 \ + $${KUBELET_IMAGE} \ --anonymous-auth=false \ --authentication-token-webhook \ --authorization-mode=Webhook \ diff --git a/azure/container-linux/kubernetes/cl/controller.yaml b/azure/container-linux/kubernetes/cl/controller.yaml index 17357966a..b3603ae4c 100644 --- a/azure/container-linux/kubernetes/cl/controller.yaml +++ b/azure/container-linux/kubernetes/cl/controller.yaml @@ -52,6 +52,7 @@ systemd: Description=Kubelet Wants=rpc-statd.service [Service] + Environment=KUBELET_IMAGE=docker://quay.io/poseidon/kubelet:v1.18.2 ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests ExecStartPre=/bin/mkdir -p /opt/cni/bin @@ -90,7 +91,7 @@ systemd: --mount volume=var-log,target=/var/log \ --volume opt-cni-bin,kind=host,source=/opt/cni/bin \ --mount volume=opt-cni-bin,target=/opt/cni/bin \ - docker://quay.io/poseidon/kubelet:v1.18.2 -- \ + $${KUBELET_IMAGE} -- \ --anonymous-auth=false \ --authentication-token-webhook \ --authorization-mode=Webhook \ diff --git a/azure/container-linux/kubernetes/workers/cl/worker.yaml b/azure/container-linux/kubernetes/workers/cl/worker.yaml index 2e1d74b2c..d0a0267bb 100644 --- a/azure/container-linux/kubernetes/workers/cl/worker.yaml +++ b/azure/container-linux/kubernetes/workers/cl/worker.yaml @@ -25,6 +25,7 @@ systemd: Description=Kubelet Wants=rpc-statd.service [Service] + Environment=KUBELET_IMAGE=docker://quay.io/poseidon/kubelet:v1.18.2 ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests ExecStartPre=/bin/mkdir -p /opt/cni/bin @@ -63,7 +64,7 @@ systemd: --mount volume=var-log,target=/var/log \ --volume opt-cni-bin,kind=host,source=/opt/cni/bin \ --mount volume=opt-cni-bin,target=/opt/cni/bin \ - docker://quay.io/poseidon/kubelet:v1.18.2 -- \ + $${KUBELET_IMAGE} -- \ --anonymous-auth=false \ --authentication-token-webhook \ --authorization-mode=Webhook \ diff --git a/azure/fedora-coreos/kubernetes/fcc/controller.yaml b/azure/fedora-coreos/kubernetes/fcc/controller.yaml index aaedb1611..faa8faed1 100644 --- a/azure/fedora-coreos/kubernetes/fcc/controller.yaml +++ b/azure/fedora-coreos/kubernetes/fcc/controller.yaml @@ -54,6 +54,7 @@ systemd: Description=Kubelet (System Container) Wants=rpc-statd.service [Service] + Environment=KUBELET_IMAGE=quay.io/poseidon/kubelet:v1.18.2 ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests ExecStartPre=/bin/mkdir -p /opt/cni/bin @@ -79,7 +80,7 @@ systemd: --volume /var/log:/var/log \ --volume /var/run/lock:/var/run/lock:z \ --volume /opt/cni/bin:/opt/cni/bin:z \ - quay.io/poseidon/kubelet:v1.18.2 \ + $${KUBELET_IMAGE} \ --anonymous-auth=false \ --authentication-token-webhook \ --authorization-mode=Webhook \ diff --git a/azure/fedora-coreos/kubernetes/workers/fcc/worker.yaml b/azure/fedora-coreos/kubernetes/workers/fcc/worker.yaml index 056cbafdb..b8cff9278 100644 --- a/azure/fedora-coreos/kubernetes/workers/fcc/worker.yaml +++ b/azure/fedora-coreos/kubernetes/workers/fcc/worker.yaml @@ -24,6 +24,7 @@ systemd: Description=Kubelet (System Container) Wants=rpc-statd.service [Service] + Environment=KUBELET_IMAGE=quay.io/poseidon/kubelet:v1.18.2 ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests ExecStartPre=/bin/mkdir -p /opt/cni/bin @@ -49,7 +50,7 @@ systemd: --volume /var/log:/var/log \ --volume /var/run/lock:/var/run/lock:z \ --volume /opt/cni/bin:/opt/cni/bin:z \ - quay.io/poseidon/kubelet:v1.18.2 \ + $${KUBELET_IMAGE} \ --anonymous-auth=false \ --authentication-token-webhook \ --authorization-mode=Webhook \ diff --git a/bare-metal/container-linux/kubernetes/cl/controller.yaml b/bare-metal/container-linux/kubernetes/cl/controller.yaml index f4f0e4617..f3d3cedb0 100644 --- a/bare-metal/container-linux/kubernetes/cl/controller.yaml +++ b/bare-metal/container-linux/kubernetes/cl/controller.yaml @@ -60,6 +60,7 @@ systemd: Description=Kubelet Wants=rpc-statd.service [Service] + Environment=KUBELET_IMAGE=docker://quay.io/poseidon/kubelet:v1.18.2 Environment=KUBELET_CGROUP_DRIVER=${cgroup_driver} ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests @@ -103,7 +104,7 @@ systemd: --mount volume=etc-iscsi,target=/etc/iscsi \ --volume usr-sbin-iscsiadm,kind=host,source=/usr/sbin/iscsiadm \ --mount volume=usr-sbin-iscsiadm,target=/sbin/iscsiadm \ - docker://quay.io/poseidon/kubelet:v1.18.2 -- \ + $${KUBELET_IMAGE} -- \ --anonymous-auth=false \ --authentication-token-webhook \ --authorization-mode=Webhook \ diff --git a/bare-metal/container-linux/kubernetes/cl/worker.yaml b/bare-metal/container-linux/kubernetes/cl/worker.yaml index dde9d7586..0a67ecc4c 100644 --- a/bare-metal/container-linux/kubernetes/cl/worker.yaml +++ b/bare-metal/container-linux/kubernetes/cl/worker.yaml @@ -33,6 +33,7 @@ systemd: Description=Kubelet Wants=rpc-statd.service [Service] + Environment=KUBELET_IMAGE=docker://quay.io/poseidon/kubelet:v1.18.2 Environment=KUBELET_CGROUP_DRIVER=${cgroup_driver} ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests @@ -76,7 +77,7 @@ systemd: --mount volume=etc-iscsi,target=/etc/iscsi \ --volume usr-sbin-iscsiadm,kind=host,source=/usr/sbin/iscsiadm \ --mount volume=usr-sbin-iscsiadm,target=/sbin/iscsiadm \ - docker://quay.io/poseidon/kubelet:v1.18.2 -- \ + $${KUBELET_IMAGE} -- \ --anonymous-auth=false \ --authentication-token-webhook \ --authorization-mode=Webhook \ diff --git a/bare-metal/fedora-coreos/kubernetes/fcc/controller.yaml b/bare-metal/fedora-coreos/kubernetes/fcc/controller.yaml index b8b068113..14ca2135e 100644 --- a/bare-metal/fedora-coreos/kubernetes/fcc/controller.yaml +++ b/bare-metal/fedora-coreos/kubernetes/fcc/controller.yaml @@ -53,6 +53,7 @@ systemd: Description=Kubelet (System Container) Wants=rpc-statd.service [Service] + Environment=KUBELET_IMAGE=quay.io/poseidon/kubelet:v1.18.2 ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests ExecStartPre=/bin/mkdir -p /opt/cni/bin @@ -80,7 +81,7 @@ systemd: --volume /opt/cni/bin:/opt/cni/bin:z \ --volume /etc/iscsi:/etc/iscsi \ --volume /sbin/iscsiadm:/sbin/iscsiadm \ - quay.io/poseidon/kubelet:v1.18.2 \ + $${KUBELET_IMAGE} \ --anonymous-auth=false \ --authentication-token-webhook \ --authorization-mode=Webhook \ diff --git a/bare-metal/fedora-coreos/kubernetes/fcc/worker.yaml b/bare-metal/fedora-coreos/kubernetes/fcc/worker.yaml index f61fe8d69..9b7feb235 100644 --- a/bare-metal/fedora-coreos/kubernetes/fcc/worker.yaml +++ b/bare-metal/fedora-coreos/kubernetes/fcc/worker.yaml @@ -23,6 +23,7 @@ systemd: Description=Kubelet (System Container) Wants=rpc-statd.service [Service] + Environment=KUBELET_IMAGE=quay.io/poseidon/kubelet:v1.18.2 ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests ExecStartPre=/bin/mkdir -p /opt/cni/bin @@ -50,7 +51,7 @@ systemd: --volume /opt/cni/bin:/opt/cni/bin:z \ --volume /etc/iscsi:/etc/iscsi \ --volume /sbin/iscsiadm:/sbin/iscsiadm \ - quay.io/poseidon/kubelet:v1.18.2 \ + $${KUBELET_IMAGE} \ --anonymous-auth=false \ --authentication-token-webhook \ --authorization-mode=Webhook \ diff --git a/digital-ocean/container-linux/kubernetes/cl/controller.yaml b/digital-ocean/container-linux/kubernetes/cl/controller.yaml index 254e22de7..73bc8bf7e 100644 --- a/digital-ocean/container-linux/kubernetes/cl/controller.yaml +++ b/digital-ocean/container-linux/kubernetes/cl/controller.yaml @@ -62,6 +62,7 @@ systemd: After=coreos-metadata.service Wants=rpc-statd.service [Service] + Environment=KUBELET_IMAGE=docker://quay.io/poseidon/kubelet:v1.18.2 EnvironmentFile=/run/metadata/coreos ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests @@ -101,7 +102,7 @@ systemd: --mount volume=var-log,target=/var/log \ --volume opt-cni-bin,kind=host,source=/opt/cni/bin \ --mount volume=opt-cni-bin,target=/opt/cni/bin \ - docker://quay.io/poseidon/kubelet:v1.18.2 -- \ + $${KUBELET_IMAGE} -- \ --anonymous-auth=false \ --authentication-token-webhook \ --authorization-mode=Webhook \ diff --git a/digital-ocean/container-linux/kubernetes/cl/worker.yaml b/digital-ocean/container-linux/kubernetes/cl/worker.yaml index c2f9419df..28686e5b2 100644 --- a/digital-ocean/container-linux/kubernetes/cl/worker.yaml +++ b/digital-ocean/container-linux/kubernetes/cl/worker.yaml @@ -35,6 +35,7 @@ systemd: After=coreos-metadata.service Wants=rpc-statd.service [Service] + Environment=KUBELET_IMAGE=docker://quay.io/poseidon/kubelet:v1.18.2 EnvironmentFile=/run/metadata/coreos ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests @@ -74,7 +75,7 @@ systemd: --mount volume=var-log,target=/var/log \ --volume opt-cni-bin,kind=host,source=/opt/cni/bin \ --mount volume=opt-cni-bin,target=/opt/cni/bin \ - docker://quay.io/poseidon/kubelet:v1.18.2 -- \ + $${KUBELET_IMAGE} -- \ --anonymous-auth=false \ --authentication-token-webhook \ --authorization-mode=Webhook \ diff --git a/digital-ocean/fedora-coreos/kubernetes/fcc/controller.yaml b/digital-ocean/fedora-coreos/kubernetes/fcc/controller.yaml index 9fde71b6a..6f9cc7070 100644 --- a/digital-ocean/fedora-coreos/kubernetes/fcc/controller.yaml +++ b/digital-ocean/fedora-coreos/kubernetes/fcc/controller.yaml @@ -55,6 +55,7 @@ systemd: After=afterburn.service Wants=rpc-statd.service [Service] + Environment=KUBELET_IMAGE=quay.io/poseidon/kubelet:v1.18.2 EnvironmentFile=/run/metadata/afterburn ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests @@ -81,7 +82,7 @@ systemd: --volume /var/log:/var/log \ --volume /var/run/lock:/var/run/lock:z \ --volume /opt/cni/bin:/opt/cni/bin:z \ - quay.io/poseidon/kubelet:v1.18.2 \ + $${KUBELET_IMAGE} \ --anonymous-auth=false \ --authentication-token-webhook \ --authorization-mode=Webhook \ diff --git a/digital-ocean/fedora-coreos/kubernetes/fcc/worker.yaml b/digital-ocean/fedora-coreos/kubernetes/fcc/worker.yaml index a18b5052a..45faab4cf 100644 --- a/digital-ocean/fedora-coreos/kubernetes/fcc/worker.yaml +++ b/digital-ocean/fedora-coreos/kubernetes/fcc/worker.yaml @@ -26,6 +26,7 @@ systemd: After=afterburn.service Wants=rpc-statd.service [Service] + Environment=KUBELET_IMAGE=quay.io/poseidon/kubelet:v1.18.2 EnvironmentFile=/run/metadata/afterburn ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests @@ -52,7 +53,7 @@ systemd: --volume /var/log:/var/log \ --volume /var/run/lock:/var/run/lock:z \ --volume /opt/cni/bin:/opt/cni/bin:z \ - quay.io/poseidon/kubelet:v1.18.2 \ + $${KUBELET_IMAGE} \ --anonymous-auth=false \ --authentication-token-webhook \ --authorization-mode=Webhook \ diff --git a/google-cloud/container-linux/kubernetes/cl/controller.yaml b/google-cloud/container-linux/kubernetes/cl/controller.yaml index 2c8f5d5f0..baf1bd4ec 100644 --- a/google-cloud/container-linux/kubernetes/cl/controller.yaml +++ b/google-cloud/container-linux/kubernetes/cl/controller.yaml @@ -52,6 +52,7 @@ systemd: Description=Kubelet Wants=rpc-statd.service [Service] + Environment=KUBELET_IMAGE=docker://quay.io/poseidon/kubelet:v1.18.2 ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests ExecStartPre=/bin/mkdir -p /opt/cni/bin @@ -90,7 +91,7 @@ systemd: --mount volume=var-log,target=/var/log \ --volume opt-cni-bin,kind=host,source=/opt/cni/bin \ --mount volume=opt-cni-bin,target=/opt/cni/bin \ - docker://quay.io/poseidon/kubelet:v1.18.2 -- \ + $${KUBELET_IMAGE} -- \ --anonymous-auth=false \ --authentication-token-webhook \ --authorization-mode=Webhook \ diff --git a/google-cloud/container-linux/kubernetes/workers/cl/worker.yaml b/google-cloud/container-linux/kubernetes/workers/cl/worker.yaml index 0bb704e2c..d5e7ae852 100644 --- a/google-cloud/container-linux/kubernetes/workers/cl/worker.yaml +++ b/google-cloud/container-linux/kubernetes/workers/cl/worker.yaml @@ -25,6 +25,7 @@ systemd: Description=Kubelet Wants=rpc-statd.service [Service] + Environment=KUBELET_IMAGE=docker://quay.io/poseidon/kubelet:v1.18.2 ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests ExecStartPre=/bin/mkdir -p /opt/cni/bin @@ -63,7 +64,7 @@ systemd: --mount volume=var-log,target=/var/log \ --volume opt-cni-bin,kind=host,source=/opt/cni/bin \ --mount volume=opt-cni-bin,target=/opt/cni/bin \ - docker://quay.io/poseidon/kubelet:v1.18.2 -- \ + $${KUBELET_IMAGE} -- \ --anonymous-auth=false \ --authentication-token-webhook \ --authorization-mode=Webhook \ diff --git a/google-cloud/fedora-coreos/kubernetes/fcc/controller.yaml b/google-cloud/fedora-coreos/kubernetes/fcc/controller.yaml index d06663239..f5c80984f 100644 --- a/google-cloud/fedora-coreos/kubernetes/fcc/controller.yaml +++ b/google-cloud/fedora-coreos/kubernetes/fcc/controller.yaml @@ -54,6 +54,7 @@ systemd: Description=Kubelet (System Container) Wants=rpc-statd.service [Service] + Environment=KUBELET_IMAGE=quay.io/poseidon/kubelet:v1.18.2 ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests ExecStartPre=/bin/mkdir -p /opt/cni/bin @@ -79,7 +80,7 @@ systemd: --volume /var/log:/var/log \ --volume /var/run/lock:/var/run/lock:z \ --volume /opt/cni/bin:/opt/cni/bin:z \ - quay.io/poseidon/kubelet:v1.18.2 \ + $${KUBELET_IMAGE} \ --anonymous-auth=false \ --authentication-token-webhook \ --authorization-mode=Webhook \ diff --git a/google-cloud/fedora-coreos/kubernetes/workers/fcc/worker.yaml b/google-cloud/fedora-coreos/kubernetes/workers/fcc/worker.yaml index 865fb9d8e..2fbebc733 100644 --- a/google-cloud/fedora-coreos/kubernetes/workers/fcc/worker.yaml +++ b/google-cloud/fedora-coreos/kubernetes/workers/fcc/worker.yaml @@ -24,6 +24,7 @@ systemd: Description=Kubelet (System Container) Wants=rpc-statd.service [Service] + Environment=KUBELET_IMAGE=quay.io/poseidon/kubelet:v1.18.2 ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests ExecStartPre=/bin/mkdir -p /opt/cni/bin @@ -49,7 +50,7 @@ systemd: --volume /var/log:/var/log \ --volume /var/run/lock:/var/run/lock:z \ --volume /opt/cni/bin:/opt/cni/bin:z \ - quay.io/poseidon/kubelet:v1.18.2 \ + $${KUBELET_IMAGE} \ --anonymous-auth=false \ --authentication-token-webhook \ --authorization-mode=Webhook \ From ecae6679ffdb9805f22a2fabc023f23d14b768c1 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Wed, 20 May 2020 20:37:39 -0700 Subject: [PATCH 427/523] Update Kubernetes from v1.18.2 to v1.18.3 * https://github.com/kubernetes/kubernetes/blob/master/CHANGELOG/CHANGELOG-1.18.md --- README.md | 10 +++++----- aws/container-linux/kubernetes/README.md | 2 +- aws/container-linux/kubernetes/bootstrap.tf | 2 +- aws/container-linux/kubernetes/cl/controller.yaml | 4 ++-- .../kubernetes/workers/cl/worker.yaml | 4 ++-- aws/fedora-coreos/kubernetes/README.md | 2 +- aws/fedora-coreos/kubernetes/bootstrap.tf | 2 +- aws/fedora-coreos/kubernetes/fcc/controller.yaml | 4 ++-- .../kubernetes/workers/fcc/worker.yaml | 4 ++-- azure/container-linux/kubernetes/README.md | 2 +- azure/container-linux/kubernetes/bootstrap.tf | 2 +- .../container-linux/kubernetes/cl/controller.yaml | 4 ++-- .../kubernetes/workers/cl/worker.yaml | 4 ++-- azure/fedora-coreos/kubernetes/README.md | 2 +- azure/fedora-coreos/kubernetes/bootstrap.tf | 2 +- azure/fedora-coreos/kubernetes/fcc/controller.yaml | 4 ++-- .../kubernetes/workers/fcc/worker.yaml | 4 ++-- bare-metal/container-linux/kubernetes/README.md | 2 +- bare-metal/container-linux/kubernetes/bootstrap.tf | 2 +- .../container-linux/kubernetes/cl/controller.yaml | 4 ++-- .../container-linux/kubernetes/cl/worker.yaml | 2 +- bare-metal/fedora-coreos/kubernetes/README.md | 2 +- bare-metal/fedora-coreos/kubernetes/bootstrap.tf | 2 +- .../fedora-coreos/kubernetes/fcc/controller.yaml | 4 ++-- .../fedora-coreos/kubernetes/fcc/worker.yaml | 2 +- digital-ocean/container-linux/kubernetes/README.md | 2 +- .../container-linux/kubernetes/bootstrap.tf | 2 +- .../container-linux/kubernetes/cl/controller.yaml | 4 ++-- .../container-linux/kubernetes/cl/worker.yaml | 4 ++-- digital-ocean/fedora-coreos/kubernetes/README.md | 2 +- .../fedora-coreos/kubernetes/bootstrap.tf | 2 +- .../fedora-coreos/kubernetes/fcc/controller.yaml | 4 ++-- .../fedora-coreos/kubernetes/fcc/worker.yaml | 4 ++-- docs/advanced/worker-pools.md | 14 +++++++------- docs/cl/aws.md | 10 +++++----- docs/cl/azure.md | 10 +++++----- docs/cl/bare-metal.md | 10 +++++----- docs/cl/digital-ocean.md | 10 +++++----- docs/cl/google-cloud.md | 10 +++++----- docs/fedora-coreos/aws.md | 10 +++++----- docs/fedora-coreos/azure.md | 10 +++++----- docs/fedora-coreos/bare-metal.md | 10 +++++----- docs/fedora-coreos/digitalocean.md | 10 +++++----- docs/fedora-coreos/google-cloud.md | 8 ++++---- docs/index.md | 10 +++++----- docs/topics/maintenance.md | 8 ++++---- google-cloud/container-linux/kubernetes/README.md | 2 +- .../container-linux/kubernetes/bootstrap.tf | 2 +- .../container-linux/kubernetes/cl/controller.yaml | 4 ++-- .../kubernetes/workers/cl/worker.yaml | 4 ++-- google-cloud/fedora-coreos/kubernetes/README.md | 2 +- google-cloud/fedora-coreos/kubernetes/bootstrap.tf | 2 +- .../fedora-coreos/kubernetes/fcc/controller.yaml | 4 ++-- .../kubernetes/workers/fcc/worker.yaml | 4 ++-- 54 files changed, 128 insertions(+), 128 deletions(-) diff --git a/README.md b/README.md index 79c40e4f7..5d5557e14 100644 --- a/README.md +++ b/README.md @@ -11,7 +11,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features -* Kubernetes v1.18.2 (upstream) +* Kubernetes v1.18.3 (upstream) * Single or multi-master, [Calico](https://www.projectcalico.org/) or [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/), SELinux enforcing * Advanced features like [worker pools](https://typhoon.psdn.io/advanced/worker-pools/), [preemptible](https://typhoon.psdn.io/cl/google-cloud/#preemption) workers, and [snippets](https://typhoon.psdn.io/advanced/customization/#container-linux) customization @@ -64,7 +64,7 @@ Define a Kubernetes cluster by using the Terraform module for your chosen platfo ```tf module "yavin" { - source = "git::https://github.com/poseidon/typhoon//google-cloud/fedora-coreos/kubernetes?ref=v1.18.2" + source = "git::https://github.com/poseidon/typhoon//google-cloud/fedora-coreos/kubernetes?ref=v1.18.3" # Google Cloud cluster_name = "yavin" @@ -103,9 +103,9 @@ In 4-8 minutes (varies by platform), the cluster will be ready. This Google Clou $ export KUBECONFIG=/home/user/.kube/configs/yavin-config $ kubectl get nodes NAME ROLES STATUS AGE VERSION -yavin-controller-0.c.example-com.internal Ready 6m v1.18.2 -yavin-worker-jrbf.c.example-com.internal Ready 5m v1.18.2 -yavin-worker-mzdm.c.example-com.internal Ready 5m v1.18.2 +yavin-controller-0.c.example-com.internal Ready 6m v1.18.3 +yavin-worker-jrbf.c.example-com.internal Ready 5m v1.18.3 +yavin-worker-mzdm.c.example-com.internal Ready 5m v1.18.3 ``` List the pods. diff --git a/aws/container-linux/kubernetes/README.md b/aws/container-linux/kubernetes/README.md index 053433256..cea72768b 100644 --- a/aws/container-linux/kubernetes/README.md +++ b/aws/container-linux/kubernetes/README.md @@ -11,7 +11,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features -* Kubernetes v1.18.2 (upstream) +* Kubernetes v1.18.3 (upstream) * Single or multi-master, [Calico](https://www.projectcalico.org/) or [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/) * Advanced features like [worker pools](https://typhoon.psdn.io/advanced/worker-pools/), [spot](https://typhoon.psdn.io/cl/aws/#spot) workers, and [snippets](https://typhoon.psdn.io/advanced/customization/#container-linux) customization diff --git a/aws/container-linux/kubernetes/bootstrap.tf b/aws/container-linux/kubernetes/bootstrap.tf index ae27f5b91..8a8fc8c30 100644 --- a/aws/container-linux/kubernetes/bootstrap.tf +++ b/aws/container-linux/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=157336db92daf1f1b7f09ab2b3ccbe16a4b66c57" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=ff7ec52d0a5e97b8ca6b86a80a7e5e1ea8570487" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/aws/container-linux/kubernetes/cl/controller.yaml b/aws/container-linux/kubernetes/cl/controller.yaml index 6c70396fa..362835017 100644 --- a/aws/container-linux/kubernetes/cl/controller.yaml +++ b/aws/container-linux/kubernetes/cl/controller.yaml @@ -52,7 +52,7 @@ systemd: Description=Kubelet Wants=rpc-statd.service [Service] - Environment=KUBELET_IMAGE=docker://quay.io/poseidon/kubelet:v1.18.2 + Environment=KUBELET_IMAGE=docker://quay.io/poseidon/kubelet:v1.18.3 Environment=KUBELET_CGROUP_DRIVER=${cgroup_driver} ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests @@ -137,7 +137,7 @@ systemd: --volume script,kind=host,source=/opt/bootstrap/apply \ --mount volume=script,target=/apply \ --insecure-options=image \ - docker://quay.io/poseidon/kubelet:v1.18.2 \ + docker://quay.io/poseidon/kubelet:v1.18.3 \ --net=host \ --dns=host \ --exec=/apply diff --git a/aws/container-linux/kubernetes/workers/cl/worker.yaml b/aws/container-linux/kubernetes/workers/cl/worker.yaml index 30bfc70eb..f4c7129af 100644 --- a/aws/container-linux/kubernetes/workers/cl/worker.yaml +++ b/aws/container-linux/kubernetes/workers/cl/worker.yaml @@ -25,7 +25,7 @@ systemd: Description=Kubelet Wants=rpc-statd.service [Service] - Environment=KUBELET_IMAGE=docker://quay.io/poseidon/kubelet:v1.18.2 + Environment=KUBELET_IMAGE=docker://quay.io/poseidon/kubelet:v1.18.3 Environment=KUBELET_CGROUP_DRIVER=${cgroup_driver} ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests @@ -130,7 +130,7 @@ storage: --volume config,kind=host,source=/etc/kubernetes \ --mount volume=config,target=/etc/kubernetes \ --insecure-options=image \ - docker://quay.io/poseidon/kubelet:v1.18.2 \ + docker://quay.io/poseidon/kubelet:v1.18.3 \ --net=host \ --dns=host \ --exec=/usr/local/bin/kubectl -- --kubeconfig=/etc/kubernetes/kubeconfig delete node $(hostname) diff --git a/aws/fedora-coreos/kubernetes/README.md b/aws/fedora-coreos/kubernetes/README.md index a10826b46..ab5aad720 100644 --- a/aws/fedora-coreos/kubernetes/README.md +++ b/aws/fedora-coreos/kubernetes/README.md @@ -11,7 +11,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features -* Kubernetes v1.18.2 (upstream) +* Kubernetes v1.18.3 (upstream) * Single or multi-master, [Calico](https://www.projectcalico.org/) or [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/), SELinux enforcing * Advanced features like [worker pools](https://typhoon.psdn.io/advanced/worker-pools/), [spot](https://typhoon.psdn.io/cl/aws/#spot) workers, and [snippets](https://typhoon.psdn.io/advanced/customization/#container-linux) customization diff --git a/aws/fedora-coreos/kubernetes/bootstrap.tf b/aws/fedora-coreos/kubernetes/bootstrap.tf index bde0ea923..c702bf139 100644 --- a/aws/fedora-coreos/kubernetes/bootstrap.tf +++ b/aws/fedora-coreos/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=157336db92daf1f1b7f09ab2b3ccbe16a4b66c57" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=ff7ec52d0a5e97b8ca6b86a80a7e5e1ea8570487" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/aws/fedora-coreos/kubernetes/fcc/controller.yaml b/aws/fedora-coreos/kubernetes/fcc/controller.yaml index f5c80984f..a98424223 100644 --- a/aws/fedora-coreos/kubernetes/fcc/controller.yaml +++ b/aws/fedora-coreos/kubernetes/fcc/controller.yaml @@ -54,7 +54,7 @@ systemd: Description=Kubelet (System Container) Wants=rpc-statd.service [Service] - Environment=KUBELET_IMAGE=quay.io/poseidon/kubelet:v1.18.2 + Environment=KUBELET_IMAGE=quay.io/poseidon/kubelet:v1.18.3 ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests ExecStartPre=/bin/mkdir -p /opt/cni/bin @@ -126,7 +126,7 @@ systemd: --volume /opt/bootstrap/assets:/assets:ro,Z \ --volume /opt/bootstrap/apply:/apply:ro,Z \ --entrypoint=/apply \ - quay.io/poseidon/kubelet:v1.18.2 + quay.io/poseidon/kubelet:v1.18.3 ExecStartPost=/bin/touch /opt/bootstrap/bootstrap.done ExecStartPost=-/usr/bin/podman stop bootstrap storage: diff --git a/aws/fedora-coreos/kubernetes/workers/fcc/worker.yaml b/aws/fedora-coreos/kubernetes/workers/fcc/worker.yaml index 2fbebc733..d33c43db5 100644 --- a/aws/fedora-coreos/kubernetes/workers/fcc/worker.yaml +++ b/aws/fedora-coreos/kubernetes/workers/fcc/worker.yaml @@ -24,7 +24,7 @@ systemd: Description=Kubelet (System Container) Wants=rpc-statd.service [Service] - Environment=KUBELET_IMAGE=quay.io/poseidon/kubelet:v1.18.2 + Environment=KUBELET_IMAGE=quay.io/poseidon/kubelet:v1.18.3 ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests ExecStartPre=/bin/mkdir -p /opt/cni/bin @@ -90,7 +90,7 @@ systemd: Type=oneshot RemainAfterExit=true ExecStart=/bin/true - ExecStop=/bin/bash -c '/usr/bin/podman run --volume /etc/kubernetes:/etc/kubernetes:ro,z --entrypoint /usr/local/bin/kubectl quay.io/poseidon/kubelet:v1.18.2 --kubeconfig=/etc/kubernetes/kubeconfig delete node $HOSTNAME' + ExecStop=/bin/bash -c '/usr/bin/podman run --volume /etc/kubernetes:/etc/kubernetes:ro,z --entrypoint /usr/local/bin/kubectl quay.io/poseidon/kubelet:v1.18.3 --kubeconfig=/etc/kubernetes/kubeconfig delete node $HOSTNAME' [Install] WantedBy=multi-user.target storage: diff --git a/azure/container-linux/kubernetes/README.md b/azure/container-linux/kubernetes/README.md index 29bff5080..a6d9d0d3f 100644 --- a/azure/container-linux/kubernetes/README.md +++ b/azure/container-linux/kubernetes/README.md @@ -11,7 +11,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features -* Kubernetes v1.18.2 (upstream) +* Kubernetes v1.18.3 (upstream) * Single or multi-master, [Calico](https://www.projectcalico.org/) or [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/) * Advanced features like [worker pools](https://typhoon.psdn.io/advanced/worker-pools/), [low-priority](https://typhoon.psdn.io/cl/azure/#low-priority) workers, and [snippets](https://typhoon.psdn.io/advanced/customization/#container-linux) customization diff --git a/azure/container-linux/kubernetes/bootstrap.tf b/azure/container-linux/kubernetes/bootstrap.tf index 7a8252786..95ab99172 100644 --- a/azure/container-linux/kubernetes/bootstrap.tf +++ b/azure/container-linux/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=157336db92daf1f1b7f09ab2b3ccbe16a4b66c57" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=ff7ec52d0a5e97b8ca6b86a80a7e5e1ea8570487" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/azure/container-linux/kubernetes/cl/controller.yaml b/azure/container-linux/kubernetes/cl/controller.yaml index b3603ae4c..c819592fb 100644 --- a/azure/container-linux/kubernetes/cl/controller.yaml +++ b/azure/container-linux/kubernetes/cl/controller.yaml @@ -52,7 +52,7 @@ systemd: Description=Kubelet Wants=rpc-statd.service [Service] - Environment=KUBELET_IMAGE=docker://quay.io/poseidon/kubelet:v1.18.2 + Environment=KUBELET_IMAGE=docker://quay.io/poseidon/kubelet:v1.18.3 ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests ExecStartPre=/bin/mkdir -p /opt/cni/bin @@ -135,7 +135,7 @@ systemd: --volume script,kind=host,source=/opt/bootstrap/apply \ --mount volume=script,target=/apply \ --insecure-options=image \ - docker://quay.io/poseidon/kubelet:v1.18.2 \ + docker://quay.io/poseidon/kubelet:v1.18.3 \ --net=host \ --dns=host \ --exec=/apply diff --git a/azure/container-linux/kubernetes/workers/cl/worker.yaml b/azure/container-linux/kubernetes/workers/cl/worker.yaml index d0a0267bb..11691e1fc 100644 --- a/azure/container-linux/kubernetes/workers/cl/worker.yaml +++ b/azure/container-linux/kubernetes/workers/cl/worker.yaml @@ -25,7 +25,7 @@ systemd: Description=Kubelet Wants=rpc-statd.service [Service] - Environment=KUBELET_IMAGE=docker://quay.io/poseidon/kubelet:v1.18.2 + Environment=KUBELET_IMAGE=docker://quay.io/poseidon/kubelet:v1.18.3 ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests ExecStartPre=/bin/mkdir -p /opt/cni/bin @@ -128,7 +128,7 @@ storage: --volume config,kind=host,source=/etc/kubernetes \ --mount volume=config,target=/etc/kubernetes \ --insecure-options=image \ - docker://quay.io/poseidon/kubelet:v1.18.2 \ + docker://quay.io/poseidon/kubelet:v1.18.3 \ --net=host \ --dns=host \ --exec=/usr/local/bin/kubectl -- --kubeconfig=/etc/kubernetes/kubeconfig delete node $(hostname | tr '[:upper:]' '[:lower:]') diff --git a/azure/fedora-coreos/kubernetes/README.md b/azure/fedora-coreos/kubernetes/README.md index 2b493533e..6ae73dfbf 100644 --- a/azure/fedora-coreos/kubernetes/README.md +++ b/azure/fedora-coreos/kubernetes/README.md @@ -11,7 +11,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features -* Kubernetes v1.18.2 (upstream) +* Kubernetes v1.18.3 (upstream) * Single or multi-master, [Calico](https://www.projectcalico.org/) or [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/), SELinux enforcing * Advanced features like [worker pools](https://typhoon.psdn.io/advanced/worker-pools/), [spot priority](https://typhoon.psdn.io/fedora-coreos/azure/#low-priority) workers, and [snippets](https://typhoon.psdn.io/advanced/customization/) customization diff --git a/azure/fedora-coreos/kubernetes/bootstrap.tf b/azure/fedora-coreos/kubernetes/bootstrap.tf index a95d3ebfc..baec21879 100644 --- a/azure/fedora-coreos/kubernetes/bootstrap.tf +++ b/azure/fedora-coreos/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=157336db92daf1f1b7f09ab2b3ccbe16a4b66c57" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=ff7ec52d0a5e97b8ca6b86a80a7e5e1ea8570487" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/azure/fedora-coreos/kubernetes/fcc/controller.yaml b/azure/fedora-coreos/kubernetes/fcc/controller.yaml index faa8faed1..9255564e0 100644 --- a/azure/fedora-coreos/kubernetes/fcc/controller.yaml +++ b/azure/fedora-coreos/kubernetes/fcc/controller.yaml @@ -54,7 +54,7 @@ systemd: Description=Kubelet (System Container) Wants=rpc-statd.service [Service] - Environment=KUBELET_IMAGE=quay.io/poseidon/kubelet:v1.18.2 + Environment=KUBELET_IMAGE=quay.io/poseidon/kubelet:v1.18.3 ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests ExecStartPre=/bin/mkdir -p /opt/cni/bin @@ -126,7 +126,7 @@ systemd: --volume /opt/bootstrap/assets:/assets:ro,Z \ --volume /opt/bootstrap/apply:/apply:ro,Z \ --entrypoint=/apply \ - quay.io/poseidon/kubelet:v1.18.2 + quay.io/poseidon/kubelet:v1.18.3 ExecStartPost=/bin/touch /opt/bootstrap/bootstrap.done ExecStartPost=-/usr/bin/podman stop bootstrap storage: diff --git a/azure/fedora-coreos/kubernetes/workers/fcc/worker.yaml b/azure/fedora-coreos/kubernetes/workers/fcc/worker.yaml index b8cff9278..814e51248 100644 --- a/azure/fedora-coreos/kubernetes/workers/fcc/worker.yaml +++ b/azure/fedora-coreos/kubernetes/workers/fcc/worker.yaml @@ -24,7 +24,7 @@ systemd: Description=Kubelet (System Container) Wants=rpc-statd.service [Service] - Environment=KUBELET_IMAGE=quay.io/poseidon/kubelet:v1.18.2 + Environment=KUBELET_IMAGE=quay.io/poseidon/kubelet:v1.18.3 ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests ExecStartPre=/bin/mkdir -p /opt/cni/bin @@ -90,7 +90,7 @@ systemd: Type=oneshot RemainAfterExit=true ExecStart=/bin/true - ExecStop=/bin/bash -c '/usr/bin/podman run --volume /etc/kubernetes:/etc/kubernetes:ro,z --entrypoint /usr/local/bin/kubectl quay.io/poseidon/kubelet:v1.18.2 --kubeconfig=/etc/kubernetes/kubeconfig delete node $HOSTNAME' + ExecStop=/bin/bash -c '/usr/bin/podman run --volume /etc/kubernetes:/etc/kubernetes:ro,z --entrypoint /usr/local/bin/kubectl quay.io/poseidon/kubelet:v1.18.3 --kubeconfig=/etc/kubernetes/kubeconfig delete node $HOSTNAME' [Install] WantedBy=multi-user.target storage: diff --git a/bare-metal/container-linux/kubernetes/README.md b/bare-metal/container-linux/kubernetes/README.md index 5c52402e8..301c08a13 100644 --- a/bare-metal/container-linux/kubernetes/README.md +++ b/bare-metal/container-linux/kubernetes/README.md @@ -11,7 +11,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features -* Kubernetes v1.18.2 (upstream) +* Kubernetes v1.18.3 (upstream) * Single or multi-master, [Calico](https://www.projectcalico.org/) or [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/) * Advanced features like [snippets](https://typhoon.psdn.io/advanced/customization/#container-linux) customization diff --git a/bare-metal/container-linux/kubernetes/bootstrap.tf b/bare-metal/container-linux/kubernetes/bootstrap.tf index 45097295a..34d68d1cb 100644 --- a/bare-metal/container-linux/kubernetes/bootstrap.tf +++ b/bare-metal/container-linux/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=157336db92daf1f1b7f09ab2b3ccbe16a4b66c57" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=ff7ec52d0a5e97b8ca6b86a80a7e5e1ea8570487" cluster_name = var.cluster_name api_servers = [var.k8s_domain_name] diff --git a/bare-metal/container-linux/kubernetes/cl/controller.yaml b/bare-metal/container-linux/kubernetes/cl/controller.yaml index f3d3cedb0..10b5a9c30 100644 --- a/bare-metal/container-linux/kubernetes/cl/controller.yaml +++ b/bare-metal/container-linux/kubernetes/cl/controller.yaml @@ -60,7 +60,7 @@ systemd: Description=Kubelet Wants=rpc-statd.service [Service] - Environment=KUBELET_IMAGE=docker://quay.io/poseidon/kubelet:v1.18.2 + Environment=KUBELET_IMAGE=docker://quay.io/poseidon/kubelet:v1.18.3 Environment=KUBELET_CGROUP_DRIVER=${cgroup_driver} ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests @@ -150,7 +150,7 @@ systemd: --volume script,kind=host,source=/opt/bootstrap/apply \ --mount volume=script,target=/apply \ --insecure-options=image \ - docker://quay.io/poseidon/kubelet:v1.18.2 \ + docker://quay.io/poseidon/kubelet:v1.18.3 \ --net=host \ --dns=host \ --exec=/apply diff --git a/bare-metal/container-linux/kubernetes/cl/worker.yaml b/bare-metal/container-linux/kubernetes/cl/worker.yaml index 0a67ecc4c..337070061 100644 --- a/bare-metal/container-linux/kubernetes/cl/worker.yaml +++ b/bare-metal/container-linux/kubernetes/cl/worker.yaml @@ -33,7 +33,7 @@ systemd: Description=Kubelet Wants=rpc-statd.service [Service] - Environment=KUBELET_IMAGE=docker://quay.io/poseidon/kubelet:v1.18.2 + Environment=KUBELET_IMAGE=docker://quay.io/poseidon/kubelet:v1.18.3 Environment=KUBELET_CGROUP_DRIVER=${cgroup_driver} ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests diff --git a/bare-metal/fedora-coreos/kubernetes/README.md b/bare-metal/fedora-coreos/kubernetes/README.md index 70d6e5b6e..427c38b2f 100644 --- a/bare-metal/fedora-coreos/kubernetes/README.md +++ b/bare-metal/fedora-coreos/kubernetes/README.md @@ -11,7 +11,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features -* Kubernetes v1.18.2 (upstream) +* Kubernetes v1.18.3 (upstream) * Single or multi-master, [Calico](https://www.projectcalico.org/) or [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/), SELinux enforcing * Advanced features like [snippets](https://typhoon.psdn.io/advanced/customization/#container-linux) customization diff --git a/bare-metal/fedora-coreos/kubernetes/bootstrap.tf b/bare-metal/fedora-coreos/kubernetes/bootstrap.tf index ae8084b80..e2a9c5317 100644 --- a/bare-metal/fedora-coreos/kubernetes/bootstrap.tf +++ b/bare-metal/fedora-coreos/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=157336db92daf1f1b7f09ab2b3ccbe16a4b66c57" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=ff7ec52d0a5e97b8ca6b86a80a7e5e1ea8570487" cluster_name = var.cluster_name api_servers = [var.k8s_domain_name] diff --git a/bare-metal/fedora-coreos/kubernetes/fcc/controller.yaml b/bare-metal/fedora-coreos/kubernetes/fcc/controller.yaml index 14ca2135e..f1baa6fd2 100644 --- a/bare-metal/fedora-coreos/kubernetes/fcc/controller.yaml +++ b/bare-metal/fedora-coreos/kubernetes/fcc/controller.yaml @@ -53,7 +53,7 @@ systemd: Description=Kubelet (System Container) Wants=rpc-statd.service [Service] - Environment=KUBELET_IMAGE=quay.io/poseidon/kubelet:v1.18.2 + Environment=KUBELET_IMAGE=quay.io/poseidon/kubelet:v1.18.3 ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests ExecStartPre=/bin/mkdir -p /opt/cni/bin @@ -137,7 +137,7 @@ systemd: --volume /opt/bootstrap/assets:/assets:ro,Z \ --volume /opt/bootstrap/apply:/apply:ro,Z \ --entrypoint=/apply \ - quay.io/poseidon/kubelet:v1.18.2 + quay.io/poseidon/kubelet:v1.18.3 ExecStartPost=/bin/touch /opt/bootstrap/bootstrap.done ExecStartPost=-/usr/bin/podman stop bootstrap storage: diff --git a/bare-metal/fedora-coreos/kubernetes/fcc/worker.yaml b/bare-metal/fedora-coreos/kubernetes/fcc/worker.yaml index 9b7feb235..f3526e6a3 100644 --- a/bare-metal/fedora-coreos/kubernetes/fcc/worker.yaml +++ b/bare-metal/fedora-coreos/kubernetes/fcc/worker.yaml @@ -23,7 +23,7 @@ systemd: Description=Kubelet (System Container) Wants=rpc-statd.service [Service] - Environment=KUBELET_IMAGE=quay.io/poseidon/kubelet:v1.18.2 + Environment=KUBELET_IMAGE=quay.io/poseidon/kubelet:v1.18.3 ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests ExecStartPre=/bin/mkdir -p /opt/cni/bin diff --git a/digital-ocean/container-linux/kubernetes/README.md b/digital-ocean/container-linux/kubernetes/README.md index 73b5fad91..2289778be 100644 --- a/digital-ocean/container-linux/kubernetes/README.md +++ b/digital-ocean/container-linux/kubernetes/README.md @@ -11,7 +11,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features -* Kubernetes v1.18.2 (upstream) +* Kubernetes v1.18.3 (upstream) * Single or multi-master, [Calico](https://www.projectcalico.org/) or [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/) * Advanced features like [snippets](https://typhoon.psdn.io/advanced/customization/#container-linux) customization diff --git a/digital-ocean/container-linux/kubernetes/bootstrap.tf b/digital-ocean/container-linux/kubernetes/bootstrap.tf index 0a636748c..cf04ebbff 100644 --- a/digital-ocean/container-linux/kubernetes/bootstrap.tf +++ b/digital-ocean/container-linux/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=157336db92daf1f1b7f09ab2b3ccbe16a4b66c57" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=ff7ec52d0a5e97b8ca6b86a80a7e5e1ea8570487" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/digital-ocean/container-linux/kubernetes/cl/controller.yaml b/digital-ocean/container-linux/kubernetes/cl/controller.yaml index 73bc8bf7e..c3c7eee97 100644 --- a/digital-ocean/container-linux/kubernetes/cl/controller.yaml +++ b/digital-ocean/container-linux/kubernetes/cl/controller.yaml @@ -62,7 +62,7 @@ systemd: After=coreos-metadata.service Wants=rpc-statd.service [Service] - Environment=KUBELET_IMAGE=docker://quay.io/poseidon/kubelet:v1.18.2 + Environment=KUBELET_IMAGE=docker://quay.io/poseidon/kubelet:v1.18.3 EnvironmentFile=/run/metadata/coreos ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests @@ -147,7 +147,7 @@ systemd: --volume script,kind=host,source=/opt/bootstrap/apply \ --mount volume=script,target=/apply \ --insecure-options=image \ - docker://quay.io/poseidon/kubelet:v1.18.2 \ + docker://quay.io/poseidon/kubelet:v1.18.3 \ --net=host \ --dns=host \ --exec=/apply diff --git a/digital-ocean/container-linux/kubernetes/cl/worker.yaml b/digital-ocean/container-linux/kubernetes/cl/worker.yaml index 28686e5b2..4c841c69a 100644 --- a/digital-ocean/container-linux/kubernetes/cl/worker.yaml +++ b/digital-ocean/container-linux/kubernetes/cl/worker.yaml @@ -35,7 +35,7 @@ systemd: After=coreos-metadata.service Wants=rpc-statd.service [Service] - Environment=KUBELET_IMAGE=docker://quay.io/poseidon/kubelet:v1.18.2 + Environment=KUBELET_IMAGE=docker://quay.io/poseidon/kubelet:v1.18.3 EnvironmentFile=/run/metadata/coreos ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests @@ -134,7 +134,7 @@ storage: --volume config,kind=host,source=/etc/kubernetes \ --mount volume=config,target=/etc/kubernetes \ --insecure-options=image \ - docker://quay.io/poseidon/kubelet:v1.18.2 \ + docker://quay.io/poseidon/kubelet:v1.18.3 \ --net=host \ --dns=host \ --exec=/usr/local/bin/kubectl -- --kubeconfig=/etc/kubernetes/kubeconfig delete node $(hostname) diff --git a/digital-ocean/fedora-coreos/kubernetes/README.md b/digital-ocean/fedora-coreos/kubernetes/README.md index 3945c7c8c..e1f7bb901 100644 --- a/digital-ocean/fedora-coreos/kubernetes/README.md +++ b/digital-ocean/fedora-coreos/kubernetes/README.md @@ -11,7 +11,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features -* Kubernetes v1.18.2 (upstream) +* Kubernetes v1.18.3 (upstream) * Single or multi-master, [Calico](https://www.projectcalico.org/) or [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/), SELinux enforcing * Advanced features like [snippets](https://typhoon.psdn.io/advanced/customization/) customization diff --git a/digital-ocean/fedora-coreos/kubernetes/bootstrap.tf b/digital-ocean/fedora-coreos/kubernetes/bootstrap.tf index c39cdeea4..62cf353e2 100644 --- a/digital-ocean/fedora-coreos/kubernetes/bootstrap.tf +++ b/digital-ocean/fedora-coreos/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=157336db92daf1f1b7f09ab2b3ccbe16a4b66c57" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=ff7ec52d0a5e97b8ca6b86a80a7e5e1ea8570487" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/digital-ocean/fedora-coreos/kubernetes/fcc/controller.yaml b/digital-ocean/fedora-coreos/kubernetes/fcc/controller.yaml index 6f9cc7070..3ae2c5823 100644 --- a/digital-ocean/fedora-coreos/kubernetes/fcc/controller.yaml +++ b/digital-ocean/fedora-coreos/kubernetes/fcc/controller.yaml @@ -55,7 +55,7 @@ systemd: After=afterburn.service Wants=rpc-statd.service [Service] - Environment=KUBELET_IMAGE=quay.io/poseidon/kubelet:v1.18.2 + Environment=KUBELET_IMAGE=quay.io/poseidon/kubelet:v1.18.3 EnvironmentFile=/run/metadata/afterburn ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests @@ -138,7 +138,7 @@ systemd: --volume /opt/bootstrap/assets:/assets:ro,Z \ --volume /opt/bootstrap/apply:/apply:ro,Z \ --entrypoint=/apply \ - quay.io/poseidon/kubelet:v1.18.2 + quay.io/poseidon/kubelet:v1.18.3 ExecStartPost=/bin/touch /opt/bootstrap/bootstrap.done ExecStartPost=-/usr/bin/podman stop bootstrap storage: diff --git a/digital-ocean/fedora-coreos/kubernetes/fcc/worker.yaml b/digital-ocean/fedora-coreos/kubernetes/fcc/worker.yaml index 45faab4cf..98554d610 100644 --- a/digital-ocean/fedora-coreos/kubernetes/fcc/worker.yaml +++ b/digital-ocean/fedora-coreos/kubernetes/fcc/worker.yaml @@ -26,7 +26,7 @@ systemd: After=afterburn.service Wants=rpc-statd.service [Service] - Environment=KUBELET_IMAGE=quay.io/poseidon/kubelet:v1.18.2 + Environment=KUBELET_IMAGE=quay.io/poseidon/kubelet:v1.18.3 EnvironmentFile=/run/metadata/afterburn ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests @@ -100,7 +100,7 @@ systemd: Type=oneshot RemainAfterExit=true ExecStart=/bin/true - ExecStop=/bin/bash -c '/usr/bin/podman run --volume /etc/kubernetes:/etc/kubernetes:ro,z --entrypoint /usr/local/bin/kubectl quay.io/poseidon/kubelet:v1.18.2 --kubeconfig=/etc/kubernetes/kubeconfig delete node $HOSTNAME' + ExecStop=/bin/bash -c '/usr/bin/podman run --volume /etc/kubernetes:/etc/kubernetes:ro,z --entrypoint /usr/local/bin/kubectl quay.io/poseidon/kubelet:v1.18.3 --kubeconfig=/etc/kubernetes/kubeconfig delete node $HOSTNAME' [Install] WantedBy=multi-user.target storage: diff --git a/docs/advanced/worker-pools.md b/docs/advanced/worker-pools.md index 6face16f0..77d351f59 100644 --- a/docs/advanced/worker-pools.md +++ b/docs/advanced/worker-pools.md @@ -82,7 +82,7 @@ Create a cluster following the Azure [tutorial](../cl/azure.md#cluster). Define ```tf module "ramius-worker-pool" { - source = "git::https://github.com/poseidon/typhoon//azure/container-linux/kubernetes/workers?ref=v1.18.2" + source = "git::https://github.com/poseidon/typhoon//azure/container-linux/kubernetes/workers?ref=v1.18.3" # Azure region = module.ramius.region @@ -148,7 +148,7 @@ Create a cluster following the Google Cloud [tutorial](../cl/google-cloud.md#clu ```tf module "yavin-worker-pool" { - source = "git::https://github.com/poseidon/typhoon//google-cloud/container-linux/kubernetes/workers?ref=v1.18.2" + source = "git::https://github.com/poseidon/typhoon//google-cloud/container-linux/kubernetes/workers?ref=v1.18.3" # Google Cloud region = "europe-west2" @@ -179,11 +179,11 @@ Verify a managed instance group of workers joins the cluster within a few minute ``` $ kubectl get nodes NAME STATUS AGE VERSION -yavin-controller-0.c.example-com.internal Ready 6m v1.18.2 -yavin-worker-jrbf.c.example-com.internal Ready 5m v1.18.2 -yavin-worker-mzdm.c.example-com.internal Ready 5m v1.18.2 -yavin-16x-worker-jrbf.c.example-com.internal Ready 3m v1.18.2 -yavin-16x-worker-mzdm.c.example-com.internal Ready 3m v1.18.2 +yavin-controller-0.c.example-com.internal Ready 6m v1.18.3 +yavin-worker-jrbf.c.example-com.internal Ready 5m v1.18.3 +yavin-worker-mzdm.c.example-com.internal Ready 5m v1.18.3 +yavin-16x-worker-jrbf.c.example-com.internal Ready 3m v1.18.3 +yavin-16x-worker-mzdm.c.example-com.internal Ready 3m v1.18.3 ``` ### Variables diff --git a/docs/cl/aws.md b/docs/cl/aws.md index 0e028ac67..2ab50462e 100644 --- a/docs/cl/aws.md +++ b/docs/cl/aws.md @@ -1,6 +1,6 @@ # AWS -In this tutorial, we'll create a Kubernetes v1.18.2 cluster on AWS with CoreOS Container Linux or Flatcar Linux. +In this tutorial, we'll create a Kubernetes v1.18.3 cluster on AWS with CoreOS Container Linux or Flatcar Linux. We'll declare a Kubernetes cluster using the Typhoon Terraform module. Then apply the changes to create a VPC, gateway, subnets, security groups, controller instances, worker auto-scaling group, network load balancer, and TLS assets. @@ -70,7 +70,7 @@ Define a Kubernetes cluster using the module `aws/container-linux/kubernetes`. ```tf module "tempest" { - source = "git::https://github.com/poseidon/typhoon//aws/container-linux/kubernetes?ref=v1.18.2" + source = "git::https://github.com/poseidon/typhoon//aws/container-linux/kubernetes?ref=v1.18.3" # AWS cluster_name = "tempest" @@ -143,9 +143,9 @@ List nodes in the cluster. $ export KUBECONFIG=/home/user/.kube/configs/tempest-config $ kubectl get nodes NAME STATUS ROLES AGE VERSION -ip-10-0-3-155 Ready 10m v1.18.2 -ip-10-0-26-65 Ready 10m v1.18.2 -ip-10-0-41-21 Ready 10m v1.18.2 +ip-10-0-3-155 Ready 10m v1.18.3 +ip-10-0-26-65 Ready 10m v1.18.3 +ip-10-0-41-21 Ready 10m v1.18.3 ``` List the pods. diff --git a/docs/cl/azure.md b/docs/cl/azure.md index ec69ec9e8..b9fbae197 100644 --- a/docs/cl/azure.md +++ b/docs/cl/azure.md @@ -1,6 +1,6 @@ # Azure -In this tutorial, we'll create a Kubernetes v1.18.2 cluster on Azure with CoreOS Container Linux or Flatcar Linux. +In this tutorial, we'll create a Kubernetes v1.18.3 cluster on Azure with CoreOS Container Linux or Flatcar Linux. We'll declare a Kubernetes cluster using the Typhoon Terraform module. Then apply the changes to create a resource group, virtual network, subnets, security groups, controller availability set, worker scale set, load balancer, and TLS assets. @@ -72,7 +72,7 @@ Define a Kubernetes cluster using the module `azure/container-linux/kubernetes`. ```tf module "ramius" { - source = "git::https://github.com/poseidon/typhoon//azure/container-linux/kubernetes?ref=v1.18.2" + source = "git::https://github.com/poseidon/typhoon//azure/container-linux/kubernetes?ref=v1.18.3" # Azure cluster_name = "ramius" @@ -146,9 +146,9 @@ List nodes in the cluster. $ export KUBECONFIG=/home/user/.kube/configs/ramius-config $ kubectl get nodes NAME STATUS ROLES AGE VERSION -ramius-controller-0 Ready 24m v1.18.2 -ramius-worker-000001 Ready 25m v1.18.2 -ramius-worker-000002 Ready 24m v1.18.2 +ramius-controller-0 Ready 24m v1.18.3 +ramius-worker-000001 Ready 25m v1.18.3 +ramius-worker-000002 Ready 24m v1.18.3 ``` List the pods. diff --git a/docs/cl/bare-metal.md b/docs/cl/bare-metal.md index e9b8ab1ab..f580c25e1 100644 --- a/docs/cl/bare-metal.md +++ b/docs/cl/bare-metal.md @@ -1,6 +1,6 @@ # Bare-Metal -In this tutorial, we'll network boot and provision a Kubernetes v1.18.2 cluster on bare-metal with CoreOS Container Linux or Flatcar Linux. +In this tutorial, we'll network boot and provision a Kubernetes v1.18.3 cluster on bare-metal with CoreOS Container Linux or Flatcar Linux. First, we'll deploy a [Matchbox](https://github.com/poseidon/matchbox) service and setup a network boot environment. Then, we'll declare a Kubernetes cluster using the Typhoon Terraform module and power on machines. On PXE boot, machines will install Container Linux to disk, reboot into the disk install, and provision themselves as Kubernetes controllers or workers via Ignition. @@ -160,7 +160,7 @@ Define a Kubernetes cluster using the module `bare-metal/container-linux/kuberne ```tf module "mercury" { - source = "git::https://github.com/poseidon/typhoon//bare-metal/container-linux/kubernetes?ref=v1.18.2" + source = "git::https://github.com/poseidon/typhoon//bare-metal/container-linux/kubernetes?ref=v1.18.3" # bare-metal cluster_name = "mercury" @@ -299,9 +299,9 @@ List nodes in the cluster. $ export KUBECONFIG=/home/user/.kube/configs/mercury-config $ kubectl get nodes NAME STATUS ROLES AGE VERSION -node1.example.com Ready 10m v1.18.2 -node2.example.com Ready 10m v1.18.2 -node3.example.com Ready 10m v1.18.2 +node1.example.com Ready 10m v1.18.3 +node2.example.com Ready 10m v1.18.3 +node3.example.com Ready 10m v1.18.3 ``` List the pods. diff --git a/docs/cl/digital-ocean.md b/docs/cl/digital-ocean.md index 929e63f4c..24933bfe0 100644 --- a/docs/cl/digital-ocean.md +++ b/docs/cl/digital-ocean.md @@ -1,6 +1,6 @@ # Digital Ocean -In this tutorial, we'll create a Kubernetes v1.18.2 cluster on DigitalOcean with CoreOS Container Linux or Flatcar Linux. +In this tutorial, we'll create a Kubernetes v1.18.3 cluster on DigitalOcean with CoreOS Container Linux or Flatcar Linux. We'll declare a Kubernetes cluster using the Typhoon Terraform module. Then apply the changes to create controller droplets, worker droplets, DNS records, tags, and TLS assets. @@ -79,7 +79,7 @@ Define a Kubernetes cluster using the module `digital-ocean/container-linux/kube ```tf module "nemo" { - source = "git::https://github.com/poseidon/typhoon//digital-ocean/container-linux/kubernetes?ref=v1.18.2" + source = "git::https://github.com/poseidon/typhoon//digital-ocean/container-linux/kubernetes?ref=v1.18.3" # Digital Ocean cluster_name = "nemo" @@ -153,9 +153,9 @@ List nodes in the cluster. $ export KUBECONFIG=/home/user/.kube/configs/nemo-config $ kubectl get nodes NAME STATUS ROLES AGE VERSION -10.132.110.130 Ready 10m v1.18.2 -10.132.115.81 Ready 10m v1.18.2 -10.132.124.107 Ready 10m v1.18.2 +10.132.110.130 Ready 10m v1.18.3 +10.132.115.81 Ready 10m v1.18.3 +10.132.124.107 Ready 10m v1.18.3 ``` List the pods. diff --git a/docs/cl/google-cloud.md b/docs/cl/google-cloud.md index a1cbb7b43..655f6ff57 100644 --- a/docs/cl/google-cloud.md +++ b/docs/cl/google-cloud.md @@ -1,6 +1,6 @@ # Google Cloud -In this tutorial, we'll create a Kubernetes v1.18.2 cluster on Google Compute Engine with CoreOS Container Linux or Flatcar Linux. +In this tutorial, we'll create a Kubernetes v1.18.3 cluster on Google Compute Engine with CoreOS Container Linux or Flatcar Linux. We'll declare a Kubernetes cluster using the Typhoon Terraform module. Then apply the changes to create a network, firewall rules, health checks, controller instances, worker managed instance group, load balancers, and TLS assets. @@ -90,7 +90,7 @@ Define a Kubernetes cluster using the module `google-cloud/container-linux/kuber ```tf module "yavin" { - source = "git::https://github.com/poseidon/typhoon//google-cloud/container-linux/kubernetes?ref=v1.18.2" + source = "git::https://github.com/poseidon/typhoon//google-cloud/container-linux/kubernetes?ref=v1.18.3" # Google Cloud cluster_name = "yavin" @@ -165,9 +165,9 @@ List nodes in the cluster. $ export KUBECONFIG=/home/user/.kube/configs/yavin-config $ kubectl get nodes NAME ROLES STATUS AGE VERSION -yavin-controller-0.c.example-com.internal Ready 6m v1.18.2 -yavin-worker-jrbf.c.example-com.internal Ready 5m v1.18.2 -yavin-worker-mzdm.c.example-com.internal Ready 5m v1.18.2 +yavin-controller-0.c.example-com.internal Ready 6m v1.18.3 +yavin-worker-jrbf.c.example-com.internal Ready 5m v1.18.3 +yavin-worker-mzdm.c.example-com.internal Ready 5m v1.18.3 ``` List the pods. diff --git a/docs/fedora-coreos/aws.md b/docs/fedora-coreos/aws.md index 5df5b1f17..56f8abc07 100644 --- a/docs/fedora-coreos/aws.md +++ b/docs/fedora-coreos/aws.md @@ -1,6 +1,6 @@ # AWS -In this tutorial, we'll create a Kubernetes v1.18.2 cluster on AWS with Fedora CoreOS. +In this tutorial, we'll create a Kubernetes v1.18.3 cluster on AWS with Fedora CoreOS. We'll declare a Kubernetes cluster using the Typhoon Terraform module. Then apply the changes to create a VPC, gateway, subnets, security groups, controller instances, worker auto-scaling group, network load balancer, and TLS assets. @@ -70,7 +70,7 @@ Define a Kubernetes cluster using the module `aws/fedora-coreos/kubernetes`. ```tf module "tempest" { - source = "git::https://github.com/poseidon/typhoon//aws/fedora-coreos/kubernetes?ref=v1.18.2" + source = "git::https://github.com/poseidon/typhoon//aws/fedora-coreos/kubernetes?ref=v1.18.3" # AWS cluster_name = "tempest" @@ -143,9 +143,9 @@ List nodes in the cluster. $ export KUBECONFIG=/home/user/.kube/configs/tempest-config $ kubectl get nodes NAME STATUS ROLES AGE VERSION -ip-10-0-3-155 Ready 10m v1.18.2 -ip-10-0-26-65 Ready 10m v1.18.2 -ip-10-0-41-21 Ready 10m v1.18.2 +ip-10-0-3-155 Ready 10m v1.18.3 +ip-10-0-26-65 Ready 10m v1.18.3 +ip-10-0-41-21 Ready 10m v1.18.3 ``` List the pods. diff --git a/docs/fedora-coreos/azure.md b/docs/fedora-coreos/azure.md index 3a49620d2..45d44e88a 100644 --- a/docs/fedora-coreos/azure.md +++ b/docs/fedora-coreos/azure.md @@ -1,6 +1,6 @@ # Azure -In this tutorial, we'll create a Kubernetes v1.18.2 cluster on Azure with Fedora CoreOS. +In this tutorial, we'll create a Kubernetes v1.18.3 cluster on Azure with Fedora CoreOS. We'll declare a Kubernetes cluster using the Typhoon Terraform module. Then apply the changes to create a resource group, virtual network, subnets, security groups, controller availability set, worker scale set, load balancer, and TLS assets. @@ -83,7 +83,7 @@ Define a Kubernetes cluster using the module `azure/fedora-coreos/kubernetes`. ```tf module "ramius" { - source = "git::https://github.com/poseidon/typhoon//azure/fedora-coreos/kubernetes?ref=v1.18.2" + source = "git::https://github.com/poseidon/typhoon//azure/fedora-coreos/kubernetes?ref=v1.18.3" # Azure cluster_name = "ramius" @@ -158,9 +158,9 @@ List nodes in the cluster. $ export KUBECONFIG=/home/user/.kube/configs/ramius-config $ kubectl get nodes NAME STATUS ROLES AGE VERSION -ramius-controller-0 Ready 24m v1.18.2 -ramius-worker-000001 Ready 25m v1.18.2 -ramius-worker-000002 Ready 24m v1.18.2 +ramius-controller-0 Ready 24m v1.18.3 +ramius-worker-000001 Ready 25m v1.18.3 +ramius-worker-000002 Ready 24m v1.18.3 ``` List the pods. diff --git a/docs/fedora-coreos/bare-metal.md b/docs/fedora-coreos/bare-metal.md index f789db64a..0a50e13ba 100644 --- a/docs/fedora-coreos/bare-metal.md +++ b/docs/fedora-coreos/bare-metal.md @@ -1,6 +1,6 @@ # Bare-Metal -In this tutorial, we'll network boot and provision a Kubernetes v1.18.2 cluster on bare-metal with Fedora CoreOS. +In this tutorial, we'll network boot and provision a Kubernetes v1.18.3 cluster on bare-metal with Fedora CoreOS. First, we'll deploy a [Matchbox](https://github.com/poseidon/matchbox) service and setup a network boot environment. Then, we'll declare a Kubernetes cluster using the Typhoon Terraform module and power on machines. On PXE boot, machines will install Fedora CoreOS to disk, reboot into the disk install, and provision themselves as Kubernetes controllers or workers via Ignition. @@ -160,7 +160,7 @@ Define a Kubernetes cluster using the module `bare-metal/fedora-coreos/kubernete ```tf module "mercury" { - source = "git::https://github.com/poseidon/typhoon//bare-metal/fedora-coreos/kubernetes?ref=v1.18.2" + source = "git::https://github.com/poseidon/typhoon//bare-metal/fedora-coreos/kubernetes?ref=v1.18.3" # bare-metal cluster_name = "mercury" @@ -289,9 +289,9 @@ List nodes in the cluster. $ export KUBECONFIG=/home/user/.kube/configs/mercury-config $ kubectl get nodes NAME STATUS ROLES AGE VERSION -node1.example.com Ready 10m v1.18.2 -node2.example.com Ready 10m v1.18.2 -node3.example.com Ready 10m v1.18.2 +node1.example.com Ready 10m v1.18.3 +node2.example.com Ready 10m v1.18.3 +node3.example.com Ready 10m v1.18.3 ``` List the pods. diff --git a/docs/fedora-coreos/digitalocean.md b/docs/fedora-coreos/digitalocean.md index a13db6616..faffac252 100644 --- a/docs/fedora-coreos/digitalocean.md +++ b/docs/fedora-coreos/digitalocean.md @@ -1,6 +1,6 @@ # Digital Ocean -In this tutorial, we'll create a Kubernetes v1.18.2 cluster on DigitalOcean with Fedora CoreOS. +In this tutorial, we'll create a Kubernetes v1.18.3 cluster on DigitalOcean with Fedora CoreOS. We'll declare a Kubernetes cluster using the Typhoon Terraform module. Then apply the changes to create controller droplets, worker droplets, DNS records, tags, and TLS assets. @@ -79,7 +79,7 @@ Define a Kubernetes cluster using the module `digital-ocean/fedora-coreos/kubern ```tf module "nemo" { - source = "git::https://github.com/poseidon/typhoon//digital-ocean/fedora-coreos/kubernetes?ref=v1.18.2" + source = "git::https://github.com/poseidon/typhoon//digital-ocean/fedora-coreos/kubernetes?ref=v1.18.3" # Digital Ocean cluster_name = "nemo" @@ -153,9 +153,9 @@ List nodes in the cluster. $ export KUBECONFIG=/home/user/.kube/configs/nemo-config $ kubectl get nodes NAME STATUS ROLES AGE VERSION -10.132.110.130 Ready 10m v1.18.2 -10.132.115.81 Ready 10m v1.18.2 -10.132.124.107 Ready 10m v1.18.2 +10.132.110.130 Ready 10m v1.18.3 +10.132.115.81 Ready 10m v1.18.3 +10.132.124.107 Ready 10m v1.18.3 ``` List the pods. diff --git a/docs/fedora-coreos/google-cloud.md b/docs/fedora-coreos/google-cloud.md index 6dba7819b..005ccfaae 100644 --- a/docs/fedora-coreos/google-cloud.md +++ b/docs/fedora-coreos/google-cloud.md @@ -1,6 +1,6 @@ # Google Cloud -In this tutorial, we'll create a Kubernetes v1.18.2 cluster on Google Compute Engine with Fedora CoreOS. +In this tutorial, we'll create a Kubernetes v1.18.3 cluster on Google Compute Engine with Fedora CoreOS. We'll declare a Kubernetes cluster using the Typhoon Terraform module. Then apply the changes to create a network, firewall rules, health checks, controller instances, worker managed instance group, load balancers, and TLS assets. @@ -145,9 +145,9 @@ List nodes in the cluster. $ export KUBECONFIG=/home/user/.kube/configs/yavin-config $ kubectl get nodes NAME ROLES STATUS AGE VERSION -yavin-controller-0.c.example-com.internal Ready 6m v1.18.2 -yavin-worker-jrbf.c.example-com.internal Ready 5m v1.18.2 -yavin-worker-mzdm.c.example-com.internal Ready 5m v1.18.2 +yavin-controller-0.c.example-com.internal Ready 6m v1.18.3 +yavin-worker-jrbf.c.example-com.internal Ready 5m v1.18.3 +yavin-worker-mzdm.c.example-com.internal Ready 5m v1.18.3 ``` List the pods. diff --git a/docs/index.md b/docs/index.md index dc0d3fdda..d8f35ef71 100644 --- a/docs/index.md +++ b/docs/index.md @@ -11,7 +11,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features -* Kubernetes v1.18.2 (upstream) +* Kubernetes v1.18.3 (upstream) * Single or multi-master, [Calico](https://www.projectcalico.org/) or [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/), SELinux enforcing * Advanced features like [worker pools](advanced/worker-pools/), [preemptible](cl/google-cloud/#preemption) workers, and [snippets](advanced/customization/#container-linux) customization @@ -64,7 +64,7 @@ Define a Kubernetes cluster by using the Terraform module for your chosen platfo ```tf module "yavin" { - source = "git::https://github.com/poseidon/typhoon//google-cloud/fedora-coreos/kubernetes?ref=v1.18.2" + source = "git::https://github.com/poseidon/typhoon//google-cloud/fedora-coreos/kubernetes?ref=v1.18.3" # Google Cloud cluster_name = "yavin" @@ -102,9 +102,9 @@ In 4-8 minutes (varies by platform), the cluster will be ready. This Google Clou $ export KUBECONFIG=/home/user/.kube/configs/yavin-config $ kubectl get nodes NAME ROLES STATUS AGE VERSION -yavin-controller-0.c.example-com.internal Ready 6m v1.18.2 -yavin-worker-jrbf.c.example-com.internal Ready 5m v1.18.2 -yavin-worker-mzdm.c.example-com.internal Ready 5m v1.18.2 +yavin-controller-0.c.example-com.internal Ready 6m v1.18.3 +yavin-worker-jrbf.c.example-com.internal Ready 5m v1.18.3 +yavin-worker-mzdm.c.example-com.internal Ready 5m v1.18.3 ``` List the pods. diff --git a/docs/topics/maintenance.md b/docs/topics/maintenance.md index b01a610f8..f020a293c 100644 --- a/docs/topics/maintenance.md +++ b/docs/topics/maintenance.md @@ -18,7 +18,7 @@ module "yavin" { } module "mercury" { - source = "git::https://github.com/poseidon/typhoon//bare-metal/container-linux/kubernetes?ref=v1.18.2" + source = "git::https://github.com/poseidon/typhoon//bare-metal/container-linux/kubernetes?ref=v1.18.3" ... } ``` @@ -279,15 +279,15 @@ Typhoon modules have been adapted for Terraform v0.12. Provider plugins requirem | Typhoon Release | Terraform version | |-------------------|---------------------| -| v1.18.2 - ? | v0.12.x | -| v1.10.3 - v1.18.2 | v0.11.x | +| v1.18.3 - ? | v0.12.x | +| v1.10.3 - v1.18.3 | v0.11.x | | v1.9.2 - v1.10.2 | v0.10.4+ or v0.11.x | | v1.7.3 - v1.9.1 | v0.10.x | | v1.6.4 - v1.7.2 | v0.9.x | ### New users -New users can start with Terraform v0.12.x and follow the docs for Typhoon v1.18.2+ without issue. +New users can start with Terraform v0.12.x and follow the docs for Typhoon v1.18.3+ without issue. ### Existing users diff --git a/google-cloud/container-linux/kubernetes/README.md b/google-cloud/container-linux/kubernetes/README.md index 1035adfae..43e8b17ff 100644 --- a/google-cloud/container-linux/kubernetes/README.md +++ b/google-cloud/container-linux/kubernetes/README.md @@ -11,7 +11,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features -* Kubernetes v1.18.2 (upstream) +* Kubernetes v1.18.3 (upstream) * Single or multi-master, [Calico](https://www.projectcalico.org/) or [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/) * Advanced features like [worker pools](https://typhoon.psdn.io/advanced/worker-pools/), [preemptible](https://typhoon.psdn.io/cl/google-cloud/#preemption) workers, and [snippets](https://typhoon.psdn.io/advanced/customization/#container-linux) customization diff --git a/google-cloud/container-linux/kubernetes/bootstrap.tf b/google-cloud/container-linux/kubernetes/bootstrap.tf index 452c96748..f7cc894d9 100644 --- a/google-cloud/container-linux/kubernetes/bootstrap.tf +++ b/google-cloud/container-linux/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=157336db92daf1f1b7f09ab2b3ccbe16a4b66c57" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=ff7ec52d0a5e97b8ca6b86a80a7e5e1ea8570487" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/google-cloud/container-linux/kubernetes/cl/controller.yaml b/google-cloud/container-linux/kubernetes/cl/controller.yaml index baf1bd4ec..994dfb7b5 100644 --- a/google-cloud/container-linux/kubernetes/cl/controller.yaml +++ b/google-cloud/container-linux/kubernetes/cl/controller.yaml @@ -52,7 +52,7 @@ systemd: Description=Kubelet Wants=rpc-statd.service [Service] - Environment=KUBELET_IMAGE=docker://quay.io/poseidon/kubelet:v1.18.2 + Environment=KUBELET_IMAGE=docker://quay.io/poseidon/kubelet:v1.18.3 ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests ExecStartPre=/bin/mkdir -p /opt/cni/bin @@ -135,7 +135,7 @@ systemd: --volume script,kind=host,source=/opt/bootstrap/apply \ --mount volume=script,target=/apply \ --insecure-options=image \ - docker://quay.io/poseidon/kubelet:v1.18.2 \ + docker://quay.io/poseidon/kubelet:v1.18.3 \ --net=host \ --dns=host \ --exec=/apply diff --git a/google-cloud/container-linux/kubernetes/workers/cl/worker.yaml b/google-cloud/container-linux/kubernetes/workers/cl/worker.yaml index d5e7ae852..f050d52ee 100644 --- a/google-cloud/container-linux/kubernetes/workers/cl/worker.yaml +++ b/google-cloud/container-linux/kubernetes/workers/cl/worker.yaml @@ -25,7 +25,7 @@ systemd: Description=Kubelet Wants=rpc-statd.service [Service] - Environment=KUBELET_IMAGE=docker://quay.io/poseidon/kubelet:v1.18.2 + Environment=KUBELET_IMAGE=docker://quay.io/poseidon/kubelet:v1.18.3 ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests ExecStartPre=/bin/mkdir -p /opt/cni/bin @@ -128,7 +128,7 @@ storage: --volume config,kind=host,source=/etc/kubernetes \ --mount volume=config,target=/etc/kubernetes \ --insecure-options=image \ - docker://quay.io/poseidon/kubelet:v1.18.2 \ + docker://quay.io/poseidon/kubelet:v1.18.3 \ --net=host \ --dns=host \ --exec=/usr/local/bin/kubectl -- --kubeconfig=/etc/kubernetes/kubeconfig delete node $(hostname) diff --git a/google-cloud/fedora-coreos/kubernetes/README.md b/google-cloud/fedora-coreos/kubernetes/README.md index 69c5e585a..3ed7b3cbf 100644 --- a/google-cloud/fedora-coreos/kubernetes/README.md +++ b/google-cloud/fedora-coreos/kubernetes/README.md @@ -11,7 +11,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features -* Kubernetes v1.18.2 (upstream) +* Kubernetes v1.18.3 (upstream) * Single or multi-master, [Calico](https://www.projectcalico.org/) or [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/), SELinux enforcing * Advanced features like [worker pools](https://typhoon.psdn.io/advanced/worker-pools/), [preemptible](https://typhoon.psdn.io/fedora-coreos/google-cloud/#preemption) workers, and [snippets](https://typhoon.psdn.io/advanced/customization/) customization diff --git a/google-cloud/fedora-coreos/kubernetes/bootstrap.tf b/google-cloud/fedora-coreos/kubernetes/bootstrap.tf index 2358bd744..8f9c4c033 100644 --- a/google-cloud/fedora-coreos/kubernetes/bootstrap.tf +++ b/google-cloud/fedora-coreos/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=157336db92daf1f1b7f09ab2b3ccbe16a4b66c57" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=ff7ec52d0a5e97b8ca6b86a80a7e5e1ea8570487" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/google-cloud/fedora-coreos/kubernetes/fcc/controller.yaml b/google-cloud/fedora-coreos/kubernetes/fcc/controller.yaml index f5c80984f..a98424223 100644 --- a/google-cloud/fedora-coreos/kubernetes/fcc/controller.yaml +++ b/google-cloud/fedora-coreos/kubernetes/fcc/controller.yaml @@ -54,7 +54,7 @@ systemd: Description=Kubelet (System Container) Wants=rpc-statd.service [Service] - Environment=KUBELET_IMAGE=quay.io/poseidon/kubelet:v1.18.2 + Environment=KUBELET_IMAGE=quay.io/poseidon/kubelet:v1.18.3 ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests ExecStartPre=/bin/mkdir -p /opt/cni/bin @@ -126,7 +126,7 @@ systemd: --volume /opt/bootstrap/assets:/assets:ro,Z \ --volume /opt/bootstrap/apply:/apply:ro,Z \ --entrypoint=/apply \ - quay.io/poseidon/kubelet:v1.18.2 + quay.io/poseidon/kubelet:v1.18.3 ExecStartPost=/bin/touch /opt/bootstrap/bootstrap.done ExecStartPost=-/usr/bin/podman stop bootstrap storage: diff --git a/google-cloud/fedora-coreos/kubernetes/workers/fcc/worker.yaml b/google-cloud/fedora-coreos/kubernetes/workers/fcc/worker.yaml index 2fbebc733..d33c43db5 100644 --- a/google-cloud/fedora-coreos/kubernetes/workers/fcc/worker.yaml +++ b/google-cloud/fedora-coreos/kubernetes/workers/fcc/worker.yaml @@ -24,7 +24,7 @@ systemd: Description=Kubelet (System Container) Wants=rpc-statd.service [Service] - Environment=KUBELET_IMAGE=quay.io/poseidon/kubelet:v1.18.2 + Environment=KUBELET_IMAGE=quay.io/poseidon/kubelet:v1.18.3 ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests ExecStartPre=/bin/mkdir -p /opt/cni/bin @@ -90,7 +90,7 @@ systemd: Type=oneshot RemainAfterExit=true ExecStart=/bin/true - ExecStop=/bin/bash -c '/usr/bin/podman run --volume /etc/kubernetes:/etc/kubernetes:ro,z --entrypoint /usr/local/bin/kubectl quay.io/poseidon/kubelet:v1.18.2 --kubeconfig=/etc/kubernetes/kubeconfig delete node $HOSTNAME' + ExecStop=/bin/bash -c '/usr/bin/podman run --volume /etc/kubernetes:/etc/kubernetes:ro,z --entrypoint /usr/local/bin/kubectl quay.io/poseidon/kubelet:v1.18.3 --kubeconfig=/etc/kubernetes/kubeconfig delete node $HOSTNAME' [Install] WantedBy=multi-user.target storage: From c52f9f8d08adb8bf1fec440fa150f9ec13ad69ea Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Wed, 20 May 2020 22:53:51 -0700 Subject: [PATCH 428/523] Upgrade docs packages and refresh content * Promote DigitalOcean from alpha to beta for Fedora CoreOS and Flatcar Linux * Upgrade mkdocs-material and PyPI packages for docs * Replace docs mentions of Container Linux with Flatcar Linux and move docs/cl to docs/flatcar-linux * Deprecate CoreOS Container Linux support. Its still usable for some time, but start removing docs --- CHANGES.md | 18 +- README.md | 20 +- docs/advanced/worker-pools.md | 6 +- docs/architecture/operating-systems.md | 2 +- docs/{cl => flatcar-linux}/aws.md | 0 docs/{cl => flatcar-linux}/azure.md | 0 docs/{cl => flatcar-linux}/bare-metal.md | 0 .../digitalocean.md} | 0 docs/{cl => flatcar-linux}/google-cloud.md | 0 docs/index.md | 29 +-- docs/topics/faq.md | 9 - docs/topics/maintenance.md | 240 +----------------- mkdocs.yml | 44 ++-- requirements.txt | 6 +- 14 files changed, 73 insertions(+), 301 deletions(-) rename docs/{cl => flatcar-linux}/aws.md (100%) rename docs/{cl => flatcar-linux}/azure.md (100%) rename docs/{cl => flatcar-linux}/bare-metal.md (100%) rename docs/{cl/digital-ocean.md => flatcar-linux/digitalocean.md} (100%) rename docs/{cl => flatcar-linux}/google-cloud.md (100%) diff --git a/CHANGES.md b/CHANGES.md index 5b7d9b878..d5a8a0849 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -2,12 +2,16 @@ Notable changes between versions. -## Latest +## v1.18.3 * Use Kubelet [TLS bootstrap](https://kubernetes.io/docs/reference/command-line-tools-reference/kubelet-tls-bootstrapping/) with bootstrap token authentication ([#713](https://github.com/poseidon/typhoon/pull/713)) * Enable Node [Authorization](https://kubernetes.io/docs/reference/access-authn-authz/node/) and [NodeRestriction](https://kubernetes.io/docs/reference/access-authn-authz/admission-controllers/#noderestriction) to reduce authorization scope * Renew Kubelet certificates every 72 hours +* Add CoreDNS node affinity preference for controller nodes ([#188](https://github.com/poseidon/terraform-render-bootstrap/pull/188)) * Update Calico from v3.13.1 to [v3.14.0](https://docs.projectcalico.org/v3.14/release-notes/) +* Deprecate CoreOS Container Linux support (no OS [updates](https://coreos.com/os/eol/) after May 2020) + * Use a `fedora-coreos` module for Fedora CoreOS + * Use a `container-linux` module for Flatcar Linux ### AWS @@ -20,6 +24,10 @@ Notable changes between versions. * Fix warning that `address_prefix` is deprecated * Require `terraform-provider-azurerm` v2.8.0+ (action required) +### DigitalOcean + +* Promote DigitalOcean to beta on both Fedora CoreOS and Flatcar Linux + ### Fedora CoreOS * Fix Calico `install-cni` crashloop on Pod restarts ([#724](https://github.com/poseidon/typhoon/pull/724)) @@ -28,13 +36,13 @@ Notable changes between versions. #### AWS -* Support Fedora CoreOS official [image streams](https://docs.fedoraproject.org/en-US/fedora-coreos/update-streams/) ([#727](https://github.com/poseidon/typhoon/pull/727)) +* Support Fedora CoreOS [image streams](https://docs.fedoraproject.org/en-US/fedora-coreos/update-streams/) ([#727](https://github.com/poseidon/typhoon/pull/727)) * Add `os_stream` variable to set the stream to `stable` (default), `testing`, or `next` * Remove unused `os_image` variable #### Google -* Support Fedora CoreOS official [image streams](https://docs.fedoraproject.org/en-US/fedora-coreos/update-streams/) ([#723](https://github.com/poseidon/typhoon/pull/722)) +* Support Fedora CoreOS [image streams](https://docs.fedoraproject.org/en-US/fedora-coreos/update-streams/) ([#723](https://github.com/poseidon/typhoon/pull/722)) * Add `os_stream` variable to set the stream to `stable` (default), `testing`, or `next` * Deprecate `os_image` variable. Manual image uploads are no longer needed @@ -46,6 +54,10 @@ Notable changes between versions. * Restore [#664](https://github.com/poseidon/typhoon/pull/664) (reverted in [#707](https://github.com/poseidon/typhoon/pull/707)) but use Flatcar Linux new free offer (not byol) * Change `os_image` to use a `flatcar-stable` default +#### Google + +* Promote Flatcar Linux to beta + ### Addons * Update nginx-ingress from v0.30.0 to [v0.32.0](https://github.com/kubernetes/ingress-nginx/releases/tag/nginx-0.32.0) diff --git a/README.md b/README.md index 5d5557e14..bd592f4b3 100644 --- a/README.md +++ b/README.md @@ -28,35 +28,25 @@ Typhoon is available for [Fedora CoreOS](https://getfedora.org/coreos/). | AWS | Fedora CoreOS | [aws/fedora-coreos/kubernetes](aws/fedora-coreos/kubernetes) | stable | | Azure | Fedora CoreOS | [azure/fedora-coreos/kubernetes](azure/fedora-coreos/kubernetes) | alpha | | Bare-Metal | Fedora CoreOS | [bare-metal/fedora-coreos/kubernetes](bare-metal/fedora-coreos/kubernetes) | beta | -| DigitalOcean | Fedora CoreOS | [digital-ocean/fedora-coreos/kubernetes](digital-ocean/fedora-coreos/kubernetes) | alpha | +| DigitalOcean | Fedora CoreOS | [digital-ocean/fedora-coreos/kubernetes](digital-ocean/fedora-coreos/kubernetes) | beta | | Google Cloud | Fedora CoreOS | [google-cloud/fedora-coreos/kubernetes](google-cloud/fedora-coreos/kubernetes) | beta | -Typhoon is available for [Flatcar Container Linux](https://www.flatcar-linux.org/releases/). +Typhoon is available for [Flatcar Linux](https://www.flatcar-linux.org/releases/). | Platform | Operating System | Terraform Module | Status | |---------------|------------------|------------------|--------| | AWS | Flatcar Linux | [aws/container-linux/kubernetes](aws/container-linux/kubernetes) | stable | | Azure | Flatcar Linux | [azure/container-linux/kubernetes](azure/container-linux/kubernetes) | alpha | | Bare-Metal | Flatcar Linux | [bare-metal/container-linux/kubernetes](bare-metal/container-linux/kubernetes) | stable | -| DigitalOcean | Flatcar Linux | [digital-ocean/container-linux/kubernetes](digital-ocean/container-linux/kubernetes) | alpha | -| Google Cloud | Flatcar Linux | [google-cloud/container-linux/kubernetes](google-cloud/container-linux/kubernetes) | alpha | - -Typhoon is available for CoreOS Container Linux ([no updates](https://coreos.com/os/eol/) after May 2020). - -| Platform | Operating System | Terraform Module | Status | -|---------------|------------------|------------------|--------| -| AWS | Container Linux | [aws/container-linux/kubernetes](aws/container-linux/kubernetes) | stable | -| Azure | Container Linux | [azure/container-linux/kubernetes](azure/container-linux/kubernetes) | alpha | -| Bare-Metal | Container Linux | [bare-metal/container-linux/kubernetes](bare-metal/container-linux/kubernetes) | stable | -| Digital Ocean | Container Linux | [digital-ocean/container-linux/kubernetes](digital-ocean/container-linux/kubernetes) | beta | -| Google Cloud | Container Linux | [google-cloud/container-linux/kubernetes](google-cloud/container-linux/kubernetes) | stable | +| DigitalOcean | Flatcar Linux | [digital-ocean/container-linux/kubernetes](digital-ocean/container-linux/kubernetes) | beta | +| Google Cloud | Flatcar Linux | [google-cloud/container-linux/kubernetes](google-cloud/container-linux/kubernetes) | beta | ## Documentation * [Docs](https://typhoon.psdn.io) * Architecture [concepts](https://typhoon.psdn.io/architecture/concepts/) and [operating systems](https://typhoon.psdn.io/architecture/operating-systems/) * Fedora CoreOS tutorials for [AWS](docs/fedora-coreos/aws.md), [Azure](docs/fedora-coreos/azure.md), [Bare-Metal](docs/fedora-coreos/bare-metal.md), [DigitalOcean](docs/fedora-coreos/digitalocean.md), and [Google Cloud](docs/fedora-coreos/google-cloud.md) -* Flatcar Linux tutorials for [AWS](docs/cl/aws.md), [Azure](docs/cl/azure.md), [Bare-Metal](docs/cl/bare-metal.md), [DigitalOcean](docs/cl/digital-ocean.md), and [Google Cloud](docs/cl/google-cloud.md) +* Flatcar Linux tutorials for [AWS](docs/flatcar-linux/aws.md), [Azure](docs/flatcar-linux/azure.md), [Bare-Metal](docs/flatcar-linux/bare-metal.md), [DigitalOcean](docs/flatcar-linux/digitalocean.md), and [Google Cloud](docs/flatcar-linux/google-cloud.md) ## Usage diff --git a/docs/advanced/worker-pools.md b/docs/advanced/worker-pools.md index 77d351f59..bd0046c2d 100644 --- a/docs/advanced/worker-pools.md +++ b/docs/advanced/worker-pools.md @@ -13,7 +13,7 @@ Internal Terraform Modules: ## AWS -Create a cluster following the AWS [tutorial](../cl/aws.md#cluster). Define a worker pool using the AWS internal `workers` module. +Create a cluster following the AWS [tutorial](../flatcar-linux/aws.md#cluster). Define a worker pool using the AWS internal `workers` module. ```tf module "tempest-worker-pool" { @@ -78,7 +78,7 @@ Check the list of valid [instance types](https://aws.amazon.com/ec2/instance-typ ## Azure -Create a cluster following the Azure [tutorial](../cl/azure.md#cluster). Define a worker pool using the Azure internal `workers` module. +Create a cluster following the Azure [tutorial](../flatcar-linux/azure.md#cluster). Define a worker pool using the Azure internal `workers` module. ```tf module "ramius-worker-pool" { @@ -144,7 +144,7 @@ Check the list of valid [machine types](https://azure.microsoft.com/en-us/pricin ## Google Cloud -Create a cluster following the Google Cloud [tutorial](../cl/google-cloud.md#cluster). Define a worker pool using the Google Cloud internal `workers` module. +Create a cluster following the Google Cloud [tutorial](../flatcar-linux/google-cloud.md#cluster). Define a worker pool using the Google Cloud internal `workers` module. ```tf module "yavin-worker-pool" { diff --git a/docs/architecture/operating-systems.md b/docs/architecture/operating-systems.md index 276aff1cb..3ebdbd051 100644 --- a/docs/architecture/operating-systems.md +++ b/docs/architecture/operating-systems.md @@ -1,6 +1,6 @@ # Operating Systems -Typhoon supports [Fedora CoreOS](https://getfedora.org/coreos/), [Flatcar Linux](https://www.flatcar-linux.org/) and Container Linux (EOL in May 2020). These operating systems were chosen because they offer: +Typhoon supports [Fedora CoreOS](https://getfedora.org/coreos/) and [Flatcar Linux](https://www.flatcar-linux.org/). These operating systems were chosen because they offer: * Minimalism and focus on clustered operation * Automated and atomic operating system upgrades diff --git a/docs/cl/aws.md b/docs/flatcar-linux/aws.md similarity index 100% rename from docs/cl/aws.md rename to docs/flatcar-linux/aws.md diff --git a/docs/cl/azure.md b/docs/flatcar-linux/azure.md similarity index 100% rename from docs/cl/azure.md rename to docs/flatcar-linux/azure.md diff --git a/docs/cl/bare-metal.md b/docs/flatcar-linux/bare-metal.md similarity index 100% rename from docs/cl/bare-metal.md rename to docs/flatcar-linux/bare-metal.md diff --git a/docs/cl/digital-ocean.md b/docs/flatcar-linux/digitalocean.md similarity index 100% rename from docs/cl/digital-ocean.md rename to docs/flatcar-linux/digitalocean.md diff --git a/docs/cl/google-cloud.md b/docs/flatcar-linux/google-cloud.md similarity index 100% rename from docs/cl/google-cloud.md rename to docs/flatcar-linux/google-cloud.md diff --git a/docs/index.md b/docs/index.md index d8f35ef71..7bd25fd5b 100644 --- a/docs/index.md +++ b/docs/index.md @@ -14,7 +14,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster * Kubernetes v1.18.3 (upstream) * Single or multi-master, [Calico](https://www.projectcalico.org/) or [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/), SELinux enforcing -* Advanced features like [worker pools](advanced/worker-pools/), [preemptible](cl/google-cloud/#preemption) workers, and [snippets](advanced/customization/#container-linux) customization +* Advanced features like [worker pools](advanced/worker-pools/), [preemptible](fedora-coreos/google-cloud/#preemption) workers, and [snippets](advanced/customization/#container-linux) customization * Ready for Ingress, Prometheus, Grafana, CSI, or other [addons](addons/overview/) ## Modules @@ -28,35 +28,24 @@ Typhoon is available for [Fedora CoreOS](https://getfedora.org/coreos/). | AWS | Fedora CoreOS | [aws/fedora-coreos/kubernetes](fedora-coreos/aws.md) | stable | | Azure | Fedora CoreOS | [azure/fedora-coreos/kubernetes](fedora-coreos/azure.md) | alpha | | Bare-Metal | Fedora CoreOS | [bare-metal/fedora-coreos/kubernetes](fedora-coreos/bare-metal.md) | beta | -| DigitalOcean | Fedora CoreOS | [digital-ocean/fedora-coreos/kubernetes](fedora-coreos/digitalocean.md) | alpha | +| DigitalOcean | Fedora CoreOS | [digital-ocean/fedora-coreos/kubernetes](fedora-coreos/digitalocean.md) | beta | | Google Cloud | Fedora CoreOS | [google-cloud/fedora-coreos/kubernetes](google-cloud/fedora-coreos/kubernetes) | beta | -Typhoon is available for [Flatcar Container Linux](https://www.flatcar-linux.org/releases/). +Typhoon is available for [Flatcar Linux](https://www.flatcar-linux.org/releases/). | Platform | Operating System | Terraform Module | Status | |---------------|------------------|------------------|--------| -| AWS | Flatcar Linux | [aws/container-linux/kubernetes](cl/aws.md) | stable | -| Azure | Flatcar Linux | [azure/container-linux/kubernetes](cl/azure.md) | alpha | -| Bare-Metal | Flatcar Linux | [bare-metal/container-linux/kubernetes](cl/bare-metal.md) | stable | -| DigitalOcean | Flatcar Linux | [digital-ocean/container-linux/kubernetes](cl/digital-ocean.md) | alpha | -| Google Cloud | Flatcar Linux | [google-cloud/container-linux/kubernetes](cl/google-cloud.md) | alpha | - -Typhoon is available for CoreOS Container Linux ([no updates](https://coreos.com/os/eol/) after May 2020). - -| Platform | Operating System | Terraform Module | Status | -|---------------|------------------|------------------|--------| -| AWS | Container Linux | [aws/container-linux/kubernetes](cl/aws.md) | stable | -| Azure | Container Linux | [azure/container-linux/kubernetes](cl/azure.md) | alpha | -| Bare-Metal | Container Linux | [bare-metal/container-linux/kubernetes](cl/bare-metal.md) | stable | -| Digital Ocean | Container Linux | [digital-ocean/container-linux/kubernetes](cl/digital-ocean.md) | beta | -| Google Cloud | Container Linux | [google-cloud/container-linux/kubernetes](cl/google-cloud.md) | stable | - +| AWS | Flatcar Linux | [aws/container-linux/kubernetes](flatcar-linux/aws.md) | stable | +| Azure | Flatcar Linux | [azure/container-linux/kubernetes](flatcar-linux/azure.md) | alpha | +| Bare-Metal | Flatcar Linux | [bare-metal/container-linux/kubernetes](flatcar-linux/bare-metal.md) | stable | +| DigitalOcean | Flatcar Linux | [digital-ocean/container-linux/kubernetes](flatcar-linux/digitalocean.md) | beta | +| Google Cloud | Flatcar Linux | [google-cloud/container-linux/kubernetes](flatcar-linux/google-cloud.md) | beta | ## Documentation * Architecture [concepts](architecture/concepts.md) and [operating-systems](architecture/operating-systems.md) * Fedora CoreOS tutorials for [AWS](fedora-coreos/aws.md), [Azure](fedora-coreos/azure.md), [Bare-Metal](fedora-coreos/bare-metal.md), [DigitalOcean](fedora-coreos/digitalocean.md), and [Google Cloud](fedora-coreos/google-cloud.md) -* Flatcar Linux tutorials for [AWS](cl/aws.md), [Azure](cl/azure.md), [Bare-Metal](cl/bare-metal.md), [DigitalOcean](cl/digital-ocean.md), and [Google Cloud](cl/google-cloud.md) +* Flatcar Linux tutorials for [AWS](flatcar-linux/aws.md), [Azure](flatcar-linux/azure.md), [Bare-Metal](flatcar-linux/bare-metal.md), [DigitalOcean](flatcar-linux/digitalocean.md), and [Google Cloud](flatcar-linux/google-cloud.md) ## Example diff --git a/docs/topics/faq.md b/docs/topics/faq.md index 1a8eef263..d661e7aa3 100644 --- a/docs/topics/faq.md +++ b/docs/topics/faq.md @@ -6,15 +6,6 @@ Typhoon provides a Terraform Module for each supported operating system and plat Formats rise and evolve. Typhoon may choose to adapt the format over time (with lots of forewarning). However, the authors' have built several Kubernetes "distros" before and learned from mistakes - Terraform modules are the right format for now. -## Operating Systems - -Typhoon supports Container Linux and the Flatcar Linux derivative. These operating systems were chosen because they offer: - -* Minimalism and focus on clustered operation -* Automated and atomic operating system upgrades -* Declarative and immutable configuration -* Optimization for containerized applications - ## Get Help Ask questions on the IRC #typhoon channel on [freenode.net](http://freenode.net/). diff --git a/docs/topics/maintenance.md b/docs/topics/maintenance.md index f020a293c..b583b6413 100644 --- a/docs/topics/maintenance.md +++ b/docs/topics/maintenance.md @@ -13,7 +13,7 @@ Typhoon provides tagged releases to allow clusters to be versioned using ordinar ``` module "yavin" { - source = "git::https://github.com/poseidon/typhoon//google-cloud/container-linux/kubernetes?ref=v1.8.6" + source = "git::https://github.com/poseidon/typhoon//google-cloud/fedora-coreos/kubernetes?ref=v1.18.3" ... } @@ -74,11 +74,11 @@ Delete or comment the Terraform config for the cluster. Apply to delete old provisioning configs from Matchbox. ``` -$ terraform apply +$ terraform apply Apply complete! Resources: 0 added, 0 changed, 55 destroyed. ``` -Re-provision a new cluster by following the bare-metal [tutorial](../cl/bare-metal.md#cluster). +Re-provision a new cluster by following the bare-metal [tutorial](../fedora-coreos/bare-metal.md#cluster). ### Cloud @@ -102,7 +102,7 @@ Once you're confident in the new cluster, delete the Terraform config for the ol Apply to delete the cluster. ``` -$ terraform apply +$ terraform apply Apply complete! Resources: 0 added, 0 changed, 55 destroyed. ``` @@ -125,86 +125,18 @@ In certain scenarios, in-place edits can be useful for quickly rolling out secur Typhoon supports multi-controller clusters, so it is possible to upgrade a cluster by deleting and replacing nodes one by one. !!! warning - Typhoon does not support or document node replacement as an upgrade strategy. It limits Typhoon's ability to make infrastructure and architectural changes between tagged releases. - -### Terraform Plugins Directory - -Use the Terraform 3rd-party [plugin directory](https://www.terraform.io/docs/configuration/providers.html#third-party-plugins) `~/.terraform.d/plugins` to keep versioned copies of the `terraform-provider-ct` and `terraform-provider-matchbox` plugins. The plugin directory replaces the `~/.terraformrc` file to allow 3rd party plugins to be defined and versioned independently (rather than globally). - -``` -# ~/.terraformrc (DEPRECATED) -providers { - ct = "/usr/local/bin/terraform-provider-ct" - matchbox = "/usr/local/bin/terraform-provider-matchbox" -} -``` - -Migrate to using the Terraform plugin directory. Move `~/.terraformrc` to a backup location. - -``` -mv ~/.terraformrc ~/.terraform-backup -``` - -Add the [terraform-provider-ct](https://github.com/poseidon/terraform-provider-ct) plugin binary for your system to `~/.terraform.d/plugins/`. Download the **same version** of `terraform-provider-ct` you were using with `~/.terraformrc`, updating only be done as a followup and is **only** safe for v1.12.2+ clusters! - -```sh -wget https://github.com/poseidon/terraform-provider-ct/releases/download/v0.2.1/terraform-provider-ct-v0.2.1-linux-amd64.tar.gz -tar xzf terraform-provider-ct-v0.2.1-linux-amd64.tar.gz -mv terraform-provider-ct-v0.2.1-linux-amd64/terraform-provider-ct ~/.terraform.d/plugins/terraform-provider-ct_v0.2.1 -``` - -If you use bare-metal, add the [terraform-provider-matchbox](https://github.com/poseidon/terraform-provider-matchbox) plugin binary for your system to `~/.terraform.d/plugins/`, noting the versioned name. - -```sh -wget https://github.com/poseidon/terraform-provider-matchbox/releases/download/v0.2.3/terraform-provider-matchbox-v0.2.3-linux-amd64.tar.gz -tar xzf terraform-provider-matchbox-v0.2.3-linux-amd64.tar.gz -mv terraform-provider-matchbox-v0.2.3-linux-amd64/terraform-provider-matchbox ~/.terraform.d/plugins/terraform-provider-matchbox_v0.2.3 -``` - -Binary names are versioned. This enables the ability to upgrade different plugins and have clusters pin different versions. - -``` -$ tree ~/.terraform.d/ -/home/user/.terraform.d/ -└── plugins - ├── terraform-provider-ct_v0.2.1 - └── terraform-provider-matchbox_v0.2.3 -``` - -In each Terraform working directory, set the version of each provider. - -``` -# providers.tf - -provider "matchbox" { - version = "0.2.3" - ... -} - -provider "ct" { - version = "0.2.1" -} -``` - -Run `terraform init` to ensure plugin version requirements are met. Verify `terraform plan` does not produce a diff, since the plugin versions should be the same as previously. - -``` -$ terraform init -$ terraform plan -``` + Typhoon does not support or document node replacement as an upgrade strategy. It limits Typhoon's ability to make infrastructure and architectural changes between tagged releases. ### Upgrade terraform-provider-ct -The [terraform-provider-ct](https://github.com/poseidon/terraform-provider-ct) plugin parses, validates, and converts Container Linux Configs into Ignition user-data for provisioning instances. Previously, updating the plugin re-provisioned controller nodes and was destructive to clusters. With Typhoon v1.12.2+, the plugin can be updated in-place and on apply, only workers will be replaced. - -First, [migrate](#terraform-plugins-directory) to the Terraform 3rd-party plugin directory to allow 3rd-party plugins to be defined and versioned independently (rather than globally). +The [terraform-provider-ct](https://github.com/poseidon/terraform-provider-ct) plugin parses, validates, and converts Container Linux Configs into Ignition user-data for provisioning instances. Since Typhoon v1.12.2+, the plugin can be updated in-place so that on apply, only workers will be replaced. Add the [terraform-provider-ct](https://github.com/poseidon/terraform-provider-ct) plugin binary for your system to `~/.terraform.d/plugins/`, noting the final name. ```sh -wget https://github.com/poseidon/terraform-provider-ct/releases/download/v0.3.1/terraform-provider-ct-v0.3.1-linux-amd64.tar.gz -tar xzf terraform-provider-ct-v0.3.1-linux-amd64.tar.gz -mv terraform-provider-ct-v0.3.1-linux-amd64/terraform-provider-ct ~/.terraform.d/plugins/terraform-provider-ct_v0.3.1 +wget https://github.com/poseidon/terraform-provider-ct/releases/download/v0.5.0/terraform-provider-ct-v0.5.0-linux-amd64.tar.gz +tar xzf terraform-provider-ct-v0.5.0-linux-amd64.tar.gz +mv terraform-provider-ct-v0.5.0-linux-amd64/terraform-provider-ct ~/.terraform.d/plugins/terraform-provider-ct_v0.5.0 ``` Binary names are versioned. This enables the ability to upgrade different plugins and have clusters pin different versions. @@ -215,8 +147,8 @@ $ tree ~/.terraform.d/ └── plugins ├── terraform-provider-ct_v0.2.1 ├── terraform-provider-ct_v0.3.0 - ├── terraform-provider-ct_v0.3.1 - └── terraform-provider-matchbox_v0.2.3 + ├── terraform-provider-ct_v0.5.0 + └── terraform-provider-matchbox_v0.3.0 ``` @@ -225,7 +157,7 @@ Update the version of the `ct` plugin in each Terraform working directory. Typho ``` # providers.tf provider "ct" { - version = "0.3.0" + version = "0.5.0" } ``` @@ -279,153 +211,9 @@ Typhoon modules have been adapted for Terraform v0.12. Provider plugins requirem | Typhoon Release | Terraform version | |-------------------|---------------------| -| v1.18.3 - ? | v0.12.x | -| v1.10.3 - v1.18.3 | v0.11.x | +| v1.15.0 - ? | v0.12.x | +| v1.10.3 - v1.15.0 | v0.11.x | | v1.9.2 - v1.10.2 | v0.10.4+ or v0.11.x | | v1.7.3 - v1.9.1 | v0.10.x | | v1.6.4 - v1.7.2 | v0.9.x | -### New users - -New users can start with Terraform v0.12.x and follow the docs for Typhoon v1.18.3+ without issue. - -### Existing users - -Migrate from Terraform v0.11 to v0.12 either **in-place** (easier, riskier) or by **moving resources** (safer, tedious). - -Install [Terraform](https://www.terraform.io/downloads.html) v0.12.x on your system alongside Terraform v0.11.x. - -```shell -sudo ln -sf ~/Downloads/terraform-0.12.0/terraform /usr/local/bin/terraform12 -``` - -!!! note - For example, `terraform` may refer Terraform v0.11.14, while `terraform12` is symlinked to Terraform v0.12.1. Once migration is complete, Terraform v0.11.x can be deleted and `terraform12` renamed. - -#### In-place - -For existing Typhoon v1.14.2 or v1.14.3 clusters, edit the Typhoon `ref` to first SHA that introduced Terraform v0.12 support (`3276bf587850218b8f967978a4bf2b05d5f440a2`). The aim is to minimize the diff and convert to using Terraform v0.12.x. For example: - -```tf - module "mercury" { -- source = "git::https://github.com/poseidon/typhoon//bare-metal/container-linux/kubernetes?ref=v1.14.3" -+ source = "git::https://github.com/poseidon/typhoon//bare-metal/container-linux/kubernetes?ref=3276bf587850218b8f967978a4bf2b05d5f440a2" - ... -``` - -With Terraform v0.12, Typhoon clusters no longer require the `providers` block (unless you actually need to pass an [aliased provider](https://www.terraform.io/docs/configuration/providers.html#alias-multiple-provider-instances)). A regression in Terraform v0.11 made it neccessary to explicitly pass aliased providers in order for Typhoon to continue to enforce constraints (see [terraform#16824](https://github.com/hashicorp/terraform/issues/16824)). Terraform v0.12 resolves this issue. - -```tf - module "mercury" { - source = "git::https://github.com/poseidon/typhoon//bare-metal/container-linux/kubernetes?ref=3276bf587850218b8f967978a4bf2b05d5f440a2" - -- providers = { -- local = "local.default" -- null = "null.default" -- template = "template.default" -- tls = "tls.default" -- } -``` - -Provider constrains ensure suitable plugin versions are used. Install new versions of `terraform-provider-ct` (v0.3.2+) and `terraform-provider-matchbox` (bare-metal only, v0.3.0+) according to the [changelog](https://github.com/poseidon/typhoon/blob/master/CHANGES.md#v1144) or tutorial docs. The `local`, `null`, `template`, and `tls` blocks in `providers.tf` are no longer needed. - -```tf - provider "matchbox" { -- version = "0.2.3" -+ version = "0.3.0" - endpoint = "matchbox.example.com:8081" - client_cert = "${file("~/.config/matchbox/client.crt")}" - client_key = "${file("~/.config/matchbox/client.key")}" - } - - provider "ct" { -- version = "0.3.2" -+ version = "0.3.3" - } -- --provider "local" { -- version = "~> 1.0" -- alias = "default" --} -- --provider "null" { -- version = "~> 1.0" -- alias = "default" --} -- --provider "template" { -- version = "~> 1.0" -- alias = "default" --} -- --provider "tls" { -- version = "~> 1.0" -- alias = "default" --} -``` - -Within the Terraform config directory (i.e. working directory), initialize to fetch suitable provider plugins. - -```shell -terraform12 init # using Terraform v0.12 binary, not v0.11 -``` - -Use the Terraform v0.12 upgrade subcommand to convert v0.11 syntax to v0.12. This _will_ edit resource definitions in `*.tf` files in the working directory. Start from a clean version control state. Inspect the changes. Resolve any "TODO" items. - -```shell -terraform12 0.12upgrade -git diff -``` - -Finally, plan. - -```shell -terraform12 plan -``` - -Verify no changes are proposed and commit changes to version control. You've migrated to Terraform v0.12! Repeat for other config directories. Use the Terraform v0.12 binary going forward. - -!!! note - It is known that plan may propose re-creating `template_dir` resources. This is harmless. - -!!! error - If plan produced errors, seek to address them (they may be in non-Typhoon resources). If plan proposed a diff, you'll need to evaluate whether that's expected and safe to apply. In-place edits between Typhoon releases aren't supported (favoring blue/green replacement). The larger the version skew, the greater the risk. Use good judgement. If in doubt, abandon the generated changes, delete `.terraform` as [suggested](https://www.terraform.io/upgrade-guides/0-12.html#upgrading-to-terraform-0-12), and try the move resources approach. - -#### Moving Resources - -Alternately, continue maintaining existing clusters using Terraform v0.11.x and existing Terraform configuration directory(ies). Create new Terraform directory(ies) and move resources there to be managed with Terraform v0.12. This approach allows resources to be migrated incrementally and ensures existing resources can always be managed (e.g. emergency patches). - -Create a new Terraform [config directory](/architecture/concepts/#organize) for *new* resources. - -```shell -mkdir infra2 -tree . -├── infraA <- existing Terraform v0.11.x configs -└── infraB <- new Terraform v0.12.x configs -``` - -Define Typhoon clusters in the new config directory using Terraform v0.12 syntax. Follow the Typhoon v1.15.0+ docs (e.g. use `terraform12` in the `infraB` dir). See [AWS](/cl/aws), [Azure](/cl/azure), [Bare-Metal](/cl/bare-metal), [Digital Ocean](/cl/digital-ocean), or [Google-Cloud](/cl/google-cloud)) to create new clusters. Follow the usual [upgrade](/topics/maintenance/#upgrades) process to apply workloads and shift traffic. Later, switch back to the old config directory and deprovision clusters with Terraform v0.11. - -```shell -terraform12 init -terraform12 plan -terraform12 apply -``` - -Your Terraform configuration directory likely defines resources other than just Typhoon modules (e.g. application DNS records, firewall rules, etc.). While such migrations are outside Typhoon's scope, you'll probably want to move existing resource definitions into your new Terraform configuration directory. Use Terraform v0.12 to import the resource into the state associated with the new config directory (to avoid trying to recreate a resource that exists). Then with Terraform v0.11 in the old directory, remove the resource from the state (to avoid trying to delete the resource). Verify neither `plan` produces a diff. - -```sh -# move google_dns_record_set.some-app from infraA to infraB -cd infraA -terraform state list -terraform state show google_dns_record_set.some-app - -cd ../infraB -terraform12 import google_dns_record_set.some-app SOMEID -terraform12 plan - -cd ../infraA -terraform state rm google_dns_record_set.some-app -terraform plan -``` - diff --git a/mkdocs.yml b/mkdocs.yml index 49d470db5..d975b56c7 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -1,26 +1,28 @@ -site_name: 'Typhoon' -site_description: 'A minimal and free Kubernetes distribution' -site_author: 'Dalton Hubble' -repo_name: 'poseidon/typhoon' -repo_url: 'https://github.com/poseidon/typhoon' +site_name: Typhoon +site_description: A minimal and free Kubernetes distribution +site_author: Dalton Hubble +repo_name: poseidon/typhoon +repo_url: https://github.com/poseidon/typhoon theme: - name: 'material' - feature: - tabs: 'true' - palette: - primary: 'blue' - accent: 'pink' + name: material + features: + - tabs logo: 'img/spin.png' favicon: 'img/favicon.ico' + icon: + repo: fontawesome/brands/github-alt + palette: + primary: blue + accent: pink font: text: 'Roboto Slab' code: 'Roboto Mono' extra: social: - - type: 'github' - link: 'https://github.com/poseidon' - - type: 'twitter' - link: 'https://twitter.com/typhoon8s' + - icon: fontawesome/brands/github-alt + link: https://github.com/poseidon + - icon: fontawesome/brands/twitter + link: https://twitter.com/typhoon8s markdown_extensions: - admonition - codehilite @@ -59,12 +61,12 @@ nav: - 'Bare-Metal': 'fedora-coreos/bare-metal.md' - 'Digital Ocean': 'fedora-coreos/digitalocean.md' - 'Google Cloud': 'fedora-coreos/google-cloud.md' - - 'Container Linux': - - 'AWS': 'cl/aws.md' - - 'Azure': 'cl/azure.md' - - 'Bare-Metal': 'cl/bare-metal.md' - - 'Digital Ocean': 'cl/digital-ocean.md' - - 'Google Cloud': 'cl/google-cloud.md' + - 'Flatcar Linux': + - 'AWS': 'flatcar-linux/aws.md' + - 'Azure': 'flatcar-linux/azure.md' + - 'Bare-Metal': 'flatcar-linux/bare-metal.md' + - 'Digital Ocean': 'flatcar-linux/digitalocean.md' + - 'Google Cloud': 'flatcar-linux/google-cloud.md' - 'Topics': - 'Maintenance': 'topics/maintenance.md' - 'Hardware': 'topics/hardware.md' diff --git a/requirements.txt b/requirements.txt index e8189b359..ba487e102 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -mkdocs==1.0.4 -mkdocs-material==4.6.3 +mkdocs==1.1.2 +mkdocs-material==5.2.0 pygments==2.5.2 -pymdown-extensions==6.3.0 +pymdown-extensions==7.1.0 From e72f916c8d97bcf95a60455e6156dcb62184c9ac Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Fri, 22 May 2020 00:50:30 -0700 Subject: [PATCH 429/523] Update etcd from v3.4.8 to v3.4.9 * https://github.com/etcd-io/etcd/blob/master/CHANGELOG-3.4.md#v349-2020-05-20 --- CHANGES.md | 3 ++- aws/container-linux/kubernetes/cl/controller.yaml | 2 +- aws/fedora-coreos/kubernetes/fcc/controller.yaml | 2 +- azure/container-linux/kubernetes/cl/controller.yaml | 2 +- azure/fedora-coreos/kubernetes/fcc/controller.yaml | 2 +- bare-metal/container-linux/kubernetes/cl/controller.yaml | 2 +- bare-metal/fedora-coreos/kubernetes/fcc/controller.yaml | 2 +- digital-ocean/container-linux/kubernetes/cl/controller.yaml | 2 +- digital-ocean/fedora-coreos/kubernetes/fcc/controller.yaml | 2 +- google-cloud/container-linux/kubernetes/cl/controller.yaml | 2 +- google-cloud/fedora-coreos/kubernetes/fcc/controller.yaml | 2 +- 11 files changed, 12 insertions(+), 11 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index d5a8a0849..4611ecdb9 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -7,8 +7,9 @@ Notable changes between versions. * Use Kubelet [TLS bootstrap](https://kubernetes.io/docs/reference/command-line-tools-reference/kubelet-tls-bootstrapping/) with bootstrap token authentication ([#713](https://github.com/poseidon/typhoon/pull/713)) * Enable Node [Authorization](https://kubernetes.io/docs/reference/access-authn-authz/node/) and [NodeRestriction](https://kubernetes.io/docs/reference/access-authn-authz/admission-controllers/#noderestriction) to reduce authorization scope * Renew Kubelet certificates every 72 hours -* Add CoreDNS node affinity preference for controller nodes ([#188](https://github.com/poseidon/terraform-render-bootstrap/pull/188)) +* Update etcd from v3.4.7 to [v3.4.9](https://github.com/etcd-io/etcd/releases/tag/v3.4.9) * Update Calico from v3.13.1 to [v3.14.0](https://docs.projectcalico.org/v3.14/release-notes/) +* Add CoreDNS node affinity preference for controller nodes ([#188](https://github.com/poseidon/terraform-render-bootstrap/pull/188)) * Deprecate CoreOS Container Linux support (no OS [updates](https://coreos.com/os/eol/) after May 2020) * Use a `fedora-coreos` module for Fedora CoreOS * Use a `container-linux` module for Flatcar Linux diff --git a/aws/container-linux/kubernetes/cl/controller.yaml b/aws/container-linux/kubernetes/cl/controller.yaml index 362835017..a62691be3 100644 --- a/aws/container-linux/kubernetes/cl/controller.yaml +++ b/aws/container-linux/kubernetes/cl/controller.yaml @@ -7,7 +7,7 @@ systemd: - name: 40-etcd-cluster.conf contents: | [Service] - Environment="ETCD_IMAGE_TAG=v3.4.8" + Environment="ETCD_IMAGE_TAG=v3.4.9" Environment="ETCD_IMAGE_URL=docker://quay.io/coreos/etcd" Environment="RKT_RUN_ARGS=--insecure-options=image" Environment="ETCD_NAME=${etcd_name}" diff --git a/aws/fedora-coreos/kubernetes/fcc/controller.yaml b/aws/fedora-coreos/kubernetes/fcc/controller.yaml index a98424223..129a8d925 100644 --- a/aws/fedora-coreos/kubernetes/fcc/controller.yaml +++ b/aws/fedora-coreos/kubernetes/fcc/controller.yaml @@ -28,7 +28,7 @@ systemd: --network host \ --volume /var/lib/etcd:/var/lib/etcd:rw,Z \ --volume /etc/ssl/etcd:/etc/ssl/certs:ro,Z \ - quay.io/coreos/etcd:v3.4.8 + quay.io/coreos/etcd:v3.4.9 ExecStop=/usr/bin/podman stop etcd [Install] WantedBy=multi-user.target diff --git a/azure/container-linux/kubernetes/cl/controller.yaml b/azure/container-linux/kubernetes/cl/controller.yaml index c819592fb..77090bd9f 100644 --- a/azure/container-linux/kubernetes/cl/controller.yaml +++ b/azure/container-linux/kubernetes/cl/controller.yaml @@ -7,7 +7,7 @@ systemd: - name: 40-etcd-cluster.conf contents: | [Service] - Environment="ETCD_IMAGE_TAG=v3.4.8" + Environment="ETCD_IMAGE_TAG=v3.4.9" Environment="ETCD_IMAGE_URL=docker://quay.io/coreos/etcd" Environment="RKT_RUN_ARGS=--insecure-options=image" Environment="ETCD_NAME=${etcd_name}" diff --git a/azure/fedora-coreos/kubernetes/fcc/controller.yaml b/azure/fedora-coreos/kubernetes/fcc/controller.yaml index 9255564e0..c38b2df36 100644 --- a/azure/fedora-coreos/kubernetes/fcc/controller.yaml +++ b/azure/fedora-coreos/kubernetes/fcc/controller.yaml @@ -28,7 +28,7 @@ systemd: --network host \ --volume /var/lib/etcd:/var/lib/etcd:rw,Z \ --volume /etc/ssl/etcd:/etc/ssl/certs:ro,Z \ - quay.io/coreos/etcd:v3.4.8 + quay.io/coreos/etcd:v3.4.9 ExecStop=/usr/bin/podman stop etcd [Install] WantedBy=multi-user.target diff --git a/bare-metal/container-linux/kubernetes/cl/controller.yaml b/bare-metal/container-linux/kubernetes/cl/controller.yaml index 10b5a9c30..387ced5ea 100644 --- a/bare-metal/container-linux/kubernetes/cl/controller.yaml +++ b/bare-metal/container-linux/kubernetes/cl/controller.yaml @@ -7,7 +7,7 @@ systemd: - name: 40-etcd-cluster.conf contents: | [Service] - Environment="ETCD_IMAGE_TAG=v3.4.8" + Environment="ETCD_IMAGE_TAG=v3.4.9" Environment="ETCD_IMAGE_URL=docker://quay.io/coreos/etcd" Environment="RKT_RUN_ARGS=--insecure-options=image" Environment="ETCD_NAME=${etcd_name}" diff --git a/bare-metal/fedora-coreos/kubernetes/fcc/controller.yaml b/bare-metal/fedora-coreos/kubernetes/fcc/controller.yaml index f1baa6fd2..2ac16271d 100644 --- a/bare-metal/fedora-coreos/kubernetes/fcc/controller.yaml +++ b/bare-metal/fedora-coreos/kubernetes/fcc/controller.yaml @@ -28,7 +28,7 @@ systemd: --network host \ --volume /var/lib/etcd:/var/lib/etcd:rw,Z \ --volume /etc/ssl/etcd:/etc/ssl/certs:ro,Z \ - quay.io/coreos/etcd:v3.4.8 + quay.io/coreos/etcd:v3.4.9 ExecStop=/usr/bin/podman stop etcd [Install] WantedBy=multi-user.target diff --git a/digital-ocean/container-linux/kubernetes/cl/controller.yaml b/digital-ocean/container-linux/kubernetes/cl/controller.yaml index c3c7eee97..9656b827a 100644 --- a/digital-ocean/container-linux/kubernetes/cl/controller.yaml +++ b/digital-ocean/container-linux/kubernetes/cl/controller.yaml @@ -7,7 +7,7 @@ systemd: - name: 40-etcd-cluster.conf contents: | [Service] - Environment="ETCD_IMAGE_TAG=v3.4.8" + Environment="ETCD_IMAGE_TAG=v3.4.9" Environment="ETCD_IMAGE_URL=docker://quay.io/coreos/etcd" Environment="RKT_RUN_ARGS=--insecure-options=image" Environment="ETCD_NAME=${etcd_name}" diff --git a/digital-ocean/fedora-coreos/kubernetes/fcc/controller.yaml b/digital-ocean/fedora-coreos/kubernetes/fcc/controller.yaml index 3ae2c5823..cd1575c4b 100644 --- a/digital-ocean/fedora-coreos/kubernetes/fcc/controller.yaml +++ b/digital-ocean/fedora-coreos/kubernetes/fcc/controller.yaml @@ -28,7 +28,7 @@ systemd: --network host \ --volume /var/lib/etcd:/var/lib/etcd:rw,Z \ --volume /etc/ssl/etcd:/etc/ssl/certs:ro,Z \ - quay.io/coreos/etcd:v3.4.8 + quay.io/coreos/etcd:v3.4.9 ExecStop=/usr/bin/podman stop etcd [Install] WantedBy=multi-user.target diff --git a/google-cloud/container-linux/kubernetes/cl/controller.yaml b/google-cloud/container-linux/kubernetes/cl/controller.yaml index 994dfb7b5..eef508f57 100644 --- a/google-cloud/container-linux/kubernetes/cl/controller.yaml +++ b/google-cloud/container-linux/kubernetes/cl/controller.yaml @@ -7,7 +7,7 @@ systemd: - name: 40-etcd-cluster.conf contents: | [Service] - Environment="ETCD_IMAGE_TAG=v3.4.8" + Environment="ETCD_IMAGE_TAG=v3.4.9" Environment="ETCD_IMAGE_URL=docker://quay.io/coreos/etcd" Environment="RKT_RUN_ARGS=--insecure-options=image" Environment="ETCD_NAME=${etcd_name}" diff --git a/google-cloud/fedora-coreos/kubernetes/fcc/controller.yaml b/google-cloud/fedora-coreos/kubernetes/fcc/controller.yaml index a98424223..129a8d925 100644 --- a/google-cloud/fedora-coreos/kubernetes/fcc/controller.yaml +++ b/google-cloud/fedora-coreos/kubernetes/fcc/controller.yaml @@ -28,7 +28,7 @@ systemd: --network host \ --volume /var/lib/etcd:/var/lib/etcd:rw,Z \ --volume /etc/ssl/etcd:/etc/ssl/certs:ro,Z \ - quay.io/coreos/etcd:v3.4.8 + quay.io/coreos/etcd:v3.4.9 ExecStop=/usr/bin/podman stop etcd [Install] WantedBy=multi-user.target From 283e14f3e01dc31bb5589c7372859129d4ee1a16 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Fri, 22 May 2020 01:11:13 -0700 Subject: [PATCH 430/523] Update recommended Terraform provider versions * Sync Terraform provider plugin versions to those actively used internally * Fix terraform fmt --- aws/fedora-coreos/kubernetes/workers.tf | 2 +- docs/fedora-coreos/aws.md | 2 +- docs/fedora-coreos/azure.md | 2 +- docs/fedora-coreos/digitalocean.md | 2 +- docs/fedora-coreos/google-cloud.md | 2 +- docs/flatcar-linux/aws.md | 2 +- docs/flatcar-linux/azure.md | 2 +- docs/flatcar-linux/digitalocean.md | 2 +- docs/flatcar-linux/google-cloud.md | 2 +- google-cloud/fedora-coreos/kubernetes/variables.tf | 4 ++-- google-cloud/fedora-coreos/kubernetes/workers/variables.tf | 4 ++-- 11 files changed, 13 insertions(+), 13 deletions(-) diff --git a/aws/fedora-coreos/kubernetes/workers.tf b/aws/fedora-coreos/kubernetes/workers.tf index d02e5288c..dcfc05d9a 100644 --- a/aws/fedora-coreos/kubernetes/workers.tf +++ b/aws/fedora-coreos/kubernetes/workers.tf @@ -8,7 +8,7 @@ module "workers" { security_groups = [aws_security_group.worker.id] worker_count = var.worker_count instance_type = var.worker_type - os_stream = var.os_stream + os_stream = var.os_stream disk_size = var.disk_size spot_price = var.worker_price target_groups = var.worker_target_groups diff --git a/docs/fedora-coreos/aws.md b/docs/fedora-coreos/aws.md index 56f8abc07..7f302374c 100644 --- a/docs/fedora-coreos/aws.md +++ b/docs/fedora-coreos/aws.md @@ -49,7 +49,7 @@ Configure the AWS provider to use your access key credentials in a `providers.tf ```tf provider "aws" { - version = "2.59.0" + version = "2.63.0" region = "eu-central-1" shared_credentials_file = "/home/user/.config/aws/credentials" } diff --git a/docs/fedora-coreos/azure.md b/docs/fedora-coreos/azure.md index 45d44e88a..45ce5c1c8 100644 --- a/docs/fedora-coreos/azure.md +++ b/docs/fedora-coreos/azure.md @@ -47,7 +47,7 @@ Configure the Azure provider in a `providers.tf` file. ```tf provider "azurerm" { - version = "2.7.0" + version = "2.11.0" } provider "ct" { diff --git a/docs/fedora-coreos/digitalocean.md b/docs/fedora-coreos/digitalocean.md index faffac252..95de04fb1 100644 --- a/docs/fedora-coreos/digitalocean.md +++ b/docs/fedora-coreos/digitalocean.md @@ -50,7 +50,7 @@ Configure the DigitalOcean provider to use your token in a `providers.tf` file. ```tf provider "digitalocean" { - version = "1.17.0" + version = "1.18.0" token = "${chomp(file("~/.config/digital-ocean/token"))}" } diff --git a/docs/fedora-coreos/google-cloud.md b/docs/fedora-coreos/google-cloud.md index 005ccfaae..6f883dea6 100644 --- a/docs/fedora-coreos/google-cloud.md +++ b/docs/fedora-coreos/google-cloud.md @@ -49,7 +49,7 @@ Configure the Google Cloud provider to use your service account key, project-id, ```tf provider "google" { - version = "3.19.0" + version = "3.22.0" project = "project-id" region = "us-central1" credentials = file("~/.config/google-cloud/terraform.json") diff --git a/docs/flatcar-linux/aws.md b/docs/flatcar-linux/aws.md index 2ab50462e..1f15aa699 100644 --- a/docs/flatcar-linux/aws.md +++ b/docs/flatcar-linux/aws.md @@ -49,7 +49,7 @@ Configure the AWS provider to use your access key credentials in a `providers.tf ```tf provider "aws" { - version = "2.59.0" + version = "2.63.0" region = "eu-central-1" shared_credentials_file = "/home/user/.config/aws/credentials" } diff --git a/docs/flatcar-linux/azure.md b/docs/flatcar-linux/azure.md index b9fbae197..2c9a08932 100644 --- a/docs/flatcar-linux/azure.md +++ b/docs/flatcar-linux/azure.md @@ -47,7 +47,7 @@ Configure the Azure provider in a `providers.tf` file. ```tf provider "azurerm" { - version = "2.7.0" + version = "2.11.0" } provider "ct" { diff --git a/docs/flatcar-linux/digitalocean.md b/docs/flatcar-linux/digitalocean.md index 24933bfe0..ede6a1654 100644 --- a/docs/flatcar-linux/digitalocean.md +++ b/docs/flatcar-linux/digitalocean.md @@ -50,7 +50,7 @@ Configure the DigitalOcean provider to use your token in a `providers.tf` file. ```tf provider "digitalocean" { - version = "1.17.0" + version = "1.18.0" token = "${chomp(file("~/.config/digital-ocean/token"))}" } diff --git a/docs/flatcar-linux/google-cloud.md b/docs/flatcar-linux/google-cloud.md index 655f6ff57..dfef04dda 100644 --- a/docs/flatcar-linux/google-cloud.md +++ b/docs/flatcar-linux/google-cloud.md @@ -49,7 +49,7 @@ Configure the Google Cloud provider to use your service account key, project-id, ```tf provider "google" { - version = "3.19.0" + version = "3.22.0" project = "project-id" region = "us-central1" credentials = file("~/.config/google-cloud/terraform.json") diff --git a/google-cloud/fedora-coreos/kubernetes/variables.tf b/google-cloud/fedora-coreos/kubernetes/variables.tf index 59626a88c..927be9d0d 100644 --- a/google-cloud/fedora-coreos/kubernetes/variables.tf +++ b/google-cloud/fedora-coreos/kubernetes/variables.tf @@ -49,14 +49,14 @@ variable "worker_type" { variable "os_stream" { type = string description = "Fedora CoreOS stream for compute instances (e.g. stable, testing, next)" - default = "stable" + default = "stable" } # Deprecated variable "os_image" { type = string description = "Fedora CoreOS image for compute instances (e.g. fedora-coreos)" - default = "" + default = "" } variable "disk_size" { diff --git a/google-cloud/fedora-coreos/kubernetes/workers/variables.tf b/google-cloud/fedora-coreos/kubernetes/workers/variables.tf index 1a47e9547..4c49badf5 100644 --- a/google-cloud/fedora-coreos/kubernetes/workers/variables.tf +++ b/google-cloud/fedora-coreos/kubernetes/workers/variables.tf @@ -37,14 +37,14 @@ variable "machine_type" { variable "os_stream" { type = string description = "Fedora CoreOS stream for compute instances (e.g. stable, testing, next)" - default = "stable" + default = "stable" } # Deprecated variable "os_image" { type = string description = "Fedora CoreOS image for compute instances (e.g. fedora-coreos)" - default = "" + default = "" } variable "disk_size" { From abc31c37116cd0c706aff701d6d20bd7944202a0 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Wed, 27 May 2020 21:33:03 -0700 Subject: [PATCH 431/523] Update node-exporter from v1.0.0-rc.1 to v1.0.0 * https://github.com/prometheus/node_exporter/releases/tag/v1.0.0 --- CHANGES.md | 6 ++++++ addons/prometheus/exporters/node-exporter/daemonset.yaml | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index 4611ecdb9..acfb07d34 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -2,6 +2,12 @@ Notable changes between versions. +## Latest + +### Addons + +* Update node-exporter from v1.0.0-rc.1 to [v1.0.0](https://github.com/prometheus/node_exporter/releases/tag/v1.0.0) + ## v1.18.3 * Use Kubelet [TLS bootstrap](https://kubernetes.io/docs/reference/command-line-tools-reference/kubelet-tls-bootstrapping/) with bootstrap token authentication ([#713](https://github.com/poseidon/typhoon/pull/713)) diff --git a/addons/prometheus/exporters/node-exporter/daemonset.yaml b/addons/prometheus/exporters/node-exporter/daemonset.yaml index 3b4199f0a..963dcb6dd 100644 --- a/addons/prometheus/exporters/node-exporter/daemonset.yaml +++ b/addons/prometheus/exporters/node-exporter/daemonset.yaml @@ -28,7 +28,7 @@ spec: hostPID: true containers: - name: node-exporter - image: quay.io/prometheus/node-exporter:v1.0.0-rc.1 + image: quay.io/prometheus/node-exporter:v1.0.0 args: - --path.procfs=/host/proc - --path.sysfs=/host/sys From 187bb17d394c9a69e5b0274ba13654fc2eed9d91 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Wed, 27 May 2020 21:35:24 -0700 Subject: [PATCH 432/523] Update Grafana from v7.0.0 to v7.0.1 * https://github.com/grafana/grafana/releases/tag/v7.0.1 --- CHANGES.md | 1 + addons/grafana/deployment.yaml | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index acfb07d34..b717fdaae 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -7,6 +7,7 @@ Notable changes between versions. ### Addons * Update node-exporter from v1.0.0-rc.1 to [v1.0.0](https://github.com/prometheus/node_exporter/releases/tag/v1.0.0) +* Update Grafana from v7.0.0 to v7.0.1 ## v1.18.3 diff --git a/addons/grafana/deployment.yaml b/addons/grafana/deployment.yaml index 47262b0f1..49bf94f39 100644 --- a/addons/grafana/deployment.yaml +++ b/addons/grafana/deployment.yaml @@ -23,7 +23,7 @@ spec: spec: containers: - name: grafana - image: docker.io/grafana/grafana:7.0.0 + image: docker.io/grafana/grafana:7.0.1 env: - name: GF_PATHS_CONFIG value: "/etc/grafana/custom.ini" From 907a96916f6e525ed2414be635b1cdf4e08153be Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Wed, 27 May 2020 21:49:40 -0700 Subject: [PATCH 433/523] Update mkdocs-material from v5.2.0 to v5.2.2 * https://github.com/squidfunk/mkdocs-material/releases/tag/5.2.2 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index ba487e102..647bd31fa 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ mkdocs==1.1.2 -mkdocs-material==5.2.0 +mkdocs-material==5.2.2 pygments==2.5.2 pymdown-extensions==7.1.0 From d45804b1f69907dfdba4ea6c34b111a83472f502 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Wed, 27 May 2020 23:49:25 -0700 Subject: [PATCH 434/523] Update Github issue template to use drop-downs (#747) * Create a stricter bug report template * Highlight topics that are not accepted in issues: operation, support, debugging, advice, or Kubernetes concepts * Add a section to strongly suggest bug reports link a PR or describe a solution. This may be able to weed out topics that aren't focused bug reports --- .github/ISSUE_TEMPLATE/bug_report.md | 39 ++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) create mode 100644 .github/ISSUE_TEMPLATE/bug_report.md diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md new file mode 100644 index 000000000..b346e332b --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -0,0 +1,39 @@ +--- +name: Bug report +about: Report a bug to improve the project +title: '' +labels: '' +assignees: '' + +--- + + + +**Description** + +A clear and concise description of what the bug is. + +**Steps to Reproduce** + +Provide clear steps to reproduce the bug. + +- [ ] Relevant error messages if appropriate (concise, not a dump of everything). +- [ ] Explored using a vanilla cluster from the [tutorials](https://typhoon.psdn.io/#documentation). Ruled out [customizations](https://typhoon.psdn.io/advanced/customization/). + +**Expected behavior** + +A clear and concise description of what you expected to happen. + +**Environment** + +* Platform: aws, azure, bare-metal, google-cloud, digital-ocean +* OS: fedora-coreos, flatcar-linux (include release version) +* Release: Typhoon version or Git SHA (reporting latest is **not** helpful) +* Terraform: `terraform version` (reporting latest is **not** helpful) +* Plugins: Provider plugin versions (reporting latest is **not** helpful) + +### Possible Solution + + + +Link to a PR or description. From 455175d9e6c23b11e4c4e55cd43126ac12982c30 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Thu, 28 May 2020 00:06:59 -0700 Subject: [PATCH 435/523] Update the fallback issue template * Even "blank" issues need to fill out the fallback template --- .github/ISSUE_TEMPLATE.md | 33 ---------------------------- .github/ISSUE_TEMPLATE/bug_report.md | 4 ++-- .github/ISSUE_TEMPLATE/config.yml | 5 +++++ .github/issue_template.md | 15 +++++++++++++ 4 files changed, 22 insertions(+), 35 deletions(-) delete mode 100644 .github/ISSUE_TEMPLATE.md create mode 100644 .github/ISSUE_TEMPLATE/config.yml create mode 100644 .github/issue_template.md diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md deleted file mode 100644 index 6f9dfaf37..000000000 --- a/.github/ISSUE_TEMPLATE.md +++ /dev/null @@ -1,33 +0,0 @@ - - -## Bug - -### Environment - -* Platform: aws, azure, bare-metal, google-cloud, digital-ocean -* OS: fedora-coreos, flatcar-linux -* Release: Typhoon version or Git SHA (reporting latest is **not** helpful) -* Terraform: `terraform version` (reporting latest is **not** helpful) -* Plugins: Provider plugin versions (reporting latest is **not** helpful) - -### Problem - -Describe the problem. - -### Desired Behavior - -Describe the goal. - -### Steps to Reproduce - -Provide clear steps to reproduce the issue unless already covered. - -## Feature Request - -### Feature - -Describe the feature and what problem it solves. - -### Tradeoffs - -What are the pros and cons of this feature? How will it be exercised and maintained? diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md index b346e332b..715538624 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.md +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -7,7 +7,7 @@ assignees: '' --- - + **Description** @@ -34,6 +34,6 @@ A clear and concise description of what you expected to happen. ### Possible Solution - + Link to a PR or description. diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml new file mode 100644 index 000000000..15a10dce0 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -0,0 +1,5 @@ +blank_issues_enabled: true +contact_links: + - name: Typhoon Security + url: https://typhoon.psdn.io/topics/security/ + about: Please report security vulnerabilities via email diff --git a/.github/issue_template.md b/.github/issue_template.md new file mode 100644 index 000000000..b494b1ee7 --- /dev/null +++ b/.github/issue_template.md @@ -0,0 +1,15 @@ + + +## Enhancement + +### Overview + +One paragraph explanation of the enhancement. + +### Motivation + +Describe the motivation and what problem this solves. + +### Tradeoffs + +What are the pros and cons of this feature? How will it be exercised and maintained? From ba44408b76cdc4b9ebfa0f0616e9d477a4a8cc8d Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Sat, 30 May 2020 22:03:53 -0700 Subject: [PATCH 436/523] Update Calico from v3.14.0 to v3.14.1 * https://docs.projectcalico.org/v3.14/release-notes/ --- CHANGES.md | 2 ++ aws/container-linux/kubernetes/bootstrap.tf | 2 +- aws/fedora-coreos/kubernetes/bootstrap.tf | 2 +- azure/container-linux/kubernetes/bootstrap.tf | 2 +- azure/fedora-coreos/kubernetes/bootstrap.tf | 2 +- bare-metal/container-linux/kubernetes/bootstrap.tf | 2 +- bare-metal/fedora-coreos/kubernetes/bootstrap.tf | 2 +- digital-ocean/container-linux/kubernetes/bootstrap.tf | 2 +- digital-ocean/fedora-coreos/kubernetes/bootstrap.tf | 2 +- google-cloud/container-linux/kubernetes/bootstrap.tf | 2 +- google-cloud/fedora-coreos/kubernetes/bootstrap.tf | 2 +- 11 files changed, 12 insertions(+), 10 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index b717fdaae..00556fb3e 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -4,6 +4,8 @@ Notable changes between versions. ## Latest +* Update Calico from v3.14.0 to [v3.14.1](https://docs.projectcalico.org/v3.14/release-notes/) + ### Addons * Update node-exporter from v1.0.0-rc.1 to [v1.0.0](https://github.com/prometheus/node_exporter/releases/tag/v1.0.0) diff --git a/aws/container-linux/kubernetes/bootstrap.tf b/aws/container-linux/kubernetes/bootstrap.tf index 8a8fc8c30..16ed8dcf1 100644 --- a/aws/container-linux/kubernetes/bootstrap.tf +++ b/aws/container-linux/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=ff7ec52d0a5e97b8ca6b86a80a7e5e1ea8570487" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=c3b1f23b5daa77c8ff96a2294924aa138a1433a4" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/aws/fedora-coreos/kubernetes/bootstrap.tf b/aws/fedora-coreos/kubernetes/bootstrap.tf index c702bf139..483a0ba11 100644 --- a/aws/fedora-coreos/kubernetes/bootstrap.tf +++ b/aws/fedora-coreos/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=ff7ec52d0a5e97b8ca6b86a80a7e5e1ea8570487" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=c3b1f23b5daa77c8ff96a2294924aa138a1433a4" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/azure/container-linux/kubernetes/bootstrap.tf b/azure/container-linux/kubernetes/bootstrap.tf index 95ab99172..c5ffc2541 100644 --- a/azure/container-linux/kubernetes/bootstrap.tf +++ b/azure/container-linux/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=ff7ec52d0a5e97b8ca6b86a80a7e5e1ea8570487" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=c3b1f23b5daa77c8ff96a2294924aa138a1433a4" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/azure/fedora-coreos/kubernetes/bootstrap.tf b/azure/fedora-coreos/kubernetes/bootstrap.tf index baec21879..b2372ed01 100644 --- a/azure/fedora-coreos/kubernetes/bootstrap.tf +++ b/azure/fedora-coreos/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=ff7ec52d0a5e97b8ca6b86a80a7e5e1ea8570487" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=c3b1f23b5daa77c8ff96a2294924aa138a1433a4" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/bare-metal/container-linux/kubernetes/bootstrap.tf b/bare-metal/container-linux/kubernetes/bootstrap.tf index 34d68d1cb..9a3262407 100644 --- a/bare-metal/container-linux/kubernetes/bootstrap.tf +++ b/bare-metal/container-linux/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=ff7ec52d0a5e97b8ca6b86a80a7e5e1ea8570487" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=c3b1f23b5daa77c8ff96a2294924aa138a1433a4" cluster_name = var.cluster_name api_servers = [var.k8s_domain_name] diff --git a/bare-metal/fedora-coreos/kubernetes/bootstrap.tf b/bare-metal/fedora-coreos/kubernetes/bootstrap.tf index e2a9c5317..dad4f2b7d 100644 --- a/bare-metal/fedora-coreos/kubernetes/bootstrap.tf +++ b/bare-metal/fedora-coreos/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=ff7ec52d0a5e97b8ca6b86a80a7e5e1ea8570487" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=c3b1f23b5daa77c8ff96a2294924aa138a1433a4" cluster_name = var.cluster_name api_servers = [var.k8s_domain_name] diff --git a/digital-ocean/container-linux/kubernetes/bootstrap.tf b/digital-ocean/container-linux/kubernetes/bootstrap.tf index cf04ebbff..3e0d0efb8 100644 --- a/digital-ocean/container-linux/kubernetes/bootstrap.tf +++ b/digital-ocean/container-linux/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=ff7ec52d0a5e97b8ca6b86a80a7e5e1ea8570487" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=c3b1f23b5daa77c8ff96a2294924aa138a1433a4" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/digital-ocean/fedora-coreos/kubernetes/bootstrap.tf b/digital-ocean/fedora-coreos/kubernetes/bootstrap.tf index 62cf353e2..8531436dc 100644 --- a/digital-ocean/fedora-coreos/kubernetes/bootstrap.tf +++ b/digital-ocean/fedora-coreos/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=ff7ec52d0a5e97b8ca6b86a80a7e5e1ea8570487" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=c3b1f23b5daa77c8ff96a2294924aa138a1433a4" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/google-cloud/container-linux/kubernetes/bootstrap.tf b/google-cloud/container-linux/kubernetes/bootstrap.tf index f7cc894d9..85456c156 100644 --- a/google-cloud/container-linux/kubernetes/bootstrap.tf +++ b/google-cloud/container-linux/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=ff7ec52d0a5e97b8ca6b86a80a7e5e1ea8570487" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=c3b1f23b5daa77c8ff96a2294924aa138a1433a4" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/google-cloud/fedora-coreos/kubernetes/bootstrap.tf b/google-cloud/fedora-coreos/kubernetes/bootstrap.tf index 8f9c4c033..4f60b87ee 100644 --- a/google-cloud/fedora-coreos/kubernetes/bootstrap.tf +++ b/google-cloud/fedora-coreos/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=ff7ec52d0a5e97b8ca6b86a80a7e5e1ea8570487" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=c3b1f23b5daa77c8ff96a2294924aa138a1433a4" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] From 20bfd697804bc504bacf05a6f10772bc5f4e21c0 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Thu, 28 May 2020 01:06:26 -0700 Subject: [PATCH 437/523] Change Kubelet container image publishing * Build Kubelet container images internally and publish to Quay and Dockerhub (new) as an alternative in case of registry outage or breach * Use our infra to provide single and multi-arch (default) Kublet images for possible future use * Docs: Show how to use alternative Kubelet images via snippets and a systemd dropin (builds on #737) Changes: * Update docs with changes to Kubelet image building * If you prefer to trust images built by Quay/Dockerhub, automated image builds are still available with unique tags (albeit with some limitations): * Quay automated builds are tagged `build-{short_sha}` (limit: only amd64) * Dockerhub automated builts are tagged `build-{tag}` and `build-master` (limit: only amd64, no shas) Links: * Kubelet: https://github.com/poseidon/kubelet * Docs: https://typhoon.psdn.io/topics/security/#container-images * Registries: * quay.io/poseidon/kubelet * docker.io/psdn/kubelet --- CHANGES.md | 8 +++++ .../kubernetes/cl/controller.yaml | 4 +-- .../kubernetes/workers/cl/worker.yaml | 4 +-- .../kubernetes/fcc/controller.yaml | 4 +-- .../kubernetes/workers/fcc/worker.yaml | 4 +-- .../kubernetes/cl/controller.yaml | 4 +-- .../kubernetes/workers/cl/worker.yaml | 4 +-- .../kubernetes/fcc/controller.yaml | 4 +-- .../kubernetes/workers/fcc/worker.yaml | 4 +-- .../kubernetes/cl/controller.yaml | 4 +-- .../container-linux/kubernetes/cl/worker.yaml | 2 +- .../kubernetes/fcc/controller.yaml | 4 +-- .../fedora-coreos/kubernetes/fcc/worker.yaml | 2 +- .../kubernetes/cl/controller.yaml | 4 +-- .../container-linux/kubernetes/cl/worker.yaml | 4 +-- .../kubernetes/fcc/controller.yaml | 4 +-- .../fedora-coreos/kubernetes/fcc/worker.yaml | 4 +-- docs/advanced/customization.md | 31 +++++++++++++++++++ docs/topics/security.md | 14 ++++++++- .../kubernetes/cl/controller.yaml | 4 +-- .../kubernetes/workers/cl/worker.yaml | 4 +-- .../kubernetes/fcc/controller.yaml | 4 +-- .../kubernetes/workers/fcc/worker.yaml | 4 +-- 23 files changed, 90 insertions(+), 39 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 00556fb3e..4d420cbbc 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -4,6 +4,14 @@ Notable changes between versions. ## Latest +* Update Kubelet image build infra and publishing ([#749](https://github.com/poseidon/typhoon/pull/749)) + * Publish Kubelet images from internal infra to Quay and Dockerhub + * [quay.io/poseidon/kubelet](https://quay.io/repository/poseidon/kubelet) (official) + * [docker.io/psdn/kubelet](https://hub.docker.com/r/psdn/kubelet) (fallback) + * Document use of alternate Kubelet images during registry incidents + * For those preferring to trust images built by Quay/Dockerhub, + automated image builds are still available with an alternate tag + strategy (see [docs](https://typhoon.psdn.io/topics/security/#container-images)) * Update Calico from v3.14.0 to [v3.14.1](https://docs.projectcalico.org/v3.14/release-notes/) ### Addons diff --git a/aws/container-linux/kubernetes/cl/controller.yaml b/aws/container-linux/kubernetes/cl/controller.yaml index a62691be3..49873f6fa 100644 --- a/aws/container-linux/kubernetes/cl/controller.yaml +++ b/aws/container-linux/kubernetes/cl/controller.yaml @@ -52,7 +52,7 @@ systemd: Description=Kubelet Wants=rpc-statd.service [Service] - Environment=KUBELET_IMAGE=docker://quay.io/poseidon/kubelet:v1.18.3 + Environment=KUBELET_IMAGE=docker://quay.io/poseidon/kubelet:v1.18.3-4-g57fa88a Environment=KUBELET_CGROUP_DRIVER=${cgroup_driver} ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests @@ -137,7 +137,7 @@ systemd: --volume script,kind=host,source=/opt/bootstrap/apply \ --mount volume=script,target=/apply \ --insecure-options=image \ - docker://quay.io/poseidon/kubelet:v1.18.3 \ + docker://quay.io/poseidon/kubelet:v1.18.3-4-g57fa88a \ --net=host \ --dns=host \ --exec=/apply diff --git a/aws/container-linux/kubernetes/workers/cl/worker.yaml b/aws/container-linux/kubernetes/workers/cl/worker.yaml index f4c7129af..e6272747b 100644 --- a/aws/container-linux/kubernetes/workers/cl/worker.yaml +++ b/aws/container-linux/kubernetes/workers/cl/worker.yaml @@ -25,7 +25,7 @@ systemd: Description=Kubelet Wants=rpc-statd.service [Service] - Environment=KUBELET_IMAGE=docker://quay.io/poseidon/kubelet:v1.18.3 + Environment=KUBELET_IMAGE=docker://quay.io/poseidon/kubelet:v1.18.3-4-g57fa88a Environment=KUBELET_CGROUP_DRIVER=${cgroup_driver} ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests @@ -130,7 +130,7 @@ storage: --volume config,kind=host,source=/etc/kubernetes \ --mount volume=config,target=/etc/kubernetes \ --insecure-options=image \ - docker://quay.io/poseidon/kubelet:v1.18.3 \ + docker://quay.io/poseidon/kubelet:v1.18.3-4-g57fa88a \ --net=host \ --dns=host \ --exec=/usr/local/bin/kubectl -- --kubeconfig=/etc/kubernetes/kubeconfig delete node $(hostname) diff --git a/aws/fedora-coreos/kubernetes/fcc/controller.yaml b/aws/fedora-coreos/kubernetes/fcc/controller.yaml index 129a8d925..7bea44b02 100644 --- a/aws/fedora-coreos/kubernetes/fcc/controller.yaml +++ b/aws/fedora-coreos/kubernetes/fcc/controller.yaml @@ -54,7 +54,7 @@ systemd: Description=Kubelet (System Container) Wants=rpc-statd.service [Service] - Environment=KUBELET_IMAGE=quay.io/poseidon/kubelet:v1.18.3 + Environment=KUBELET_IMAGE=quay.io/poseidon/kubelet:v1.18.3-4-g57fa88a ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests ExecStartPre=/bin/mkdir -p /opt/cni/bin @@ -126,7 +126,7 @@ systemd: --volume /opt/bootstrap/assets:/assets:ro,Z \ --volume /opt/bootstrap/apply:/apply:ro,Z \ --entrypoint=/apply \ - quay.io/poseidon/kubelet:v1.18.3 + quay.io/poseidon/kubelet:v1.18.3-4-g57fa88a ExecStartPost=/bin/touch /opt/bootstrap/bootstrap.done ExecStartPost=-/usr/bin/podman stop bootstrap storage: diff --git a/aws/fedora-coreos/kubernetes/workers/fcc/worker.yaml b/aws/fedora-coreos/kubernetes/workers/fcc/worker.yaml index d33c43db5..062f117a2 100644 --- a/aws/fedora-coreos/kubernetes/workers/fcc/worker.yaml +++ b/aws/fedora-coreos/kubernetes/workers/fcc/worker.yaml @@ -24,7 +24,7 @@ systemd: Description=Kubelet (System Container) Wants=rpc-statd.service [Service] - Environment=KUBELET_IMAGE=quay.io/poseidon/kubelet:v1.18.3 + Environment=KUBELET_IMAGE=quay.io/poseidon/kubelet:v1.18.3-4-g57fa88a ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests ExecStartPre=/bin/mkdir -p /opt/cni/bin @@ -90,7 +90,7 @@ systemd: Type=oneshot RemainAfterExit=true ExecStart=/bin/true - ExecStop=/bin/bash -c '/usr/bin/podman run --volume /etc/kubernetes:/etc/kubernetes:ro,z --entrypoint /usr/local/bin/kubectl quay.io/poseidon/kubelet:v1.18.3 --kubeconfig=/etc/kubernetes/kubeconfig delete node $HOSTNAME' + ExecStop=/bin/bash -c '/usr/bin/podman run --volume /etc/kubernetes:/etc/kubernetes:ro,z --entrypoint /usr/local/bin/kubectl quay.io/poseidon/kubelet:v1.18.3-4-g57fa88a --kubeconfig=/etc/kubernetes/kubeconfig delete node $HOSTNAME' [Install] WantedBy=multi-user.target storage: diff --git a/azure/container-linux/kubernetes/cl/controller.yaml b/azure/container-linux/kubernetes/cl/controller.yaml index 77090bd9f..b1ddb75cc 100644 --- a/azure/container-linux/kubernetes/cl/controller.yaml +++ b/azure/container-linux/kubernetes/cl/controller.yaml @@ -52,7 +52,7 @@ systemd: Description=Kubelet Wants=rpc-statd.service [Service] - Environment=KUBELET_IMAGE=docker://quay.io/poseidon/kubelet:v1.18.3 + Environment=KUBELET_IMAGE=docker://quay.io/poseidon/kubelet:v1.18.3-4-g57fa88a ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests ExecStartPre=/bin/mkdir -p /opt/cni/bin @@ -135,7 +135,7 @@ systemd: --volume script,kind=host,source=/opt/bootstrap/apply \ --mount volume=script,target=/apply \ --insecure-options=image \ - docker://quay.io/poseidon/kubelet:v1.18.3 \ + docker://quay.io/poseidon/kubelet:v1.18.3-4-g57fa88a \ --net=host \ --dns=host \ --exec=/apply diff --git a/azure/container-linux/kubernetes/workers/cl/worker.yaml b/azure/container-linux/kubernetes/workers/cl/worker.yaml index 11691e1fc..87e33e682 100644 --- a/azure/container-linux/kubernetes/workers/cl/worker.yaml +++ b/azure/container-linux/kubernetes/workers/cl/worker.yaml @@ -25,7 +25,7 @@ systemd: Description=Kubelet Wants=rpc-statd.service [Service] - Environment=KUBELET_IMAGE=docker://quay.io/poseidon/kubelet:v1.18.3 + Environment=KUBELET_IMAGE=docker://quay.io/poseidon/kubelet:v1.18.3-4-g57fa88a ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests ExecStartPre=/bin/mkdir -p /opt/cni/bin @@ -128,7 +128,7 @@ storage: --volume config,kind=host,source=/etc/kubernetes \ --mount volume=config,target=/etc/kubernetes \ --insecure-options=image \ - docker://quay.io/poseidon/kubelet:v1.18.3 \ + docker://quay.io/poseidon/kubelet:v1.18.3-4-g57fa88a \ --net=host \ --dns=host \ --exec=/usr/local/bin/kubectl -- --kubeconfig=/etc/kubernetes/kubeconfig delete node $(hostname | tr '[:upper:]' '[:lower:]') diff --git a/azure/fedora-coreos/kubernetes/fcc/controller.yaml b/azure/fedora-coreos/kubernetes/fcc/controller.yaml index c38b2df36..4831048c3 100644 --- a/azure/fedora-coreos/kubernetes/fcc/controller.yaml +++ b/azure/fedora-coreos/kubernetes/fcc/controller.yaml @@ -54,7 +54,7 @@ systemd: Description=Kubelet (System Container) Wants=rpc-statd.service [Service] - Environment=KUBELET_IMAGE=quay.io/poseidon/kubelet:v1.18.3 + Environment=KUBELET_IMAGE=quay.io/poseidon/kubelet:v1.18.3-4-g57fa88a ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests ExecStartPre=/bin/mkdir -p /opt/cni/bin @@ -126,7 +126,7 @@ systemd: --volume /opt/bootstrap/assets:/assets:ro,Z \ --volume /opt/bootstrap/apply:/apply:ro,Z \ --entrypoint=/apply \ - quay.io/poseidon/kubelet:v1.18.3 + quay.io/poseidon/kubelet:v1.18.3-4-g57fa88a ExecStartPost=/bin/touch /opt/bootstrap/bootstrap.done ExecStartPost=-/usr/bin/podman stop bootstrap storage: diff --git a/azure/fedora-coreos/kubernetes/workers/fcc/worker.yaml b/azure/fedora-coreos/kubernetes/workers/fcc/worker.yaml index 814e51248..815a67a7f 100644 --- a/azure/fedora-coreos/kubernetes/workers/fcc/worker.yaml +++ b/azure/fedora-coreos/kubernetes/workers/fcc/worker.yaml @@ -24,7 +24,7 @@ systemd: Description=Kubelet (System Container) Wants=rpc-statd.service [Service] - Environment=KUBELET_IMAGE=quay.io/poseidon/kubelet:v1.18.3 + Environment=KUBELET_IMAGE=quay.io/poseidon/kubelet:v1.18.3-4-g57fa88a ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests ExecStartPre=/bin/mkdir -p /opt/cni/bin @@ -90,7 +90,7 @@ systemd: Type=oneshot RemainAfterExit=true ExecStart=/bin/true - ExecStop=/bin/bash -c '/usr/bin/podman run --volume /etc/kubernetes:/etc/kubernetes:ro,z --entrypoint /usr/local/bin/kubectl quay.io/poseidon/kubelet:v1.18.3 --kubeconfig=/etc/kubernetes/kubeconfig delete node $HOSTNAME' + ExecStop=/bin/bash -c '/usr/bin/podman run --volume /etc/kubernetes:/etc/kubernetes:ro,z --entrypoint /usr/local/bin/kubectl quay.io/poseidon/kubelet:v1.18.3-4-g57fa88a --kubeconfig=/etc/kubernetes/kubeconfig delete node $HOSTNAME' [Install] WantedBy=multi-user.target storage: diff --git a/bare-metal/container-linux/kubernetes/cl/controller.yaml b/bare-metal/container-linux/kubernetes/cl/controller.yaml index 387ced5ea..8ac9368b3 100644 --- a/bare-metal/container-linux/kubernetes/cl/controller.yaml +++ b/bare-metal/container-linux/kubernetes/cl/controller.yaml @@ -60,7 +60,7 @@ systemd: Description=Kubelet Wants=rpc-statd.service [Service] - Environment=KUBELET_IMAGE=docker://quay.io/poseidon/kubelet:v1.18.3 + Environment=KUBELET_IMAGE=docker://quay.io/poseidon/kubelet:v1.18.3-4-g57fa88a Environment=KUBELET_CGROUP_DRIVER=${cgroup_driver} ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests @@ -150,7 +150,7 @@ systemd: --volume script,kind=host,source=/opt/bootstrap/apply \ --mount volume=script,target=/apply \ --insecure-options=image \ - docker://quay.io/poseidon/kubelet:v1.18.3 \ + docker://quay.io/poseidon/kubelet:v1.18.3-4-g57fa88a \ --net=host \ --dns=host \ --exec=/apply diff --git a/bare-metal/container-linux/kubernetes/cl/worker.yaml b/bare-metal/container-linux/kubernetes/cl/worker.yaml index 337070061..73bdea996 100644 --- a/bare-metal/container-linux/kubernetes/cl/worker.yaml +++ b/bare-metal/container-linux/kubernetes/cl/worker.yaml @@ -33,7 +33,7 @@ systemd: Description=Kubelet Wants=rpc-statd.service [Service] - Environment=KUBELET_IMAGE=docker://quay.io/poseidon/kubelet:v1.18.3 + Environment=KUBELET_IMAGE=docker://quay.io/poseidon/kubelet:v1.18.3-4-g57fa88a Environment=KUBELET_CGROUP_DRIVER=${cgroup_driver} ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests diff --git a/bare-metal/fedora-coreos/kubernetes/fcc/controller.yaml b/bare-metal/fedora-coreos/kubernetes/fcc/controller.yaml index 2ac16271d..d2c756eec 100644 --- a/bare-metal/fedora-coreos/kubernetes/fcc/controller.yaml +++ b/bare-metal/fedora-coreos/kubernetes/fcc/controller.yaml @@ -53,7 +53,7 @@ systemd: Description=Kubelet (System Container) Wants=rpc-statd.service [Service] - Environment=KUBELET_IMAGE=quay.io/poseidon/kubelet:v1.18.3 + Environment=KUBELET_IMAGE=quay.io/poseidon/kubelet:v1.18.3-4-g57fa88a ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests ExecStartPre=/bin/mkdir -p /opt/cni/bin @@ -137,7 +137,7 @@ systemd: --volume /opt/bootstrap/assets:/assets:ro,Z \ --volume /opt/bootstrap/apply:/apply:ro,Z \ --entrypoint=/apply \ - quay.io/poseidon/kubelet:v1.18.3 + quay.io/poseidon/kubelet:v1.18.3-4-g57fa88a ExecStartPost=/bin/touch /opt/bootstrap/bootstrap.done ExecStartPost=-/usr/bin/podman stop bootstrap storage: diff --git a/bare-metal/fedora-coreos/kubernetes/fcc/worker.yaml b/bare-metal/fedora-coreos/kubernetes/fcc/worker.yaml index f3526e6a3..0c7e114a9 100644 --- a/bare-metal/fedora-coreos/kubernetes/fcc/worker.yaml +++ b/bare-metal/fedora-coreos/kubernetes/fcc/worker.yaml @@ -23,7 +23,7 @@ systemd: Description=Kubelet (System Container) Wants=rpc-statd.service [Service] - Environment=KUBELET_IMAGE=quay.io/poseidon/kubelet:v1.18.3 + Environment=KUBELET_IMAGE=quay.io/poseidon/kubelet:v1.18.3-4-g57fa88a ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests ExecStartPre=/bin/mkdir -p /opt/cni/bin diff --git a/digital-ocean/container-linux/kubernetes/cl/controller.yaml b/digital-ocean/container-linux/kubernetes/cl/controller.yaml index 9656b827a..dbcacebf4 100644 --- a/digital-ocean/container-linux/kubernetes/cl/controller.yaml +++ b/digital-ocean/container-linux/kubernetes/cl/controller.yaml @@ -62,7 +62,7 @@ systemd: After=coreos-metadata.service Wants=rpc-statd.service [Service] - Environment=KUBELET_IMAGE=docker://quay.io/poseidon/kubelet:v1.18.3 + Environment=KUBELET_IMAGE=docker://quay.io/poseidon/kubelet:v1.18.3-4-g57fa88a EnvironmentFile=/run/metadata/coreos ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests @@ -147,7 +147,7 @@ systemd: --volume script,kind=host,source=/opt/bootstrap/apply \ --mount volume=script,target=/apply \ --insecure-options=image \ - docker://quay.io/poseidon/kubelet:v1.18.3 \ + docker://quay.io/poseidon/kubelet:v1.18.3-4-g57fa88a \ --net=host \ --dns=host \ --exec=/apply diff --git a/digital-ocean/container-linux/kubernetes/cl/worker.yaml b/digital-ocean/container-linux/kubernetes/cl/worker.yaml index 4c841c69a..b1803b2da 100644 --- a/digital-ocean/container-linux/kubernetes/cl/worker.yaml +++ b/digital-ocean/container-linux/kubernetes/cl/worker.yaml @@ -35,7 +35,7 @@ systemd: After=coreos-metadata.service Wants=rpc-statd.service [Service] - Environment=KUBELET_IMAGE=docker://quay.io/poseidon/kubelet:v1.18.3 + Environment=KUBELET_IMAGE=docker://quay.io/poseidon/kubelet:v1.18.3-4-g57fa88a EnvironmentFile=/run/metadata/coreos ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests @@ -134,7 +134,7 @@ storage: --volume config,kind=host,source=/etc/kubernetes \ --mount volume=config,target=/etc/kubernetes \ --insecure-options=image \ - docker://quay.io/poseidon/kubelet:v1.18.3 \ + docker://quay.io/poseidon/kubelet:v1.18.3-4-g57fa88a \ --net=host \ --dns=host \ --exec=/usr/local/bin/kubectl -- --kubeconfig=/etc/kubernetes/kubeconfig delete node $(hostname) diff --git a/digital-ocean/fedora-coreos/kubernetes/fcc/controller.yaml b/digital-ocean/fedora-coreos/kubernetes/fcc/controller.yaml index cd1575c4b..d8a9bd92d 100644 --- a/digital-ocean/fedora-coreos/kubernetes/fcc/controller.yaml +++ b/digital-ocean/fedora-coreos/kubernetes/fcc/controller.yaml @@ -55,7 +55,7 @@ systemd: After=afterburn.service Wants=rpc-statd.service [Service] - Environment=KUBELET_IMAGE=quay.io/poseidon/kubelet:v1.18.3 + Environment=KUBELET_IMAGE=quay.io/poseidon/kubelet:v1.18.3-4-g57fa88a EnvironmentFile=/run/metadata/afterburn ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests @@ -138,7 +138,7 @@ systemd: --volume /opt/bootstrap/assets:/assets:ro,Z \ --volume /opt/bootstrap/apply:/apply:ro,Z \ --entrypoint=/apply \ - quay.io/poseidon/kubelet:v1.18.3 + quay.io/poseidon/kubelet:v1.18.3-4-g57fa88a ExecStartPost=/bin/touch /opt/bootstrap/bootstrap.done ExecStartPost=-/usr/bin/podman stop bootstrap storage: diff --git a/digital-ocean/fedora-coreos/kubernetes/fcc/worker.yaml b/digital-ocean/fedora-coreos/kubernetes/fcc/worker.yaml index 98554d610..02667f877 100644 --- a/digital-ocean/fedora-coreos/kubernetes/fcc/worker.yaml +++ b/digital-ocean/fedora-coreos/kubernetes/fcc/worker.yaml @@ -26,7 +26,7 @@ systemd: After=afterburn.service Wants=rpc-statd.service [Service] - Environment=KUBELET_IMAGE=quay.io/poseidon/kubelet:v1.18.3 + Environment=KUBELET_IMAGE=quay.io/poseidon/kubelet:v1.18.3-4-g57fa88a EnvironmentFile=/run/metadata/afterburn ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests @@ -100,7 +100,7 @@ systemd: Type=oneshot RemainAfterExit=true ExecStart=/bin/true - ExecStop=/bin/bash -c '/usr/bin/podman run --volume /etc/kubernetes:/etc/kubernetes:ro,z --entrypoint /usr/local/bin/kubectl quay.io/poseidon/kubelet:v1.18.3 --kubeconfig=/etc/kubernetes/kubeconfig delete node $HOSTNAME' + ExecStop=/bin/bash -c '/usr/bin/podman run --volume /etc/kubernetes:/etc/kubernetes:ro,z --entrypoint /usr/local/bin/kubectl quay.io/poseidon/kubelet:v1.18.3-4-g57fa88a --kubeconfig=/etc/kubernetes/kubeconfig delete node $HOSTNAME' [Install] WantedBy=multi-user.target storage: diff --git a/docs/advanced/customization.md b/docs/advanced/customization.md index 91552eb40..2001a59b8 100644 --- a/docs/advanced/customization.md +++ b/docs/advanced/customization.md @@ -174,3 +174,34 @@ module "nemo" { To customize low-level Kubernetes control plane bootstrapping, see the [poseidon/terraform-render-bootstrap](https://github.com/poseidon/terraform-render-bootstrap) Terraform module. +## Kubelet + +Typhoon publishes Kubelet [container images](/topics/security.md#container-images) to Quay.io (default) and to Dockerhub (in case of a Quay [outage](https://github.com/poseidon/typhoon/issues/735) or breach). Quay automated builds also provide the option for fully verifiable tagged images (`build-{short_sha}`). + +To set an alternative Kubelet image, use a snippet to set a systemd dropin. + +``` +# host-image-override.yaml +variant: fcos <- remove for Flatcar Linux +version: 1.0.0 <- remove for Flatcar Linux +systemd: + units: + - name: kubelet.service + dropins: + - name: 10-image-override.conf + contents: | + [Service] + Environment=KUBELET_IMAGE=docker.io/psdn/kubelet:v1.18.3 +``` + +``` +module "nemo" { + ... + + worker_snippets = [ + file("./snippets/host-image-override.yaml") + ] + ... +} +``` + diff --git a/docs/topics/security.md b/docs/topics/security.md index 5542a98fc..78e5f4fb1 100644 --- a/docs/topics/security.md +++ b/docs/topics/security.md @@ -52,7 +52,19 @@ Typhoon uses upstream container images (where possible) and upstream binaries. !!! note Kubernetes releases `kubelet` as a binary for distros to package, either as a DEB/RPM on traditional distros or as a container image for container-optimized operating systems. -Typhoon [packages](https://github.com/poseidon/kubelet) the upstream Kubelet and its dependencies as a [container image](https://quay.io/repository/poseidon/kubelet) for use in Typhoon. The upstream Kubelet binary is checksummed and packaged directly. Quay automated builds provide verifiability and confidence in image contents. +Typhoon [packages](https://github.com/poseidon/kubelet) the upstream Kubelet and its dependencies as a [container image](https://quay.io/repository/poseidon/kubelet). Builds fetch the upstream Kubelet binary and verify its checksum. + +The Kubelet image is published to Quay.io and Dockerhub. + +* [quay.io/poseidon/kubelet](https://quay.io/repository/poseidon/kubelet) (official) +* [docker.io/psdn/kubelet](https://hub.docker.com/r/psdn/kubelet) (fallback) + +Two tag styles indicate the build strategy used. + +* Typhoon internal infra publishes single and multi-arch images (e.g. `v1.18.3`, `v1.18.3-amd64`, `v1.18.3-arm64`, `v1.18.3-2-g23228e6-amd64`, `v1.18.3-2-g23228e6-arm64`) +* Quay and Dockerhub automated builds publish verifiable images (e.g. `build-SHA` on Quay, `build-TAG` on Dockerhub) + +The Typhoon-built Kubelet image is used as the official image. Automated builds provide an alternative image for those preferring to trust images built by Quay/Dockerhub (albeit lacking multi-arch). To use the fallback registry or an alternative tag, see [customization](/advanced/customization.md#kubelet). ## Disclosures diff --git a/google-cloud/container-linux/kubernetes/cl/controller.yaml b/google-cloud/container-linux/kubernetes/cl/controller.yaml index eef508f57..53cd7a747 100644 --- a/google-cloud/container-linux/kubernetes/cl/controller.yaml +++ b/google-cloud/container-linux/kubernetes/cl/controller.yaml @@ -52,7 +52,7 @@ systemd: Description=Kubelet Wants=rpc-statd.service [Service] - Environment=KUBELET_IMAGE=docker://quay.io/poseidon/kubelet:v1.18.3 + Environment=KUBELET_IMAGE=docker://quay.io/poseidon/kubelet:v1.18.3-4-g57fa88a ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests ExecStartPre=/bin/mkdir -p /opt/cni/bin @@ -135,7 +135,7 @@ systemd: --volume script,kind=host,source=/opt/bootstrap/apply \ --mount volume=script,target=/apply \ --insecure-options=image \ - docker://quay.io/poseidon/kubelet:v1.18.3 \ + docker://quay.io/poseidon/kubelet:v1.18.3-4-g57fa88a \ --net=host \ --dns=host \ --exec=/apply diff --git a/google-cloud/container-linux/kubernetes/workers/cl/worker.yaml b/google-cloud/container-linux/kubernetes/workers/cl/worker.yaml index f050d52ee..1351cb91f 100644 --- a/google-cloud/container-linux/kubernetes/workers/cl/worker.yaml +++ b/google-cloud/container-linux/kubernetes/workers/cl/worker.yaml @@ -25,7 +25,7 @@ systemd: Description=Kubelet Wants=rpc-statd.service [Service] - Environment=KUBELET_IMAGE=docker://quay.io/poseidon/kubelet:v1.18.3 + Environment=KUBELET_IMAGE=docker://quay.io/poseidon/kubelet:v1.18.3-4-g57fa88a ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests ExecStartPre=/bin/mkdir -p /opt/cni/bin @@ -128,7 +128,7 @@ storage: --volume config,kind=host,source=/etc/kubernetes \ --mount volume=config,target=/etc/kubernetes \ --insecure-options=image \ - docker://quay.io/poseidon/kubelet:v1.18.3 \ + docker://quay.io/poseidon/kubelet:v1.18.3-4-g57fa88a \ --net=host \ --dns=host \ --exec=/usr/local/bin/kubectl -- --kubeconfig=/etc/kubernetes/kubeconfig delete node $(hostname) diff --git a/google-cloud/fedora-coreos/kubernetes/fcc/controller.yaml b/google-cloud/fedora-coreos/kubernetes/fcc/controller.yaml index 129a8d925..7bea44b02 100644 --- a/google-cloud/fedora-coreos/kubernetes/fcc/controller.yaml +++ b/google-cloud/fedora-coreos/kubernetes/fcc/controller.yaml @@ -54,7 +54,7 @@ systemd: Description=Kubelet (System Container) Wants=rpc-statd.service [Service] - Environment=KUBELET_IMAGE=quay.io/poseidon/kubelet:v1.18.3 + Environment=KUBELET_IMAGE=quay.io/poseidon/kubelet:v1.18.3-4-g57fa88a ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests ExecStartPre=/bin/mkdir -p /opt/cni/bin @@ -126,7 +126,7 @@ systemd: --volume /opt/bootstrap/assets:/assets:ro,Z \ --volume /opt/bootstrap/apply:/apply:ro,Z \ --entrypoint=/apply \ - quay.io/poseidon/kubelet:v1.18.3 + quay.io/poseidon/kubelet:v1.18.3-4-g57fa88a ExecStartPost=/bin/touch /opt/bootstrap/bootstrap.done ExecStartPost=-/usr/bin/podman stop bootstrap storage: diff --git a/google-cloud/fedora-coreos/kubernetes/workers/fcc/worker.yaml b/google-cloud/fedora-coreos/kubernetes/workers/fcc/worker.yaml index d33c43db5..062f117a2 100644 --- a/google-cloud/fedora-coreos/kubernetes/workers/fcc/worker.yaml +++ b/google-cloud/fedora-coreos/kubernetes/workers/fcc/worker.yaml @@ -24,7 +24,7 @@ systemd: Description=Kubelet (System Container) Wants=rpc-statd.service [Service] - Environment=KUBELET_IMAGE=quay.io/poseidon/kubelet:v1.18.3 + Environment=KUBELET_IMAGE=quay.io/poseidon/kubelet:v1.18.3-4-g57fa88a ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests ExecStartPre=/bin/mkdir -p /opt/cni/bin @@ -90,7 +90,7 @@ systemd: Type=oneshot RemainAfterExit=true ExecStart=/bin/true - ExecStop=/bin/bash -c '/usr/bin/podman run --volume /etc/kubernetes:/etc/kubernetes:ro,z --entrypoint /usr/local/bin/kubectl quay.io/poseidon/kubelet:v1.18.3 --kubeconfig=/etc/kubernetes/kubeconfig delete node $HOSTNAME' + ExecStop=/bin/bash -c '/usr/bin/podman run --volume /etc/kubernetes:/etc/kubernetes:ro,z --entrypoint /usr/local/bin/kubectl quay.io/poseidon/kubelet:v1.18.3-4-g57fa88a --kubeconfig=/etc/kubernetes/kubeconfig delete node $HOSTNAME' [Install] WantedBy=multi-user.target storage: From 99dbce67a39e77839fb723c90874cc892287644e Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Sun, 31 May 2020 16:18:49 -0700 Subject: [PATCH 438/523] Tweak minor style elements of issue templates --- .github/ISSUE_TEMPLATE/bug_report.md | 2 +- .github/ISSUE_TEMPLATE/config.yml | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md index 715538624..cca945fa7 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.md +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -32,7 +32,7 @@ A clear and concise description of what you expected to happen. * Terraform: `terraform version` (reporting latest is **not** helpful) * Plugins: Provider plugin versions (reporting latest is **not** helpful) -### Possible Solution +**Possible Solution** diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml index 15a10dce0..1a270766e 100644 --- a/.github/ISSUE_TEMPLATE/config.yml +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -1,5 +1,5 @@ blank_issues_enabled: true contact_links: - - name: Typhoon Security + - name: Security url: https://typhoon.psdn.io/topics/security/ - about: Please report security vulnerabilities via email + about: Report security vulnerabilities From 16c0b9152b5a476941a8fbaad5bac27a33b7c4c0 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Wed, 3 Jun 2020 11:35:10 -0700 Subject: [PATCH 439/523] Update kube-state-metrics from v1.9.6 to v1.9.7 * https://github.com/kubernetes/kube-state-metrics/releases/tag/v1.9.7 --- CHANGES.md | 1 + addons/prometheus/exporters/kube-state-metrics/deployment.yaml | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index 4d420cbbc..a342a6a23 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -17,6 +17,7 @@ Notable changes between versions. ### Addons * Update node-exporter from v1.0.0-rc.1 to [v1.0.0](https://github.com/prometheus/node_exporter/releases/tag/v1.0.0) +* Update kube-state-metrics from v1.9.6 to v1.9.7 * Update Grafana from v7.0.0 to v7.0.1 ## v1.18.3 diff --git a/addons/prometheus/exporters/kube-state-metrics/deployment.yaml b/addons/prometheus/exporters/kube-state-metrics/deployment.yaml index d05f811a8..fb5389a57 100644 --- a/addons/prometheus/exporters/kube-state-metrics/deployment.yaml +++ b/addons/prometheus/exporters/kube-state-metrics/deployment.yaml @@ -24,7 +24,7 @@ spec: serviceAccountName: kube-state-metrics containers: - name: kube-state-metrics - image: quay.io/coreos/kube-state-metrics:v1.9.6 + image: quay.io/coreos/kube-state-metrics:v1.9.7 ports: - name: metrics containerPort: 8080 From 8f875f80f5118b45fbc1a89cde8a5ee58a20cf2d Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Wed, 3 Jun 2020 12:31:58 -0700 Subject: [PATCH 440/523] Update Grafana from v7.0.1 to v7.0.3 * https://github.com/grafana/grafana/releases/tag/v7.0.2 * https://github.com/grafana/grafana/releases/tag/v7.0.3 --- CHANGES.md | 2 +- addons/grafana/deployment.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index a342a6a23..338bd7deb 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -18,7 +18,7 @@ Notable changes between versions. * Update node-exporter from v1.0.0-rc.1 to [v1.0.0](https://github.com/prometheus/node_exporter/releases/tag/v1.0.0) * Update kube-state-metrics from v1.9.6 to v1.9.7 -* Update Grafana from v7.0.0 to v7.0.1 +* Update Grafana from v7.0.0 to v7.0.3 ## v1.18.3 diff --git a/addons/grafana/deployment.yaml b/addons/grafana/deployment.yaml index 49bf94f39..197e535d4 100644 --- a/addons/grafana/deployment.yaml +++ b/addons/grafana/deployment.yaml @@ -23,7 +23,7 @@ spec: spec: containers: - name: grafana - image: docker.io/grafana/grafana:7.0.1 + image: docker.io/grafana/grafana:7.0.3 env: - name: GF_PATHS_CONFIG value: "/etc/grafana/custom.ini" From 31d02b022130d3273f26b7d3b94fb561376f9730 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Fri, 5 Jun 2020 00:16:45 -0700 Subject: [PATCH 441/523] Update Prometheus from v2.18.1 to v2.19.0-rc.0 * https://github.com/prometheus/prometheus/releases/tag/v2.19.0-rc.0 --- CHANGES.md | 1 + addons/prometheus/deployment.yaml | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index 338bd7deb..e27f4156b 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -16,6 +16,7 @@ Notable changes between versions. ### Addons +* Update Prometheus from v2.18.1 to [v2.19.0-rc.0](https://github.com/prometheus/prometheus/releases/tag/v2.19.0-rc.0) * Update node-exporter from v1.0.0-rc.1 to [v1.0.0](https://github.com/prometheus/node_exporter/releases/tag/v1.0.0) * Update kube-state-metrics from v1.9.6 to v1.9.7 * Update Grafana from v7.0.0 to v7.0.3 diff --git a/addons/prometheus/deployment.yaml b/addons/prometheus/deployment.yaml index 59fd27698..591e003ec 100644 --- a/addons/prometheus/deployment.yaml +++ b/addons/prometheus/deployment.yaml @@ -20,7 +20,7 @@ spec: serviceAccountName: prometheus containers: - name: prometheus - image: quay.io/prometheus/prometheus:v2.18.1 + image: quay.io/prometheus/prometheus:v2.19.0-rc.0 args: - --web.listen-address=0.0.0.0:9090 - --config.file=/etc/prometheus/prometheus.yaml From aed1a5f33df635a3151ed21ed4d9c5a5d8cd2041 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Mon, 8 Jun 2020 12:25:57 -0700 Subject: [PATCH 442/523] Fix Fedora CoreOS docs for selecting a stream * Fedora CoreOS image `os_stream` stable, testing, and next have been configurable since v1.18.3 * Remove mention of outdated `os_image` variable --- docs/fedora-coreos/aws.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/fedora-coreos/aws.md b/docs/fedora-coreos/aws.md index 7f302374c..57e8724f8 100644 --- a/docs/fedora-coreos/aws.md +++ b/docs/fedora-coreos/aws.md @@ -208,7 +208,7 @@ Reference the DNS zone id with `aws_route53_zone.zone-for-clusters.zone_id`. | worker_count | Number of workers | 1 | 3 | | controller_type | EC2 instance type for controllers | "t3.small" | See below | | worker_type | EC2 instance type for workers | "t3.small" | See below | -| os_image | AMI channel for Fedora CoreOS | not yet used | ? | +| os_stream | Fedora CoreOS stream for compute instances | "stable" | "testing", "next" | | disk_size | Size of the EBS volume in GB | 40 | 100 | | disk_type | Type of the EBS volume | "gp2" | standard, gp2, io1 | | disk_iops | IOPS of the EBS volume | 0 (i.e. auto) | 400 | From 8dc170b9d9e56d1b7491c429badd7f722bb27987 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Mon, 8 Jun 2020 12:37:09 -0700 Subject: [PATCH 443/523] Update security disclosure contact email * Use security@psdn.io across github.com/poseidon projects --- docs/advanced/customization.md | 2 +- docs/topics/security.md | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/advanced/customization.md b/docs/advanced/customization.md index 2001a59b8..c92d1adbf 100644 --- a/docs/advanced/customization.md +++ b/docs/advanced/customization.md @@ -176,7 +176,7 @@ To customize low-level Kubernetes control plane bootstrapping, see the [poseidon ## Kubelet -Typhoon publishes Kubelet [container images](/topics/security.md#container-images) to Quay.io (default) and to Dockerhub (in case of a Quay [outage](https://github.com/poseidon/typhoon/issues/735) or breach). Quay automated builds also provide the option for fully verifiable tagged images (`build-{short_sha}`). +Typhoon publishes Kubelet [container images](/topics/security/#container-images) to Quay.io (default) and to Dockerhub (in case of a Quay [outage](https://github.com/poseidon/typhoon/issues/735) or breach). Quay automated builds also provide the option for fully verifiable tagged images (`build-{short_sha}`). To set an alternative Kubelet image, use a snippet to set a systemd dropin. diff --git a/docs/topics/security.md b/docs/topics/security.md index 78e5f4fb1..e793c7a08 100644 --- a/docs/topics/security.md +++ b/docs/topics/security.md @@ -64,9 +64,9 @@ Two tag styles indicate the build strategy used. * Typhoon internal infra publishes single and multi-arch images (e.g. `v1.18.3`, `v1.18.3-amd64`, `v1.18.3-arm64`, `v1.18.3-2-g23228e6-amd64`, `v1.18.3-2-g23228e6-arm64`) * Quay and Dockerhub automated builds publish verifiable images (e.g. `build-SHA` on Quay, `build-TAG` on Dockerhub) -The Typhoon-built Kubelet image is used as the official image. Automated builds provide an alternative image for those preferring to trust images built by Quay/Dockerhub (albeit lacking multi-arch). To use the fallback registry or an alternative tag, see [customization](/advanced/customization.md#kubelet). +The Typhoon-built Kubelet image is used as the official image. Automated builds provide an alternative image for those preferring to trust images built by Quay/Dockerhub (albeit lacking multi-arch). To use the fallback registry or an alternative tag, see [customization](/advanced/customization/#kubelet). ## Disclosures -If you find security issues, please email dghubble at gmail. If the issue lies in upstream Kubernetes, please inform upstream Kubernetes as well. +If you find security issues, please email `security@psdn.io`. If the issue lies in upstream Kubernetes, please inform upstream Kubernetes as well. From a287920169659a8d103c8f9a6a82af237824d464 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Tue, 9 Jun 2020 22:38:32 -0700 Subject: [PATCH 444/523] Use strict mode for Container Linux Configs * Enable terraform-provider-ct `strict` mode for parsing Container Linux Configs and snippets * Fix Container Linux Config systemd unit syntax `enable` (old) to `enabled` * Align with Fedora CoreOS which uses strict mode already --- CHANGES.md | 11 +++++++++++ .../kubernetes/cl/controller.yaml | 9 +++++---- aws/container-linux/kubernetes/controllers.tf | 8 ++++---- aws/container-linux/kubernetes/versions.tf | 2 +- .../kubernetes/workers/cl/worker.yaml | 7 ++++--- .../kubernetes/workers/workers.tf | 6 +++--- .../kubernetes/cl/controller.yaml | 9 +++++---- azure/container-linux/kubernetes/controllers.tf | 8 ++++---- azure/container-linux/kubernetes/versions.tf | 2 +- .../kubernetes/workers/cl/worker.yaml | 9 +++++---- .../kubernetes/workers/workers.tf | 6 +++--- azure/fedora-coreos/kubernetes/controllers.tf | 8 ++++---- azure/fedora-coreos/kubernetes/versions.tf | 2 +- .../fedora-coreos/kubernetes/workers/workers.tf | 6 +++--- .../kubernetes/cl/controller.yaml | 10 ++++++---- .../container-linux/kubernetes/cl/install.yaml | 2 +- .../container-linux/kubernetes/cl/worker.yaml | 8 +++++--- .../container-linux/kubernetes/profiles.tf | 16 ++++++++-------- .../container-linux/kubernetes/versions.tf | 2 +- .../kubernetes/cl/controller.yaml | 10 ++++++---- .../container-linux/kubernetes/cl/worker.yaml | 10 ++++++---- .../container-linux/kubernetes/controllers.tf | 8 ++++---- .../container-linux/kubernetes/versions.tf | 2 +- .../container-linux/kubernetes/workers.tf | 6 +++--- .../fedora-coreos/kubernetes/versions.tf | 2 +- .../kubernetes/cl/controller.yaml | 9 +++++---- .../container-linux/kubernetes/controllers.tf | 8 ++++---- .../container-linux/kubernetes/versions.tf | 2 +- .../kubernetes/workers/cl/worker.yaml | 9 +++++---- .../kubernetes/workers/workers.tf | 6 +++--- .../fedora-coreos/kubernetes/versions.tf | 2 +- 31 files changed, 115 insertions(+), 90 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index e27f4156b..ab1c52b9c 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -14,6 +14,17 @@ Notable changes between versions. strategy (see [docs](https://typhoon.psdn.io/topics/security/#container-images)) * Update Calico from v3.14.0 to [v3.14.1](https://docs.projectcalico.org/v3.14/release-notes/) +### Fedora CoreOS + +#### Azure + +* Use `strict` Fedora CoreOS Config (FCC) snippet parsing ([#755](https://github.com/poseidon/typhoon/pull/755)) + +### Flatcar Linux + +* Use `strict` Container Linux Config (CLC) snippet parsing ([#755](https://github.com/poseidon/typhoon/pull/755)) + * Require `terraform-provider-ct` v0.4+, recommend v0.5+ (**action required**) + ### Addons * Update Prometheus from v2.18.1 to [v2.19.0-rc.0](https://github.com/prometheus/prometheus/releases/tag/v2.19.0-rc.0) diff --git a/aws/container-linux/kubernetes/cl/controller.yaml b/aws/container-linux/kubernetes/cl/controller.yaml index 49873f6fa..ac7efce42 100644 --- a/aws/container-linux/kubernetes/cl/controller.yaml +++ b/aws/container-linux/kubernetes/cl/controller.yaml @@ -2,7 +2,7 @@ systemd: units: - name: etcd-member.service - enable: true + enabled: true dropins: - name: 40-etcd-cluster.conf contents: | @@ -28,11 +28,11 @@ systemd: Environment="ETCD_PEER_KEY_FILE=/etc/ssl/certs/etcd/peer.key" Environment="ETCD_PEER_CLIENT_CERT_AUTH=true" - name: docker.service - enable: true + enabled: true - name: locksmithd.service mask: true - name: wait-for-dns.service - enable: true + enabled: true contents: | [Unit] Description=Wait for DNS entries @@ -46,7 +46,7 @@ systemd: RequiredBy=kubelet.service RequiredBy=etcd-member.service - name: kubelet.service - enable: true + enabled: true contents: | [Unit] Description=Kubelet @@ -191,6 +191,7 @@ storage: done - path: /etc/sysctl.d/max-user-watches.conf filesystem: root + mode: 0644 contents: inline: | fs.inotify.max_user_watches=16184 diff --git a/aws/container-linux/kubernetes/controllers.tf b/aws/container-linux/kubernetes/controllers.tf index 66a0a9bc2..252c6b9d8 100644 --- a/aws/container-linux/kubernetes/controllers.tf +++ b/aws/container-linux/kubernetes/controllers.tf @@ -49,10 +49,10 @@ resource "aws_instance" "controllers" { # Controller Ignition configs data "ct_config" "controller-ignitions" { - count = var.controller_count - content = data.template_file.controller-configs.*.rendered[count.index] - pretty_print = false - snippets = var.controller_snippets + count = var.controller_count + content = data.template_file.controller-configs.*.rendered[count.index] + strict = true + snippets = var.controller_snippets } # Controller Container Linux configs diff --git a/aws/container-linux/kubernetes/versions.tf b/aws/container-linux/kubernetes/versions.tf index f7a10ff15..bd2776f08 100644 --- a/aws/container-linux/kubernetes/versions.tf +++ b/aws/container-linux/kubernetes/versions.tf @@ -4,7 +4,7 @@ terraform { required_version = "~> 0.12.6" required_providers { aws = "~> 2.23" - ct = "~> 0.3" + ct = "~> 0.4" template = "~> 2.1" null = "~> 2.1" } diff --git a/aws/container-linux/kubernetes/workers/cl/worker.yaml b/aws/container-linux/kubernetes/workers/cl/worker.yaml index e6272747b..3743b6b67 100644 --- a/aws/container-linux/kubernetes/workers/cl/worker.yaml +++ b/aws/container-linux/kubernetes/workers/cl/worker.yaml @@ -2,11 +2,11 @@ systemd: units: - name: docker.service - enable: true + enabled: true - name: locksmithd.service mask: true - name: wait-for-dns.service - enable: true + enabled: true contents: | [Unit] Description=Wait for DNS entries @@ -19,7 +19,7 @@ systemd: [Install] RequiredBy=kubelet.service - name: kubelet.service - enable: true + enabled: true contents: | [Unit] Description=Kubelet @@ -115,6 +115,7 @@ storage: ${kubeconfig} - path: /etc/sysctl.d/max-user-watches.conf filesystem: root + mode: 0644 contents: inline: | fs.inotify.max_user_watches=16184 diff --git a/aws/container-linux/kubernetes/workers/workers.tf b/aws/container-linux/kubernetes/workers/workers.tf index 0d6ab035f..5b28c1052 100644 --- a/aws/container-linux/kubernetes/workers/workers.tf +++ b/aws/container-linux/kubernetes/workers/workers.tf @@ -71,9 +71,9 @@ resource "aws_launch_configuration" "worker" { # Worker Ignition config data "ct_config" "worker-ignition" { - content = data.template_file.worker-config.rendered - pretty_print = false - snippets = var.snippets + content = data.template_file.worker-config.rendered + strict = true + snippets = var.snippets } # Worker Container Linux config diff --git a/azure/container-linux/kubernetes/cl/controller.yaml b/azure/container-linux/kubernetes/cl/controller.yaml index b1ddb75cc..babca9b98 100644 --- a/azure/container-linux/kubernetes/cl/controller.yaml +++ b/azure/container-linux/kubernetes/cl/controller.yaml @@ -2,7 +2,7 @@ systemd: units: - name: etcd-member.service - enable: true + enabled: true dropins: - name: 40-etcd-cluster.conf contents: | @@ -28,11 +28,11 @@ systemd: Environment="ETCD_PEER_KEY_FILE=/etc/ssl/certs/etcd/peer.key" Environment="ETCD_PEER_CLIENT_CERT_AUTH=true" - name: docker.service - enable: true + enabled: true - name: locksmithd.service mask: true - name: wait-for-dns.service - enable: true + enabled: true contents: | [Unit] Description=Wait for DNS entries @@ -46,7 +46,7 @@ systemd: RequiredBy=kubelet.service RequiredBy=etcd-member.service - name: kubelet.service - enable: true + enabled: true contents: | [Unit] Description=Kubelet @@ -189,6 +189,7 @@ storage: done - path: /etc/sysctl.d/max-user-watches.conf filesystem: root + mode: 0644 contents: inline: | fs.inotify.max_user_watches=16184 diff --git a/azure/container-linux/kubernetes/controllers.tf b/azure/container-linux/kubernetes/controllers.tf index ad329b0f7..876ae7ca1 100644 --- a/azure/container-linux/kubernetes/controllers.tf +++ b/azure/container-linux/kubernetes/controllers.tf @@ -139,10 +139,10 @@ resource "azurerm_network_interface_backend_address_pool_association" "controlle # Controller Ignition configs data "ct_config" "controller-ignitions" { - count = var.controller_count - content = data.template_file.controller-configs.*.rendered[count.index] - pretty_print = false - snippets = var.controller_snippets + count = var.controller_count + content = data.template_file.controller-configs.*.rendered[count.index] + strict = true + snippets = var.controller_snippets } # Controller Container Linux configs diff --git a/azure/container-linux/kubernetes/versions.tf b/azure/container-linux/kubernetes/versions.tf index 09f4f3e2d..5da5fff00 100644 --- a/azure/container-linux/kubernetes/versions.tf +++ b/azure/container-linux/kubernetes/versions.tf @@ -4,7 +4,7 @@ terraform { required_version = "~> 0.12.6" required_providers { azurerm = "~> 2.8" - ct = "~> 0.3" + ct = "~> 0.4" template = "~> 2.1" null = "~> 2.1" } diff --git a/azure/container-linux/kubernetes/workers/cl/worker.yaml b/azure/container-linux/kubernetes/workers/cl/worker.yaml index 87e33e682..402e21a6f 100644 --- a/azure/container-linux/kubernetes/workers/cl/worker.yaml +++ b/azure/container-linux/kubernetes/workers/cl/worker.yaml @@ -2,11 +2,11 @@ systemd: units: - name: docker.service - enable: true + enabled: true - name: locksmithd.service mask: true - name: wait-for-dns.service - enable: true + enabled: true contents: | [Unit] Description=Wait for DNS entries @@ -19,7 +19,7 @@ systemd: [Install] RequiredBy=kubelet.service - name: kubelet.service - enable: true + enabled: true contents: | [Unit] Description=Kubelet @@ -92,7 +92,7 @@ systemd: [Install] WantedBy=multi-user.target - name: delete-node.service - enable: true + enabled: true contents: | [Unit] Description=Waiting to delete Kubernetes node on shutdown @@ -113,6 +113,7 @@ storage: ${kubeconfig} - path: /etc/sysctl.d/max-user-watches.conf filesystem: root + mode: 0644 contents: inline: | fs.inotify.max_user_watches=16184 diff --git a/azure/container-linux/kubernetes/workers/workers.tf b/azure/container-linux/kubernetes/workers/workers.tf index 025045541..7e8f008d2 100644 --- a/azure/container-linux/kubernetes/workers/workers.tf +++ b/azure/container-linux/kubernetes/workers/workers.tf @@ -97,9 +97,9 @@ resource "azurerm_monitor_autoscale_setting" "workers" { # Worker Ignition configs data "ct_config" "worker-ignition" { - content = data.template_file.worker-config.rendered - pretty_print = false - snippets = var.snippets + content = data.template_file.worker-config.rendered + strict = true + snippets = var.snippets } # Worker Container Linux configs diff --git a/azure/fedora-coreos/kubernetes/controllers.tf b/azure/fedora-coreos/kubernetes/controllers.tf index b82992315..02853e33a 100644 --- a/azure/fedora-coreos/kubernetes/controllers.tf +++ b/azure/fedora-coreos/kubernetes/controllers.tf @@ -113,10 +113,10 @@ resource "azurerm_network_interface_backend_address_pool_association" "controlle # Controller Ignition configs data "ct_config" "controller-ignitions" { - count = var.controller_count - content = data.template_file.controller-configs.*.rendered[count.index] - pretty_print = false - snippets = var.controller_snippets + count = var.controller_count + content = data.template_file.controller-configs.*.rendered[count.index] + strict = true + snippets = var.controller_snippets } # Controller Fedora CoreOS configs diff --git a/azure/fedora-coreos/kubernetes/versions.tf b/azure/fedora-coreos/kubernetes/versions.tf index 09f4f3e2d..5da5fff00 100644 --- a/azure/fedora-coreos/kubernetes/versions.tf +++ b/azure/fedora-coreos/kubernetes/versions.tf @@ -4,7 +4,7 @@ terraform { required_version = "~> 0.12.6" required_providers { azurerm = "~> 2.8" - ct = "~> 0.3" + ct = "~> 0.4" template = "~> 2.1" null = "~> 2.1" } diff --git a/azure/fedora-coreos/kubernetes/workers/workers.tf b/azure/fedora-coreos/kubernetes/workers/workers.tf index a5c2c209c..6ecb81a50 100644 --- a/azure/fedora-coreos/kubernetes/workers/workers.tf +++ b/azure/fedora-coreos/kubernetes/workers/workers.tf @@ -72,9 +72,9 @@ resource "azurerm_monitor_autoscale_setting" "workers" { # Worker Ignition configs data "ct_config" "worker-ignition" { - content = data.template_file.worker-config.rendered - pretty_print = false - snippets = var.snippets + content = data.template_file.worker-config.rendered + strict = true + snippets = var.snippets } # Worker Fedora CoreOS configs diff --git a/bare-metal/container-linux/kubernetes/cl/controller.yaml b/bare-metal/container-linux/kubernetes/cl/controller.yaml index 8ac9368b3..399430502 100644 --- a/bare-metal/container-linux/kubernetes/cl/controller.yaml +++ b/bare-metal/container-linux/kubernetes/cl/controller.yaml @@ -2,7 +2,7 @@ systemd: units: - name: etcd-member.service - enable: true + enabled: true dropins: - name: 40-etcd-cluster.conf contents: | @@ -28,11 +28,11 @@ systemd: Environment="ETCD_PEER_KEY_FILE=/etc/ssl/certs/etcd/peer.key" Environment="ETCD_PEER_CLIENT_CERT_AUTH=true" - name: docker.service - enable: true + enabled: true - name: locksmithd.service mask: true - name: kubelet.path - enable: true + enabled: true contents: | [Unit] Description=Watch for kubeconfig @@ -41,7 +41,7 @@ systemd: [Install] WantedBy=multi-user.target - name: wait-for-dns.service - enable: true + enabled: true contents: | [Unit] Description=Wait for DNS entries @@ -161,6 +161,7 @@ storage: directories: - path: /etc/kubernetes filesystem: root + mode: 0755 files: - path: /etc/hostname filesystem: root @@ -207,6 +208,7 @@ storage: done - path: /etc/sysctl.d/max-user-watches.conf filesystem: root + mode: 0644 contents: inline: | fs.inotify.max_user_watches=16184 diff --git a/bare-metal/container-linux/kubernetes/cl/install.yaml b/bare-metal/container-linux/kubernetes/cl/install.yaml index e8562c932..6a36f19dc 100644 --- a/bare-metal/container-linux/kubernetes/cl/install.yaml +++ b/bare-metal/container-linux/kubernetes/cl/install.yaml @@ -2,7 +2,7 @@ systemd: units: - name: installer.service - enable: true + enabled: true contents: | [Unit] Requires=network-online.target diff --git a/bare-metal/container-linux/kubernetes/cl/worker.yaml b/bare-metal/container-linux/kubernetes/cl/worker.yaml index 73bdea996..e83bf3898 100644 --- a/bare-metal/container-linux/kubernetes/cl/worker.yaml +++ b/bare-metal/container-linux/kubernetes/cl/worker.yaml @@ -2,11 +2,11 @@ systemd: units: - name: docker.service - enable: true + enabled: true - name: locksmithd.service mask: true - name: kubelet.path - enable: true + enabled: true contents: | [Unit] Description=Watch for kubeconfig @@ -15,7 +15,7 @@ systemd: [Install] WantedBy=multi-user.target - name: wait-for-dns.service - enable: true + enabled: true contents: | [Unit] Description=Wait for DNS entries @@ -114,6 +114,7 @@ storage: directories: - path: /etc/kubernetes filesystem: root + mode: 0755 files: - path: /etc/hostname filesystem: root @@ -123,6 +124,7 @@ storage: ${domain_name} - path: /etc/sysctl.d/max-user-watches.conf filesystem: root + mode: 0644 contents: inline: | fs.inotify.max_user_watches=16184 diff --git a/bare-metal/container-linux/kubernetes/profiles.tf b/bare-metal/container-linux/kubernetes/profiles.tf index 7ec6cc3a7..eee772539 100644 --- a/bare-metal/container-linux/kubernetes/profiles.tf +++ b/bare-metal/container-linux/kubernetes/profiles.tf @@ -141,10 +141,10 @@ resource "matchbox_profile" "controllers" { } data "ct_config" "controller-ignitions" { - count = length(var.controllers) - content = data.template_file.controller-configs.*.rendered[count.index] - pretty_print = false - snippets = lookup(var.snippets, var.controllers.*.name[count.index], []) + count = length(var.controllers) + content = data.template_file.controller-configs.*.rendered[count.index] + strict = true + snippets = lookup(var.snippets, var.controllers.*.name[count.index], []) } data "template_file" "controller-configs" { @@ -171,10 +171,10 @@ resource "matchbox_profile" "workers" { } data "ct_config" "worker-ignitions" { - count = length(var.workers) - content = data.template_file.worker-configs.*.rendered[count.index] - pretty_print = false - snippets = lookup(var.snippets, var.workers.*.name[count.index], []) + count = length(var.workers) + content = data.template_file.worker-configs.*.rendered[count.index] + strict = true + snippets = lookup(var.snippets, var.workers.*.name[count.index], []) } data "template_file" "worker-configs" { diff --git a/bare-metal/container-linux/kubernetes/versions.tf b/bare-metal/container-linux/kubernetes/versions.tf index f7f7aaf69..cf547f19d 100644 --- a/bare-metal/container-linux/kubernetes/versions.tf +++ b/bare-metal/container-linux/kubernetes/versions.tf @@ -4,7 +4,7 @@ terraform { required_version = "~> 0.12.6" required_providers { matchbox = "~> 0.3.0" - ct = "~> 0.3" + ct = "~> 0.4" template = "~> 2.1" null = "~> 2.1" } diff --git a/digital-ocean/container-linux/kubernetes/cl/controller.yaml b/digital-ocean/container-linux/kubernetes/cl/controller.yaml index dbcacebf4..ef1395122 100644 --- a/digital-ocean/container-linux/kubernetes/cl/controller.yaml +++ b/digital-ocean/container-linux/kubernetes/cl/controller.yaml @@ -2,7 +2,7 @@ systemd: units: - name: etcd-member.service - enable: true + enabled: true dropins: - name: 40-etcd-cluster.conf contents: | @@ -28,11 +28,11 @@ systemd: Environment="ETCD_PEER_KEY_FILE=/etc/ssl/certs/etcd/peer.key" Environment="ETCD_PEER_CLIENT_CERT_AUTH=true" - name: docker.service - enable: true + enabled: true - name: locksmithd.service mask: true - name: kubelet.path - enable: true + enabled: true contents: | [Unit] Description=Watch for kubeconfig @@ -41,7 +41,7 @@ systemd: [Install] WantedBy=multi-user.target - name: wait-for-dns.service - enable: true + enabled: true contents: | [Unit] Description=Wait for DNS entries @@ -158,6 +158,7 @@ storage: directories: - path: /etc/kubernetes filesystem: root + mode: 0755 files: - path: /opt/bootstrap/layout filesystem: root @@ -198,6 +199,7 @@ storage: done - path: /etc/sysctl.d/max-user-watches.conf filesystem: root + mode: 0644 contents: inline: | fs.inotify.max_user_watches=16184 diff --git a/digital-ocean/container-linux/kubernetes/cl/worker.yaml b/digital-ocean/container-linux/kubernetes/cl/worker.yaml index b1803b2da..387f0f034 100644 --- a/digital-ocean/container-linux/kubernetes/cl/worker.yaml +++ b/digital-ocean/container-linux/kubernetes/cl/worker.yaml @@ -2,11 +2,11 @@ systemd: units: - name: docker.service - enable: true + enabled: true - name: locksmithd.service mask: true - name: kubelet.path - enable: true + enabled: true contents: | [Unit] Description=Watch for kubeconfig @@ -15,7 +15,7 @@ systemd: [Install] WantedBy=multi-user.target - name: wait-for-dns.service - enable: true + enabled: true contents: | [Unit] Description=Wait for DNS entries @@ -101,7 +101,7 @@ systemd: [Install] WantedBy=multi-user.target - name: delete-node.service - enable: true + enabled: true contents: | [Unit] Description=Waiting to delete Kubernetes node on shutdown @@ -116,9 +116,11 @@ storage: directories: - path: /etc/kubernetes filesystem: root + mode: 0755 files: - path: /etc/sysctl.d/max-user-watches.conf filesystem: root + mode: 0644 contents: inline: | fs.inotify.max_user_watches=16184 diff --git a/digital-ocean/container-linux/kubernetes/controllers.tf b/digital-ocean/container-linux/kubernetes/controllers.tf index 6d183b952..9efe84270 100644 --- a/digital-ocean/container-linux/kubernetes/controllers.tf +++ b/digital-ocean/container-linux/kubernetes/controllers.tf @@ -69,10 +69,10 @@ resource "digitalocean_tag" "controllers" { # Controller Ignition configs data "ct_config" "controller-ignitions" { - count = var.controller_count - content = data.template_file.controller-configs.*.rendered[count.index] - pretty_print = false - snippets = var.controller_snippets + count = var.controller_count + content = data.template_file.controller-configs.*.rendered[count.index] + strict = true + snippets = var.controller_snippets } # Controller Container Linux configs diff --git a/digital-ocean/container-linux/kubernetes/versions.tf b/digital-ocean/container-linux/kubernetes/versions.tf index c0e31a276..38b32c8d1 100644 --- a/digital-ocean/container-linux/kubernetes/versions.tf +++ b/digital-ocean/container-linux/kubernetes/versions.tf @@ -4,7 +4,7 @@ terraform { required_version = "~> 0.12.6" required_providers { digitalocean = "~> 1.3" - ct = "~> 0.3" + ct = "~> 0.4" template = "~> 2.1" null = "~> 2.1" } diff --git a/digital-ocean/container-linux/kubernetes/workers.tf b/digital-ocean/container-linux/kubernetes/workers.tf index fa9b85d61..cdd94ebae 100644 --- a/digital-ocean/container-linux/kubernetes/workers.tf +++ b/digital-ocean/container-linux/kubernetes/workers.tf @@ -58,9 +58,9 @@ resource "digitalocean_tag" "workers" { # Worker Ignition config data "ct_config" "worker-ignition" { - content = data.template_file.worker-config.rendered - pretty_print = false - snippets = var.worker_snippets + content = data.template_file.worker-config.rendered + strict = true + snippets = var.worker_snippets } # Worker Container Linux config diff --git a/digital-ocean/fedora-coreos/kubernetes/versions.tf b/digital-ocean/fedora-coreos/kubernetes/versions.tf index c0e31a276..38b32c8d1 100644 --- a/digital-ocean/fedora-coreos/kubernetes/versions.tf +++ b/digital-ocean/fedora-coreos/kubernetes/versions.tf @@ -4,7 +4,7 @@ terraform { required_version = "~> 0.12.6" required_providers { digitalocean = "~> 1.3" - ct = "~> 0.3" + ct = "~> 0.4" template = "~> 2.1" null = "~> 2.1" } diff --git a/google-cloud/container-linux/kubernetes/cl/controller.yaml b/google-cloud/container-linux/kubernetes/cl/controller.yaml index 53cd7a747..28bc25102 100644 --- a/google-cloud/container-linux/kubernetes/cl/controller.yaml +++ b/google-cloud/container-linux/kubernetes/cl/controller.yaml @@ -2,7 +2,7 @@ systemd: units: - name: etcd-member.service - enable: true + enabled: true dropins: - name: 40-etcd-cluster.conf contents: | @@ -28,11 +28,11 @@ systemd: Environment="ETCD_PEER_KEY_FILE=/etc/ssl/certs/etcd/peer.key" Environment="ETCD_PEER_CLIENT_CERT_AUTH=true" - name: docker.service - enable: true + enabled: true - name: locksmithd.service mask: true - name: wait-for-dns.service - enable: true + enabled: true contents: | [Unit] Description=Wait for DNS entries @@ -46,7 +46,7 @@ systemd: RequiredBy=kubelet.service RequiredBy=etcd-member.service - name: kubelet.service - enable: true + enabled: true contents: | [Unit] Description=Kubelet @@ -189,6 +189,7 @@ storage: done - path: /etc/sysctl.d/max-user-watches.conf filesystem: root + mode: 0644 contents: inline: | fs.inotify.max_user_watches=16184 diff --git a/google-cloud/container-linux/kubernetes/controllers.tf b/google-cloud/container-linux/kubernetes/controllers.tf index 6ab09219d..58fb93121 100644 --- a/google-cloud/container-linux/kubernetes/controllers.tf +++ b/google-cloud/container-linux/kubernetes/controllers.tf @@ -65,10 +65,10 @@ resource "google_compute_instance" "controllers" { # Controller Ignition configs data "ct_config" "controller-ignitions" { - count = var.controller_count - content = data.template_file.controller-configs.*.rendered[count.index] - pretty_print = false - snippets = var.controller_snippets + count = var.controller_count + content = data.template_file.controller-configs.*.rendered[count.index] + strict = true + snippets = var.controller_snippets } # Controller Container Linux configs diff --git a/google-cloud/container-linux/kubernetes/versions.tf b/google-cloud/container-linux/kubernetes/versions.tf index 26ea74cac..7210a4b33 100644 --- a/google-cloud/container-linux/kubernetes/versions.tf +++ b/google-cloud/container-linux/kubernetes/versions.tf @@ -4,7 +4,7 @@ terraform { required_version = "~> 0.12.6" required_providers { google = ">= 2.19, < 4.0" - ct = "~> 0.3" + ct = "~> 0.4" template = "~> 2.1" null = "~> 2.1" } diff --git a/google-cloud/container-linux/kubernetes/workers/cl/worker.yaml b/google-cloud/container-linux/kubernetes/workers/cl/worker.yaml index 1351cb91f..048b667e1 100644 --- a/google-cloud/container-linux/kubernetes/workers/cl/worker.yaml +++ b/google-cloud/container-linux/kubernetes/workers/cl/worker.yaml @@ -2,11 +2,11 @@ systemd: units: - name: docker.service - enable: true + enabled: true - name: locksmithd.service mask: true - name: wait-for-dns.service - enable: true + enabled: true contents: | [Unit] Description=Wait for DNS entries @@ -19,7 +19,7 @@ systemd: [Install] RequiredBy=kubelet.service - name: kubelet.service - enable: true + enabled: true contents: | [Unit] Description=Kubelet @@ -92,7 +92,7 @@ systemd: [Install] WantedBy=multi-user.target - name: delete-node.service - enable: true + enabled: true contents: | [Unit] Description=Waiting to delete Kubernetes node on shutdown @@ -113,6 +113,7 @@ storage: ${kubeconfig} - path: /etc/sysctl.d/max-user-watches.conf filesystem: root + mode: 0644 contents: inline: | fs.inotify.max_user_watches=16184 diff --git a/google-cloud/container-linux/kubernetes/workers/workers.tf b/google-cloud/container-linux/kubernetes/workers/workers.tf index 7c130ed20..e635592ff 100644 --- a/google-cloud/container-linux/kubernetes/workers/workers.tf +++ b/google-cloud/container-linux/kubernetes/workers/workers.tf @@ -71,9 +71,9 @@ resource "google_compute_instance_template" "worker" { # Worker Ignition config data "ct_config" "worker-ignition" { - content = data.template_file.worker-config.rendered - pretty_print = false - snippets = var.snippets + content = data.template_file.worker-config.rendered + strict = true + snippets = var.snippets } # Worker Container Linux config diff --git a/google-cloud/fedora-coreos/kubernetes/versions.tf b/google-cloud/fedora-coreos/kubernetes/versions.tf index 26ea74cac..7210a4b33 100644 --- a/google-cloud/fedora-coreos/kubernetes/versions.tf +++ b/google-cloud/fedora-coreos/kubernetes/versions.tf @@ -4,7 +4,7 @@ terraform { required_version = "~> 0.12.6" required_providers { google = ">= 2.19, < 4.0" - ct = "~> 0.3" + ct = "~> 0.4" template = "~> 2.1" null = "~> 2.1" } From c9059d3fe9d3d7e822981bd23433eaede7980b5e Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Tue, 9 Jun 2020 23:05:03 -0700 Subject: [PATCH 445/523] Update Prometheus from v2.19.0-rc.0 to v2.19.0 * https://github.com/prometheus/prometheus/releases/tag/v2.19.0 --- CHANGES.md | 2 +- addons/prometheus/deployment.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index ab1c52b9c..cbdf16cb7 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -27,7 +27,7 @@ Notable changes between versions. ### Addons -* Update Prometheus from v2.18.1 to [v2.19.0-rc.0](https://github.com/prometheus/prometheus/releases/tag/v2.19.0-rc.0) +* Update Prometheus from v2.18.1 to [v2.19.0](https://github.com/prometheus/prometheus/releases/tag/v2.19.0) * Update node-exporter from v1.0.0-rc.1 to [v1.0.0](https://github.com/prometheus/node_exporter/releases/tag/v1.0.0) * Update kube-state-metrics from v1.9.6 to v1.9.7 * Update Grafana from v7.0.0 to v7.0.3 diff --git a/addons/prometheus/deployment.yaml b/addons/prometheus/deployment.yaml index 591e003ec..80acb9bf1 100644 --- a/addons/prometheus/deployment.yaml +++ b/addons/prometheus/deployment.yaml @@ -20,7 +20,7 @@ spec: serviceAccountName: prometheus containers: - name: prometheus - image: quay.io/prometheus/prometheus:v2.19.0-rc.0 + image: quay.io/prometheus/prometheus:v2.19.0 args: - --web.listen-address=0.0.0.0:9090 - --config.file=/etc/prometheus/prometheus.yaml From 96711d7f17d829409bef0da1a4b9a4d36f766944 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Thu, 11 Jun 2020 21:24:36 -0700 Subject: [PATCH 446/523] Remove unused Kubelet cert / key Terraform state * Generated Kubelet TLS certificate and key are not longer used or distributed to machines since Kubelet TLS bootstrap is used instead. Remove the certificate and key from state --- aws/container-linux/kubernetes/bootstrap.tf | 2 +- aws/fedora-coreos/kubernetes/bootstrap.tf | 2 +- azure/container-linux/kubernetes/bootstrap.tf | 2 +- azure/fedora-coreos/kubernetes/bootstrap.tf | 2 +- bare-metal/container-linux/kubernetes/bootstrap.tf | 2 +- bare-metal/fedora-coreos/kubernetes/bootstrap.tf | 2 +- digital-ocean/container-linux/kubernetes/bootstrap.tf | 2 +- digital-ocean/fedora-coreos/kubernetes/bootstrap.tf | 2 +- google-cloud/container-linux/kubernetes/bootstrap.tf | 2 +- google-cloud/fedora-coreos/kubernetes/bootstrap.tf | 2 +- 10 files changed, 10 insertions(+), 10 deletions(-) diff --git a/aws/container-linux/kubernetes/bootstrap.tf b/aws/container-linux/kubernetes/bootstrap.tf index 16ed8dcf1..b7944e21a 100644 --- a/aws/container-linux/kubernetes/bootstrap.tf +++ b/aws/container-linux/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=c3b1f23b5daa77c8ff96a2294924aa138a1433a4" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=fc1a7bac89c6c95082334e0f79e478ee321596c0" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/aws/fedora-coreos/kubernetes/bootstrap.tf b/aws/fedora-coreos/kubernetes/bootstrap.tf index 483a0ba11..4df01331d 100644 --- a/aws/fedora-coreos/kubernetes/bootstrap.tf +++ b/aws/fedora-coreos/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=c3b1f23b5daa77c8ff96a2294924aa138a1433a4" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=fc1a7bac89c6c95082334e0f79e478ee321596c0" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/azure/container-linux/kubernetes/bootstrap.tf b/azure/container-linux/kubernetes/bootstrap.tf index c5ffc2541..a829118da 100644 --- a/azure/container-linux/kubernetes/bootstrap.tf +++ b/azure/container-linux/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=c3b1f23b5daa77c8ff96a2294924aa138a1433a4" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=fc1a7bac89c6c95082334e0f79e478ee321596c0" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/azure/fedora-coreos/kubernetes/bootstrap.tf b/azure/fedora-coreos/kubernetes/bootstrap.tf index b2372ed01..3828f06bf 100644 --- a/azure/fedora-coreos/kubernetes/bootstrap.tf +++ b/azure/fedora-coreos/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=c3b1f23b5daa77c8ff96a2294924aa138a1433a4" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=fc1a7bac89c6c95082334e0f79e478ee321596c0" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/bare-metal/container-linux/kubernetes/bootstrap.tf b/bare-metal/container-linux/kubernetes/bootstrap.tf index 9a3262407..d1da2e668 100644 --- a/bare-metal/container-linux/kubernetes/bootstrap.tf +++ b/bare-metal/container-linux/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=c3b1f23b5daa77c8ff96a2294924aa138a1433a4" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=fc1a7bac89c6c95082334e0f79e478ee321596c0" cluster_name = var.cluster_name api_servers = [var.k8s_domain_name] diff --git a/bare-metal/fedora-coreos/kubernetes/bootstrap.tf b/bare-metal/fedora-coreos/kubernetes/bootstrap.tf index dad4f2b7d..815ae9040 100644 --- a/bare-metal/fedora-coreos/kubernetes/bootstrap.tf +++ b/bare-metal/fedora-coreos/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=c3b1f23b5daa77c8ff96a2294924aa138a1433a4" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=fc1a7bac89c6c95082334e0f79e478ee321596c0" cluster_name = var.cluster_name api_servers = [var.k8s_domain_name] diff --git a/digital-ocean/container-linux/kubernetes/bootstrap.tf b/digital-ocean/container-linux/kubernetes/bootstrap.tf index 3e0d0efb8..20434da55 100644 --- a/digital-ocean/container-linux/kubernetes/bootstrap.tf +++ b/digital-ocean/container-linux/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=c3b1f23b5daa77c8ff96a2294924aa138a1433a4" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=fc1a7bac89c6c95082334e0f79e478ee321596c0" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/digital-ocean/fedora-coreos/kubernetes/bootstrap.tf b/digital-ocean/fedora-coreos/kubernetes/bootstrap.tf index 8531436dc..14cb127cc 100644 --- a/digital-ocean/fedora-coreos/kubernetes/bootstrap.tf +++ b/digital-ocean/fedora-coreos/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=c3b1f23b5daa77c8ff96a2294924aa138a1433a4" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=fc1a7bac89c6c95082334e0f79e478ee321596c0" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/google-cloud/container-linux/kubernetes/bootstrap.tf b/google-cloud/container-linux/kubernetes/bootstrap.tf index 85456c156..89deafdab 100644 --- a/google-cloud/container-linux/kubernetes/bootstrap.tf +++ b/google-cloud/container-linux/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=c3b1f23b5daa77c8ff96a2294924aa138a1433a4" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=fc1a7bac89c6c95082334e0f79e478ee321596c0" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/google-cloud/fedora-coreos/kubernetes/bootstrap.tf b/google-cloud/fedora-coreos/kubernetes/bootstrap.tf index 4f60b87ee..76af04099 100644 --- a/google-cloud/fedora-coreos/kubernetes/bootstrap.tf +++ b/google-cloud/fedora-coreos/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=c3b1f23b5daa77c8ff96a2294924aa138a1433a4" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=fc1a7bac89c6c95082334e0f79e478ee321596c0" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] From 413585681b45b8861a48158abdd5698dabef76c1 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Thu, 11 Jun 2020 23:55:58 -0700 Subject: [PATCH 447/523] Remove unused Kubelet lock-file and exit-on-lock-contention * Kubelet `--lock-file` and `--exit-on-lock-contention` date back to usage of bootkube and at one point running Kubelet in a "self-hosted" style whereby an on-host Kubelet (rkt) started pods, but then a Kubelet DaemonSet was scheduled and able to take over (hence self-hosted). `lock-file` and `exit-on-lock-contention` flags supported this pivot. The pattern has been out of favor (in bootkube too) for years because of dueling Kubelet complexity * Typhoon runs Kubelet as a container via an on-host systemd unit using podman (Fedora CoreOS) or rkt (Flatcar Linux). In fact, Typhoon no longer uses bootkube or control plane pivot (let alone Kubelet pivot) and uses static pods since v1.16.0 * https://github.com/poseidon/typhoon/pull/536 --- CHANGES.md | 1 + aws/container-linux/kubernetes/cl/controller.yaml | 2 -- aws/container-linux/kubernetes/workers/cl/worker.yaml | 2 -- aws/fedora-coreos/kubernetes/fcc/controller.yaml | 2 -- aws/fedora-coreos/kubernetes/workers/fcc/worker.yaml | 2 -- azure/container-linux/kubernetes/cl/controller.yaml | 2 -- azure/container-linux/kubernetes/workers/cl/worker.yaml | 2 -- azure/fedora-coreos/kubernetes/fcc/controller.yaml | 2 -- azure/fedora-coreos/kubernetes/workers/fcc/worker.yaml | 2 -- bare-metal/container-linux/kubernetes/cl/controller.yaml | 2 -- bare-metal/container-linux/kubernetes/cl/worker.yaml | 2 -- bare-metal/fedora-coreos/kubernetes/fcc/controller.yaml | 2 -- bare-metal/fedora-coreos/kubernetes/fcc/worker.yaml | 2 -- digital-ocean/container-linux/kubernetes/cl/controller.yaml | 2 -- digital-ocean/container-linux/kubernetes/cl/worker.yaml | 2 -- digital-ocean/fedora-coreos/kubernetes/fcc/controller.yaml | 2 -- digital-ocean/fedora-coreos/kubernetes/fcc/worker.yaml | 2 -- google-cloud/container-linux/kubernetes/cl/controller.yaml | 2 -- google-cloud/container-linux/kubernetes/workers/cl/worker.yaml | 2 -- google-cloud/fedora-coreos/kubernetes/fcc/controller.yaml | 2 -- google-cloud/fedora-coreos/kubernetes/workers/fcc/worker.yaml | 2 -- 21 files changed, 1 insertion(+), 40 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index cbdf16cb7..f5d55bedd 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -13,6 +13,7 @@ Notable changes between versions. automated image builds are still available with an alternate tag strategy (see [docs](https://typhoon.psdn.io/topics/security/#container-images)) * Update Calico from v3.14.0 to [v3.14.1](https://docs.projectcalico.org/v3.14/release-notes/) +* Remove unused Kubelet `-lock-file` and `-exit-on-lock-contention` ### Fedora CoreOS diff --git a/aws/container-linux/kubernetes/cl/controller.yaml b/aws/container-linux/kubernetes/cl/controller.yaml index ac7efce42..77774e600 100644 --- a/aws/container-linux/kubernetes/cl/controller.yaml +++ b/aws/container-linux/kubernetes/cl/controller.yaml @@ -102,10 +102,8 @@ systemd: --cluster_dns=${cluster_dns_service_ip} \ --cluster_domain=${cluster_domain_suffix} \ --cni-conf-dir=/etc/kubernetes/cni/net.d \ - --exit-on-lock-contention \ --healthz-port=0 \ --kubeconfig=/var/lib/kubelet/kubeconfig \ - --lock-file=/var/run/lock/kubelet.lock \ --network-plugin=cni \ --node-labels=node.kubernetes.io/master \ --node-labels=node.kubernetes.io/controller="true" \ diff --git a/aws/container-linux/kubernetes/workers/cl/worker.yaml b/aws/container-linux/kubernetes/workers/cl/worker.yaml index 3743b6b67..77bd79d1b 100644 --- a/aws/container-linux/kubernetes/workers/cl/worker.yaml +++ b/aws/container-linux/kubernetes/workers/cl/worker.yaml @@ -75,10 +75,8 @@ systemd: --cluster_dns=${cluster_dns_service_ip} \ --cluster_domain=${cluster_domain_suffix} \ --cni-conf-dir=/etc/kubernetes/cni/net.d \ - --exit-on-lock-contention \ --healthz-port=0 \ --kubeconfig=/var/lib/kubelet/kubeconfig \ - --lock-file=/var/run/lock/kubelet.lock \ --network-plugin=cni \ --node-labels=node.kubernetes.io/node \ %{~ for label in split(",", node_labels) ~} diff --git a/aws/fedora-coreos/kubernetes/fcc/controller.yaml b/aws/fedora-coreos/kubernetes/fcc/controller.yaml index 7bea44b02..116a91625 100644 --- a/aws/fedora-coreos/kubernetes/fcc/controller.yaml +++ b/aws/fedora-coreos/kubernetes/fcc/controller.yaml @@ -92,10 +92,8 @@ systemd: --cluster_dns=${cluster_dns_service_ip} \ --cluster_domain=${cluster_domain_suffix} \ --cni-conf-dir=/etc/kubernetes/cni/net.d \ - --exit-on-lock-contention \ --healthz-port=0 \ --kubeconfig=/var/lib/kubelet/kubeconfig \ - --lock-file=/var/run/lock/kubelet.lock \ --network-plugin=cni \ --node-labels=node.kubernetes.io/master \ --node-labels=node.kubernetes.io/controller="true" \ diff --git a/aws/fedora-coreos/kubernetes/workers/fcc/worker.yaml b/aws/fedora-coreos/kubernetes/workers/fcc/worker.yaml index 062f117a2..ec8687940 100644 --- a/aws/fedora-coreos/kubernetes/workers/fcc/worker.yaml +++ b/aws/fedora-coreos/kubernetes/workers/fcc/worker.yaml @@ -62,10 +62,8 @@ systemd: --cluster_dns=${cluster_dns_service_ip} \ --cluster_domain=${cluster_domain_suffix} \ --cni-conf-dir=/etc/kubernetes/cni/net.d \ - --exit-on-lock-contention \ --healthz-port=0 \ --kubeconfig=/var/lib/kubelet/kubeconfig \ - --lock-file=/var/run/lock/kubelet.lock \ --network-plugin=cni \ --node-labels=node.kubernetes.io/node \ %{~ for label in split(",", node_labels) ~} diff --git a/azure/container-linux/kubernetes/cl/controller.yaml b/azure/container-linux/kubernetes/cl/controller.yaml index babca9b98..0418f8deb 100644 --- a/azure/container-linux/kubernetes/cl/controller.yaml +++ b/azure/container-linux/kubernetes/cl/controller.yaml @@ -100,10 +100,8 @@ systemd: --cluster_dns=${cluster_dns_service_ip} \ --cluster_domain=${cluster_domain_suffix} \ --cni-conf-dir=/etc/kubernetes/cni/net.d \ - --exit-on-lock-contention \ --healthz-port=0 \ --kubeconfig=/var/lib/kubelet/kubeconfig \ - --lock-file=/var/run/lock/kubelet.lock \ --network-plugin=cni \ --node-labels=node.kubernetes.io/master \ --node-labels=node.kubernetes.io/controller="true" \ diff --git a/azure/container-linux/kubernetes/workers/cl/worker.yaml b/azure/container-linux/kubernetes/workers/cl/worker.yaml index 402e21a6f..3ca249625 100644 --- a/azure/container-linux/kubernetes/workers/cl/worker.yaml +++ b/azure/container-linux/kubernetes/workers/cl/worker.yaml @@ -73,10 +73,8 @@ systemd: --cluster_dns=${cluster_dns_service_ip} \ --cluster_domain=${cluster_domain_suffix} \ --cni-conf-dir=/etc/kubernetes/cni/net.d \ - --exit-on-lock-contention \ --healthz-port=0 \ --kubeconfig=/var/lib/kubelet/kubeconfig \ - --lock-file=/var/run/lock/kubelet.lock \ --network-plugin=cni \ --node-labels=node.kubernetes.io/node \ %{~ for label in split(",", node_labels) ~} diff --git a/azure/fedora-coreos/kubernetes/fcc/controller.yaml b/azure/fedora-coreos/kubernetes/fcc/controller.yaml index 4831048c3..47fc9eddb 100644 --- a/azure/fedora-coreos/kubernetes/fcc/controller.yaml +++ b/azure/fedora-coreos/kubernetes/fcc/controller.yaml @@ -92,10 +92,8 @@ systemd: --cluster_dns=${cluster_dns_service_ip} \ --cluster_domain=${cluster_domain_suffix} \ --cni-conf-dir=/etc/kubernetes/cni/net.d \ - --exit-on-lock-contention \ --healthz-port=0 \ --kubeconfig=/var/lib/kubelet/kubeconfig \ - --lock-file=/var/run/lock/kubelet.lock \ --network-plugin=cni \ --node-labels=node.kubernetes.io/master \ --node-labels=node.kubernetes.io/controller="true" \ diff --git a/azure/fedora-coreos/kubernetes/workers/fcc/worker.yaml b/azure/fedora-coreos/kubernetes/workers/fcc/worker.yaml index 815a67a7f..8da2e5079 100644 --- a/azure/fedora-coreos/kubernetes/workers/fcc/worker.yaml +++ b/azure/fedora-coreos/kubernetes/workers/fcc/worker.yaml @@ -62,10 +62,8 @@ systemd: --cluster_dns=${cluster_dns_service_ip} \ --cluster_domain=${cluster_domain_suffix} \ --cni-conf-dir=/etc/kubernetes/cni/net.d \ - --exit-on-lock-contention \ --healthz-port=0 \ --kubeconfig=/var/lib/kubelet/kubeconfig \ - --lock-file=/var/run/lock/kubelet.lock \ --network-plugin=cni \ --node-labels=node.kubernetes.io/node \ %{~ for label in split(",", node_labels) ~} diff --git a/bare-metal/container-linux/kubernetes/cl/controller.yaml b/bare-metal/container-linux/kubernetes/cl/controller.yaml index 399430502..cdf38214b 100644 --- a/bare-metal/container-linux/kubernetes/cl/controller.yaml +++ b/bare-metal/container-linux/kubernetes/cl/controller.yaml @@ -114,11 +114,9 @@ systemd: --cluster_dns=${cluster_dns_service_ip} \ --cluster_domain=${cluster_domain_suffix} \ --cni-conf-dir=/etc/kubernetes/cni/net.d \ - --exit-on-lock-contention \ --healthz-port=0 \ --hostname-override=${domain_name} \ --kubeconfig=/var/lib/kubelet/kubeconfig \ - --lock-file=/var/run/lock/kubelet.lock \ --network-plugin=cni \ --node-labels=node.kubernetes.io/master \ --node-labels=node.kubernetes.io/controller="true" \ diff --git a/bare-metal/container-linux/kubernetes/cl/worker.yaml b/bare-metal/container-linux/kubernetes/cl/worker.yaml index e83bf3898..a06716414 100644 --- a/bare-metal/container-linux/kubernetes/cl/worker.yaml +++ b/bare-metal/container-linux/kubernetes/cl/worker.yaml @@ -87,11 +87,9 @@ systemd: --cluster_dns=${cluster_dns_service_ip} \ --cluster_domain=${cluster_domain_suffix} \ --cni-conf-dir=/etc/kubernetes/cni/net.d \ - --exit-on-lock-contention \ --healthz-port=0 \ --hostname-override=${domain_name} \ --kubeconfig=/var/lib/kubelet/kubeconfig \ - --lock-file=/var/run/lock/kubelet.lock \ --network-plugin=cni \ --node-labels=node.kubernetes.io/node \ %{~ for label in compact(split(",", node_labels)) ~} diff --git a/bare-metal/fedora-coreos/kubernetes/fcc/controller.yaml b/bare-metal/fedora-coreos/kubernetes/fcc/controller.yaml index d2c756eec..63f3acda2 100644 --- a/bare-metal/fedora-coreos/kubernetes/fcc/controller.yaml +++ b/bare-metal/fedora-coreos/kubernetes/fcc/controller.yaml @@ -93,11 +93,9 @@ systemd: --cluster_dns=${cluster_dns_service_ip} \ --cluster_domain=${cluster_domain_suffix} \ --cni-conf-dir=/etc/kubernetes/cni/net.d \ - --exit-on-lock-contention \ --healthz-port=0 \ --hostname-override=${domain_name} \ --kubeconfig=/var/lib/kubelet/kubeconfig \ - --lock-file=/var/run/lock/kubelet.lock \ --network-plugin=cni \ --node-labels=node.kubernetes.io/master \ --node-labels=node.kubernetes.io/controller="true" \ diff --git a/bare-metal/fedora-coreos/kubernetes/fcc/worker.yaml b/bare-metal/fedora-coreos/kubernetes/fcc/worker.yaml index 0c7e114a9..80d3771a4 100644 --- a/bare-metal/fedora-coreos/kubernetes/fcc/worker.yaml +++ b/bare-metal/fedora-coreos/kubernetes/fcc/worker.yaml @@ -63,11 +63,9 @@ systemd: --cluster_dns=${cluster_dns_service_ip} \ --cluster_domain=${cluster_domain_suffix} \ --cni-conf-dir=/etc/kubernetes/cni/net.d \ - --exit-on-lock-contention \ --healthz-port=0 \ --hostname-override=${domain_name} \ --kubeconfig=/var/lib/kubelet/kubeconfig \ - --lock-file=/var/run/lock/kubelet.lock \ --network-plugin=cni \ --node-labels=node.kubernetes.io/node \ %{~ for label in compact(split(",", node_labels)) ~} diff --git a/digital-ocean/container-linux/kubernetes/cl/controller.yaml b/digital-ocean/container-linux/kubernetes/cl/controller.yaml index ef1395122..ca1d725f3 100644 --- a/digital-ocean/container-linux/kubernetes/cl/controller.yaml +++ b/digital-ocean/container-linux/kubernetes/cl/controller.yaml @@ -111,11 +111,9 @@ systemd: --cluster_dns=${cluster_dns_service_ip} \ --cluster_domain=${cluster_domain_suffix} \ --cni-conf-dir=/etc/kubernetes/cni/net.d \ - --exit-on-lock-contention \ --healthz-port=0 \ --hostname-override=$${COREOS_DIGITALOCEAN_IPV4_PRIVATE_0} \ --kubeconfig=/var/lib/kubelet/kubeconfig \ - --lock-file=/var/run/lock/kubelet.lock \ --network-plugin=cni \ --node-labels=node.kubernetes.io/master \ --node-labels=node.kubernetes.io/controller="true" \ diff --git a/digital-ocean/container-linux/kubernetes/cl/worker.yaml b/digital-ocean/container-linux/kubernetes/cl/worker.yaml index 387f0f034..e3dc92f69 100644 --- a/digital-ocean/container-linux/kubernetes/cl/worker.yaml +++ b/digital-ocean/container-linux/kubernetes/cl/worker.yaml @@ -84,11 +84,9 @@ systemd: --cluster_dns=${cluster_dns_service_ip} \ --cluster_domain=${cluster_domain_suffix} \ --cni-conf-dir=/etc/kubernetes/cni/net.d \ - --exit-on-lock-contention \ --healthz-port=0 \ --hostname-override=$${COREOS_DIGITALOCEAN_IPV4_PRIVATE_0} \ --kubeconfig=/var/lib/kubelet/kubeconfig \ - --lock-file=/var/run/lock/kubelet.lock \ --network-plugin=cni \ --node-labels=node.kubernetes.io/node \ --pod-manifest-path=/etc/kubernetes/manifests \ diff --git a/digital-ocean/fedora-coreos/kubernetes/fcc/controller.yaml b/digital-ocean/fedora-coreos/kubernetes/fcc/controller.yaml index d8a9bd92d..9a70c5a8f 100644 --- a/digital-ocean/fedora-coreos/kubernetes/fcc/controller.yaml +++ b/digital-ocean/fedora-coreos/kubernetes/fcc/controller.yaml @@ -94,11 +94,9 @@ systemd: --cluster_dns=${cluster_dns_service_ip} \ --cluster_domain=${cluster_domain_suffix} \ --cni-conf-dir=/etc/kubernetes/cni/net.d \ - --exit-on-lock-contention \ --healthz-port=0 \ --hostname-override=$${AFTERBURN_DIGITALOCEAN_IPV4_PRIVATE_0} \ --kubeconfig=/var/lib/kubelet/kubeconfig \ - --lock-file=/var/run/lock/kubelet.lock \ --network-plugin=cni \ --node-labels=node.kubernetes.io/master \ --node-labels=node.kubernetes.io/controller="true" \ diff --git a/digital-ocean/fedora-coreos/kubernetes/fcc/worker.yaml b/digital-ocean/fedora-coreos/kubernetes/fcc/worker.yaml index 02667f877..f5187ed98 100644 --- a/digital-ocean/fedora-coreos/kubernetes/fcc/worker.yaml +++ b/digital-ocean/fedora-coreos/kubernetes/fcc/worker.yaml @@ -65,11 +65,9 @@ systemd: --cluster_dns=${cluster_dns_service_ip} \ --cluster_domain=${cluster_domain_suffix} \ --cni-conf-dir=/etc/kubernetes/cni/net.d \ - --exit-on-lock-contention \ --healthz-port=0 \ --hostname-override=$${AFTERBURN_DIGITALOCEAN_IPV4_PRIVATE_0} \ --kubeconfig=/var/lib/kubelet/kubeconfig \ - --lock-file=/var/run/lock/kubelet.lock \ --network-plugin=cni \ --node-labels=node.kubernetes.io/node \ --pod-manifest-path=/etc/kubernetes/manifests \ diff --git a/google-cloud/container-linux/kubernetes/cl/controller.yaml b/google-cloud/container-linux/kubernetes/cl/controller.yaml index 28bc25102..2960aa591 100644 --- a/google-cloud/container-linux/kubernetes/cl/controller.yaml +++ b/google-cloud/container-linux/kubernetes/cl/controller.yaml @@ -100,10 +100,8 @@ systemd: --cluster_dns=${cluster_dns_service_ip} \ --cluster_domain=${cluster_domain_suffix} \ --cni-conf-dir=/etc/kubernetes/cni/net.d \ - --exit-on-lock-contention \ --healthz-port=0 \ --kubeconfig=/var/lib/kubelet/kubeconfig \ - --lock-file=/var/run/lock/kubelet.lock \ --network-plugin=cni \ --node-labels=node.kubernetes.io/master \ --node-labels=node.kubernetes.io/controller="true" \ diff --git a/google-cloud/container-linux/kubernetes/workers/cl/worker.yaml b/google-cloud/container-linux/kubernetes/workers/cl/worker.yaml index 048b667e1..a53ab0ae8 100644 --- a/google-cloud/container-linux/kubernetes/workers/cl/worker.yaml +++ b/google-cloud/container-linux/kubernetes/workers/cl/worker.yaml @@ -73,10 +73,8 @@ systemd: --cluster_dns=${cluster_dns_service_ip} \ --cluster_domain=${cluster_domain_suffix} \ --cni-conf-dir=/etc/kubernetes/cni/net.d \ - --exit-on-lock-contention \ --healthz-port=0 \ --kubeconfig=/var/lib/kubelet/kubeconfig \ - --lock-file=/var/run/lock/kubelet.lock \ --network-plugin=cni \ --node-labels=node.kubernetes.io/node \ %{~ for label in split(",", node_labels) ~} diff --git a/google-cloud/fedora-coreos/kubernetes/fcc/controller.yaml b/google-cloud/fedora-coreos/kubernetes/fcc/controller.yaml index 7bea44b02..116a91625 100644 --- a/google-cloud/fedora-coreos/kubernetes/fcc/controller.yaml +++ b/google-cloud/fedora-coreos/kubernetes/fcc/controller.yaml @@ -92,10 +92,8 @@ systemd: --cluster_dns=${cluster_dns_service_ip} \ --cluster_domain=${cluster_domain_suffix} \ --cni-conf-dir=/etc/kubernetes/cni/net.d \ - --exit-on-lock-contention \ --healthz-port=0 \ --kubeconfig=/var/lib/kubelet/kubeconfig \ - --lock-file=/var/run/lock/kubelet.lock \ --network-plugin=cni \ --node-labels=node.kubernetes.io/master \ --node-labels=node.kubernetes.io/controller="true" \ diff --git a/google-cloud/fedora-coreos/kubernetes/workers/fcc/worker.yaml b/google-cloud/fedora-coreos/kubernetes/workers/fcc/worker.yaml index 062f117a2..ec8687940 100644 --- a/google-cloud/fedora-coreos/kubernetes/workers/fcc/worker.yaml +++ b/google-cloud/fedora-coreos/kubernetes/workers/fcc/worker.yaml @@ -62,10 +62,8 @@ systemd: --cluster_dns=${cluster_dns_service_ip} \ --cluster_domain=${cluster_domain_suffix} \ --cni-conf-dir=/etc/kubernetes/cni/net.d \ - --exit-on-lock-contention \ --healthz-port=0 \ --kubeconfig=/var/lib/kubelet/kubeconfig \ - --lock-file=/var/run/lock/kubelet.lock \ --network-plugin=cni \ --node-labels=node.kubernetes.io/node \ %{~ for label in split(",", node_labels) ~} From 04520e447ca7c4714f2eb878feea2890e5798946 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Tue, 16 Jun 2020 17:57:09 -0700 Subject: [PATCH 448/523] Update node-exporter from v1.0.0 to v1.0.1 * https://github.com/prometheus/node_exporter/releases/tag/v1.0.1 --- CHANGES.md | 2 +- addons/prometheus/exporters/node-exporter/daemonset.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index f5d55bedd..40e584cd2 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -29,7 +29,7 @@ Notable changes between versions. ### Addons * Update Prometheus from v2.18.1 to [v2.19.0](https://github.com/prometheus/prometheus/releases/tag/v2.19.0) -* Update node-exporter from v1.0.0-rc.1 to [v1.0.0](https://github.com/prometheus/node_exporter/releases/tag/v1.0.0) +* Update node-exporter from v1.0.0-rc.1 to [v1.0.1](https://github.com/prometheus/node_exporter/releases/tag/v1.0.1) * Update kube-state-metrics from v1.9.6 to v1.9.7 * Update Grafana from v7.0.0 to v7.0.3 diff --git a/addons/prometheus/exporters/node-exporter/daemonset.yaml b/addons/prometheus/exporters/node-exporter/daemonset.yaml index 963dcb6dd..2a30c37be 100644 --- a/addons/prometheus/exporters/node-exporter/daemonset.yaml +++ b/addons/prometheus/exporters/node-exporter/daemonset.yaml @@ -28,7 +28,7 @@ spec: hostPID: true containers: - name: node-exporter - image: quay.io/prometheus/node-exporter:v1.0.0 + image: quay.io/prometheus/node-exporter:v1.0.1 args: - --path.procfs=/host/proc - --path.sysfs=/host/sys From 331566e1f774129fef3b4d200853848dfecf1d61 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Tue, 16 Jun 2020 18:20:19 -0700 Subject: [PATCH 449/523] Update mkdocs packages for website --- requirements.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index 647bd31fa..e9461384d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ mkdocs==1.1.2 -mkdocs-material==5.2.2 -pygments==2.5.2 +mkdocs-material==5.3.0 +pygments==2.6.1 pymdown-extensions==7.1.0 From 4b0203fdb2981bf06f8c0d679d226925940a7f20 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Tue, 16 Jun 2020 18:23:27 -0700 Subject: [PATCH 450/523] Fix typo in DigitalOcean docs title --- CHANGES.md | 8 +++----- docs/fedora-coreos/digitalocean.md | 2 +- docs/flatcar-linux/digitalocean.md | 2 +- mkdocs.yml | 4 ++-- 4 files changed, 7 insertions(+), 9 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 40e584cd2..f684de082 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -8,12 +8,10 @@ Notable changes between versions. * Publish Kubelet images from internal infra to Quay and Dockerhub * [quay.io/poseidon/kubelet](https://quay.io/repository/poseidon/kubelet) (official) * [docker.io/psdn/kubelet](https://hub.docker.com/r/psdn/kubelet) (fallback) - * Document use of alternate Kubelet images during registry incidents - * For those preferring to trust images built by Quay/Dockerhub, - automated image builds are still available with an alternate tag - strategy (see [docs](https://typhoon.psdn.io/topics/security/#container-images)) + * [Document](https://typhoon.psdn.io/advanced/customization/#kubelet) use of alternate Kubelet images during registry incidents + * For those preferring to trust images built by Quay/Dockerhub, automated image builds are still available with an alternate tag strategy (see [docs](https://typhoon.psdn.io/topics/security/#container-images)) * Update Calico from v3.14.0 to [v3.14.1](https://docs.projectcalico.org/v3.14/release-notes/) -* Remove unused Kubelet `-lock-file` and `-exit-on-lock-contention` +* Remove unused Kubelet `-lock-file` and `-exit-on-lock-contention` ([#758](https://github.com/poseidon/typhoon/pull/758)) ### Fedora CoreOS diff --git a/docs/fedora-coreos/digitalocean.md b/docs/fedora-coreos/digitalocean.md index 95de04fb1..5d4d93bb0 100644 --- a/docs/fedora-coreos/digitalocean.md +++ b/docs/fedora-coreos/digitalocean.md @@ -1,4 +1,4 @@ -# Digital Ocean +# DigitalOcean In this tutorial, we'll create a Kubernetes v1.18.3 cluster on DigitalOcean with Fedora CoreOS. diff --git a/docs/flatcar-linux/digitalocean.md b/docs/flatcar-linux/digitalocean.md index ede6a1654..1a8bee296 100644 --- a/docs/flatcar-linux/digitalocean.md +++ b/docs/flatcar-linux/digitalocean.md @@ -1,4 +1,4 @@ -# Digital Ocean +# DigitalOcean In this tutorial, we'll create a Kubernetes v1.18.3 cluster on DigitalOcean with CoreOS Container Linux or Flatcar Linux. diff --git a/mkdocs.yml b/mkdocs.yml index d975b56c7..c5dc1f445 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -59,13 +59,13 @@ nav: - 'AWS': 'fedora-coreos/aws.md' - 'Azure': 'fedora-coreos/azure.md' - 'Bare-Metal': 'fedora-coreos/bare-metal.md' - - 'Digital Ocean': 'fedora-coreos/digitalocean.md' + - 'DigitalOcean': 'fedora-coreos/digitalocean.md' - 'Google Cloud': 'fedora-coreos/google-cloud.md' - 'Flatcar Linux': - 'AWS': 'flatcar-linux/aws.md' - 'Azure': 'flatcar-linux/azure.md' - 'Bare-Metal': 'flatcar-linux/bare-metal.md' - - 'Digital Ocean': 'flatcar-linux/digitalocean.md' + - 'DigitalOcean': 'flatcar-linux/digitalocean.md' - 'Google Cloud': 'flatcar-linux/google-cloud.md' - 'Topics': - 'Maintenance': 'topics/maintenance.md' From bc9b808d446404376a0bbde4bcb726cf7de131b0 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Tue, 16 Jun 2020 18:44:40 -0700 Subject: [PATCH 451/523] Update nginx-ingress from v0.32.0 to v0.33.0 * https://github.com/kubernetes/ingress-nginx/releases/tag/controller-0.33.0 --- CHANGES.md | 1 + addons/nginx-ingress/aws/deployment.yaml | 2 +- addons/nginx-ingress/azure/deployment.yaml | 2 +- addons/nginx-ingress/bare-metal/deployment.yaml | 2 +- addons/nginx-ingress/digital-ocean/daemonset.yaml | 2 +- addons/nginx-ingress/google-cloud/deployment.yaml | 2 +- 6 files changed, 6 insertions(+), 5 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index f684de082..5ec23daf8 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -26,6 +26,7 @@ Notable changes between versions. ### Addons +* Update nginx-ingress from v0.32.0 to [v0.33.0](https://github.com/kubernetes/ingress-nginx/releases/tag/nginx-0.33.0) * Update Prometheus from v2.18.1 to [v2.19.0](https://github.com/prometheus/prometheus/releases/tag/v2.19.0) * Update node-exporter from v1.0.0-rc.1 to [v1.0.1](https://github.com/prometheus/node_exporter/releases/tag/v1.0.1) * Update kube-state-metrics from v1.9.6 to v1.9.7 diff --git a/addons/nginx-ingress/aws/deployment.yaml b/addons/nginx-ingress/aws/deployment.yaml index 3f3b2fe4d..ba2bb4e9a 100644 --- a/addons/nginx-ingress/aws/deployment.yaml +++ b/addons/nginx-ingress/aws/deployment.yaml @@ -22,7 +22,7 @@ spec: spec: containers: - name: nginx-ingress-controller - image: quay.io/kubernetes-ingress-controller/nginx-ingress-controller:0.32.0 + image: quay.io/kubernetes-ingress-controller/nginx-ingress-controller:0.33.0 args: - /nginx-ingress-controller - --ingress-class=public diff --git a/addons/nginx-ingress/azure/deployment.yaml b/addons/nginx-ingress/azure/deployment.yaml index 3f3b2fe4d..ba2bb4e9a 100644 --- a/addons/nginx-ingress/azure/deployment.yaml +++ b/addons/nginx-ingress/azure/deployment.yaml @@ -22,7 +22,7 @@ spec: spec: containers: - name: nginx-ingress-controller - image: quay.io/kubernetes-ingress-controller/nginx-ingress-controller:0.32.0 + image: quay.io/kubernetes-ingress-controller/nginx-ingress-controller:0.33.0 args: - /nginx-ingress-controller - --ingress-class=public diff --git a/addons/nginx-ingress/bare-metal/deployment.yaml b/addons/nginx-ingress/bare-metal/deployment.yaml index 5cabf818a..8dcaa6a93 100644 --- a/addons/nginx-ingress/bare-metal/deployment.yaml +++ b/addons/nginx-ingress/bare-metal/deployment.yaml @@ -22,7 +22,7 @@ spec: spec: containers: - name: nginx-ingress-controller - image: quay.io/kubernetes-ingress-controller/nginx-ingress-controller:0.32.0 + image: quay.io/kubernetes-ingress-controller/nginx-ingress-controller:0.33.0 args: - /nginx-ingress-controller - --ingress-class=public diff --git a/addons/nginx-ingress/digital-ocean/daemonset.yaml b/addons/nginx-ingress/digital-ocean/daemonset.yaml index 4edd51b07..afa5bf6ad 100644 --- a/addons/nginx-ingress/digital-ocean/daemonset.yaml +++ b/addons/nginx-ingress/digital-ocean/daemonset.yaml @@ -22,7 +22,7 @@ spec: spec: containers: - name: nginx-ingress-controller - image: quay.io/kubernetes-ingress-controller/nginx-ingress-controller:0.32.0 + image: quay.io/kubernetes-ingress-controller/nginx-ingress-controller:0.33.0 args: - /nginx-ingress-controller - --ingress-class=public diff --git a/addons/nginx-ingress/google-cloud/deployment.yaml b/addons/nginx-ingress/google-cloud/deployment.yaml index 3f3b2fe4d..ba2bb4e9a 100644 --- a/addons/nginx-ingress/google-cloud/deployment.yaml +++ b/addons/nginx-ingress/google-cloud/deployment.yaml @@ -22,7 +22,7 @@ spec: spec: containers: - name: nginx-ingress-controller - image: quay.io/kubernetes-ingress-controller/nginx-ingress-controller:0.32.0 + image: quay.io/kubernetes-ingress-controller/nginx-ingress-controller:0.33.0 args: - /nginx-ingress-controller - --ingress-class=public From c25c59058cb97892505a37007497544ec0f8a8fb Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Wed, 17 Jun 2020 19:34:50 -0700 Subject: [PATCH 452/523] Update Kubernetes from v1.18.3 to v1.18.4 * https://github.com/kubernetes/kubernetes/blob/master/CHANGELOG/CHANGELOG-1.18.md#v1184 --- CHANGES.md | 9 ++++++--- README.md | 10 +++++----- aws/container-linux/kubernetes/README.md | 2 +- aws/container-linux/kubernetes/bootstrap.tf | 2 +- aws/container-linux/kubernetes/cl/controller.yaml | 4 ++-- .../kubernetes/workers/cl/worker.yaml | 4 ++-- aws/fedora-coreos/kubernetes/README.md | 2 +- aws/fedora-coreos/kubernetes/bootstrap.tf | 2 +- aws/fedora-coreos/kubernetes/fcc/controller.yaml | 4 ++-- .../kubernetes/workers/fcc/worker.yaml | 4 ++-- azure/container-linux/kubernetes/README.md | 2 +- azure/container-linux/kubernetes/bootstrap.tf | 2 +- .../container-linux/kubernetes/cl/controller.yaml | 4 ++-- .../kubernetes/workers/cl/worker.yaml | 4 ++-- azure/fedora-coreos/kubernetes/README.md | 2 +- azure/fedora-coreos/kubernetes/bootstrap.tf | 2 +- azure/fedora-coreos/kubernetes/fcc/controller.yaml | 4 ++-- .../kubernetes/workers/fcc/worker.yaml | 4 ++-- bare-metal/container-linux/kubernetes/README.md | 2 +- bare-metal/container-linux/kubernetes/bootstrap.tf | 2 +- .../container-linux/kubernetes/cl/controller.yaml | 4 ++-- .../container-linux/kubernetes/cl/worker.yaml | 2 +- bare-metal/fedora-coreos/kubernetes/README.md | 2 +- bare-metal/fedora-coreos/kubernetes/bootstrap.tf | 2 +- .../fedora-coreos/kubernetes/fcc/controller.yaml | 4 ++-- .../fedora-coreos/kubernetes/fcc/worker.yaml | 2 +- digital-ocean/container-linux/kubernetes/README.md | 2 +- .../container-linux/kubernetes/bootstrap.tf | 2 +- .../container-linux/kubernetes/cl/controller.yaml | 4 ++-- .../container-linux/kubernetes/cl/worker.yaml | 4 ++-- digital-ocean/fedora-coreos/kubernetes/README.md | 2 +- .../fedora-coreos/kubernetes/bootstrap.tf | 2 +- .../fedora-coreos/kubernetes/fcc/controller.yaml | 4 ++-- .../fedora-coreos/kubernetes/fcc/worker.yaml | 4 ++-- docs/advanced/worker-pools.md | 14 +++++++------- docs/fedora-coreos/aws.md | 10 +++++----- docs/fedora-coreos/azure.md | 10 +++++----- docs/fedora-coreos/bare-metal.md | 10 +++++----- docs/fedora-coreos/digitalocean.md | 10 +++++----- docs/fedora-coreos/google-cloud.md | 8 ++++---- docs/flatcar-linux/aws.md | 10 +++++----- docs/flatcar-linux/azure.md | 10 +++++----- docs/flatcar-linux/bare-metal.md | 10 +++++----- docs/flatcar-linux/digitalocean.md | 10 +++++----- docs/flatcar-linux/google-cloud.md | 10 +++++----- docs/index.md | 10 +++++----- docs/topics/maintenance.md | 4 ++-- docs/topics/security.md | 2 +- google-cloud/container-linux/kubernetes/README.md | 2 +- .../container-linux/kubernetes/bootstrap.tf | 2 +- .../container-linux/kubernetes/cl/controller.yaml | 4 ++-- .../kubernetes/workers/cl/worker.yaml | 4 ++-- google-cloud/fedora-coreos/kubernetes/README.md | 2 +- google-cloud/fedora-coreos/kubernetes/bootstrap.tf | 2 +- .../fedora-coreos/kubernetes/fcc/controller.yaml | 4 ++-- .../kubernetes/workers/fcc/worker.yaml | 4 ++-- 56 files changed, 133 insertions(+), 130 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 5ec23daf8..d1b37b3f7 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -4,13 +4,15 @@ Notable changes between versions. ## Latest -* Update Kubelet image build infra and publishing ([#749](https://github.com/poseidon/typhoon/pull/749)) - * Publish Kubelet images from internal infra to Quay and Dockerhub +* Kubernetes [v1.18.4](https://github.com/kubernetes/kubernetes/blob/master/CHANGELOG/CHANGELOG-1.18.md#v1184) +* Update Kubelet image publishing ([#749](https://github.com/poseidon/typhoon/pull/749)) + * Build Kubelet images internally and publish to Quay and Dockerhub * [quay.io/poseidon/kubelet](https://quay.io/repository/poseidon/kubelet) (official) * [docker.io/psdn/kubelet](https://hub.docker.com/r/psdn/kubelet) (fallback) + * Continue offering automated image builds with an alternate tag strategy (see [docs](https://typhoon.psdn.io/topics/security/#container-images)) * [Document](https://typhoon.psdn.io/advanced/customization/#kubelet) use of alternate Kubelet images during registry incidents - * For those preferring to trust images built by Quay/Dockerhub, automated image builds are still available with an alternate tag strategy (see [docs](https://typhoon.psdn.io/topics/security/#container-images)) * Update Calico from v3.14.0 to [v3.14.1](https://docs.projectcalico.org/v3.14/release-notes/) + * Fix [CVE-2020-13597](https://github.com/kubernetes/kubernetes/issues/91507) * Remove unused Kubelet `-lock-file` and `-exit-on-lock-contention` ([#758](https://github.com/poseidon/typhoon/pull/758)) ### Fedora CoreOS @@ -34,6 +36,7 @@ Notable changes between versions. ## v1.18.3 +* Kubernetes [v1.18.3](https://github.com/kubernetes/kubernetes/blob/master/CHANGELOG/CHANGELOG-1.18.md#v1183) * Use Kubelet [TLS bootstrap](https://kubernetes.io/docs/reference/command-line-tools-reference/kubelet-tls-bootstrapping/) with bootstrap token authentication ([#713](https://github.com/poseidon/typhoon/pull/713)) * Enable Node [Authorization](https://kubernetes.io/docs/reference/access-authn-authz/node/) and [NodeRestriction](https://kubernetes.io/docs/reference/access-authn-authz/admission-controllers/#noderestriction) to reduce authorization scope * Renew Kubelet certificates every 72 hours diff --git a/README.md b/README.md index bd592f4b3..b7d9b0874 100644 --- a/README.md +++ b/README.md @@ -11,7 +11,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features -* Kubernetes v1.18.3 (upstream) +* Kubernetes v1.18.4 (upstream) * Single or multi-master, [Calico](https://www.projectcalico.org/) or [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/), SELinux enforcing * Advanced features like [worker pools](https://typhoon.psdn.io/advanced/worker-pools/), [preemptible](https://typhoon.psdn.io/cl/google-cloud/#preemption) workers, and [snippets](https://typhoon.psdn.io/advanced/customization/#container-linux) customization @@ -54,7 +54,7 @@ Define a Kubernetes cluster by using the Terraform module for your chosen platfo ```tf module "yavin" { - source = "git::https://github.com/poseidon/typhoon//google-cloud/fedora-coreos/kubernetes?ref=v1.18.3" + source = "git::https://github.com/poseidon/typhoon//google-cloud/fedora-coreos/kubernetes?ref=v1.18.4" # Google Cloud cluster_name = "yavin" @@ -93,9 +93,9 @@ In 4-8 minutes (varies by platform), the cluster will be ready. This Google Clou $ export KUBECONFIG=/home/user/.kube/configs/yavin-config $ kubectl get nodes NAME ROLES STATUS AGE VERSION -yavin-controller-0.c.example-com.internal Ready 6m v1.18.3 -yavin-worker-jrbf.c.example-com.internal Ready 5m v1.18.3 -yavin-worker-mzdm.c.example-com.internal Ready 5m v1.18.3 +yavin-controller-0.c.example-com.internal Ready 6m v1.18.4 +yavin-worker-jrbf.c.example-com.internal Ready 5m v1.18.4 +yavin-worker-mzdm.c.example-com.internal Ready 5m v1.18.4 ``` List the pods. diff --git a/aws/container-linux/kubernetes/README.md b/aws/container-linux/kubernetes/README.md index cea72768b..cf480174e 100644 --- a/aws/container-linux/kubernetes/README.md +++ b/aws/container-linux/kubernetes/README.md @@ -11,7 +11,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features -* Kubernetes v1.18.3 (upstream) +* Kubernetes v1.18.4 (upstream) * Single or multi-master, [Calico](https://www.projectcalico.org/) or [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/) * Advanced features like [worker pools](https://typhoon.psdn.io/advanced/worker-pools/), [spot](https://typhoon.psdn.io/cl/aws/#spot) workers, and [snippets](https://typhoon.psdn.io/advanced/customization/#container-linux) customization diff --git a/aws/container-linux/kubernetes/bootstrap.tf b/aws/container-linux/kubernetes/bootstrap.tf index b7944e21a..fe5d14210 100644 --- a/aws/container-linux/kubernetes/bootstrap.tf +++ b/aws/container-linux/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=fc1a7bac89c6c95082334e0f79e478ee321596c0" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=3fe903d0accd71d198415cf46f2f6f53c5c4f699" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/aws/container-linux/kubernetes/cl/controller.yaml b/aws/container-linux/kubernetes/cl/controller.yaml index 77774e600..e0340a89a 100644 --- a/aws/container-linux/kubernetes/cl/controller.yaml +++ b/aws/container-linux/kubernetes/cl/controller.yaml @@ -52,7 +52,7 @@ systemd: Description=Kubelet Wants=rpc-statd.service [Service] - Environment=KUBELET_IMAGE=docker://quay.io/poseidon/kubelet:v1.18.3-4-g57fa88a + Environment=KUBELET_IMAGE=docker://quay.io/poseidon/kubelet:v1.18.4 Environment=KUBELET_CGROUP_DRIVER=${cgroup_driver} ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests @@ -135,7 +135,7 @@ systemd: --volume script,kind=host,source=/opt/bootstrap/apply \ --mount volume=script,target=/apply \ --insecure-options=image \ - docker://quay.io/poseidon/kubelet:v1.18.3-4-g57fa88a \ + docker://quay.io/poseidon/kubelet:v1.18.4 \ --net=host \ --dns=host \ --exec=/apply diff --git a/aws/container-linux/kubernetes/workers/cl/worker.yaml b/aws/container-linux/kubernetes/workers/cl/worker.yaml index 77bd79d1b..d4f5f7a43 100644 --- a/aws/container-linux/kubernetes/workers/cl/worker.yaml +++ b/aws/container-linux/kubernetes/workers/cl/worker.yaml @@ -25,7 +25,7 @@ systemd: Description=Kubelet Wants=rpc-statd.service [Service] - Environment=KUBELET_IMAGE=docker://quay.io/poseidon/kubelet:v1.18.3-4-g57fa88a + Environment=KUBELET_IMAGE=docker://quay.io/poseidon/kubelet:v1.18.4 Environment=KUBELET_CGROUP_DRIVER=${cgroup_driver} ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests @@ -129,7 +129,7 @@ storage: --volume config,kind=host,source=/etc/kubernetes \ --mount volume=config,target=/etc/kubernetes \ --insecure-options=image \ - docker://quay.io/poseidon/kubelet:v1.18.3-4-g57fa88a \ + docker://quay.io/poseidon/kubelet:v1.18.4 \ --net=host \ --dns=host \ --exec=/usr/local/bin/kubectl -- --kubeconfig=/etc/kubernetes/kubeconfig delete node $(hostname) diff --git a/aws/fedora-coreos/kubernetes/README.md b/aws/fedora-coreos/kubernetes/README.md index ab5aad720..ad40e80fa 100644 --- a/aws/fedora-coreos/kubernetes/README.md +++ b/aws/fedora-coreos/kubernetes/README.md @@ -11,7 +11,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features -* Kubernetes v1.18.3 (upstream) +* Kubernetes v1.18.4 (upstream) * Single or multi-master, [Calico](https://www.projectcalico.org/) or [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/), SELinux enforcing * Advanced features like [worker pools](https://typhoon.psdn.io/advanced/worker-pools/), [spot](https://typhoon.psdn.io/cl/aws/#spot) workers, and [snippets](https://typhoon.psdn.io/advanced/customization/#container-linux) customization diff --git a/aws/fedora-coreos/kubernetes/bootstrap.tf b/aws/fedora-coreos/kubernetes/bootstrap.tf index 4df01331d..cd0619658 100644 --- a/aws/fedora-coreos/kubernetes/bootstrap.tf +++ b/aws/fedora-coreos/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=fc1a7bac89c6c95082334e0f79e478ee321596c0" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=3fe903d0accd71d198415cf46f2f6f53c5c4f699" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/aws/fedora-coreos/kubernetes/fcc/controller.yaml b/aws/fedora-coreos/kubernetes/fcc/controller.yaml index 116a91625..c46e244d6 100644 --- a/aws/fedora-coreos/kubernetes/fcc/controller.yaml +++ b/aws/fedora-coreos/kubernetes/fcc/controller.yaml @@ -54,7 +54,7 @@ systemd: Description=Kubelet (System Container) Wants=rpc-statd.service [Service] - Environment=KUBELET_IMAGE=quay.io/poseidon/kubelet:v1.18.3-4-g57fa88a + Environment=KUBELET_IMAGE=quay.io/poseidon/kubelet:v1.18.4 ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests ExecStartPre=/bin/mkdir -p /opt/cni/bin @@ -124,7 +124,7 @@ systemd: --volume /opt/bootstrap/assets:/assets:ro,Z \ --volume /opt/bootstrap/apply:/apply:ro,Z \ --entrypoint=/apply \ - quay.io/poseidon/kubelet:v1.18.3-4-g57fa88a + quay.io/poseidon/kubelet:v1.18.4 ExecStartPost=/bin/touch /opt/bootstrap/bootstrap.done ExecStartPost=-/usr/bin/podman stop bootstrap storage: diff --git a/aws/fedora-coreos/kubernetes/workers/fcc/worker.yaml b/aws/fedora-coreos/kubernetes/workers/fcc/worker.yaml index ec8687940..3febec95a 100644 --- a/aws/fedora-coreos/kubernetes/workers/fcc/worker.yaml +++ b/aws/fedora-coreos/kubernetes/workers/fcc/worker.yaml @@ -24,7 +24,7 @@ systemd: Description=Kubelet (System Container) Wants=rpc-statd.service [Service] - Environment=KUBELET_IMAGE=quay.io/poseidon/kubelet:v1.18.3-4-g57fa88a + Environment=KUBELET_IMAGE=quay.io/poseidon/kubelet:v1.18.4 ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests ExecStartPre=/bin/mkdir -p /opt/cni/bin @@ -88,7 +88,7 @@ systemd: Type=oneshot RemainAfterExit=true ExecStart=/bin/true - ExecStop=/bin/bash -c '/usr/bin/podman run --volume /etc/kubernetes:/etc/kubernetes:ro,z --entrypoint /usr/local/bin/kubectl quay.io/poseidon/kubelet:v1.18.3-4-g57fa88a --kubeconfig=/etc/kubernetes/kubeconfig delete node $HOSTNAME' + ExecStop=/bin/bash -c '/usr/bin/podman run --volume /etc/kubernetes:/etc/kubernetes:ro,z --entrypoint /usr/local/bin/kubectl quay.io/poseidon/kubelet:v1.18.4 --kubeconfig=/etc/kubernetes/kubeconfig delete node $HOSTNAME' [Install] WantedBy=multi-user.target storage: diff --git a/azure/container-linux/kubernetes/README.md b/azure/container-linux/kubernetes/README.md index a6d9d0d3f..bd50f056d 100644 --- a/azure/container-linux/kubernetes/README.md +++ b/azure/container-linux/kubernetes/README.md @@ -11,7 +11,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features -* Kubernetes v1.18.3 (upstream) +* Kubernetes v1.18.4 (upstream) * Single or multi-master, [Calico](https://www.projectcalico.org/) or [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/) * Advanced features like [worker pools](https://typhoon.psdn.io/advanced/worker-pools/), [low-priority](https://typhoon.psdn.io/cl/azure/#low-priority) workers, and [snippets](https://typhoon.psdn.io/advanced/customization/#container-linux) customization diff --git a/azure/container-linux/kubernetes/bootstrap.tf b/azure/container-linux/kubernetes/bootstrap.tf index a829118da..0bfcde357 100644 --- a/azure/container-linux/kubernetes/bootstrap.tf +++ b/azure/container-linux/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=fc1a7bac89c6c95082334e0f79e478ee321596c0" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=3fe903d0accd71d198415cf46f2f6f53c5c4f699" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/azure/container-linux/kubernetes/cl/controller.yaml b/azure/container-linux/kubernetes/cl/controller.yaml index 0418f8deb..6141ce09e 100644 --- a/azure/container-linux/kubernetes/cl/controller.yaml +++ b/azure/container-linux/kubernetes/cl/controller.yaml @@ -52,7 +52,7 @@ systemd: Description=Kubelet Wants=rpc-statd.service [Service] - Environment=KUBELET_IMAGE=docker://quay.io/poseidon/kubelet:v1.18.3-4-g57fa88a + Environment=KUBELET_IMAGE=docker://quay.io/poseidon/kubelet:v1.18.4 ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests ExecStartPre=/bin/mkdir -p /opt/cni/bin @@ -133,7 +133,7 @@ systemd: --volume script,kind=host,source=/opt/bootstrap/apply \ --mount volume=script,target=/apply \ --insecure-options=image \ - docker://quay.io/poseidon/kubelet:v1.18.3-4-g57fa88a \ + docker://quay.io/poseidon/kubelet:v1.18.4 \ --net=host \ --dns=host \ --exec=/apply diff --git a/azure/container-linux/kubernetes/workers/cl/worker.yaml b/azure/container-linux/kubernetes/workers/cl/worker.yaml index 3ca249625..6a9cab44c 100644 --- a/azure/container-linux/kubernetes/workers/cl/worker.yaml +++ b/azure/container-linux/kubernetes/workers/cl/worker.yaml @@ -25,7 +25,7 @@ systemd: Description=Kubelet Wants=rpc-statd.service [Service] - Environment=KUBELET_IMAGE=docker://quay.io/poseidon/kubelet:v1.18.3-4-g57fa88a + Environment=KUBELET_IMAGE=docker://quay.io/poseidon/kubelet:v1.18.4 ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests ExecStartPre=/bin/mkdir -p /opt/cni/bin @@ -127,7 +127,7 @@ storage: --volume config,kind=host,source=/etc/kubernetes \ --mount volume=config,target=/etc/kubernetes \ --insecure-options=image \ - docker://quay.io/poseidon/kubelet:v1.18.3-4-g57fa88a \ + docker://quay.io/poseidon/kubelet:v1.18.4 \ --net=host \ --dns=host \ --exec=/usr/local/bin/kubectl -- --kubeconfig=/etc/kubernetes/kubeconfig delete node $(hostname | tr '[:upper:]' '[:lower:]') diff --git a/azure/fedora-coreos/kubernetes/README.md b/azure/fedora-coreos/kubernetes/README.md index 6ae73dfbf..3d699145f 100644 --- a/azure/fedora-coreos/kubernetes/README.md +++ b/azure/fedora-coreos/kubernetes/README.md @@ -11,7 +11,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features -* Kubernetes v1.18.3 (upstream) +* Kubernetes v1.18.4 (upstream) * Single or multi-master, [Calico](https://www.projectcalico.org/) or [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/), SELinux enforcing * Advanced features like [worker pools](https://typhoon.psdn.io/advanced/worker-pools/), [spot priority](https://typhoon.psdn.io/fedora-coreos/azure/#low-priority) workers, and [snippets](https://typhoon.psdn.io/advanced/customization/) customization diff --git a/azure/fedora-coreos/kubernetes/bootstrap.tf b/azure/fedora-coreos/kubernetes/bootstrap.tf index 3828f06bf..b5fe8b8b0 100644 --- a/azure/fedora-coreos/kubernetes/bootstrap.tf +++ b/azure/fedora-coreos/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=fc1a7bac89c6c95082334e0f79e478ee321596c0" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=3fe903d0accd71d198415cf46f2f6f53c5c4f699" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/azure/fedora-coreos/kubernetes/fcc/controller.yaml b/azure/fedora-coreos/kubernetes/fcc/controller.yaml index 47fc9eddb..d94d644f7 100644 --- a/azure/fedora-coreos/kubernetes/fcc/controller.yaml +++ b/azure/fedora-coreos/kubernetes/fcc/controller.yaml @@ -54,7 +54,7 @@ systemd: Description=Kubelet (System Container) Wants=rpc-statd.service [Service] - Environment=KUBELET_IMAGE=quay.io/poseidon/kubelet:v1.18.3-4-g57fa88a + Environment=KUBELET_IMAGE=quay.io/poseidon/kubelet:v1.18.4 ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests ExecStartPre=/bin/mkdir -p /opt/cni/bin @@ -124,7 +124,7 @@ systemd: --volume /opt/bootstrap/assets:/assets:ro,Z \ --volume /opt/bootstrap/apply:/apply:ro,Z \ --entrypoint=/apply \ - quay.io/poseidon/kubelet:v1.18.3-4-g57fa88a + quay.io/poseidon/kubelet:v1.18.4 ExecStartPost=/bin/touch /opt/bootstrap/bootstrap.done ExecStartPost=-/usr/bin/podman stop bootstrap storage: diff --git a/azure/fedora-coreos/kubernetes/workers/fcc/worker.yaml b/azure/fedora-coreos/kubernetes/workers/fcc/worker.yaml index 8da2e5079..81e0c0bac 100644 --- a/azure/fedora-coreos/kubernetes/workers/fcc/worker.yaml +++ b/azure/fedora-coreos/kubernetes/workers/fcc/worker.yaml @@ -24,7 +24,7 @@ systemd: Description=Kubelet (System Container) Wants=rpc-statd.service [Service] - Environment=KUBELET_IMAGE=quay.io/poseidon/kubelet:v1.18.3-4-g57fa88a + Environment=KUBELET_IMAGE=quay.io/poseidon/kubelet:v1.18.4 ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests ExecStartPre=/bin/mkdir -p /opt/cni/bin @@ -88,7 +88,7 @@ systemd: Type=oneshot RemainAfterExit=true ExecStart=/bin/true - ExecStop=/bin/bash -c '/usr/bin/podman run --volume /etc/kubernetes:/etc/kubernetes:ro,z --entrypoint /usr/local/bin/kubectl quay.io/poseidon/kubelet:v1.18.3-4-g57fa88a --kubeconfig=/etc/kubernetes/kubeconfig delete node $HOSTNAME' + ExecStop=/bin/bash -c '/usr/bin/podman run --volume /etc/kubernetes:/etc/kubernetes:ro,z --entrypoint /usr/local/bin/kubectl quay.io/poseidon/kubelet:v1.18.4 --kubeconfig=/etc/kubernetes/kubeconfig delete node $HOSTNAME' [Install] WantedBy=multi-user.target storage: diff --git a/bare-metal/container-linux/kubernetes/README.md b/bare-metal/container-linux/kubernetes/README.md index 301c08a13..c7bf09f42 100644 --- a/bare-metal/container-linux/kubernetes/README.md +++ b/bare-metal/container-linux/kubernetes/README.md @@ -11,7 +11,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features -* Kubernetes v1.18.3 (upstream) +* Kubernetes v1.18.4 (upstream) * Single or multi-master, [Calico](https://www.projectcalico.org/) or [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/) * Advanced features like [snippets](https://typhoon.psdn.io/advanced/customization/#container-linux) customization diff --git a/bare-metal/container-linux/kubernetes/bootstrap.tf b/bare-metal/container-linux/kubernetes/bootstrap.tf index d1da2e668..58e9933c4 100644 --- a/bare-metal/container-linux/kubernetes/bootstrap.tf +++ b/bare-metal/container-linux/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=fc1a7bac89c6c95082334e0f79e478ee321596c0" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=3fe903d0accd71d198415cf46f2f6f53c5c4f699" cluster_name = var.cluster_name api_servers = [var.k8s_domain_name] diff --git a/bare-metal/container-linux/kubernetes/cl/controller.yaml b/bare-metal/container-linux/kubernetes/cl/controller.yaml index cdf38214b..398845538 100644 --- a/bare-metal/container-linux/kubernetes/cl/controller.yaml +++ b/bare-metal/container-linux/kubernetes/cl/controller.yaml @@ -60,7 +60,7 @@ systemd: Description=Kubelet Wants=rpc-statd.service [Service] - Environment=KUBELET_IMAGE=docker://quay.io/poseidon/kubelet:v1.18.3-4-g57fa88a + Environment=KUBELET_IMAGE=docker://quay.io/poseidon/kubelet:v1.18.4 Environment=KUBELET_CGROUP_DRIVER=${cgroup_driver} ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests @@ -148,7 +148,7 @@ systemd: --volume script,kind=host,source=/opt/bootstrap/apply \ --mount volume=script,target=/apply \ --insecure-options=image \ - docker://quay.io/poseidon/kubelet:v1.18.3-4-g57fa88a \ + docker://quay.io/poseidon/kubelet:v1.18.4 \ --net=host \ --dns=host \ --exec=/apply diff --git a/bare-metal/container-linux/kubernetes/cl/worker.yaml b/bare-metal/container-linux/kubernetes/cl/worker.yaml index a06716414..7396e1c8a 100644 --- a/bare-metal/container-linux/kubernetes/cl/worker.yaml +++ b/bare-metal/container-linux/kubernetes/cl/worker.yaml @@ -33,7 +33,7 @@ systemd: Description=Kubelet Wants=rpc-statd.service [Service] - Environment=KUBELET_IMAGE=docker://quay.io/poseidon/kubelet:v1.18.3-4-g57fa88a + Environment=KUBELET_IMAGE=docker://quay.io/poseidon/kubelet:v1.18.4 Environment=KUBELET_CGROUP_DRIVER=${cgroup_driver} ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests diff --git a/bare-metal/fedora-coreos/kubernetes/README.md b/bare-metal/fedora-coreos/kubernetes/README.md index 427c38b2f..2d79cb83a 100644 --- a/bare-metal/fedora-coreos/kubernetes/README.md +++ b/bare-metal/fedora-coreos/kubernetes/README.md @@ -11,7 +11,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features -* Kubernetes v1.18.3 (upstream) +* Kubernetes v1.18.4 (upstream) * Single or multi-master, [Calico](https://www.projectcalico.org/) or [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/), SELinux enforcing * Advanced features like [snippets](https://typhoon.psdn.io/advanced/customization/#container-linux) customization diff --git a/bare-metal/fedora-coreos/kubernetes/bootstrap.tf b/bare-metal/fedora-coreos/kubernetes/bootstrap.tf index 815ae9040..0095ae83e 100644 --- a/bare-metal/fedora-coreos/kubernetes/bootstrap.tf +++ b/bare-metal/fedora-coreos/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=fc1a7bac89c6c95082334e0f79e478ee321596c0" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=3fe903d0accd71d198415cf46f2f6f53c5c4f699" cluster_name = var.cluster_name api_servers = [var.k8s_domain_name] diff --git a/bare-metal/fedora-coreos/kubernetes/fcc/controller.yaml b/bare-metal/fedora-coreos/kubernetes/fcc/controller.yaml index 63f3acda2..6623a8c6e 100644 --- a/bare-metal/fedora-coreos/kubernetes/fcc/controller.yaml +++ b/bare-metal/fedora-coreos/kubernetes/fcc/controller.yaml @@ -53,7 +53,7 @@ systemd: Description=Kubelet (System Container) Wants=rpc-statd.service [Service] - Environment=KUBELET_IMAGE=quay.io/poseidon/kubelet:v1.18.3-4-g57fa88a + Environment=KUBELET_IMAGE=quay.io/poseidon/kubelet:v1.18.4 ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests ExecStartPre=/bin/mkdir -p /opt/cni/bin @@ -135,7 +135,7 @@ systemd: --volume /opt/bootstrap/assets:/assets:ro,Z \ --volume /opt/bootstrap/apply:/apply:ro,Z \ --entrypoint=/apply \ - quay.io/poseidon/kubelet:v1.18.3-4-g57fa88a + quay.io/poseidon/kubelet:v1.18.4 ExecStartPost=/bin/touch /opt/bootstrap/bootstrap.done ExecStartPost=-/usr/bin/podman stop bootstrap storage: diff --git a/bare-metal/fedora-coreos/kubernetes/fcc/worker.yaml b/bare-metal/fedora-coreos/kubernetes/fcc/worker.yaml index 80d3771a4..26d2cd94e 100644 --- a/bare-metal/fedora-coreos/kubernetes/fcc/worker.yaml +++ b/bare-metal/fedora-coreos/kubernetes/fcc/worker.yaml @@ -23,7 +23,7 @@ systemd: Description=Kubelet (System Container) Wants=rpc-statd.service [Service] - Environment=KUBELET_IMAGE=quay.io/poseidon/kubelet:v1.18.3-4-g57fa88a + Environment=KUBELET_IMAGE=quay.io/poseidon/kubelet:v1.18.4 ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests ExecStartPre=/bin/mkdir -p /opt/cni/bin diff --git a/digital-ocean/container-linux/kubernetes/README.md b/digital-ocean/container-linux/kubernetes/README.md index 2289778be..622a9ccfe 100644 --- a/digital-ocean/container-linux/kubernetes/README.md +++ b/digital-ocean/container-linux/kubernetes/README.md @@ -11,7 +11,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features -* Kubernetes v1.18.3 (upstream) +* Kubernetes v1.18.4 (upstream) * Single or multi-master, [Calico](https://www.projectcalico.org/) or [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/) * Advanced features like [snippets](https://typhoon.psdn.io/advanced/customization/#container-linux) customization diff --git a/digital-ocean/container-linux/kubernetes/bootstrap.tf b/digital-ocean/container-linux/kubernetes/bootstrap.tf index 20434da55..aca9cc89f 100644 --- a/digital-ocean/container-linux/kubernetes/bootstrap.tf +++ b/digital-ocean/container-linux/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=fc1a7bac89c6c95082334e0f79e478ee321596c0" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=3fe903d0accd71d198415cf46f2f6f53c5c4f699" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/digital-ocean/container-linux/kubernetes/cl/controller.yaml b/digital-ocean/container-linux/kubernetes/cl/controller.yaml index ca1d725f3..de45a7892 100644 --- a/digital-ocean/container-linux/kubernetes/cl/controller.yaml +++ b/digital-ocean/container-linux/kubernetes/cl/controller.yaml @@ -62,7 +62,7 @@ systemd: After=coreos-metadata.service Wants=rpc-statd.service [Service] - Environment=KUBELET_IMAGE=docker://quay.io/poseidon/kubelet:v1.18.3-4-g57fa88a + Environment=KUBELET_IMAGE=docker://quay.io/poseidon/kubelet:v1.18.4 EnvironmentFile=/run/metadata/coreos ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests @@ -145,7 +145,7 @@ systemd: --volume script,kind=host,source=/opt/bootstrap/apply \ --mount volume=script,target=/apply \ --insecure-options=image \ - docker://quay.io/poseidon/kubelet:v1.18.3-4-g57fa88a \ + docker://quay.io/poseidon/kubelet:v1.18.4 \ --net=host \ --dns=host \ --exec=/apply diff --git a/digital-ocean/container-linux/kubernetes/cl/worker.yaml b/digital-ocean/container-linux/kubernetes/cl/worker.yaml index e3dc92f69..05c4ded3b 100644 --- a/digital-ocean/container-linux/kubernetes/cl/worker.yaml +++ b/digital-ocean/container-linux/kubernetes/cl/worker.yaml @@ -35,7 +35,7 @@ systemd: After=coreos-metadata.service Wants=rpc-statd.service [Service] - Environment=KUBELET_IMAGE=docker://quay.io/poseidon/kubelet:v1.18.3-4-g57fa88a + Environment=KUBELET_IMAGE=docker://quay.io/poseidon/kubelet:v1.18.4 EnvironmentFile=/run/metadata/coreos ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests @@ -134,7 +134,7 @@ storage: --volume config,kind=host,source=/etc/kubernetes \ --mount volume=config,target=/etc/kubernetes \ --insecure-options=image \ - docker://quay.io/poseidon/kubelet:v1.18.3-4-g57fa88a \ + docker://quay.io/poseidon/kubelet:v1.18.4 \ --net=host \ --dns=host \ --exec=/usr/local/bin/kubectl -- --kubeconfig=/etc/kubernetes/kubeconfig delete node $(hostname) diff --git a/digital-ocean/fedora-coreos/kubernetes/README.md b/digital-ocean/fedora-coreos/kubernetes/README.md index e1f7bb901..44ef73523 100644 --- a/digital-ocean/fedora-coreos/kubernetes/README.md +++ b/digital-ocean/fedora-coreos/kubernetes/README.md @@ -11,7 +11,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features -* Kubernetes v1.18.3 (upstream) +* Kubernetes v1.18.4 (upstream) * Single or multi-master, [Calico](https://www.projectcalico.org/) or [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/), SELinux enforcing * Advanced features like [snippets](https://typhoon.psdn.io/advanced/customization/) customization diff --git a/digital-ocean/fedora-coreos/kubernetes/bootstrap.tf b/digital-ocean/fedora-coreos/kubernetes/bootstrap.tf index 14cb127cc..81dabda5a 100644 --- a/digital-ocean/fedora-coreos/kubernetes/bootstrap.tf +++ b/digital-ocean/fedora-coreos/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=fc1a7bac89c6c95082334e0f79e478ee321596c0" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=3fe903d0accd71d198415cf46f2f6f53c5c4f699" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/digital-ocean/fedora-coreos/kubernetes/fcc/controller.yaml b/digital-ocean/fedora-coreos/kubernetes/fcc/controller.yaml index 9a70c5a8f..878837e6f 100644 --- a/digital-ocean/fedora-coreos/kubernetes/fcc/controller.yaml +++ b/digital-ocean/fedora-coreos/kubernetes/fcc/controller.yaml @@ -55,7 +55,7 @@ systemd: After=afterburn.service Wants=rpc-statd.service [Service] - Environment=KUBELET_IMAGE=quay.io/poseidon/kubelet:v1.18.3-4-g57fa88a + Environment=KUBELET_IMAGE=quay.io/poseidon/kubelet:v1.18.4 EnvironmentFile=/run/metadata/afterburn ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests @@ -136,7 +136,7 @@ systemd: --volume /opt/bootstrap/assets:/assets:ro,Z \ --volume /opt/bootstrap/apply:/apply:ro,Z \ --entrypoint=/apply \ - quay.io/poseidon/kubelet:v1.18.3-4-g57fa88a + quay.io/poseidon/kubelet:v1.18.4 ExecStartPost=/bin/touch /opt/bootstrap/bootstrap.done ExecStartPost=-/usr/bin/podman stop bootstrap storage: diff --git a/digital-ocean/fedora-coreos/kubernetes/fcc/worker.yaml b/digital-ocean/fedora-coreos/kubernetes/fcc/worker.yaml index f5187ed98..44be2761e 100644 --- a/digital-ocean/fedora-coreos/kubernetes/fcc/worker.yaml +++ b/digital-ocean/fedora-coreos/kubernetes/fcc/worker.yaml @@ -26,7 +26,7 @@ systemd: After=afterburn.service Wants=rpc-statd.service [Service] - Environment=KUBELET_IMAGE=quay.io/poseidon/kubelet:v1.18.3-4-g57fa88a + Environment=KUBELET_IMAGE=quay.io/poseidon/kubelet:v1.18.4 EnvironmentFile=/run/metadata/afterburn ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests @@ -98,7 +98,7 @@ systemd: Type=oneshot RemainAfterExit=true ExecStart=/bin/true - ExecStop=/bin/bash -c '/usr/bin/podman run --volume /etc/kubernetes:/etc/kubernetes:ro,z --entrypoint /usr/local/bin/kubectl quay.io/poseidon/kubelet:v1.18.3-4-g57fa88a --kubeconfig=/etc/kubernetes/kubeconfig delete node $HOSTNAME' + ExecStop=/bin/bash -c '/usr/bin/podman run --volume /etc/kubernetes:/etc/kubernetes:ro,z --entrypoint /usr/local/bin/kubectl quay.io/poseidon/kubelet:v1.18.4 --kubeconfig=/etc/kubernetes/kubeconfig delete node $HOSTNAME' [Install] WantedBy=multi-user.target storage: diff --git a/docs/advanced/worker-pools.md b/docs/advanced/worker-pools.md index bd0046c2d..a2f368617 100644 --- a/docs/advanced/worker-pools.md +++ b/docs/advanced/worker-pools.md @@ -82,7 +82,7 @@ Create a cluster following the Azure [tutorial](../flatcar-linux/azure.md#cluste ```tf module "ramius-worker-pool" { - source = "git::https://github.com/poseidon/typhoon//azure/container-linux/kubernetes/workers?ref=v1.18.3" + source = "git::https://github.com/poseidon/typhoon//azure/container-linux/kubernetes/workers?ref=v1.18.4" # Azure region = module.ramius.region @@ -148,7 +148,7 @@ Create a cluster following the Google Cloud [tutorial](../flatcar-linux/google-c ```tf module "yavin-worker-pool" { - source = "git::https://github.com/poseidon/typhoon//google-cloud/container-linux/kubernetes/workers?ref=v1.18.3" + source = "git::https://github.com/poseidon/typhoon//google-cloud/container-linux/kubernetes/workers?ref=v1.18.4" # Google Cloud region = "europe-west2" @@ -179,11 +179,11 @@ Verify a managed instance group of workers joins the cluster within a few minute ``` $ kubectl get nodes NAME STATUS AGE VERSION -yavin-controller-0.c.example-com.internal Ready 6m v1.18.3 -yavin-worker-jrbf.c.example-com.internal Ready 5m v1.18.3 -yavin-worker-mzdm.c.example-com.internal Ready 5m v1.18.3 -yavin-16x-worker-jrbf.c.example-com.internal Ready 3m v1.18.3 -yavin-16x-worker-mzdm.c.example-com.internal Ready 3m v1.18.3 +yavin-controller-0.c.example-com.internal Ready 6m v1.18.4 +yavin-worker-jrbf.c.example-com.internal Ready 5m v1.18.4 +yavin-worker-mzdm.c.example-com.internal Ready 5m v1.18.4 +yavin-16x-worker-jrbf.c.example-com.internal Ready 3m v1.18.4 +yavin-16x-worker-mzdm.c.example-com.internal Ready 3m v1.18.4 ``` ### Variables diff --git a/docs/fedora-coreos/aws.md b/docs/fedora-coreos/aws.md index 57e8724f8..3911a5760 100644 --- a/docs/fedora-coreos/aws.md +++ b/docs/fedora-coreos/aws.md @@ -1,6 +1,6 @@ # AWS -In this tutorial, we'll create a Kubernetes v1.18.3 cluster on AWS with Fedora CoreOS. +In this tutorial, we'll create a Kubernetes v1.18.4 cluster on AWS with Fedora CoreOS. We'll declare a Kubernetes cluster using the Typhoon Terraform module. Then apply the changes to create a VPC, gateway, subnets, security groups, controller instances, worker auto-scaling group, network load balancer, and TLS assets. @@ -70,7 +70,7 @@ Define a Kubernetes cluster using the module `aws/fedora-coreos/kubernetes`. ```tf module "tempest" { - source = "git::https://github.com/poseidon/typhoon//aws/fedora-coreos/kubernetes?ref=v1.18.3" + source = "git::https://github.com/poseidon/typhoon//aws/fedora-coreos/kubernetes?ref=v1.18.4" # AWS cluster_name = "tempest" @@ -143,9 +143,9 @@ List nodes in the cluster. $ export KUBECONFIG=/home/user/.kube/configs/tempest-config $ kubectl get nodes NAME STATUS ROLES AGE VERSION -ip-10-0-3-155 Ready 10m v1.18.3 -ip-10-0-26-65 Ready 10m v1.18.3 -ip-10-0-41-21 Ready 10m v1.18.3 +ip-10-0-3-155 Ready 10m v1.18.4 +ip-10-0-26-65 Ready 10m v1.18.4 +ip-10-0-41-21 Ready 10m v1.18.4 ``` List the pods. diff --git a/docs/fedora-coreos/azure.md b/docs/fedora-coreos/azure.md index 45ce5c1c8..879038c8d 100644 --- a/docs/fedora-coreos/azure.md +++ b/docs/fedora-coreos/azure.md @@ -1,6 +1,6 @@ # Azure -In this tutorial, we'll create a Kubernetes v1.18.3 cluster on Azure with Fedora CoreOS. +In this tutorial, we'll create a Kubernetes v1.18.4 cluster on Azure with Fedora CoreOS. We'll declare a Kubernetes cluster using the Typhoon Terraform module. Then apply the changes to create a resource group, virtual network, subnets, security groups, controller availability set, worker scale set, load balancer, and TLS assets. @@ -83,7 +83,7 @@ Define a Kubernetes cluster using the module `azure/fedora-coreos/kubernetes`. ```tf module "ramius" { - source = "git::https://github.com/poseidon/typhoon//azure/fedora-coreos/kubernetes?ref=v1.18.3" + source = "git::https://github.com/poseidon/typhoon//azure/fedora-coreos/kubernetes?ref=v1.18.4" # Azure cluster_name = "ramius" @@ -158,9 +158,9 @@ List nodes in the cluster. $ export KUBECONFIG=/home/user/.kube/configs/ramius-config $ kubectl get nodes NAME STATUS ROLES AGE VERSION -ramius-controller-0 Ready 24m v1.18.3 -ramius-worker-000001 Ready 25m v1.18.3 -ramius-worker-000002 Ready 24m v1.18.3 +ramius-controller-0 Ready 24m v1.18.4 +ramius-worker-000001 Ready 25m v1.18.4 +ramius-worker-000002 Ready 24m v1.18.4 ``` List the pods. diff --git a/docs/fedora-coreos/bare-metal.md b/docs/fedora-coreos/bare-metal.md index 0a50e13ba..cbab8a111 100644 --- a/docs/fedora-coreos/bare-metal.md +++ b/docs/fedora-coreos/bare-metal.md @@ -1,6 +1,6 @@ # Bare-Metal -In this tutorial, we'll network boot and provision a Kubernetes v1.18.3 cluster on bare-metal with Fedora CoreOS. +In this tutorial, we'll network boot and provision a Kubernetes v1.18.4 cluster on bare-metal with Fedora CoreOS. First, we'll deploy a [Matchbox](https://github.com/poseidon/matchbox) service and setup a network boot environment. Then, we'll declare a Kubernetes cluster using the Typhoon Terraform module and power on machines. On PXE boot, machines will install Fedora CoreOS to disk, reboot into the disk install, and provision themselves as Kubernetes controllers or workers via Ignition. @@ -160,7 +160,7 @@ Define a Kubernetes cluster using the module `bare-metal/fedora-coreos/kubernete ```tf module "mercury" { - source = "git::https://github.com/poseidon/typhoon//bare-metal/fedora-coreos/kubernetes?ref=v1.18.3" + source = "git::https://github.com/poseidon/typhoon//bare-metal/fedora-coreos/kubernetes?ref=v1.18.4" # bare-metal cluster_name = "mercury" @@ -289,9 +289,9 @@ List nodes in the cluster. $ export KUBECONFIG=/home/user/.kube/configs/mercury-config $ kubectl get nodes NAME STATUS ROLES AGE VERSION -node1.example.com Ready 10m v1.18.3 -node2.example.com Ready 10m v1.18.3 -node3.example.com Ready 10m v1.18.3 +node1.example.com Ready 10m v1.18.4 +node2.example.com Ready 10m v1.18.4 +node3.example.com Ready 10m v1.18.4 ``` List the pods. diff --git a/docs/fedora-coreos/digitalocean.md b/docs/fedora-coreos/digitalocean.md index 5d4d93bb0..e78e904a0 100644 --- a/docs/fedora-coreos/digitalocean.md +++ b/docs/fedora-coreos/digitalocean.md @@ -1,6 +1,6 @@ # DigitalOcean -In this tutorial, we'll create a Kubernetes v1.18.3 cluster on DigitalOcean with Fedora CoreOS. +In this tutorial, we'll create a Kubernetes v1.18.4 cluster on DigitalOcean with Fedora CoreOS. We'll declare a Kubernetes cluster using the Typhoon Terraform module. Then apply the changes to create controller droplets, worker droplets, DNS records, tags, and TLS assets. @@ -79,7 +79,7 @@ Define a Kubernetes cluster using the module `digital-ocean/fedora-coreos/kubern ```tf module "nemo" { - source = "git::https://github.com/poseidon/typhoon//digital-ocean/fedora-coreos/kubernetes?ref=v1.18.3" + source = "git::https://github.com/poseidon/typhoon//digital-ocean/fedora-coreos/kubernetes?ref=v1.18.4" # Digital Ocean cluster_name = "nemo" @@ -153,9 +153,9 @@ List nodes in the cluster. $ export KUBECONFIG=/home/user/.kube/configs/nemo-config $ kubectl get nodes NAME STATUS ROLES AGE VERSION -10.132.110.130 Ready 10m v1.18.3 -10.132.115.81 Ready 10m v1.18.3 -10.132.124.107 Ready 10m v1.18.3 +10.132.110.130 Ready 10m v1.18.4 +10.132.115.81 Ready 10m v1.18.4 +10.132.124.107 Ready 10m v1.18.4 ``` List the pods. diff --git a/docs/fedora-coreos/google-cloud.md b/docs/fedora-coreos/google-cloud.md index 6f883dea6..4ad86de9a 100644 --- a/docs/fedora-coreos/google-cloud.md +++ b/docs/fedora-coreos/google-cloud.md @@ -1,6 +1,6 @@ # Google Cloud -In this tutorial, we'll create a Kubernetes v1.18.3 cluster on Google Compute Engine with Fedora CoreOS. +In this tutorial, we'll create a Kubernetes v1.18.4 cluster on Google Compute Engine with Fedora CoreOS. We'll declare a Kubernetes cluster using the Typhoon Terraform module. Then apply the changes to create a network, firewall rules, health checks, controller instances, worker managed instance group, load balancers, and TLS assets. @@ -145,9 +145,9 @@ List nodes in the cluster. $ export KUBECONFIG=/home/user/.kube/configs/yavin-config $ kubectl get nodes NAME ROLES STATUS AGE VERSION -yavin-controller-0.c.example-com.internal Ready 6m v1.18.3 -yavin-worker-jrbf.c.example-com.internal Ready 5m v1.18.3 -yavin-worker-mzdm.c.example-com.internal Ready 5m v1.18.3 +yavin-controller-0.c.example-com.internal Ready 6m v1.18.4 +yavin-worker-jrbf.c.example-com.internal Ready 5m v1.18.4 +yavin-worker-mzdm.c.example-com.internal Ready 5m v1.18.4 ``` List the pods. diff --git a/docs/flatcar-linux/aws.md b/docs/flatcar-linux/aws.md index 1f15aa699..6264df1ae 100644 --- a/docs/flatcar-linux/aws.md +++ b/docs/flatcar-linux/aws.md @@ -1,6 +1,6 @@ # AWS -In this tutorial, we'll create a Kubernetes v1.18.3 cluster on AWS with CoreOS Container Linux or Flatcar Linux. +In this tutorial, we'll create a Kubernetes v1.18.4 cluster on AWS with CoreOS Container Linux or Flatcar Linux. We'll declare a Kubernetes cluster using the Typhoon Terraform module. Then apply the changes to create a VPC, gateway, subnets, security groups, controller instances, worker auto-scaling group, network load balancer, and TLS assets. @@ -70,7 +70,7 @@ Define a Kubernetes cluster using the module `aws/container-linux/kubernetes`. ```tf module "tempest" { - source = "git::https://github.com/poseidon/typhoon//aws/container-linux/kubernetes?ref=v1.18.3" + source = "git::https://github.com/poseidon/typhoon//aws/container-linux/kubernetes?ref=v1.18.4" # AWS cluster_name = "tempest" @@ -143,9 +143,9 @@ List nodes in the cluster. $ export KUBECONFIG=/home/user/.kube/configs/tempest-config $ kubectl get nodes NAME STATUS ROLES AGE VERSION -ip-10-0-3-155 Ready 10m v1.18.3 -ip-10-0-26-65 Ready 10m v1.18.3 -ip-10-0-41-21 Ready 10m v1.18.3 +ip-10-0-3-155 Ready 10m v1.18.4 +ip-10-0-26-65 Ready 10m v1.18.4 +ip-10-0-41-21 Ready 10m v1.18.4 ``` List the pods. diff --git a/docs/flatcar-linux/azure.md b/docs/flatcar-linux/azure.md index 2c9a08932..1b816dc12 100644 --- a/docs/flatcar-linux/azure.md +++ b/docs/flatcar-linux/azure.md @@ -1,6 +1,6 @@ # Azure -In this tutorial, we'll create a Kubernetes v1.18.3 cluster on Azure with CoreOS Container Linux or Flatcar Linux. +In this tutorial, we'll create a Kubernetes v1.18.4 cluster on Azure with CoreOS Container Linux or Flatcar Linux. We'll declare a Kubernetes cluster using the Typhoon Terraform module. Then apply the changes to create a resource group, virtual network, subnets, security groups, controller availability set, worker scale set, load balancer, and TLS assets. @@ -72,7 +72,7 @@ Define a Kubernetes cluster using the module `azure/container-linux/kubernetes`. ```tf module "ramius" { - source = "git::https://github.com/poseidon/typhoon//azure/container-linux/kubernetes?ref=v1.18.3" + source = "git::https://github.com/poseidon/typhoon//azure/container-linux/kubernetes?ref=v1.18.4" # Azure cluster_name = "ramius" @@ -146,9 +146,9 @@ List nodes in the cluster. $ export KUBECONFIG=/home/user/.kube/configs/ramius-config $ kubectl get nodes NAME STATUS ROLES AGE VERSION -ramius-controller-0 Ready 24m v1.18.3 -ramius-worker-000001 Ready 25m v1.18.3 -ramius-worker-000002 Ready 24m v1.18.3 +ramius-controller-0 Ready 24m v1.18.4 +ramius-worker-000001 Ready 25m v1.18.4 +ramius-worker-000002 Ready 24m v1.18.4 ``` List the pods. diff --git a/docs/flatcar-linux/bare-metal.md b/docs/flatcar-linux/bare-metal.md index f580c25e1..024826e92 100644 --- a/docs/flatcar-linux/bare-metal.md +++ b/docs/flatcar-linux/bare-metal.md @@ -1,6 +1,6 @@ # Bare-Metal -In this tutorial, we'll network boot and provision a Kubernetes v1.18.3 cluster on bare-metal with CoreOS Container Linux or Flatcar Linux. +In this tutorial, we'll network boot and provision a Kubernetes v1.18.4 cluster on bare-metal with CoreOS Container Linux or Flatcar Linux. First, we'll deploy a [Matchbox](https://github.com/poseidon/matchbox) service and setup a network boot environment. Then, we'll declare a Kubernetes cluster using the Typhoon Terraform module and power on machines. On PXE boot, machines will install Container Linux to disk, reboot into the disk install, and provision themselves as Kubernetes controllers or workers via Ignition. @@ -160,7 +160,7 @@ Define a Kubernetes cluster using the module `bare-metal/container-linux/kuberne ```tf module "mercury" { - source = "git::https://github.com/poseidon/typhoon//bare-metal/container-linux/kubernetes?ref=v1.18.3" + source = "git::https://github.com/poseidon/typhoon//bare-metal/container-linux/kubernetes?ref=v1.18.4" # bare-metal cluster_name = "mercury" @@ -299,9 +299,9 @@ List nodes in the cluster. $ export KUBECONFIG=/home/user/.kube/configs/mercury-config $ kubectl get nodes NAME STATUS ROLES AGE VERSION -node1.example.com Ready 10m v1.18.3 -node2.example.com Ready 10m v1.18.3 -node3.example.com Ready 10m v1.18.3 +node1.example.com Ready 10m v1.18.4 +node2.example.com Ready 10m v1.18.4 +node3.example.com Ready 10m v1.18.4 ``` List the pods. diff --git a/docs/flatcar-linux/digitalocean.md b/docs/flatcar-linux/digitalocean.md index 1a8bee296..73a00340d 100644 --- a/docs/flatcar-linux/digitalocean.md +++ b/docs/flatcar-linux/digitalocean.md @@ -1,6 +1,6 @@ # DigitalOcean -In this tutorial, we'll create a Kubernetes v1.18.3 cluster on DigitalOcean with CoreOS Container Linux or Flatcar Linux. +In this tutorial, we'll create a Kubernetes v1.18.4 cluster on DigitalOcean with CoreOS Container Linux or Flatcar Linux. We'll declare a Kubernetes cluster using the Typhoon Terraform module. Then apply the changes to create controller droplets, worker droplets, DNS records, tags, and TLS assets. @@ -79,7 +79,7 @@ Define a Kubernetes cluster using the module `digital-ocean/container-linux/kube ```tf module "nemo" { - source = "git::https://github.com/poseidon/typhoon//digital-ocean/container-linux/kubernetes?ref=v1.18.3" + source = "git::https://github.com/poseidon/typhoon//digital-ocean/container-linux/kubernetes?ref=v1.18.4" # Digital Ocean cluster_name = "nemo" @@ -153,9 +153,9 @@ List nodes in the cluster. $ export KUBECONFIG=/home/user/.kube/configs/nemo-config $ kubectl get nodes NAME STATUS ROLES AGE VERSION -10.132.110.130 Ready 10m v1.18.3 -10.132.115.81 Ready 10m v1.18.3 -10.132.124.107 Ready 10m v1.18.3 +10.132.110.130 Ready 10m v1.18.4 +10.132.115.81 Ready 10m v1.18.4 +10.132.124.107 Ready 10m v1.18.4 ``` List the pods. diff --git a/docs/flatcar-linux/google-cloud.md b/docs/flatcar-linux/google-cloud.md index dfef04dda..5d4580814 100644 --- a/docs/flatcar-linux/google-cloud.md +++ b/docs/flatcar-linux/google-cloud.md @@ -1,6 +1,6 @@ # Google Cloud -In this tutorial, we'll create a Kubernetes v1.18.3 cluster on Google Compute Engine with CoreOS Container Linux or Flatcar Linux. +In this tutorial, we'll create a Kubernetes v1.18.4 cluster on Google Compute Engine with CoreOS Container Linux or Flatcar Linux. We'll declare a Kubernetes cluster using the Typhoon Terraform module. Then apply the changes to create a network, firewall rules, health checks, controller instances, worker managed instance group, load balancers, and TLS assets. @@ -90,7 +90,7 @@ Define a Kubernetes cluster using the module `google-cloud/container-linux/kuber ```tf module "yavin" { - source = "git::https://github.com/poseidon/typhoon//google-cloud/container-linux/kubernetes?ref=v1.18.3" + source = "git::https://github.com/poseidon/typhoon//google-cloud/container-linux/kubernetes?ref=v1.18.4" # Google Cloud cluster_name = "yavin" @@ -165,9 +165,9 @@ List nodes in the cluster. $ export KUBECONFIG=/home/user/.kube/configs/yavin-config $ kubectl get nodes NAME ROLES STATUS AGE VERSION -yavin-controller-0.c.example-com.internal Ready 6m v1.18.3 -yavin-worker-jrbf.c.example-com.internal Ready 5m v1.18.3 -yavin-worker-mzdm.c.example-com.internal Ready 5m v1.18.3 +yavin-controller-0.c.example-com.internal Ready 6m v1.18.4 +yavin-worker-jrbf.c.example-com.internal Ready 5m v1.18.4 +yavin-worker-mzdm.c.example-com.internal Ready 5m v1.18.4 ``` List the pods. diff --git a/docs/index.md b/docs/index.md index 7bd25fd5b..9efc3b4d1 100644 --- a/docs/index.md +++ b/docs/index.md @@ -11,7 +11,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features -* Kubernetes v1.18.3 (upstream) +* Kubernetes v1.18.4 (upstream) * Single or multi-master, [Calico](https://www.projectcalico.org/) or [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/), SELinux enforcing * Advanced features like [worker pools](advanced/worker-pools/), [preemptible](fedora-coreos/google-cloud/#preemption) workers, and [snippets](advanced/customization/#container-linux) customization @@ -53,7 +53,7 @@ Define a Kubernetes cluster by using the Terraform module for your chosen platfo ```tf module "yavin" { - source = "git::https://github.com/poseidon/typhoon//google-cloud/fedora-coreos/kubernetes?ref=v1.18.3" + source = "git::https://github.com/poseidon/typhoon//google-cloud/fedora-coreos/kubernetes?ref=v1.18.4" # Google Cloud cluster_name = "yavin" @@ -91,9 +91,9 @@ In 4-8 minutes (varies by platform), the cluster will be ready. This Google Clou $ export KUBECONFIG=/home/user/.kube/configs/yavin-config $ kubectl get nodes NAME ROLES STATUS AGE VERSION -yavin-controller-0.c.example-com.internal Ready 6m v1.18.3 -yavin-worker-jrbf.c.example-com.internal Ready 5m v1.18.3 -yavin-worker-mzdm.c.example-com.internal Ready 5m v1.18.3 +yavin-controller-0.c.example-com.internal Ready 6m v1.18.4 +yavin-worker-jrbf.c.example-com.internal Ready 5m v1.18.4 +yavin-worker-mzdm.c.example-com.internal Ready 5m v1.18.4 ``` List the pods. diff --git a/docs/topics/maintenance.md b/docs/topics/maintenance.md index b583b6413..0f2a7df8a 100644 --- a/docs/topics/maintenance.md +++ b/docs/topics/maintenance.md @@ -13,12 +13,12 @@ Typhoon provides tagged releases to allow clusters to be versioned using ordinar ``` module "yavin" { - source = "git::https://github.com/poseidon/typhoon//google-cloud/fedora-coreos/kubernetes?ref=v1.18.3" + source = "git::https://github.com/poseidon/typhoon//google-cloud/fedora-coreos/kubernetes?ref=v1.18.4" ... } module "mercury" { - source = "git::https://github.com/poseidon/typhoon//bare-metal/container-linux/kubernetes?ref=v1.18.3" + source = "git::https://github.com/poseidon/typhoon//bare-metal/container-linux/kubernetes?ref=v1.18.4" ... } ``` diff --git a/docs/topics/security.md b/docs/topics/security.md index e793c7a08..5041dd99b 100644 --- a/docs/topics/security.md +++ b/docs/topics/security.md @@ -61,7 +61,7 @@ The Kubelet image is published to Quay.io and Dockerhub. Two tag styles indicate the build strategy used. -* Typhoon internal infra publishes single and multi-arch images (e.g. `v1.18.3`, `v1.18.3-amd64`, `v1.18.3-arm64`, `v1.18.3-2-g23228e6-amd64`, `v1.18.3-2-g23228e6-arm64`) +* Typhoon internal infra publishes single and multi-arch images (e.g. `v1.18.4`, `v1.18.4-amd64`, `v1.18.4-arm64`, `v1.18.4-2-g23228e6-amd64`, `v1.18.4-2-g23228e6-arm64`) * Quay and Dockerhub automated builds publish verifiable images (e.g. `build-SHA` on Quay, `build-TAG` on Dockerhub) The Typhoon-built Kubelet image is used as the official image. Automated builds provide an alternative image for those preferring to trust images built by Quay/Dockerhub (albeit lacking multi-arch). To use the fallback registry or an alternative tag, see [customization](/advanced/customization/#kubelet). diff --git a/google-cloud/container-linux/kubernetes/README.md b/google-cloud/container-linux/kubernetes/README.md index 43e8b17ff..b2ee7f2bb 100644 --- a/google-cloud/container-linux/kubernetes/README.md +++ b/google-cloud/container-linux/kubernetes/README.md @@ -11,7 +11,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features -* Kubernetes v1.18.3 (upstream) +* Kubernetes v1.18.4 (upstream) * Single or multi-master, [Calico](https://www.projectcalico.org/) or [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/) * Advanced features like [worker pools](https://typhoon.psdn.io/advanced/worker-pools/), [preemptible](https://typhoon.psdn.io/cl/google-cloud/#preemption) workers, and [snippets](https://typhoon.psdn.io/advanced/customization/#container-linux) customization diff --git a/google-cloud/container-linux/kubernetes/bootstrap.tf b/google-cloud/container-linux/kubernetes/bootstrap.tf index 89deafdab..fd4103d60 100644 --- a/google-cloud/container-linux/kubernetes/bootstrap.tf +++ b/google-cloud/container-linux/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=fc1a7bac89c6c95082334e0f79e478ee321596c0" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=3fe903d0accd71d198415cf46f2f6f53c5c4f699" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/google-cloud/container-linux/kubernetes/cl/controller.yaml b/google-cloud/container-linux/kubernetes/cl/controller.yaml index 2960aa591..74d509d6d 100644 --- a/google-cloud/container-linux/kubernetes/cl/controller.yaml +++ b/google-cloud/container-linux/kubernetes/cl/controller.yaml @@ -52,7 +52,7 @@ systemd: Description=Kubelet Wants=rpc-statd.service [Service] - Environment=KUBELET_IMAGE=docker://quay.io/poseidon/kubelet:v1.18.3-4-g57fa88a + Environment=KUBELET_IMAGE=docker://quay.io/poseidon/kubelet:v1.18.4 ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests ExecStartPre=/bin/mkdir -p /opt/cni/bin @@ -133,7 +133,7 @@ systemd: --volume script,kind=host,source=/opt/bootstrap/apply \ --mount volume=script,target=/apply \ --insecure-options=image \ - docker://quay.io/poseidon/kubelet:v1.18.3-4-g57fa88a \ + docker://quay.io/poseidon/kubelet:v1.18.4 \ --net=host \ --dns=host \ --exec=/apply diff --git a/google-cloud/container-linux/kubernetes/workers/cl/worker.yaml b/google-cloud/container-linux/kubernetes/workers/cl/worker.yaml index a53ab0ae8..94e1d68fd 100644 --- a/google-cloud/container-linux/kubernetes/workers/cl/worker.yaml +++ b/google-cloud/container-linux/kubernetes/workers/cl/worker.yaml @@ -25,7 +25,7 @@ systemd: Description=Kubelet Wants=rpc-statd.service [Service] - Environment=KUBELET_IMAGE=docker://quay.io/poseidon/kubelet:v1.18.3-4-g57fa88a + Environment=KUBELET_IMAGE=docker://quay.io/poseidon/kubelet:v1.18.4 ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests ExecStartPre=/bin/mkdir -p /opt/cni/bin @@ -127,7 +127,7 @@ storage: --volume config,kind=host,source=/etc/kubernetes \ --mount volume=config,target=/etc/kubernetes \ --insecure-options=image \ - docker://quay.io/poseidon/kubelet:v1.18.3-4-g57fa88a \ + docker://quay.io/poseidon/kubelet:v1.18.4 \ --net=host \ --dns=host \ --exec=/usr/local/bin/kubectl -- --kubeconfig=/etc/kubernetes/kubeconfig delete node $(hostname) diff --git a/google-cloud/fedora-coreos/kubernetes/README.md b/google-cloud/fedora-coreos/kubernetes/README.md index 3ed7b3cbf..6a3af1601 100644 --- a/google-cloud/fedora-coreos/kubernetes/README.md +++ b/google-cloud/fedora-coreos/kubernetes/README.md @@ -11,7 +11,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features -* Kubernetes v1.18.3 (upstream) +* Kubernetes v1.18.4 (upstream) * Single or multi-master, [Calico](https://www.projectcalico.org/) or [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/), SELinux enforcing * Advanced features like [worker pools](https://typhoon.psdn.io/advanced/worker-pools/), [preemptible](https://typhoon.psdn.io/fedora-coreos/google-cloud/#preemption) workers, and [snippets](https://typhoon.psdn.io/advanced/customization/) customization diff --git a/google-cloud/fedora-coreos/kubernetes/bootstrap.tf b/google-cloud/fedora-coreos/kubernetes/bootstrap.tf index 76af04099..a404e26a7 100644 --- a/google-cloud/fedora-coreos/kubernetes/bootstrap.tf +++ b/google-cloud/fedora-coreos/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=fc1a7bac89c6c95082334e0f79e478ee321596c0" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=3fe903d0accd71d198415cf46f2f6f53c5c4f699" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/google-cloud/fedora-coreos/kubernetes/fcc/controller.yaml b/google-cloud/fedora-coreos/kubernetes/fcc/controller.yaml index 116a91625..c46e244d6 100644 --- a/google-cloud/fedora-coreos/kubernetes/fcc/controller.yaml +++ b/google-cloud/fedora-coreos/kubernetes/fcc/controller.yaml @@ -54,7 +54,7 @@ systemd: Description=Kubelet (System Container) Wants=rpc-statd.service [Service] - Environment=KUBELET_IMAGE=quay.io/poseidon/kubelet:v1.18.3-4-g57fa88a + Environment=KUBELET_IMAGE=quay.io/poseidon/kubelet:v1.18.4 ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests ExecStartPre=/bin/mkdir -p /opt/cni/bin @@ -124,7 +124,7 @@ systemd: --volume /opt/bootstrap/assets:/assets:ro,Z \ --volume /opt/bootstrap/apply:/apply:ro,Z \ --entrypoint=/apply \ - quay.io/poseidon/kubelet:v1.18.3-4-g57fa88a + quay.io/poseidon/kubelet:v1.18.4 ExecStartPost=/bin/touch /opt/bootstrap/bootstrap.done ExecStartPost=-/usr/bin/podman stop bootstrap storage: diff --git a/google-cloud/fedora-coreos/kubernetes/workers/fcc/worker.yaml b/google-cloud/fedora-coreos/kubernetes/workers/fcc/worker.yaml index ec8687940..3febec95a 100644 --- a/google-cloud/fedora-coreos/kubernetes/workers/fcc/worker.yaml +++ b/google-cloud/fedora-coreos/kubernetes/workers/fcc/worker.yaml @@ -24,7 +24,7 @@ systemd: Description=Kubelet (System Container) Wants=rpc-statd.service [Service] - Environment=KUBELET_IMAGE=quay.io/poseidon/kubelet:v1.18.3-4-g57fa88a + Environment=KUBELET_IMAGE=quay.io/poseidon/kubelet:v1.18.4 ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests ExecStartPre=/bin/mkdir -p /opt/cni/bin @@ -88,7 +88,7 @@ systemd: Type=oneshot RemainAfterExit=true ExecStart=/bin/true - ExecStop=/bin/bash -c '/usr/bin/podman run --volume /etc/kubernetes:/etc/kubernetes:ro,z --entrypoint /usr/local/bin/kubectl quay.io/poseidon/kubelet:v1.18.3-4-g57fa88a --kubeconfig=/etc/kubernetes/kubeconfig delete node $HOSTNAME' + ExecStop=/bin/bash -c '/usr/bin/podman run --volume /etc/kubernetes:/etc/kubernetes:ro,z --entrypoint /usr/local/bin/kubectl quay.io/poseidon/kubelet:v1.18.4 --kubeconfig=/etc/kubernetes/kubeconfig delete node $HOSTNAME' [Install] WantedBy=multi-user.target storage: From 62341479487cbe90d00b407cbf2166d60e22b7b7 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Thu, 18 Jun 2020 01:03:37 -0700 Subject: [PATCH 453/523] Update recommended Terraform provider versions * Sync Terraform provider plugin versions with those used internally --- docs/fedora-coreos/aws.md | 2 +- docs/fedora-coreos/azure.md | 2 +- docs/fedora-coreos/digitalocean.md | 2 +- docs/fedora-coreos/google-cloud.md | 2 +- docs/flatcar-linux/aws.md | 2 +- docs/flatcar-linux/azure.md | 2 +- docs/flatcar-linux/digitalocean.md | 2 +- docs/flatcar-linux/google-cloud.md | 2 +- 8 files changed, 8 insertions(+), 8 deletions(-) diff --git a/docs/fedora-coreos/aws.md b/docs/fedora-coreos/aws.md index 3911a5760..d409decde 100644 --- a/docs/fedora-coreos/aws.md +++ b/docs/fedora-coreos/aws.md @@ -49,7 +49,7 @@ Configure the AWS provider to use your access key credentials in a `providers.tf ```tf provider "aws" { - version = "2.63.0" + version = "2.66.0" region = "eu-central-1" shared_credentials_file = "/home/user/.config/aws/credentials" } diff --git a/docs/fedora-coreos/azure.md b/docs/fedora-coreos/azure.md index 879038c8d..e7938ac8a 100644 --- a/docs/fedora-coreos/azure.md +++ b/docs/fedora-coreos/azure.md @@ -47,7 +47,7 @@ Configure the Azure provider in a `providers.tf` file. ```tf provider "azurerm" { - version = "2.11.0" + version = "2.14.0" } provider "ct" { diff --git a/docs/fedora-coreos/digitalocean.md b/docs/fedora-coreos/digitalocean.md index e78e904a0..b2b641b3d 100644 --- a/docs/fedora-coreos/digitalocean.md +++ b/docs/fedora-coreos/digitalocean.md @@ -50,7 +50,7 @@ Configure the DigitalOcean provider to use your token in a `providers.tf` file. ```tf provider "digitalocean" { - version = "1.18.0" + version = "1.20.0" token = "${chomp(file("~/.config/digital-ocean/token"))}" } diff --git a/docs/fedora-coreos/google-cloud.md b/docs/fedora-coreos/google-cloud.md index 4ad86de9a..5e755e842 100644 --- a/docs/fedora-coreos/google-cloud.md +++ b/docs/fedora-coreos/google-cloud.md @@ -49,7 +49,7 @@ Configure the Google Cloud provider to use your service account key, project-id, ```tf provider "google" { - version = "3.22.0" + version = "3.26.0" project = "project-id" region = "us-central1" credentials = file("~/.config/google-cloud/terraform.json") diff --git a/docs/flatcar-linux/aws.md b/docs/flatcar-linux/aws.md index 6264df1ae..02243536a 100644 --- a/docs/flatcar-linux/aws.md +++ b/docs/flatcar-linux/aws.md @@ -49,7 +49,7 @@ Configure the AWS provider to use your access key credentials in a `providers.tf ```tf provider "aws" { - version = "2.63.0" + version = "2.66.0" region = "eu-central-1" shared_credentials_file = "/home/user/.config/aws/credentials" } diff --git a/docs/flatcar-linux/azure.md b/docs/flatcar-linux/azure.md index 1b816dc12..246e4a869 100644 --- a/docs/flatcar-linux/azure.md +++ b/docs/flatcar-linux/azure.md @@ -47,7 +47,7 @@ Configure the Azure provider in a `providers.tf` file. ```tf provider "azurerm" { - version = "2.11.0" + version = "2.14.0" } provider "ct" { diff --git a/docs/flatcar-linux/digitalocean.md b/docs/flatcar-linux/digitalocean.md index 73a00340d..272ac2410 100644 --- a/docs/flatcar-linux/digitalocean.md +++ b/docs/flatcar-linux/digitalocean.md @@ -50,7 +50,7 @@ Configure the DigitalOcean provider to use your token in a `providers.tf` file. ```tf provider "digitalocean" { - version = "1.18.0" + version = "1.20.0" token = "${chomp(file("~/.config/digital-ocean/token"))}" } diff --git a/docs/flatcar-linux/google-cloud.md b/docs/flatcar-linux/google-cloud.md index 5d4580814..f22b27f0d 100644 --- a/docs/flatcar-linux/google-cloud.md +++ b/docs/flatcar-linux/google-cloud.md @@ -49,7 +49,7 @@ Configure the Google Cloud provider to use your service account key, project-id, ```tf provider "google" { - version = "3.22.0" + version = "3.26.0" project = "project-id" region = "us-central1" credentials = file("~/.config/google-cloud/terraform.json") From 90e23f5822917d7b08db21ca38ac43d6c7a1c98e Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Wed, 17 Jun 2020 22:50:12 -0700 Subject: [PATCH 454/523] Rename controller node label and NoSchedule taint * Remove node label `node.kubernetes.io/master` from controller nodes * Use `node.kubernetes.io/controller` (present since v1.9.5, [#160](https://github.com/poseidon/typhoon/pull/160)) to node select controllers * Rename controller NoSchedule taint from `node-role.kubernetes.io/master` to `node-role.kubernetes.io/controller` * Tolerate the new taint name for workloads that may run on controller nodes and stop tolerating `node-role.kubernetes.io/master` taint --- CHANGES.md | 6 ++++++ aws/container-linux/kubernetes/bootstrap.tf | 2 +- aws/container-linux/kubernetes/cl/controller.yaml | 3 +-- aws/fedora-coreos/kubernetes/bootstrap.tf | 2 +- aws/fedora-coreos/kubernetes/fcc/controller.yaml | 3 +-- azure/container-linux/kubernetes/bootstrap.tf | 2 +- azure/container-linux/kubernetes/cl/controller.yaml | 3 +-- azure/fedora-coreos/kubernetes/bootstrap.tf | 2 +- azure/fedora-coreos/kubernetes/fcc/controller.yaml | 3 +-- bare-metal/container-linux/kubernetes/bootstrap.tf | 2 +- bare-metal/container-linux/kubernetes/cl/controller.yaml | 3 +-- bare-metal/fedora-coreos/kubernetes/bootstrap.tf | 2 +- bare-metal/fedora-coreos/kubernetes/fcc/controller.yaml | 3 +-- digital-ocean/container-linux/kubernetes/bootstrap.tf | 2 +- digital-ocean/container-linux/kubernetes/cl/controller.yaml | 3 +-- digital-ocean/fedora-coreos/kubernetes/bootstrap.tf | 2 +- digital-ocean/fedora-coreos/kubernetes/fcc/controller.yaml | 3 +-- google-cloud/container-linux/kubernetes/bootstrap.tf | 2 +- google-cloud/container-linux/kubernetes/cl/controller.yaml | 3 +-- google-cloud/fedora-coreos/kubernetes/bootstrap.tf | 2 +- google-cloud/fedora-coreos/kubernetes/fcc/controller.yaml | 3 +-- 21 files changed, 26 insertions(+), 30 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index d1b37b3f7..731e85e55 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -4,6 +4,8 @@ Notable changes between versions. ## Latest +## v1.18.4 + * Kubernetes [v1.18.4](https://github.com/kubernetes/kubernetes/blob/master/CHANGELOG/CHANGELOG-1.18.md#v1184) * Update Kubelet image publishing ([#749](https://github.com/poseidon/typhoon/pull/749)) * Build Kubelet images internally and publish to Quay and Dockerhub @@ -13,6 +15,10 @@ Notable changes between versions. * [Document](https://typhoon.psdn.io/advanced/customization/#kubelet) use of alternate Kubelet images during registry incidents * Update Calico from v3.14.0 to [v3.14.1](https://docs.projectcalico.org/v3.14/release-notes/) * Fix [CVE-2020-13597](https://github.com/kubernetes/kubernetes/issues/91507) +* Rename controller NoSchedule taint from `node-role.kubernetes.io/master` to `node-role.kubernetes.io/controller` ([#764](https://github.com/poseidon/typhoon/pull/764)) + * Tolerate the new taint name for workloads that may run on controller nodes +* Remove node label `node.kubernetes.io/master` from controller nodes ([#764](https://github.com/poseidon/typhoon/pull/764)) + * Use `node.kubernetes.io/controller` (present since v1.9.5, [#160](https://github.com/poseidon/typhoon/pull/160)) to node select controllers * Remove unused Kubelet `-lock-file` and `-exit-on-lock-contention` ([#758](https://github.com/poseidon/typhoon/pull/758)) ### Fedora CoreOS diff --git a/aws/container-linux/kubernetes/bootstrap.tf b/aws/container-linux/kubernetes/bootstrap.tf index fe5d14210..abd8c0fb1 100644 --- a/aws/container-linux/kubernetes/bootstrap.tf +++ b/aws/container-linux/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=3fe903d0accd71d198415cf46f2f6f53c5c4f699" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=e75697ce35d7773705f0b9b28ce1ffbe99f9493c" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/aws/container-linux/kubernetes/cl/controller.yaml b/aws/container-linux/kubernetes/cl/controller.yaml index e0340a89a..771f09686 100644 --- a/aws/container-linux/kubernetes/cl/controller.yaml +++ b/aws/container-linux/kubernetes/cl/controller.yaml @@ -105,11 +105,10 @@ systemd: --healthz-port=0 \ --kubeconfig=/var/lib/kubelet/kubeconfig \ --network-plugin=cni \ - --node-labels=node.kubernetes.io/master \ --node-labels=node.kubernetes.io/controller="true" \ --pod-manifest-path=/etc/kubernetes/manifests \ --read-only-port=0 \ - --register-with-taints=node-role.kubernetes.io/master=:NoSchedule \ + --register-with-taints=node-role.kubernetes.io/controller=:NoSchedule \ --rotate-certificates \ --volume-plugin-dir=/var/lib/kubelet/volumeplugins ExecStop=-/usr/bin/rkt stop --uuid-file=/var/cache/kubelet-pod.uuid diff --git a/aws/fedora-coreos/kubernetes/bootstrap.tf b/aws/fedora-coreos/kubernetes/bootstrap.tf index cd0619658..5dd4c6774 100644 --- a/aws/fedora-coreos/kubernetes/bootstrap.tf +++ b/aws/fedora-coreos/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=3fe903d0accd71d198415cf46f2f6f53c5c4f699" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=e75697ce35d7773705f0b9b28ce1ffbe99f9493c" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/aws/fedora-coreos/kubernetes/fcc/controller.yaml b/aws/fedora-coreos/kubernetes/fcc/controller.yaml index c46e244d6..5e0119b2d 100644 --- a/aws/fedora-coreos/kubernetes/fcc/controller.yaml +++ b/aws/fedora-coreos/kubernetes/fcc/controller.yaml @@ -95,11 +95,10 @@ systemd: --healthz-port=0 \ --kubeconfig=/var/lib/kubelet/kubeconfig \ --network-plugin=cni \ - --node-labels=node.kubernetes.io/master \ --node-labels=node.kubernetes.io/controller="true" \ --pod-manifest-path=/etc/kubernetes/manifests \ --read-only-port=0 \ - --register-with-taints=node-role.kubernetes.io/master=:NoSchedule \ + --register-with-taints=node-role.kubernetes.io/controller=:NoSchedule \ --rotate-certificates \ --volume-plugin-dir=/var/lib/kubelet/volumeplugins ExecStop=-/usr/bin/podman stop kubelet diff --git a/azure/container-linux/kubernetes/bootstrap.tf b/azure/container-linux/kubernetes/bootstrap.tf index 0bfcde357..62012c199 100644 --- a/azure/container-linux/kubernetes/bootstrap.tf +++ b/azure/container-linux/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=3fe903d0accd71d198415cf46f2f6f53c5c4f699" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=e75697ce35d7773705f0b9b28ce1ffbe99f9493c" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/azure/container-linux/kubernetes/cl/controller.yaml b/azure/container-linux/kubernetes/cl/controller.yaml index 6141ce09e..d19c55679 100644 --- a/azure/container-linux/kubernetes/cl/controller.yaml +++ b/azure/container-linux/kubernetes/cl/controller.yaml @@ -103,11 +103,10 @@ systemd: --healthz-port=0 \ --kubeconfig=/var/lib/kubelet/kubeconfig \ --network-plugin=cni \ - --node-labels=node.kubernetes.io/master \ --node-labels=node.kubernetes.io/controller="true" \ --pod-manifest-path=/etc/kubernetes/manifests \ --read-only-port=0 \ - --register-with-taints=node-role.kubernetes.io/master=:NoSchedule \ + --register-with-taints=node-role.kubernetes.io/controller=:NoSchedule \ --rotate-certificates \ --volume-plugin-dir=/var/lib/kubelet/volumeplugins ExecStop=-/usr/bin/rkt stop --uuid-file=/var/cache/kubelet-pod.uuid diff --git a/azure/fedora-coreos/kubernetes/bootstrap.tf b/azure/fedora-coreos/kubernetes/bootstrap.tf index b5fe8b8b0..76391f6eb 100644 --- a/azure/fedora-coreos/kubernetes/bootstrap.tf +++ b/azure/fedora-coreos/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=3fe903d0accd71d198415cf46f2f6f53c5c4f699" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=e75697ce35d7773705f0b9b28ce1ffbe99f9493c" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/azure/fedora-coreos/kubernetes/fcc/controller.yaml b/azure/fedora-coreos/kubernetes/fcc/controller.yaml index d94d644f7..f64b298d2 100644 --- a/azure/fedora-coreos/kubernetes/fcc/controller.yaml +++ b/azure/fedora-coreos/kubernetes/fcc/controller.yaml @@ -95,11 +95,10 @@ systemd: --healthz-port=0 \ --kubeconfig=/var/lib/kubelet/kubeconfig \ --network-plugin=cni \ - --node-labels=node.kubernetes.io/master \ --node-labels=node.kubernetes.io/controller="true" \ --pod-manifest-path=/etc/kubernetes/manifests \ --read-only-port=0 \ - --register-with-taints=node-role.kubernetes.io/master=:NoSchedule \ + --register-with-taints=node-role.kubernetes.io/controller=:NoSchedule \ --rotate-certificates \ --volume-plugin-dir=/var/lib/kubelet/volumeplugins ExecStop=-/usr/bin/podman stop kubelet diff --git a/bare-metal/container-linux/kubernetes/bootstrap.tf b/bare-metal/container-linux/kubernetes/bootstrap.tf index 58e9933c4..abe58d1a8 100644 --- a/bare-metal/container-linux/kubernetes/bootstrap.tf +++ b/bare-metal/container-linux/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=3fe903d0accd71d198415cf46f2f6f53c5c4f699" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=e75697ce35d7773705f0b9b28ce1ffbe99f9493c" cluster_name = var.cluster_name api_servers = [var.k8s_domain_name] diff --git a/bare-metal/container-linux/kubernetes/cl/controller.yaml b/bare-metal/container-linux/kubernetes/cl/controller.yaml index 398845538..dc68e651b 100644 --- a/bare-metal/container-linux/kubernetes/cl/controller.yaml +++ b/bare-metal/container-linux/kubernetes/cl/controller.yaml @@ -118,11 +118,10 @@ systemd: --hostname-override=${domain_name} \ --kubeconfig=/var/lib/kubelet/kubeconfig \ --network-plugin=cni \ - --node-labels=node.kubernetes.io/master \ --node-labels=node.kubernetes.io/controller="true" \ --pod-manifest-path=/etc/kubernetes/manifests \ --read-only-port=0 \ - --register-with-taints=node-role.kubernetes.io/master=:NoSchedule \ + --register-with-taints=node-role.kubernetes.io/controller=:NoSchedule \ --rotate-certificates \ --volume-plugin-dir=/var/lib/kubelet/volumeplugins ExecStop=-/usr/bin/rkt stop --uuid-file=/var/cache/kubelet-pod.uuid diff --git a/bare-metal/fedora-coreos/kubernetes/bootstrap.tf b/bare-metal/fedora-coreos/kubernetes/bootstrap.tf index 0095ae83e..41d0c7677 100644 --- a/bare-metal/fedora-coreos/kubernetes/bootstrap.tf +++ b/bare-metal/fedora-coreos/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=3fe903d0accd71d198415cf46f2f6f53c5c4f699" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=e75697ce35d7773705f0b9b28ce1ffbe99f9493c" cluster_name = var.cluster_name api_servers = [var.k8s_domain_name] diff --git a/bare-metal/fedora-coreos/kubernetes/fcc/controller.yaml b/bare-metal/fedora-coreos/kubernetes/fcc/controller.yaml index 6623a8c6e..d56c144ed 100644 --- a/bare-metal/fedora-coreos/kubernetes/fcc/controller.yaml +++ b/bare-metal/fedora-coreos/kubernetes/fcc/controller.yaml @@ -97,11 +97,10 @@ systemd: --hostname-override=${domain_name} \ --kubeconfig=/var/lib/kubelet/kubeconfig \ --network-plugin=cni \ - --node-labels=node.kubernetes.io/master \ --node-labels=node.kubernetes.io/controller="true" \ --pod-manifest-path=/etc/kubernetes/manifests \ --read-only-port=0 \ - --register-with-taints=node-role.kubernetes.io/master=:NoSchedule \ + --register-with-taints=node-role.kubernetes.io/controller=:NoSchedule \ --rotate-certificates \ --volume-plugin-dir=/var/lib/kubelet/volumeplugins ExecStop=-/usr/bin/podman stop kubelet diff --git a/digital-ocean/container-linux/kubernetes/bootstrap.tf b/digital-ocean/container-linux/kubernetes/bootstrap.tf index aca9cc89f..87a8519ec 100644 --- a/digital-ocean/container-linux/kubernetes/bootstrap.tf +++ b/digital-ocean/container-linux/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=3fe903d0accd71d198415cf46f2f6f53c5c4f699" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=e75697ce35d7773705f0b9b28ce1ffbe99f9493c" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/digital-ocean/container-linux/kubernetes/cl/controller.yaml b/digital-ocean/container-linux/kubernetes/cl/controller.yaml index de45a7892..ef64b4e6a 100644 --- a/digital-ocean/container-linux/kubernetes/cl/controller.yaml +++ b/digital-ocean/container-linux/kubernetes/cl/controller.yaml @@ -115,11 +115,10 @@ systemd: --hostname-override=$${COREOS_DIGITALOCEAN_IPV4_PRIVATE_0} \ --kubeconfig=/var/lib/kubelet/kubeconfig \ --network-plugin=cni \ - --node-labels=node.kubernetes.io/master \ --node-labels=node.kubernetes.io/controller="true" \ --pod-manifest-path=/etc/kubernetes/manifests \ --read-only-port=0 \ - --register-with-taints=node-role.kubernetes.io/master=:NoSchedule \ + --register-with-taints=node-role.kubernetes.io/controller=:NoSchedule \ --rotate-certificates \ --volume-plugin-dir=/var/lib/kubelet/volumeplugins ExecStop=-/usr/bin/rkt stop --uuid-file=/var/cache/kubelet-pod.uuid diff --git a/digital-ocean/fedora-coreos/kubernetes/bootstrap.tf b/digital-ocean/fedora-coreos/kubernetes/bootstrap.tf index 81dabda5a..92cebf464 100644 --- a/digital-ocean/fedora-coreos/kubernetes/bootstrap.tf +++ b/digital-ocean/fedora-coreos/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=3fe903d0accd71d198415cf46f2f6f53c5c4f699" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=e75697ce35d7773705f0b9b28ce1ffbe99f9493c" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/digital-ocean/fedora-coreos/kubernetes/fcc/controller.yaml b/digital-ocean/fedora-coreos/kubernetes/fcc/controller.yaml index 878837e6f..6e32f50cb 100644 --- a/digital-ocean/fedora-coreos/kubernetes/fcc/controller.yaml +++ b/digital-ocean/fedora-coreos/kubernetes/fcc/controller.yaml @@ -98,11 +98,10 @@ systemd: --hostname-override=$${AFTERBURN_DIGITALOCEAN_IPV4_PRIVATE_0} \ --kubeconfig=/var/lib/kubelet/kubeconfig \ --network-plugin=cni \ - --node-labels=node.kubernetes.io/master \ --node-labels=node.kubernetes.io/controller="true" \ --pod-manifest-path=/etc/kubernetes/manifests \ --read-only-port=0 \ - --register-with-taints=node-role.kubernetes.io/master=:NoSchedule \ + --register-with-taints=node-role.kubernetes.io/controller=:NoSchedule \ --rotate-certificates \ --volume-plugin-dir=/var/lib/kubelet/volumeplugins ExecStop=-/usr/bin/podman stop kubelet diff --git a/google-cloud/container-linux/kubernetes/bootstrap.tf b/google-cloud/container-linux/kubernetes/bootstrap.tf index fd4103d60..9b906522c 100644 --- a/google-cloud/container-linux/kubernetes/bootstrap.tf +++ b/google-cloud/container-linux/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=3fe903d0accd71d198415cf46f2f6f53c5c4f699" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=e75697ce35d7773705f0b9b28ce1ffbe99f9493c" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/google-cloud/container-linux/kubernetes/cl/controller.yaml b/google-cloud/container-linux/kubernetes/cl/controller.yaml index 74d509d6d..31f861b97 100644 --- a/google-cloud/container-linux/kubernetes/cl/controller.yaml +++ b/google-cloud/container-linux/kubernetes/cl/controller.yaml @@ -103,10 +103,9 @@ systemd: --healthz-port=0 \ --kubeconfig=/var/lib/kubelet/kubeconfig \ --network-plugin=cni \ - --node-labels=node.kubernetes.io/master \ --node-labels=node.kubernetes.io/controller="true" \ --pod-manifest-path=/etc/kubernetes/manifests \ - --register-with-taints=node-role.kubernetes.io/master=:NoSchedule \ + --register-with-taints=node-role.kubernetes.io/controller=:NoSchedule \ --read-only-port=0 \ --rotate-certificates \ --volume-plugin-dir=/var/lib/kubelet/volumeplugins diff --git a/google-cloud/fedora-coreos/kubernetes/bootstrap.tf b/google-cloud/fedora-coreos/kubernetes/bootstrap.tf index a404e26a7..92ac39acf 100644 --- a/google-cloud/fedora-coreos/kubernetes/bootstrap.tf +++ b/google-cloud/fedora-coreos/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=3fe903d0accd71d198415cf46f2f6f53c5c4f699" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=e75697ce35d7773705f0b9b28ce1ffbe99f9493c" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/google-cloud/fedora-coreos/kubernetes/fcc/controller.yaml b/google-cloud/fedora-coreos/kubernetes/fcc/controller.yaml index c46e244d6..5e0119b2d 100644 --- a/google-cloud/fedora-coreos/kubernetes/fcc/controller.yaml +++ b/google-cloud/fedora-coreos/kubernetes/fcc/controller.yaml @@ -95,11 +95,10 @@ systemd: --healthz-port=0 \ --kubeconfig=/var/lib/kubelet/kubeconfig \ --network-plugin=cni \ - --node-labels=node.kubernetes.io/master \ --node-labels=node.kubernetes.io/controller="true" \ --pod-manifest-path=/etc/kubernetes/manifests \ --read-only-port=0 \ - --register-with-taints=node-role.kubernetes.io/master=:NoSchedule \ + --register-with-taints=node-role.kubernetes.io/controller=:NoSchedule \ --rotate-certificates \ --volume-plugin-dir=/var/lib/kubelet/volumeplugins ExecStop=-/usr/bin/podman stop kubelet From 4cfafeaa07b5f18272906c1655eeeb06676c02e1 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Thu, 18 Jun 2020 23:08:01 -0700 Subject: [PATCH 455/523] Fix Kubelet starting before hostname set on FCOS AWS * Fedora CoreOS `kubelet.service` can start before the hostname is set. Kubelet reads the hostname to determine the node name to register. If the hostname was read as localhost, Kubelet will continue trying to register as localhost (problem) * This race manifests as a node that appears NotReady, the Kubelet is trying to register as localhost, while the host itself (by then) has an AWS provided hostname. Restarting kubelet.service is a manual fix so Kubelet re-reads the hostname * This race could only be shown on AWS, not on Google Cloud or Azure despite attempts. Bare-metal and DigitalOcean differ and use hostname-override (e.g. afterburn) so they're not affected * Wait for nodes to have a non-localhost hostname in the oneshot that awaits /etc/resolve.conf. Typhoon has no valid cases for a node hostname being localhost (not even single-node clusters) Related Openshift: https://github.com/openshift/machine-config-operator/pull/1813 Close https://github.com/poseidon/typhoon/issues/765 --- CHANGES.md | 5 +++++ aws/fedora-coreos/kubernetes/fcc/controller.yaml | 3 ++- aws/fedora-coreos/kubernetes/workers/fcc/worker.yaml | 3 ++- 3 files changed, 9 insertions(+), 2 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 731e85e55..f35e44fec 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -27,6 +27,11 @@ Notable changes between versions. * Use `strict` Fedora CoreOS Config (FCC) snippet parsing ([#755](https://github.com/poseidon/typhoon/pull/755)) +#### AWS + +* Fix Kubelet service race with hostname update ([#766](https://github.com/poseidon/typhoon/pull/766)) + * Wait for a hostname to avoid Kubelet trying to register as `localhost` + ### Flatcar Linux * Use `strict` Container Linux Config (CLC) snippet parsing ([#755](https://github.com/poseidon/typhoon/pull/755)) diff --git a/aws/fedora-coreos/kubernetes/fcc/controller.yaml b/aws/fedora-coreos/kubernetes/fcc/controller.yaml index 5e0119b2d..2bc88e1ee 100644 --- a/aws/fedora-coreos/kubernetes/fcc/controller.yaml +++ b/aws/fedora-coreos/kubernetes/fcc/controller.yaml @@ -38,11 +38,12 @@ systemd: enabled: true contents: | [Unit] - Description=Wait for DNS entries + Description=Wait for DNS and hostname Before=kubelet.service [Service] Type=oneshot RemainAfterExit=true + ExecStartPre=/bin/sh -c 'while [ `hostname -s` == "localhost" ]; do sleep 1; done;' ExecStart=/bin/sh -c 'while ! /usr/bin/grep '^[^#[:space:]]' /etc/resolv.conf > /dev/null; do sleep 1; done' [Install] RequiredBy=kubelet.service diff --git a/aws/fedora-coreos/kubernetes/workers/fcc/worker.yaml b/aws/fedora-coreos/kubernetes/workers/fcc/worker.yaml index 3febec95a..a5645dea7 100644 --- a/aws/fedora-coreos/kubernetes/workers/fcc/worker.yaml +++ b/aws/fedora-coreos/kubernetes/workers/fcc/worker.yaml @@ -9,11 +9,12 @@ systemd: enabled: true contents: | [Unit] - Description=Wait for DNS entries + Description=Wait for DNS and hostname Before=kubelet.service [Service] Type=oneshot RemainAfterExit=true + ExecStartPre=/bin/sh -c 'while [ `hostname -s` == "localhost" ]; do sleep 1; done;' ExecStart=/bin/sh -c 'while ! /usr/bin/grep '^[^#[:space:]]' /etc/resolv.conf > /dev/null; do sleep 1; done' [Install] RequiredBy=kubelet.service From 37f00a38827ddd775513b46b548cc507ff4f74b3 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Thu, 18 Jun 2020 23:47:35 -0700 Subject: [PATCH 456/523] Reduce Calcio MTU on Fedora CoreOS Azure * Change the Calico VXLAN interface for MTU from 1450 to 1410 * VXLAN on Azure should support MTU 1450. However, there is history where performance measures have shown that 1410 is needed to have expected performance. Flatcar Linux has the same MTU 1410 override and note * FCOS 31.20200323.3.2 was known to perform fine with 1450, but now in 31.20200517.3.0 the right value seems to be 1410 --- CHANGES.md | 1 + azure/fedora-coreos/kubernetes/bootstrap.tf | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index f35e44fec..af856a579 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -26,6 +26,7 @@ Notable changes between versions. #### Azure * Use `strict` Fedora CoreOS Config (FCC) snippet parsing ([#755](https://github.com/poseidon/typhoon/pull/755)) +* Reduce Calico vxlan interface MTU to maintain performance ([#767](https://github.com/poseidon/typhoon/pull/766)) #### AWS diff --git a/azure/fedora-coreos/kubernetes/bootstrap.tf b/azure/fedora-coreos/kubernetes/bootstrap.tf index 76391f6eb..6e8d6e79b 100644 --- a/azure/fedora-coreos/kubernetes/bootstrap.tf +++ b/azure/fedora-coreos/kubernetes/bootstrap.tf @@ -10,8 +10,9 @@ module "bootstrap" { networking = var.networking # only effective with Calico networking + # we should be able to use 1450 MTU, but in practice, 1410 was needed network_encapsulation = "vxlan" - network_mtu = "1450" + network_mtu = "1410" pod_cidr = var.pod_cidr service_cidr = var.service_cidr From e9c8520359b2c1f3fd17a5d990f1bf5333e71608 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Mon, 15 Jun 2020 22:42:57 -0700 Subject: [PATCH 457/523] Add experimental Cilium CNI provider * Accept experimental CNI `networking` mode "cilium" * Run Cilium v1.8.0-rc4 with overlay vxlan tunnels and a minimal set of features. We're interested in: * IPAM: Divide pod_cidr into /24 subnets per node * CNI networking pod-to-pod, pod-to-external * BPF masquerade * NetworkPolicy as defined by Kubernetes (no L7 Policy) * Continue using kube-proxy with Cilium probe mode * Firewall changes: * Require UDP 8472 for vxlan (Linux kernel default) between nodes * Optional ICMP echo(8) between nodes for host reachability (health) * Optional TCP 4240 between nodes for endpoint reachability (health) Known Issues: * Containers with `hostPort` don't listen on all host addresses, these workloads must use `hostNetwork` for now https://github.com/cilium/cilium/issues/12116 * Erroneous warning on Fedora CoreOS https://github.com/cilium/cilium/issues/10256 Note: This is experimental. It is not listed in docs and may be changed or removed without a deprecation notice Related: * https://github.com/poseidon/terraform-render-bootstrap/pull/192 * https://github.com/cilium/cilium/issues/12217 --- aws/container-linux/kubernetes/bootstrap.tf | 2 +- aws/container-linux/kubernetes/security.tf | 190 ++++++++++++++++-- aws/fedora-coreos/kubernetes/bootstrap.tf | 2 +- .../kubernetes/fcc/controller.yaml | 5 + aws/fedora-coreos/kubernetes/security.tf | 190 ++++++++++++++++-- .../kubernetes/workers/fcc/worker.yaml | 5 + azure/container-linux/kubernetes/bootstrap.tf | 2 +- azure/container-linux/kubernetes/security.tf | 92 +++++++++ azure/fedora-coreos/kubernetes/bootstrap.tf | 2 +- .../kubernetes/fcc/controller.yaml | 5 + azure/fedora-coreos/kubernetes/security.tf | 92 +++++++++ .../kubernetes/workers/fcc/worker.yaml | 5 + .../container-linux/kubernetes/bootstrap.tf | 2 +- .../fedora-coreos/kubernetes/bootstrap.tf | 2 +- .../kubernetes/fcc/controller.yaml | 5 + .../fedora-coreos/kubernetes/fcc/worker.yaml | 5 + .../container-linux/kubernetes/bootstrap.tf | 2 +- .../container-linux/kubernetes/network.tf | 21 ++ .../fedora-coreos/kubernetes/bootstrap.tf | 2 +- .../kubernetes/fcc/controller.yaml | 5 + .../fedora-coreos/kubernetes/fcc/worker.yaml | 5 + .../fedora-coreos/kubernetes/network.tf | 21 ++ .../container-linux/kubernetes/bootstrap.tf | 2 +- .../container-linux/kubernetes/network.tf | 26 +++ .../fedora-coreos/kubernetes/bootstrap.tf | 2 +- .../kubernetes/fcc/controller.yaml | 5 + .../fedora-coreos/kubernetes/network.tf | 26 +++ .../kubernetes/workers/fcc/worker.yaml | 5 + 28 files changed, 676 insertions(+), 52 deletions(-) diff --git a/aws/container-linux/kubernetes/bootstrap.tf b/aws/container-linux/kubernetes/bootstrap.tf index abd8c0fb1..32ff2a46e 100644 --- a/aws/container-linux/kubernetes/bootstrap.tf +++ b/aws/container-linux/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=e75697ce35d7773705f0b9b28ce1ffbe99f9493c" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=af36c539360696f5ca6cf5b06bb729477a003602" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/aws/container-linux/kubernetes/security.tf b/aws/container-linux/kubernetes/security.tf index 60727af85..5a19930db 100644 --- a/aws/container-linux/kubernetes/security.tf +++ b/aws/container-linux/kubernetes/security.tf @@ -13,6 +13,30 @@ resource "aws_security_group" "controller" { } } +resource "aws_security_group_rule" "controller-icmp" { + count = var.networking == "cilium" ? 1 : 0 + + security_group_id = aws_security_group.controller.id + + type = "ingress" + protocol = "icmp" + from_port = 8 + to_port = 0 + source_security_group_id = aws_security_group.worker.id +} + +resource "aws_security_group_rule" "controller-icmp-self" { + count = var.networking == "cilium" ? 1 : 0 + + security_group_id = aws_security_group.controller.id + + type = "ingress" + protocol = "icmp" + from_port = 8 + to_port = 0 + self = true +} + resource "aws_security_group_rule" "controller-ssh" { security_group_id = aws_security_group.controller.id @@ -44,39 +68,31 @@ resource "aws_security_group_rule" "controller-etcd-metrics" { source_security_group_id = aws_security_group.worker.id } -# Allow Prometheus to scrape kube-proxy -resource "aws_security_group_rule" "kube-proxy-metrics" { - security_group_id = aws_security_group.controller.id +resource "aws_security_group_rule" "controller-cilium-health" { + count = var.networking == "cilium" ? 1 : 0 - type = "ingress" - protocol = "tcp" - from_port = 10249 - to_port = 10249 - source_security_group_id = aws_security_group.worker.id -} - -# Allow Prometheus to scrape kube-scheduler -resource "aws_security_group_rule" "controller-scheduler-metrics" { security_group_id = aws_security_group.controller.id type = "ingress" protocol = "tcp" - from_port = 10251 - to_port = 10251 + from_port = 4240 + to_port = 4240 source_security_group_id = aws_security_group.worker.id } -# Allow Prometheus to scrape kube-controller-manager -resource "aws_security_group_rule" "controller-manager-metrics" { +resource "aws_security_group_rule" "controller-cilium-health-self" { + count = var.networking == "cilium" ? 1 : 0 + security_group_id = aws_security_group.controller.id - type = "ingress" - protocol = "tcp" - from_port = 10252 - to_port = 10252 - source_security_group_id = aws_security_group.worker.id + type = "ingress" + protocol = "tcp" + from_port = 4240 + to_port = 4240 + self = true } +# IANA VXLAN default resource "aws_security_group_rule" "controller-vxlan" { count = var.networking == "flannel" ? 1 : 0 @@ -111,6 +127,31 @@ resource "aws_security_group_rule" "controller-apiserver" { cidr_blocks = ["0.0.0.0/0"] } +# Linux VXLAN default +resource "aws_security_group_rule" "controller-linux-vxlan" { + count = var.networking == "cilium" ? 1 : 0 + + security_group_id = aws_security_group.controller.id + + type = "ingress" + protocol = "udp" + from_port = 8472 + to_port = 8472 + source_security_group_id = aws_security_group.worker.id +} + +resource "aws_security_group_rule" "controller-linux-vxlan-self" { + count = var.networking == "cilium" ? 1 : 0 + + security_group_id = aws_security_group.controller.id + + type = "ingress" + protocol = "udp" + from_port = 8472 + to_port = 8472 + self = true +} + # Allow Prometheus to scrape node-exporter daemonset resource "aws_security_group_rule" "controller-node-exporter" { security_group_id = aws_security_group.controller.id @@ -122,6 +163,17 @@ resource "aws_security_group_rule" "controller-node-exporter" { source_security_group_id = aws_security_group.worker.id } +# Allow Prometheus to scrape kube-proxy +resource "aws_security_group_rule" "kube-proxy-metrics" { + security_group_id = aws_security_group.controller.id + + type = "ingress" + protocol = "tcp" + from_port = 10249 + to_port = 10249 + source_security_group_id = aws_security_group.worker.id +} + # Allow apiserver to access kubelets for exec, log, port-forward resource "aws_security_group_rule" "controller-kubelet" { security_group_id = aws_security_group.controller.id @@ -143,6 +195,28 @@ resource "aws_security_group_rule" "controller-kubelet-self" { self = true } +# Allow Prometheus to scrape kube-scheduler +resource "aws_security_group_rule" "controller-scheduler-metrics" { + security_group_id = aws_security_group.controller.id + + type = "ingress" + protocol = "tcp" + from_port = 10251 + to_port = 10251 + source_security_group_id = aws_security_group.worker.id +} + +# Allow Prometheus to scrape kube-controller-manager +resource "aws_security_group_rule" "controller-manager-metrics" { + security_group_id = aws_security_group.controller.id + + type = "ingress" + protocol = "tcp" + from_port = 10252 + to_port = 10252 + source_security_group_id = aws_security_group.worker.id +} + resource "aws_security_group_rule" "controller-bgp" { security_group_id = aws_security_group.controller.id @@ -227,6 +301,30 @@ resource "aws_security_group" "worker" { } } +resource "aws_security_group_rule" "worker-icmp" { + count = var.networking == "cilium" ? 1 : 0 + + security_group_id = aws_security_group.worker.id + + type = "ingress" + protocol = "icmp" + from_port = 8 + to_port = 0 + source_security_group_id = aws_security_group.controller.id +} + +resource "aws_security_group_rule" "worker-icmp-self" { + count = var.networking == "cilium" ? 1 : 0 + + security_group_id = aws_security_group.worker.id + + type = "ingress" + protocol = "icmp" + from_port = 8 + to_port = 0 + self = true +} + resource "aws_security_group_rule" "worker-ssh" { security_group_id = aws_security_group.worker.id @@ -257,6 +355,31 @@ resource "aws_security_group_rule" "worker-https" { cidr_blocks = ["0.0.0.0/0"] } +resource "aws_security_group_rule" "worker-cilium-health" { + count = var.networking == "cilium" ? 1 : 0 + + security_group_id = aws_security_group.worker.id + + type = "ingress" + protocol = "tcp" + from_port = 4240 + to_port = 4240 + source_security_group_id = aws_security_group.controller.id +} + +resource "aws_security_group_rule" "worker-cilium-health-self" { + count = var.networking == "cilium" ? 1 : 0 + + security_group_id = aws_security_group.worker.id + + type = "ingress" + protocol = "tcp" + from_port = 4240 + to_port = 4240 + self = true +} + +# IANA VXLAN default resource "aws_security_group_rule" "worker-vxlan" { count = var.networking == "flannel" ? 1 : 0 @@ -281,6 +404,31 @@ resource "aws_security_group_rule" "worker-vxlan-self" { self = true } +# Linux VXLAN default +resource "aws_security_group_rule" "worker-linux-vxlan" { + count = var.networking == "cilium" ? 1 : 0 + + security_group_id = aws_security_group.worker.id + + type = "ingress" + protocol = "udp" + from_port = 8472 + to_port = 8472 + source_security_group_id = aws_security_group.controller.id +} + +resource "aws_security_group_rule" "worker-linux-vxlan-self" { + count = var.networking == "cilium" ? 1 : 0 + + security_group_id = aws_security_group.worker.id + + type = "ingress" + protocol = "udp" + from_port = 8472 + to_port = 8472 + self = true +} + # Allow Prometheus to scrape node-exporter daemonset resource "aws_security_group_rule" "worker-node-exporter" { security_group_id = aws_security_group.worker.id diff --git a/aws/fedora-coreos/kubernetes/bootstrap.tf b/aws/fedora-coreos/kubernetes/bootstrap.tf index 5dd4c6774..3af19ef4c 100644 --- a/aws/fedora-coreos/kubernetes/bootstrap.tf +++ b/aws/fedora-coreos/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=e75697ce35d7773705f0b9b28ce1ffbe99f9493c" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=af36c539360696f5ca6cf5b06bb729477a003602" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/aws/fedora-coreos/kubernetes/fcc/controller.yaml b/aws/fedora-coreos/kubernetes/fcc/controller.yaml index 2bc88e1ee..1db4af008 100644 --- a/aws/fedora-coreos/kubernetes/fcc/controller.yaml +++ b/aws/fedora-coreos/kubernetes/fcc/controller.yaml @@ -176,6 +176,11 @@ storage: contents: inline: | fs.inotify.max_user_watches=16184 + - path: /etc/sysctl.d/reverse-path-filter.conf + contents: + inline: | + net.ipv4.conf.default.rp_filter=0 + net.ipv4.conf.*.rp_filter=0 - path: /etc/systemd/system.conf.d/accounting.conf contents: inline: | diff --git a/aws/fedora-coreos/kubernetes/security.tf b/aws/fedora-coreos/kubernetes/security.tf index 60727af85..5a19930db 100644 --- a/aws/fedora-coreos/kubernetes/security.tf +++ b/aws/fedora-coreos/kubernetes/security.tf @@ -13,6 +13,30 @@ resource "aws_security_group" "controller" { } } +resource "aws_security_group_rule" "controller-icmp" { + count = var.networking == "cilium" ? 1 : 0 + + security_group_id = aws_security_group.controller.id + + type = "ingress" + protocol = "icmp" + from_port = 8 + to_port = 0 + source_security_group_id = aws_security_group.worker.id +} + +resource "aws_security_group_rule" "controller-icmp-self" { + count = var.networking == "cilium" ? 1 : 0 + + security_group_id = aws_security_group.controller.id + + type = "ingress" + protocol = "icmp" + from_port = 8 + to_port = 0 + self = true +} + resource "aws_security_group_rule" "controller-ssh" { security_group_id = aws_security_group.controller.id @@ -44,39 +68,31 @@ resource "aws_security_group_rule" "controller-etcd-metrics" { source_security_group_id = aws_security_group.worker.id } -# Allow Prometheus to scrape kube-proxy -resource "aws_security_group_rule" "kube-proxy-metrics" { - security_group_id = aws_security_group.controller.id +resource "aws_security_group_rule" "controller-cilium-health" { + count = var.networking == "cilium" ? 1 : 0 - type = "ingress" - protocol = "tcp" - from_port = 10249 - to_port = 10249 - source_security_group_id = aws_security_group.worker.id -} - -# Allow Prometheus to scrape kube-scheduler -resource "aws_security_group_rule" "controller-scheduler-metrics" { security_group_id = aws_security_group.controller.id type = "ingress" protocol = "tcp" - from_port = 10251 - to_port = 10251 + from_port = 4240 + to_port = 4240 source_security_group_id = aws_security_group.worker.id } -# Allow Prometheus to scrape kube-controller-manager -resource "aws_security_group_rule" "controller-manager-metrics" { +resource "aws_security_group_rule" "controller-cilium-health-self" { + count = var.networking == "cilium" ? 1 : 0 + security_group_id = aws_security_group.controller.id - type = "ingress" - protocol = "tcp" - from_port = 10252 - to_port = 10252 - source_security_group_id = aws_security_group.worker.id + type = "ingress" + protocol = "tcp" + from_port = 4240 + to_port = 4240 + self = true } +# IANA VXLAN default resource "aws_security_group_rule" "controller-vxlan" { count = var.networking == "flannel" ? 1 : 0 @@ -111,6 +127,31 @@ resource "aws_security_group_rule" "controller-apiserver" { cidr_blocks = ["0.0.0.0/0"] } +# Linux VXLAN default +resource "aws_security_group_rule" "controller-linux-vxlan" { + count = var.networking == "cilium" ? 1 : 0 + + security_group_id = aws_security_group.controller.id + + type = "ingress" + protocol = "udp" + from_port = 8472 + to_port = 8472 + source_security_group_id = aws_security_group.worker.id +} + +resource "aws_security_group_rule" "controller-linux-vxlan-self" { + count = var.networking == "cilium" ? 1 : 0 + + security_group_id = aws_security_group.controller.id + + type = "ingress" + protocol = "udp" + from_port = 8472 + to_port = 8472 + self = true +} + # Allow Prometheus to scrape node-exporter daemonset resource "aws_security_group_rule" "controller-node-exporter" { security_group_id = aws_security_group.controller.id @@ -122,6 +163,17 @@ resource "aws_security_group_rule" "controller-node-exporter" { source_security_group_id = aws_security_group.worker.id } +# Allow Prometheus to scrape kube-proxy +resource "aws_security_group_rule" "kube-proxy-metrics" { + security_group_id = aws_security_group.controller.id + + type = "ingress" + protocol = "tcp" + from_port = 10249 + to_port = 10249 + source_security_group_id = aws_security_group.worker.id +} + # Allow apiserver to access kubelets for exec, log, port-forward resource "aws_security_group_rule" "controller-kubelet" { security_group_id = aws_security_group.controller.id @@ -143,6 +195,28 @@ resource "aws_security_group_rule" "controller-kubelet-self" { self = true } +# Allow Prometheus to scrape kube-scheduler +resource "aws_security_group_rule" "controller-scheduler-metrics" { + security_group_id = aws_security_group.controller.id + + type = "ingress" + protocol = "tcp" + from_port = 10251 + to_port = 10251 + source_security_group_id = aws_security_group.worker.id +} + +# Allow Prometheus to scrape kube-controller-manager +resource "aws_security_group_rule" "controller-manager-metrics" { + security_group_id = aws_security_group.controller.id + + type = "ingress" + protocol = "tcp" + from_port = 10252 + to_port = 10252 + source_security_group_id = aws_security_group.worker.id +} + resource "aws_security_group_rule" "controller-bgp" { security_group_id = aws_security_group.controller.id @@ -227,6 +301,30 @@ resource "aws_security_group" "worker" { } } +resource "aws_security_group_rule" "worker-icmp" { + count = var.networking == "cilium" ? 1 : 0 + + security_group_id = aws_security_group.worker.id + + type = "ingress" + protocol = "icmp" + from_port = 8 + to_port = 0 + source_security_group_id = aws_security_group.controller.id +} + +resource "aws_security_group_rule" "worker-icmp-self" { + count = var.networking == "cilium" ? 1 : 0 + + security_group_id = aws_security_group.worker.id + + type = "ingress" + protocol = "icmp" + from_port = 8 + to_port = 0 + self = true +} + resource "aws_security_group_rule" "worker-ssh" { security_group_id = aws_security_group.worker.id @@ -257,6 +355,31 @@ resource "aws_security_group_rule" "worker-https" { cidr_blocks = ["0.0.0.0/0"] } +resource "aws_security_group_rule" "worker-cilium-health" { + count = var.networking == "cilium" ? 1 : 0 + + security_group_id = aws_security_group.worker.id + + type = "ingress" + protocol = "tcp" + from_port = 4240 + to_port = 4240 + source_security_group_id = aws_security_group.controller.id +} + +resource "aws_security_group_rule" "worker-cilium-health-self" { + count = var.networking == "cilium" ? 1 : 0 + + security_group_id = aws_security_group.worker.id + + type = "ingress" + protocol = "tcp" + from_port = 4240 + to_port = 4240 + self = true +} + +# IANA VXLAN default resource "aws_security_group_rule" "worker-vxlan" { count = var.networking == "flannel" ? 1 : 0 @@ -281,6 +404,31 @@ resource "aws_security_group_rule" "worker-vxlan-self" { self = true } +# Linux VXLAN default +resource "aws_security_group_rule" "worker-linux-vxlan" { + count = var.networking == "cilium" ? 1 : 0 + + security_group_id = aws_security_group.worker.id + + type = "ingress" + protocol = "udp" + from_port = 8472 + to_port = 8472 + source_security_group_id = aws_security_group.controller.id +} + +resource "aws_security_group_rule" "worker-linux-vxlan-self" { + count = var.networking == "cilium" ? 1 : 0 + + security_group_id = aws_security_group.worker.id + + type = "ingress" + protocol = "udp" + from_port = 8472 + to_port = 8472 + self = true +} + # Allow Prometheus to scrape node-exporter daemonset resource "aws_security_group_rule" "worker-node-exporter" { security_group_id = aws_security_group.worker.id diff --git a/aws/fedora-coreos/kubernetes/workers/fcc/worker.yaml b/aws/fedora-coreos/kubernetes/workers/fcc/worker.yaml index a5645dea7..e99a22482 100644 --- a/aws/fedora-coreos/kubernetes/workers/fcc/worker.yaml +++ b/aws/fedora-coreos/kubernetes/workers/fcc/worker.yaml @@ -105,6 +105,11 @@ storage: contents: inline: | fs.inotify.max_user_watches=16184 + - path: /etc/sysctl.d/reverse-path-filter.conf + contents: + inline: | + net.ipv4.conf.default.rp_filter=0 + net.ipv4.conf.*.rp_filter=0 - path: /etc/systemd/system.conf.d/accounting.conf contents: inline: | diff --git a/azure/container-linux/kubernetes/bootstrap.tf b/azure/container-linux/kubernetes/bootstrap.tf index 62012c199..81fae090b 100644 --- a/azure/container-linux/kubernetes/bootstrap.tf +++ b/azure/container-linux/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=e75697ce35d7773705f0b9b28ce1ffbe99f9493c" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=af36c539360696f5ca6cf5b06bb729477a003602" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/azure/container-linux/kubernetes/security.tf b/azure/container-linux/kubernetes/security.tf index feb6fef54..c31d00145 100644 --- a/azure/container-linux/kubernetes/security.tf +++ b/azure/container-linux/kubernetes/security.tf @@ -7,6 +7,21 @@ resource "azurerm_network_security_group" "controller" { location = azurerm_resource_group.cluster.location } +resource "azurerm_network_security_rule" "controller-icmp" { + resource_group_name = azurerm_resource_group.cluster.name + + name = "allow-icmp" + network_security_group_name = azurerm_network_security_group.controller.name + priority = "1995" + access = "Allow" + direction = "Inbound" + protocol = "Icmp" + source_port_range = "*" + destination_port_range = "*" + source_address_prefixes = [azurerm_subnet.controller.address_prefix, azurerm_subnet.worker.address_prefix] + destination_address_prefix = azurerm_subnet.controller.address_prefix +} + resource "azurerm_network_security_rule" "controller-ssh" { resource_group_name = azurerm_resource_group.cluster.name @@ -100,6 +115,22 @@ resource "azurerm_network_security_rule" "controller-apiserver" { destination_address_prefix = azurerm_subnet.controller.address_prefix } +resource "azurerm_network_security_rule" "controller-cilium-health" { + resource_group_name = azurerm_resource_group.cluster.name + count = var.networking == "cilium" ? 1 : 0 + + name = "allow-cilium-health" + network_security_group_name = azurerm_network_security_group.controller.name + priority = "2019" + access = "Allow" + direction = "Inbound" + protocol = "Tcp" + source_port_range = "*" + destination_port_range = "4240" + source_address_prefixes = [azurerm_subnet.controller.address_prefix, azurerm_subnet.worker.address_prefix] + destination_address_prefix = azurerm_subnet.controller.address_prefix +} + resource "azurerm_network_security_rule" "controller-vxlan" { resource_group_name = azurerm_resource_group.cluster.name @@ -115,6 +146,21 @@ resource "azurerm_network_security_rule" "controller-vxlan" { destination_address_prefix = azurerm_subnet.controller.address_prefix } +resource "azurerm_network_security_rule" "controller-linux-vxlan" { + resource_group_name = azurerm_resource_group.cluster.name + + name = "allow-linux-vxlan" + network_security_group_name = azurerm_network_security_group.controller.name + priority = "2021" + access = "Allow" + direction = "Inbound" + protocol = "Udp" + source_port_range = "*" + destination_port_range = "8472" + source_address_prefixes = [azurerm_subnet.controller.address_prefix, azurerm_subnet.worker.address_prefix] + destination_address_prefix = azurerm_subnet.controller.address_prefix +} + # Allow Prometheus to scrape node-exporter daemonset resource "azurerm_network_security_rule" "controller-node-exporter" { resource_group_name = azurerm_resource_group.cluster.name @@ -191,6 +237,21 @@ resource "azurerm_network_security_group" "worker" { location = azurerm_resource_group.cluster.location } +resource "azurerm_network_security_rule" "worker-icmp" { + resource_group_name = azurerm_resource_group.cluster.name + + name = "allow-icmp" + network_security_group_name = azurerm_network_security_group.worker.name + priority = "1995" + access = "Allow" + direction = "Inbound" + protocol = "Icmp" + source_port_range = "*" + destination_port_range = "*" + source_address_prefixes = [azurerm_subnet.controller.address_prefix, azurerm_subnet.worker.address_prefix] + destination_address_prefix = azurerm_subnet.worker.address_prefix +} + resource "azurerm_network_security_rule" "worker-ssh" { resource_group_name = azurerm_resource_group.cluster.name @@ -236,6 +297,22 @@ resource "azurerm_network_security_rule" "worker-https" { destination_address_prefix = azurerm_subnet.worker.address_prefix } +resource "azurerm_network_security_rule" "worker-cilium-health" { + resource_group_name = azurerm_resource_group.cluster.name + count = var.networking == "cilium" ? 1 : 0 + + name = "allow-cilium-health" + network_security_group_name = azurerm_network_security_group.worker.name + priority = "2014" + access = "Allow" + direction = "Inbound" + protocol = "Tcp" + source_port_range = "*" + destination_port_range = "4240" + source_address_prefixes = [azurerm_subnet.controller.address_prefix, azurerm_subnet.worker.address_prefix] + destination_address_prefix = azurerm_subnet.worker.address_prefix +} + resource "azurerm_network_security_rule" "worker-vxlan" { resource_group_name = azurerm_resource_group.cluster.name @@ -251,6 +328,21 @@ resource "azurerm_network_security_rule" "worker-vxlan" { destination_address_prefix = azurerm_subnet.worker.address_prefix } +resource "azurerm_network_security_rule" "worker-linux-vxlan" { + resource_group_name = azurerm_resource_group.cluster.name + + name = "allow-linux-vxlan" + network_security_group_name = azurerm_network_security_group.worker.name + priority = "2016" + access = "Allow" + direction = "Inbound" + protocol = "Udp" + source_port_range = "*" + destination_port_range = "8472" + source_address_prefixes = [azurerm_subnet.controller.address_prefix, azurerm_subnet.worker.address_prefix] + destination_address_prefix = azurerm_subnet.worker.address_prefix +} + # Allow Prometheus to scrape node-exporter daemonset resource "azurerm_network_security_rule" "worker-node-exporter" { resource_group_name = azurerm_resource_group.cluster.name diff --git a/azure/fedora-coreos/kubernetes/bootstrap.tf b/azure/fedora-coreos/kubernetes/bootstrap.tf index 6e8d6e79b..a4b9ec2d4 100644 --- a/azure/fedora-coreos/kubernetes/bootstrap.tf +++ b/azure/fedora-coreos/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=e75697ce35d7773705f0b9b28ce1ffbe99f9493c" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=af36c539360696f5ca6cf5b06bb729477a003602" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/azure/fedora-coreos/kubernetes/fcc/controller.yaml b/azure/fedora-coreos/kubernetes/fcc/controller.yaml index f64b298d2..f5c0e902f 100644 --- a/azure/fedora-coreos/kubernetes/fcc/controller.yaml +++ b/azure/fedora-coreos/kubernetes/fcc/controller.yaml @@ -175,6 +175,11 @@ storage: contents: inline: | fs.inotify.max_user_watches=16184 + - path: /etc/sysctl.d/reverse-path-filter.conf + contents: + inline: | + net.ipv4.conf.default.rp_filter=0 + net.ipv4.conf.*.rp_filter=0 - path: /etc/systemd/system.conf.d/accounting.conf contents: inline: | diff --git a/azure/fedora-coreos/kubernetes/security.tf b/azure/fedora-coreos/kubernetes/security.tf index feb6fef54..c31d00145 100644 --- a/azure/fedora-coreos/kubernetes/security.tf +++ b/azure/fedora-coreos/kubernetes/security.tf @@ -7,6 +7,21 @@ resource "azurerm_network_security_group" "controller" { location = azurerm_resource_group.cluster.location } +resource "azurerm_network_security_rule" "controller-icmp" { + resource_group_name = azurerm_resource_group.cluster.name + + name = "allow-icmp" + network_security_group_name = azurerm_network_security_group.controller.name + priority = "1995" + access = "Allow" + direction = "Inbound" + protocol = "Icmp" + source_port_range = "*" + destination_port_range = "*" + source_address_prefixes = [azurerm_subnet.controller.address_prefix, azurerm_subnet.worker.address_prefix] + destination_address_prefix = azurerm_subnet.controller.address_prefix +} + resource "azurerm_network_security_rule" "controller-ssh" { resource_group_name = azurerm_resource_group.cluster.name @@ -100,6 +115,22 @@ resource "azurerm_network_security_rule" "controller-apiserver" { destination_address_prefix = azurerm_subnet.controller.address_prefix } +resource "azurerm_network_security_rule" "controller-cilium-health" { + resource_group_name = azurerm_resource_group.cluster.name + count = var.networking == "cilium" ? 1 : 0 + + name = "allow-cilium-health" + network_security_group_name = azurerm_network_security_group.controller.name + priority = "2019" + access = "Allow" + direction = "Inbound" + protocol = "Tcp" + source_port_range = "*" + destination_port_range = "4240" + source_address_prefixes = [azurerm_subnet.controller.address_prefix, azurerm_subnet.worker.address_prefix] + destination_address_prefix = azurerm_subnet.controller.address_prefix +} + resource "azurerm_network_security_rule" "controller-vxlan" { resource_group_name = azurerm_resource_group.cluster.name @@ -115,6 +146,21 @@ resource "azurerm_network_security_rule" "controller-vxlan" { destination_address_prefix = azurerm_subnet.controller.address_prefix } +resource "azurerm_network_security_rule" "controller-linux-vxlan" { + resource_group_name = azurerm_resource_group.cluster.name + + name = "allow-linux-vxlan" + network_security_group_name = azurerm_network_security_group.controller.name + priority = "2021" + access = "Allow" + direction = "Inbound" + protocol = "Udp" + source_port_range = "*" + destination_port_range = "8472" + source_address_prefixes = [azurerm_subnet.controller.address_prefix, azurerm_subnet.worker.address_prefix] + destination_address_prefix = azurerm_subnet.controller.address_prefix +} + # Allow Prometheus to scrape node-exporter daemonset resource "azurerm_network_security_rule" "controller-node-exporter" { resource_group_name = azurerm_resource_group.cluster.name @@ -191,6 +237,21 @@ resource "azurerm_network_security_group" "worker" { location = azurerm_resource_group.cluster.location } +resource "azurerm_network_security_rule" "worker-icmp" { + resource_group_name = azurerm_resource_group.cluster.name + + name = "allow-icmp" + network_security_group_name = azurerm_network_security_group.worker.name + priority = "1995" + access = "Allow" + direction = "Inbound" + protocol = "Icmp" + source_port_range = "*" + destination_port_range = "*" + source_address_prefixes = [azurerm_subnet.controller.address_prefix, azurerm_subnet.worker.address_prefix] + destination_address_prefix = azurerm_subnet.worker.address_prefix +} + resource "azurerm_network_security_rule" "worker-ssh" { resource_group_name = azurerm_resource_group.cluster.name @@ -236,6 +297,22 @@ resource "azurerm_network_security_rule" "worker-https" { destination_address_prefix = azurerm_subnet.worker.address_prefix } +resource "azurerm_network_security_rule" "worker-cilium-health" { + resource_group_name = azurerm_resource_group.cluster.name + count = var.networking == "cilium" ? 1 : 0 + + name = "allow-cilium-health" + network_security_group_name = azurerm_network_security_group.worker.name + priority = "2014" + access = "Allow" + direction = "Inbound" + protocol = "Tcp" + source_port_range = "*" + destination_port_range = "4240" + source_address_prefixes = [azurerm_subnet.controller.address_prefix, azurerm_subnet.worker.address_prefix] + destination_address_prefix = azurerm_subnet.worker.address_prefix +} + resource "azurerm_network_security_rule" "worker-vxlan" { resource_group_name = azurerm_resource_group.cluster.name @@ -251,6 +328,21 @@ resource "azurerm_network_security_rule" "worker-vxlan" { destination_address_prefix = azurerm_subnet.worker.address_prefix } +resource "azurerm_network_security_rule" "worker-linux-vxlan" { + resource_group_name = azurerm_resource_group.cluster.name + + name = "allow-linux-vxlan" + network_security_group_name = azurerm_network_security_group.worker.name + priority = "2016" + access = "Allow" + direction = "Inbound" + protocol = "Udp" + source_port_range = "*" + destination_port_range = "8472" + source_address_prefixes = [azurerm_subnet.controller.address_prefix, azurerm_subnet.worker.address_prefix] + destination_address_prefix = azurerm_subnet.worker.address_prefix +} + # Allow Prometheus to scrape node-exporter daemonset resource "azurerm_network_security_rule" "worker-node-exporter" { resource_group_name = azurerm_resource_group.cluster.name diff --git a/azure/fedora-coreos/kubernetes/workers/fcc/worker.yaml b/azure/fedora-coreos/kubernetes/workers/fcc/worker.yaml index 81e0c0bac..b901b1627 100644 --- a/azure/fedora-coreos/kubernetes/workers/fcc/worker.yaml +++ b/azure/fedora-coreos/kubernetes/workers/fcc/worker.yaml @@ -104,6 +104,11 @@ storage: contents: inline: | fs.inotify.max_user_watches=16184 + - path: /etc/sysctl.d/reverse-path-filter.conf + contents: + inline: | + net.ipv4.conf.default.rp_filter=0 + net.ipv4.conf.*.rp_filter=0 - path: /etc/systemd/system.conf.d/accounting.conf contents: inline: | diff --git a/bare-metal/container-linux/kubernetes/bootstrap.tf b/bare-metal/container-linux/kubernetes/bootstrap.tf index abe58d1a8..4fa6b0872 100644 --- a/bare-metal/container-linux/kubernetes/bootstrap.tf +++ b/bare-metal/container-linux/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=e75697ce35d7773705f0b9b28ce1ffbe99f9493c" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=af36c539360696f5ca6cf5b06bb729477a003602" cluster_name = var.cluster_name api_servers = [var.k8s_domain_name] diff --git a/bare-metal/fedora-coreos/kubernetes/bootstrap.tf b/bare-metal/fedora-coreos/kubernetes/bootstrap.tf index 41d0c7677..89901e5f6 100644 --- a/bare-metal/fedora-coreos/kubernetes/bootstrap.tf +++ b/bare-metal/fedora-coreos/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=e75697ce35d7773705f0b9b28ce1ffbe99f9493c" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=af36c539360696f5ca6cf5b06bb729477a003602" cluster_name = var.cluster_name api_servers = [var.k8s_domain_name] diff --git a/bare-metal/fedora-coreos/kubernetes/fcc/controller.yaml b/bare-metal/fedora-coreos/kubernetes/fcc/controller.yaml index d56c144ed..b31cddbaa 100644 --- a/bare-metal/fedora-coreos/kubernetes/fcc/controller.yaml +++ b/bare-metal/fedora-coreos/kubernetes/fcc/controller.yaml @@ -186,6 +186,11 @@ storage: contents: inline: | fs.inotify.max_user_watches=16184 + - path: /etc/sysctl.d/reverse-path-filter.conf + contents: + inline: | + net.ipv4.conf.default.rp_filter=0 + net.ipv4.conf.*.rp_filter=0 - path: /etc/systemd/system.conf.d/accounting.conf contents: inline: | diff --git a/bare-metal/fedora-coreos/kubernetes/fcc/worker.yaml b/bare-metal/fedora-coreos/kubernetes/fcc/worker.yaml index 26d2cd94e..ed026ba23 100644 --- a/bare-metal/fedora-coreos/kubernetes/fcc/worker.yaml +++ b/bare-metal/fedora-coreos/kubernetes/fcc/worker.yaml @@ -106,6 +106,11 @@ storage: contents: inline: | fs.inotify.max_user_watches=16184 + - path: /etc/sysctl.d/reverse-path-filter.conf + contents: + inline: | + net.ipv4.conf.default.rp_filter=0 + net.ipv4.conf.*.rp_filter=0 - path: /etc/systemd/system.conf.d/accounting.conf contents: inline: | diff --git a/digital-ocean/container-linux/kubernetes/bootstrap.tf b/digital-ocean/container-linux/kubernetes/bootstrap.tf index 87a8519ec..f228b4257 100644 --- a/digital-ocean/container-linux/kubernetes/bootstrap.tf +++ b/digital-ocean/container-linux/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=e75697ce35d7773705f0b9b28ce1ffbe99f9493c" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=af36c539360696f5ca6cf5b06bb729477a003602" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/digital-ocean/container-linux/kubernetes/network.tf b/digital-ocean/container-linux/kubernetes/network.tf index e8b0564db..4be6827aa 100644 --- a/digital-ocean/container-linux/kubernetes/network.tf +++ b/digital-ocean/container-linux/kubernetes/network.tf @@ -6,6 +6,11 @@ resource "digitalocean_firewall" "rules" { digitalocean_tag.workers.name ] + inbound_rule { + protocol = "icmp" + source_tags = [digitalocean_tag.controllers.name, digitalocean_tag.workers.name] + } + # allow ssh, internal flannel, internal node-exporter, internal kubelet inbound_rule { protocol = "tcp" @@ -13,12 +18,27 @@ resource "digitalocean_firewall" "rules" { source_addresses = ["0.0.0.0/0", "::/0"] } + # Cilium health + inbound_rule { + protocol = "tcp" + port_range = "4240" + source_tags = [digitalocean_tag.controllers.name, digitalocean_tag.workers.name] + } + + # IANA vxlan (flannel, calico) inbound_rule { protocol = "udp" port_range = "4789" source_tags = [digitalocean_tag.controllers.name, digitalocean_tag.workers.name] } + # Linux vxlan (Cilium) + inbound_rule { + protocol = "udp" + port_range = "8472" + source_tags = [digitalocean_tag.controllers.name, digitalocean_tag.workers.name] + } + # Allow Prometheus to scrape node-exporter inbound_rule { protocol = "tcp" @@ -33,6 +53,7 @@ resource "digitalocean_firewall" "rules" { source_tags = [digitalocean_tag.workers.name] } + # Kubelet inbound_rule { protocol = "tcp" port_range = "10250" diff --git a/digital-ocean/fedora-coreos/kubernetes/bootstrap.tf b/digital-ocean/fedora-coreos/kubernetes/bootstrap.tf index 92cebf464..53266d376 100644 --- a/digital-ocean/fedora-coreos/kubernetes/bootstrap.tf +++ b/digital-ocean/fedora-coreos/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=e75697ce35d7773705f0b9b28ce1ffbe99f9493c" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=af36c539360696f5ca6cf5b06bb729477a003602" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/digital-ocean/fedora-coreos/kubernetes/fcc/controller.yaml b/digital-ocean/fedora-coreos/kubernetes/fcc/controller.yaml index 6e32f50cb..54e4000c8 100644 --- a/digital-ocean/fedora-coreos/kubernetes/fcc/controller.yaml +++ b/digital-ocean/fedora-coreos/kubernetes/fcc/controller.yaml @@ -182,6 +182,11 @@ storage: contents: inline: | fs.inotify.max_user_watches=16184 + - path: /etc/sysctl.d/reverse-path-filter.conf + contents: + inline: | + net.ipv4.conf.default.rp_filter=0 + net.ipv4.conf.*.rp_filter=0 - path: /etc/systemd/system.conf.d/accounting.conf contents: inline: | diff --git a/digital-ocean/fedora-coreos/kubernetes/fcc/worker.yaml b/digital-ocean/fedora-coreos/kubernetes/fcc/worker.yaml index 44be2761e..ae360545e 100644 --- a/digital-ocean/fedora-coreos/kubernetes/fcc/worker.yaml +++ b/digital-ocean/fedora-coreos/kubernetes/fcc/worker.yaml @@ -109,6 +109,11 @@ storage: contents: inline: | fs.inotify.max_user_watches=16184 + - path: /etc/sysctl.d/reverse-path-filter.conf + contents: + inline: | + net.ipv4.conf.default.rp_filter=0 + net.ipv4.conf.*.rp_filter=0 - path: /etc/systemd/system.conf.d/accounting.conf contents: inline: | diff --git a/digital-ocean/fedora-coreos/kubernetes/network.tf b/digital-ocean/fedora-coreos/kubernetes/network.tf index e8b0564db..4be6827aa 100644 --- a/digital-ocean/fedora-coreos/kubernetes/network.tf +++ b/digital-ocean/fedora-coreos/kubernetes/network.tf @@ -6,6 +6,11 @@ resource "digitalocean_firewall" "rules" { digitalocean_tag.workers.name ] + inbound_rule { + protocol = "icmp" + source_tags = [digitalocean_tag.controllers.name, digitalocean_tag.workers.name] + } + # allow ssh, internal flannel, internal node-exporter, internal kubelet inbound_rule { protocol = "tcp" @@ -13,12 +18,27 @@ resource "digitalocean_firewall" "rules" { source_addresses = ["0.0.0.0/0", "::/0"] } + # Cilium health + inbound_rule { + protocol = "tcp" + port_range = "4240" + source_tags = [digitalocean_tag.controllers.name, digitalocean_tag.workers.name] + } + + # IANA vxlan (flannel, calico) inbound_rule { protocol = "udp" port_range = "4789" source_tags = [digitalocean_tag.controllers.name, digitalocean_tag.workers.name] } + # Linux vxlan (Cilium) + inbound_rule { + protocol = "udp" + port_range = "8472" + source_tags = [digitalocean_tag.controllers.name, digitalocean_tag.workers.name] + } + # Allow Prometheus to scrape node-exporter inbound_rule { protocol = "tcp" @@ -33,6 +53,7 @@ resource "digitalocean_firewall" "rules" { source_tags = [digitalocean_tag.workers.name] } + # Kubelet inbound_rule { protocol = "tcp" port_range = "10250" diff --git a/google-cloud/container-linux/kubernetes/bootstrap.tf b/google-cloud/container-linux/kubernetes/bootstrap.tf index 9b906522c..31fcfc22b 100644 --- a/google-cloud/container-linux/kubernetes/bootstrap.tf +++ b/google-cloud/container-linux/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=e75697ce35d7773705f0b9b28ce1ffbe99f9493c" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=af36c539360696f5ca6cf5b06bb729477a003602" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/google-cloud/container-linux/kubernetes/network.tf b/google-cloud/container-linux/kubernetes/network.tf index bd7067d7b..67c6afdee 100644 --- a/google-cloud/container-linux/kubernetes/network.tf +++ b/google-cloud/container-linux/kubernetes/network.tf @@ -112,6 +112,32 @@ resource "google_compute_firewall" "internal-vxlan" { target_tags = ["${var.cluster_name}-controller", "${var.cluster_name}-worker"] } +# Cilium VXLAN +resource "google_compute_firewall" "internal-linux-vxlan" { + count = var.networking == "cilium" ? 1 : 0 + + name = "${var.cluster_name}-linux-vxlan" + network = google_compute_network.network.name + + allow { + protocol = "udp" + ports = [8472] + } + + # Cilium health + allow { + protocol = "icmp" + } + + allow { + protocol = "tcp" + ports = [4240] + } + + source_tags = ["${var.cluster_name}-controller", "${var.cluster_name}-worker"] + target_tags = ["${var.cluster_name}-controller", "${var.cluster_name}-worker"] +} + # Allow Prometheus to scrape node-exporter daemonset resource "google_compute_firewall" "internal-node-exporter" { name = "${var.cluster_name}-internal-node-exporter" diff --git a/google-cloud/fedora-coreos/kubernetes/bootstrap.tf b/google-cloud/fedora-coreos/kubernetes/bootstrap.tf index 92ac39acf..3da0c90f8 100644 --- a/google-cloud/fedora-coreos/kubernetes/bootstrap.tf +++ b/google-cloud/fedora-coreos/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=e75697ce35d7773705f0b9b28ce1ffbe99f9493c" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=af36c539360696f5ca6cf5b06bb729477a003602" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/google-cloud/fedora-coreos/kubernetes/fcc/controller.yaml b/google-cloud/fedora-coreos/kubernetes/fcc/controller.yaml index 5e0119b2d..6d6c3745e 100644 --- a/google-cloud/fedora-coreos/kubernetes/fcc/controller.yaml +++ b/google-cloud/fedora-coreos/kubernetes/fcc/controller.yaml @@ -175,6 +175,11 @@ storage: contents: inline: | fs.inotify.max_user_watches=16184 + - path: /etc/sysctl.d/reverse-path-filter.conf + contents: + inline: | + net.ipv4.conf.default.rp_filter=0 + net.ipv4.conf.*.rp_filter=0 - path: /etc/systemd/system.conf.d/accounting.conf contents: inline: | diff --git a/google-cloud/fedora-coreos/kubernetes/network.tf b/google-cloud/fedora-coreos/kubernetes/network.tf index bd7067d7b..67c6afdee 100644 --- a/google-cloud/fedora-coreos/kubernetes/network.tf +++ b/google-cloud/fedora-coreos/kubernetes/network.tf @@ -112,6 +112,32 @@ resource "google_compute_firewall" "internal-vxlan" { target_tags = ["${var.cluster_name}-controller", "${var.cluster_name}-worker"] } +# Cilium VXLAN +resource "google_compute_firewall" "internal-linux-vxlan" { + count = var.networking == "cilium" ? 1 : 0 + + name = "${var.cluster_name}-linux-vxlan" + network = google_compute_network.network.name + + allow { + protocol = "udp" + ports = [8472] + } + + # Cilium health + allow { + protocol = "icmp" + } + + allow { + protocol = "tcp" + ports = [4240] + } + + source_tags = ["${var.cluster_name}-controller", "${var.cluster_name}-worker"] + target_tags = ["${var.cluster_name}-controller", "${var.cluster_name}-worker"] +} + # Allow Prometheus to scrape node-exporter daemonset resource "google_compute_firewall" "internal-node-exporter" { name = "${var.cluster_name}-internal-node-exporter" diff --git a/google-cloud/fedora-coreos/kubernetes/workers/fcc/worker.yaml b/google-cloud/fedora-coreos/kubernetes/workers/fcc/worker.yaml index 3febec95a..147cea9b9 100644 --- a/google-cloud/fedora-coreos/kubernetes/workers/fcc/worker.yaml +++ b/google-cloud/fedora-coreos/kubernetes/workers/fcc/worker.yaml @@ -104,6 +104,11 @@ storage: contents: inline: | fs.inotify.max_user_watches=16184 + - path: /etc/sysctl.d/reverse-path-filter.conf + contents: + inline: | + net.ipv4.conf.default.rp_filter=0 + net.ipv4.conf.*.rp_filter=0 - path: /etc/systemd/system.conf.d/accounting.conf contents: inline: | From d27f367004a4018b70a908924402afb914a4deae Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Mon, 22 Jun 2020 22:26:49 -0700 Subject: [PATCH 458/523] Update Cilium from v1.8.0-rc4 to v1.8.0 * https://github.com/cilium/cilium/releases/tag/v1.8.0 --- aws/container-linux/kubernetes/bootstrap.tf | 2 +- aws/fedora-coreos/kubernetes/bootstrap.tf | 2 +- azure/container-linux/kubernetes/bootstrap.tf | 2 +- azure/fedora-coreos/kubernetes/bootstrap.tf | 2 +- bare-metal/container-linux/kubernetes/bootstrap.tf | 2 +- bare-metal/fedora-coreos/kubernetes/bootstrap.tf | 2 +- digital-ocean/container-linux/kubernetes/bootstrap.tf | 2 +- digital-ocean/fedora-coreos/kubernetes/bootstrap.tf | 2 +- google-cloud/container-linux/kubernetes/bootstrap.tf | 2 +- google-cloud/fedora-coreos/kubernetes/bootstrap.tf | 2 +- 10 files changed, 10 insertions(+), 10 deletions(-) diff --git a/aws/container-linux/kubernetes/bootstrap.tf b/aws/container-linux/kubernetes/bootstrap.tf index 32ff2a46e..9dcf0c002 100644 --- a/aws/container-linux/kubernetes/bootstrap.tf +++ b/aws/container-linux/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=af36c539360696f5ca6cf5b06bb729477a003602" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=c014b770901405fa61c1158897c745d818f684e4" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/aws/fedora-coreos/kubernetes/bootstrap.tf b/aws/fedora-coreos/kubernetes/bootstrap.tf index 3af19ef4c..e33186a75 100644 --- a/aws/fedora-coreos/kubernetes/bootstrap.tf +++ b/aws/fedora-coreos/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=af36c539360696f5ca6cf5b06bb729477a003602" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=c014b770901405fa61c1158897c745d818f684e4" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/azure/container-linux/kubernetes/bootstrap.tf b/azure/container-linux/kubernetes/bootstrap.tf index 81fae090b..087cbd66b 100644 --- a/azure/container-linux/kubernetes/bootstrap.tf +++ b/azure/container-linux/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=af36c539360696f5ca6cf5b06bb729477a003602" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=c014b770901405fa61c1158897c745d818f684e4" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/azure/fedora-coreos/kubernetes/bootstrap.tf b/azure/fedora-coreos/kubernetes/bootstrap.tf index a4b9ec2d4..2fe1840f0 100644 --- a/azure/fedora-coreos/kubernetes/bootstrap.tf +++ b/azure/fedora-coreos/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=af36c539360696f5ca6cf5b06bb729477a003602" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=c014b770901405fa61c1158897c745d818f684e4" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/bare-metal/container-linux/kubernetes/bootstrap.tf b/bare-metal/container-linux/kubernetes/bootstrap.tf index 4fa6b0872..d2d63a450 100644 --- a/bare-metal/container-linux/kubernetes/bootstrap.tf +++ b/bare-metal/container-linux/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=af36c539360696f5ca6cf5b06bb729477a003602" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=c014b770901405fa61c1158897c745d818f684e4" cluster_name = var.cluster_name api_servers = [var.k8s_domain_name] diff --git a/bare-metal/fedora-coreos/kubernetes/bootstrap.tf b/bare-metal/fedora-coreos/kubernetes/bootstrap.tf index 89901e5f6..85db06ec5 100644 --- a/bare-metal/fedora-coreos/kubernetes/bootstrap.tf +++ b/bare-metal/fedora-coreos/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=af36c539360696f5ca6cf5b06bb729477a003602" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=c014b770901405fa61c1158897c745d818f684e4" cluster_name = var.cluster_name api_servers = [var.k8s_domain_name] diff --git a/digital-ocean/container-linux/kubernetes/bootstrap.tf b/digital-ocean/container-linux/kubernetes/bootstrap.tf index f228b4257..c7952dd29 100644 --- a/digital-ocean/container-linux/kubernetes/bootstrap.tf +++ b/digital-ocean/container-linux/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=af36c539360696f5ca6cf5b06bb729477a003602" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=c014b770901405fa61c1158897c745d818f684e4" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/digital-ocean/fedora-coreos/kubernetes/bootstrap.tf b/digital-ocean/fedora-coreos/kubernetes/bootstrap.tf index 53266d376..a2818559c 100644 --- a/digital-ocean/fedora-coreos/kubernetes/bootstrap.tf +++ b/digital-ocean/fedora-coreos/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=af36c539360696f5ca6cf5b06bb729477a003602" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=c014b770901405fa61c1158897c745d818f684e4" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/google-cloud/container-linux/kubernetes/bootstrap.tf b/google-cloud/container-linux/kubernetes/bootstrap.tf index 31fcfc22b..3015ad0a4 100644 --- a/google-cloud/container-linux/kubernetes/bootstrap.tf +++ b/google-cloud/container-linux/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=af36c539360696f5ca6cf5b06bb729477a003602" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=c014b770901405fa61c1158897c745d818f684e4" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/google-cloud/fedora-coreos/kubernetes/bootstrap.tf b/google-cloud/fedora-coreos/kubernetes/bootstrap.tf index 3da0c90f8..9ccf49211 100644 --- a/google-cloud/fedora-coreos/kubernetes/bootstrap.tf +++ b/google-cloud/fedora-coreos/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=af36c539360696f5ca6cf5b06bb729477a003602" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=c014b770901405fa61c1158897c745d818f684e4" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] From 99a11442c72a3a73409c8803f0d50206c0e049a4 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Fri, 26 Jun 2020 01:55:49 -0700 Subject: [PATCH 459/523] Update Prometheus from v2.19.0 to v2.19.1 * https://github.com/prometheus/prometheus/releases/tag/v2.19.1 --- CHANGES.md | 7 +++++++ addons/prometheus/deployment.yaml | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index af856a579..12e323ff7 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -4,6 +4,13 @@ Notable changes between versions. ## Latest +* Add Cilium v1.8.0 as a (experimental) CNI provider option ([#760](https://github.com/poseidon/typhoon/pull/760)) + * Set `networking` to "cilium" to enable + +#### Addons + +* Update Prometheus from v2.19.0 to [v2.19.1](https://github.com/prometheus/prometheus/releases/tag/v2.19.1) + ## v1.18.4 * Kubernetes [v1.18.4](https://github.com/kubernetes/kubernetes/blob/master/CHANGELOG/CHANGELOG-1.18.md#v1184) diff --git a/addons/prometheus/deployment.yaml b/addons/prometheus/deployment.yaml index 80acb9bf1..48a798d53 100644 --- a/addons/prometheus/deployment.yaml +++ b/addons/prometheus/deployment.yaml @@ -20,7 +20,7 @@ spec: serviceAccountName: prometheus containers: - name: prometheus - image: quay.io/prometheus/prometheus:v2.19.0 + image: quay.io/prometheus/prometheus:v2.19.1 args: - --web.listen-address=0.0.0.0:9090 - --config.file=/etc/prometheus/prometheus.yaml From a79ad34ba31fb6ae8de3862edbddb77fd077d07a Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Fri, 26 Jun 2020 02:06:38 -0700 Subject: [PATCH 460/523] Update Grafana from v7.0.3 to v7.0.4 * https://github.com/grafana/grafana/releases/tag/v7.0.4 --- CHANGES.md | 1 + addons/grafana/deployment.yaml | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index 12e323ff7..834adf50f 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -10,6 +10,7 @@ Notable changes between versions. #### Addons * Update Prometheus from v2.19.0 to [v2.19.1](https://github.com/prometheus/prometheus/releases/tag/v2.19.1) +* Update Grafana from v7.0.3 to [v7.0.4](https://github.com/grafana/grafana/releases/tag/v7.0.4) ## v1.18.4 diff --git a/addons/grafana/deployment.yaml b/addons/grafana/deployment.yaml index 197e535d4..8047f5fa6 100644 --- a/addons/grafana/deployment.yaml +++ b/addons/grafana/deployment.yaml @@ -23,7 +23,7 @@ spec: spec: containers: - name: grafana - image: docker.io/grafana/grafana:7.0.3 + image: docker.io/grafana/grafana:7.0.4 env: - name: GF_PATHS_CONFIG value: "/etc/grafana/custom.ini" From a10a1cee9f7cbdc4d520b636117c27716574d7d9 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Fri, 26 Jun 2020 02:24:37 -0700 Subject: [PATCH 461/523] Update mkdocs-material from v5.3.0 to v5.3.3 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index e9461384d..92d6eb802 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ mkdocs==1.1.2 -mkdocs-material==5.3.0 +mkdocs-material==5.3.3 pygments==2.6.1 pymdown-extensions==7.1.0 From 1f83ae7dbb052367e3a599b6b15cabf3eeb033c3 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Fri, 26 Jun 2020 02:40:12 -0700 Subject: [PATCH 462/523] Update Calico from v3.14.1 to v3.15.0 * https://docs.projectcalico.org/v3.15/release-notes/ --- CHANGES.md | 1 + aws/container-linux/kubernetes/bootstrap.tf | 2 +- aws/fedora-coreos/kubernetes/bootstrap.tf | 2 +- azure/container-linux/kubernetes/bootstrap.tf | 2 +- azure/fedora-coreos/kubernetes/bootstrap.tf | 2 +- bare-metal/container-linux/kubernetes/bootstrap.tf | 2 +- bare-metal/fedora-coreos/kubernetes/bootstrap.tf | 2 +- digital-ocean/container-linux/kubernetes/bootstrap.tf | 2 +- digital-ocean/fedora-coreos/kubernetes/bootstrap.tf | 2 +- google-cloud/container-linux/kubernetes/bootstrap.tf | 2 +- google-cloud/fedora-coreos/kubernetes/bootstrap.tf | 2 +- 11 files changed, 11 insertions(+), 10 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 834adf50f..c95b88851 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -6,6 +6,7 @@ Notable changes between versions. * Add Cilium v1.8.0 as a (experimental) CNI provider option ([#760](https://github.com/poseidon/typhoon/pull/760)) * Set `networking` to "cilium" to enable +* Update Calico from v3.14.1 to [v3.15.0](https://docs.projectcalico.org/v3.15/release-notes/) #### Addons diff --git a/aws/container-linux/kubernetes/bootstrap.tf b/aws/container-linux/kubernetes/bootstrap.tf index 9dcf0c002..81276180e 100644 --- a/aws/container-linux/kubernetes/bootstrap.tf +++ b/aws/container-linux/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=c014b770901405fa61c1158897c745d818f684e4" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=5043456b05b3c3caf5ef821fcf65c8ffb7afa316" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/aws/fedora-coreos/kubernetes/bootstrap.tf b/aws/fedora-coreos/kubernetes/bootstrap.tf index e33186a75..a606f4b40 100644 --- a/aws/fedora-coreos/kubernetes/bootstrap.tf +++ b/aws/fedora-coreos/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=c014b770901405fa61c1158897c745d818f684e4" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=5043456b05b3c3caf5ef821fcf65c8ffb7afa316" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/azure/container-linux/kubernetes/bootstrap.tf b/azure/container-linux/kubernetes/bootstrap.tf index 087cbd66b..bf47fdfa4 100644 --- a/azure/container-linux/kubernetes/bootstrap.tf +++ b/azure/container-linux/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=c014b770901405fa61c1158897c745d818f684e4" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=5043456b05b3c3caf5ef821fcf65c8ffb7afa316" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/azure/fedora-coreos/kubernetes/bootstrap.tf b/azure/fedora-coreos/kubernetes/bootstrap.tf index 2fe1840f0..32f4d1c02 100644 --- a/azure/fedora-coreos/kubernetes/bootstrap.tf +++ b/azure/fedora-coreos/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=c014b770901405fa61c1158897c745d818f684e4" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=5043456b05b3c3caf5ef821fcf65c8ffb7afa316" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/bare-metal/container-linux/kubernetes/bootstrap.tf b/bare-metal/container-linux/kubernetes/bootstrap.tf index d2d63a450..34c0e39b9 100644 --- a/bare-metal/container-linux/kubernetes/bootstrap.tf +++ b/bare-metal/container-linux/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=c014b770901405fa61c1158897c745d818f684e4" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=5043456b05b3c3caf5ef821fcf65c8ffb7afa316" cluster_name = var.cluster_name api_servers = [var.k8s_domain_name] diff --git a/bare-metal/fedora-coreos/kubernetes/bootstrap.tf b/bare-metal/fedora-coreos/kubernetes/bootstrap.tf index 85db06ec5..d5776c202 100644 --- a/bare-metal/fedora-coreos/kubernetes/bootstrap.tf +++ b/bare-metal/fedora-coreos/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=c014b770901405fa61c1158897c745d818f684e4" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=5043456b05b3c3caf5ef821fcf65c8ffb7afa316" cluster_name = var.cluster_name api_servers = [var.k8s_domain_name] diff --git a/digital-ocean/container-linux/kubernetes/bootstrap.tf b/digital-ocean/container-linux/kubernetes/bootstrap.tf index c7952dd29..8b8d666a3 100644 --- a/digital-ocean/container-linux/kubernetes/bootstrap.tf +++ b/digital-ocean/container-linux/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=c014b770901405fa61c1158897c745d818f684e4" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=5043456b05b3c3caf5ef821fcf65c8ffb7afa316" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/digital-ocean/fedora-coreos/kubernetes/bootstrap.tf b/digital-ocean/fedora-coreos/kubernetes/bootstrap.tf index a2818559c..77cbed070 100644 --- a/digital-ocean/fedora-coreos/kubernetes/bootstrap.tf +++ b/digital-ocean/fedora-coreos/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=c014b770901405fa61c1158897c745d818f684e4" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=5043456b05b3c3caf5ef821fcf65c8ffb7afa316" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/google-cloud/container-linux/kubernetes/bootstrap.tf b/google-cloud/container-linux/kubernetes/bootstrap.tf index 3015ad0a4..e0974bc05 100644 --- a/google-cloud/container-linux/kubernetes/bootstrap.tf +++ b/google-cloud/container-linux/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=c014b770901405fa61c1158897c745d818f684e4" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=5043456b05b3c3caf5ef821fcf65c8ffb7afa316" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/google-cloud/fedora-coreos/kubernetes/bootstrap.tf b/google-cloud/fedora-coreos/kubernetes/bootstrap.tf index 9ccf49211..e22491cba 100644 --- a/google-cloud/fedora-coreos/kubernetes/bootstrap.tf +++ b/google-cloud/fedora-coreos/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=c014b770901405fa61c1158897c745d818f684e4" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=5043456b05b3c3caf5ef821fcf65c8ffb7afa316" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] From 7bce15975cffb18add74df674e2b526a23bc83f0 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Sat, 27 Jun 2020 13:51:20 -0700 Subject: [PATCH 463/523] Update Kubernetes from v1.18.4 to v1.18.5 * https://github.com/kubernetes/kubernetes/blob/master/CHANGELOG/CHANGELOG-1.18.md#v1185 --- CHANGES.md | 1 + README.md | 10 +++++----- aws/container-linux/kubernetes/README.md | 2 +- aws/container-linux/kubernetes/bootstrap.tf | 2 +- aws/container-linux/kubernetes/cl/controller.yaml | 4 ++-- .../kubernetes/workers/cl/worker.yaml | 4 ++-- aws/fedora-coreos/kubernetes/README.md | 2 +- aws/fedora-coreos/kubernetes/bootstrap.tf | 2 +- aws/fedora-coreos/kubernetes/fcc/controller.yaml | 4 ++-- .../kubernetes/workers/fcc/worker.yaml | 4 ++-- azure/container-linux/kubernetes/README.md | 2 +- azure/container-linux/kubernetes/bootstrap.tf | 2 +- .../container-linux/kubernetes/cl/controller.yaml | 4 ++-- .../kubernetes/workers/cl/worker.yaml | 4 ++-- azure/fedora-coreos/kubernetes/README.md | 2 +- azure/fedora-coreos/kubernetes/bootstrap.tf | 2 +- azure/fedora-coreos/kubernetes/fcc/controller.yaml | 4 ++-- .../kubernetes/workers/fcc/worker.yaml | 4 ++-- bare-metal/container-linux/kubernetes/README.md | 2 +- bare-metal/container-linux/kubernetes/bootstrap.tf | 2 +- .../container-linux/kubernetes/cl/controller.yaml | 4 ++-- .../container-linux/kubernetes/cl/worker.yaml | 2 +- bare-metal/fedora-coreos/kubernetes/README.md | 2 +- bare-metal/fedora-coreos/kubernetes/bootstrap.tf | 2 +- .../fedora-coreos/kubernetes/fcc/controller.yaml | 4 ++-- .../fedora-coreos/kubernetes/fcc/worker.yaml | 2 +- digital-ocean/container-linux/kubernetes/README.md | 2 +- .../container-linux/kubernetes/bootstrap.tf | 2 +- .../container-linux/kubernetes/cl/controller.yaml | 4 ++-- .../container-linux/kubernetes/cl/worker.yaml | 4 ++-- digital-ocean/fedora-coreos/kubernetes/README.md | 2 +- .../fedora-coreos/kubernetes/bootstrap.tf | 2 +- .../fedora-coreos/kubernetes/fcc/controller.yaml | 4 ++-- .../fedora-coreos/kubernetes/fcc/worker.yaml | 4 ++-- docs/advanced/worker-pools.md | 14 +++++++------- docs/fedora-coreos/aws.md | 10 +++++----- docs/fedora-coreos/azure.md | 10 +++++----- docs/fedora-coreos/bare-metal.md | 10 +++++----- docs/fedora-coreos/digitalocean.md | 10 +++++----- docs/fedora-coreos/google-cloud.md | 8 ++++---- docs/flatcar-linux/aws.md | 10 +++++----- docs/flatcar-linux/azure.md | 10 +++++----- docs/flatcar-linux/bare-metal.md | 10 +++++----- docs/flatcar-linux/digitalocean.md | 10 +++++----- docs/flatcar-linux/google-cloud.md | 10 +++++----- docs/index.md | 10 +++++----- docs/topics/maintenance.md | 4 ++-- google-cloud/container-linux/kubernetes/README.md | 2 +- .../container-linux/kubernetes/bootstrap.tf | 2 +- .../container-linux/kubernetes/cl/controller.yaml | 4 ++-- .../kubernetes/workers/cl/worker.yaml | 4 ++-- google-cloud/fedora-coreos/kubernetes/README.md | 2 +- google-cloud/fedora-coreos/kubernetes/bootstrap.tf | 2 +- .../fedora-coreos/kubernetes/fcc/controller.yaml | 4 ++-- .../kubernetes/workers/fcc/worker.yaml | 4 ++-- 55 files changed, 127 insertions(+), 126 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index c95b88851..23dc7c24c 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -4,6 +4,7 @@ Notable changes between versions. ## Latest +* Kubernetes [v1.18.5](https://github.com/kubernetes/kubernetes/blob/master/CHANGELOG/CHANGELOG-1.18.md#v1185) * Add Cilium v1.8.0 as a (experimental) CNI provider option ([#760](https://github.com/poseidon/typhoon/pull/760)) * Set `networking` to "cilium" to enable * Update Calico from v3.14.1 to [v3.15.0](https://docs.projectcalico.org/v3.15/release-notes/) diff --git a/README.md b/README.md index b7d9b0874..bf19cf34a 100644 --- a/README.md +++ b/README.md @@ -11,7 +11,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features -* Kubernetes v1.18.4 (upstream) +* Kubernetes v1.18.5 (upstream) * Single or multi-master, [Calico](https://www.projectcalico.org/) or [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/), SELinux enforcing * Advanced features like [worker pools](https://typhoon.psdn.io/advanced/worker-pools/), [preemptible](https://typhoon.psdn.io/cl/google-cloud/#preemption) workers, and [snippets](https://typhoon.psdn.io/advanced/customization/#container-linux) customization @@ -54,7 +54,7 @@ Define a Kubernetes cluster by using the Terraform module for your chosen platfo ```tf module "yavin" { - source = "git::https://github.com/poseidon/typhoon//google-cloud/fedora-coreos/kubernetes?ref=v1.18.4" + source = "git::https://github.com/poseidon/typhoon//google-cloud/fedora-coreos/kubernetes?ref=v1.18.5" # Google Cloud cluster_name = "yavin" @@ -93,9 +93,9 @@ In 4-8 minutes (varies by platform), the cluster will be ready. This Google Clou $ export KUBECONFIG=/home/user/.kube/configs/yavin-config $ kubectl get nodes NAME ROLES STATUS AGE VERSION -yavin-controller-0.c.example-com.internal Ready 6m v1.18.4 -yavin-worker-jrbf.c.example-com.internal Ready 5m v1.18.4 -yavin-worker-mzdm.c.example-com.internal Ready 5m v1.18.4 +yavin-controller-0.c.example-com.internal Ready 6m v1.18.5 +yavin-worker-jrbf.c.example-com.internal Ready 5m v1.18.5 +yavin-worker-mzdm.c.example-com.internal Ready 5m v1.18.5 ``` List the pods. diff --git a/aws/container-linux/kubernetes/README.md b/aws/container-linux/kubernetes/README.md index cf480174e..edde1eec3 100644 --- a/aws/container-linux/kubernetes/README.md +++ b/aws/container-linux/kubernetes/README.md @@ -11,7 +11,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features -* Kubernetes v1.18.4 (upstream) +* Kubernetes v1.18.5 (upstream) * Single or multi-master, [Calico](https://www.projectcalico.org/) or [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/) * Advanced features like [worker pools](https://typhoon.psdn.io/advanced/worker-pools/), [spot](https://typhoon.psdn.io/cl/aws/#spot) workers, and [snippets](https://typhoon.psdn.io/advanced/customization/#container-linux) customization diff --git a/aws/container-linux/kubernetes/bootstrap.tf b/aws/container-linux/kubernetes/bootstrap.tf index 81276180e..c197e1d52 100644 --- a/aws/container-linux/kubernetes/bootstrap.tf +++ b/aws/container-linux/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=5043456b05b3c3caf5ef821fcf65c8ffb7afa316" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=5a7c963caf59740891df2aeae4b1561ccb3b9db6" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/aws/container-linux/kubernetes/cl/controller.yaml b/aws/container-linux/kubernetes/cl/controller.yaml index 771f09686..ded14bf5d 100644 --- a/aws/container-linux/kubernetes/cl/controller.yaml +++ b/aws/container-linux/kubernetes/cl/controller.yaml @@ -52,7 +52,7 @@ systemd: Description=Kubelet Wants=rpc-statd.service [Service] - Environment=KUBELET_IMAGE=docker://quay.io/poseidon/kubelet:v1.18.4 + Environment=KUBELET_IMAGE=docker://quay.io/poseidon/kubelet:v1.18.5 Environment=KUBELET_CGROUP_DRIVER=${cgroup_driver} ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests @@ -134,7 +134,7 @@ systemd: --volume script,kind=host,source=/opt/bootstrap/apply \ --mount volume=script,target=/apply \ --insecure-options=image \ - docker://quay.io/poseidon/kubelet:v1.18.4 \ + docker://quay.io/poseidon/kubelet:v1.18.5 \ --net=host \ --dns=host \ --exec=/apply diff --git a/aws/container-linux/kubernetes/workers/cl/worker.yaml b/aws/container-linux/kubernetes/workers/cl/worker.yaml index d4f5f7a43..3841f951b 100644 --- a/aws/container-linux/kubernetes/workers/cl/worker.yaml +++ b/aws/container-linux/kubernetes/workers/cl/worker.yaml @@ -25,7 +25,7 @@ systemd: Description=Kubelet Wants=rpc-statd.service [Service] - Environment=KUBELET_IMAGE=docker://quay.io/poseidon/kubelet:v1.18.4 + Environment=KUBELET_IMAGE=docker://quay.io/poseidon/kubelet:v1.18.5 Environment=KUBELET_CGROUP_DRIVER=${cgroup_driver} ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests @@ -129,7 +129,7 @@ storage: --volume config,kind=host,source=/etc/kubernetes \ --mount volume=config,target=/etc/kubernetes \ --insecure-options=image \ - docker://quay.io/poseidon/kubelet:v1.18.4 \ + docker://quay.io/poseidon/kubelet:v1.18.5 \ --net=host \ --dns=host \ --exec=/usr/local/bin/kubectl -- --kubeconfig=/etc/kubernetes/kubeconfig delete node $(hostname) diff --git a/aws/fedora-coreos/kubernetes/README.md b/aws/fedora-coreos/kubernetes/README.md index ad40e80fa..71ff7fcd4 100644 --- a/aws/fedora-coreos/kubernetes/README.md +++ b/aws/fedora-coreos/kubernetes/README.md @@ -11,7 +11,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features -* Kubernetes v1.18.4 (upstream) +* Kubernetes v1.18.5 (upstream) * Single or multi-master, [Calico](https://www.projectcalico.org/) or [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/), SELinux enforcing * Advanced features like [worker pools](https://typhoon.psdn.io/advanced/worker-pools/), [spot](https://typhoon.psdn.io/cl/aws/#spot) workers, and [snippets](https://typhoon.psdn.io/advanced/customization/#container-linux) customization diff --git a/aws/fedora-coreos/kubernetes/bootstrap.tf b/aws/fedora-coreos/kubernetes/bootstrap.tf index a606f4b40..bcedda803 100644 --- a/aws/fedora-coreos/kubernetes/bootstrap.tf +++ b/aws/fedora-coreos/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=5043456b05b3c3caf5ef821fcf65c8ffb7afa316" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=5a7c963caf59740891df2aeae4b1561ccb3b9db6" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/aws/fedora-coreos/kubernetes/fcc/controller.yaml b/aws/fedora-coreos/kubernetes/fcc/controller.yaml index 1db4af008..33b7ecc28 100644 --- a/aws/fedora-coreos/kubernetes/fcc/controller.yaml +++ b/aws/fedora-coreos/kubernetes/fcc/controller.yaml @@ -55,7 +55,7 @@ systemd: Description=Kubelet (System Container) Wants=rpc-statd.service [Service] - Environment=KUBELET_IMAGE=quay.io/poseidon/kubelet:v1.18.4 + Environment=KUBELET_IMAGE=quay.io/poseidon/kubelet:v1.18.5 ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests ExecStartPre=/bin/mkdir -p /opt/cni/bin @@ -124,7 +124,7 @@ systemd: --volume /opt/bootstrap/assets:/assets:ro,Z \ --volume /opt/bootstrap/apply:/apply:ro,Z \ --entrypoint=/apply \ - quay.io/poseidon/kubelet:v1.18.4 + quay.io/poseidon/kubelet:v1.18.5 ExecStartPost=/bin/touch /opt/bootstrap/bootstrap.done ExecStartPost=-/usr/bin/podman stop bootstrap storage: diff --git a/aws/fedora-coreos/kubernetes/workers/fcc/worker.yaml b/aws/fedora-coreos/kubernetes/workers/fcc/worker.yaml index e99a22482..69911b7fe 100644 --- a/aws/fedora-coreos/kubernetes/workers/fcc/worker.yaml +++ b/aws/fedora-coreos/kubernetes/workers/fcc/worker.yaml @@ -25,7 +25,7 @@ systemd: Description=Kubelet (System Container) Wants=rpc-statd.service [Service] - Environment=KUBELET_IMAGE=quay.io/poseidon/kubelet:v1.18.4 + Environment=KUBELET_IMAGE=quay.io/poseidon/kubelet:v1.18.5 ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests ExecStartPre=/bin/mkdir -p /opt/cni/bin @@ -89,7 +89,7 @@ systemd: Type=oneshot RemainAfterExit=true ExecStart=/bin/true - ExecStop=/bin/bash -c '/usr/bin/podman run --volume /etc/kubernetes:/etc/kubernetes:ro,z --entrypoint /usr/local/bin/kubectl quay.io/poseidon/kubelet:v1.18.4 --kubeconfig=/etc/kubernetes/kubeconfig delete node $HOSTNAME' + ExecStop=/bin/bash -c '/usr/bin/podman run --volume /etc/kubernetes:/etc/kubernetes:ro,z --entrypoint /usr/local/bin/kubectl quay.io/poseidon/kubelet:v1.18.5 --kubeconfig=/etc/kubernetes/kubeconfig delete node $HOSTNAME' [Install] WantedBy=multi-user.target storage: diff --git a/azure/container-linux/kubernetes/README.md b/azure/container-linux/kubernetes/README.md index bd50f056d..28b88a2e7 100644 --- a/azure/container-linux/kubernetes/README.md +++ b/azure/container-linux/kubernetes/README.md @@ -11,7 +11,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features -* Kubernetes v1.18.4 (upstream) +* Kubernetes v1.18.5 (upstream) * Single or multi-master, [Calico](https://www.projectcalico.org/) or [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/) * Advanced features like [worker pools](https://typhoon.psdn.io/advanced/worker-pools/), [low-priority](https://typhoon.psdn.io/cl/azure/#low-priority) workers, and [snippets](https://typhoon.psdn.io/advanced/customization/#container-linux) customization diff --git a/azure/container-linux/kubernetes/bootstrap.tf b/azure/container-linux/kubernetes/bootstrap.tf index bf47fdfa4..bdef479ac 100644 --- a/azure/container-linux/kubernetes/bootstrap.tf +++ b/azure/container-linux/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=5043456b05b3c3caf5ef821fcf65c8ffb7afa316" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=5a7c963caf59740891df2aeae4b1561ccb3b9db6" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/azure/container-linux/kubernetes/cl/controller.yaml b/azure/container-linux/kubernetes/cl/controller.yaml index d19c55679..db71d8a80 100644 --- a/azure/container-linux/kubernetes/cl/controller.yaml +++ b/azure/container-linux/kubernetes/cl/controller.yaml @@ -52,7 +52,7 @@ systemd: Description=Kubelet Wants=rpc-statd.service [Service] - Environment=KUBELET_IMAGE=docker://quay.io/poseidon/kubelet:v1.18.4 + Environment=KUBELET_IMAGE=docker://quay.io/poseidon/kubelet:v1.18.5 ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests ExecStartPre=/bin/mkdir -p /opt/cni/bin @@ -132,7 +132,7 @@ systemd: --volume script,kind=host,source=/opt/bootstrap/apply \ --mount volume=script,target=/apply \ --insecure-options=image \ - docker://quay.io/poseidon/kubelet:v1.18.4 \ + docker://quay.io/poseidon/kubelet:v1.18.5 \ --net=host \ --dns=host \ --exec=/apply diff --git a/azure/container-linux/kubernetes/workers/cl/worker.yaml b/azure/container-linux/kubernetes/workers/cl/worker.yaml index 6a9cab44c..549fcffe5 100644 --- a/azure/container-linux/kubernetes/workers/cl/worker.yaml +++ b/azure/container-linux/kubernetes/workers/cl/worker.yaml @@ -25,7 +25,7 @@ systemd: Description=Kubelet Wants=rpc-statd.service [Service] - Environment=KUBELET_IMAGE=docker://quay.io/poseidon/kubelet:v1.18.4 + Environment=KUBELET_IMAGE=docker://quay.io/poseidon/kubelet:v1.18.5 ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests ExecStartPre=/bin/mkdir -p /opt/cni/bin @@ -127,7 +127,7 @@ storage: --volume config,kind=host,source=/etc/kubernetes \ --mount volume=config,target=/etc/kubernetes \ --insecure-options=image \ - docker://quay.io/poseidon/kubelet:v1.18.4 \ + docker://quay.io/poseidon/kubelet:v1.18.5 \ --net=host \ --dns=host \ --exec=/usr/local/bin/kubectl -- --kubeconfig=/etc/kubernetes/kubeconfig delete node $(hostname | tr '[:upper:]' '[:lower:]') diff --git a/azure/fedora-coreos/kubernetes/README.md b/azure/fedora-coreos/kubernetes/README.md index 3d699145f..058df8f6a 100644 --- a/azure/fedora-coreos/kubernetes/README.md +++ b/azure/fedora-coreos/kubernetes/README.md @@ -11,7 +11,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features -* Kubernetes v1.18.4 (upstream) +* Kubernetes v1.18.5 (upstream) * Single or multi-master, [Calico](https://www.projectcalico.org/) or [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/), SELinux enforcing * Advanced features like [worker pools](https://typhoon.psdn.io/advanced/worker-pools/), [spot priority](https://typhoon.psdn.io/fedora-coreos/azure/#low-priority) workers, and [snippets](https://typhoon.psdn.io/advanced/customization/) customization diff --git a/azure/fedora-coreos/kubernetes/bootstrap.tf b/azure/fedora-coreos/kubernetes/bootstrap.tf index 32f4d1c02..56f19623b 100644 --- a/azure/fedora-coreos/kubernetes/bootstrap.tf +++ b/azure/fedora-coreos/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=5043456b05b3c3caf5ef821fcf65c8ffb7afa316" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=5a7c963caf59740891df2aeae4b1561ccb3b9db6" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/azure/fedora-coreos/kubernetes/fcc/controller.yaml b/azure/fedora-coreos/kubernetes/fcc/controller.yaml index f5c0e902f..f2d6e5b4a 100644 --- a/azure/fedora-coreos/kubernetes/fcc/controller.yaml +++ b/azure/fedora-coreos/kubernetes/fcc/controller.yaml @@ -54,7 +54,7 @@ systemd: Description=Kubelet (System Container) Wants=rpc-statd.service [Service] - Environment=KUBELET_IMAGE=quay.io/poseidon/kubelet:v1.18.4 + Environment=KUBELET_IMAGE=quay.io/poseidon/kubelet:v1.18.5 ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests ExecStartPre=/bin/mkdir -p /opt/cni/bin @@ -123,7 +123,7 @@ systemd: --volume /opt/bootstrap/assets:/assets:ro,Z \ --volume /opt/bootstrap/apply:/apply:ro,Z \ --entrypoint=/apply \ - quay.io/poseidon/kubelet:v1.18.4 + quay.io/poseidon/kubelet:v1.18.5 ExecStartPost=/bin/touch /opt/bootstrap/bootstrap.done ExecStartPost=-/usr/bin/podman stop bootstrap storage: diff --git a/azure/fedora-coreos/kubernetes/workers/fcc/worker.yaml b/azure/fedora-coreos/kubernetes/workers/fcc/worker.yaml index b901b1627..abb47553f 100644 --- a/azure/fedora-coreos/kubernetes/workers/fcc/worker.yaml +++ b/azure/fedora-coreos/kubernetes/workers/fcc/worker.yaml @@ -24,7 +24,7 @@ systemd: Description=Kubelet (System Container) Wants=rpc-statd.service [Service] - Environment=KUBELET_IMAGE=quay.io/poseidon/kubelet:v1.18.4 + Environment=KUBELET_IMAGE=quay.io/poseidon/kubelet:v1.18.5 ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests ExecStartPre=/bin/mkdir -p /opt/cni/bin @@ -88,7 +88,7 @@ systemd: Type=oneshot RemainAfterExit=true ExecStart=/bin/true - ExecStop=/bin/bash -c '/usr/bin/podman run --volume /etc/kubernetes:/etc/kubernetes:ro,z --entrypoint /usr/local/bin/kubectl quay.io/poseidon/kubelet:v1.18.4 --kubeconfig=/etc/kubernetes/kubeconfig delete node $HOSTNAME' + ExecStop=/bin/bash -c '/usr/bin/podman run --volume /etc/kubernetes:/etc/kubernetes:ro,z --entrypoint /usr/local/bin/kubectl quay.io/poseidon/kubelet:v1.18.5 --kubeconfig=/etc/kubernetes/kubeconfig delete node $HOSTNAME' [Install] WantedBy=multi-user.target storage: diff --git a/bare-metal/container-linux/kubernetes/README.md b/bare-metal/container-linux/kubernetes/README.md index c7bf09f42..b7a2ebcc7 100644 --- a/bare-metal/container-linux/kubernetes/README.md +++ b/bare-metal/container-linux/kubernetes/README.md @@ -11,7 +11,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features -* Kubernetes v1.18.4 (upstream) +* Kubernetes v1.18.5 (upstream) * Single or multi-master, [Calico](https://www.projectcalico.org/) or [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/) * Advanced features like [snippets](https://typhoon.psdn.io/advanced/customization/#container-linux) customization diff --git a/bare-metal/container-linux/kubernetes/bootstrap.tf b/bare-metal/container-linux/kubernetes/bootstrap.tf index 34c0e39b9..7bc3175e7 100644 --- a/bare-metal/container-linux/kubernetes/bootstrap.tf +++ b/bare-metal/container-linux/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=5043456b05b3c3caf5ef821fcf65c8ffb7afa316" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=5a7c963caf59740891df2aeae4b1561ccb3b9db6" cluster_name = var.cluster_name api_servers = [var.k8s_domain_name] diff --git a/bare-metal/container-linux/kubernetes/cl/controller.yaml b/bare-metal/container-linux/kubernetes/cl/controller.yaml index dc68e651b..ca87fac57 100644 --- a/bare-metal/container-linux/kubernetes/cl/controller.yaml +++ b/bare-metal/container-linux/kubernetes/cl/controller.yaml @@ -60,7 +60,7 @@ systemd: Description=Kubelet Wants=rpc-statd.service [Service] - Environment=KUBELET_IMAGE=docker://quay.io/poseidon/kubelet:v1.18.4 + Environment=KUBELET_IMAGE=docker://quay.io/poseidon/kubelet:v1.18.5 Environment=KUBELET_CGROUP_DRIVER=${cgroup_driver} ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests @@ -147,7 +147,7 @@ systemd: --volume script,kind=host,source=/opt/bootstrap/apply \ --mount volume=script,target=/apply \ --insecure-options=image \ - docker://quay.io/poseidon/kubelet:v1.18.4 \ + docker://quay.io/poseidon/kubelet:v1.18.5 \ --net=host \ --dns=host \ --exec=/apply diff --git a/bare-metal/container-linux/kubernetes/cl/worker.yaml b/bare-metal/container-linux/kubernetes/cl/worker.yaml index 7396e1c8a..407eec8b7 100644 --- a/bare-metal/container-linux/kubernetes/cl/worker.yaml +++ b/bare-metal/container-linux/kubernetes/cl/worker.yaml @@ -33,7 +33,7 @@ systemd: Description=Kubelet Wants=rpc-statd.service [Service] - Environment=KUBELET_IMAGE=docker://quay.io/poseidon/kubelet:v1.18.4 + Environment=KUBELET_IMAGE=docker://quay.io/poseidon/kubelet:v1.18.5 Environment=KUBELET_CGROUP_DRIVER=${cgroup_driver} ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests diff --git a/bare-metal/fedora-coreos/kubernetes/README.md b/bare-metal/fedora-coreos/kubernetes/README.md index 2d79cb83a..357501cf2 100644 --- a/bare-metal/fedora-coreos/kubernetes/README.md +++ b/bare-metal/fedora-coreos/kubernetes/README.md @@ -11,7 +11,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features -* Kubernetes v1.18.4 (upstream) +* Kubernetes v1.18.5 (upstream) * Single or multi-master, [Calico](https://www.projectcalico.org/) or [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/), SELinux enforcing * Advanced features like [snippets](https://typhoon.psdn.io/advanced/customization/#container-linux) customization diff --git a/bare-metal/fedora-coreos/kubernetes/bootstrap.tf b/bare-metal/fedora-coreos/kubernetes/bootstrap.tf index d5776c202..55232279a 100644 --- a/bare-metal/fedora-coreos/kubernetes/bootstrap.tf +++ b/bare-metal/fedora-coreos/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=5043456b05b3c3caf5ef821fcf65c8ffb7afa316" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=5a7c963caf59740891df2aeae4b1561ccb3b9db6" cluster_name = var.cluster_name api_servers = [var.k8s_domain_name] diff --git a/bare-metal/fedora-coreos/kubernetes/fcc/controller.yaml b/bare-metal/fedora-coreos/kubernetes/fcc/controller.yaml index b31cddbaa..9e296bb96 100644 --- a/bare-metal/fedora-coreos/kubernetes/fcc/controller.yaml +++ b/bare-metal/fedora-coreos/kubernetes/fcc/controller.yaml @@ -53,7 +53,7 @@ systemd: Description=Kubelet (System Container) Wants=rpc-statd.service [Service] - Environment=KUBELET_IMAGE=quay.io/poseidon/kubelet:v1.18.4 + Environment=KUBELET_IMAGE=quay.io/poseidon/kubelet:v1.18.5 ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests ExecStartPre=/bin/mkdir -p /opt/cni/bin @@ -134,7 +134,7 @@ systemd: --volume /opt/bootstrap/assets:/assets:ro,Z \ --volume /opt/bootstrap/apply:/apply:ro,Z \ --entrypoint=/apply \ - quay.io/poseidon/kubelet:v1.18.4 + quay.io/poseidon/kubelet:v1.18.5 ExecStartPost=/bin/touch /opt/bootstrap/bootstrap.done ExecStartPost=-/usr/bin/podman stop bootstrap storage: diff --git a/bare-metal/fedora-coreos/kubernetes/fcc/worker.yaml b/bare-metal/fedora-coreos/kubernetes/fcc/worker.yaml index ed026ba23..b86e70168 100644 --- a/bare-metal/fedora-coreos/kubernetes/fcc/worker.yaml +++ b/bare-metal/fedora-coreos/kubernetes/fcc/worker.yaml @@ -23,7 +23,7 @@ systemd: Description=Kubelet (System Container) Wants=rpc-statd.service [Service] - Environment=KUBELET_IMAGE=quay.io/poseidon/kubelet:v1.18.4 + Environment=KUBELET_IMAGE=quay.io/poseidon/kubelet:v1.18.5 ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests ExecStartPre=/bin/mkdir -p /opt/cni/bin diff --git a/digital-ocean/container-linux/kubernetes/README.md b/digital-ocean/container-linux/kubernetes/README.md index 622a9ccfe..626d35bbd 100644 --- a/digital-ocean/container-linux/kubernetes/README.md +++ b/digital-ocean/container-linux/kubernetes/README.md @@ -11,7 +11,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features -* Kubernetes v1.18.4 (upstream) +* Kubernetes v1.18.5 (upstream) * Single or multi-master, [Calico](https://www.projectcalico.org/) or [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/) * Advanced features like [snippets](https://typhoon.psdn.io/advanced/customization/#container-linux) customization diff --git a/digital-ocean/container-linux/kubernetes/bootstrap.tf b/digital-ocean/container-linux/kubernetes/bootstrap.tf index 8b8d666a3..ac191838c 100644 --- a/digital-ocean/container-linux/kubernetes/bootstrap.tf +++ b/digital-ocean/container-linux/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=5043456b05b3c3caf5ef821fcf65c8ffb7afa316" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=5a7c963caf59740891df2aeae4b1561ccb3b9db6" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/digital-ocean/container-linux/kubernetes/cl/controller.yaml b/digital-ocean/container-linux/kubernetes/cl/controller.yaml index ef64b4e6a..7342042fc 100644 --- a/digital-ocean/container-linux/kubernetes/cl/controller.yaml +++ b/digital-ocean/container-linux/kubernetes/cl/controller.yaml @@ -62,7 +62,7 @@ systemd: After=coreos-metadata.service Wants=rpc-statd.service [Service] - Environment=KUBELET_IMAGE=docker://quay.io/poseidon/kubelet:v1.18.4 + Environment=KUBELET_IMAGE=docker://quay.io/poseidon/kubelet:v1.18.5 EnvironmentFile=/run/metadata/coreos ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests @@ -144,7 +144,7 @@ systemd: --volume script,kind=host,source=/opt/bootstrap/apply \ --mount volume=script,target=/apply \ --insecure-options=image \ - docker://quay.io/poseidon/kubelet:v1.18.4 \ + docker://quay.io/poseidon/kubelet:v1.18.5 \ --net=host \ --dns=host \ --exec=/apply diff --git a/digital-ocean/container-linux/kubernetes/cl/worker.yaml b/digital-ocean/container-linux/kubernetes/cl/worker.yaml index 05c4ded3b..ac028b47e 100644 --- a/digital-ocean/container-linux/kubernetes/cl/worker.yaml +++ b/digital-ocean/container-linux/kubernetes/cl/worker.yaml @@ -35,7 +35,7 @@ systemd: After=coreos-metadata.service Wants=rpc-statd.service [Service] - Environment=KUBELET_IMAGE=docker://quay.io/poseidon/kubelet:v1.18.4 + Environment=KUBELET_IMAGE=docker://quay.io/poseidon/kubelet:v1.18.5 EnvironmentFile=/run/metadata/coreos ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests @@ -134,7 +134,7 @@ storage: --volume config,kind=host,source=/etc/kubernetes \ --mount volume=config,target=/etc/kubernetes \ --insecure-options=image \ - docker://quay.io/poseidon/kubelet:v1.18.4 \ + docker://quay.io/poseidon/kubelet:v1.18.5 \ --net=host \ --dns=host \ --exec=/usr/local/bin/kubectl -- --kubeconfig=/etc/kubernetes/kubeconfig delete node $(hostname) diff --git a/digital-ocean/fedora-coreos/kubernetes/README.md b/digital-ocean/fedora-coreos/kubernetes/README.md index 44ef73523..52e1131e3 100644 --- a/digital-ocean/fedora-coreos/kubernetes/README.md +++ b/digital-ocean/fedora-coreos/kubernetes/README.md @@ -11,7 +11,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features -* Kubernetes v1.18.4 (upstream) +* Kubernetes v1.18.5 (upstream) * Single or multi-master, [Calico](https://www.projectcalico.org/) or [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/), SELinux enforcing * Advanced features like [snippets](https://typhoon.psdn.io/advanced/customization/) customization diff --git a/digital-ocean/fedora-coreos/kubernetes/bootstrap.tf b/digital-ocean/fedora-coreos/kubernetes/bootstrap.tf index 77cbed070..89404db28 100644 --- a/digital-ocean/fedora-coreos/kubernetes/bootstrap.tf +++ b/digital-ocean/fedora-coreos/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=5043456b05b3c3caf5ef821fcf65c8ffb7afa316" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=5a7c963caf59740891df2aeae4b1561ccb3b9db6" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/digital-ocean/fedora-coreos/kubernetes/fcc/controller.yaml b/digital-ocean/fedora-coreos/kubernetes/fcc/controller.yaml index 54e4000c8..be92764cb 100644 --- a/digital-ocean/fedora-coreos/kubernetes/fcc/controller.yaml +++ b/digital-ocean/fedora-coreos/kubernetes/fcc/controller.yaml @@ -55,7 +55,7 @@ systemd: After=afterburn.service Wants=rpc-statd.service [Service] - Environment=KUBELET_IMAGE=quay.io/poseidon/kubelet:v1.18.4 + Environment=KUBELET_IMAGE=quay.io/poseidon/kubelet:v1.18.5 EnvironmentFile=/run/metadata/afterburn ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests @@ -135,7 +135,7 @@ systemd: --volume /opt/bootstrap/assets:/assets:ro,Z \ --volume /opt/bootstrap/apply:/apply:ro,Z \ --entrypoint=/apply \ - quay.io/poseidon/kubelet:v1.18.4 + quay.io/poseidon/kubelet:v1.18.5 ExecStartPost=/bin/touch /opt/bootstrap/bootstrap.done ExecStartPost=-/usr/bin/podman stop bootstrap storage: diff --git a/digital-ocean/fedora-coreos/kubernetes/fcc/worker.yaml b/digital-ocean/fedora-coreos/kubernetes/fcc/worker.yaml index ae360545e..231cf8439 100644 --- a/digital-ocean/fedora-coreos/kubernetes/fcc/worker.yaml +++ b/digital-ocean/fedora-coreos/kubernetes/fcc/worker.yaml @@ -26,7 +26,7 @@ systemd: After=afterburn.service Wants=rpc-statd.service [Service] - Environment=KUBELET_IMAGE=quay.io/poseidon/kubelet:v1.18.4 + Environment=KUBELET_IMAGE=quay.io/poseidon/kubelet:v1.18.5 EnvironmentFile=/run/metadata/afterburn ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests @@ -98,7 +98,7 @@ systemd: Type=oneshot RemainAfterExit=true ExecStart=/bin/true - ExecStop=/bin/bash -c '/usr/bin/podman run --volume /etc/kubernetes:/etc/kubernetes:ro,z --entrypoint /usr/local/bin/kubectl quay.io/poseidon/kubelet:v1.18.4 --kubeconfig=/etc/kubernetes/kubeconfig delete node $HOSTNAME' + ExecStop=/bin/bash -c '/usr/bin/podman run --volume /etc/kubernetes:/etc/kubernetes:ro,z --entrypoint /usr/local/bin/kubectl quay.io/poseidon/kubelet:v1.18.5 --kubeconfig=/etc/kubernetes/kubeconfig delete node $HOSTNAME' [Install] WantedBy=multi-user.target storage: diff --git a/docs/advanced/worker-pools.md b/docs/advanced/worker-pools.md index a2f368617..4f880ed0c 100644 --- a/docs/advanced/worker-pools.md +++ b/docs/advanced/worker-pools.md @@ -82,7 +82,7 @@ Create a cluster following the Azure [tutorial](../flatcar-linux/azure.md#cluste ```tf module "ramius-worker-pool" { - source = "git::https://github.com/poseidon/typhoon//azure/container-linux/kubernetes/workers?ref=v1.18.4" + source = "git::https://github.com/poseidon/typhoon//azure/container-linux/kubernetes/workers?ref=v1.18.5" # Azure region = module.ramius.region @@ -148,7 +148,7 @@ Create a cluster following the Google Cloud [tutorial](../flatcar-linux/google-c ```tf module "yavin-worker-pool" { - source = "git::https://github.com/poseidon/typhoon//google-cloud/container-linux/kubernetes/workers?ref=v1.18.4" + source = "git::https://github.com/poseidon/typhoon//google-cloud/container-linux/kubernetes/workers?ref=v1.18.5" # Google Cloud region = "europe-west2" @@ -179,11 +179,11 @@ Verify a managed instance group of workers joins the cluster within a few minute ``` $ kubectl get nodes NAME STATUS AGE VERSION -yavin-controller-0.c.example-com.internal Ready 6m v1.18.4 -yavin-worker-jrbf.c.example-com.internal Ready 5m v1.18.4 -yavin-worker-mzdm.c.example-com.internal Ready 5m v1.18.4 -yavin-16x-worker-jrbf.c.example-com.internal Ready 3m v1.18.4 -yavin-16x-worker-mzdm.c.example-com.internal Ready 3m v1.18.4 +yavin-controller-0.c.example-com.internal Ready 6m v1.18.5 +yavin-worker-jrbf.c.example-com.internal Ready 5m v1.18.5 +yavin-worker-mzdm.c.example-com.internal Ready 5m v1.18.5 +yavin-16x-worker-jrbf.c.example-com.internal Ready 3m v1.18.5 +yavin-16x-worker-mzdm.c.example-com.internal Ready 3m v1.18.5 ``` ### Variables diff --git a/docs/fedora-coreos/aws.md b/docs/fedora-coreos/aws.md index d409decde..9f2e4841d 100644 --- a/docs/fedora-coreos/aws.md +++ b/docs/fedora-coreos/aws.md @@ -1,6 +1,6 @@ # AWS -In this tutorial, we'll create a Kubernetes v1.18.4 cluster on AWS with Fedora CoreOS. +In this tutorial, we'll create a Kubernetes v1.18.5 cluster on AWS with Fedora CoreOS. We'll declare a Kubernetes cluster using the Typhoon Terraform module. Then apply the changes to create a VPC, gateway, subnets, security groups, controller instances, worker auto-scaling group, network load balancer, and TLS assets. @@ -70,7 +70,7 @@ Define a Kubernetes cluster using the module `aws/fedora-coreos/kubernetes`. ```tf module "tempest" { - source = "git::https://github.com/poseidon/typhoon//aws/fedora-coreos/kubernetes?ref=v1.18.4" + source = "git::https://github.com/poseidon/typhoon//aws/fedora-coreos/kubernetes?ref=v1.18.5" # AWS cluster_name = "tempest" @@ -143,9 +143,9 @@ List nodes in the cluster. $ export KUBECONFIG=/home/user/.kube/configs/tempest-config $ kubectl get nodes NAME STATUS ROLES AGE VERSION -ip-10-0-3-155 Ready 10m v1.18.4 -ip-10-0-26-65 Ready 10m v1.18.4 -ip-10-0-41-21 Ready 10m v1.18.4 +ip-10-0-3-155 Ready 10m v1.18.5 +ip-10-0-26-65 Ready 10m v1.18.5 +ip-10-0-41-21 Ready 10m v1.18.5 ``` List the pods. diff --git a/docs/fedora-coreos/azure.md b/docs/fedora-coreos/azure.md index e7938ac8a..93eecdda1 100644 --- a/docs/fedora-coreos/azure.md +++ b/docs/fedora-coreos/azure.md @@ -1,6 +1,6 @@ # Azure -In this tutorial, we'll create a Kubernetes v1.18.4 cluster on Azure with Fedora CoreOS. +In this tutorial, we'll create a Kubernetes v1.18.5 cluster on Azure with Fedora CoreOS. We'll declare a Kubernetes cluster using the Typhoon Terraform module. Then apply the changes to create a resource group, virtual network, subnets, security groups, controller availability set, worker scale set, load balancer, and TLS assets. @@ -83,7 +83,7 @@ Define a Kubernetes cluster using the module `azure/fedora-coreos/kubernetes`. ```tf module "ramius" { - source = "git::https://github.com/poseidon/typhoon//azure/fedora-coreos/kubernetes?ref=v1.18.4" + source = "git::https://github.com/poseidon/typhoon//azure/fedora-coreos/kubernetes?ref=v1.18.5" # Azure cluster_name = "ramius" @@ -158,9 +158,9 @@ List nodes in the cluster. $ export KUBECONFIG=/home/user/.kube/configs/ramius-config $ kubectl get nodes NAME STATUS ROLES AGE VERSION -ramius-controller-0 Ready 24m v1.18.4 -ramius-worker-000001 Ready 25m v1.18.4 -ramius-worker-000002 Ready 24m v1.18.4 +ramius-controller-0 Ready 24m v1.18.5 +ramius-worker-000001 Ready 25m v1.18.5 +ramius-worker-000002 Ready 24m v1.18.5 ``` List the pods. diff --git a/docs/fedora-coreos/bare-metal.md b/docs/fedora-coreos/bare-metal.md index cbab8a111..061746787 100644 --- a/docs/fedora-coreos/bare-metal.md +++ b/docs/fedora-coreos/bare-metal.md @@ -1,6 +1,6 @@ # Bare-Metal -In this tutorial, we'll network boot and provision a Kubernetes v1.18.4 cluster on bare-metal with Fedora CoreOS. +In this tutorial, we'll network boot and provision a Kubernetes v1.18.5 cluster on bare-metal with Fedora CoreOS. First, we'll deploy a [Matchbox](https://github.com/poseidon/matchbox) service and setup a network boot environment. Then, we'll declare a Kubernetes cluster using the Typhoon Terraform module and power on machines. On PXE boot, machines will install Fedora CoreOS to disk, reboot into the disk install, and provision themselves as Kubernetes controllers or workers via Ignition. @@ -160,7 +160,7 @@ Define a Kubernetes cluster using the module `bare-metal/fedora-coreos/kubernete ```tf module "mercury" { - source = "git::https://github.com/poseidon/typhoon//bare-metal/fedora-coreos/kubernetes?ref=v1.18.4" + source = "git::https://github.com/poseidon/typhoon//bare-metal/fedora-coreos/kubernetes?ref=v1.18.5" # bare-metal cluster_name = "mercury" @@ -289,9 +289,9 @@ List nodes in the cluster. $ export KUBECONFIG=/home/user/.kube/configs/mercury-config $ kubectl get nodes NAME STATUS ROLES AGE VERSION -node1.example.com Ready 10m v1.18.4 -node2.example.com Ready 10m v1.18.4 -node3.example.com Ready 10m v1.18.4 +node1.example.com Ready 10m v1.18.5 +node2.example.com Ready 10m v1.18.5 +node3.example.com Ready 10m v1.18.5 ``` List the pods. diff --git a/docs/fedora-coreos/digitalocean.md b/docs/fedora-coreos/digitalocean.md index b2b641b3d..7884af438 100644 --- a/docs/fedora-coreos/digitalocean.md +++ b/docs/fedora-coreos/digitalocean.md @@ -1,6 +1,6 @@ # DigitalOcean -In this tutorial, we'll create a Kubernetes v1.18.4 cluster on DigitalOcean with Fedora CoreOS. +In this tutorial, we'll create a Kubernetes v1.18.5 cluster on DigitalOcean with Fedora CoreOS. We'll declare a Kubernetes cluster using the Typhoon Terraform module. Then apply the changes to create controller droplets, worker droplets, DNS records, tags, and TLS assets. @@ -79,7 +79,7 @@ Define a Kubernetes cluster using the module `digital-ocean/fedora-coreos/kubern ```tf module "nemo" { - source = "git::https://github.com/poseidon/typhoon//digital-ocean/fedora-coreos/kubernetes?ref=v1.18.4" + source = "git::https://github.com/poseidon/typhoon//digital-ocean/fedora-coreos/kubernetes?ref=v1.18.5" # Digital Ocean cluster_name = "nemo" @@ -153,9 +153,9 @@ List nodes in the cluster. $ export KUBECONFIG=/home/user/.kube/configs/nemo-config $ kubectl get nodes NAME STATUS ROLES AGE VERSION -10.132.110.130 Ready 10m v1.18.4 -10.132.115.81 Ready 10m v1.18.4 -10.132.124.107 Ready 10m v1.18.4 +10.132.110.130 Ready 10m v1.18.5 +10.132.115.81 Ready 10m v1.18.5 +10.132.124.107 Ready 10m v1.18.5 ``` List the pods. diff --git a/docs/fedora-coreos/google-cloud.md b/docs/fedora-coreos/google-cloud.md index 5e755e842..dfb192727 100644 --- a/docs/fedora-coreos/google-cloud.md +++ b/docs/fedora-coreos/google-cloud.md @@ -1,6 +1,6 @@ # Google Cloud -In this tutorial, we'll create a Kubernetes v1.18.4 cluster on Google Compute Engine with Fedora CoreOS. +In this tutorial, we'll create a Kubernetes v1.18.5 cluster on Google Compute Engine with Fedora CoreOS. We'll declare a Kubernetes cluster using the Typhoon Terraform module. Then apply the changes to create a network, firewall rules, health checks, controller instances, worker managed instance group, load balancers, and TLS assets. @@ -145,9 +145,9 @@ List nodes in the cluster. $ export KUBECONFIG=/home/user/.kube/configs/yavin-config $ kubectl get nodes NAME ROLES STATUS AGE VERSION -yavin-controller-0.c.example-com.internal Ready 6m v1.18.4 -yavin-worker-jrbf.c.example-com.internal Ready 5m v1.18.4 -yavin-worker-mzdm.c.example-com.internal Ready 5m v1.18.4 +yavin-controller-0.c.example-com.internal Ready 6m v1.18.5 +yavin-worker-jrbf.c.example-com.internal Ready 5m v1.18.5 +yavin-worker-mzdm.c.example-com.internal Ready 5m v1.18.5 ``` List the pods. diff --git a/docs/flatcar-linux/aws.md b/docs/flatcar-linux/aws.md index 02243536a..25d04fa0f 100644 --- a/docs/flatcar-linux/aws.md +++ b/docs/flatcar-linux/aws.md @@ -1,6 +1,6 @@ # AWS -In this tutorial, we'll create a Kubernetes v1.18.4 cluster on AWS with CoreOS Container Linux or Flatcar Linux. +In this tutorial, we'll create a Kubernetes v1.18.5 cluster on AWS with CoreOS Container Linux or Flatcar Linux. We'll declare a Kubernetes cluster using the Typhoon Terraform module. Then apply the changes to create a VPC, gateway, subnets, security groups, controller instances, worker auto-scaling group, network load balancer, and TLS assets. @@ -70,7 +70,7 @@ Define a Kubernetes cluster using the module `aws/container-linux/kubernetes`. ```tf module "tempest" { - source = "git::https://github.com/poseidon/typhoon//aws/container-linux/kubernetes?ref=v1.18.4" + source = "git::https://github.com/poseidon/typhoon//aws/container-linux/kubernetes?ref=v1.18.5" # AWS cluster_name = "tempest" @@ -143,9 +143,9 @@ List nodes in the cluster. $ export KUBECONFIG=/home/user/.kube/configs/tempest-config $ kubectl get nodes NAME STATUS ROLES AGE VERSION -ip-10-0-3-155 Ready 10m v1.18.4 -ip-10-0-26-65 Ready 10m v1.18.4 -ip-10-0-41-21 Ready 10m v1.18.4 +ip-10-0-3-155 Ready 10m v1.18.5 +ip-10-0-26-65 Ready 10m v1.18.5 +ip-10-0-41-21 Ready 10m v1.18.5 ``` List the pods. diff --git a/docs/flatcar-linux/azure.md b/docs/flatcar-linux/azure.md index 246e4a869..c0b6f9cf2 100644 --- a/docs/flatcar-linux/azure.md +++ b/docs/flatcar-linux/azure.md @@ -1,6 +1,6 @@ # Azure -In this tutorial, we'll create a Kubernetes v1.18.4 cluster on Azure with CoreOS Container Linux or Flatcar Linux. +In this tutorial, we'll create a Kubernetes v1.18.5 cluster on Azure with CoreOS Container Linux or Flatcar Linux. We'll declare a Kubernetes cluster using the Typhoon Terraform module. Then apply the changes to create a resource group, virtual network, subnets, security groups, controller availability set, worker scale set, load balancer, and TLS assets. @@ -72,7 +72,7 @@ Define a Kubernetes cluster using the module `azure/container-linux/kubernetes`. ```tf module "ramius" { - source = "git::https://github.com/poseidon/typhoon//azure/container-linux/kubernetes?ref=v1.18.4" + source = "git::https://github.com/poseidon/typhoon//azure/container-linux/kubernetes?ref=v1.18.5" # Azure cluster_name = "ramius" @@ -146,9 +146,9 @@ List nodes in the cluster. $ export KUBECONFIG=/home/user/.kube/configs/ramius-config $ kubectl get nodes NAME STATUS ROLES AGE VERSION -ramius-controller-0 Ready 24m v1.18.4 -ramius-worker-000001 Ready 25m v1.18.4 -ramius-worker-000002 Ready 24m v1.18.4 +ramius-controller-0 Ready 24m v1.18.5 +ramius-worker-000001 Ready 25m v1.18.5 +ramius-worker-000002 Ready 24m v1.18.5 ``` List the pods. diff --git a/docs/flatcar-linux/bare-metal.md b/docs/flatcar-linux/bare-metal.md index 024826e92..b2624d91a 100644 --- a/docs/flatcar-linux/bare-metal.md +++ b/docs/flatcar-linux/bare-metal.md @@ -1,6 +1,6 @@ # Bare-Metal -In this tutorial, we'll network boot and provision a Kubernetes v1.18.4 cluster on bare-metal with CoreOS Container Linux or Flatcar Linux. +In this tutorial, we'll network boot and provision a Kubernetes v1.18.5 cluster on bare-metal with CoreOS Container Linux or Flatcar Linux. First, we'll deploy a [Matchbox](https://github.com/poseidon/matchbox) service and setup a network boot environment. Then, we'll declare a Kubernetes cluster using the Typhoon Terraform module and power on machines. On PXE boot, machines will install Container Linux to disk, reboot into the disk install, and provision themselves as Kubernetes controllers or workers via Ignition. @@ -160,7 +160,7 @@ Define a Kubernetes cluster using the module `bare-metal/container-linux/kuberne ```tf module "mercury" { - source = "git::https://github.com/poseidon/typhoon//bare-metal/container-linux/kubernetes?ref=v1.18.4" + source = "git::https://github.com/poseidon/typhoon//bare-metal/container-linux/kubernetes?ref=v1.18.5" # bare-metal cluster_name = "mercury" @@ -299,9 +299,9 @@ List nodes in the cluster. $ export KUBECONFIG=/home/user/.kube/configs/mercury-config $ kubectl get nodes NAME STATUS ROLES AGE VERSION -node1.example.com Ready 10m v1.18.4 -node2.example.com Ready 10m v1.18.4 -node3.example.com Ready 10m v1.18.4 +node1.example.com Ready 10m v1.18.5 +node2.example.com Ready 10m v1.18.5 +node3.example.com Ready 10m v1.18.5 ``` List the pods. diff --git a/docs/flatcar-linux/digitalocean.md b/docs/flatcar-linux/digitalocean.md index 272ac2410..4418fdc2e 100644 --- a/docs/flatcar-linux/digitalocean.md +++ b/docs/flatcar-linux/digitalocean.md @@ -1,6 +1,6 @@ # DigitalOcean -In this tutorial, we'll create a Kubernetes v1.18.4 cluster on DigitalOcean with CoreOS Container Linux or Flatcar Linux. +In this tutorial, we'll create a Kubernetes v1.18.5 cluster on DigitalOcean with CoreOS Container Linux or Flatcar Linux. We'll declare a Kubernetes cluster using the Typhoon Terraform module. Then apply the changes to create controller droplets, worker droplets, DNS records, tags, and TLS assets. @@ -79,7 +79,7 @@ Define a Kubernetes cluster using the module `digital-ocean/container-linux/kube ```tf module "nemo" { - source = "git::https://github.com/poseidon/typhoon//digital-ocean/container-linux/kubernetes?ref=v1.18.4" + source = "git::https://github.com/poseidon/typhoon//digital-ocean/container-linux/kubernetes?ref=v1.18.5" # Digital Ocean cluster_name = "nemo" @@ -153,9 +153,9 @@ List nodes in the cluster. $ export KUBECONFIG=/home/user/.kube/configs/nemo-config $ kubectl get nodes NAME STATUS ROLES AGE VERSION -10.132.110.130 Ready 10m v1.18.4 -10.132.115.81 Ready 10m v1.18.4 -10.132.124.107 Ready 10m v1.18.4 +10.132.110.130 Ready 10m v1.18.5 +10.132.115.81 Ready 10m v1.18.5 +10.132.124.107 Ready 10m v1.18.5 ``` List the pods. diff --git a/docs/flatcar-linux/google-cloud.md b/docs/flatcar-linux/google-cloud.md index f22b27f0d..d10755788 100644 --- a/docs/flatcar-linux/google-cloud.md +++ b/docs/flatcar-linux/google-cloud.md @@ -1,6 +1,6 @@ # Google Cloud -In this tutorial, we'll create a Kubernetes v1.18.4 cluster on Google Compute Engine with CoreOS Container Linux or Flatcar Linux. +In this tutorial, we'll create a Kubernetes v1.18.5 cluster on Google Compute Engine with CoreOS Container Linux or Flatcar Linux. We'll declare a Kubernetes cluster using the Typhoon Terraform module. Then apply the changes to create a network, firewall rules, health checks, controller instances, worker managed instance group, load balancers, and TLS assets. @@ -90,7 +90,7 @@ Define a Kubernetes cluster using the module `google-cloud/container-linux/kuber ```tf module "yavin" { - source = "git::https://github.com/poseidon/typhoon//google-cloud/container-linux/kubernetes?ref=v1.18.4" + source = "git::https://github.com/poseidon/typhoon//google-cloud/container-linux/kubernetes?ref=v1.18.5" # Google Cloud cluster_name = "yavin" @@ -165,9 +165,9 @@ List nodes in the cluster. $ export KUBECONFIG=/home/user/.kube/configs/yavin-config $ kubectl get nodes NAME ROLES STATUS AGE VERSION -yavin-controller-0.c.example-com.internal Ready 6m v1.18.4 -yavin-worker-jrbf.c.example-com.internal Ready 5m v1.18.4 -yavin-worker-mzdm.c.example-com.internal Ready 5m v1.18.4 +yavin-controller-0.c.example-com.internal Ready 6m v1.18.5 +yavin-worker-jrbf.c.example-com.internal Ready 5m v1.18.5 +yavin-worker-mzdm.c.example-com.internal Ready 5m v1.18.5 ``` List the pods. diff --git a/docs/index.md b/docs/index.md index 9efc3b4d1..0b682ebef 100644 --- a/docs/index.md +++ b/docs/index.md @@ -11,7 +11,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features -* Kubernetes v1.18.4 (upstream) +* Kubernetes v1.18.5 (upstream) * Single or multi-master, [Calico](https://www.projectcalico.org/) or [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/), SELinux enforcing * Advanced features like [worker pools](advanced/worker-pools/), [preemptible](fedora-coreos/google-cloud/#preemption) workers, and [snippets](advanced/customization/#container-linux) customization @@ -53,7 +53,7 @@ Define a Kubernetes cluster by using the Terraform module for your chosen platfo ```tf module "yavin" { - source = "git::https://github.com/poseidon/typhoon//google-cloud/fedora-coreos/kubernetes?ref=v1.18.4" + source = "git::https://github.com/poseidon/typhoon//google-cloud/fedora-coreos/kubernetes?ref=v1.18.5" # Google Cloud cluster_name = "yavin" @@ -91,9 +91,9 @@ In 4-8 minutes (varies by platform), the cluster will be ready. This Google Clou $ export KUBECONFIG=/home/user/.kube/configs/yavin-config $ kubectl get nodes NAME ROLES STATUS AGE VERSION -yavin-controller-0.c.example-com.internal Ready 6m v1.18.4 -yavin-worker-jrbf.c.example-com.internal Ready 5m v1.18.4 -yavin-worker-mzdm.c.example-com.internal Ready 5m v1.18.4 +yavin-controller-0.c.example-com.internal Ready 6m v1.18.5 +yavin-worker-jrbf.c.example-com.internal Ready 5m v1.18.5 +yavin-worker-mzdm.c.example-com.internal Ready 5m v1.18.5 ``` List the pods. diff --git a/docs/topics/maintenance.md b/docs/topics/maintenance.md index 0f2a7df8a..cb6303e36 100644 --- a/docs/topics/maintenance.md +++ b/docs/topics/maintenance.md @@ -13,12 +13,12 @@ Typhoon provides tagged releases to allow clusters to be versioned using ordinar ``` module "yavin" { - source = "git::https://github.com/poseidon/typhoon//google-cloud/fedora-coreos/kubernetes?ref=v1.18.4" + source = "git::https://github.com/poseidon/typhoon//google-cloud/fedora-coreos/kubernetes?ref=v1.18.5" ... } module "mercury" { - source = "git::https://github.com/poseidon/typhoon//bare-metal/container-linux/kubernetes?ref=v1.18.4" + source = "git::https://github.com/poseidon/typhoon//bare-metal/container-linux/kubernetes?ref=v1.18.5" ... } ``` diff --git a/google-cloud/container-linux/kubernetes/README.md b/google-cloud/container-linux/kubernetes/README.md index b2ee7f2bb..834e34454 100644 --- a/google-cloud/container-linux/kubernetes/README.md +++ b/google-cloud/container-linux/kubernetes/README.md @@ -11,7 +11,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features -* Kubernetes v1.18.4 (upstream) +* Kubernetes v1.18.5 (upstream) * Single or multi-master, [Calico](https://www.projectcalico.org/) or [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/) * Advanced features like [worker pools](https://typhoon.psdn.io/advanced/worker-pools/), [preemptible](https://typhoon.psdn.io/cl/google-cloud/#preemption) workers, and [snippets](https://typhoon.psdn.io/advanced/customization/#container-linux) customization diff --git a/google-cloud/container-linux/kubernetes/bootstrap.tf b/google-cloud/container-linux/kubernetes/bootstrap.tf index e0974bc05..7510c04bf 100644 --- a/google-cloud/container-linux/kubernetes/bootstrap.tf +++ b/google-cloud/container-linux/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=5043456b05b3c3caf5ef821fcf65c8ffb7afa316" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=5a7c963caf59740891df2aeae4b1561ccb3b9db6" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/google-cloud/container-linux/kubernetes/cl/controller.yaml b/google-cloud/container-linux/kubernetes/cl/controller.yaml index 31f861b97..084ed1be0 100644 --- a/google-cloud/container-linux/kubernetes/cl/controller.yaml +++ b/google-cloud/container-linux/kubernetes/cl/controller.yaml @@ -52,7 +52,7 @@ systemd: Description=Kubelet Wants=rpc-statd.service [Service] - Environment=KUBELET_IMAGE=docker://quay.io/poseidon/kubelet:v1.18.4 + Environment=KUBELET_IMAGE=docker://quay.io/poseidon/kubelet:v1.18.5 ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests ExecStartPre=/bin/mkdir -p /opt/cni/bin @@ -132,7 +132,7 @@ systemd: --volume script,kind=host,source=/opt/bootstrap/apply \ --mount volume=script,target=/apply \ --insecure-options=image \ - docker://quay.io/poseidon/kubelet:v1.18.4 \ + docker://quay.io/poseidon/kubelet:v1.18.5 \ --net=host \ --dns=host \ --exec=/apply diff --git a/google-cloud/container-linux/kubernetes/workers/cl/worker.yaml b/google-cloud/container-linux/kubernetes/workers/cl/worker.yaml index 94e1d68fd..c78f6a04e 100644 --- a/google-cloud/container-linux/kubernetes/workers/cl/worker.yaml +++ b/google-cloud/container-linux/kubernetes/workers/cl/worker.yaml @@ -25,7 +25,7 @@ systemd: Description=Kubelet Wants=rpc-statd.service [Service] - Environment=KUBELET_IMAGE=docker://quay.io/poseidon/kubelet:v1.18.4 + Environment=KUBELET_IMAGE=docker://quay.io/poseidon/kubelet:v1.18.5 ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests ExecStartPre=/bin/mkdir -p /opt/cni/bin @@ -127,7 +127,7 @@ storage: --volume config,kind=host,source=/etc/kubernetes \ --mount volume=config,target=/etc/kubernetes \ --insecure-options=image \ - docker://quay.io/poseidon/kubelet:v1.18.4 \ + docker://quay.io/poseidon/kubelet:v1.18.5 \ --net=host \ --dns=host \ --exec=/usr/local/bin/kubectl -- --kubeconfig=/etc/kubernetes/kubeconfig delete node $(hostname) diff --git a/google-cloud/fedora-coreos/kubernetes/README.md b/google-cloud/fedora-coreos/kubernetes/README.md index 6a3af1601..b78aa67ef 100644 --- a/google-cloud/fedora-coreos/kubernetes/README.md +++ b/google-cloud/fedora-coreos/kubernetes/README.md @@ -11,7 +11,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features -* Kubernetes v1.18.4 (upstream) +* Kubernetes v1.18.5 (upstream) * Single or multi-master, [Calico](https://www.projectcalico.org/) or [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/), SELinux enforcing * Advanced features like [worker pools](https://typhoon.psdn.io/advanced/worker-pools/), [preemptible](https://typhoon.psdn.io/fedora-coreos/google-cloud/#preemption) workers, and [snippets](https://typhoon.psdn.io/advanced/customization/) customization diff --git a/google-cloud/fedora-coreos/kubernetes/bootstrap.tf b/google-cloud/fedora-coreos/kubernetes/bootstrap.tf index e22491cba..ef7375637 100644 --- a/google-cloud/fedora-coreos/kubernetes/bootstrap.tf +++ b/google-cloud/fedora-coreos/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=5043456b05b3c3caf5ef821fcf65c8ffb7afa316" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=5a7c963caf59740891df2aeae4b1561ccb3b9db6" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/google-cloud/fedora-coreos/kubernetes/fcc/controller.yaml b/google-cloud/fedora-coreos/kubernetes/fcc/controller.yaml index 6d6c3745e..4acedbdce 100644 --- a/google-cloud/fedora-coreos/kubernetes/fcc/controller.yaml +++ b/google-cloud/fedora-coreos/kubernetes/fcc/controller.yaml @@ -54,7 +54,7 @@ systemd: Description=Kubelet (System Container) Wants=rpc-statd.service [Service] - Environment=KUBELET_IMAGE=quay.io/poseidon/kubelet:v1.18.4 + Environment=KUBELET_IMAGE=quay.io/poseidon/kubelet:v1.18.5 ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests ExecStartPre=/bin/mkdir -p /opt/cni/bin @@ -123,7 +123,7 @@ systemd: --volume /opt/bootstrap/assets:/assets:ro,Z \ --volume /opt/bootstrap/apply:/apply:ro,Z \ --entrypoint=/apply \ - quay.io/poseidon/kubelet:v1.18.4 + quay.io/poseidon/kubelet:v1.18.5 ExecStartPost=/bin/touch /opt/bootstrap/bootstrap.done ExecStartPost=-/usr/bin/podman stop bootstrap storage: diff --git a/google-cloud/fedora-coreos/kubernetes/workers/fcc/worker.yaml b/google-cloud/fedora-coreos/kubernetes/workers/fcc/worker.yaml index 147cea9b9..b29aa4f2c 100644 --- a/google-cloud/fedora-coreos/kubernetes/workers/fcc/worker.yaml +++ b/google-cloud/fedora-coreos/kubernetes/workers/fcc/worker.yaml @@ -24,7 +24,7 @@ systemd: Description=Kubelet (System Container) Wants=rpc-statd.service [Service] - Environment=KUBELET_IMAGE=quay.io/poseidon/kubelet:v1.18.4 + Environment=KUBELET_IMAGE=quay.io/poseidon/kubelet:v1.18.5 ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests ExecStartPre=/bin/mkdir -p /opt/cni/bin @@ -88,7 +88,7 @@ systemd: Type=oneshot RemainAfterExit=true ExecStart=/bin/true - ExecStop=/bin/bash -c '/usr/bin/podman run --volume /etc/kubernetes:/etc/kubernetes:ro,z --entrypoint /usr/local/bin/kubectl quay.io/poseidon/kubelet:v1.18.4 --kubeconfig=/etc/kubernetes/kubeconfig delete node $HOSTNAME' + ExecStop=/bin/bash -c '/usr/bin/podman run --volume /etc/kubernetes:/etc/kubernetes:ro,z --entrypoint /usr/local/bin/kubectl quay.io/poseidon/kubelet:v1.18.5 --kubeconfig=/etc/kubernetes/kubeconfig delete node $HOSTNAME' [Install] WantedBy=multi-user.target storage: From 81b6f54169119702c3cc6a3ecabca77f8646b444 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Sat, 27 Jun 2020 14:34:30 -0700 Subject: [PATCH 464/523] Update Prometheus from v2.19.1 to v2.19.2 * https://github.com/prometheus/prometheus/releases/tag/v2.19.2 --- CHANGES.md | 2 +- addons/prometheus/deployment.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 23dc7c24c..534be03f4 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -11,7 +11,7 @@ Notable changes between versions. #### Addons -* Update Prometheus from v2.19.0 to [v2.19.1](https://github.com/prometheus/prometheus/releases/tag/v2.19.1) +* Update Prometheus from v2.19.0 to [v2.19.2](https://github.com/prometheus/prometheus/releases/tag/v2.19.2) * Update Grafana from v7.0.3 to [v7.0.4](https://github.com/grafana/grafana/releases/tag/v7.0.4) ## v1.18.4 diff --git a/addons/prometheus/deployment.yaml b/addons/prometheus/deployment.yaml index 48a798d53..d9077b27e 100644 --- a/addons/prometheus/deployment.yaml +++ b/addons/prometheus/deployment.yaml @@ -20,7 +20,7 @@ spec: serviceAccountName: prometheus containers: - name: prometheus - image: quay.io/prometheus/prometheus:v2.19.1 + image: quay.io/prometheus/prometheus:v2.19.2 args: - --web.listen-address=0.0.0.0:9090 - --config.file=/etc/prometheus/prometheus.yaml From 9dcf35e393ee5829e0aeec0af9cf5b46491e7773 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Sat, 27 Jun 2020 14:44:18 -0700 Subject: [PATCH 465/523] Update recommended Terraform provider versions * Sync Terraform provider plugin versions with those used internally --- docs/fedora-coreos/aws.md | 2 +- docs/fedora-coreos/azure.md | 2 +- docs/fedora-coreos/google-cloud.md | 2 +- docs/flatcar-linux/aws.md | 2 +- docs/flatcar-linux/azure.md | 2 +- docs/flatcar-linux/google-cloud.md | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/docs/fedora-coreos/aws.md b/docs/fedora-coreos/aws.md index 9f2e4841d..d2455f779 100644 --- a/docs/fedora-coreos/aws.md +++ b/docs/fedora-coreos/aws.md @@ -49,7 +49,7 @@ Configure the AWS provider to use your access key credentials in a `providers.tf ```tf provider "aws" { - version = "2.66.0" + version = "2.68.0" region = "eu-central-1" shared_credentials_file = "/home/user/.config/aws/credentials" } diff --git a/docs/fedora-coreos/azure.md b/docs/fedora-coreos/azure.md index 93eecdda1..2c94212a1 100644 --- a/docs/fedora-coreos/azure.md +++ b/docs/fedora-coreos/azure.md @@ -47,7 +47,7 @@ Configure the Azure provider in a `providers.tf` file. ```tf provider "azurerm" { - version = "2.14.0" + version = "2.16.0" } provider "ct" { diff --git a/docs/fedora-coreos/google-cloud.md b/docs/fedora-coreos/google-cloud.md index dfb192727..515b6cdca 100644 --- a/docs/fedora-coreos/google-cloud.md +++ b/docs/fedora-coreos/google-cloud.md @@ -49,7 +49,7 @@ Configure the Google Cloud provider to use your service account key, project-id, ```tf provider "google" { - version = "3.26.0" + version = "3.27.0" project = "project-id" region = "us-central1" credentials = file("~/.config/google-cloud/terraform.json") diff --git a/docs/flatcar-linux/aws.md b/docs/flatcar-linux/aws.md index 25d04fa0f..a68e0c814 100644 --- a/docs/flatcar-linux/aws.md +++ b/docs/flatcar-linux/aws.md @@ -49,7 +49,7 @@ Configure the AWS provider to use your access key credentials in a `providers.tf ```tf provider "aws" { - version = "2.66.0" + version = "2.68.0" region = "eu-central-1" shared_credentials_file = "/home/user/.config/aws/credentials" } diff --git a/docs/flatcar-linux/azure.md b/docs/flatcar-linux/azure.md index c0b6f9cf2..3b2b8d014 100644 --- a/docs/flatcar-linux/azure.md +++ b/docs/flatcar-linux/azure.md @@ -47,7 +47,7 @@ Configure the Azure provider in a `providers.tf` file. ```tf provider "azurerm" { - version = "2.14.0" + version = "2.16.0" } provider "ct" { diff --git a/docs/flatcar-linux/google-cloud.md b/docs/flatcar-linux/google-cloud.md index d10755788..0efa4c856 100644 --- a/docs/flatcar-linux/google-cloud.md +++ b/docs/flatcar-linux/google-cloud.md @@ -49,7 +49,7 @@ Configure the Google Cloud provider to use your service account key, project-id, ```tf provider "google" { - version = "3.26.0" + version = "3.27.0" project = "project-id" region = "us-central1" credentials = file("~/.config/google-cloud/terraform.json") From 21178868db7997b4d63e601a1a6f4dc4db346621 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Sat, 27 Jun 2020 14:53:58 -0700 Subject: [PATCH 466/523] Revert "Update Prometheus from v2.19.1 to v2.19.2" * Prometheus has not published the v1.19.2 * This reverts commit 81b6f54169119702c3cc6a3ecabca77f8646b444. --- CHANGES.md | 2 +- addons/prometheus/deployment.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 534be03f4..23dc7c24c 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -11,7 +11,7 @@ Notable changes between versions. #### Addons -* Update Prometheus from v2.19.0 to [v2.19.2](https://github.com/prometheus/prometheus/releases/tag/v2.19.2) +* Update Prometheus from v2.19.0 to [v2.19.1](https://github.com/prometheus/prometheus/releases/tag/v2.19.1) * Update Grafana from v7.0.3 to [v7.0.4](https://github.com/grafana/grafana/releases/tag/v7.0.4) ## v1.18.4 diff --git a/addons/prometheus/deployment.yaml b/addons/prometheus/deployment.yaml index d9077b27e..48a798d53 100644 --- a/addons/prometheus/deployment.yaml +++ b/addons/prometheus/deployment.yaml @@ -20,7 +20,7 @@ spec: serviceAccountName: prometheus containers: - name: prometheus - image: quay.io/prometheus/prometheus:v2.19.2 + image: quay.io/prometheus/prometheus:v2.19.1 args: - --web.listen-address=0.0.0.0:9090 - --config.file=/etc/prometheus/prometheus.yaml From 7c6ab21b9477333bf3c643117450b8906b9568ae Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Sun, 28 Jun 2020 23:12:54 -0700 Subject: [PATCH 467/523] Isolate each DigitalOcean cluster in its own VPC * DigitalOcean introduced Virtual Private Cloud (VPC) support to match other clouds and enhance the prior "private networking" feature. Before, droplet's belonging to different clusters (but residing in the same region) could reach one another (although Typhoon firewall rules prohibit this). Now, droplets in a VPC reside in their own network * https://www.digitalocean.com/docs/networking/vpc/ * Create droplet instances in a VPC per cluster. This matches the design of Typhoon AWS, Azure, and GCP. * Require `terraform-provider-digitalocean` v1.16.0+ (action required) * Output `vpc_id` for use with an attached DigitalOcean loadbalancer --- CHANGES.md | 7 +++++++ digital-ocean/container-linux/kubernetes/controllers.tf | 5 +++-- digital-ocean/container-linux/kubernetes/network.tf | 7 +++++++ digital-ocean/container-linux/kubernetes/outputs.tf | 9 +++++++++ digital-ocean/container-linux/kubernetes/versions.tf | 2 +- digital-ocean/container-linux/kubernetes/workers.tf | 5 +++-- digital-ocean/fedora-coreos/kubernetes/controllers.tf | 5 +++-- digital-ocean/fedora-coreos/kubernetes/network.tf | 7 +++++++ digital-ocean/fedora-coreos/kubernetes/outputs.tf | 8 ++++++++ digital-ocean/fedora-coreos/kubernetes/versions.tf | 2 +- digital-ocean/fedora-coreos/kubernetes/workers.tf | 5 +++-- docs/architecture/digitalocean.md | 1 + 12 files changed, 53 insertions(+), 10 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 23dc7c24c..dd9af6ae2 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -9,6 +9,13 @@ Notable changes between versions. * Set `networking` to "cilium" to enable * Update Calico from v3.14.1 to [v3.15.0](https://docs.projectcalico.org/v3.15/release-notes/) +#### DigitalOcean + +* Isolate each cluster in an independent DigitalOcean VPC ([#776](https://github.com/poseidon/typhoon/pull/776)) + * Create droplets in a VPC per cluster (matches Typhoon AWS, Azure, and GCP) + * Require `terraform-provider-digitalocean` v1.16.0+ (action required) + * Output `vpc_id` for use with an attached DigitalOcean [loadbalancer](https://github.com/poseidon/typhoon/blob/v1.18.5/docs/architecture/digitalocean.md#custom-load-balancer) + #### Addons * Update Prometheus from v2.19.0 to [v2.19.1](https://github.com/prometheus/prometheus/releases/tag/v2.19.1) diff --git a/digital-ocean/container-linux/kubernetes/controllers.tf b/digital-ocean/container-linux/kubernetes/controllers.tf index 9efe84270..dcb6f0dda 100644 --- a/digital-ocean/container-linux/kubernetes/controllers.tf +++ b/digital-ocean/container-linux/kubernetes/controllers.tf @@ -46,9 +46,10 @@ resource "digitalocean_droplet" "controllers" { size = var.controller_type # network - # only official DigitalOcean images support IPv6 - ipv6 = local.is_official_image private_networking = true + vpc_uuid = digitalocean_vpc.network.id + # TODO: Only official DigitalOcean images support IPv6 + ipv6 = false user_data = data.ct_config.controller-ignitions.*.rendered[count.index] ssh_keys = var.ssh_fingerprints diff --git a/digital-ocean/container-linux/kubernetes/network.tf b/digital-ocean/container-linux/kubernetes/network.tf index 4be6827aa..0d3438bbf 100644 --- a/digital-ocean/container-linux/kubernetes/network.tf +++ b/digital-ocean/container-linux/kubernetes/network.tf @@ -1,3 +1,10 @@ +# Network VPC +resource "digitalocean_vpc" "network" { + name = var.cluster_name + region = var.region + description = "Network for ${var.cluster_name} cluster" +} + resource "digitalocean_firewall" "rules" { name = var.cluster_name diff --git a/digital-ocean/container-linux/kubernetes/outputs.tf b/digital-ocean/container-linux/kubernetes/outputs.tf index 429893c58..4cadc8860 100644 --- a/digital-ocean/container-linux/kubernetes/outputs.tf +++ b/digital-ocean/container-linux/kubernetes/outputs.tf @@ -2,6 +2,8 @@ output "kubeconfig-admin" { value = module.bootstrap.kubeconfig-admin } +# Outputs for Kubernetes Ingress + output "controllers_dns" { value = digitalocean_record.controllers[0].fqdn } @@ -45,3 +47,10 @@ output "worker_tag" { value = digitalocean_tag.workers.name } +# Outputs for custom load balancing + +output "vpc_id" { + description = "ID of the cluster VPC" + value = digitalocean_vpc.network.id +} + diff --git a/digital-ocean/container-linux/kubernetes/versions.tf b/digital-ocean/container-linux/kubernetes/versions.tf index 38b32c8d1..14ab23b30 100644 --- a/digital-ocean/container-linux/kubernetes/versions.tf +++ b/digital-ocean/container-linux/kubernetes/versions.tf @@ -3,7 +3,7 @@ terraform { required_version = "~> 0.12.6" required_providers { - digitalocean = "~> 1.3" + digitalocean = "~> 1.16" ct = "~> 0.4" template = "~> 2.1" null = "~> 2.1" diff --git a/digital-ocean/container-linux/kubernetes/workers.tf b/digital-ocean/container-linux/kubernetes/workers.tf index cdd94ebae..c89898090 100644 --- a/digital-ocean/container-linux/kubernetes/workers.tf +++ b/digital-ocean/container-linux/kubernetes/workers.tf @@ -35,9 +35,10 @@ resource "digitalocean_droplet" "workers" { size = var.worker_type # network - # only official DigitalOcean images support IPv6 - ipv6 = local.is_official_image private_networking = true + vpc_uuid = digitalocean_vpc.network.id + # only official DigitalOcean images support IPv6 + ipv6 = local.is_official_image user_data = data.ct_config.worker-ignition.rendered ssh_keys = var.ssh_fingerprints diff --git a/digital-ocean/fedora-coreos/kubernetes/controllers.tf b/digital-ocean/fedora-coreos/kubernetes/controllers.tf index 2c0964c93..95d43bace 100644 --- a/digital-ocean/fedora-coreos/kubernetes/controllers.tf +++ b/digital-ocean/fedora-coreos/kubernetes/controllers.tf @@ -41,9 +41,10 @@ resource "digitalocean_droplet" "controllers" { size = var.controller_type # network - # TODO: Only official DigitalOcean images support IPv6 - ipv6 = false private_networking = true + vpc_uuid = digitalocean_vpc.network.id + # TODO: Only official DigitalOcean images support IPv6 + ipv6 = false user_data = data.ct_config.controller-ignitions.*.rendered[count.index] ssh_keys = var.ssh_fingerprints diff --git a/digital-ocean/fedora-coreos/kubernetes/network.tf b/digital-ocean/fedora-coreos/kubernetes/network.tf index 4be6827aa..0d3438bbf 100644 --- a/digital-ocean/fedora-coreos/kubernetes/network.tf +++ b/digital-ocean/fedora-coreos/kubernetes/network.tf @@ -1,3 +1,10 @@ +# Network VPC +resource "digitalocean_vpc" "network" { + name = var.cluster_name + region = var.region + description = "Network for ${var.cluster_name} cluster" +} + resource "digitalocean_firewall" "rules" { name = var.cluster_name diff --git a/digital-ocean/fedora-coreos/kubernetes/outputs.tf b/digital-ocean/fedora-coreos/kubernetes/outputs.tf index 429893c58..616eaf48a 100644 --- a/digital-ocean/fedora-coreos/kubernetes/outputs.tf +++ b/digital-ocean/fedora-coreos/kubernetes/outputs.tf @@ -2,6 +2,8 @@ output "kubeconfig-admin" { value = module.bootstrap.kubeconfig-admin } +# Outputs for Kubernetes Ingress + output "controllers_dns" { value = digitalocean_record.controllers[0].fqdn } @@ -45,3 +47,9 @@ output "worker_tag" { value = digitalocean_tag.workers.name } +# Outputs for custom load balancing + +output "vpc_id" { + description = "ID of the cluster VPC" + value = digitalocean_vpc.network.id +} diff --git a/digital-ocean/fedora-coreos/kubernetes/versions.tf b/digital-ocean/fedora-coreos/kubernetes/versions.tf index 38b32c8d1..14ab23b30 100644 --- a/digital-ocean/fedora-coreos/kubernetes/versions.tf +++ b/digital-ocean/fedora-coreos/kubernetes/versions.tf @@ -3,7 +3,7 @@ terraform { required_version = "~> 0.12.6" required_providers { - digitalocean = "~> 1.3" + digitalocean = "~> 1.16" ct = "~> 0.4" template = "~> 2.1" null = "~> 2.1" diff --git a/digital-ocean/fedora-coreos/kubernetes/workers.tf b/digital-ocean/fedora-coreos/kubernetes/workers.tf index c1a087ae6..f4db41af0 100644 --- a/digital-ocean/fedora-coreos/kubernetes/workers.tf +++ b/digital-ocean/fedora-coreos/kubernetes/workers.tf @@ -37,9 +37,10 @@ resource "digitalocean_droplet" "workers" { size = var.worker_type # network - # TODO: Only official DigitalOcean images support IPv6 - ipv6 = false private_networking = true + vpc_uuid = digitalocean_vpc.network.id + # TODO: Only official DigitalOcean images support IPv6 + ipv6 = false user_data = data.ct_config.worker-ignition.rendered ssh_keys = var.ssh_fingerprints diff --git a/docs/architecture/digitalocean.md b/docs/architecture/digitalocean.md index ba8a21dec..5bb045563 100644 --- a/docs/architecture/digitalocean.md +++ b/docs/architecture/digitalocean.md @@ -30,6 +30,7 @@ Add a DigitalOcean load balancer to distribute IPv4 TCP traffic (HTTP/HTTPS Ingr resource "digitalocean_loadbalancer" "ingress" { name = "ingress" region = "fra1" + vpc_uuid = module.nemo.vpc_id droplet_tag = module.nemo.worker_tag healthcheck { From 430d139a5b287b22e2f497e240d03dd1492b2fdd Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Mon, 29 Jun 2020 22:57:11 -0700 Subject: [PATCH 468/523] Remove os_image variable on Google Cloud Fedora CoreOS * In v1.18.3, the `os_stream` variable was added to select a Fedora CoreOS image stream (stable, testing, next) on AWS and Google Cloud (which publish official streams) * Remove `os_image` variable deprecated in v1.18.3. Manually uploaded images are no longer needed --- CHANGES.md | 9 ++++++++- google-cloud/fedora-coreos/kubernetes/controllers.tf | 2 +- google-cloud/fedora-coreos/kubernetes/variables.tf | 7 ------- google-cloud/fedora-coreos/kubernetes/workers.tf | 1 - .../fedora-coreos/kubernetes/workers/variables.tf | 7 ------- google-cloud/fedora-coreos/kubernetes/workers/workers.tf | 2 +- 6 files changed, 10 insertions(+), 18 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index dd9af6ae2..414d07bb1 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -16,6 +16,13 @@ Notable changes between versions. * Require `terraform-provider-digitalocean` v1.16.0+ (action required) * Output `vpc_id` for use with an attached DigitalOcean [loadbalancer](https://github.com/poseidon/typhoon/blob/v1.18.5/docs/architecture/digitalocean.md#custom-load-balancer) +### Fedora CoreOS + +#### Google + +* Remove `os_image` variable (deprecated in v1.18.3) + * Use `os_stream` to select a Fedora CoreOS image stream + #### Addons * Update Prometheus from v2.19.0 to [v2.19.1](https://github.com/prometheus/prometheus/releases/tag/v2.19.1) @@ -105,7 +112,7 @@ Notable changes between versions. #### Google -* Support Fedora CoreOS [image streams](https://docs.fedoraproject.org/en-US/fedora-coreos/update-streams/) ([#723](https://github.com/poseidon/typhoon/pull/722)) +* Support Fedora CoreOS [image streams](https://docs.fedoraproject.org/en-US/fedora-coreos/update-streams/) ([#723](https://github.com/poseidon/typhoon/pull/723)) * Add `os_stream` variable to set the stream to `stable` (default), `testing`, or `next` * Deprecate `os_image` variable. Manual image uploads are no longer needed diff --git a/google-cloud/fedora-coreos/kubernetes/controllers.tf b/google-cloud/fedora-coreos/kubernetes/controllers.tf index 3dc96a3e9..cbf1b7f8c 100644 --- a/google-cloud/fedora-coreos/kubernetes/controllers.tf +++ b/google-cloud/fedora-coreos/kubernetes/controllers.tf @@ -42,7 +42,7 @@ resource "google_compute_instance" "controllers" { auto_delete = true initialize_params { - image = var.os_image == "" ? data.google_compute_image.fedora-coreos.self_link : var.os_image + image = data.google_compute_image.fedora-coreos.self_link size = var.disk_size } } diff --git a/google-cloud/fedora-coreos/kubernetes/variables.tf b/google-cloud/fedora-coreos/kubernetes/variables.tf index 927be9d0d..74b59d41b 100644 --- a/google-cloud/fedora-coreos/kubernetes/variables.tf +++ b/google-cloud/fedora-coreos/kubernetes/variables.tf @@ -52,13 +52,6 @@ variable "os_stream" { default = "stable" } -# Deprecated -variable "os_image" { - type = string - description = "Fedora CoreOS image for compute instances (e.g. fedora-coreos)" - default = "" -} - variable "disk_size" { type = number description = "Size of the disk in GB" diff --git a/google-cloud/fedora-coreos/kubernetes/workers.tf b/google-cloud/fedora-coreos/kubernetes/workers.tf index 5afec0826..d35db25f5 100644 --- a/google-cloud/fedora-coreos/kubernetes/workers.tf +++ b/google-cloud/fedora-coreos/kubernetes/workers.tf @@ -9,7 +9,6 @@ module "workers" { worker_count = var.worker_count machine_type = var.worker_type os_stream = var.os_stream - os_image = var.os_image disk_size = var.disk_size preemptible = var.worker_preemptible diff --git a/google-cloud/fedora-coreos/kubernetes/workers/variables.tf b/google-cloud/fedora-coreos/kubernetes/workers/variables.tf index 4c49badf5..b3802388b 100644 --- a/google-cloud/fedora-coreos/kubernetes/workers/variables.tf +++ b/google-cloud/fedora-coreos/kubernetes/workers/variables.tf @@ -40,13 +40,6 @@ variable "os_stream" { default = "stable" } -# Deprecated -variable "os_image" { - type = string - description = "Fedora CoreOS image for compute instances (e.g. fedora-coreos)" - default = "" -} - variable "disk_size" { type = number description = "Size of the disk in GB" diff --git a/google-cloud/fedora-coreos/kubernetes/workers/workers.tf b/google-cloud/fedora-coreos/kubernetes/workers/workers.tf index 78367c673..3c36b1aa9 100644 --- a/google-cloud/fedora-coreos/kubernetes/workers/workers.tf +++ b/google-cloud/fedora-coreos/kubernetes/workers/workers.tf @@ -43,7 +43,7 @@ resource "google_compute_instance_template" "worker" { disk { auto_delete = true boot = true - source_image = var.os_image == "" ? data.google_compute_image.fedora-coreos.self_link : var.os_image + source_image = data.google_compute_image.fedora-coreos.self_link disk_size_gb = var.disk_size } From 0ba2c1a4dace6efaf9e1bc035ae9cff0ca20ca55 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Mon, 29 Jun 2020 23:04:54 -0700 Subject: [PATCH 469/523] Fix terraform fmt in firewall rules --- azure/container-linux/kubernetes/security.tf | 4 ++-- azure/fedora-coreos/kubernetes/security.tf | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/azure/container-linux/kubernetes/security.tf b/azure/container-linux/kubernetes/security.tf index c31d00145..c258ec2d6 100644 --- a/azure/container-linux/kubernetes/security.tf +++ b/azure/container-linux/kubernetes/security.tf @@ -117,7 +117,7 @@ resource "azurerm_network_security_rule" "controller-apiserver" { resource "azurerm_network_security_rule" "controller-cilium-health" { resource_group_name = azurerm_resource_group.cluster.name - count = var.networking == "cilium" ? 1 : 0 + count = var.networking == "cilium" ? 1 : 0 name = "allow-cilium-health" network_security_group_name = azurerm_network_security_group.controller.name @@ -299,7 +299,7 @@ resource "azurerm_network_security_rule" "worker-https" { resource "azurerm_network_security_rule" "worker-cilium-health" { resource_group_name = azurerm_resource_group.cluster.name - count = var.networking == "cilium" ? 1 : 0 + count = var.networking == "cilium" ? 1 : 0 name = "allow-cilium-health" network_security_group_name = azurerm_network_security_group.worker.name diff --git a/azure/fedora-coreos/kubernetes/security.tf b/azure/fedora-coreos/kubernetes/security.tf index c31d00145..c258ec2d6 100644 --- a/azure/fedora-coreos/kubernetes/security.tf +++ b/azure/fedora-coreos/kubernetes/security.tf @@ -117,7 +117,7 @@ resource "azurerm_network_security_rule" "controller-apiserver" { resource "azurerm_network_security_rule" "controller-cilium-health" { resource_group_name = azurerm_resource_group.cluster.name - count = var.networking == "cilium" ? 1 : 0 + count = var.networking == "cilium" ? 1 : 0 name = "allow-cilium-health" network_security_group_name = azurerm_network_security_group.controller.name @@ -299,7 +299,7 @@ resource "azurerm_network_security_rule" "worker-https" { resource "azurerm_network_security_rule" "worker-cilium-health" { resource_group_name = azurerm_resource_group.cluster.name - count = var.networking == "cilium" ? 1 : 0 + count = var.networking == "cilium" ? 1 : 0 name = "allow-cilium-health" network_security_group_name = azurerm_network_security_group.worker.name From 32886cfba159a5d4c51df861f9118939587c3f96 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Mon, 29 Jun 2020 23:09:11 -0700 Subject: [PATCH 470/523] Promote Fedora CoreOS on Google Cloud to stable status --- CHANGES.md | 1 + README.md | 2 +- docs/index.md | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 414d07bb1..9733ed23b 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -20,6 +20,7 @@ Notable changes between versions. #### Google +* Promote Fedora CoreOS to stable * Remove `os_image` variable (deprecated in v1.18.3) * Use `os_stream` to select a Fedora CoreOS image stream diff --git a/README.md b/README.md index bf19cf34a..14803ea29 100644 --- a/README.md +++ b/README.md @@ -29,7 +29,7 @@ Typhoon is available for [Fedora CoreOS](https://getfedora.org/coreos/). | Azure | Fedora CoreOS | [azure/fedora-coreos/kubernetes](azure/fedora-coreos/kubernetes) | alpha | | Bare-Metal | Fedora CoreOS | [bare-metal/fedora-coreos/kubernetes](bare-metal/fedora-coreos/kubernetes) | beta | | DigitalOcean | Fedora CoreOS | [digital-ocean/fedora-coreos/kubernetes](digital-ocean/fedora-coreos/kubernetes) | beta | -| Google Cloud | Fedora CoreOS | [google-cloud/fedora-coreos/kubernetes](google-cloud/fedora-coreos/kubernetes) | beta | +| Google Cloud | Fedora CoreOS | [google-cloud/fedora-coreos/kubernetes](google-cloud/fedora-coreos/kubernetes) | stable | Typhoon is available for [Flatcar Linux](https://www.flatcar-linux.org/releases/). diff --git a/docs/index.md b/docs/index.md index 0b682ebef..9dde5cc92 100644 --- a/docs/index.md +++ b/docs/index.md @@ -29,7 +29,7 @@ Typhoon is available for [Fedora CoreOS](https://getfedora.org/coreos/). | Azure | Fedora CoreOS | [azure/fedora-coreos/kubernetes](fedora-coreos/azure.md) | alpha | | Bare-Metal | Fedora CoreOS | [bare-metal/fedora-coreos/kubernetes](fedora-coreos/bare-metal.md) | beta | | DigitalOcean | Fedora CoreOS | [digital-ocean/fedora-coreos/kubernetes](fedora-coreos/digitalocean.md) | beta | -| Google Cloud | Fedora CoreOS | [google-cloud/fedora-coreos/kubernetes](google-cloud/fedora-coreos/kubernetes) | beta | +| Google Cloud | Fedora CoreOS | [google-cloud/fedora-coreos/kubernetes](google-cloud/fedora-coreos/kubernetes) | stable | Typhoon is available for [Flatcar Linux](https://www.flatcar-linux.org/releases/). From df3f40bcce88936df71f38342d66c444a2c90f11 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Tue, 30 Jun 2020 01:16:24 -0700 Subject: [PATCH 471/523] Allow using Flatcar Linux edge on Azure * Set Kubelet cgroup driver to systemd when Flatcar Linux edge is chosen Note: Typhoon module status assumes use of the stable variant of an OS channel/stream. Its possible to use earlier variants and those are sometimes tested or developed against, but stable is the recommendation --- CHANGES.md | 10 ++++++++-- azure/container-linux/kubernetes/cl/controller.yaml | 2 ++ azure/container-linux/kubernetes/controllers.tf | 1 + .../container-linux/kubernetes/workers/cl/worker.yaml | 2 ++ azure/container-linux/kubernetes/workers/workers.tf | 1 + 5 files changed, 14 insertions(+), 2 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 9733ed23b..73afe4602 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -18,12 +18,18 @@ Notable changes between versions. ### Fedora CoreOS -#### Google +#### Google Cloud * Promote Fedora CoreOS to stable -* Remove `os_image` variable (deprecated in v1.18.3) +* Remove `os_image` variable deprecated in v1.18.3 ([#777](https://github.com/poseidon/typhoon/pull/777)) * Use `os_stream` to select a Fedora CoreOS image stream +### Flatcar Linux + +#### Azure + +* Allow using Flatcar Linux Edge by setting `os_image` to "flatcar-edge" ([#778](https://github.com/poseidon/typhoon/pull/778)) + #### Addons * Update Prometheus from v2.19.0 to [v2.19.1](https://github.com/prometheus/prometheus/releases/tag/v2.19.1) diff --git a/azure/container-linux/kubernetes/cl/controller.yaml b/azure/container-linux/kubernetes/cl/controller.yaml index db71d8a80..ded14bf5d 100644 --- a/azure/container-linux/kubernetes/cl/controller.yaml +++ b/azure/container-linux/kubernetes/cl/controller.yaml @@ -53,6 +53,7 @@ systemd: Wants=rpc-statd.service [Service] Environment=KUBELET_IMAGE=docker://quay.io/poseidon/kubelet:v1.18.5 + Environment=KUBELET_CGROUP_DRIVER=${cgroup_driver} ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests ExecStartPre=/bin/mkdir -p /opt/cni/bin @@ -96,6 +97,7 @@ systemd: --authentication-token-webhook \ --authorization-mode=Webhook \ --bootstrap-kubeconfig=/etc/kubernetes/kubeconfig \ + --cgroup-driver=$${KUBELET_CGROUP_DRIVER} \ --client-ca-file=/etc/kubernetes/ca.crt \ --cluster_dns=${cluster_dns_service_ip} \ --cluster_domain=${cluster_domain_suffix} \ diff --git a/azure/container-linux/kubernetes/controllers.tf b/azure/container-linux/kubernetes/controllers.tf index 876ae7ca1..38a79b983 100644 --- a/azure/container-linux/kubernetes/controllers.tf +++ b/azure/container-linux/kubernetes/controllers.tf @@ -157,6 +157,7 @@ data "template_file" "controller-configs" { etcd_domain = "${var.cluster_name}-etcd${count.index}.${var.dns_zone}" # etcd0=https://cluster-etcd0.example.com,etcd1=https://cluster-etcd1.example.com,... etcd_initial_cluster = join(",", data.template_file.etcds.*.rendered) + cgroup_driver = local.flavor == "flatcar" && local.channel == "edge" ? "systemd" : "cgroupfs" kubeconfig = indent(10, module.bootstrap.kubeconfig-kubelet) ssh_authorized_key = var.ssh_authorized_key cluster_dns_service_ip = cidrhost(var.service_cidr, 10) diff --git a/azure/container-linux/kubernetes/workers/cl/worker.yaml b/azure/container-linux/kubernetes/workers/cl/worker.yaml index 549fcffe5..6817232f6 100644 --- a/azure/container-linux/kubernetes/workers/cl/worker.yaml +++ b/azure/container-linux/kubernetes/workers/cl/worker.yaml @@ -26,6 +26,7 @@ systemd: Wants=rpc-statd.service [Service] Environment=KUBELET_IMAGE=docker://quay.io/poseidon/kubelet:v1.18.5 + Environment=KUBELET_CGROUP_DRIVER=${cgroup_driver} ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests ExecStartPre=/bin/mkdir -p /opt/cni/bin @@ -69,6 +70,7 @@ systemd: --authentication-token-webhook \ --authorization-mode=Webhook \ --bootstrap-kubeconfig=/etc/kubernetes/kubeconfig \ + --cgroup-driver=$${KUBELET_CGROUP_DRIVER} \ --client-ca-file=/etc/kubernetes/ca.crt \ --cluster_dns=${cluster_dns_service_ip} \ --cluster_domain=${cluster_domain_suffix} \ diff --git a/azure/container-linux/kubernetes/workers/workers.tf b/azure/container-linux/kubernetes/workers/workers.tf index 7e8f008d2..9070051d8 100644 --- a/azure/container-linux/kubernetes/workers/workers.tf +++ b/azure/container-linux/kubernetes/workers/workers.tf @@ -111,6 +111,7 @@ data "template_file" "worker-config" { ssh_authorized_key = var.ssh_authorized_key cluster_dns_service_ip = cidrhost(var.service_cidr, 10) cluster_domain_suffix = var.cluster_domain_suffix + cgroup_driver = local.flavor == "flatcar" && local.channel == "edge" ? "systemd" : "cgroupfs" node_labels = join(",", var.node_labels) } } From 257a49ce3709394a67a6eac0e5255a8dde9c53ba Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Tue, 30 Jun 2020 01:30:18 -0700 Subject: [PATCH 472/523] Remove CoreOS Container Linux image names from docs * Remove coreos-stable, coreos-beta, and coreos-alpha channel references from docs * CoreOS Container Linux is end of life (see changelog) --- docs/advanced/worker-pools.md | 7 ++++--- docs/fedora-coreos/google-cloud.md | 2 +- docs/flatcar-linux/aws.md | 2 +- docs/flatcar-linux/azure.md | 2 +- docs/flatcar-linux/bare-metal.md | 2 +- docs/flatcar-linux/digitalocean.md | 2 +- docs/flatcar-linux/google-cloud.md | 2 +- 7 files changed, 10 insertions(+), 9 deletions(-) diff --git a/docs/advanced/worker-pools.md b/docs/advanced/worker-pools.md index 4f880ed0c..f9b72594a 100644 --- a/docs/advanced/worker-pools.md +++ b/docs/advanced/worker-pools.md @@ -65,7 +65,8 @@ The AWS internal `workers` module supports a number of [variables](https://githu |:-----|:------------|:--------|:--------| | worker_count | Number of instances | 1 | 3 | | instance_type | EC2 instance type | "t3.small" | "t3.medium" | -| os_image | AMI channel for a Container Linux derivative | "flatcar-stable" | flatcar-stable, flatcar-beta, flatcar-alph, coreos-stable, coreos-beta, coreos-alpha | +| os_image | AMI channel for a Container Linux derivative | "flatcar-stable" | flatcar-stable, flatcar-beta, flatcar-alph, flatcar-edge | +| os_stream | Fedora CoreOS stream for compute instances | "stable" | "testing", "next" | | disk_size | Size of the EBS volume in GB | 40 | 100 | | disk_type | Type of the EBS volume | "gp2" | standard, gp2, io1 | | disk_iops | IOPS of the EBS volume | 0 (i.e. auto) | 400 | @@ -134,7 +135,7 @@ The Azure internal `workers` module supports a number of [variables](https://git |:-----|:------------|:--------|:--------| | worker_count | Number of instances | 1 | 3 | | vm_type | Machine type for instances | "Standard_DS1_v2" | See below | -| os_image | Channel for a Container Linux derivative | "flatcar-stable" | flatcar-stable, flatcar-beta, flatcar-alpha, flatcar-edge, coreos-stable, coreos-beta, coreos-alpha | +| os_image | Channel for a Container Linux derivative | "flatcar-stable" | flatcar-stable, flatcar-beta, flatcar-alpha, flatcar-edge | | priority | Set priority to Spot to use reduced cost surplus capacity, with the tradeoff that instances can be deallocated at any time | "Regular" | "Spot" | | snippets | Container Linux Config snippets | [] | [examples](/advanced/customization/) | | service_cidr | CIDR IPv4 range to assign to Kubernetes services | "10.3.0.0/16" | "10.3.0.0/24" | @@ -199,7 +200,7 @@ The Google Cloud internal `workers` module supports a number of [variables](http | region | Region for the worker pool instances. May differ from the cluster's region | "europe-west2" | | network | Must be set to `network_name` output by cluster | module.cluster.network_name | | kubeconfig | Must be set to `kubeconfig` output by cluster | module.cluster.kubeconfig | -| os_image | Container Linux image for compute instances | "fedora-coreos-or-flatcar-image", coreos-stable, coreos-beta, coreos-alpha | +| os_image | Container Linux image for compute instances | "uploaded-flatcar-image" | | ssh_authorized_key | SSH public key for user 'core' | "ssh-rsa AAAAB3NZ..." | Check the list of regions [docs](https://cloud.google.com/compute/docs/regions-zones/regions-zones) or with `gcloud compute regions list`. diff --git a/docs/fedora-coreos/google-cloud.md b/docs/fedora-coreos/google-cloud.md index 515b6cdca..414be8027 100644 --- a/docs/fedora-coreos/google-cloud.md +++ b/docs/fedora-coreos/google-cloud.md @@ -213,7 +213,7 @@ resource "google_dns_managed_zone" "zone-for-clusters" { | worker_count | Number of workers | 1 | 3 | | controller_type | Machine type for controllers | "n1-standard-1" | See below | | worker_type | Machine type for workers | "n1-standard-1" | See below | -| os_stream | Fedora CoreOS stream for compute instances | "stable" | "testing", "next" | +| os_stream | Fedora CoreOS stream for compute instances | "stable" | "stable", "testing", "next" | | disk_size | Size of the disk in GB | 40 | 100 | | worker_preemptible | If enabled, Compute Engine will terminate workers randomly within 24 hours | false | true | | controller_snippets | Controller Fedora CoreOS Config snippets | [] | [examples](/advanced/customization/) | diff --git a/docs/flatcar-linux/aws.md b/docs/flatcar-linux/aws.md index a68e0c814..c2c3bf7ff 100644 --- a/docs/flatcar-linux/aws.md +++ b/docs/flatcar-linux/aws.md @@ -208,7 +208,7 @@ Reference the DNS zone id with `aws_route53_zone.zone-for-clusters.zone_id`. | worker_count | Number of workers | 1 | 3 | | controller_type | EC2 instance type for controllers | "t3.small" | See below | | worker_type | EC2 instance type for workers | "t3.small" | See below | -| os_image | AMI channel for a Container Linux derivative | "flatcar-stable" | coreos-stable, coreos-beta, coreos-alpha, flatcar-stable, flatcar-beta, flatcar-alpha, flatcar-edge | +| os_image | AMI channel for a Container Linux derivative | "flatcar-stable" | flatcar-stable, flatcar-beta, flatcar-alpha, flatcar-edge | | disk_size | Size of the EBS volume in GB | 40 | 100 | | disk_type | Type of the EBS volume | "gp2" | standard, gp2, io1 | | disk_iops | IOPS of the EBS volume | 0 (i.e. auto) | 400 | diff --git a/docs/flatcar-linux/azure.md b/docs/flatcar-linux/azure.md index 3b2b8d014..4f5b2babf 100644 --- a/docs/flatcar-linux/azure.md +++ b/docs/flatcar-linux/azure.md @@ -225,7 +225,7 @@ Reference the DNS zone with `azurerm_dns_zone.clusters.name` and its resource gr | worker_count | Number of workers | 1 | 3 | | controller_type | Machine type for controllers | "Standard_B2s" | See below | | worker_type | Machine type for workers | "Standard_DS1_v2" | See below | -| os_image | Channel for a Container Linux derivative | "flatcar-stable" | flatcar-stable, flatcar-beta, flatcar-alpha, flatcar-edge, coreos-stable, coreos-beta, coreos-alpha | +| os_image | Channel for a Container Linux derivative | "flatcar-stable" | flatcar-stable, flatcar-beta, flatcar-alpha, flatcar-edge | | disk_size | Size of the disk in GB | 40 | 100 | | worker_priority | Set priority to Spot to use reduced cost surplus capacity, with the tradeoff that instances can be deallocated at any time | Regular | Spot | | controller_snippets | Controller Container Linux Config snippets | [] | [example](/advanced/customization/#usage) | diff --git a/docs/flatcar-linux/bare-metal.md b/docs/flatcar-linux/bare-metal.md index b2624d91a..73ce0f1cc 100644 --- a/docs/flatcar-linux/bare-metal.md +++ b/docs/flatcar-linux/bare-metal.md @@ -336,7 +336,7 @@ Check the [variables.tf](https://github.com/poseidon/typhoon/blob/master/bare-me |:-----|:------------|:--------| | cluster_name | Unique cluster name | "mercury" | | matchbox_http_endpoint | Matchbox HTTP read-only endpoint | "http://matchbox.example.com:port" | -| os_channel | Channel for a Container Linux derivative | coreos-stable, coreos-beta, coreos-alpha, flatcar-stable, flatcar-beta, flatcar-alpha, flatcar-edge | +| os_channel | Channel for a Container Linux derivative | flatcar-stable, flatcar-beta, flatcar-alpha, flatcar-edge | | os_version | Version for a Container Linux derivative to PXE and install | "2345.3.1" | | k8s_domain_name | FQDN resolving to the controller(s) nodes. Workers and kubectl will communicate with this endpoint | "myk8s.example.com" | | ssh_authorized_key | SSH public key for user 'core' | "ssh-rsa AAAAB3Nz..." | diff --git a/docs/flatcar-linux/digitalocean.md b/docs/flatcar-linux/digitalocean.md index 4418fdc2e..5360efb0e 100644 --- a/docs/flatcar-linux/digitalocean.md +++ b/docs/flatcar-linux/digitalocean.md @@ -190,7 +190,7 @@ Check the [variables.tf](https://github.com/poseidon/typhoon/blob/master/digital | cluster_name | Unique cluster name (prepended to dns_zone) | "nemo" | | region | Digital Ocean region | "nyc1", "sfo2", "fra1", tor1" | | dns_zone | Digital Ocean domain (i.e. DNS zone) | "do.example.com" | -| os_image | Container Linux image for instances | "custom-image-id", coreos-stable, coreos-beta, coreos-alpha | +| os_image | Container Linux image for instances | "uploaded-flatcar-image-id" | | ssh_fingerprints | SSH public key fingerprints | ["d7:9d..."] | #### DNS Zone diff --git a/docs/flatcar-linux/google-cloud.md b/docs/flatcar-linux/google-cloud.md index 0efa4c856..76f61b021 100644 --- a/docs/flatcar-linux/google-cloud.md +++ b/docs/flatcar-linux/google-cloud.md @@ -204,7 +204,7 @@ Check the [variables.tf](https://github.com/poseidon/typhoon/blob/master/google- | region | Google Cloud region | "us-central1" | | dns_zone | Google Cloud DNS zone | "google-cloud.example.com" | | dns_zone_name | Google Cloud DNS zone name | "example-zone" | -| os_image | Container Linux image for compute instances | "flatcar-linux-2303-4-0", coreos-stable, coreos-beta, coreos-alpha | +| os_image | Container Linux image for compute instances | "flatcar-linux-2303-4-0" | | ssh_authorized_key | SSH public key for user 'core' | "ssh-rsa AAAAB3NZ..." | Check the list of valid [regions](https://cloud.google.com/compute/docs/regions-zones/regions-zones) and list Container Linux [images](https://cloud.google.com/compute/docs/images) with `gcloud compute images list | grep coreos`. From 74e025c9e49005ae22f3846bae1158eb5d4358d9 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Sat, 4 Jul 2020 16:53:21 -0700 Subject: [PATCH 473/523] Update Grafana from v7.0.4 to v7.0.5 * https://github.com/grafana/grafana/releases/tag/v7.0.5 --- CHANGES.md | 4 ++++ addons/grafana/deployment.yaml | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index 73afe4602..aabf13ba2 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -4,6 +4,10 @@ Notable changes between versions. ## Latest +* Update Grafana from v7.0.4 to [v7.0.5](https://github.com/grafana/grafana/releases/tag/v7.0.5) + +## v1.18.5 + * Kubernetes [v1.18.5](https://github.com/kubernetes/kubernetes/blob/master/CHANGELOG/CHANGELOG-1.18.md#v1185) * Add Cilium v1.8.0 as a (experimental) CNI provider option ([#760](https://github.com/poseidon/typhoon/pull/760)) * Set `networking` to "cilium" to enable diff --git a/addons/grafana/deployment.yaml b/addons/grafana/deployment.yaml index 8047f5fa6..fc9a051a9 100644 --- a/addons/grafana/deployment.yaml +++ b/addons/grafana/deployment.yaml @@ -23,7 +23,7 @@ spec: spec: containers: - name: grafana - image: docker.io/grafana/grafana:7.0.4 + image: docker.io/grafana/grafana:7.0.5 env: - name: GF_PATHS_CONFIG value: "/etc/grafana/custom.ini" From 49050320ce349622f0da2e778f4519614b4adebc Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Sun, 5 Jul 2020 15:58:34 -0700 Subject: [PATCH 474/523] Update Cilium from v1.8.0 to v1.8.1 * https://github.com/cilium/cilium/releases/tag/v1.8.1 --- CHANGES.md | 1 + aws/container-linux/kubernetes/bootstrap.tf | 2 +- aws/fedora-coreos/kubernetes/bootstrap.tf | 2 +- azure/container-linux/kubernetes/bootstrap.tf | 2 +- azure/fedora-coreos/kubernetes/bootstrap.tf | 2 +- bare-metal/container-linux/kubernetes/bootstrap.tf | 2 +- bare-metal/fedora-coreos/kubernetes/bootstrap.tf | 2 +- digital-ocean/container-linux/kubernetes/bootstrap.tf | 2 +- digital-ocean/fedora-coreos/kubernetes/bootstrap.tf | 2 +- google-cloud/container-linux/kubernetes/bootstrap.tf | 2 +- google-cloud/fedora-coreos/kubernetes/bootstrap.tf | 2 +- 11 files changed, 11 insertions(+), 10 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index aabf13ba2..2284a6bcd 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -5,6 +5,7 @@ Notable changes between versions. ## Latest * Update Grafana from v7.0.4 to [v7.0.5](https://github.com/grafana/grafana/releases/tag/v7.0.5) +* Update Cilium from v1.8.0 to [v1.8.1](https://github.com/cilium/cilium/releases/tag/v1.8.1) ## v1.18.5 diff --git a/aws/container-linux/kubernetes/bootstrap.tf b/aws/container-linux/kubernetes/bootstrap.tf index c197e1d52..ba52066b8 100644 --- a/aws/container-linux/kubernetes/bootstrap.tf +++ b/aws/container-linux/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=5a7c963caf59740891df2aeae4b1561ccb3b9db6" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=9a5132b2ad199ccb87e69b1b203cbb3caa9a755e" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/aws/fedora-coreos/kubernetes/bootstrap.tf b/aws/fedora-coreos/kubernetes/bootstrap.tf index bcedda803..d3d9cc304 100644 --- a/aws/fedora-coreos/kubernetes/bootstrap.tf +++ b/aws/fedora-coreos/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=5a7c963caf59740891df2aeae4b1561ccb3b9db6" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=9a5132b2ad199ccb87e69b1b203cbb3caa9a755e" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/azure/container-linux/kubernetes/bootstrap.tf b/azure/container-linux/kubernetes/bootstrap.tf index bdef479ac..3de7add51 100644 --- a/azure/container-linux/kubernetes/bootstrap.tf +++ b/azure/container-linux/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=5a7c963caf59740891df2aeae4b1561ccb3b9db6" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=9a5132b2ad199ccb87e69b1b203cbb3caa9a755e" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/azure/fedora-coreos/kubernetes/bootstrap.tf b/azure/fedora-coreos/kubernetes/bootstrap.tf index 56f19623b..e765cddb9 100644 --- a/azure/fedora-coreos/kubernetes/bootstrap.tf +++ b/azure/fedora-coreos/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=5a7c963caf59740891df2aeae4b1561ccb3b9db6" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=9a5132b2ad199ccb87e69b1b203cbb3caa9a755e" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/bare-metal/container-linux/kubernetes/bootstrap.tf b/bare-metal/container-linux/kubernetes/bootstrap.tf index 7bc3175e7..baa8bdc68 100644 --- a/bare-metal/container-linux/kubernetes/bootstrap.tf +++ b/bare-metal/container-linux/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=5a7c963caf59740891df2aeae4b1561ccb3b9db6" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=9a5132b2ad199ccb87e69b1b203cbb3caa9a755e" cluster_name = var.cluster_name api_servers = [var.k8s_domain_name] diff --git a/bare-metal/fedora-coreos/kubernetes/bootstrap.tf b/bare-metal/fedora-coreos/kubernetes/bootstrap.tf index 55232279a..bd91b8b2e 100644 --- a/bare-metal/fedora-coreos/kubernetes/bootstrap.tf +++ b/bare-metal/fedora-coreos/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=5a7c963caf59740891df2aeae4b1561ccb3b9db6" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=9a5132b2ad199ccb87e69b1b203cbb3caa9a755e" cluster_name = var.cluster_name api_servers = [var.k8s_domain_name] diff --git a/digital-ocean/container-linux/kubernetes/bootstrap.tf b/digital-ocean/container-linux/kubernetes/bootstrap.tf index ac191838c..dc7adc3b8 100644 --- a/digital-ocean/container-linux/kubernetes/bootstrap.tf +++ b/digital-ocean/container-linux/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=5a7c963caf59740891df2aeae4b1561ccb3b9db6" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=9a5132b2ad199ccb87e69b1b203cbb3caa9a755e" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/digital-ocean/fedora-coreos/kubernetes/bootstrap.tf b/digital-ocean/fedora-coreos/kubernetes/bootstrap.tf index 89404db28..41ad74560 100644 --- a/digital-ocean/fedora-coreos/kubernetes/bootstrap.tf +++ b/digital-ocean/fedora-coreos/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=5a7c963caf59740891df2aeae4b1561ccb3b9db6" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=9a5132b2ad199ccb87e69b1b203cbb3caa9a755e" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/google-cloud/container-linux/kubernetes/bootstrap.tf b/google-cloud/container-linux/kubernetes/bootstrap.tf index 7510c04bf..03a98751d 100644 --- a/google-cloud/container-linux/kubernetes/bootstrap.tf +++ b/google-cloud/container-linux/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=5a7c963caf59740891df2aeae4b1561ccb3b9db6" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=9a5132b2ad199ccb87e69b1b203cbb3caa9a755e" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/google-cloud/fedora-coreos/kubernetes/bootstrap.tf b/google-cloud/fedora-coreos/kubernetes/bootstrap.tf index ef7375637..851546eac 100644 --- a/google-cloud/fedora-coreos/kubernetes/bootstrap.tf +++ b/google-cloud/fedora-coreos/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=5a7c963caf59740891df2aeae4b1561ccb3b9db6" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=9a5132b2ad199ccb87e69b1b203cbb3caa9a755e" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] From e3bf7d8f9bfbcd0ad5b7fa4ca85122318f520295 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Thu, 9 Jul 2020 21:08:55 -0700 Subject: [PATCH 475/523] Update Prometheus from v2.19.1 to v2.19.2 * https://github.com/prometheus/prometheus/releases/tag/v2.19.2 --- CHANGES.md | 4 ++++ addons/prometheus/deployment.yaml | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index 2284a6bcd..c764775d4 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -7,6 +7,10 @@ Notable changes between versions. * Update Grafana from v7.0.4 to [v7.0.5](https://github.com/grafana/grafana/releases/tag/v7.0.5) * Update Cilium from v1.8.0 to [v1.8.1](https://github.com/cilium/cilium/releases/tag/v1.8.1) +#### Addons + +* Update Prometheus from v2.19.1 to [v2.19.2](https://github.com/prometheus/prometheus/releases/tag/v2.19.2) + ## v1.18.5 * Kubernetes [v1.18.5](https://github.com/kubernetes/kubernetes/blob/master/CHANGELOG/CHANGELOG-1.18.md#v1185) diff --git a/addons/prometheus/deployment.yaml b/addons/prometheus/deployment.yaml index 48a798d53..d9077b27e 100644 --- a/addons/prometheus/deployment.yaml +++ b/addons/prometheus/deployment.yaml @@ -20,7 +20,7 @@ spec: serviceAccountName: prometheus containers: - name: prometheus - image: quay.io/prometheus/prometheus:v2.19.1 + image: quay.io/prometheus/prometheus:v2.19.2 args: - --web.listen-address=0.0.0.0:9090 - --config.file=/etc/prometheus/prometheus.yaml From dfd2a0ec23c36ffda15e810e7d2ebe4435f1b0e6 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Thu, 9 Jul 2020 21:10:48 -0700 Subject: [PATCH 476/523] Update Grafana from v7.0.5 to v7.0.6 * https://github.com/grafana/grafana/releases/tag/v7.0.6 --- CHANGES.md | 2 +- addons/grafana/deployment.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index c764775d4..1cdac3192 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -4,12 +4,12 @@ Notable changes between versions. ## Latest -* Update Grafana from v7.0.4 to [v7.0.5](https://github.com/grafana/grafana/releases/tag/v7.0.5) * Update Cilium from v1.8.0 to [v1.8.1](https://github.com/cilium/cilium/releases/tag/v1.8.1) #### Addons * Update Prometheus from v2.19.1 to [v2.19.2](https://github.com/prometheus/prometheus/releases/tag/v2.19.2) +* Update Grafana from v7.0.4 to [v7.0.6](https://github.com/grafana/grafana/releases/tag/v7.0.6) ## v1.18.5 diff --git a/addons/grafana/deployment.yaml b/addons/grafana/deployment.yaml index fc9a051a9..f7d68eb2a 100644 --- a/addons/grafana/deployment.yaml +++ b/addons/grafana/deployment.yaml @@ -23,7 +23,7 @@ spec: spec: containers: - name: grafana - image: docker.io/grafana/grafana:7.0.5 + image: docker.io/grafana/grafana:7.0.6 env: - name: GF_PATHS_CONFIG value: "/etc/grafana/custom.ini" From 507aac9b78a202f6cc198178523d3b9479907677 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Sat, 11 Jul 2020 22:56:41 -0700 Subject: [PATCH 477/523] Update mkdocs-material from v5.3.3 to v5.4.0 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 92d6eb802..edd58d2c1 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ mkdocs==1.1.2 -mkdocs-material==5.3.3 +mkdocs-material==5.4.0 pygments==2.6.1 pymdown-extensions==7.1.0 From 9ea6d2c2457844d4c6e11304f063c8c1076f4845 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Wed, 15 Jul 2020 22:05:57 -0700 Subject: [PATCH 478/523] Update Kubernetes from v1.18.5 to v1.18.6 * https://github.com/kubernetes/kubernetes/blob/master/CHANGELOG/CHANGELOG-1.18.md#v1186 * https://github.com/poseidon/terraform-render-bootstrap/pull/201 --- README.md | 10 +++++----- aws/container-linux/kubernetes/README.md | 2 +- aws/container-linux/kubernetes/bootstrap.tf | 2 +- aws/container-linux/kubernetes/cl/controller.yaml | 4 ++-- .../kubernetes/workers/cl/worker.yaml | 4 ++-- aws/fedora-coreos/kubernetes/README.md | 2 +- aws/fedora-coreos/kubernetes/bootstrap.tf | 2 +- aws/fedora-coreos/kubernetes/fcc/controller.yaml | 4 ++-- .../kubernetes/workers/fcc/worker.yaml | 4 ++-- azure/container-linux/kubernetes/README.md | 2 +- azure/container-linux/kubernetes/bootstrap.tf | 2 +- .../container-linux/kubernetes/cl/controller.yaml | 4 ++-- .../kubernetes/workers/cl/worker.yaml | 4 ++-- azure/fedora-coreos/kubernetes/README.md | 2 +- azure/fedora-coreos/kubernetes/bootstrap.tf | 2 +- azure/fedora-coreos/kubernetes/fcc/controller.yaml | 4 ++-- .../kubernetes/workers/fcc/worker.yaml | 4 ++-- bare-metal/container-linux/kubernetes/README.md | 2 +- bare-metal/container-linux/kubernetes/bootstrap.tf | 2 +- .../container-linux/kubernetes/cl/controller.yaml | 4 ++-- .../container-linux/kubernetes/cl/worker.yaml | 2 +- bare-metal/fedora-coreos/kubernetes/README.md | 2 +- bare-metal/fedora-coreos/kubernetes/bootstrap.tf | 2 +- .../fedora-coreos/kubernetes/fcc/controller.yaml | 4 ++-- .../fedora-coreos/kubernetes/fcc/worker.yaml | 2 +- digital-ocean/container-linux/kubernetes/README.md | 2 +- .../container-linux/kubernetes/bootstrap.tf | 2 +- .../container-linux/kubernetes/cl/controller.yaml | 4 ++-- .../container-linux/kubernetes/cl/worker.yaml | 4 ++-- digital-ocean/fedora-coreos/kubernetes/README.md | 2 +- .../fedora-coreos/kubernetes/bootstrap.tf | 2 +- .../fedora-coreos/kubernetes/fcc/controller.yaml | 4 ++-- .../fedora-coreos/kubernetes/fcc/worker.yaml | 4 ++-- docs/advanced/worker-pools.md | 14 +++++++------- docs/fedora-coreos/aws.md | 10 +++++----- docs/fedora-coreos/azure.md | 10 +++++----- docs/fedora-coreos/bare-metal.md | 10 +++++----- docs/fedora-coreos/digitalocean.md | 10 +++++----- docs/fedora-coreos/google-cloud.md | 8 ++++---- docs/flatcar-linux/aws.md | 10 +++++----- docs/flatcar-linux/azure.md | 10 +++++----- docs/flatcar-linux/bare-metal.md | 10 +++++----- docs/flatcar-linux/digitalocean.md | 10 +++++----- docs/flatcar-linux/google-cloud.md | 10 +++++----- docs/index.md | 10 +++++----- docs/topics/maintenance.md | 4 ++-- google-cloud/container-linux/kubernetes/README.md | 2 +- .../container-linux/kubernetes/bootstrap.tf | 2 +- .../container-linux/kubernetes/cl/controller.yaml | 4 ++-- .../kubernetes/workers/cl/worker.yaml | 4 ++-- google-cloud/fedora-coreos/kubernetes/README.md | 2 +- google-cloud/fedora-coreos/kubernetes/bootstrap.tf | 2 +- .../fedora-coreos/kubernetes/fcc/controller.yaml | 4 ++-- .../kubernetes/workers/fcc/worker.yaml | 4 ++-- 54 files changed, 126 insertions(+), 126 deletions(-) diff --git a/README.md b/README.md index 14803ea29..f8d6fadef 100644 --- a/README.md +++ b/README.md @@ -11,7 +11,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features -* Kubernetes v1.18.5 (upstream) +* Kubernetes v1.18.6 (upstream) * Single or multi-master, [Calico](https://www.projectcalico.org/) or [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/), SELinux enforcing * Advanced features like [worker pools](https://typhoon.psdn.io/advanced/worker-pools/), [preemptible](https://typhoon.psdn.io/cl/google-cloud/#preemption) workers, and [snippets](https://typhoon.psdn.io/advanced/customization/#container-linux) customization @@ -54,7 +54,7 @@ Define a Kubernetes cluster by using the Terraform module for your chosen platfo ```tf module "yavin" { - source = "git::https://github.com/poseidon/typhoon//google-cloud/fedora-coreos/kubernetes?ref=v1.18.5" + source = "git::https://github.com/poseidon/typhoon//google-cloud/fedora-coreos/kubernetes?ref=v1.18.6" # Google Cloud cluster_name = "yavin" @@ -93,9 +93,9 @@ In 4-8 minutes (varies by platform), the cluster will be ready. This Google Clou $ export KUBECONFIG=/home/user/.kube/configs/yavin-config $ kubectl get nodes NAME ROLES STATUS AGE VERSION -yavin-controller-0.c.example-com.internal Ready 6m v1.18.5 -yavin-worker-jrbf.c.example-com.internal Ready 5m v1.18.5 -yavin-worker-mzdm.c.example-com.internal Ready 5m v1.18.5 +yavin-controller-0.c.example-com.internal Ready 6m v1.18.6 +yavin-worker-jrbf.c.example-com.internal Ready 5m v1.18.6 +yavin-worker-mzdm.c.example-com.internal Ready 5m v1.18.6 ``` List the pods. diff --git a/aws/container-linux/kubernetes/README.md b/aws/container-linux/kubernetes/README.md index edde1eec3..eeee14dbb 100644 --- a/aws/container-linux/kubernetes/README.md +++ b/aws/container-linux/kubernetes/README.md @@ -11,7 +11,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features -* Kubernetes v1.18.5 (upstream) +* Kubernetes v1.18.6 (upstream) * Single or multi-master, [Calico](https://www.projectcalico.org/) or [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/) * Advanced features like [worker pools](https://typhoon.psdn.io/advanced/worker-pools/), [spot](https://typhoon.psdn.io/cl/aws/#spot) workers, and [snippets](https://typhoon.psdn.io/advanced/customization/#container-linux) customization diff --git a/aws/container-linux/kubernetes/bootstrap.tf b/aws/container-linux/kubernetes/bootstrap.tf index ba52066b8..8235977ba 100644 --- a/aws/container-linux/kubernetes/bootstrap.tf +++ b/aws/container-linux/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=9a5132b2ad199ccb87e69b1b203cbb3caa9a755e" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=835890025bfb3499aa0cd5a9704e9e7d3e7e5fe5" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/aws/container-linux/kubernetes/cl/controller.yaml b/aws/container-linux/kubernetes/cl/controller.yaml index ded14bf5d..62a744803 100644 --- a/aws/container-linux/kubernetes/cl/controller.yaml +++ b/aws/container-linux/kubernetes/cl/controller.yaml @@ -52,7 +52,7 @@ systemd: Description=Kubelet Wants=rpc-statd.service [Service] - Environment=KUBELET_IMAGE=docker://quay.io/poseidon/kubelet:v1.18.5 + Environment=KUBELET_IMAGE=docker://quay.io/poseidon/kubelet:v1.18.6 Environment=KUBELET_CGROUP_DRIVER=${cgroup_driver} ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests @@ -134,7 +134,7 @@ systemd: --volume script,kind=host,source=/opt/bootstrap/apply \ --mount volume=script,target=/apply \ --insecure-options=image \ - docker://quay.io/poseidon/kubelet:v1.18.5 \ + docker://quay.io/poseidon/kubelet:v1.18.6 \ --net=host \ --dns=host \ --exec=/apply diff --git a/aws/container-linux/kubernetes/workers/cl/worker.yaml b/aws/container-linux/kubernetes/workers/cl/worker.yaml index 3841f951b..fc8e85c6f 100644 --- a/aws/container-linux/kubernetes/workers/cl/worker.yaml +++ b/aws/container-linux/kubernetes/workers/cl/worker.yaml @@ -25,7 +25,7 @@ systemd: Description=Kubelet Wants=rpc-statd.service [Service] - Environment=KUBELET_IMAGE=docker://quay.io/poseidon/kubelet:v1.18.5 + Environment=KUBELET_IMAGE=docker://quay.io/poseidon/kubelet:v1.18.6 Environment=KUBELET_CGROUP_DRIVER=${cgroup_driver} ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests @@ -129,7 +129,7 @@ storage: --volume config,kind=host,source=/etc/kubernetes \ --mount volume=config,target=/etc/kubernetes \ --insecure-options=image \ - docker://quay.io/poseidon/kubelet:v1.18.5 \ + docker://quay.io/poseidon/kubelet:v1.18.6 \ --net=host \ --dns=host \ --exec=/usr/local/bin/kubectl -- --kubeconfig=/etc/kubernetes/kubeconfig delete node $(hostname) diff --git a/aws/fedora-coreos/kubernetes/README.md b/aws/fedora-coreos/kubernetes/README.md index 71ff7fcd4..56d092584 100644 --- a/aws/fedora-coreos/kubernetes/README.md +++ b/aws/fedora-coreos/kubernetes/README.md @@ -11,7 +11,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features -* Kubernetes v1.18.5 (upstream) +* Kubernetes v1.18.6 (upstream) * Single or multi-master, [Calico](https://www.projectcalico.org/) or [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/), SELinux enforcing * Advanced features like [worker pools](https://typhoon.psdn.io/advanced/worker-pools/), [spot](https://typhoon.psdn.io/cl/aws/#spot) workers, and [snippets](https://typhoon.psdn.io/advanced/customization/#container-linux) customization diff --git a/aws/fedora-coreos/kubernetes/bootstrap.tf b/aws/fedora-coreos/kubernetes/bootstrap.tf index d3d9cc304..2d13be75b 100644 --- a/aws/fedora-coreos/kubernetes/bootstrap.tf +++ b/aws/fedora-coreos/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=9a5132b2ad199ccb87e69b1b203cbb3caa9a755e" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=835890025bfb3499aa0cd5a9704e9e7d3e7e5fe5" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/aws/fedora-coreos/kubernetes/fcc/controller.yaml b/aws/fedora-coreos/kubernetes/fcc/controller.yaml index 33b7ecc28..3bd633056 100644 --- a/aws/fedora-coreos/kubernetes/fcc/controller.yaml +++ b/aws/fedora-coreos/kubernetes/fcc/controller.yaml @@ -55,7 +55,7 @@ systemd: Description=Kubelet (System Container) Wants=rpc-statd.service [Service] - Environment=KUBELET_IMAGE=quay.io/poseidon/kubelet:v1.18.5 + Environment=KUBELET_IMAGE=quay.io/poseidon/kubelet:v1.18.6 ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests ExecStartPre=/bin/mkdir -p /opt/cni/bin @@ -124,7 +124,7 @@ systemd: --volume /opt/bootstrap/assets:/assets:ro,Z \ --volume /opt/bootstrap/apply:/apply:ro,Z \ --entrypoint=/apply \ - quay.io/poseidon/kubelet:v1.18.5 + quay.io/poseidon/kubelet:v1.18.6 ExecStartPost=/bin/touch /opt/bootstrap/bootstrap.done ExecStartPost=-/usr/bin/podman stop bootstrap storage: diff --git a/aws/fedora-coreos/kubernetes/workers/fcc/worker.yaml b/aws/fedora-coreos/kubernetes/workers/fcc/worker.yaml index 69911b7fe..c178f896f 100644 --- a/aws/fedora-coreos/kubernetes/workers/fcc/worker.yaml +++ b/aws/fedora-coreos/kubernetes/workers/fcc/worker.yaml @@ -25,7 +25,7 @@ systemd: Description=Kubelet (System Container) Wants=rpc-statd.service [Service] - Environment=KUBELET_IMAGE=quay.io/poseidon/kubelet:v1.18.5 + Environment=KUBELET_IMAGE=quay.io/poseidon/kubelet:v1.18.6 ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests ExecStartPre=/bin/mkdir -p /opt/cni/bin @@ -89,7 +89,7 @@ systemd: Type=oneshot RemainAfterExit=true ExecStart=/bin/true - ExecStop=/bin/bash -c '/usr/bin/podman run --volume /etc/kubernetes:/etc/kubernetes:ro,z --entrypoint /usr/local/bin/kubectl quay.io/poseidon/kubelet:v1.18.5 --kubeconfig=/etc/kubernetes/kubeconfig delete node $HOSTNAME' + ExecStop=/bin/bash -c '/usr/bin/podman run --volume /etc/kubernetes:/etc/kubernetes:ro,z --entrypoint /usr/local/bin/kubectl quay.io/poseidon/kubelet:v1.18.6 --kubeconfig=/etc/kubernetes/kubeconfig delete node $HOSTNAME' [Install] WantedBy=multi-user.target storage: diff --git a/azure/container-linux/kubernetes/README.md b/azure/container-linux/kubernetes/README.md index 28b88a2e7..8eae38750 100644 --- a/azure/container-linux/kubernetes/README.md +++ b/azure/container-linux/kubernetes/README.md @@ -11,7 +11,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features -* Kubernetes v1.18.5 (upstream) +* Kubernetes v1.18.6 (upstream) * Single or multi-master, [Calico](https://www.projectcalico.org/) or [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/) * Advanced features like [worker pools](https://typhoon.psdn.io/advanced/worker-pools/), [low-priority](https://typhoon.psdn.io/cl/azure/#low-priority) workers, and [snippets](https://typhoon.psdn.io/advanced/customization/#container-linux) customization diff --git a/azure/container-linux/kubernetes/bootstrap.tf b/azure/container-linux/kubernetes/bootstrap.tf index 3de7add51..209c303c4 100644 --- a/azure/container-linux/kubernetes/bootstrap.tf +++ b/azure/container-linux/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=9a5132b2ad199ccb87e69b1b203cbb3caa9a755e" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=835890025bfb3499aa0cd5a9704e9e7d3e7e5fe5" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/azure/container-linux/kubernetes/cl/controller.yaml b/azure/container-linux/kubernetes/cl/controller.yaml index ded14bf5d..62a744803 100644 --- a/azure/container-linux/kubernetes/cl/controller.yaml +++ b/azure/container-linux/kubernetes/cl/controller.yaml @@ -52,7 +52,7 @@ systemd: Description=Kubelet Wants=rpc-statd.service [Service] - Environment=KUBELET_IMAGE=docker://quay.io/poseidon/kubelet:v1.18.5 + Environment=KUBELET_IMAGE=docker://quay.io/poseidon/kubelet:v1.18.6 Environment=KUBELET_CGROUP_DRIVER=${cgroup_driver} ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests @@ -134,7 +134,7 @@ systemd: --volume script,kind=host,source=/opt/bootstrap/apply \ --mount volume=script,target=/apply \ --insecure-options=image \ - docker://quay.io/poseidon/kubelet:v1.18.5 \ + docker://quay.io/poseidon/kubelet:v1.18.6 \ --net=host \ --dns=host \ --exec=/apply diff --git a/azure/container-linux/kubernetes/workers/cl/worker.yaml b/azure/container-linux/kubernetes/workers/cl/worker.yaml index 6817232f6..35f7b6798 100644 --- a/azure/container-linux/kubernetes/workers/cl/worker.yaml +++ b/azure/container-linux/kubernetes/workers/cl/worker.yaml @@ -25,7 +25,7 @@ systemd: Description=Kubelet Wants=rpc-statd.service [Service] - Environment=KUBELET_IMAGE=docker://quay.io/poseidon/kubelet:v1.18.5 + Environment=KUBELET_IMAGE=docker://quay.io/poseidon/kubelet:v1.18.6 Environment=KUBELET_CGROUP_DRIVER=${cgroup_driver} ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests @@ -129,7 +129,7 @@ storage: --volume config,kind=host,source=/etc/kubernetes \ --mount volume=config,target=/etc/kubernetes \ --insecure-options=image \ - docker://quay.io/poseidon/kubelet:v1.18.5 \ + docker://quay.io/poseidon/kubelet:v1.18.6 \ --net=host \ --dns=host \ --exec=/usr/local/bin/kubectl -- --kubeconfig=/etc/kubernetes/kubeconfig delete node $(hostname | tr '[:upper:]' '[:lower:]') diff --git a/azure/fedora-coreos/kubernetes/README.md b/azure/fedora-coreos/kubernetes/README.md index 058df8f6a..ce6367768 100644 --- a/azure/fedora-coreos/kubernetes/README.md +++ b/azure/fedora-coreos/kubernetes/README.md @@ -11,7 +11,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features -* Kubernetes v1.18.5 (upstream) +* Kubernetes v1.18.6 (upstream) * Single or multi-master, [Calico](https://www.projectcalico.org/) or [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/), SELinux enforcing * Advanced features like [worker pools](https://typhoon.psdn.io/advanced/worker-pools/), [spot priority](https://typhoon.psdn.io/fedora-coreos/azure/#low-priority) workers, and [snippets](https://typhoon.psdn.io/advanced/customization/) customization diff --git a/azure/fedora-coreos/kubernetes/bootstrap.tf b/azure/fedora-coreos/kubernetes/bootstrap.tf index e765cddb9..33349e136 100644 --- a/azure/fedora-coreos/kubernetes/bootstrap.tf +++ b/azure/fedora-coreos/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=9a5132b2ad199ccb87e69b1b203cbb3caa9a755e" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=835890025bfb3499aa0cd5a9704e9e7d3e7e5fe5" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/azure/fedora-coreos/kubernetes/fcc/controller.yaml b/azure/fedora-coreos/kubernetes/fcc/controller.yaml index f2d6e5b4a..84d5ba16f 100644 --- a/azure/fedora-coreos/kubernetes/fcc/controller.yaml +++ b/azure/fedora-coreos/kubernetes/fcc/controller.yaml @@ -54,7 +54,7 @@ systemd: Description=Kubelet (System Container) Wants=rpc-statd.service [Service] - Environment=KUBELET_IMAGE=quay.io/poseidon/kubelet:v1.18.5 + Environment=KUBELET_IMAGE=quay.io/poseidon/kubelet:v1.18.6 ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests ExecStartPre=/bin/mkdir -p /opt/cni/bin @@ -123,7 +123,7 @@ systemd: --volume /opt/bootstrap/assets:/assets:ro,Z \ --volume /opt/bootstrap/apply:/apply:ro,Z \ --entrypoint=/apply \ - quay.io/poseidon/kubelet:v1.18.5 + quay.io/poseidon/kubelet:v1.18.6 ExecStartPost=/bin/touch /opt/bootstrap/bootstrap.done ExecStartPost=-/usr/bin/podman stop bootstrap storage: diff --git a/azure/fedora-coreos/kubernetes/workers/fcc/worker.yaml b/azure/fedora-coreos/kubernetes/workers/fcc/worker.yaml index abb47553f..06904d40e 100644 --- a/azure/fedora-coreos/kubernetes/workers/fcc/worker.yaml +++ b/azure/fedora-coreos/kubernetes/workers/fcc/worker.yaml @@ -24,7 +24,7 @@ systemd: Description=Kubelet (System Container) Wants=rpc-statd.service [Service] - Environment=KUBELET_IMAGE=quay.io/poseidon/kubelet:v1.18.5 + Environment=KUBELET_IMAGE=quay.io/poseidon/kubelet:v1.18.6 ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests ExecStartPre=/bin/mkdir -p /opt/cni/bin @@ -88,7 +88,7 @@ systemd: Type=oneshot RemainAfterExit=true ExecStart=/bin/true - ExecStop=/bin/bash -c '/usr/bin/podman run --volume /etc/kubernetes:/etc/kubernetes:ro,z --entrypoint /usr/local/bin/kubectl quay.io/poseidon/kubelet:v1.18.5 --kubeconfig=/etc/kubernetes/kubeconfig delete node $HOSTNAME' + ExecStop=/bin/bash -c '/usr/bin/podman run --volume /etc/kubernetes:/etc/kubernetes:ro,z --entrypoint /usr/local/bin/kubectl quay.io/poseidon/kubelet:v1.18.6 --kubeconfig=/etc/kubernetes/kubeconfig delete node $HOSTNAME' [Install] WantedBy=multi-user.target storage: diff --git a/bare-metal/container-linux/kubernetes/README.md b/bare-metal/container-linux/kubernetes/README.md index b7a2ebcc7..3b8a5f347 100644 --- a/bare-metal/container-linux/kubernetes/README.md +++ b/bare-metal/container-linux/kubernetes/README.md @@ -11,7 +11,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features -* Kubernetes v1.18.5 (upstream) +* Kubernetes v1.18.6 (upstream) * Single or multi-master, [Calico](https://www.projectcalico.org/) or [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/) * Advanced features like [snippets](https://typhoon.psdn.io/advanced/customization/#container-linux) customization diff --git a/bare-metal/container-linux/kubernetes/bootstrap.tf b/bare-metal/container-linux/kubernetes/bootstrap.tf index baa8bdc68..57cdfb728 100644 --- a/bare-metal/container-linux/kubernetes/bootstrap.tf +++ b/bare-metal/container-linux/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=9a5132b2ad199ccb87e69b1b203cbb3caa9a755e" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=835890025bfb3499aa0cd5a9704e9e7d3e7e5fe5" cluster_name = var.cluster_name api_servers = [var.k8s_domain_name] diff --git a/bare-metal/container-linux/kubernetes/cl/controller.yaml b/bare-metal/container-linux/kubernetes/cl/controller.yaml index ca87fac57..b316d579e 100644 --- a/bare-metal/container-linux/kubernetes/cl/controller.yaml +++ b/bare-metal/container-linux/kubernetes/cl/controller.yaml @@ -60,7 +60,7 @@ systemd: Description=Kubelet Wants=rpc-statd.service [Service] - Environment=KUBELET_IMAGE=docker://quay.io/poseidon/kubelet:v1.18.5 + Environment=KUBELET_IMAGE=docker://quay.io/poseidon/kubelet:v1.18.6 Environment=KUBELET_CGROUP_DRIVER=${cgroup_driver} ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests @@ -147,7 +147,7 @@ systemd: --volume script,kind=host,source=/opt/bootstrap/apply \ --mount volume=script,target=/apply \ --insecure-options=image \ - docker://quay.io/poseidon/kubelet:v1.18.5 \ + docker://quay.io/poseidon/kubelet:v1.18.6 \ --net=host \ --dns=host \ --exec=/apply diff --git a/bare-metal/container-linux/kubernetes/cl/worker.yaml b/bare-metal/container-linux/kubernetes/cl/worker.yaml index 407eec8b7..91424a39a 100644 --- a/bare-metal/container-linux/kubernetes/cl/worker.yaml +++ b/bare-metal/container-linux/kubernetes/cl/worker.yaml @@ -33,7 +33,7 @@ systemd: Description=Kubelet Wants=rpc-statd.service [Service] - Environment=KUBELET_IMAGE=docker://quay.io/poseidon/kubelet:v1.18.5 + Environment=KUBELET_IMAGE=docker://quay.io/poseidon/kubelet:v1.18.6 Environment=KUBELET_CGROUP_DRIVER=${cgroup_driver} ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests diff --git a/bare-metal/fedora-coreos/kubernetes/README.md b/bare-metal/fedora-coreos/kubernetes/README.md index 357501cf2..a7e1a3ca5 100644 --- a/bare-metal/fedora-coreos/kubernetes/README.md +++ b/bare-metal/fedora-coreos/kubernetes/README.md @@ -11,7 +11,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features -* Kubernetes v1.18.5 (upstream) +* Kubernetes v1.18.6 (upstream) * Single or multi-master, [Calico](https://www.projectcalico.org/) or [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/), SELinux enforcing * Advanced features like [snippets](https://typhoon.psdn.io/advanced/customization/#container-linux) customization diff --git a/bare-metal/fedora-coreos/kubernetes/bootstrap.tf b/bare-metal/fedora-coreos/kubernetes/bootstrap.tf index bd91b8b2e..54769fdaa 100644 --- a/bare-metal/fedora-coreos/kubernetes/bootstrap.tf +++ b/bare-metal/fedora-coreos/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=9a5132b2ad199ccb87e69b1b203cbb3caa9a755e" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=835890025bfb3499aa0cd5a9704e9e7d3e7e5fe5" cluster_name = var.cluster_name api_servers = [var.k8s_domain_name] diff --git a/bare-metal/fedora-coreos/kubernetes/fcc/controller.yaml b/bare-metal/fedora-coreos/kubernetes/fcc/controller.yaml index 9e296bb96..ef40b396f 100644 --- a/bare-metal/fedora-coreos/kubernetes/fcc/controller.yaml +++ b/bare-metal/fedora-coreos/kubernetes/fcc/controller.yaml @@ -53,7 +53,7 @@ systemd: Description=Kubelet (System Container) Wants=rpc-statd.service [Service] - Environment=KUBELET_IMAGE=quay.io/poseidon/kubelet:v1.18.5 + Environment=KUBELET_IMAGE=quay.io/poseidon/kubelet:v1.18.6 ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests ExecStartPre=/bin/mkdir -p /opt/cni/bin @@ -134,7 +134,7 @@ systemd: --volume /opt/bootstrap/assets:/assets:ro,Z \ --volume /opt/bootstrap/apply:/apply:ro,Z \ --entrypoint=/apply \ - quay.io/poseidon/kubelet:v1.18.5 + quay.io/poseidon/kubelet:v1.18.6 ExecStartPost=/bin/touch /opt/bootstrap/bootstrap.done ExecStartPost=-/usr/bin/podman stop bootstrap storage: diff --git a/bare-metal/fedora-coreos/kubernetes/fcc/worker.yaml b/bare-metal/fedora-coreos/kubernetes/fcc/worker.yaml index b86e70168..79a9275bc 100644 --- a/bare-metal/fedora-coreos/kubernetes/fcc/worker.yaml +++ b/bare-metal/fedora-coreos/kubernetes/fcc/worker.yaml @@ -23,7 +23,7 @@ systemd: Description=Kubelet (System Container) Wants=rpc-statd.service [Service] - Environment=KUBELET_IMAGE=quay.io/poseidon/kubelet:v1.18.5 + Environment=KUBELET_IMAGE=quay.io/poseidon/kubelet:v1.18.6 ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests ExecStartPre=/bin/mkdir -p /opt/cni/bin diff --git a/digital-ocean/container-linux/kubernetes/README.md b/digital-ocean/container-linux/kubernetes/README.md index 626d35bbd..dc91f0a24 100644 --- a/digital-ocean/container-linux/kubernetes/README.md +++ b/digital-ocean/container-linux/kubernetes/README.md @@ -11,7 +11,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features -* Kubernetes v1.18.5 (upstream) +* Kubernetes v1.18.6 (upstream) * Single or multi-master, [Calico](https://www.projectcalico.org/) or [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/) * Advanced features like [snippets](https://typhoon.psdn.io/advanced/customization/#container-linux) customization diff --git a/digital-ocean/container-linux/kubernetes/bootstrap.tf b/digital-ocean/container-linux/kubernetes/bootstrap.tf index dc7adc3b8..e4874eb5c 100644 --- a/digital-ocean/container-linux/kubernetes/bootstrap.tf +++ b/digital-ocean/container-linux/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=9a5132b2ad199ccb87e69b1b203cbb3caa9a755e" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=835890025bfb3499aa0cd5a9704e9e7d3e7e5fe5" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/digital-ocean/container-linux/kubernetes/cl/controller.yaml b/digital-ocean/container-linux/kubernetes/cl/controller.yaml index 7342042fc..719dbc0b1 100644 --- a/digital-ocean/container-linux/kubernetes/cl/controller.yaml +++ b/digital-ocean/container-linux/kubernetes/cl/controller.yaml @@ -62,7 +62,7 @@ systemd: After=coreos-metadata.service Wants=rpc-statd.service [Service] - Environment=KUBELET_IMAGE=docker://quay.io/poseidon/kubelet:v1.18.5 + Environment=KUBELET_IMAGE=docker://quay.io/poseidon/kubelet:v1.18.6 EnvironmentFile=/run/metadata/coreos ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests @@ -144,7 +144,7 @@ systemd: --volume script,kind=host,source=/opt/bootstrap/apply \ --mount volume=script,target=/apply \ --insecure-options=image \ - docker://quay.io/poseidon/kubelet:v1.18.5 \ + docker://quay.io/poseidon/kubelet:v1.18.6 \ --net=host \ --dns=host \ --exec=/apply diff --git a/digital-ocean/container-linux/kubernetes/cl/worker.yaml b/digital-ocean/container-linux/kubernetes/cl/worker.yaml index ac028b47e..1d638d8aa 100644 --- a/digital-ocean/container-linux/kubernetes/cl/worker.yaml +++ b/digital-ocean/container-linux/kubernetes/cl/worker.yaml @@ -35,7 +35,7 @@ systemd: After=coreos-metadata.service Wants=rpc-statd.service [Service] - Environment=KUBELET_IMAGE=docker://quay.io/poseidon/kubelet:v1.18.5 + Environment=KUBELET_IMAGE=docker://quay.io/poseidon/kubelet:v1.18.6 EnvironmentFile=/run/metadata/coreos ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests @@ -134,7 +134,7 @@ storage: --volume config,kind=host,source=/etc/kubernetes \ --mount volume=config,target=/etc/kubernetes \ --insecure-options=image \ - docker://quay.io/poseidon/kubelet:v1.18.5 \ + docker://quay.io/poseidon/kubelet:v1.18.6 \ --net=host \ --dns=host \ --exec=/usr/local/bin/kubectl -- --kubeconfig=/etc/kubernetes/kubeconfig delete node $(hostname) diff --git a/digital-ocean/fedora-coreos/kubernetes/README.md b/digital-ocean/fedora-coreos/kubernetes/README.md index 52e1131e3..76fc2db7f 100644 --- a/digital-ocean/fedora-coreos/kubernetes/README.md +++ b/digital-ocean/fedora-coreos/kubernetes/README.md @@ -11,7 +11,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features -* Kubernetes v1.18.5 (upstream) +* Kubernetes v1.18.6 (upstream) * Single or multi-master, [Calico](https://www.projectcalico.org/) or [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/), SELinux enforcing * Advanced features like [snippets](https://typhoon.psdn.io/advanced/customization/) customization diff --git a/digital-ocean/fedora-coreos/kubernetes/bootstrap.tf b/digital-ocean/fedora-coreos/kubernetes/bootstrap.tf index 41ad74560..857221705 100644 --- a/digital-ocean/fedora-coreos/kubernetes/bootstrap.tf +++ b/digital-ocean/fedora-coreos/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=9a5132b2ad199ccb87e69b1b203cbb3caa9a755e" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=835890025bfb3499aa0cd5a9704e9e7d3e7e5fe5" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/digital-ocean/fedora-coreos/kubernetes/fcc/controller.yaml b/digital-ocean/fedora-coreos/kubernetes/fcc/controller.yaml index be92764cb..4948a8861 100644 --- a/digital-ocean/fedora-coreos/kubernetes/fcc/controller.yaml +++ b/digital-ocean/fedora-coreos/kubernetes/fcc/controller.yaml @@ -55,7 +55,7 @@ systemd: After=afterburn.service Wants=rpc-statd.service [Service] - Environment=KUBELET_IMAGE=quay.io/poseidon/kubelet:v1.18.5 + Environment=KUBELET_IMAGE=quay.io/poseidon/kubelet:v1.18.6 EnvironmentFile=/run/metadata/afterburn ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests @@ -135,7 +135,7 @@ systemd: --volume /opt/bootstrap/assets:/assets:ro,Z \ --volume /opt/bootstrap/apply:/apply:ro,Z \ --entrypoint=/apply \ - quay.io/poseidon/kubelet:v1.18.5 + quay.io/poseidon/kubelet:v1.18.6 ExecStartPost=/bin/touch /opt/bootstrap/bootstrap.done ExecStartPost=-/usr/bin/podman stop bootstrap storage: diff --git a/digital-ocean/fedora-coreos/kubernetes/fcc/worker.yaml b/digital-ocean/fedora-coreos/kubernetes/fcc/worker.yaml index 231cf8439..f4b605e50 100644 --- a/digital-ocean/fedora-coreos/kubernetes/fcc/worker.yaml +++ b/digital-ocean/fedora-coreos/kubernetes/fcc/worker.yaml @@ -26,7 +26,7 @@ systemd: After=afterburn.service Wants=rpc-statd.service [Service] - Environment=KUBELET_IMAGE=quay.io/poseidon/kubelet:v1.18.5 + Environment=KUBELET_IMAGE=quay.io/poseidon/kubelet:v1.18.6 EnvironmentFile=/run/metadata/afterburn ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests @@ -98,7 +98,7 @@ systemd: Type=oneshot RemainAfterExit=true ExecStart=/bin/true - ExecStop=/bin/bash -c '/usr/bin/podman run --volume /etc/kubernetes:/etc/kubernetes:ro,z --entrypoint /usr/local/bin/kubectl quay.io/poseidon/kubelet:v1.18.5 --kubeconfig=/etc/kubernetes/kubeconfig delete node $HOSTNAME' + ExecStop=/bin/bash -c '/usr/bin/podman run --volume /etc/kubernetes:/etc/kubernetes:ro,z --entrypoint /usr/local/bin/kubectl quay.io/poseidon/kubelet:v1.18.6 --kubeconfig=/etc/kubernetes/kubeconfig delete node $HOSTNAME' [Install] WantedBy=multi-user.target storage: diff --git a/docs/advanced/worker-pools.md b/docs/advanced/worker-pools.md index f9b72594a..7caed3f58 100644 --- a/docs/advanced/worker-pools.md +++ b/docs/advanced/worker-pools.md @@ -83,7 +83,7 @@ Create a cluster following the Azure [tutorial](../flatcar-linux/azure.md#cluste ```tf module "ramius-worker-pool" { - source = "git::https://github.com/poseidon/typhoon//azure/container-linux/kubernetes/workers?ref=v1.18.5" + source = "git::https://github.com/poseidon/typhoon//azure/container-linux/kubernetes/workers?ref=v1.18.6" # Azure region = module.ramius.region @@ -149,7 +149,7 @@ Create a cluster following the Google Cloud [tutorial](../flatcar-linux/google-c ```tf module "yavin-worker-pool" { - source = "git::https://github.com/poseidon/typhoon//google-cloud/container-linux/kubernetes/workers?ref=v1.18.5" + source = "git::https://github.com/poseidon/typhoon//google-cloud/container-linux/kubernetes/workers?ref=v1.18.6" # Google Cloud region = "europe-west2" @@ -180,11 +180,11 @@ Verify a managed instance group of workers joins the cluster within a few minute ``` $ kubectl get nodes NAME STATUS AGE VERSION -yavin-controller-0.c.example-com.internal Ready 6m v1.18.5 -yavin-worker-jrbf.c.example-com.internal Ready 5m v1.18.5 -yavin-worker-mzdm.c.example-com.internal Ready 5m v1.18.5 -yavin-16x-worker-jrbf.c.example-com.internal Ready 3m v1.18.5 -yavin-16x-worker-mzdm.c.example-com.internal Ready 3m v1.18.5 +yavin-controller-0.c.example-com.internal Ready 6m v1.18.6 +yavin-worker-jrbf.c.example-com.internal Ready 5m v1.18.6 +yavin-worker-mzdm.c.example-com.internal Ready 5m v1.18.6 +yavin-16x-worker-jrbf.c.example-com.internal Ready 3m v1.18.6 +yavin-16x-worker-mzdm.c.example-com.internal Ready 3m v1.18.6 ``` ### Variables diff --git a/docs/fedora-coreos/aws.md b/docs/fedora-coreos/aws.md index d2455f779..6f4430c6b 100644 --- a/docs/fedora-coreos/aws.md +++ b/docs/fedora-coreos/aws.md @@ -1,6 +1,6 @@ # AWS -In this tutorial, we'll create a Kubernetes v1.18.5 cluster on AWS with Fedora CoreOS. +In this tutorial, we'll create a Kubernetes v1.18.6 cluster on AWS with Fedora CoreOS. We'll declare a Kubernetes cluster using the Typhoon Terraform module. Then apply the changes to create a VPC, gateway, subnets, security groups, controller instances, worker auto-scaling group, network load balancer, and TLS assets. @@ -70,7 +70,7 @@ Define a Kubernetes cluster using the module `aws/fedora-coreos/kubernetes`. ```tf module "tempest" { - source = "git::https://github.com/poseidon/typhoon//aws/fedora-coreos/kubernetes?ref=v1.18.5" + source = "git::https://github.com/poseidon/typhoon//aws/fedora-coreos/kubernetes?ref=v1.18.6" # AWS cluster_name = "tempest" @@ -143,9 +143,9 @@ List nodes in the cluster. $ export KUBECONFIG=/home/user/.kube/configs/tempest-config $ kubectl get nodes NAME STATUS ROLES AGE VERSION -ip-10-0-3-155 Ready 10m v1.18.5 -ip-10-0-26-65 Ready 10m v1.18.5 -ip-10-0-41-21 Ready 10m v1.18.5 +ip-10-0-3-155 Ready 10m v1.18.6 +ip-10-0-26-65 Ready 10m v1.18.6 +ip-10-0-41-21 Ready 10m v1.18.6 ``` List the pods. diff --git a/docs/fedora-coreos/azure.md b/docs/fedora-coreos/azure.md index 2c94212a1..820dc3a0c 100644 --- a/docs/fedora-coreos/azure.md +++ b/docs/fedora-coreos/azure.md @@ -1,6 +1,6 @@ # Azure -In this tutorial, we'll create a Kubernetes v1.18.5 cluster on Azure with Fedora CoreOS. +In this tutorial, we'll create a Kubernetes v1.18.6 cluster on Azure with Fedora CoreOS. We'll declare a Kubernetes cluster using the Typhoon Terraform module. Then apply the changes to create a resource group, virtual network, subnets, security groups, controller availability set, worker scale set, load balancer, and TLS assets. @@ -83,7 +83,7 @@ Define a Kubernetes cluster using the module `azure/fedora-coreos/kubernetes`. ```tf module "ramius" { - source = "git::https://github.com/poseidon/typhoon//azure/fedora-coreos/kubernetes?ref=v1.18.5" + source = "git::https://github.com/poseidon/typhoon//azure/fedora-coreos/kubernetes?ref=v1.18.6" # Azure cluster_name = "ramius" @@ -158,9 +158,9 @@ List nodes in the cluster. $ export KUBECONFIG=/home/user/.kube/configs/ramius-config $ kubectl get nodes NAME STATUS ROLES AGE VERSION -ramius-controller-0 Ready 24m v1.18.5 -ramius-worker-000001 Ready 25m v1.18.5 -ramius-worker-000002 Ready 24m v1.18.5 +ramius-controller-0 Ready 24m v1.18.6 +ramius-worker-000001 Ready 25m v1.18.6 +ramius-worker-000002 Ready 24m v1.18.6 ``` List the pods. diff --git a/docs/fedora-coreos/bare-metal.md b/docs/fedora-coreos/bare-metal.md index 061746787..5ae6bf880 100644 --- a/docs/fedora-coreos/bare-metal.md +++ b/docs/fedora-coreos/bare-metal.md @@ -1,6 +1,6 @@ # Bare-Metal -In this tutorial, we'll network boot and provision a Kubernetes v1.18.5 cluster on bare-metal with Fedora CoreOS. +In this tutorial, we'll network boot and provision a Kubernetes v1.18.6 cluster on bare-metal with Fedora CoreOS. First, we'll deploy a [Matchbox](https://github.com/poseidon/matchbox) service and setup a network boot environment. Then, we'll declare a Kubernetes cluster using the Typhoon Terraform module and power on machines. On PXE boot, machines will install Fedora CoreOS to disk, reboot into the disk install, and provision themselves as Kubernetes controllers or workers via Ignition. @@ -160,7 +160,7 @@ Define a Kubernetes cluster using the module `bare-metal/fedora-coreos/kubernete ```tf module "mercury" { - source = "git::https://github.com/poseidon/typhoon//bare-metal/fedora-coreos/kubernetes?ref=v1.18.5" + source = "git::https://github.com/poseidon/typhoon//bare-metal/fedora-coreos/kubernetes?ref=v1.18.6" # bare-metal cluster_name = "mercury" @@ -289,9 +289,9 @@ List nodes in the cluster. $ export KUBECONFIG=/home/user/.kube/configs/mercury-config $ kubectl get nodes NAME STATUS ROLES AGE VERSION -node1.example.com Ready 10m v1.18.5 -node2.example.com Ready 10m v1.18.5 -node3.example.com Ready 10m v1.18.5 +node1.example.com Ready 10m v1.18.6 +node2.example.com Ready 10m v1.18.6 +node3.example.com Ready 10m v1.18.6 ``` List the pods. diff --git a/docs/fedora-coreos/digitalocean.md b/docs/fedora-coreos/digitalocean.md index 7884af438..3d3bd9474 100644 --- a/docs/fedora-coreos/digitalocean.md +++ b/docs/fedora-coreos/digitalocean.md @@ -1,6 +1,6 @@ # DigitalOcean -In this tutorial, we'll create a Kubernetes v1.18.5 cluster on DigitalOcean with Fedora CoreOS. +In this tutorial, we'll create a Kubernetes v1.18.6 cluster on DigitalOcean with Fedora CoreOS. We'll declare a Kubernetes cluster using the Typhoon Terraform module. Then apply the changes to create controller droplets, worker droplets, DNS records, tags, and TLS assets. @@ -79,7 +79,7 @@ Define a Kubernetes cluster using the module `digital-ocean/fedora-coreos/kubern ```tf module "nemo" { - source = "git::https://github.com/poseidon/typhoon//digital-ocean/fedora-coreos/kubernetes?ref=v1.18.5" + source = "git::https://github.com/poseidon/typhoon//digital-ocean/fedora-coreos/kubernetes?ref=v1.18.6" # Digital Ocean cluster_name = "nemo" @@ -153,9 +153,9 @@ List nodes in the cluster. $ export KUBECONFIG=/home/user/.kube/configs/nemo-config $ kubectl get nodes NAME STATUS ROLES AGE VERSION -10.132.110.130 Ready 10m v1.18.5 -10.132.115.81 Ready 10m v1.18.5 -10.132.124.107 Ready 10m v1.18.5 +10.132.110.130 Ready 10m v1.18.6 +10.132.115.81 Ready 10m v1.18.6 +10.132.124.107 Ready 10m v1.18.6 ``` List the pods. diff --git a/docs/fedora-coreos/google-cloud.md b/docs/fedora-coreos/google-cloud.md index 414be8027..98426f001 100644 --- a/docs/fedora-coreos/google-cloud.md +++ b/docs/fedora-coreos/google-cloud.md @@ -1,6 +1,6 @@ # Google Cloud -In this tutorial, we'll create a Kubernetes v1.18.5 cluster on Google Compute Engine with Fedora CoreOS. +In this tutorial, we'll create a Kubernetes v1.18.6 cluster on Google Compute Engine with Fedora CoreOS. We'll declare a Kubernetes cluster using the Typhoon Terraform module. Then apply the changes to create a network, firewall rules, health checks, controller instances, worker managed instance group, load balancers, and TLS assets. @@ -145,9 +145,9 @@ List nodes in the cluster. $ export KUBECONFIG=/home/user/.kube/configs/yavin-config $ kubectl get nodes NAME ROLES STATUS AGE VERSION -yavin-controller-0.c.example-com.internal Ready 6m v1.18.5 -yavin-worker-jrbf.c.example-com.internal Ready 5m v1.18.5 -yavin-worker-mzdm.c.example-com.internal Ready 5m v1.18.5 +yavin-controller-0.c.example-com.internal Ready 6m v1.18.6 +yavin-worker-jrbf.c.example-com.internal Ready 5m v1.18.6 +yavin-worker-mzdm.c.example-com.internal Ready 5m v1.18.6 ``` List the pods. diff --git a/docs/flatcar-linux/aws.md b/docs/flatcar-linux/aws.md index c2c3bf7ff..749628ee8 100644 --- a/docs/flatcar-linux/aws.md +++ b/docs/flatcar-linux/aws.md @@ -1,6 +1,6 @@ # AWS -In this tutorial, we'll create a Kubernetes v1.18.5 cluster on AWS with CoreOS Container Linux or Flatcar Linux. +In this tutorial, we'll create a Kubernetes v1.18.6 cluster on AWS with CoreOS Container Linux or Flatcar Linux. We'll declare a Kubernetes cluster using the Typhoon Terraform module. Then apply the changes to create a VPC, gateway, subnets, security groups, controller instances, worker auto-scaling group, network load balancer, and TLS assets. @@ -70,7 +70,7 @@ Define a Kubernetes cluster using the module `aws/container-linux/kubernetes`. ```tf module "tempest" { - source = "git::https://github.com/poseidon/typhoon//aws/container-linux/kubernetes?ref=v1.18.5" + source = "git::https://github.com/poseidon/typhoon//aws/container-linux/kubernetes?ref=v1.18.6" # AWS cluster_name = "tempest" @@ -143,9 +143,9 @@ List nodes in the cluster. $ export KUBECONFIG=/home/user/.kube/configs/tempest-config $ kubectl get nodes NAME STATUS ROLES AGE VERSION -ip-10-0-3-155 Ready 10m v1.18.5 -ip-10-0-26-65 Ready 10m v1.18.5 -ip-10-0-41-21 Ready 10m v1.18.5 +ip-10-0-3-155 Ready 10m v1.18.6 +ip-10-0-26-65 Ready 10m v1.18.6 +ip-10-0-41-21 Ready 10m v1.18.6 ``` List the pods. diff --git a/docs/flatcar-linux/azure.md b/docs/flatcar-linux/azure.md index 4f5b2babf..e5f33d271 100644 --- a/docs/flatcar-linux/azure.md +++ b/docs/flatcar-linux/azure.md @@ -1,6 +1,6 @@ # Azure -In this tutorial, we'll create a Kubernetes v1.18.5 cluster on Azure with CoreOS Container Linux or Flatcar Linux. +In this tutorial, we'll create a Kubernetes v1.18.6 cluster on Azure with CoreOS Container Linux or Flatcar Linux. We'll declare a Kubernetes cluster using the Typhoon Terraform module. Then apply the changes to create a resource group, virtual network, subnets, security groups, controller availability set, worker scale set, load balancer, and TLS assets. @@ -72,7 +72,7 @@ Define a Kubernetes cluster using the module `azure/container-linux/kubernetes`. ```tf module "ramius" { - source = "git::https://github.com/poseidon/typhoon//azure/container-linux/kubernetes?ref=v1.18.5" + source = "git::https://github.com/poseidon/typhoon//azure/container-linux/kubernetes?ref=v1.18.6" # Azure cluster_name = "ramius" @@ -146,9 +146,9 @@ List nodes in the cluster. $ export KUBECONFIG=/home/user/.kube/configs/ramius-config $ kubectl get nodes NAME STATUS ROLES AGE VERSION -ramius-controller-0 Ready 24m v1.18.5 -ramius-worker-000001 Ready 25m v1.18.5 -ramius-worker-000002 Ready 24m v1.18.5 +ramius-controller-0 Ready 24m v1.18.6 +ramius-worker-000001 Ready 25m v1.18.6 +ramius-worker-000002 Ready 24m v1.18.6 ``` List the pods. diff --git a/docs/flatcar-linux/bare-metal.md b/docs/flatcar-linux/bare-metal.md index 73ce0f1cc..46cc984b4 100644 --- a/docs/flatcar-linux/bare-metal.md +++ b/docs/flatcar-linux/bare-metal.md @@ -1,6 +1,6 @@ # Bare-Metal -In this tutorial, we'll network boot and provision a Kubernetes v1.18.5 cluster on bare-metal with CoreOS Container Linux or Flatcar Linux. +In this tutorial, we'll network boot and provision a Kubernetes v1.18.6 cluster on bare-metal with CoreOS Container Linux or Flatcar Linux. First, we'll deploy a [Matchbox](https://github.com/poseidon/matchbox) service and setup a network boot environment. Then, we'll declare a Kubernetes cluster using the Typhoon Terraform module and power on machines. On PXE boot, machines will install Container Linux to disk, reboot into the disk install, and provision themselves as Kubernetes controllers or workers via Ignition. @@ -160,7 +160,7 @@ Define a Kubernetes cluster using the module `bare-metal/container-linux/kuberne ```tf module "mercury" { - source = "git::https://github.com/poseidon/typhoon//bare-metal/container-linux/kubernetes?ref=v1.18.5" + source = "git::https://github.com/poseidon/typhoon//bare-metal/container-linux/kubernetes?ref=v1.18.6" # bare-metal cluster_name = "mercury" @@ -299,9 +299,9 @@ List nodes in the cluster. $ export KUBECONFIG=/home/user/.kube/configs/mercury-config $ kubectl get nodes NAME STATUS ROLES AGE VERSION -node1.example.com Ready 10m v1.18.5 -node2.example.com Ready 10m v1.18.5 -node3.example.com Ready 10m v1.18.5 +node1.example.com Ready 10m v1.18.6 +node2.example.com Ready 10m v1.18.6 +node3.example.com Ready 10m v1.18.6 ``` List the pods. diff --git a/docs/flatcar-linux/digitalocean.md b/docs/flatcar-linux/digitalocean.md index 5360efb0e..e5de490e2 100644 --- a/docs/flatcar-linux/digitalocean.md +++ b/docs/flatcar-linux/digitalocean.md @@ -1,6 +1,6 @@ # DigitalOcean -In this tutorial, we'll create a Kubernetes v1.18.5 cluster on DigitalOcean with CoreOS Container Linux or Flatcar Linux. +In this tutorial, we'll create a Kubernetes v1.18.6 cluster on DigitalOcean with CoreOS Container Linux or Flatcar Linux. We'll declare a Kubernetes cluster using the Typhoon Terraform module. Then apply the changes to create controller droplets, worker droplets, DNS records, tags, and TLS assets. @@ -79,7 +79,7 @@ Define a Kubernetes cluster using the module `digital-ocean/container-linux/kube ```tf module "nemo" { - source = "git::https://github.com/poseidon/typhoon//digital-ocean/container-linux/kubernetes?ref=v1.18.5" + source = "git::https://github.com/poseidon/typhoon//digital-ocean/container-linux/kubernetes?ref=v1.18.6" # Digital Ocean cluster_name = "nemo" @@ -153,9 +153,9 @@ List nodes in the cluster. $ export KUBECONFIG=/home/user/.kube/configs/nemo-config $ kubectl get nodes NAME STATUS ROLES AGE VERSION -10.132.110.130 Ready 10m v1.18.5 -10.132.115.81 Ready 10m v1.18.5 -10.132.124.107 Ready 10m v1.18.5 +10.132.110.130 Ready 10m v1.18.6 +10.132.115.81 Ready 10m v1.18.6 +10.132.124.107 Ready 10m v1.18.6 ``` List the pods. diff --git a/docs/flatcar-linux/google-cloud.md b/docs/flatcar-linux/google-cloud.md index 76f61b021..8770fb747 100644 --- a/docs/flatcar-linux/google-cloud.md +++ b/docs/flatcar-linux/google-cloud.md @@ -1,6 +1,6 @@ # Google Cloud -In this tutorial, we'll create a Kubernetes v1.18.5 cluster on Google Compute Engine with CoreOS Container Linux or Flatcar Linux. +In this tutorial, we'll create a Kubernetes v1.18.6 cluster on Google Compute Engine with CoreOS Container Linux or Flatcar Linux. We'll declare a Kubernetes cluster using the Typhoon Terraform module. Then apply the changes to create a network, firewall rules, health checks, controller instances, worker managed instance group, load balancers, and TLS assets. @@ -90,7 +90,7 @@ Define a Kubernetes cluster using the module `google-cloud/container-linux/kuber ```tf module "yavin" { - source = "git::https://github.com/poseidon/typhoon//google-cloud/container-linux/kubernetes?ref=v1.18.5" + source = "git::https://github.com/poseidon/typhoon//google-cloud/container-linux/kubernetes?ref=v1.18.6" # Google Cloud cluster_name = "yavin" @@ -165,9 +165,9 @@ List nodes in the cluster. $ export KUBECONFIG=/home/user/.kube/configs/yavin-config $ kubectl get nodes NAME ROLES STATUS AGE VERSION -yavin-controller-0.c.example-com.internal Ready 6m v1.18.5 -yavin-worker-jrbf.c.example-com.internal Ready 5m v1.18.5 -yavin-worker-mzdm.c.example-com.internal Ready 5m v1.18.5 +yavin-controller-0.c.example-com.internal Ready 6m v1.18.6 +yavin-worker-jrbf.c.example-com.internal Ready 5m v1.18.6 +yavin-worker-mzdm.c.example-com.internal Ready 5m v1.18.6 ``` List the pods. diff --git a/docs/index.md b/docs/index.md index 9dde5cc92..f843b1c44 100644 --- a/docs/index.md +++ b/docs/index.md @@ -11,7 +11,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features -* Kubernetes v1.18.5 (upstream) +* Kubernetes v1.18.6 (upstream) * Single or multi-master, [Calico](https://www.projectcalico.org/) or [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/), SELinux enforcing * Advanced features like [worker pools](advanced/worker-pools/), [preemptible](fedora-coreos/google-cloud/#preemption) workers, and [snippets](advanced/customization/#container-linux) customization @@ -53,7 +53,7 @@ Define a Kubernetes cluster by using the Terraform module for your chosen platfo ```tf module "yavin" { - source = "git::https://github.com/poseidon/typhoon//google-cloud/fedora-coreos/kubernetes?ref=v1.18.5" + source = "git::https://github.com/poseidon/typhoon//google-cloud/fedora-coreos/kubernetes?ref=v1.18.6" # Google Cloud cluster_name = "yavin" @@ -91,9 +91,9 @@ In 4-8 minutes (varies by platform), the cluster will be ready. This Google Clou $ export KUBECONFIG=/home/user/.kube/configs/yavin-config $ kubectl get nodes NAME ROLES STATUS AGE VERSION -yavin-controller-0.c.example-com.internal Ready 6m v1.18.5 -yavin-worker-jrbf.c.example-com.internal Ready 5m v1.18.5 -yavin-worker-mzdm.c.example-com.internal Ready 5m v1.18.5 +yavin-controller-0.c.example-com.internal Ready 6m v1.18.6 +yavin-worker-jrbf.c.example-com.internal Ready 5m v1.18.6 +yavin-worker-mzdm.c.example-com.internal Ready 5m v1.18.6 ``` List the pods. diff --git a/docs/topics/maintenance.md b/docs/topics/maintenance.md index cb6303e36..65b650a13 100644 --- a/docs/topics/maintenance.md +++ b/docs/topics/maintenance.md @@ -13,12 +13,12 @@ Typhoon provides tagged releases to allow clusters to be versioned using ordinar ``` module "yavin" { - source = "git::https://github.com/poseidon/typhoon//google-cloud/fedora-coreos/kubernetes?ref=v1.18.5" + source = "git::https://github.com/poseidon/typhoon//google-cloud/fedora-coreos/kubernetes?ref=v1.18.6" ... } module "mercury" { - source = "git::https://github.com/poseidon/typhoon//bare-metal/container-linux/kubernetes?ref=v1.18.5" + source = "git::https://github.com/poseidon/typhoon//bare-metal/container-linux/kubernetes?ref=v1.18.6" ... } ``` diff --git a/google-cloud/container-linux/kubernetes/README.md b/google-cloud/container-linux/kubernetes/README.md index 834e34454..a40f991b8 100644 --- a/google-cloud/container-linux/kubernetes/README.md +++ b/google-cloud/container-linux/kubernetes/README.md @@ -11,7 +11,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features -* Kubernetes v1.18.5 (upstream) +* Kubernetes v1.18.6 (upstream) * Single or multi-master, [Calico](https://www.projectcalico.org/) or [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/) * Advanced features like [worker pools](https://typhoon.psdn.io/advanced/worker-pools/), [preemptible](https://typhoon.psdn.io/cl/google-cloud/#preemption) workers, and [snippets](https://typhoon.psdn.io/advanced/customization/#container-linux) customization diff --git a/google-cloud/container-linux/kubernetes/bootstrap.tf b/google-cloud/container-linux/kubernetes/bootstrap.tf index 03a98751d..7b9222258 100644 --- a/google-cloud/container-linux/kubernetes/bootstrap.tf +++ b/google-cloud/container-linux/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=9a5132b2ad199ccb87e69b1b203cbb3caa9a755e" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=835890025bfb3499aa0cd5a9704e9e7d3e7e5fe5" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/google-cloud/container-linux/kubernetes/cl/controller.yaml b/google-cloud/container-linux/kubernetes/cl/controller.yaml index 084ed1be0..fbca032ae 100644 --- a/google-cloud/container-linux/kubernetes/cl/controller.yaml +++ b/google-cloud/container-linux/kubernetes/cl/controller.yaml @@ -52,7 +52,7 @@ systemd: Description=Kubelet Wants=rpc-statd.service [Service] - Environment=KUBELET_IMAGE=docker://quay.io/poseidon/kubelet:v1.18.5 + Environment=KUBELET_IMAGE=docker://quay.io/poseidon/kubelet:v1.18.6 ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests ExecStartPre=/bin/mkdir -p /opt/cni/bin @@ -132,7 +132,7 @@ systemd: --volume script,kind=host,source=/opt/bootstrap/apply \ --mount volume=script,target=/apply \ --insecure-options=image \ - docker://quay.io/poseidon/kubelet:v1.18.5 \ + docker://quay.io/poseidon/kubelet:v1.18.6 \ --net=host \ --dns=host \ --exec=/apply diff --git a/google-cloud/container-linux/kubernetes/workers/cl/worker.yaml b/google-cloud/container-linux/kubernetes/workers/cl/worker.yaml index c78f6a04e..41e375539 100644 --- a/google-cloud/container-linux/kubernetes/workers/cl/worker.yaml +++ b/google-cloud/container-linux/kubernetes/workers/cl/worker.yaml @@ -25,7 +25,7 @@ systemd: Description=Kubelet Wants=rpc-statd.service [Service] - Environment=KUBELET_IMAGE=docker://quay.io/poseidon/kubelet:v1.18.5 + Environment=KUBELET_IMAGE=docker://quay.io/poseidon/kubelet:v1.18.6 ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests ExecStartPre=/bin/mkdir -p /opt/cni/bin @@ -127,7 +127,7 @@ storage: --volume config,kind=host,source=/etc/kubernetes \ --mount volume=config,target=/etc/kubernetes \ --insecure-options=image \ - docker://quay.io/poseidon/kubelet:v1.18.5 \ + docker://quay.io/poseidon/kubelet:v1.18.6 \ --net=host \ --dns=host \ --exec=/usr/local/bin/kubectl -- --kubeconfig=/etc/kubernetes/kubeconfig delete node $(hostname) diff --git a/google-cloud/fedora-coreos/kubernetes/README.md b/google-cloud/fedora-coreos/kubernetes/README.md index b78aa67ef..915647de5 100644 --- a/google-cloud/fedora-coreos/kubernetes/README.md +++ b/google-cloud/fedora-coreos/kubernetes/README.md @@ -11,7 +11,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features -* Kubernetes v1.18.5 (upstream) +* Kubernetes v1.18.6 (upstream) * Single or multi-master, [Calico](https://www.projectcalico.org/) or [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/), SELinux enforcing * Advanced features like [worker pools](https://typhoon.psdn.io/advanced/worker-pools/), [preemptible](https://typhoon.psdn.io/fedora-coreos/google-cloud/#preemption) workers, and [snippets](https://typhoon.psdn.io/advanced/customization/) customization diff --git a/google-cloud/fedora-coreos/kubernetes/bootstrap.tf b/google-cloud/fedora-coreos/kubernetes/bootstrap.tf index 851546eac..fe3b10276 100644 --- a/google-cloud/fedora-coreos/kubernetes/bootstrap.tf +++ b/google-cloud/fedora-coreos/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=9a5132b2ad199ccb87e69b1b203cbb3caa9a755e" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=835890025bfb3499aa0cd5a9704e9e7d3e7e5fe5" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/google-cloud/fedora-coreos/kubernetes/fcc/controller.yaml b/google-cloud/fedora-coreos/kubernetes/fcc/controller.yaml index 4acedbdce..6055eea47 100644 --- a/google-cloud/fedora-coreos/kubernetes/fcc/controller.yaml +++ b/google-cloud/fedora-coreos/kubernetes/fcc/controller.yaml @@ -54,7 +54,7 @@ systemd: Description=Kubelet (System Container) Wants=rpc-statd.service [Service] - Environment=KUBELET_IMAGE=quay.io/poseidon/kubelet:v1.18.5 + Environment=KUBELET_IMAGE=quay.io/poseidon/kubelet:v1.18.6 ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests ExecStartPre=/bin/mkdir -p /opt/cni/bin @@ -123,7 +123,7 @@ systemd: --volume /opt/bootstrap/assets:/assets:ro,Z \ --volume /opt/bootstrap/apply:/apply:ro,Z \ --entrypoint=/apply \ - quay.io/poseidon/kubelet:v1.18.5 + quay.io/poseidon/kubelet:v1.18.6 ExecStartPost=/bin/touch /opt/bootstrap/bootstrap.done ExecStartPost=-/usr/bin/podman stop bootstrap storage: diff --git a/google-cloud/fedora-coreos/kubernetes/workers/fcc/worker.yaml b/google-cloud/fedora-coreos/kubernetes/workers/fcc/worker.yaml index b29aa4f2c..dff706ce6 100644 --- a/google-cloud/fedora-coreos/kubernetes/workers/fcc/worker.yaml +++ b/google-cloud/fedora-coreos/kubernetes/workers/fcc/worker.yaml @@ -24,7 +24,7 @@ systemd: Description=Kubelet (System Container) Wants=rpc-statd.service [Service] - Environment=KUBELET_IMAGE=quay.io/poseidon/kubelet:v1.18.5 + Environment=KUBELET_IMAGE=quay.io/poseidon/kubelet:v1.18.6 ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests ExecStartPre=/bin/mkdir -p /opt/cni/bin @@ -88,7 +88,7 @@ systemd: Type=oneshot RemainAfterExit=true ExecStart=/bin/true - ExecStop=/bin/bash -c '/usr/bin/podman run --volume /etc/kubernetes:/etc/kubernetes:ro,z --entrypoint /usr/local/bin/kubectl quay.io/poseidon/kubelet:v1.18.5 --kubeconfig=/etc/kubernetes/kubeconfig delete node $HOSTNAME' + ExecStop=/bin/bash -c '/usr/bin/podman run --volume /etc/kubernetes:/etc/kubernetes:ro,z --entrypoint /usr/local/bin/kubectl quay.io/poseidon/kubelet:v1.18.6 --kubeconfig=/etc/kubernetes/kubeconfig delete node $HOSTNAME' [Install] WantedBy=multi-user.target storage: From a8d3d3bb1265ef74e1d3f52539d4da1e81ee0818 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Wed, 15 Jul 2020 22:43:49 -0700 Subject: [PATCH 479/523] Update ingress-nginx from v0.33.0 to v0.34.1 * Switch to ingress-nginx controller images from us.grc.io (eu, asia can also be used if desired) * https://github.com/kubernetes/ingress-nginx/releases/tag/controller-v0.34.1 * https://github.com/kubernetes/ingress-nginx/releases/tag/controller-v0.34.0 --- CHANGES.md | 6 +++++- addons/nginx-ingress/aws/deployment.yaml | 2 +- addons/nginx-ingress/azure/deployment.yaml | 2 +- addons/nginx-ingress/bare-metal/deployment.yaml | 2 +- addons/nginx-ingress/digital-ocean/daemonset.yaml | 2 +- addons/nginx-ingress/google-cloud/deployment.yaml | 2 +- 6 files changed, 10 insertions(+), 6 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 1cdac3192..fc464027d 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -2,12 +2,16 @@ Notable changes between versions. -## Latest +## v1.18.6 +* Kubernetes [v1.18.6](https://github.com/kubernetes/kubernetes/blob/master/CHANGELOG/CHANGELOG-1.18.md#v1186) +* Update Calico from v3.15.0 to [v3.15.1](https://docs.projectcalico.org/v3.15/release-notes/) * Update Cilium from v1.8.0 to [v1.8.1](https://github.com/cilium/cilium/releases/tag/v1.8.1) #### Addons +* Update nginx-ingress from v0.33.0 to [v0.34.1](https://github.com/kubernetes/ingress-nginx/releases/tag/nginx-0.34.1) + * [ingress-nginx](https://github.com/kubernetes/ingress-nginx/releases/tag/controller-v0.34.0) will publish images only to gcr.io * Update Prometheus from v2.19.1 to [v2.19.2](https://github.com/prometheus/prometheus/releases/tag/v2.19.2) * Update Grafana from v7.0.4 to [v7.0.6](https://github.com/grafana/grafana/releases/tag/v7.0.6) diff --git a/addons/nginx-ingress/aws/deployment.yaml b/addons/nginx-ingress/aws/deployment.yaml index ba2bb4e9a..e07337d52 100644 --- a/addons/nginx-ingress/aws/deployment.yaml +++ b/addons/nginx-ingress/aws/deployment.yaml @@ -22,7 +22,7 @@ spec: spec: containers: - name: nginx-ingress-controller - image: quay.io/kubernetes-ingress-controller/nginx-ingress-controller:0.33.0 + image: us.gcr.io/k8s-artifacts-prod/ingress-nginx/controller:v0.34.1 args: - /nginx-ingress-controller - --ingress-class=public diff --git a/addons/nginx-ingress/azure/deployment.yaml b/addons/nginx-ingress/azure/deployment.yaml index ba2bb4e9a..e07337d52 100644 --- a/addons/nginx-ingress/azure/deployment.yaml +++ b/addons/nginx-ingress/azure/deployment.yaml @@ -22,7 +22,7 @@ spec: spec: containers: - name: nginx-ingress-controller - image: quay.io/kubernetes-ingress-controller/nginx-ingress-controller:0.33.0 + image: us.gcr.io/k8s-artifacts-prod/ingress-nginx/controller:v0.34.1 args: - /nginx-ingress-controller - --ingress-class=public diff --git a/addons/nginx-ingress/bare-metal/deployment.yaml b/addons/nginx-ingress/bare-metal/deployment.yaml index 8dcaa6a93..6a49f3d81 100644 --- a/addons/nginx-ingress/bare-metal/deployment.yaml +++ b/addons/nginx-ingress/bare-metal/deployment.yaml @@ -22,7 +22,7 @@ spec: spec: containers: - name: nginx-ingress-controller - image: quay.io/kubernetes-ingress-controller/nginx-ingress-controller:0.33.0 + image: us.gcr.io/k8s-artifacts-prod/ingress-nginx/controller:v0.34.1 args: - /nginx-ingress-controller - --ingress-class=public diff --git a/addons/nginx-ingress/digital-ocean/daemonset.yaml b/addons/nginx-ingress/digital-ocean/daemonset.yaml index afa5bf6ad..538efbbe3 100644 --- a/addons/nginx-ingress/digital-ocean/daemonset.yaml +++ b/addons/nginx-ingress/digital-ocean/daemonset.yaml @@ -22,7 +22,7 @@ spec: spec: containers: - name: nginx-ingress-controller - image: quay.io/kubernetes-ingress-controller/nginx-ingress-controller:0.33.0 + image: us.gcr.io/k8s-artifacts-prod/ingress-nginx/controller:v0.34.1 args: - /nginx-ingress-controller - --ingress-class=public diff --git a/addons/nginx-ingress/google-cloud/deployment.yaml b/addons/nginx-ingress/google-cloud/deployment.yaml index ba2bb4e9a..e07337d52 100644 --- a/addons/nginx-ingress/google-cloud/deployment.yaml +++ b/addons/nginx-ingress/google-cloud/deployment.yaml @@ -22,7 +22,7 @@ spec: spec: containers: - name: nginx-ingress-controller - image: quay.io/kubernetes-ingress-controller/nginx-ingress-controller:0.33.0 + image: us.gcr.io/k8s-artifacts-prod/ingress-nginx/controller:v0.34.1 args: - /nginx-ingress-controller - --ingress-class=public From 5fba20d3583ac089cfb697ba5450afa9aca3029a Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Sat, 18 Jul 2020 13:18:45 -0700 Subject: [PATCH 480/523] Update recommended Terraform provider versions * Sync Terraform provider plugin versions with those used internally --- CHANGES.md | 2 ++ docs/fedora-coreos/aws.md | 2 +- docs/fedora-coreos/azure.md | 2 +- docs/fedora-coreos/google-cloud.md | 2 +- docs/flatcar-linux/aws.md | 2 +- docs/flatcar-linux/azure.md | 2 +- docs/flatcar-linux/google-cloud.md | 2 +- 7 files changed, 8 insertions(+), 6 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index fc464027d..06d5aae78 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -2,6 +2,8 @@ Notable changes between versions. +## Latest + ## v1.18.6 * Kubernetes [v1.18.6](https://github.com/kubernetes/kubernetes/blob/master/CHANGELOG/CHANGELOG-1.18.md#v1186) diff --git a/docs/fedora-coreos/aws.md b/docs/fedora-coreos/aws.md index 6f4430c6b..0ae8a7d6d 100644 --- a/docs/fedora-coreos/aws.md +++ b/docs/fedora-coreos/aws.md @@ -49,7 +49,7 @@ Configure the AWS provider to use your access key credentials in a `providers.tf ```tf provider "aws" { - version = "2.68.0" + version = "2.70.0" region = "eu-central-1" shared_credentials_file = "/home/user/.config/aws/credentials" } diff --git a/docs/fedora-coreos/azure.md b/docs/fedora-coreos/azure.md index 820dc3a0c..38b87dc1a 100644 --- a/docs/fedora-coreos/azure.md +++ b/docs/fedora-coreos/azure.md @@ -47,7 +47,7 @@ Configure the Azure provider in a `providers.tf` file. ```tf provider "azurerm" { - version = "2.16.0" + version = "2.19.0" } provider "ct" { diff --git a/docs/fedora-coreos/google-cloud.md b/docs/fedora-coreos/google-cloud.md index 98426f001..d03c7fffe 100644 --- a/docs/fedora-coreos/google-cloud.md +++ b/docs/fedora-coreos/google-cloud.md @@ -49,7 +49,7 @@ Configure the Google Cloud provider to use your service account key, project-id, ```tf provider "google" { - version = "3.27.0" + version = "3.30.0" project = "project-id" region = "us-central1" credentials = file("~/.config/google-cloud/terraform.json") diff --git a/docs/flatcar-linux/aws.md b/docs/flatcar-linux/aws.md index 749628ee8..2d6bfe567 100644 --- a/docs/flatcar-linux/aws.md +++ b/docs/flatcar-linux/aws.md @@ -49,7 +49,7 @@ Configure the AWS provider to use your access key credentials in a `providers.tf ```tf provider "aws" { - version = "2.68.0" + version = "2.70.0" region = "eu-central-1" shared_credentials_file = "/home/user/.config/aws/credentials" } diff --git a/docs/flatcar-linux/azure.md b/docs/flatcar-linux/azure.md index e5f33d271..7f2dd7f2f 100644 --- a/docs/flatcar-linux/azure.md +++ b/docs/flatcar-linux/azure.md @@ -47,7 +47,7 @@ Configure the Azure provider in a `providers.tf` file. ```tf provider "azurerm" { - version = "2.16.0" + version = "2.19.0" } provider "ct" { diff --git a/docs/flatcar-linux/google-cloud.md b/docs/flatcar-linux/google-cloud.md index 8770fb747..7bd8566d3 100644 --- a/docs/flatcar-linux/google-cloud.md +++ b/docs/flatcar-linux/google-cloud.md @@ -49,7 +49,7 @@ Configure the Google Cloud provider to use your service account key, project-id, ```tf provider "google" { - version = "3.27.0" + version = "3.30.0" project = "project-id" region = "us-central1" credentials = file("~/.config/google-cloud/terraform.json") From 6df6bf904a5e6a44c0a86095207435870482c4e5 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Sat, 18 Jul 2020 13:27:56 -0700 Subject: [PATCH 481/523] Show Cilium as a CNI provider option in docs * Start to show Cilium as a CNI option * https://github.com/cilium/cilium --- README.md | 2 +- aws/container-linux/kubernetes/README.md | 2 +- aws/fedora-coreos/kubernetes/README.md | 2 +- azure/container-linux/kubernetes/README.md | 2 +- azure/fedora-coreos/kubernetes/README.md | 2 +- bare-metal/container-linux/kubernetes/README.md | 2 +- bare-metal/fedora-coreos/kubernetes/README.md | 2 +- digital-ocean/container-linux/kubernetes/README.md | 2 +- docs/fedora-coreos/aws.md | 2 +- docs/fedora-coreos/azure.md | 2 +- docs/fedora-coreos/bare-metal.md | 2 +- docs/fedora-coreos/digitalocean.md | 2 +- docs/fedora-coreos/google-cloud.md | 2 +- docs/flatcar-linux/aws.md | 2 +- docs/flatcar-linux/azure.md | 2 +- docs/flatcar-linux/bare-metal.md | 2 +- docs/flatcar-linux/digitalocean.md | 2 +- docs/flatcar-linux/google-cloud.md | 2 +- docs/index.md | 2 +- docs/topics/performance.md | 2 +- google-cloud/container-linux/kubernetes/README.md | 2 +- google-cloud/fedora-coreos/kubernetes/README.md | 2 +- 22 files changed, 22 insertions(+), 22 deletions(-) diff --git a/README.md b/README.md index f8d6fadef..0f9fba272 100644 --- a/README.md +++ b/README.md @@ -12,7 +12,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features * Kubernetes v1.18.6 (upstream) -* Single or multi-master, [Calico](https://www.projectcalico.org/) or [flannel](https://github.com/coreos/flannel) networking +* Single or multi-master, [Calico](https://www.projectcalico.org/) or [Cilium](https://github.com/cilium/cilium) or [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/), SELinux enforcing * Advanced features like [worker pools](https://typhoon.psdn.io/advanced/worker-pools/), [preemptible](https://typhoon.psdn.io/cl/google-cloud/#preemption) workers, and [snippets](https://typhoon.psdn.io/advanced/customization/#container-linux) customization * Ready for Ingress, Prometheus, Grafana, CSI, or other [addons](https://typhoon.psdn.io/addons/overview/) diff --git a/aws/container-linux/kubernetes/README.md b/aws/container-linux/kubernetes/README.md index eeee14dbb..2eab62dc5 100644 --- a/aws/container-linux/kubernetes/README.md +++ b/aws/container-linux/kubernetes/README.md @@ -12,7 +12,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features * Kubernetes v1.18.6 (upstream) -* Single or multi-master, [Calico](https://www.projectcalico.org/) or [flannel](https://github.com/coreos/flannel) networking +* Single or multi-master, [Calico](https://www.projectcalico.org/) or [Cilium](https://github.com/cilium/cilium) or [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/) * Advanced features like [worker pools](https://typhoon.psdn.io/advanced/worker-pools/), [spot](https://typhoon.psdn.io/cl/aws/#spot) workers, and [snippets](https://typhoon.psdn.io/advanced/customization/#container-linux) customization * Ready for Ingress, Prometheus, Grafana, CSI, and other optional [addons](https://typhoon.psdn.io/addons/overview/) diff --git a/aws/fedora-coreos/kubernetes/README.md b/aws/fedora-coreos/kubernetes/README.md index 56d092584..f0a2502f6 100644 --- a/aws/fedora-coreos/kubernetes/README.md +++ b/aws/fedora-coreos/kubernetes/README.md @@ -12,7 +12,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features * Kubernetes v1.18.6 (upstream) -* Single or multi-master, [Calico](https://www.projectcalico.org/) or [flannel](https://github.com/coreos/flannel) networking +* Single or multi-master, [Calico](https://www.projectcalico.org/) or [Cilium](https://github.com/cilium/cilium) or [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/), SELinux enforcing * Advanced features like [worker pools](https://typhoon.psdn.io/advanced/worker-pools/), [spot](https://typhoon.psdn.io/cl/aws/#spot) workers, and [snippets](https://typhoon.psdn.io/advanced/customization/#container-linux) customization * Ready for Ingress, Prometheus, Grafana, CSI, and other optional [addons](https://typhoon.psdn.io/addons/overview/) diff --git a/azure/container-linux/kubernetes/README.md b/azure/container-linux/kubernetes/README.md index 8eae38750..64ce749ab 100644 --- a/azure/container-linux/kubernetes/README.md +++ b/azure/container-linux/kubernetes/README.md @@ -12,7 +12,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features * Kubernetes v1.18.6 (upstream) -* Single or multi-master, [Calico](https://www.projectcalico.org/) or [flannel](https://github.com/coreos/flannel) networking +* Single or multi-master, [Calico](https://www.projectcalico.org/) or [Cilium](https://github.com/cilium/cilium) or [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/) * Advanced features like [worker pools](https://typhoon.psdn.io/advanced/worker-pools/), [low-priority](https://typhoon.psdn.io/cl/azure/#low-priority) workers, and [snippets](https://typhoon.psdn.io/advanced/customization/#container-linux) customization * Ready for Ingress, Prometheus, Grafana, and other optional [addons](https://typhoon.psdn.io/addons/overview/) diff --git a/azure/fedora-coreos/kubernetes/README.md b/azure/fedora-coreos/kubernetes/README.md index ce6367768..29ed3f649 100644 --- a/azure/fedora-coreos/kubernetes/README.md +++ b/azure/fedora-coreos/kubernetes/README.md @@ -12,7 +12,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features * Kubernetes v1.18.6 (upstream) -* Single or multi-master, [Calico](https://www.projectcalico.org/) or [flannel](https://github.com/coreos/flannel) networking +* Single or multi-master, [Calico](https://www.projectcalico.org/) or [Cilium](https://github.com/cilium/cilium) or [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/), SELinux enforcing * Advanced features like [worker pools](https://typhoon.psdn.io/advanced/worker-pools/), [spot priority](https://typhoon.psdn.io/fedora-coreos/azure/#low-priority) workers, and [snippets](https://typhoon.psdn.io/advanced/customization/) customization * Ready for Ingress, Prometheus, Grafana, and other optional [addons](https://typhoon.psdn.io/addons/overview/) diff --git a/bare-metal/container-linux/kubernetes/README.md b/bare-metal/container-linux/kubernetes/README.md index 3b8a5f347..8a60c63a6 100644 --- a/bare-metal/container-linux/kubernetes/README.md +++ b/bare-metal/container-linux/kubernetes/README.md @@ -12,7 +12,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features * Kubernetes v1.18.6 (upstream) -* Single or multi-master, [Calico](https://www.projectcalico.org/) or [flannel](https://github.com/coreos/flannel) networking +* Single or multi-master, [Calico](https://www.projectcalico.org/) or [Cilium](https://github.com/cilium/cilium) or [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/) * Advanced features like [snippets](https://typhoon.psdn.io/advanced/customization/#container-linux) customization * Ready for Ingress, Prometheus, Grafana, and other optional [addons](https://typhoon.psdn.io/addons/overview/) diff --git a/bare-metal/fedora-coreos/kubernetes/README.md b/bare-metal/fedora-coreos/kubernetes/README.md index a7e1a3ca5..64fd3def2 100644 --- a/bare-metal/fedora-coreos/kubernetes/README.md +++ b/bare-metal/fedora-coreos/kubernetes/README.md @@ -12,7 +12,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features * Kubernetes v1.18.6 (upstream) -* Single or multi-master, [Calico](https://www.projectcalico.org/) or [flannel](https://github.com/coreos/flannel) networking +* Single or multi-master, [Calico](https://www.projectcalico.org/) or [Cilium](https://github.com/cilium/cilium) or [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/), SELinux enforcing * Advanced features like [snippets](https://typhoon.psdn.io/advanced/customization/#container-linux) customization * Ready for Ingress, Prometheus, Grafana, and other optional [addons](https://typhoon.psdn.io/addons/overview/) diff --git a/digital-ocean/container-linux/kubernetes/README.md b/digital-ocean/container-linux/kubernetes/README.md index dc91f0a24..6b1a1e49f 100644 --- a/digital-ocean/container-linux/kubernetes/README.md +++ b/digital-ocean/container-linux/kubernetes/README.md @@ -12,7 +12,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features * Kubernetes v1.18.6 (upstream) -* Single or multi-master, [Calico](https://www.projectcalico.org/) or [flannel](https://github.com/coreos/flannel) networking +* Single or multi-master, [Calico](https://www.projectcalico.org/) or [Cilium](https://github.com/cilium/cilium) or [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/) * Advanced features like [snippets](https://typhoon.psdn.io/advanced/customization/#container-linux) customization * Ready for Ingress, Prometheus, Grafana, CSI, and other [addons](https://typhoon.psdn.io/addons/overview/) diff --git a/docs/fedora-coreos/aws.md b/docs/fedora-coreos/aws.md index 0ae8a7d6d..6f28ec4f4 100644 --- a/docs/fedora-coreos/aws.md +++ b/docs/fedora-coreos/aws.md @@ -216,7 +216,7 @@ Reference the DNS zone id with `aws_route53_zone.zone-for-clusters.zone_id`. | worker_price | Spot price in USD for worker instances or 0 to use on-demand instances | 0 | 0.10 | | controller_snippets | Controller Fedora CoreOS Config snippets | [] | [examples](/advanced/customization/) | | worker_snippets | Worker Fedora CoreOS Config snippets | [] | [examples](/advanced/customization/) | -| networking | Choice of networking provider | "calico" | "calico" or "flannel" | +| networking | Choice of networking provider | "calico" | "calico" or "cilium" or "flannel" | | network_mtu | CNI interface MTU (calico only) | 1480 | 8981 | | host_cidr | CIDR IPv4 range to assign to EC2 instances | "10.0.0.0/16" | "10.1.0.0/16" | | pod_cidr | CIDR IPv4 range to assign to Kubernetes pods | "10.2.0.0/16" | "10.22.0.0/16" | diff --git a/docs/fedora-coreos/azure.md b/docs/fedora-coreos/azure.md index 38b87dc1a..4b66a7fc8 100644 --- a/docs/fedora-coreos/azure.md +++ b/docs/fedora-coreos/azure.md @@ -242,7 +242,7 @@ Reference the DNS zone with `azurerm_dns_zone.clusters.name` and its resource gr | worker_priority | Set priority to Spot to use reduced cost surplus capacity, with the tradeoff that instances can be deallocated at any time | Regular | Spot | | controller_snippets | Controller Fedora CoreOS Config snippets | [] | [example](/advanced/customization/#usage) | | worker_snippets | Worker Fedora CoreOS Config snippets | [] | [example](/advanced/customization/#usage) | -| networking | Choice of networking provider | "calico" | "flannel" or "calico" | +| networking | Choice of networking provider | "calico" | "calico" or "cilium" or "flannel" | | host_cidr | CIDR IPv4 range to assign to instances | "10.0.0.0/16" | "10.0.0.0/20" | | pod_cidr | CIDR IPv4 range to assign to Kubernetes pods | "10.2.0.0/16" | "10.22.0.0/16" | | service_cidr | CIDR IPv4 range to assign to Kubernetes services | "10.3.0.0/16" | "10.3.0.0/24" | diff --git a/docs/fedora-coreos/bare-metal.md b/docs/fedora-coreos/bare-metal.md index 5ae6bf880..2a257d8dc 100644 --- a/docs/fedora-coreos/bare-metal.md +++ b/docs/fedora-coreos/bare-metal.md @@ -339,7 +339,7 @@ Check the [variables.tf](https://github.com/poseidon/typhoon/blob/master/bare-me |:-----|:------------|:--------|:--------| | cached_install | PXE boot and install from the Matchbox `/assets` cache. Admin MUST have downloaded Fedora CoreOS images into the cache | false | true | | install_disk | Disk device where Fedora CoreOS should be installed | "sda" (not "/dev/sda" like Container Linux) | "sdb" | -| networking | Choice of networking provider | "calico" | "calico" or "flannel" | +| networking | Choice of networking provider | "calico" | "calico" or "cilium" or "flannel" | | network_mtu | CNI interface MTU (calico-only) | 1480 | - | | snippets | Map from machine names to lists of Fedora CoreOS Config snippets | {} | [examples](/advanced/customization/) | | network_ip_autodetection_method | Method to detect host IPv4 address (calico-only) | "first-found" | "can-reach=10.0.0.1" | diff --git a/docs/fedora-coreos/digitalocean.md b/docs/fedora-coreos/digitalocean.md index 3d3bd9474..e134cce3a 100644 --- a/docs/fedora-coreos/digitalocean.md +++ b/docs/fedora-coreos/digitalocean.md @@ -238,7 +238,7 @@ Digital Ocean requires the SSH public key be uploaded to your account, so you ma | worker_type | Droplet type for workers | "s-1vcpu-2gb" | s-1vcpu-2gb, s-2vcpu-2gb, ... | | controller_snippets | Controller Fedora CoreOS Config snippets | [] | [example](/advanced/customization/) | | worker_snippets | Worker Fedora CoreOS Config snippets | [] | [example](/advanced/customization/) | -| networking | Choice of networking provider | "calico" | "flannel" or "calico" | +| networking | Choice of networking provider | "calico" | "calico" or "cilium" or "flannel" | | pod_cidr | CIDR IPv4 range to assign to Kubernetes pods | "10.2.0.0/16" | "10.22.0.0/16" | | service_cidr | CIDR IPv4 range to assign to Kubernetes services | "10.3.0.0/16" | "10.3.0.0/24" | diff --git a/docs/fedora-coreos/google-cloud.md b/docs/fedora-coreos/google-cloud.md index d03c7fffe..d11077038 100644 --- a/docs/fedora-coreos/google-cloud.md +++ b/docs/fedora-coreos/google-cloud.md @@ -218,7 +218,7 @@ resource "google_dns_managed_zone" "zone-for-clusters" { | worker_preemptible | If enabled, Compute Engine will terminate workers randomly within 24 hours | false | true | | controller_snippets | Controller Fedora CoreOS Config snippets | [] | [examples](/advanced/customization/) | | worker_snippets | Worker Fedora CoreOS Config snippets | [] | [examples](/advanced/customization/) | -| networking | Choice of networking provider | "calico" | "calico" or "flannel" | +| networking | Choice of networking provider | "calico" | "calico" or "cilium" or "flannel" | | pod_cidr | CIDR IPv4 range to assign to Kubernetes pods | "10.2.0.0/16" | "10.22.0.0/16" | | service_cidr | CIDR IPv4 range to assign to Kubernetes services | "10.3.0.0/16" | "10.3.0.0/24" | | worker_node_labels | List of initial worker node labels | [] | ["worker-pool=default"] | diff --git a/docs/flatcar-linux/aws.md b/docs/flatcar-linux/aws.md index 2d6bfe567..82d0846ae 100644 --- a/docs/flatcar-linux/aws.md +++ b/docs/flatcar-linux/aws.md @@ -216,7 +216,7 @@ Reference the DNS zone id with `aws_route53_zone.zone-for-clusters.zone_id`. | worker_price | Spot price in USD for worker instances or 0 to use on-demand instances | 0/null | 0.10 | | controller_snippets | Controller Container Linux Config snippets | [] | [example](/advanced/customization/) | | worker_snippets | Worker Container Linux Config snippets | [] | [example](/advanced/customization/) | -| networking | Choice of networking provider | "calico" | "calico" or "flannel" | +| networking | Choice of networking provider | "calico" | "calico" or "cilium" or "flannel" | | network_mtu | CNI interface MTU (calico only) | 1480 | 8981 | | host_cidr | CIDR IPv4 range to assign to EC2 instances | "10.0.0.0/16" | "10.1.0.0/16" | | pod_cidr | CIDR IPv4 range to assign to Kubernetes pods | "10.2.0.0/16" | "10.22.0.0/16" | diff --git a/docs/flatcar-linux/azure.md b/docs/flatcar-linux/azure.md index 7f2dd7f2f..bff37e2c6 100644 --- a/docs/flatcar-linux/azure.md +++ b/docs/flatcar-linux/azure.md @@ -230,7 +230,7 @@ Reference the DNS zone with `azurerm_dns_zone.clusters.name` and its resource gr | worker_priority | Set priority to Spot to use reduced cost surplus capacity, with the tradeoff that instances can be deallocated at any time | Regular | Spot | | controller_snippets | Controller Container Linux Config snippets | [] | [example](/advanced/customization/#usage) | | worker_snippets | Worker Container Linux Config snippets | [] | [example](/advanced/customization/#usage) | -| networking | Choice of networking provider | "calico" | "flannel" or "calico" | +| networking | Choice of networking provider | "calico" | "calico" or "cilium" or "flannel" | | host_cidr | CIDR IPv4 range to assign to instances | "10.0.0.0/16" | "10.0.0.0/20" | | pod_cidr | CIDR IPv4 range to assign to Kubernetes pods | "10.2.0.0/16" | "10.22.0.0/16" | | service_cidr | CIDR IPv4 range to assign to Kubernetes services | "10.3.0.0/16" | "10.3.0.0/24" | diff --git a/docs/flatcar-linux/bare-metal.md b/docs/flatcar-linux/bare-metal.md index 46cc984b4..b41e69fbe 100644 --- a/docs/flatcar-linux/bare-metal.md +++ b/docs/flatcar-linux/bare-metal.md @@ -350,7 +350,7 @@ Check the [variables.tf](https://github.com/poseidon/typhoon/blob/master/bare-me | download_protocol | Protocol iPXE uses to download the kernel and initrd. iPXE must be compiled with [crypto](https://ipxe.org/crypto) support for https. Unused if cached_install is true | "https" | "http" | | cached_install | PXE boot and install from the Matchbox `/assets` cache. Admin MUST have downloaded Container Linux or Flatcar images into the cache | false | true | | install_disk | Disk device where Container Linux should be installed | "/dev/sda" | "/dev/sdb" | -| networking | Choice of networking provider | "calico" | "calico" or "flannel" | +| networking | Choice of networking provider | "calico" | "calico" or "cilium" or "flannel" | | network_mtu | CNI interface MTU (calico-only) | 1480 | - | | snippets | Map from machine names to lists of Container Linux Config snippets | {} | [examples](/advanced/customization/) | | network_ip_autodetection_method | Method to detect host IPv4 address (calico-only) | "first-found" | "can-reach=10.0.0.1" | diff --git a/docs/flatcar-linux/digitalocean.md b/docs/flatcar-linux/digitalocean.md index e5de490e2..52dc04240 100644 --- a/docs/flatcar-linux/digitalocean.md +++ b/docs/flatcar-linux/digitalocean.md @@ -238,7 +238,7 @@ Digital Ocean requires the SSH public key be uploaded to your account, so you ma | worker_type | Droplet type for workers | "s-1vcpu-2gb" | s-1vcpu-2gb, s-2vcpu-2gb, ... | | controller_snippets | Controller Container Linux Config snippets | [] | [example](/advanced/customization/) | | worker_snippets | Worker Container Linux Config snippets | [] | [example](/advanced/customization/) | -| networking | Choice of networking provider | "calico" | "flannel" or "calico" | +| networking | Choice of networking provider | "calico" | "calico" or "cilium" or "flannel" | | pod_cidr | CIDR IPv4 range to assign to Kubernetes pods | "10.2.0.0/16" | "10.22.0.0/16" | | service_cidr | CIDR IPv4 range to assign to Kubernetes services | "10.3.0.0/16" | "10.3.0.0/24" | diff --git a/docs/flatcar-linux/google-cloud.md b/docs/flatcar-linux/google-cloud.md index 7bd8566d3..92afabfaf 100644 --- a/docs/flatcar-linux/google-cloud.md +++ b/docs/flatcar-linux/google-cloud.md @@ -238,7 +238,7 @@ resource "google_dns_managed_zone" "zone-for-clusters" { | worker_preemptible | If enabled, Compute Engine will terminate workers randomly within 24 hours | false | true | | controller_snippets | Controller Container Linux Config snippets | [] | [example](/advanced/customization/) | | worker_snippets | Worker Container Linux Config snippets | [] | [example](/advanced/customization/) | -| networking | Choice of networking provider | "calico" | "calico" or "flannel" | +| networking | Choice of networking provider | "calico" | "calico" or "cilium" or "flannel" | | pod_cidr | CIDR IPv4 range to assign to Kubernetes pods | "10.2.0.0/16" | "10.22.0.0/16" | | service_cidr | CIDR IPv4 range to assign to Kubernetes services | "10.3.0.0/16" | "10.3.0.0/24" | | worker_node_labels | List of initial worker node labels | [] | ["worker-pool=default"] | diff --git a/docs/index.md b/docs/index.md index f843b1c44..10a598254 100644 --- a/docs/index.md +++ b/docs/index.md @@ -12,7 +12,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features * Kubernetes v1.18.6 (upstream) -* Single or multi-master, [Calico](https://www.projectcalico.org/) or [flannel](https://github.com/coreos/flannel) networking +* Single or multi-master, [Calico](https://www.projectcalico.org/) or [Cilium](https://github.com/cilium/cilium) or [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/), SELinux enforcing * Advanced features like [worker pools](advanced/worker-pools/), [preemptible](fedora-coreos/google-cloud/#preemption) workers, and [snippets](advanced/customization/#container-linux) customization * Ready for Ingress, Prometheus, Grafana, CSI, or other [addons](addons/overview/) diff --git a/docs/topics/performance.md b/docs/topics/performance.md index d7aa60c04..364ac15e1 100644 --- a/docs/topics/performance.md +++ b/docs/topics/performance.md @@ -38,7 +38,7 @@ Network performance varies based on the platform and CNI plugin. `iperf` was use Notes: -* Calico and Flannel have comparable performance. Platform and configuration differences dominate. +* Calico, Cilium, and Flannel have comparable performance. Platform and configuration differences dominate. * Azure and DigitalOcean network performance can be quite variable or depend on machine type * Only [certain AWS EC2 instance types](http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/network_mtu.html#jumbo_frame_instances) allow jumbo frames. This is why the default MTU on AWS must be 1480. diff --git a/google-cloud/container-linux/kubernetes/README.md b/google-cloud/container-linux/kubernetes/README.md index a40f991b8..f5d3fbe60 100644 --- a/google-cloud/container-linux/kubernetes/README.md +++ b/google-cloud/container-linux/kubernetes/README.md @@ -12,7 +12,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features * Kubernetes v1.18.6 (upstream) -* Single or multi-master, [Calico](https://www.projectcalico.org/) or [flannel](https://github.com/coreos/flannel) networking +* Single or multi-master, [Calico](https://www.projectcalico.org/) or [Cilium](https://github.com/cilium/cilium) or [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/) * Advanced features like [worker pools](https://typhoon.psdn.io/advanced/worker-pools/), [preemptible](https://typhoon.psdn.io/cl/google-cloud/#preemption) workers, and [snippets](https://typhoon.psdn.io/advanced/customization/#container-linux) customization * Ready for Ingress, Prometheus, Grafana, CSI, and other optional [addons](https://typhoon.psdn.io/addons/overview/) diff --git a/google-cloud/fedora-coreos/kubernetes/README.md b/google-cloud/fedora-coreos/kubernetes/README.md index 915647de5..da99d075e 100644 --- a/google-cloud/fedora-coreos/kubernetes/README.md +++ b/google-cloud/fedora-coreos/kubernetes/README.md @@ -12,7 +12,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features * Kubernetes v1.18.6 (upstream) -* Single or multi-master, [Calico](https://www.projectcalico.org/) or [flannel](https://github.com/coreos/flannel) networking +* Single or multi-master, [Calico](https://www.projectcalico.org/) or [Cilium](https://github.com/cilium/cilium) or [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/), SELinux enforcing * Advanced features like [worker pools](https://typhoon.psdn.io/advanced/worker-pools/), [preemptible](https://typhoon.psdn.io/fedora-coreos/google-cloud/#preemption) workers, and [snippets](https://typhoon.psdn.io/advanced/customization/) customization * Ready for Ingress, Prometheus, Grafana, CSI, and other optional [addons](https://typhoon.psdn.io/addons/overview/) From efd4a0319db881512eb8a0377ad147ab833e5bd9 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Sat, 18 Jul 2020 13:54:56 -0700 Subject: [PATCH 482/523] Update Grafana from v7.0.6 to v7.1.0 * https://github.com/grafana/grafana/releases/tag/v7.1.0 --- CHANGES.md | 4 ++++ addons/grafana/deployment.yaml | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index 06d5aae78..f1604cab6 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -4,6 +4,10 @@ Notable changes between versions. ## Latest +#### Addons + +* Update Grafana from v7.0.6 to [v7.1.0](https://github.com/grafana/grafana/releases/tag/v7.1.0) + ## v1.18.6 * Kubernetes [v1.18.6](https://github.com/kubernetes/kubernetes/blob/master/CHANGELOG/CHANGELOG-1.18.md#v1186) diff --git a/addons/grafana/deployment.yaml b/addons/grafana/deployment.yaml index f7d68eb2a..96d45cc52 100644 --- a/addons/grafana/deployment.yaml +++ b/addons/grafana/deployment.yaml @@ -23,7 +23,7 @@ spec: spec: containers: - name: grafana - image: docker.io/grafana/grafana:7.0.6 + image: docker.io/grafana/grafana:7.1.0 env: - name: GF_PATHS_CONFIG value: "/etc/grafana/custom.ini" From f96e91f225657fea73357d22c1bdbfc2ab5350d1 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Sat, 18 Jul 2020 14:08:22 -0700 Subject: [PATCH 483/523] Update etcd from v3.4.9 to v3.4.10 * https://github.com/etcd-io/etcd/releases/tag/v3.4.10 --- CHANGES.md | 2 ++ aws/container-linux/kubernetes/cl/controller.yaml | 2 +- aws/fedora-coreos/kubernetes/fcc/controller.yaml | 2 +- azure/container-linux/kubernetes/cl/controller.yaml | 2 +- azure/fedora-coreos/kubernetes/fcc/controller.yaml | 2 +- bare-metal/container-linux/kubernetes/cl/controller.yaml | 2 +- bare-metal/fedora-coreos/kubernetes/fcc/controller.yaml | 2 +- digital-ocean/container-linux/kubernetes/cl/controller.yaml | 2 +- digital-ocean/fedora-coreos/kubernetes/fcc/controller.yaml | 2 +- google-cloud/container-linux/kubernetes/cl/controller.yaml | 2 +- google-cloud/fedora-coreos/kubernetes/fcc/controller.yaml | 2 +- 11 files changed, 12 insertions(+), 10 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index f1604cab6..0f5c3f478 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -4,6 +4,8 @@ Notable changes between versions. ## Latest +* Update etcd from v3.4.9 to [v3.4.10](https://github.com/etcd-io/etcd/releases/tag/v3.4.10) + #### Addons * Update Grafana from v7.0.6 to [v7.1.0](https://github.com/grafana/grafana/releases/tag/v7.1.0) diff --git a/aws/container-linux/kubernetes/cl/controller.yaml b/aws/container-linux/kubernetes/cl/controller.yaml index 62a744803..3aee27b7c 100644 --- a/aws/container-linux/kubernetes/cl/controller.yaml +++ b/aws/container-linux/kubernetes/cl/controller.yaml @@ -7,7 +7,7 @@ systemd: - name: 40-etcd-cluster.conf contents: | [Service] - Environment="ETCD_IMAGE_TAG=v3.4.9" + Environment="ETCD_IMAGE_TAG=v3.4.10" Environment="ETCD_IMAGE_URL=docker://quay.io/coreos/etcd" Environment="RKT_RUN_ARGS=--insecure-options=image" Environment="ETCD_NAME=${etcd_name}" diff --git a/aws/fedora-coreos/kubernetes/fcc/controller.yaml b/aws/fedora-coreos/kubernetes/fcc/controller.yaml index 3bd633056..f5d6e303b 100644 --- a/aws/fedora-coreos/kubernetes/fcc/controller.yaml +++ b/aws/fedora-coreos/kubernetes/fcc/controller.yaml @@ -28,7 +28,7 @@ systemd: --network host \ --volume /var/lib/etcd:/var/lib/etcd:rw,Z \ --volume /etc/ssl/etcd:/etc/ssl/certs:ro,Z \ - quay.io/coreos/etcd:v3.4.9 + quay.io/coreos/etcd:v3.4.10 ExecStop=/usr/bin/podman stop etcd [Install] WantedBy=multi-user.target diff --git a/azure/container-linux/kubernetes/cl/controller.yaml b/azure/container-linux/kubernetes/cl/controller.yaml index 62a744803..3aee27b7c 100644 --- a/azure/container-linux/kubernetes/cl/controller.yaml +++ b/azure/container-linux/kubernetes/cl/controller.yaml @@ -7,7 +7,7 @@ systemd: - name: 40-etcd-cluster.conf contents: | [Service] - Environment="ETCD_IMAGE_TAG=v3.4.9" + Environment="ETCD_IMAGE_TAG=v3.4.10" Environment="ETCD_IMAGE_URL=docker://quay.io/coreos/etcd" Environment="RKT_RUN_ARGS=--insecure-options=image" Environment="ETCD_NAME=${etcd_name}" diff --git a/azure/fedora-coreos/kubernetes/fcc/controller.yaml b/azure/fedora-coreos/kubernetes/fcc/controller.yaml index 84d5ba16f..ec532153d 100644 --- a/azure/fedora-coreos/kubernetes/fcc/controller.yaml +++ b/azure/fedora-coreos/kubernetes/fcc/controller.yaml @@ -28,7 +28,7 @@ systemd: --network host \ --volume /var/lib/etcd:/var/lib/etcd:rw,Z \ --volume /etc/ssl/etcd:/etc/ssl/certs:ro,Z \ - quay.io/coreos/etcd:v3.4.9 + quay.io/coreos/etcd:v3.4.10 ExecStop=/usr/bin/podman stop etcd [Install] WantedBy=multi-user.target diff --git a/bare-metal/container-linux/kubernetes/cl/controller.yaml b/bare-metal/container-linux/kubernetes/cl/controller.yaml index b316d579e..2ad89e7b0 100644 --- a/bare-metal/container-linux/kubernetes/cl/controller.yaml +++ b/bare-metal/container-linux/kubernetes/cl/controller.yaml @@ -7,7 +7,7 @@ systemd: - name: 40-etcd-cluster.conf contents: | [Service] - Environment="ETCD_IMAGE_TAG=v3.4.9" + Environment="ETCD_IMAGE_TAG=v3.4.10" Environment="ETCD_IMAGE_URL=docker://quay.io/coreos/etcd" Environment="RKT_RUN_ARGS=--insecure-options=image" Environment="ETCD_NAME=${etcd_name}" diff --git a/bare-metal/fedora-coreos/kubernetes/fcc/controller.yaml b/bare-metal/fedora-coreos/kubernetes/fcc/controller.yaml index ef40b396f..9c8e02862 100644 --- a/bare-metal/fedora-coreos/kubernetes/fcc/controller.yaml +++ b/bare-metal/fedora-coreos/kubernetes/fcc/controller.yaml @@ -28,7 +28,7 @@ systemd: --network host \ --volume /var/lib/etcd:/var/lib/etcd:rw,Z \ --volume /etc/ssl/etcd:/etc/ssl/certs:ro,Z \ - quay.io/coreos/etcd:v3.4.9 + quay.io/coreos/etcd:v3.4.10 ExecStop=/usr/bin/podman stop etcd [Install] WantedBy=multi-user.target diff --git a/digital-ocean/container-linux/kubernetes/cl/controller.yaml b/digital-ocean/container-linux/kubernetes/cl/controller.yaml index 719dbc0b1..80397a863 100644 --- a/digital-ocean/container-linux/kubernetes/cl/controller.yaml +++ b/digital-ocean/container-linux/kubernetes/cl/controller.yaml @@ -7,7 +7,7 @@ systemd: - name: 40-etcd-cluster.conf contents: | [Service] - Environment="ETCD_IMAGE_TAG=v3.4.9" + Environment="ETCD_IMAGE_TAG=v3.4.10" Environment="ETCD_IMAGE_URL=docker://quay.io/coreos/etcd" Environment="RKT_RUN_ARGS=--insecure-options=image" Environment="ETCD_NAME=${etcd_name}" diff --git a/digital-ocean/fedora-coreos/kubernetes/fcc/controller.yaml b/digital-ocean/fedora-coreos/kubernetes/fcc/controller.yaml index 4948a8861..372fae95b 100644 --- a/digital-ocean/fedora-coreos/kubernetes/fcc/controller.yaml +++ b/digital-ocean/fedora-coreos/kubernetes/fcc/controller.yaml @@ -28,7 +28,7 @@ systemd: --network host \ --volume /var/lib/etcd:/var/lib/etcd:rw,Z \ --volume /etc/ssl/etcd:/etc/ssl/certs:ro,Z \ - quay.io/coreos/etcd:v3.4.9 + quay.io/coreos/etcd:v3.4.10 ExecStop=/usr/bin/podman stop etcd [Install] WantedBy=multi-user.target diff --git a/google-cloud/container-linux/kubernetes/cl/controller.yaml b/google-cloud/container-linux/kubernetes/cl/controller.yaml index fbca032ae..fb11f5296 100644 --- a/google-cloud/container-linux/kubernetes/cl/controller.yaml +++ b/google-cloud/container-linux/kubernetes/cl/controller.yaml @@ -7,7 +7,7 @@ systemd: - name: 40-etcd-cluster.conf contents: | [Service] - Environment="ETCD_IMAGE_TAG=v3.4.9" + Environment="ETCD_IMAGE_TAG=v3.4.10" Environment="ETCD_IMAGE_URL=docker://quay.io/coreos/etcd" Environment="RKT_RUN_ARGS=--insecure-options=image" Environment="ETCD_NAME=${etcd_name}" diff --git a/google-cloud/fedora-coreos/kubernetes/fcc/controller.yaml b/google-cloud/fedora-coreos/kubernetes/fcc/controller.yaml index 6055eea47..61c42925c 100644 --- a/google-cloud/fedora-coreos/kubernetes/fcc/controller.yaml +++ b/google-cloud/fedora-coreos/kubernetes/fcc/controller.yaml @@ -28,7 +28,7 @@ systemd: --network host \ --volume /var/lib/etcd:/var/lib/etcd:rw,Z \ --volume /etc/ssl/etcd:/etc/ssl/certs:ro,Z \ - quay.io/coreos/etcd:v3.4.9 + quay.io/coreos/etcd:v3.4.10 ExecStop=/usr/bin/podman stop etcd [Install] WantedBy=multi-user.target From 264d23a1b5be6377358a7d35eb6e7ebc4f35584b Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Sat, 25 Jul 2020 13:50:08 -0700 Subject: [PATCH 484/523] Declare etcd data directory permissions * Set etcd data directory /var/lib/etcd permissions to 700 * On Flatcar Linux, /var/lib/etcd is pre-existing and Ignition v2 doesn't overwrite the directory. Update the Container Linux config, but add the manual chmod workaround to bootstrap for Flatcar Linux users * https://github.com/etcd-io/etcd/blob/master/CHANGELOG-3.4.md#v3410-2020-07-16 * https://github.com/etcd-io/etcd/pull/11798 --- aws/container-linux/kubernetes/cl/controller.yaml | 6 ++++++ aws/fedora-coreos/kubernetes/fcc/controller.yaml | 2 ++ azure/container-linux/kubernetes/cl/controller.yaml | 6 ++++++ azure/fedora-coreos/kubernetes/fcc/controller.yaml | 2 ++ bare-metal/container-linux/kubernetes/cl/controller.yaml | 5 +++++ bare-metal/fedora-coreos/kubernetes/fcc/controller.yaml | 2 ++ digital-ocean/container-linux/kubernetes/cl/controller.yaml | 5 +++++ digital-ocean/fedora-coreos/kubernetes/fcc/controller.yaml | 2 ++ google-cloud/container-linux/kubernetes/cl/controller.yaml | 6 ++++++ google-cloud/fedora-coreos/kubernetes/fcc/controller.yaml | 2 ++ 10 files changed, 38 insertions(+) diff --git a/aws/container-linux/kubernetes/cl/controller.yaml b/aws/container-linux/kubernetes/cl/controller.yaml index 3aee27b7c..ff15c3c6e 100644 --- a/aws/container-linux/kubernetes/cl/controller.yaml +++ b/aws/container-linux/kubernetes/cl/controller.yaml @@ -142,6 +142,11 @@ systemd: [Install] WantedBy=multi-user.target storage: + directories: + - path: /var/lib/etcd + filesystem: root + mode: 0700 + overwrite: true files: - path: /etc/kubernetes/kubeconfig filesystem: root @@ -163,6 +168,7 @@ storage: mv tls/etcd/etcd-client* /etc/kubernetes/bootstrap-secrets/ chown -R etcd:etcd /etc/ssl/etcd chmod -R 500 /etc/ssl/etcd + chmod -R 700 /var/lib/etcd mv auth/kubeconfig /etc/kubernetes/bootstrap-secrets/ mv tls/k8s/* /etc/kubernetes/bootstrap-secrets/ mkdir -p /etc/kubernetes/manifests diff --git a/aws/fedora-coreos/kubernetes/fcc/controller.yaml b/aws/fedora-coreos/kubernetes/fcc/controller.yaml index f5d6e303b..0d8b4c35c 100644 --- a/aws/fedora-coreos/kubernetes/fcc/controller.yaml +++ b/aws/fedora-coreos/kubernetes/fcc/controller.yaml @@ -129,6 +129,8 @@ systemd: ExecStartPost=-/usr/bin/podman stop bootstrap storage: directories: + - path: /var/lib/etcd + mode: 0700 - path: /etc/kubernetes - path: /opt/bootstrap files: diff --git a/azure/container-linux/kubernetes/cl/controller.yaml b/azure/container-linux/kubernetes/cl/controller.yaml index 3aee27b7c..ff15c3c6e 100644 --- a/azure/container-linux/kubernetes/cl/controller.yaml +++ b/azure/container-linux/kubernetes/cl/controller.yaml @@ -142,6 +142,11 @@ systemd: [Install] WantedBy=multi-user.target storage: + directories: + - path: /var/lib/etcd + filesystem: root + mode: 0700 + overwrite: true files: - path: /etc/kubernetes/kubeconfig filesystem: root @@ -163,6 +168,7 @@ storage: mv tls/etcd/etcd-client* /etc/kubernetes/bootstrap-secrets/ chown -R etcd:etcd /etc/ssl/etcd chmod -R 500 /etc/ssl/etcd + chmod -R 700 /var/lib/etcd mv auth/kubeconfig /etc/kubernetes/bootstrap-secrets/ mv tls/k8s/* /etc/kubernetes/bootstrap-secrets/ mkdir -p /etc/kubernetes/manifests diff --git a/azure/fedora-coreos/kubernetes/fcc/controller.yaml b/azure/fedora-coreos/kubernetes/fcc/controller.yaml index ec532153d..a337e9a86 100644 --- a/azure/fedora-coreos/kubernetes/fcc/controller.yaml +++ b/azure/fedora-coreos/kubernetes/fcc/controller.yaml @@ -128,6 +128,8 @@ systemd: ExecStartPost=-/usr/bin/podman stop bootstrap storage: directories: + - path: /var/lib/etcd + mode: 0700 - path: /etc/kubernetes - path: /opt/bootstrap files: diff --git a/bare-metal/container-linux/kubernetes/cl/controller.yaml b/bare-metal/container-linux/kubernetes/cl/controller.yaml index 2ad89e7b0..7e72223dc 100644 --- a/bare-metal/container-linux/kubernetes/cl/controller.yaml +++ b/bare-metal/container-linux/kubernetes/cl/controller.yaml @@ -156,6 +156,10 @@ systemd: WantedBy=multi-user.target storage: directories: + - path: /var/lib/etcd + filesystem: root + mode: 0700 + overwrite: true - path: /etc/kubernetes filesystem: root mode: 0755 @@ -180,6 +184,7 @@ storage: mv tls/etcd/etcd-client* /etc/kubernetes/bootstrap-secrets/ chown -R etcd:etcd /etc/ssl/etcd chmod -R 500 /etc/ssl/etcd + chmod -R 700 /var/lib/etcd mv auth/kubeconfig /etc/kubernetes/bootstrap-secrets/ mv tls/k8s/* /etc/kubernetes/bootstrap-secrets/ mkdir -p /etc/kubernetes/manifests diff --git a/bare-metal/fedora-coreos/kubernetes/fcc/controller.yaml b/bare-metal/fedora-coreos/kubernetes/fcc/controller.yaml index 9c8e02862..2f6c9ee73 100644 --- a/bare-metal/fedora-coreos/kubernetes/fcc/controller.yaml +++ b/bare-metal/fedora-coreos/kubernetes/fcc/controller.yaml @@ -139,6 +139,8 @@ systemd: ExecStartPost=-/usr/bin/podman stop bootstrap storage: directories: + - path: /var/lib/etcd + mode: 0700 - path: /etc/kubernetes - path: /opt/bootstrap files: diff --git a/digital-ocean/container-linux/kubernetes/cl/controller.yaml b/digital-ocean/container-linux/kubernetes/cl/controller.yaml index 80397a863..415f3305f 100644 --- a/digital-ocean/container-linux/kubernetes/cl/controller.yaml +++ b/digital-ocean/container-linux/kubernetes/cl/controller.yaml @@ -153,6 +153,10 @@ systemd: WantedBy=multi-user.target storage: directories: + - path: /var/lib/etcd + filesystem: root + mode: 0700 + overwrite: true - path: /etc/kubernetes filesystem: root mode: 0755 @@ -171,6 +175,7 @@ storage: mv tls/etcd/etcd-client* /etc/kubernetes/bootstrap-secrets/ chown -R etcd:etcd /etc/ssl/etcd chmod -R 500 /etc/ssl/etcd + chmod -R 700 /var/lib/etcd mv auth/kubeconfig /etc/kubernetes/bootstrap-secrets/ mv tls/k8s/* /etc/kubernetes/bootstrap-secrets/ mkdir -p /etc/kubernetes/manifests diff --git a/digital-ocean/fedora-coreos/kubernetes/fcc/controller.yaml b/digital-ocean/fedora-coreos/kubernetes/fcc/controller.yaml index 372fae95b..a819e6a66 100644 --- a/digital-ocean/fedora-coreos/kubernetes/fcc/controller.yaml +++ b/digital-ocean/fedora-coreos/kubernetes/fcc/controller.yaml @@ -140,6 +140,8 @@ systemd: ExecStartPost=-/usr/bin/podman stop bootstrap storage: directories: + - path: /var/lib/etcd + mode: 0700 - path: /etc/kubernetes - path: /opt/bootstrap files: diff --git a/google-cloud/container-linux/kubernetes/cl/controller.yaml b/google-cloud/container-linux/kubernetes/cl/controller.yaml index fb11f5296..176529490 100644 --- a/google-cloud/container-linux/kubernetes/cl/controller.yaml +++ b/google-cloud/container-linux/kubernetes/cl/controller.yaml @@ -140,6 +140,11 @@ systemd: [Install] WantedBy=multi-user.target storage: + directories: + - path: /var/lib/etcd + filesystem: root + mode: 0700 + overwrite: true files: - path: /etc/kubernetes/kubeconfig filesystem: root @@ -161,6 +166,7 @@ storage: mv tls/etcd/etcd-client* /etc/kubernetes/bootstrap-secrets/ chown -R etcd:etcd /etc/ssl/etcd chmod -R 500 /etc/ssl/etcd + chmod -R 700 /var/lib/etcd mv auth/kubeconfig /etc/kubernetes/bootstrap-secrets/ mv tls/k8s/* /etc/kubernetes/bootstrap-secrets/ mkdir -p /etc/kubernetes/manifests diff --git a/google-cloud/fedora-coreos/kubernetes/fcc/controller.yaml b/google-cloud/fedora-coreos/kubernetes/fcc/controller.yaml index 61c42925c..8edb5261e 100644 --- a/google-cloud/fedora-coreos/kubernetes/fcc/controller.yaml +++ b/google-cloud/fedora-coreos/kubernetes/fcc/controller.yaml @@ -128,6 +128,8 @@ systemd: ExecStartPost=-/usr/bin/podman stop bootstrap storage: directories: + - path: /var/lib/etcd + mode: 0700 - path: /etc/kubernetes - path: /opt/bootstrap files: From 618f8b30fdde2396962230f647791c5a9d91484a Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Sat, 25 Jul 2020 13:18:07 -0700 Subject: [PATCH 485/523] Update CoreDNS from v1.6.7 to v1.7.0 * https://coredns.io/2020/06/15/coredns-1.7.0-release/ * Update Grafana dashboard with revised metrics names --- CHANGES.md | 1 + addons/grafana/dashboards-coredns.yaml | 10 +++++----- aws/container-linux/kubernetes/bootstrap.tf | 2 +- aws/fedora-coreos/kubernetes/bootstrap.tf | 2 +- azure/container-linux/kubernetes/bootstrap.tf | 2 +- azure/fedora-coreos/kubernetes/bootstrap.tf | 2 +- bare-metal/container-linux/kubernetes/bootstrap.tf | 2 +- bare-metal/fedora-coreos/kubernetes/bootstrap.tf | 2 +- digital-ocean/container-linux/kubernetes/bootstrap.tf | 2 +- digital-ocean/fedora-coreos/kubernetes/bootstrap.tf | 2 +- google-cloud/container-linux/kubernetes/bootstrap.tf | 2 +- google-cloud/fedora-coreos/kubernetes/bootstrap.tf | 2 +- 12 files changed, 16 insertions(+), 15 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 0f5c3f478..36b58e29c 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -5,6 +5,7 @@ Notable changes between versions. ## Latest * Update etcd from v3.4.9 to [v3.4.10](https://github.com/etcd-io/etcd/releases/tag/v3.4.10) +* Update CoreDNS from v1.6.7 to [v1.7.0](https://coredns.io/2020/06/15/coredns-1.7.0-release/) #### Addons diff --git a/addons/grafana/dashboards-coredns.yaml b/addons/grafana/dashboards-coredns.yaml index eb4395b79..8bd507aaa 100644 --- a/addons/grafana/dashboards-coredns.yaml +++ b/addons/grafana/dashboards-coredns.yaml @@ -72,7 +72,7 @@ data: "steppedLine": false, "targets": [ { - "expr": "sum(rate(coredns_dns_request_count_total{instance=~\"$instance\"}[5m])) by (proto)", + "expr": "sum(rate(coredns_dns_requests_total{instance=~\"$instance\"}[5m])) by (proto)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{proto}}", @@ -163,7 +163,7 @@ data: "steppedLine": false, "targets": [ { - "expr": "sum(rate(coredns_dns_request_type_count_total{instance=~\"$instance\"}[5m])) by (type)", + "expr": "sum(rate(coredns_dns_requests_total{instance=~\"$instance\"}[5m])) by (type)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{type}}", @@ -254,7 +254,7 @@ data: "steppedLine": false, "targets": [ { - "expr": "sum(rate(coredns_dns_request_count_total{instance=~\"$instance\"}[5m])) by (zone)", + "expr": "sum(rate(coredns_dns_requests_total{instance=~\"$instance\"}[5m])) by (zone)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{zone}}", @@ -463,7 +463,7 @@ data: "steppedLine": false, "targets": [ { - "expr": "sum(rate(coredns_dns_response_rcode_count_total{instance=~\"$instance\"}[5m])) by (rcode)", + "expr": "sum(rate(coredns_dns_responses_total{instance=~\"$instance\"}[5m])) by (rcode)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{rcode}}", @@ -790,7 +790,7 @@ data: "steppedLine": false, "targets": [ { - "expr": "sum(coredns_cache_size{instance=~\"$instance\"}) by (type)", + "expr": "sum(coredns_cache_entries{instance=~\"$instance\"}) by (type)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{type}}", diff --git a/aws/container-linux/kubernetes/bootstrap.tf b/aws/container-linux/kubernetes/bootstrap.tf index 8235977ba..b2cfdc1d8 100644 --- a/aws/container-linux/kubernetes/bootstrap.tf +++ b/aws/container-linux/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=835890025bfb3499aa0cd5a9704e9e7d3e7e5fe5" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=9de4267c287135fb6ed00a72aa875033b4c5d8f6" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/aws/fedora-coreos/kubernetes/bootstrap.tf b/aws/fedora-coreos/kubernetes/bootstrap.tf index 2d13be75b..590e80859 100644 --- a/aws/fedora-coreos/kubernetes/bootstrap.tf +++ b/aws/fedora-coreos/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=835890025bfb3499aa0cd5a9704e9e7d3e7e5fe5" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=9de4267c287135fb6ed00a72aa875033b4c5d8f6" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/azure/container-linux/kubernetes/bootstrap.tf b/azure/container-linux/kubernetes/bootstrap.tf index 209c303c4..ec1884f7e 100644 --- a/azure/container-linux/kubernetes/bootstrap.tf +++ b/azure/container-linux/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=835890025bfb3499aa0cd5a9704e9e7d3e7e5fe5" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=9de4267c287135fb6ed00a72aa875033b4c5d8f6" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/azure/fedora-coreos/kubernetes/bootstrap.tf b/azure/fedora-coreos/kubernetes/bootstrap.tf index 33349e136..e051ee209 100644 --- a/azure/fedora-coreos/kubernetes/bootstrap.tf +++ b/azure/fedora-coreos/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=835890025bfb3499aa0cd5a9704e9e7d3e7e5fe5" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=9de4267c287135fb6ed00a72aa875033b4c5d8f6" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/bare-metal/container-linux/kubernetes/bootstrap.tf b/bare-metal/container-linux/kubernetes/bootstrap.tf index 57cdfb728..8f325997b 100644 --- a/bare-metal/container-linux/kubernetes/bootstrap.tf +++ b/bare-metal/container-linux/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=835890025bfb3499aa0cd5a9704e9e7d3e7e5fe5" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=9de4267c287135fb6ed00a72aa875033b4c5d8f6" cluster_name = var.cluster_name api_servers = [var.k8s_domain_name] diff --git a/bare-metal/fedora-coreos/kubernetes/bootstrap.tf b/bare-metal/fedora-coreos/kubernetes/bootstrap.tf index 54769fdaa..3224240c1 100644 --- a/bare-metal/fedora-coreos/kubernetes/bootstrap.tf +++ b/bare-metal/fedora-coreos/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=835890025bfb3499aa0cd5a9704e9e7d3e7e5fe5" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=9de4267c287135fb6ed00a72aa875033b4c5d8f6" cluster_name = var.cluster_name api_servers = [var.k8s_domain_name] diff --git a/digital-ocean/container-linux/kubernetes/bootstrap.tf b/digital-ocean/container-linux/kubernetes/bootstrap.tf index e4874eb5c..8f6b53ab2 100644 --- a/digital-ocean/container-linux/kubernetes/bootstrap.tf +++ b/digital-ocean/container-linux/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=835890025bfb3499aa0cd5a9704e9e7d3e7e5fe5" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=9de4267c287135fb6ed00a72aa875033b4c5d8f6" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/digital-ocean/fedora-coreos/kubernetes/bootstrap.tf b/digital-ocean/fedora-coreos/kubernetes/bootstrap.tf index 857221705..9539e84bd 100644 --- a/digital-ocean/fedora-coreos/kubernetes/bootstrap.tf +++ b/digital-ocean/fedora-coreos/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=835890025bfb3499aa0cd5a9704e9e7d3e7e5fe5" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=9de4267c287135fb6ed00a72aa875033b4c5d8f6" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/google-cloud/container-linux/kubernetes/bootstrap.tf b/google-cloud/container-linux/kubernetes/bootstrap.tf index 7b9222258..3c2ee7bc3 100644 --- a/google-cloud/container-linux/kubernetes/bootstrap.tf +++ b/google-cloud/container-linux/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=835890025bfb3499aa0cd5a9704e9e7d3e7e5fe5" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=9de4267c287135fb6ed00a72aa875033b4c5d8f6" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/google-cloud/fedora-coreos/kubernetes/bootstrap.tf b/google-cloud/fedora-coreos/kubernetes/bootstrap.tf index fe3b10276..4a11ae3a2 100644 --- a/google-cloud/fedora-coreos/kubernetes/bootstrap.tf +++ b/google-cloud/fedora-coreos/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=835890025bfb3499aa0cd5a9704e9e7d3e7e5fe5" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=9de4267c287135fb6ed00a72aa875033b4c5d8f6" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] From cd0a28904eb30406f0d84f556fd98f97ad7c6dd7 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Sat, 25 Jul 2020 16:06:27 -0700 Subject: [PATCH 486/523] Update Cilium from v1.8.1 to v1.8.2 * https://github.com/cilium/cilium/releases/tag/v1.8.2 --- CHANGES.md | 1 + aws/container-linux/kubernetes/bootstrap.tf | 2 +- aws/fedora-coreos/kubernetes/bootstrap.tf | 2 +- azure/container-linux/kubernetes/bootstrap.tf | 2 +- azure/fedora-coreos/kubernetes/bootstrap.tf | 2 +- bare-metal/container-linux/kubernetes/bootstrap.tf | 2 +- bare-metal/fedora-coreos/kubernetes/bootstrap.tf | 2 +- digital-ocean/container-linux/kubernetes/bootstrap.tf | 2 +- digital-ocean/fedora-coreos/kubernetes/bootstrap.tf | 2 +- google-cloud/container-linux/kubernetes/bootstrap.tf | 2 +- google-cloud/fedora-coreos/kubernetes/bootstrap.tf | 2 +- 11 files changed, 11 insertions(+), 10 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 36b58e29c..afaff6275 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -6,6 +6,7 @@ Notable changes between versions. * Update etcd from v3.4.9 to [v3.4.10](https://github.com/etcd-io/etcd/releases/tag/v3.4.10) * Update CoreDNS from v1.6.7 to [v1.7.0](https://coredns.io/2020/06/15/coredns-1.7.0-release/) +* Update Cilium from v1.8.1 to [v1.8.2](https://github.com/cilium/cilium/releases/tag/v1.8.2) #### Addons diff --git a/aws/container-linux/kubernetes/bootstrap.tf b/aws/container-linux/kubernetes/bootstrap.tf index b2cfdc1d8..480ceda36 100644 --- a/aws/container-linux/kubernetes/bootstrap.tf +++ b/aws/container-linux/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=9de4267c287135fb6ed00a72aa875033b4c5d8f6" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=45053a62cb751e2f8b371c20b7714e1604d6e57d" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/aws/fedora-coreos/kubernetes/bootstrap.tf b/aws/fedora-coreos/kubernetes/bootstrap.tf index 590e80859..cceb3bfbe 100644 --- a/aws/fedora-coreos/kubernetes/bootstrap.tf +++ b/aws/fedora-coreos/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=9de4267c287135fb6ed00a72aa875033b4c5d8f6" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=45053a62cb751e2f8b371c20b7714e1604d6e57d" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/azure/container-linux/kubernetes/bootstrap.tf b/azure/container-linux/kubernetes/bootstrap.tf index ec1884f7e..f3f59911a 100644 --- a/azure/container-linux/kubernetes/bootstrap.tf +++ b/azure/container-linux/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=9de4267c287135fb6ed00a72aa875033b4c5d8f6" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=45053a62cb751e2f8b371c20b7714e1604d6e57d" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/azure/fedora-coreos/kubernetes/bootstrap.tf b/azure/fedora-coreos/kubernetes/bootstrap.tf index e051ee209..5aad4effc 100644 --- a/azure/fedora-coreos/kubernetes/bootstrap.tf +++ b/azure/fedora-coreos/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=9de4267c287135fb6ed00a72aa875033b4c5d8f6" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=45053a62cb751e2f8b371c20b7714e1604d6e57d" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/bare-metal/container-linux/kubernetes/bootstrap.tf b/bare-metal/container-linux/kubernetes/bootstrap.tf index 8f325997b..c819e5325 100644 --- a/bare-metal/container-linux/kubernetes/bootstrap.tf +++ b/bare-metal/container-linux/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=9de4267c287135fb6ed00a72aa875033b4c5d8f6" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=45053a62cb751e2f8b371c20b7714e1604d6e57d" cluster_name = var.cluster_name api_servers = [var.k8s_domain_name] diff --git a/bare-metal/fedora-coreos/kubernetes/bootstrap.tf b/bare-metal/fedora-coreos/kubernetes/bootstrap.tf index 3224240c1..3e383ea71 100644 --- a/bare-metal/fedora-coreos/kubernetes/bootstrap.tf +++ b/bare-metal/fedora-coreos/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=9de4267c287135fb6ed00a72aa875033b4c5d8f6" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=45053a62cb751e2f8b371c20b7714e1604d6e57d" cluster_name = var.cluster_name api_servers = [var.k8s_domain_name] diff --git a/digital-ocean/container-linux/kubernetes/bootstrap.tf b/digital-ocean/container-linux/kubernetes/bootstrap.tf index 8f6b53ab2..7c6c2ce82 100644 --- a/digital-ocean/container-linux/kubernetes/bootstrap.tf +++ b/digital-ocean/container-linux/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=9de4267c287135fb6ed00a72aa875033b4c5d8f6" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=45053a62cb751e2f8b371c20b7714e1604d6e57d" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/digital-ocean/fedora-coreos/kubernetes/bootstrap.tf b/digital-ocean/fedora-coreos/kubernetes/bootstrap.tf index 9539e84bd..c89b9f6f5 100644 --- a/digital-ocean/fedora-coreos/kubernetes/bootstrap.tf +++ b/digital-ocean/fedora-coreos/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=9de4267c287135fb6ed00a72aa875033b4c5d8f6" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=45053a62cb751e2f8b371c20b7714e1604d6e57d" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/google-cloud/container-linux/kubernetes/bootstrap.tf b/google-cloud/container-linux/kubernetes/bootstrap.tf index 3c2ee7bc3..3d0a9db34 100644 --- a/google-cloud/container-linux/kubernetes/bootstrap.tf +++ b/google-cloud/container-linux/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=9de4267c287135fb6ed00a72aa875033b4c5d8f6" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=45053a62cb751e2f8b371c20b7714e1604d6e57d" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/google-cloud/fedora-coreos/kubernetes/bootstrap.tf b/google-cloud/fedora-coreos/kubernetes/bootstrap.tf index 4a11ae3a2..5697c2ddd 100644 --- a/google-cloud/fedora-coreos/kubernetes/bootstrap.tf +++ b/google-cloud/fedora-coreos/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=9de4267c287135fb6ed00a72aa875033b4c5d8f6" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=45053a62cb751e2f8b371c20b7714e1604d6e57d" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] From 26f5d2d753d0ae5a9ed0ad91bd320817a8fe4c8b Mon Sep 17 00:00:00 2001 From: Eldon Date: Sat, 25 Jul 2020 23:32:08 +0000 Subject: [PATCH 487/523] Fix some links in docs (#788) --- docs/advanced/customization.md | 6 +++--- docs/index.md | 2 +- docs/topics/faq.md | 2 +- docs/topics/hardware.md | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/docs/advanced/customization.md b/docs/advanced/customization.md index c92d1adbf..1c87aa94e 100644 --- a/docs/advanced/customization.md +++ b/docs/advanced/customization.md @@ -83,7 +83,7 @@ module "mercury" { } ``` -### Container Linux +### Flatcar Linux Define a Container Linux Config (CLC) ([config](https://github.com/coreos/container-linux-config-transpiler/blob/master/doc/configuration.md), [examples](https://github.com/coreos/container-linux-config-transpiler/blob/master/doc/examples.md)) in version control near your Terraform workspace directory (e.g. perhaps in a `snippets` subdirectory). You may organize snippets into multiple files, if desired. @@ -125,7 +125,7 @@ systemd: Environment="ETCD_LOG_PACKAGE_LEVELS=etcdserver=WARNING,security=DEBUG" ``` -Reference the CLC contents by location (e.g. `file("./custom-units.yaml")`). On [AWS](/cl/aws/#cluster), [Azure](/cl/azure/#cluster), [DigitalOcean](/cl/digital-ocean/#cluster), or [Google Cloud](/cl/google-cloud/#cluster) extend the `controller_snippets` or `worker_snippets` list variables. +Reference the CLC contents by location (e.g. `file("./custom-units.yaml")`). On [AWS](/flatcar-linux/aws/#cluster), [Azure](/flatcar-linux/azure/#cluster), [DigitalOcean](/flatcar-linux/digital-ocean/#cluster), or [Google Cloud](/flatcar-linux/google-cloud/#cluster) extend the `controller_snippets` or `worker_snippets` list variables. ```tf module "nemo" { @@ -145,7 +145,7 @@ module "nemo" { } ``` -On [Bare-Metal](/cl/bare-metal/#cluster), different CLCs may be used for each node (since hardware may be heterogeneous). Extend the `snippets` map variable by mapping a controller or worker name key to a list of snippets. +On [Bare-Metal](/flatcar-linux/bare-metal/#cluster), different CLCs may be used for each node (since hardware may be heterogeneous). Extend the `snippets` map variable by mapping a controller or worker name key to a list of snippets. ```tf module "mercury" { diff --git a/docs/index.md b/docs/index.md index 10a598254..2966ed2fd 100644 --- a/docs/index.md +++ b/docs/index.md @@ -29,7 +29,7 @@ Typhoon is available for [Fedora CoreOS](https://getfedora.org/coreos/). | Azure | Fedora CoreOS | [azure/fedora-coreos/kubernetes](fedora-coreos/azure.md) | alpha | | Bare-Metal | Fedora CoreOS | [bare-metal/fedora-coreos/kubernetes](fedora-coreos/bare-metal.md) | beta | | DigitalOcean | Fedora CoreOS | [digital-ocean/fedora-coreos/kubernetes](fedora-coreos/digitalocean.md) | beta | -| Google Cloud | Fedora CoreOS | [google-cloud/fedora-coreos/kubernetes](google-cloud/fedora-coreos/kubernetes) | stable | +| Google Cloud | Fedora CoreOS | [google-cloud/fedora-coreos/kubernetes](fedora-coreos/google-cloud/kubernetes) | stable | Typhoon is available for [Flatcar Linux](https://www.flatcar-linux.org/releases/). diff --git a/docs/topics/faq.md b/docs/topics/faq.md index d661e7aa3..3a8fa7a77 100644 --- a/docs/topics/faq.md +++ b/docs/topics/faq.md @@ -12,7 +12,7 @@ Ask questions on the IRC #typhoon channel on [freenode.net](http://freenode.net/ ## Security Issues -If you find security issues, please see [security disclosures](/topics/security.md#disclosures). +If you find security issues, please see [security disclosures](/topics/security/#disclosures). ## Maintainers diff --git a/docs/topics/hardware.md b/docs/topics/hardware.md index d95ad2275..6087c62b0 100644 --- a/docs/topics/hardware.md +++ b/docs/topics/hardware.md @@ -183,7 +183,7 @@ show ip route bgp ### Port Forwarding -Expose the [Ingress Controller](/addons/ingress.md#bare-metal) by adding `port-forward` rules that DNAT a port on the router's WAN interface to an internal IP and port. By convention, a public Ingress controller is assigned a fixed service IP (e.g. 10.3.0.12). +Expose the [Ingress Controller](/addons/ingress/#bare-metal) by adding `port-forward` rules that DNAT a port on the router's WAN interface to an internal IP and port. By convention, a public Ingress controller is assigned a fixed service IP (e.g. 10.3.0.12). ``` configure From b7d67757def4dccb6cf0e7e7f24da1848cc291a9 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Sat, 25 Jul 2020 16:33:40 -0700 Subject: [PATCH 488/523] Update Grafana from v7.1.0 to v7.1.1 * https://github.com/grafana/grafana/releases/tag/v7.1.1 --- CHANGES.md | 2 +- addons/grafana/deployment.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index afaff6275..5ea23e690 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -10,7 +10,7 @@ Notable changes between versions. #### Addons -* Update Grafana from v7.0.6 to [v7.1.0](https://github.com/grafana/grafana/releases/tag/v7.1.0) +* Update Grafana from v7.0.6 to [v7.1.1](https://github.com/grafana/grafana/releases/tag/v7.1.1) ## v1.18.6 diff --git a/addons/grafana/deployment.yaml b/addons/grafana/deployment.yaml index 96d45cc52..f25a80e08 100644 --- a/addons/grafana/deployment.yaml +++ b/addons/grafana/deployment.yaml @@ -23,7 +23,7 @@ spec: spec: containers: - name: grafana - image: docker.io/grafana/grafana:7.1.0 + image: docker.io/grafana/grafana:7.1.1 env: - name: GF_PATHS_CONFIG value: "/etc/grafana/custom.ini" From 2aef42d4f62ddf1c646a96c09d0d52d3c86c5a81 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Sat, 25 Jul 2020 16:37:28 -0700 Subject: [PATCH 489/523] Update Prometheus from v2.19.2 to v2.20.0 * https://github.com/prometheus/prometheus/releases/tag/v2.20.0 --- CHANGES.md | 1 + addons/prometheus/deployment.yaml | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index 5ea23e690..3de6d207c 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -10,6 +10,7 @@ Notable changes between versions. #### Addons +* Update Prometheus from v2.19.2 to [v2.20.0](https://github.com/prometheus/prometheus/releases/tag/v2.20.0) * Update Grafana from v7.0.6 to [v7.1.1](https://github.com/grafana/grafana/releases/tag/v7.1.1) ## v1.18.6 diff --git a/addons/prometheus/deployment.yaml b/addons/prometheus/deployment.yaml index d9077b27e..a0dbc4832 100644 --- a/addons/prometheus/deployment.yaml +++ b/addons/prometheus/deployment.yaml @@ -20,7 +20,7 @@ spec: serviceAccountName: prometheus containers: - name: prometheus - image: quay.io/prometheus/prometheus:v2.19.2 + image: quay.io/prometheus/prometheus:v2.20.0 args: - --web.listen-address=0.0.0.0:9090 - --config.file=/etc/prometheus/prometheus.yaml From bcf6f57fd5b2ba7dde89f67b86bbc49bea316ff8 Mon Sep 17 00:00:00 2001 From: Bo Huang Date: Fri, 31 Jul 2020 12:44:11 -0700 Subject: [PATCH 490/523] Migrate to Fedora CoreOS --- aws/fedora-coreos/kubernetes/bastion.tf | 247 ++++++++++++++++++ aws/fedora-coreos/kubernetes/bootstrap.tf | 3 + aws/fedora-coreos/kubernetes/controllers.tf | 98 ++++++- .../kubernetes/fcc/controller.yaml | 7 +- .../kubernetes/ignition-configs-bucket.tf | 15 ++ aws/fedora-coreos/kubernetes/network.tf | 100 ++++++- aws/fedora-coreos/kubernetes/nlb.tf | 30 +-- aws/fedora-coreos/kubernetes/outputs.tf | 79 +++++- aws/fedora-coreos/kubernetes/security.tf | 42 ++- aws/fedora-coreos/kubernetes/ssh.tf | 29 +- aws/fedora-coreos/kubernetes/variables.tf | 63 +++++ aws/fedora-coreos/kubernetes/workers.tf | 12 +- .../kubernetes/workers/fcc/worker.yaml | 7 +- .../kubernetes/workers/outputs.tf | 9 + .../kubernetes/workers/variables.tf | 33 ++- .../kubernetes/workers/workers.tf | 102 +++++++- 16 files changed, 779 insertions(+), 97 deletions(-) create mode 100644 aws/fedora-coreos/kubernetes/bastion.tf create mode 100644 aws/fedora-coreos/kubernetes/ignition-configs-bucket.tf diff --git a/aws/fedora-coreos/kubernetes/bastion.tf b/aws/fedora-coreos/kubernetes/bastion.tf new file mode 100644 index 000000000..989a7974e --- /dev/null +++ b/aws/fedora-coreos/kubernetes/bastion.tf @@ -0,0 +1,247 @@ +locals { + user_data_bastion = { + ignition = { + version = "3.0.0" + config = { + merge = [ + { + source = var.base_ignition_config_path + }, + { + source = "s3://${aws_s3_bucket.ignition_configs.id}/${aws_s3_bucket_object.bastion_config.id}" + } + ] + } + } + } +} + +resource "aws_autoscaling_group" "bastion" { + name = "${var.cluster_name}-bastion ${aws_launch_configuration.bastion.name}" + + # count + desired_capacity = var.bastion_count + min_size = var.bastion_count + max_size = var.bastion_count + default_cooldown = 30 + health_check_grace_period = 30 + + # network + vpc_zone_identifier = aws_subnet.private.*.id + + # template + launch_configuration = aws_launch_configuration.bastion.name + + # target groups to which instances should be added + target_group_arns = [ + aws_lb_target_group.bastion.id + ] + + min_elb_capacity = 1 + + lifecycle { + # override the default destroy and replace update behavior + create_before_destroy = true + ignore_changes = [launch_configuration] + } + + tags = [{ + key = "Name" + value = "${var.cluster_name}-bastion" + propagate_at_launch = true + }] +} + +resource "aws_launch_configuration" "bastion" { + image_id = coalesce(var.ami, data.aws_ami.fedora-coreos.image_id) + instance_type = var.bastion_type + + user_data = jsonencode(local.user_data_bastion) + + # network + security_groups = [ + aws_security_group.bastion_external.id + ] + + # iam + iam_instance_profile = aws_iam_instance_profile.bastion.name + + lifecycle { + // Override the default destroy and replace update behavior + create_before_destroy = true + } +} + +resource "aws_s3_bucket_object" "bastion_config" { + bucket = aws_s3_bucket.ignition_configs.id + key = "bastion.json" + content = data.ct_config.bastion_ign.rendered +} + +data "template_file" "bastion_config" { + template = file("${path.module}/cl/bastion.yaml.tmpl") +} + +data "ct_config" "bastion_ign" { + content = data.template_file.bastion_config.rendered + pretty_print = false + snippets = var.bastion_clc_snippets +} + +resource "aws_security_group" "bastion_external" { + name_prefix = "${var.cluster_name}-bastion-external-" + description = "Allows access to the bastion from the internet" + + vpc_id = aws_vpc.network.id + + tags = { + Name = "${var.cluster_name}-bastion-external" + } + + ingress { + protocol = "tcp" + from_port = 22 + to_port = 22 + cidr_blocks = ["0.0.0.0/0"] + } + + egress { + protocol = -1 + from_port = 0 + to_port = 0 + cidr_blocks = ["0.0.0.0/0"] + } + + lifecycle { + create_before_destroy = true + } +} + +resource "aws_security_group" "bastion_internal" { + name_prefix = "${var.cluster_name}-bastion-internal-" + description = "Allows access to a host from the bastion" + + vpc_id = aws_vpc.network.id + + tags = { + Name = "${var.cluster_name}-bastion-internal" + } + + ingress { + protocol = "tcp" + from_port = 22 + to_port = 22 + security_groups = [aws_security_group.bastion_external.id] + } + + lifecycle { + create_before_destroy = true + } +} + +resource "aws_lb" "bastion" { + name = "${var.cluster_name}-bastion" + load_balancer_type = "network" + + subnets = aws_subnet.public.*.id +} + + +resource "aws_lb_listener" "bastion" { + load_balancer_arn = aws_lb.bastion.arn + protocol = "TCP" + port = "22" + + default_action { + type = "forward" + target_group_arn = aws_lb_target_group.bastion.arn + } +} + +resource "aws_lb_target_group" "bastion" { + name = "${var.cluster_name}-bastion" + vpc_id = aws_vpc.network.id + target_type = "instance" + + protocol = "TCP" + port = 22 + + health_check { + protocol = "TCP" + port = 22 + + healthy_threshold = 3 + unhealthy_threshold = 3 + + interval = 10 + } +} + +resource "aws_route53_record" "bastion" { + depends_on = [aws_autoscaling_group.bastion] + + zone_id = var.dns_zone_id + + name = format("bastion.%s.%s.", var.cluster_name, var.dns_zone) + type = "A" + + alias { + name = aws_lb.bastion.dns_name + zone_id = aws_lb.bastion.zone_id + evaluate_target_health = false + } +} + +resource "aws_iam_role_policy" "bastion_read_base_ignition_config" { + name = "read-base-ignition-config" + role = aws_iam_role.bastion.id + policy = var.base_ignition_config_read_policy +} + +resource "aws_iam_role_policy" "bastion_read_ignition_config" { + name = "read-ignition-configs" + role = aws_iam_role.bastion.id + policy = data.aws_iam_policy_document.bastion_read_ignition_config.json +} + +data "aws_iam_policy_document" "bastion_read_ignition_config" { + statement { + effect = "Allow" + actions = ["s3:GetObject"] + resources = ["${aws_s3_bucket.ignition_configs.arn}/${aws_s3_bucket_object.bastion_config.id}"] + } +} + +resource "aws_iam_role_policy" "bastion_instance_read_ec2" { + name = "instance-read-ec2" + role = aws_iam_role.bastion.id + policy = data.aws_iam_policy_document.bastion_instance_read_ec2.json +} + +data "aws_iam_policy_document" "bastion_instance_read_ec2" { + statement { + actions = ["ec2:Describe*"] + resources = ["*"] + } +} + +data "aws_iam_policy_document" "bastion_assume_role" { + statement { + actions = ["sts:AssumeRole"] + + principals { + type = "Service" + identifiers = ["ec2.amazonaws.com"] + } + } +} + +resource "aws_iam_role" "bastion" { + name = "${var.cluster_name}-bastion" + assume_role_policy = data.aws_iam_policy_document.bastion_assume_role.json +} + +resource "aws_iam_instance_profile" "bastion" { + name = "${var.cluster_name}-bastion" + role = aws_iam_role.bastion.id +} diff --git a/aws/fedora-coreos/kubernetes/bootstrap.tf b/aws/fedora-coreos/kubernetes/bootstrap.tf index d35fd11ef..4d5c94974 100644 --- a/aws/fedora-coreos/kubernetes/bootstrap.tf +++ b/aws/fedora-coreos/kubernetes/bootstrap.tf @@ -15,5 +15,8 @@ module "bootstrap" { enable_aggregation = var.enable_aggregation trusted_certs_dir = "/etc/pki/tls/certs" + + # scoop + apiserver_arguments = var.apiserver_arguments } diff --git a/aws/fedora-coreos/kubernetes/controllers.tf b/aws/fedora-coreos/kubernetes/controllers.tf index e4bd9ddba..dffbf9a20 100644 --- a/aws/fedora-coreos/kubernetes/controllers.tf +++ b/aws/fedora-coreos/kubernetes/controllers.tf @@ -17,14 +17,33 @@ resource "aws_route53_record" "etcds" { resource "aws_instance" "controllers" { count = var.controller_count - tags = { - Name = "${var.cluster_name}-controller-${count.index}" - } + tags = map( + "Name", "${var.cluster_name}-controller-${count.index}", + "kubernetes.io/cluster/${var.cluster_name}", "owned" + ) instance_type = var.controller_type - ami = data.aws_ami.fedora-coreos.image_id - user_data = data.ct_config.controller-ignitions.*.rendered[count.index] + ami = coalesce(var.ami, data.aws_ami.fedora-coreos.image_id) + iam_instance_profile = aws_iam_instance_profile.controller.name + + user_data = < 0 ? var.spot_price : null enable_monitoring = false - user_data = data.ct_config.worker-ignition.rendered + user_data = jsonencode(local.user_data_worker) # storage root_block_device { @@ -62,13 +86,22 @@ resource "aws_launch_configuration" "worker" { # network security_groups = var.security_groups + # iam + iam_instance_profile = aws_iam_instance_profile.worker.name + lifecycle { // Override the default destroy and replace update behavior create_before_destroy = true - ignore_changes = [image_id] + ignore_changes = [image_id, user_data] } } +resource "aws_s3_bucket_object" "worker-ignition" { + bucket = var.ignition_config_bucket + key = "worker.json" + content = data.ct_config.worker-ignition.rendered +} + # Worker Ignition config data "ct_config" "worker-ignition" { content = data.template_file.worker-config.rendered @@ -89,3 +122,56 @@ data "template_file" "worker-config" { } } +resource "aws_iam_role_policy" "worker_read_base_ignition_config" { + name = "read-base-ignition-config" + role = aws_iam_role.worker.id + policy = var.base_ignition_config_read_policy +} + +resource "aws_iam_role_policy" "worker_read_ignition_configs" { + name = "read-ignition-configs" + role = aws_iam_role.worker.id + policy = data.aws_iam_policy_document.worker_read_ignition_configs.json +} + +data "aws_iam_policy_document" "worker_read_ignition_configs" { + statement { + effect = "Allow" + actions = ["s3:GetObject"] + resources = ["arn:aws:s3:::${var.ignition_config_bucket}/${aws_s3_bucket_object.worker-ignition.id}"] + } +} + +resource "aws_iam_role_policy" "worker_instance_read_ec2" { + name = "instance-read-ec2" + role = aws_iam_role.worker.id + policy = data.aws_iam_policy_document.worker_instance_read_ec2.json +} + +data "aws_iam_policy_document" "worker_instance_read_ec2" { + statement { + actions = ["ec2:Describe*"] + resources = ["*"] + } +} + +data "aws_iam_policy_document" "assume_role" { + statement { + actions = ["sts:AssumeRole"] + + principals { + type = "Service" + identifiers = ["ec2.amazonaws.com"] + } + } +} + +resource "aws_iam_role" "worker" { + name = "${var.name}-worker" + assume_role_policy = data.aws_iam_policy_document.assume_role.json +} + +resource "aws_iam_instance_profile" "worker" { + name = "${var.name}-worker" + role = aws_iam_role.worker.id +} From 2984f3e70be6f8ecdc6a96f70cf809b260c919d6 Mon Sep 17 00:00:00 2001 From: Bo Huang Date: Fri, 31 Jul 2020 12:51:55 -0700 Subject: [PATCH 491/523] bastion only needs base fcos config --- aws/fedora-coreos/kubernetes/bastion.tf | 41 ++----------------------- 1 file changed, 3 insertions(+), 38 deletions(-) diff --git a/aws/fedora-coreos/kubernetes/bastion.tf b/aws/fedora-coreos/kubernetes/bastion.tf index 989a7974e..531f7244b 100644 --- a/aws/fedora-coreos/kubernetes/bastion.tf +++ b/aws/fedora-coreos/kubernetes/bastion.tf @@ -3,14 +3,9 @@ locals { ignition = { version = "3.0.0" config = { - merge = [ - { - source = var.base_ignition_config_path - }, - { - source = "s3://${aws_s3_bucket.ignition_configs.id}/${aws_s3_bucket_object.bastion_config.id}" - } - ] + replace = { + source = var.base_ignition_config_path + } } } } @@ -72,22 +67,6 @@ resource "aws_launch_configuration" "bastion" { } } -resource "aws_s3_bucket_object" "bastion_config" { - bucket = aws_s3_bucket.ignition_configs.id - key = "bastion.json" - content = data.ct_config.bastion_ign.rendered -} - -data "template_file" "bastion_config" { - template = file("${path.module}/cl/bastion.yaml.tmpl") -} - -data "ct_config" "bastion_ign" { - content = data.template_file.bastion_config.rendered - pretty_print = false - snippets = var.bastion_clc_snippets -} - resource "aws_security_group" "bastion_external" { name_prefix = "${var.cluster_name}-bastion-external-" description = "Allows access to the bastion from the internet" @@ -198,20 +177,6 @@ resource "aws_iam_role_policy" "bastion_read_base_ignition_config" { policy = var.base_ignition_config_read_policy } -resource "aws_iam_role_policy" "bastion_read_ignition_config" { - name = "read-ignition-configs" - role = aws_iam_role.bastion.id - policy = data.aws_iam_policy_document.bastion_read_ignition_config.json -} - -data "aws_iam_policy_document" "bastion_read_ignition_config" { - statement { - effect = "Allow" - actions = ["s3:GetObject"] - resources = ["${aws_s3_bucket.ignition_configs.arn}/${aws_s3_bucket_object.bastion_config.id}"] - } -} - resource "aws_iam_role_policy" "bastion_instance_read_ec2" { name = "instance-read-ec2" role = aws_iam_role.bastion.id From 4ab09a058fba1214becb19455194eea0cdf2c081 Mon Sep 17 00:00:00 2001 From: Bo Huang Date: Fri, 31 Jul 2020 15:07:38 -0700 Subject: [PATCH 492/523] Revert "bastion only needs base fcos config" This reverts commit 2984f3e70be6f8ecdc6a96f70cf809b260c919d6. --- aws/fedora-coreos/kubernetes/bastion.tf | 41 +++++++++++++++++++++++-- 1 file changed, 38 insertions(+), 3 deletions(-) diff --git a/aws/fedora-coreos/kubernetes/bastion.tf b/aws/fedora-coreos/kubernetes/bastion.tf index 531f7244b..989a7974e 100644 --- a/aws/fedora-coreos/kubernetes/bastion.tf +++ b/aws/fedora-coreos/kubernetes/bastion.tf @@ -3,9 +3,14 @@ locals { ignition = { version = "3.0.0" config = { - replace = { - source = var.base_ignition_config_path - } + merge = [ + { + source = var.base_ignition_config_path + }, + { + source = "s3://${aws_s3_bucket.ignition_configs.id}/${aws_s3_bucket_object.bastion_config.id}" + } + ] } } } @@ -67,6 +72,22 @@ resource "aws_launch_configuration" "bastion" { } } +resource "aws_s3_bucket_object" "bastion_config" { + bucket = aws_s3_bucket.ignition_configs.id + key = "bastion.json" + content = data.ct_config.bastion_ign.rendered +} + +data "template_file" "bastion_config" { + template = file("${path.module}/cl/bastion.yaml.tmpl") +} + +data "ct_config" "bastion_ign" { + content = data.template_file.bastion_config.rendered + pretty_print = false + snippets = var.bastion_clc_snippets +} + resource "aws_security_group" "bastion_external" { name_prefix = "${var.cluster_name}-bastion-external-" description = "Allows access to the bastion from the internet" @@ -177,6 +198,20 @@ resource "aws_iam_role_policy" "bastion_read_base_ignition_config" { policy = var.base_ignition_config_read_policy } +resource "aws_iam_role_policy" "bastion_read_ignition_config" { + name = "read-ignition-configs" + role = aws_iam_role.bastion.id + policy = data.aws_iam_policy_document.bastion_read_ignition_config.json +} + +data "aws_iam_policy_document" "bastion_read_ignition_config" { + statement { + effect = "Allow" + actions = ["s3:GetObject"] + resources = ["${aws_s3_bucket.ignition_configs.arn}/${aws_s3_bucket_object.bastion_config.id}"] + } +} + resource "aws_iam_role_policy" "bastion_instance_read_ec2" { name = "instance-read-ec2" role = aws_iam_role.bastion.id From 652196afb1f1cd0eb363903c357374e119854507 Mon Sep 17 00:00:00 2001 From: Bo Huang Date: Fri, 31 Jul 2020 15:55:28 -0700 Subject: [PATCH 493/523] support custom bastion snippet as a variable --- aws/fedora-coreos/kubernetes/bastion.tf | 10 +++------- aws/fedora-coreos/kubernetes/fcc/bastion.yaml | 3 +++ aws/fedora-coreos/kubernetes/variables.tf | 11 ++++++----- aws/fedora-coreos/kubernetes/workers/workers.tf | 1 - 4 files changed, 12 insertions(+), 13 deletions(-) create mode 100644 aws/fedora-coreos/kubernetes/fcc/bastion.yaml diff --git a/aws/fedora-coreos/kubernetes/bastion.tf b/aws/fedora-coreos/kubernetes/bastion.tf index 989a7974e..6f8874474 100644 --- a/aws/fedora-coreos/kubernetes/bastion.tf +++ b/aws/fedora-coreos/kubernetes/bastion.tf @@ -78,14 +78,10 @@ resource "aws_s3_bucket_object" "bastion_config" { content = data.ct_config.bastion_ign.rendered } -data "template_file" "bastion_config" { - template = file("${path.module}/cl/bastion.yaml.tmpl") -} - data "ct_config" "bastion_ign" { - content = data.template_file.bastion_config.rendered - pretty_print = false - snippets = var.bastion_clc_snippets + content = file("${path.module}/fcc/bastion.yaml") + strict = true + snippets = var.bastion_snippets } resource "aws_security_group" "bastion_external" { diff --git a/aws/fedora-coreos/kubernetes/fcc/bastion.yaml b/aws/fedora-coreos/kubernetes/fcc/bastion.yaml new file mode 100644 index 000000000..6d8684b5e --- /dev/null +++ b/aws/fedora-coreos/kubernetes/fcc/bastion.yaml @@ -0,0 +1,3 @@ +--- +variant: fcos +version: 1.0.0 diff --git a/aws/fedora-coreos/kubernetes/variables.tf b/aws/fedora-coreos/kubernetes/variables.tf index c8ad7325f..b0b0db4a4 100644 --- a/aws/fedora-coreos/kubernetes/variables.tf +++ b/aws/fedora-coreos/kubernetes/variables.tf @@ -89,13 +89,14 @@ variable "worker_snippets" { default = [] } -# configuration - -variable "ssh_authorized_key" { - type = string - description = "SSH public key for user 'core'" +variable "bastion_snippets" { + type = list(string) + description = "Bastion Fedora CoreOS Config snippets" + default = [] } +# configuration + variable "asset_dir" { type = string description = "Absolute path to a directory where generated assets should be placed (contains secrets)" diff --git a/aws/fedora-coreos/kubernetes/workers/workers.tf b/aws/fedora-coreos/kubernetes/workers/workers.tf index 4c817a53d..ac973bc9a 100644 --- a/aws/fedora-coreos/kubernetes/workers/workers.tf +++ b/aws/fedora-coreos/kubernetes/workers/workers.tf @@ -115,7 +115,6 @@ data "template_file" "worker-config" { vars = { kubeconfig = indent(10, var.kubeconfig) - ssh_authorized_key = var.ssh_authorized_key cluster_dns_service_ip = cidrhost(var.service_cidr, 10) cluster_domain_suffix = var.cluster_domain_suffix node_labels = join(",", var.node_labels) From 78e6409bd0abac23451951dc93748513d238bcec Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Sat, 1 Aug 2020 21:00:39 -0700 Subject: [PATCH 494/523] Fix flannel support on Fedora CoreOS * Fedora CoreOS now ships systemd-udev's `default.link` while Flannel relies on being able to pick its own MAC address for the `flannel.1` link for tunneled traffic to reach cni0 on the destination side, without being dropped * This change first appeared in FCOS testing-devel 32.20200624.20.1 and is the behavior going forward in FCOS since it was added to align FCOS network naming / configs with the rest of Fedora and address issues related to the default being missing * Flatcar Linux (and Container Linux) has a specific flannel.link configuration builtin, so it was not affected * https://github.com/coreos/fedora-coreos-tracker/issues/574#issuecomment-665487296 Note: Typhoon's recommended and default CNI provider is Calico, unless `networking` is set to flannel directly. --- CHANGES.md | 7 +++++++ aws/fedora-coreos/kubernetes/fcc/controller.yaml | 7 +++++++ aws/fedora-coreos/kubernetes/workers/fcc/worker.yaml | 7 +++++++ azure/fedora-coreos/kubernetes/fcc/controller.yaml | 7 +++++++ azure/fedora-coreos/kubernetes/workers/fcc/worker.yaml | 7 +++++++ bare-metal/fedora-coreos/kubernetes/fcc/controller.yaml | 7 +++++++ bare-metal/fedora-coreos/kubernetes/fcc/worker.yaml | 7 +++++++ digital-ocean/fedora-coreos/kubernetes/fcc/controller.yaml | 7 +++++++ digital-ocean/fedora-coreos/kubernetes/fcc/worker.yaml | 7 +++++++ google-cloud/fedora-coreos/kubernetes/fcc/controller.yaml | 7 +++++++ .../fedora-coreos/kubernetes/workers/fcc/worker.yaml | 7 +++++++ 11 files changed, 77 insertions(+) diff --git a/CHANGES.md b/CHANGES.md index 3de6d207c..2e5a2985f 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -8,6 +8,13 @@ Notable changes between versions. * Update CoreDNS from v1.6.7 to [v1.7.0](https://coredns.io/2020/06/15/coredns-1.7.0-release/) * Update Cilium from v1.8.1 to [v1.8.2](https://github.com/cilium/cilium/releases/tag/v1.8.2) +### Fedora CoreOS + +* Fix support for Flannel with Fedora CoreOS ([#795](https://github.com/poseidon/typhoon/pull/795)) + * Fedora CoreOS fixes to align network interface defaults altered MAC address assignment for + the `flannel.1` interface in a way that caused flannel to drop pod-to-pod traffic + * Configure flannel interfaces explicitly + #### Addons * Update Prometheus from v2.19.2 to [v2.20.0](https://github.com/prometheus/prometheus/releases/tag/v2.20.0) diff --git a/aws/fedora-coreos/kubernetes/fcc/controller.yaml b/aws/fedora-coreos/kubernetes/fcc/controller.yaml index 0d8b4c35c..0bec7c812 100644 --- a/aws/fedora-coreos/kubernetes/fcc/controller.yaml +++ b/aws/fedora-coreos/kubernetes/fcc/controller.yaml @@ -183,6 +183,13 @@ storage: inline: | net.ipv4.conf.default.rp_filter=0 net.ipv4.conf.*.rp_filter=0 + - path: /etc/systemd/network/50-flannel.link + contents: + inline: | + [Match] + OriginalName=flannel* + [Link] + MACAddressPolicy=none - path: /etc/systemd/system.conf.d/accounting.conf contents: inline: | diff --git a/aws/fedora-coreos/kubernetes/workers/fcc/worker.yaml b/aws/fedora-coreos/kubernetes/workers/fcc/worker.yaml index c178f896f..5fdeca2af 100644 --- a/aws/fedora-coreos/kubernetes/workers/fcc/worker.yaml +++ b/aws/fedora-coreos/kubernetes/workers/fcc/worker.yaml @@ -110,6 +110,13 @@ storage: inline: | net.ipv4.conf.default.rp_filter=0 net.ipv4.conf.*.rp_filter=0 + - path: /etc/systemd/network/50-flannel.link + contents: + inline: | + [Match] + OriginalName=flannel* + [Link] + MACAddressPolicy=none - path: /etc/systemd/system.conf.d/accounting.conf contents: inline: | diff --git a/azure/fedora-coreos/kubernetes/fcc/controller.yaml b/azure/fedora-coreos/kubernetes/fcc/controller.yaml index a337e9a86..cdfa5f2a5 100644 --- a/azure/fedora-coreos/kubernetes/fcc/controller.yaml +++ b/azure/fedora-coreos/kubernetes/fcc/controller.yaml @@ -182,6 +182,13 @@ storage: inline: | net.ipv4.conf.default.rp_filter=0 net.ipv4.conf.*.rp_filter=0 + - path: /etc/systemd/network/50-flannel.link + contents: + inline: | + [Match] + OriginalName=flannel* + [Link] + MACAddressPolicy=none - path: /etc/systemd/system.conf.d/accounting.conf contents: inline: | diff --git a/azure/fedora-coreos/kubernetes/workers/fcc/worker.yaml b/azure/fedora-coreos/kubernetes/workers/fcc/worker.yaml index 06904d40e..e392486a5 100644 --- a/azure/fedora-coreos/kubernetes/workers/fcc/worker.yaml +++ b/azure/fedora-coreos/kubernetes/workers/fcc/worker.yaml @@ -109,6 +109,13 @@ storage: inline: | net.ipv4.conf.default.rp_filter=0 net.ipv4.conf.*.rp_filter=0 + - path: /etc/systemd/network/50-flannel.link + contents: + inline: | + [Match] + OriginalName=flannel* + [Link] + MACAddressPolicy=none - path: /etc/systemd/system.conf.d/accounting.conf contents: inline: | diff --git a/bare-metal/fedora-coreos/kubernetes/fcc/controller.yaml b/bare-metal/fedora-coreos/kubernetes/fcc/controller.yaml index 2f6c9ee73..4700a9c40 100644 --- a/bare-metal/fedora-coreos/kubernetes/fcc/controller.yaml +++ b/bare-metal/fedora-coreos/kubernetes/fcc/controller.yaml @@ -193,6 +193,13 @@ storage: inline: | net.ipv4.conf.default.rp_filter=0 net.ipv4.conf.*.rp_filter=0 + - path: /etc/systemd/network/50-flannel.link + contents: + inline: | + [Match] + OriginalName=flannel* + [Link] + MACAddressPolicy=none - path: /etc/systemd/system.conf.d/accounting.conf contents: inline: | diff --git a/bare-metal/fedora-coreos/kubernetes/fcc/worker.yaml b/bare-metal/fedora-coreos/kubernetes/fcc/worker.yaml index 79a9275bc..2b0645a12 100644 --- a/bare-metal/fedora-coreos/kubernetes/fcc/worker.yaml +++ b/bare-metal/fedora-coreos/kubernetes/fcc/worker.yaml @@ -111,6 +111,13 @@ storage: inline: | net.ipv4.conf.default.rp_filter=0 net.ipv4.conf.*.rp_filter=0 + - path: /etc/systemd/network/50-flannel.link + contents: + inline: | + [Match] + OriginalName=flannel* + [Link] + MACAddressPolicy=none - path: /etc/systemd/system.conf.d/accounting.conf contents: inline: | diff --git a/digital-ocean/fedora-coreos/kubernetes/fcc/controller.yaml b/digital-ocean/fedora-coreos/kubernetes/fcc/controller.yaml index a819e6a66..6ce76a4d2 100644 --- a/digital-ocean/fedora-coreos/kubernetes/fcc/controller.yaml +++ b/digital-ocean/fedora-coreos/kubernetes/fcc/controller.yaml @@ -189,6 +189,13 @@ storage: inline: | net.ipv4.conf.default.rp_filter=0 net.ipv4.conf.*.rp_filter=0 + - path: /etc/systemd/network/50-flannel.link + contents: + inline: | + [Match] + OriginalName=flannel* + [Link] + MACAddressPolicy=none - path: /etc/systemd/system.conf.d/accounting.conf contents: inline: | diff --git a/digital-ocean/fedora-coreos/kubernetes/fcc/worker.yaml b/digital-ocean/fedora-coreos/kubernetes/fcc/worker.yaml index f4b605e50..fbff7b339 100644 --- a/digital-ocean/fedora-coreos/kubernetes/fcc/worker.yaml +++ b/digital-ocean/fedora-coreos/kubernetes/fcc/worker.yaml @@ -114,6 +114,13 @@ storage: inline: | net.ipv4.conf.default.rp_filter=0 net.ipv4.conf.*.rp_filter=0 + - path: /etc/systemd/network/50-flannel.link + contents: + inline: | + [Match] + OriginalName=flannel* + [Link] + MACAddressPolicy=none - path: /etc/systemd/system.conf.d/accounting.conf contents: inline: | diff --git a/google-cloud/fedora-coreos/kubernetes/fcc/controller.yaml b/google-cloud/fedora-coreos/kubernetes/fcc/controller.yaml index 8edb5261e..b9fdcef24 100644 --- a/google-cloud/fedora-coreos/kubernetes/fcc/controller.yaml +++ b/google-cloud/fedora-coreos/kubernetes/fcc/controller.yaml @@ -182,6 +182,13 @@ storage: inline: | net.ipv4.conf.default.rp_filter=0 net.ipv4.conf.*.rp_filter=0 + - path: /etc/systemd/network/50-flannel.link + contents: + inline: | + [Match] + OriginalName=flannel* + [Link] + MACAddressPolicy=none - path: /etc/systemd/system.conf.d/accounting.conf contents: inline: | diff --git a/google-cloud/fedora-coreos/kubernetes/workers/fcc/worker.yaml b/google-cloud/fedora-coreos/kubernetes/workers/fcc/worker.yaml index dff706ce6..5db86926c 100644 --- a/google-cloud/fedora-coreos/kubernetes/workers/fcc/worker.yaml +++ b/google-cloud/fedora-coreos/kubernetes/workers/fcc/worker.yaml @@ -109,6 +109,13 @@ storage: inline: | net.ipv4.conf.default.rp_filter=0 net.ipv4.conf.*.rp_filter=0 + - path: /etc/systemd/network/50-flannel.link + contents: + inline: | + [Match] + OriginalName=flannel* + [Link] + MACAddressPolicy=none - path: /etc/systemd/system.conf.d/accounting.conf contents: inline: | From 8aefd4f0823c8d939356d27b0b0bb88058fb07f0 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Sun, 2 Aug 2020 01:09:28 -0700 Subject: [PATCH 495/523] Relex terraform-provider-matchbox version constraint * Allow use of terraform-provider-matchbox v0.3+ (which allows v0.3.0 <= version < v1.0) for any pre 1.0 release * Before, the requirement was v0.3.0 <= version < v0.4.0 --- bare-metal/container-linux/kubernetes/versions.tf | 2 +- bare-metal/fedora-coreos/kubernetes/versions.tf | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/bare-metal/container-linux/kubernetes/versions.tf b/bare-metal/container-linux/kubernetes/versions.tf index cf547f19d..6a31221ee 100644 --- a/bare-metal/container-linux/kubernetes/versions.tf +++ b/bare-metal/container-linux/kubernetes/versions.tf @@ -3,7 +3,7 @@ terraform { required_version = "~> 0.12.6" required_providers { - matchbox = "~> 0.3.0" + matchbox = "~> 0.3" ct = "~> 0.4" template = "~> 2.1" null = "~> 2.1" diff --git a/bare-metal/fedora-coreos/kubernetes/versions.tf b/bare-metal/fedora-coreos/kubernetes/versions.tf index fd7df2ffa..cf3f8224c 100644 --- a/bare-metal/fedora-coreos/kubernetes/versions.tf +++ b/bare-metal/fedora-coreos/kubernetes/versions.tf @@ -3,7 +3,7 @@ terraform { required_version = "~> 0.12.6" required_providers { - matchbox = "~> 0.3.0" + matchbox = "~> 0.3" ct = "~> 0.4" template = "~> 2.1" null = "~> 2.1" From ccee5d3d89b45784f68cc43935253d3b10c5e957 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Sun, 2 Aug 2020 00:45:39 -0700 Subject: [PATCH 496/523] Update from coreos/flannel-cni to poseidon/flannel-cni * Update CNI plugins from v0.6.0 to v0.8.6 to fix several CVEs * Update the base image to alpine:3.12 * Use `flannel-cni` as an init container and remove sleep * https://github.com/poseidon/terraform-render-bootstrap/pull/205 * https://github.com/poseidon/flannel-cni * https://quay.io/repository/poseidon/flannel-cni Background * Switch from github.com/coreos/flannel-cni v0.3.0 which was last published by me in 2017 and is no longer accessible to me to maintain or patch * Port to the poseidon/flannel-cni rewrite, which releases v0.4.0 to continue the prior release numbering --- CHANGES.md | 3 +++ aws/container-linux/kubernetes/bootstrap.tf | 2 +- aws/fedora-coreos/kubernetes/bootstrap.tf | 2 +- azure/container-linux/kubernetes/bootstrap.tf | 2 +- azure/fedora-coreos/kubernetes/bootstrap.tf | 2 +- bare-metal/container-linux/kubernetes/bootstrap.tf | 2 +- bare-metal/fedora-coreos/kubernetes/bootstrap.tf | 2 +- digital-ocean/container-linux/kubernetes/bootstrap.tf | 2 +- digital-ocean/fedora-coreos/kubernetes/bootstrap.tf | 2 +- docs/topics/security.md | 6 ++++++ google-cloud/container-linux/kubernetes/bootstrap.tf | 2 +- google-cloud/fedora-coreos/kubernetes/bootstrap.tf | 2 +- 12 files changed, 19 insertions(+), 10 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 2e5a2985f..d57a000d8 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -7,6 +7,9 @@ Notable changes between versions. * Update etcd from v3.4.9 to [v3.4.10](https://github.com/etcd-io/etcd/releases/tag/v3.4.10) * Update CoreDNS from v1.6.7 to [v1.7.0](https://coredns.io/2020/06/15/coredns-1.7.0-release/) * Update Cilium from v1.8.1 to [v1.8.2](https://github.com/cilium/cilium/releases/tag/v1.8.2) +* Update [coreos/flannel-cni](https://github.com/coreos/flannel-cni) to [poseidon/flannel-cni](https://github.com/poseidon/flannel-cni) ([#798](https://github.com/poseidon/typhoon/pull/798)) + * Update CNI plugins and fix CVEs with Flannel (non-default) + * Transition to a poseidon maintained container image ### Fedora CoreOS diff --git a/aws/container-linux/kubernetes/bootstrap.tf b/aws/container-linux/kubernetes/bootstrap.tf index 480ceda36..68d5f8192 100644 --- a/aws/container-linux/kubernetes/bootstrap.tf +++ b/aws/container-linux/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=45053a62cb751e2f8b371c20b7714e1604d6e57d" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=3675b3a539efd8d341277f0c03322883f97fd992" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/aws/fedora-coreos/kubernetes/bootstrap.tf b/aws/fedora-coreos/kubernetes/bootstrap.tf index cceb3bfbe..04d39b05f 100644 --- a/aws/fedora-coreos/kubernetes/bootstrap.tf +++ b/aws/fedora-coreos/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=45053a62cb751e2f8b371c20b7714e1604d6e57d" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=3675b3a539efd8d341277f0c03322883f97fd992" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/azure/container-linux/kubernetes/bootstrap.tf b/azure/container-linux/kubernetes/bootstrap.tf index f3f59911a..b51496e3d 100644 --- a/azure/container-linux/kubernetes/bootstrap.tf +++ b/azure/container-linux/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=45053a62cb751e2f8b371c20b7714e1604d6e57d" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=3675b3a539efd8d341277f0c03322883f97fd992" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/azure/fedora-coreos/kubernetes/bootstrap.tf b/azure/fedora-coreos/kubernetes/bootstrap.tf index 5aad4effc..d3f2d5ff3 100644 --- a/azure/fedora-coreos/kubernetes/bootstrap.tf +++ b/azure/fedora-coreos/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=45053a62cb751e2f8b371c20b7714e1604d6e57d" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=3675b3a539efd8d341277f0c03322883f97fd992" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/bare-metal/container-linux/kubernetes/bootstrap.tf b/bare-metal/container-linux/kubernetes/bootstrap.tf index c819e5325..ea5fed8e5 100644 --- a/bare-metal/container-linux/kubernetes/bootstrap.tf +++ b/bare-metal/container-linux/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=45053a62cb751e2f8b371c20b7714e1604d6e57d" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=3675b3a539efd8d341277f0c03322883f97fd992" cluster_name = var.cluster_name api_servers = [var.k8s_domain_name] diff --git a/bare-metal/fedora-coreos/kubernetes/bootstrap.tf b/bare-metal/fedora-coreos/kubernetes/bootstrap.tf index 3e383ea71..837c489b6 100644 --- a/bare-metal/fedora-coreos/kubernetes/bootstrap.tf +++ b/bare-metal/fedora-coreos/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=45053a62cb751e2f8b371c20b7714e1604d6e57d" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=3675b3a539efd8d341277f0c03322883f97fd992" cluster_name = var.cluster_name api_servers = [var.k8s_domain_name] diff --git a/digital-ocean/container-linux/kubernetes/bootstrap.tf b/digital-ocean/container-linux/kubernetes/bootstrap.tf index 7c6c2ce82..0dd4ac823 100644 --- a/digital-ocean/container-linux/kubernetes/bootstrap.tf +++ b/digital-ocean/container-linux/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=45053a62cb751e2f8b371c20b7714e1604d6e57d" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=3675b3a539efd8d341277f0c03322883f97fd992" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/digital-ocean/fedora-coreos/kubernetes/bootstrap.tf b/digital-ocean/fedora-coreos/kubernetes/bootstrap.tf index c89b9f6f5..2d9d825af 100644 --- a/digital-ocean/fedora-coreos/kubernetes/bootstrap.tf +++ b/digital-ocean/fedora-coreos/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=45053a62cb751e2f8b371c20b7714e1604d6e57d" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=3675b3a539efd8d341277f0c03322883f97fd992" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/docs/topics/security.md b/docs/topics/security.md index 5041dd99b..77a64748d 100644 --- a/docs/topics/security.md +++ b/docs/topics/security.md @@ -66,6 +66,12 @@ Two tag styles indicate the build strategy used. The Typhoon-built Kubelet image is used as the official image. Automated builds provide an alternative image for those preferring to trust images built by Quay/Dockerhub (albeit lacking multi-arch). To use the fallback registry or an alternative tag, see [customization](/advanced/customization/#kubelet). +### flannel-cni + +Typhoon packages the [flannel-cni](https://github.com/poseidon/flannel-cni) container image to provide security patches. + +* [quay.io/poseidon/flannel-cni](https://quay.io/repository/poseidon/flannel-cni) (official) + ## Disclosures If you find security issues, please email `security@psdn.io`. If the issue lies in upstream Kubernetes, please inform upstream Kubernetes as well. diff --git a/google-cloud/container-linux/kubernetes/bootstrap.tf b/google-cloud/container-linux/kubernetes/bootstrap.tf index 3d0a9db34..62670ff2e 100644 --- a/google-cloud/container-linux/kubernetes/bootstrap.tf +++ b/google-cloud/container-linux/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=45053a62cb751e2f8b371c20b7714e1604d6e57d" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=3675b3a539efd8d341277f0c03322883f97fd992" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/google-cloud/fedora-coreos/kubernetes/bootstrap.tf b/google-cloud/fedora-coreos/kubernetes/bootstrap.tf index 5697c2ddd..cc86ce0d0 100644 --- a/google-cloud/fedora-coreos/kubernetes/bootstrap.tf +++ b/google-cloud/fedora-coreos/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=45053a62cb751e2f8b371c20b7714e1604d6e57d" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=3675b3a539efd8d341277f0c03322883f97fd992" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] From 8b3d41d6a078c01541257babb16c8d479cdead76 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Sun, 2 Aug 2020 15:22:10 -0700 Subject: [PATCH 497/523] Update mkdocs-material from v5.4.0 to v5.5.1 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index edd58d2c1..3548568da 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ mkdocs==1.1.2 -mkdocs-material==5.4.0 +mkdocs-material==5.5.1 pygments==2.6.1 pymdown-extensions==7.1.0 From b5e7173530650707ecdf3cf4ea5098829ed7b4ab Mon Sep 17 00:00:00 2001 From: Bo Huang Date: Tue, 4 Aug 2020 10:19:01 -0700 Subject: [PATCH 498/523] use scoop's fork of terraform-render-bootstrap --- aws/fedora-coreos/kubernetes/bootstrap.tf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/aws/fedora-coreos/kubernetes/bootstrap.tf b/aws/fedora-coreos/kubernetes/bootstrap.tf index 4d5c94974..a206e8c7e 100644 --- a/aws/fedora-coreos/kubernetes/bootstrap.tf +++ b/aws/fedora-coreos/kubernetes/bootstrap.tf @@ -1,9 +1,9 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=14d0b2087962a0f2557c184f3f523548ce19bbdc" + source = "git::https://github.com/takescoop/terraform-render-bootstrap.git?ref=a7ad7d74586fd698be22b75b4cc314c2d3dc4bcd" cluster_name = var.cluster_name - api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] + api_servers = concat(list(format("%s.%s", var.cluster_name, var.dns_zone)), var.apiserver_aliases) etcd_servers = aws_route53_record.etcds.*.fqdn asset_dir = var.asset_dir networking = var.networking From 4579af8bda4a5720e791c56fa02c14ecb767a537 Mon Sep 17 00:00:00 2001 From: Bo Huang Date: Tue, 4 Aug 2020 10:47:58 -0700 Subject: [PATCH 499/523] policy arn --- aws/fedora-coreos/kubernetes/bastion.tf | 7 +++---- aws/fedora-coreos/kubernetes/controllers.tf | 7 +++---- aws/fedora-coreos/kubernetes/variables.tf | 4 ++-- aws/fedora-coreos/kubernetes/workers.tf | 8 ++++---- aws/fedora-coreos/kubernetes/workers/variables.tf | 4 ++-- aws/fedora-coreos/kubernetes/workers/workers.tf | 7 +++---- 6 files changed, 17 insertions(+), 20 deletions(-) diff --git a/aws/fedora-coreos/kubernetes/bastion.tf b/aws/fedora-coreos/kubernetes/bastion.tf index 6f8874474..7cc260d2f 100644 --- a/aws/fedora-coreos/kubernetes/bastion.tf +++ b/aws/fedora-coreos/kubernetes/bastion.tf @@ -188,10 +188,9 @@ resource "aws_route53_record" "bastion" { } } -resource "aws_iam_role_policy" "bastion_read_base_ignition_config" { - name = "read-base-ignition-config" - role = aws_iam_role.bastion.id - policy = var.base_ignition_config_read_policy +resource "aws_iam_role_policy_attachment" "bastion_read_base_ignition_config" { + role = aws_iam_role.bastion.id + policy_arn = var.base_ignition_config_read_policy_arn } resource "aws_iam_role_policy" "bastion_read_ignition_config" { diff --git a/aws/fedora-coreos/kubernetes/controllers.tf b/aws/fedora-coreos/kubernetes/controllers.tf index dffbf9a20..c79692163 100644 --- a/aws/fedora-coreos/kubernetes/controllers.tf +++ b/aws/fedora-coreos/kubernetes/controllers.tf @@ -112,10 +112,9 @@ data "template_file" "etcds" { } } -resource "aws_iam_role_policy" "controller_read_base_ignition_config" { - name = "read-base-ignition-config" - role = aws_iam_role.controller.id - policy = var.base_ignition_config_read_policy +resource "aws_iam_role_policy_attachment" "controller_read_base_ignition_config" { + role = aws_iam_role.controller.id + policy_arn = var.base_ignition_config_read_policy_arn } resource "aws_iam_role_policy" "controller_read_ignition_configs" { diff --git a/aws/fedora-coreos/kubernetes/variables.tf b/aws/fedora-coreos/kubernetes/variables.tf index b0b0db4a4..ad6724171 100644 --- a/aws/fedora-coreos/kubernetes/variables.tf +++ b/aws/fedora-coreos/kubernetes/variables.tf @@ -199,9 +199,9 @@ variable "base_ignition_config_path" { description = "The full path of the S3 object that stores base ignition config" } -variable "base_ignition_config_read_policy" { +variable "base_ignition_config_read_policy_arn" { type = string - description = "The contents of the IAM policy that allows reading base ignition config" + description = "The ARN of the IAM policy that allows reading base ignition config" } variable "ssh_user" { diff --git a/aws/fedora-coreos/kubernetes/workers.tf b/aws/fedora-coreos/kubernetes/workers.tf index ea326abd0..0d379c133 100644 --- a/aws/fedora-coreos/kubernetes/workers.tf +++ b/aws/fedora-coreos/kubernetes/workers.tf @@ -22,9 +22,9 @@ module "workers" { node_labels = var.worker_node_labels # scoop - ami = var.ami - base_ignition_config_path = var.base_ignition_config_path - base_ignition_config_read_policy = var.base_ignition_config_read_policy - ignition_config_bucket = aws_s3_bucket.ignition_configs.id + ami = var.ami + base_ignition_config_path = var.base_ignition_config_path + base_ignition_config_read_policy_arn = var.base_ignition_config_read_policy_arn + ignition_config_bucket = aws_s3_bucket.ignition_configs.id } diff --git a/aws/fedora-coreos/kubernetes/workers/variables.tf b/aws/fedora-coreos/kubernetes/workers/variables.tf index e07e216f4..5a6d8090d 100644 --- a/aws/fedora-coreos/kubernetes/workers/variables.tf +++ b/aws/fedora-coreos/kubernetes/workers/variables.tf @@ -122,9 +122,9 @@ variable "base_ignition_config_path" { description = "The full path of the S3 object that stores base ignition config" } -variable "base_ignition_config_read_policy" { +variable "base_ignition_config_read_policy_arn" { type = string - description = "The contents of the IAM policy that allows reading base ignition config" + description = "The ARN of the IAM policy that allows reading base ignition config" } variable "ignition_config_bucket" { diff --git a/aws/fedora-coreos/kubernetes/workers/workers.tf b/aws/fedora-coreos/kubernetes/workers/workers.tf index ac973bc9a..ee34a80b5 100644 --- a/aws/fedora-coreos/kubernetes/workers/workers.tf +++ b/aws/fedora-coreos/kubernetes/workers/workers.tf @@ -121,10 +121,9 @@ data "template_file" "worker-config" { } } -resource "aws_iam_role_policy" "worker_read_base_ignition_config" { - name = "read-base-ignition-config" - role = aws_iam_role.worker.id - policy = var.base_ignition_config_read_policy +resource "aws_iam_role_policy_attachment" "worker_read_base_ignition_config" { + role = aws_iam_role.worker.id + policy_arn = var.base_ignition_config_read_policy_arn } resource "aws_iam_role_policy" "worker_read_ignition_configs" { From 8ae4f6595aac912fa52cc2d71c76a992e22899bc Mon Sep 17 00:00:00 2001 From: Bo Huang Date: Tue, 4 Aug 2020 12:07:31 -0700 Subject: [PATCH 500/523] Revert "policy arn" This reverts commit 4579af8bda4a5720e791c56fa02c14ecb767a537. --- aws/fedora-coreos/kubernetes/bastion.tf | 7 ++++--- aws/fedora-coreos/kubernetes/controllers.tf | 7 ++++--- aws/fedora-coreos/kubernetes/variables.tf | 4 ++-- aws/fedora-coreos/kubernetes/workers.tf | 8 ++++---- aws/fedora-coreos/kubernetes/workers/variables.tf | 4 ++-- aws/fedora-coreos/kubernetes/workers/workers.tf | 7 ++++--- 6 files changed, 20 insertions(+), 17 deletions(-) diff --git a/aws/fedora-coreos/kubernetes/bastion.tf b/aws/fedora-coreos/kubernetes/bastion.tf index 7cc260d2f..6f8874474 100644 --- a/aws/fedora-coreos/kubernetes/bastion.tf +++ b/aws/fedora-coreos/kubernetes/bastion.tf @@ -188,9 +188,10 @@ resource "aws_route53_record" "bastion" { } } -resource "aws_iam_role_policy_attachment" "bastion_read_base_ignition_config" { - role = aws_iam_role.bastion.id - policy_arn = var.base_ignition_config_read_policy_arn +resource "aws_iam_role_policy" "bastion_read_base_ignition_config" { + name = "read-base-ignition-config" + role = aws_iam_role.bastion.id + policy = var.base_ignition_config_read_policy } resource "aws_iam_role_policy" "bastion_read_ignition_config" { diff --git a/aws/fedora-coreos/kubernetes/controllers.tf b/aws/fedora-coreos/kubernetes/controllers.tf index c79692163..dffbf9a20 100644 --- a/aws/fedora-coreos/kubernetes/controllers.tf +++ b/aws/fedora-coreos/kubernetes/controllers.tf @@ -112,9 +112,10 @@ data "template_file" "etcds" { } } -resource "aws_iam_role_policy_attachment" "controller_read_base_ignition_config" { - role = aws_iam_role.controller.id - policy_arn = var.base_ignition_config_read_policy_arn +resource "aws_iam_role_policy" "controller_read_base_ignition_config" { + name = "read-base-ignition-config" + role = aws_iam_role.controller.id + policy = var.base_ignition_config_read_policy } resource "aws_iam_role_policy" "controller_read_ignition_configs" { diff --git a/aws/fedora-coreos/kubernetes/variables.tf b/aws/fedora-coreos/kubernetes/variables.tf index ad6724171..b0b0db4a4 100644 --- a/aws/fedora-coreos/kubernetes/variables.tf +++ b/aws/fedora-coreos/kubernetes/variables.tf @@ -199,9 +199,9 @@ variable "base_ignition_config_path" { description = "The full path of the S3 object that stores base ignition config" } -variable "base_ignition_config_read_policy_arn" { +variable "base_ignition_config_read_policy" { type = string - description = "The ARN of the IAM policy that allows reading base ignition config" + description = "The contents of the IAM policy that allows reading base ignition config" } variable "ssh_user" { diff --git a/aws/fedora-coreos/kubernetes/workers.tf b/aws/fedora-coreos/kubernetes/workers.tf index 0d379c133..ea326abd0 100644 --- a/aws/fedora-coreos/kubernetes/workers.tf +++ b/aws/fedora-coreos/kubernetes/workers.tf @@ -22,9 +22,9 @@ module "workers" { node_labels = var.worker_node_labels # scoop - ami = var.ami - base_ignition_config_path = var.base_ignition_config_path - base_ignition_config_read_policy_arn = var.base_ignition_config_read_policy_arn - ignition_config_bucket = aws_s3_bucket.ignition_configs.id + ami = var.ami + base_ignition_config_path = var.base_ignition_config_path + base_ignition_config_read_policy = var.base_ignition_config_read_policy + ignition_config_bucket = aws_s3_bucket.ignition_configs.id } diff --git a/aws/fedora-coreos/kubernetes/workers/variables.tf b/aws/fedora-coreos/kubernetes/workers/variables.tf index 5a6d8090d..e07e216f4 100644 --- a/aws/fedora-coreos/kubernetes/workers/variables.tf +++ b/aws/fedora-coreos/kubernetes/workers/variables.tf @@ -122,9 +122,9 @@ variable "base_ignition_config_path" { description = "The full path of the S3 object that stores base ignition config" } -variable "base_ignition_config_read_policy_arn" { +variable "base_ignition_config_read_policy" { type = string - description = "The ARN of the IAM policy that allows reading base ignition config" + description = "The contents of the IAM policy that allows reading base ignition config" } variable "ignition_config_bucket" { diff --git a/aws/fedora-coreos/kubernetes/workers/workers.tf b/aws/fedora-coreos/kubernetes/workers/workers.tf index ee34a80b5..ac973bc9a 100644 --- a/aws/fedora-coreos/kubernetes/workers/workers.tf +++ b/aws/fedora-coreos/kubernetes/workers/workers.tf @@ -121,9 +121,10 @@ data "template_file" "worker-config" { } } -resource "aws_iam_role_policy_attachment" "worker_read_base_ignition_config" { - role = aws_iam_role.worker.id - policy_arn = var.base_ignition_config_read_policy_arn +resource "aws_iam_role_policy" "worker_read_base_ignition_config" { + name = "read-base-ignition-config" + role = aws_iam_role.worker.id + policy = var.base_ignition_config_read_policy } resource "aws_iam_role_policy" "worker_read_ignition_configs" { From 1097bc779616e6e67987ea3def419cff1274278b Mon Sep 17 00:00:00 2001 From: Bo Huang Date: Wed, 5 Aug 2020 10:58:25 -0700 Subject: [PATCH 501/523] workers and controllers need to stay private --- aws/fedora-coreos/kubernetes/controllers.tf | 2 +- aws/fedora-coreos/kubernetes/workers.tf | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/aws/fedora-coreos/kubernetes/controllers.tf b/aws/fedora-coreos/kubernetes/controllers.tf index dffbf9a20..d38509620 100644 --- a/aws/fedora-coreos/kubernetes/controllers.tf +++ b/aws/fedora-coreos/kubernetes/controllers.tf @@ -54,7 +54,7 @@ EOF } # network - subnet_id = aws_subnet.public.*.id[count.index] + subnet_id = aws_subnet.private.*.id[count.index] vpc_security_group_ids = [ aws_security_group.controller.id, aws_security_group.bastion_internal.id diff --git a/aws/fedora-coreos/kubernetes/workers.tf b/aws/fedora-coreos/kubernetes/workers.tf index ea326abd0..5c1eddd97 100644 --- a/aws/fedora-coreos/kubernetes/workers.tf +++ b/aws/fedora-coreos/kubernetes/workers.tf @@ -5,7 +5,7 @@ module "workers" { # AWS vpc_id = aws_vpc.network.id - subnet_ids = aws_subnet.public.*.id + subnet_ids = aws_subnet.private.*.id security_groups = [aws_security_group.worker.id] worker_count = var.worker_count instance_type = var.worker_type From 1411f5d4d95db277e5ba7de3056f3a395edc7980 Mon Sep 17 00:00:00 2001 From: Bo Huang Date: Wed, 5 Aug 2020 15:05:00 -0700 Subject: [PATCH 502/523] fedora coreos 32 --- aws/fedora-coreos/kubernetes/ami.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aws/fedora-coreos/kubernetes/ami.tf b/aws/fedora-coreos/kubernetes/ami.tf index e32ce159f..91b0561e6 100644 --- a/aws/fedora-coreos/kubernetes/ami.tf +++ b/aws/fedora-coreos/kubernetes/ami.tf @@ -15,7 +15,7 @@ data "aws_ami" "fedora-coreos" { filter { name = "name" - values = ["fedora-coreos-31.*.*.*-hvm"] + values = ["fedora-coreos-32.*.*.*-hvm"] } filter { From e79f7ef972704da96afb12ddc7c09f78fb952a96 Mon Sep 17 00:00:00 2001 From: Bo Huang Date: Wed, 5 Aug 2020 15:18:36 -0700 Subject: [PATCH 503/523] fedora coreos 32 --- aws/fedora-coreos/kubernetes/ami.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aws/fedora-coreos/kubernetes/ami.tf b/aws/fedora-coreos/kubernetes/ami.tf index 91b0561e6..3056d82fe 100644 --- a/aws/fedora-coreos/kubernetes/ami.tf +++ b/aws/fedora-coreos/kubernetes/ami.tf @@ -24,5 +24,5 @@ data "aws_ami" "fedora-coreos" { } # try to filter out dev images (AWS filters can't) - name_regex = "^fedora-coreos-31.[0-9]*.[0-9]*.[0-9]*-hvm*" + name_regex = "^fedora-coreos-32.[0-9]*.[0-9]*.[0-9]*-hvm*" } From 14b54e53f2aadb35c9a730861ccfda673874bf4a Mon Sep 17 00:00:00 2001 From: Bo Huang Date: Wed, 5 Aug 2020 15:34:24 -0700 Subject: [PATCH 504/523] Support Fedora CoreOS OS image streams on AWS --- aws/fedora-coreos/kubernetes/ami.tf | 10 +--------- aws/fedora-coreos/kubernetes/variables.tf | 4 ++-- aws/fedora-coreos/kubernetes/workers.tf | 2 +- aws/fedora-coreos/kubernetes/workers/ami.tf | 10 +--------- aws/fedora-coreos/kubernetes/workers/variables.tf | 4 ++-- 5 files changed, 7 insertions(+), 23 deletions(-) diff --git a/aws/fedora-coreos/kubernetes/ami.tf b/aws/fedora-coreos/kubernetes/ami.tf index 3056d82fe..a7ab184bd 100644 --- a/aws/fedora-coreos/kubernetes/ami.tf +++ b/aws/fedora-coreos/kubernetes/ami.tf @@ -13,16 +13,8 @@ data "aws_ami" "fedora-coreos" { values = ["hvm"] } - filter { - name = "name" - values = ["fedora-coreos-32.*.*.*-hvm"] - } - filter { name = "description" - values = ["Fedora CoreOS stable*"] + values = ["Fedora CoreOS ${var.os_stream} *"] } - - # try to filter out dev images (AWS filters can't) - name_regex = "^fedora-coreos-32.[0-9]*.[0-9]*.[0-9]*-hvm*" } diff --git a/aws/fedora-coreos/kubernetes/variables.tf b/aws/fedora-coreos/kubernetes/variables.tf index b0b0db4a4..5347fa2c1 100644 --- a/aws/fedora-coreos/kubernetes/variables.tf +++ b/aws/fedora-coreos/kubernetes/variables.tf @@ -41,9 +41,9 @@ variable "worker_type" { default = "t3.small" } -variable "os_image" { +variable "os_stream" { type = string - description = "AMI channel for Fedora CoreOS (not yet used)" + description = "Fedora CoreOs image stream for instances (e.g. stable, testing, next)" default = "stable" } diff --git a/aws/fedora-coreos/kubernetes/workers.tf b/aws/fedora-coreos/kubernetes/workers.tf index 5c1eddd97..41436793e 100644 --- a/aws/fedora-coreos/kubernetes/workers.tf +++ b/aws/fedora-coreos/kubernetes/workers.tf @@ -9,7 +9,7 @@ module "workers" { security_groups = [aws_security_group.worker.id] worker_count = var.worker_count instance_type = var.worker_type - os_image = var.os_image + os_stream = var.os_stream disk_size = var.disk_size spot_price = var.worker_price target_groups = var.worker_target_groups diff --git a/aws/fedora-coreos/kubernetes/workers/ami.tf b/aws/fedora-coreos/kubernetes/workers/ami.tf index e32ce159f..a7ab184bd 100644 --- a/aws/fedora-coreos/kubernetes/workers/ami.tf +++ b/aws/fedora-coreos/kubernetes/workers/ami.tf @@ -13,16 +13,8 @@ data "aws_ami" "fedora-coreos" { values = ["hvm"] } - filter { - name = "name" - values = ["fedora-coreos-31.*.*.*-hvm"] - } - filter { name = "description" - values = ["Fedora CoreOS stable*"] + values = ["Fedora CoreOS ${var.os_stream} *"] } - - # try to filter out dev images (AWS filters can't) - name_regex = "^fedora-coreos-31.[0-9]*.[0-9]*.[0-9]*-hvm*" } diff --git a/aws/fedora-coreos/kubernetes/workers/variables.tf b/aws/fedora-coreos/kubernetes/workers/variables.tf index e07e216f4..0f2f1322a 100644 --- a/aws/fedora-coreos/kubernetes/workers/variables.tf +++ b/aws/fedora-coreos/kubernetes/workers/variables.tf @@ -39,9 +39,9 @@ variable "instance_type" { default = "t3.small" } -variable "os_image" { +variable "os_stream" { type = string - description = "AMI channel for Fedora CoreOS (not yet used)" + description = "Fedora CoreOs image stream for instances (e.g. stable, testing, next)" default = "stable" } From 7bc8066622f387bccc224a18d58df0877720de8c Mon Sep 17 00:00:00 2001 From: Bo Huang Date: Fri, 7 Aug 2020 15:06:22 -0700 Subject: [PATCH 505/523] fix mistakes in resolving merging conflicts --- aws/fedora-coreos/kubernetes/nlb.tf | 30 +------ aws/fedora-coreos/kubernetes/outputs.tf | 80 ++++++++++++++++++- .../kubernetes/workers/outputs.tf | 9 +++ .../kubernetes/workers/variables.tf | 33 ++++++-- 4 files changed, 118 insertions(+), 34 deletions(-) diff --git a/aws/fedora-coreos/kubernetes/nlb.tf b/aws/fedora-coreos/kubernetes/nlb.tf index 2b87366a7..2e8b2cbab 100644 --- a/aws/fedora-coreos/kubernetes/nlb.tf +++ b/aws/fedora-coreos/kubernetes/nlb.tf @@ -19,7 +19,7 @@ resource "aws_lb" "nlb" { load_balancer_type = "network" internal = false - subnets = aws_subnet.public.*.id + subnets = aws_subnet.private.*.id enable_cross_zone_load_balancing = true } @@ -36,35 +36,11 @@ resource "aws_lb_listener" "apiserver-https" { } } -# Forward HTTP ingress traffic to workers -resource "aws_lb_listener" "ingress-http" { - load_balancer_arn = aws_lb.nlb.arn - protocol = "TCP" - port = 80 - - default_action { - type = "forward" - target_group_arn = module.workers.target_group_http - } -} - -# Forward HTTPS ingress traffic to workers -resource "aws_lb_listener" "ingress-https" { - load_balancer_arn = aws_lb.nlb.arn - protocol = "TCP" - port = 443 - - default_action { - type = "forward" - target_group_arn = module.workers.target_group_https - } -} - # Target group of controllers resource "aws_lb_target_group" "controllers" { name = "${var.cluster_name}-controllers" vpc_id = aws_vpc.network.id - target_type = "instance" + target_type = "ip" protocol = "TCP" port = 6443 @@ -88,7 +64,7 @@ resource "aws_lb_target_group_attachment" "controllers" { count = var.controller_count target_group_arn = aws_lb_target_group.controllers.arn - target_id = aws_instance.controllers.*.id[count.index] + target_id = aws_instance.controllers.*.private_ip[count.index] port = 6443 } diff --git a/aws/fedora-coreos/kubernetes/outputs.tf b/aws/fedora-coreos/kubernetes/outputs.tf index d9afc7bd3..3e605f974 100644 --- a/aws/fedora-coreos/kubernetes/outputs.tf +++ b/aws/fedora-coreos/kubernetes/outputs.tf @@ -21,9 +21,14 @@ output "vpc_id" { description = "ID of the VPC for creating worker instances" } -output "subnet_ids" { +output "private_subnet_ids" { + value = aws_subnet.private.*.id + description = "List of private subnet IDs" +} + +output "public_subnet_ids" { value = aws_subnet.public.*.id - description = "List of subnet IDs for creating worker instances" + description = "List of public subnet IDs" } output "worker_security_groups" { @@ -35,6 +40,11 @@ output "kubeconfig" { value = module.bootstrap.kubeconfig-kubelet } +output "kube_ca" { + description = "Base64-encoded CA cert data for Kubernetes apiserver" + value = module.bootstrap.ca_cert +} + # Outputs for custom load balancing output "nlb_id" { @@ -52,3 +62,69 @@ output "worker_target_group_https" { value = module.workers.target_group_https } +# Scoop outputs + +output "bastion_dns_name" { + value = aws_lb.bastion.dns_name + description = "DNS name of the network load balancer for distributing traffic to bastion hosts" + + depends_on = [ + aws_autoscaling_group.bastion + ] +} + +output "apiserver_dns_name" { + value = aws_route53_record.apiserver.fqdn + description = "DNS name of the Route53 record used to access the Kubernetes apiserver" +} + +output "bootstrap_controller_ip" { + value = aws_instance.controllers.0.private_ip + description = "IP address of the controller instance used to bootstrap the cluster" +} + +output "nat_ips" { + value = aws_eip.nat.*.public_ip + description = "List of NAT IPs where public traffic from this cluster will originate" +} + +output "private_route_tables" { + value = aws_route_table.private.*.id + description = "IDs of the private route tables that can be used to add additional private routes" +} + +output "private_route_tables_count" { + value = length(aws_route_table.private) + description = "Number of private route tables that are created" +} + +output "public_route_tables" { + value = aws_route_table.public.*.id + description = "IDs of the public route tables" +} + +output "public_route_tables_count" { + value = length(aws_route_table.public) + description = "Number of public route tables that are created" +} + +output "depends_id" { + value = null_resource.bootstrap.id + description = "Resource ID that will be defined when the cluster is ready" +} + +output "controller_role" { + value = aws_iam_role.controller.arn + description = "Instance role ARN attached to controller instances via instance profile" +} + +output "worker_role" { + value = module.workers.instance_role + description = "Instance role ARN attached to worker instances via instance profile" +} + +output "worker_autoscaling_group" { + value = module.workers.autoscaling_group + description = "Name of the workers autoscaling group" +} + diff --git a/aws/fedora-coreos/kubernetes/workers/outputs.tf b/aws/fedora-coreos/kubernetes/workers/outputs.tf index 22f378855..ea9141c33 100644 --- a/aws/fedora-coreos/kubernetes/workers/outputs.tf +++ b/aws/fedora-coreos/kubernetes/workers/outputs.tf @@ -8,3 +8,12 @@ output "target_group_https" { value = aws_lb_target_group.workers-https.arn } +output "instance_role" { + description = "IAM role ARN attached to instances via instance profile" + value = aws_iam_role.worker.arn +} + +output "autoscaling_group" { + description = "Name of the workers autoscaling group" + value = aws_autoscaling_group.workers.name +} diff --git a/aws/fedora-coreos/kubernetes/workers/variables.tf b/aws/fedora-coreos/kubernetes/workers/variables.tf index 76b33cbf9..0f2f1322a 100644 --- a/aws/fedora-coreos/kubernetes/workers/variables.tf +++ b/aws/fedora-coreos/kubernetes/workers/variables.tf @@ -1,3 +1,8 @@ +variable "cluster_name" { + type = string + description = "Name of the cluster the workers belong to" +} + variable "name" { type = string description = "Unique name for the worker pool" @@ -83,11 +88,6 @@ variable "kubeconfig" { description = "Must be set to `kubeconfig` output by cluster" } -variable "ssh_authorized_key" { - type = string - description = "SSH public key for user 'core'" -} - variable "service_cidr" { type = string description = < Date: Fri, 7 Aug 2020 16:37:47 -0700 Subject: [PATCH 506/523] add new security components --- aws/fedora-coreos/kubernetes/security.tf | 191 ++++++++++++++++++++--- 1 file changed, 170 insertions(+), 21 deletions(-) diff --git a/aws/fedora-coreos/kubernetes/security.tf b/aws/fedora-coreos/kubernetes/security.tf index 4e4e68027..fa91522d9 100644 --- a/aws/fedora-coreos/kubernetes/security.tf +++ b/aws/fedora-coreos/kubernetes/security.tf @@ -13,6 +13,30 @@ resource "aws_security_group" "controller" { } } +resource "aws_security_group_rule" "controller-icmp" { + count = var.networking == "cilium" ? 1 : 0 + + security_group_id = aws_security_group.controller.id + + type = "ingress" + protocol = "icmp" + from_port = 8 + to_port = 0 + source_security_group_id = aws_security_group.worker.id +} + +resource "aws_security_group_rule" "controller-icmp-self" { + count = var.networking == "cilium" ? 1 : 0 + + security_group_id = aws_security_group.controller.id + + type = "ingress" + protocol = "icmp" + from_port = 8 + to_port = 0 + self = true +} + resource "aws_security_group_rule" "controller-ssh" { security_group_id = aws_security_group.controller.id @@ -54,39 +78,31 @@ resource "aws_security_group_rule" "controller-etcd-metrics-self" { self = true } -# Allow Prometheus to scrape kube-proxy -resource "aws_security_group_rule" "kube-proxy-metrics" { - security_group_id = aws_security_group.controller.id +resource "aws_security_group_rule" "controller-cilium-health" { + count = var.networking == "cilium" ? 1 : 0 - type = "ingress" - protocol = "tcp" - from_port = 10249 - to_port = 10249 - source_security_group_id = aws_security_group.worker.id -} - -# Allow Prometheus to scrape kube-scheduler -resource "aws_security_group_rule" "controller-scheduler-metrics" { security_group_id = aws_security_group.controller.id type = "ingress" protocol = "tcp" - from_port = 10251 - to_port = 10251 + from_port = 4240 + to_port = 4240 source_security_group_id = aws_security_group.worker.id } -# Allow Prometheus to scrape kube-controller-manager -resource "aws_security_group_rule" "controller-manager-metrics" { +resource "aws_security_group_rule" "controller-cilium-health-self" { + count = var.networking == "cilium" ? 1 : 0 + security_group_id = aws_security_group.controller.id - type = "ingress" - protocol = "tcp" - from_port = 10252 - to_port = 10252 - source_security_group_id = aws_security_group.worker.id + type = "ingress" + protocol = "tcp" + from_port = 4240 + to_port = 4240 + self = true } +# IANA VXLAN default resource "aws_security_group_rule" "controller-vxlan" { count = var.networking == "flannel" ? 1 : 0 @@ -121,6 +137,31 @@ resource "aws_security_group_rule" "controller-apiserver" { cidr_blocks = ["0.0.0.0/0"] } +# Linux VXLAN default +resource "aws_security_group_rule" "controller-linux-vxlan" { + count = var.networking == "cilium" ? 1 : 0 + + security_group_id = aws_security_group.controller.id + + type = "ingress" + protocol = "udp" + from_port = 8472 + to_port = 8472 + source_security_group_id = aws_security_group.worker.id +} + +resource "aws_security_group_rule" "controller-linux-vxlan-self" { + count = var.networking == "cilium" ? 1 : 0 + + security_group_id = aws_security_group.controller.id + + type = "ingress" + protocol = "udp" + from_port = 8472 + to_port = 8472 + self = true +} + # Allow Prometheus to scrape node-exporter daemonset resource "aws_security_group_rule" "controller-node-exporter" { security_group_id = aws_security_group.controller.id @@ -132,6 +173,18 @@ resource "aws_security_group_rule" "controller-node-exporter" { source_security_group_id = aws_security_group.worker.id } + +# Allow Prometheus to scrape kube-proxy +resource "aws_security_group_rule" "kube-proxy-metrics" { + security_group_id = aws_security_group.controller.id + + type = "ingress" + protocol = "tcp" + from_port = 10249 + to_port = 10249 + source_security_group_id = aws_security_group.worker.id +} + # Allow apiserver to access kubelets for exec, log, port-forward resource "aws_security_group_rule" "controller-kubelet" { security_group_id = aws_security_group.controller.id @@ -153,6 +206,28 @@ resource "aws_security_group_rule" "controller-kubelet-self" { self = true } +# Allow Prometheus to scrape kube-scheduler +resource "aws_security_group_rule" "controller-scheduler-metrics" { + security_group_id = aws_security_group.controller.id + + type = "ingress" + protocol = "tcp" + from_port = 10251 + to_port = 10251 + source_security_group_id = aws_security_group.worker.id +} + +# Allow Prometheus to scrape kube-controller-manager +resource "aws_security_group_rule" "controller-manager-metrics" { + security_group_id = aws_security_group.controller.id + + type = "ingress" + protocol = "tcp" + from_port = 10252 + to_port = 10252 + source_security_group_id = aws_security_group.worker.id +} + resource "aws_security_group_rule" "controller-bgp" { security_group_id = aws_security_group.controller.id @@ -237,6 +312,30 @@ resource "aws_security_group" "worker" { } } +resource "aws_security_group_rule" "worker-icmp" { + count = var.networking == "cilium" ? 1 : 0 + + security_group_id = aws_security_group.worker.id + + type = "ingress" + protocol = "icmp" + from_port = 8 + to_port = 0 + source_security_group_id = aws_security_group.controller.id +} + +resource "aws_security_group_rule" "worker-icmp-self" { + count = var.networking == "cilium" ? 1 : 0 + + security_group_id = aws_security_group.worker.id + + type = "ingress" + protocol = "icmp" + from_port = 8 + to_port = 0 + self = true +} + resource "aws_security_group_rule" "worker-ssh-bastion" { security_group_id = aws_security_group.worker.id @@ -277,6 +376,31 @@ resource "aws_security_group_rule" "worker-https" { cidr_blocks = ["0.0.0.0/0"] } +resource "aws_security_group_rule" "worker-cilium-health" { + count = var.networking == "cilium" ? 1 : 0 + + security_group_id = aws_security_group.worker.id + + type = "ingress" + protocol = "tcp" + from_port = 4240 + to_port = 4240 + source_security_group_id = aws_security_group.controller.id +} + +resource "aws_security_group_rule" "worker-cilium-health-self" { + count = var.networking == "cilium" ? 1 : 0 + + security_group_id = aws_security_group.worker.id + + type = "ingress" + protocol = "tcp" + from_port = 4240 + to_port = 4240 + self = true +} + +# IANA VXLAN default resource "aws_security_group_rule" "worker-vxlan" { count = var.networking == "flannel" ? 1 : 0 @@ -301,6 +425,31 @@ resource "aws_security_group_rule" "worker-vxlan-self" { self = true } +# Linux VXLAN default +resource "aws_security_group_rule" "worker-linux-vxlan" { + count = var.networking == "cilium" ? 1 : 0 + + security_group_id = aws_security_group.worker.id + + type = "ingress" + protocol = "udp" + from_port = 8472 + to_port = 8472 + source_security_group_id = aws_security_group.controller.id +} + +resource "aws_security_group_rule" "worker-linux-vxlan-self" { + count = var.networking == "cilium" ? 1 : 0 + + security_group_id = aws_security_group.worker.id + + type = "ingress" + protocol = "udp" + from_port = 8472 + to_port = 8472 + self = true +} + # Allow Prometheus to scrape node-exporter daemonset resource "aws_security_group_rule" "worker-node-exporter" { security_group_id = aws_security_group.worker.id From d326a67051502ef4dede26740c4245b303316694 Mon Sep 17 00:00:00 2001 From: Bo Huang Date: Fri, 7 Aug 2020 18:55:02 -0700 Subject: [PATCH 507/523] fix json format --- aws/fedora-coreos/kubernetes/controllers.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aws/fedora-coreos/kubernetes/controllers.tf b/aws/fedora-coreos/kubernetes/controllers.tf index d38509620..ae1330601 100644 --- a/aws/fedora-coreos/kubernetes/controllers.tf +++ b/aws/fedora-coreos/kubernetes/controllers.tf @@ -34,7 +34,7 @@ resource "aws_instance" "controllers" { "config": { "merge": [ { - source = var.base_ignition_config_path + "source": "${var.base_ignition_config_path}" }, { "source": "s3://${aws_s3_bucket.ignition_configs.id}/${aws_s3_bucket_object.controller-ignitions.*.id[count.index]}" From e1d6ab2f240d7b41614447a15dd8847f41ee35c2 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Sat, 8 Aug 2020 18:59:49 -0700 Subject: [PATCH 508/523] Update Grafana from v7.1.1 to v7.1.3 * https://github.com/grafana/grafana/releases/tag/v7.1.3 * https://github.com/grafana/grafana/releases/tag/v7.1.2 --- CHANGES.md | 8 ++++---- addons/grafana/deployment.yaml | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index d57a000d8..b736b6cd8 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -14,14 +14,14 @@ Notable changes between versions. ### Fedora CoreOS * Fix support for Flannel with Fedora CoreOS ([#795](https://github.com/poseidon/typhoon/pull/795)) - * Fedora CoreOS fixes to align network interface defaults altered MAC address assignment for - the `flannel.1` interface in a way that caused flannel to drop pod-to-pod traffic - * Configure flannel interfaces explicitly + * Configure `flannel.1` link to select its own MAC address to solve flannel + pod-to-pod traffic drops starting with default link changes in Fedora CoreOS + 32.20200629.3.0 ([details](https://github.com/coreos/fedora-coreos-tracker/issues/574#issuecomment-665487296)) #### Addons * Update Prometheus from v2.19.2 to [v2.20.0](https://github.com/prometheus/prometheus/releases/tag/v2.20.0) -* Update Grafana from v7.0.6 to [v7.1.1](https://github.com/grafana/grafana/releases/tag/v7.1.1) +* Update Grafana from v7.0.6 to [v7.1.3](https://github.com/grafana/grafana/releases/tag/v7.1.3) ## v1.18.6 diff --git a/addons/grafana/deployment.yaml b/addons/grafana/deployment.yaml index f25a80e08..eb2b9ec56 100644 --- a/addons/grafana/deployment.yaml +++ b/addons/grafana/deployment.yaml @@ -23,7 +23,7 @@ spec: spec: containers: - name: grafana - image: docker.io/grafana/grafana:7.1.1 + image: docker.io/grafana/grafana:7.1.3 env: - name: GF_PATHS_CONFIG value: "/etc/grafana/custom.ini" From f6ce12766b8c488a62f5c0c0bbf723e6f2045181 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Sun, 9 Aug 2020 12:25:27 -0700 Subject: [PATCH 509/523] Allow terraform-provider-aws v3.0+ plugin * Typhoon AWS is compatible with terraform-provider-aws v3.x releases * Continue to allow v2.23+, no v3.x specific features are used * Set required provider versions in the worker module, since it can be used independently Related: * https://github.com/terraform-providers/terraform-provider-aws/releases/tag/v3.0.0 --- CHANGES.md | 6 ++++++ aws/container-linux/kubernetes/versions.tf | 2 +- aws/container-linux/kubernetes/workers/versions.tf | 5 +++++ aws/fedora-coreos/kubernetes/versions.tf | 2 +- aws/fedora-coreos/kubernetes/workers/versions.tf | 5 +++++ 5 files changed, 18 insertions(+), 2 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index b736b6cd8..0798ad870 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -11,6 +11,12 @@ Notable changes between versions. * Update CNI plugins and fix CVEs with Flannel (non-default) * Transition to a poseidon maintained container image +### AWS + +* Allow `terraform-provider-aws` v3.0+ ([#803](https://github.com/poseidon/typhoon/pull/803)) + * Recommend updating `terraform-provider-aws` to v3.0+ + * Continue to allow v2.23+, no v3.x specific features are used + ### Fedora CoreOS * Fix support for Flannel with Fedora CoreOS ([#795](https://github.com/poseidon/typhoon/pull/795)) diff --git a/aws/container-linux/kubernetes/versions.tf b/aws/container-linux/kubernetes/versions.tf index bd2776f08..b07833503 100644 --- a/aws/container-linux/kubernetes/versions.tf +++ b/aws/container-linux/kubernetes/versions.tf @@ -3,7 +3,7 @@ terraform { required_version = "~> 0.12.6" required_providers { - aws = "~> 2.23" + aws = ">= 2.23, <= 4.0" ct = "~> 0.4" template = "~> 2.1" null = "~> 2.1" diff --git a/aws/container-linux/kubernetes/workers/versions.tf b/aws/container-linux/kubernetes/workers/versions.tf index ac97c6ac8..c9c25e95b 100644 --- a/aws/container-linux/kubernetes/workers/versions.tf +++ b/aws/container-linux/kubernetes/workers/versions.tf @@ -1,4 +1,9 @@ terraform { required_version = ">= 0.12" + required_providers { + aws = ">= 2.23, <= 4.0" + ct = "~> 0.4" + template = "~> 2.1" + } } diff --git a/aws/fedora-coreos/kubernetes/versions.tf b/aws/fedora-coreos/kubernetes/versions.tf index bd2776f08..b07833503 100644 --- a/aws/fedora-coreos/kubernetes/versions.tf +++ b/aws/fedora-coreos/kubernetes/versions.tf @@ -3,7 +3,7 @@ terraform { required_version = "~> 0.12.6" required_providers { - aws = "~> 2.23" + aws = ">= 2.23, <= 4.0" ct = "~> 0.4" template = "~> 2.1" null = "~> 2.1" diff --git a/aws/fedora-coreos/kubernetes/workers/versions.tf b/aws/fedora-coreos/kubernetes/workers/versions.tf index ac97c6ac8..c9c25e95b 100644 --- a/aws/fedora-coreos/kubernetes/workers/versions.tf +++ b/aws/fedora-coreos/kubernetes/workers/versions.tf @@ -1,4 +1,9 @@ terraform { required_version = ">= 0.12" + required_providers { + aws = ">= 2.23, <= 4.0" + ct = "~> 0.4" + template = "~> 2.1" + } } From aab071309f3371942e478ff5d3cca4aff3d5a1e2 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Sun, 9 Aug 2020 12:40:22 -0700 Subject: [PATCH 510/523] Update recommended Terraform provider versions * Sync Terraform provider plugin versions to those used internally --- docs/fedora-coreos/aws.md | 2 +- docs/fedora-coreos/azure.md | 2 +- docs/fedora-coreos/bare-metal.md | 2 +- docs/fedora-coreos/google-cloud.md | 2 +- docs/flatcar-linux/aws.md | 2 +- docs/flatcar-linux/azure.md | 2 +- docs/flatcar-linux/bare-metal.md | 2 +- docs/flatcar-linux/google-cloud.md | 2 +- 8 files changed, 8 insertions(+), 8 deletions(-) diff --git a/docs/fedora-coreos/aws.md b/docs/fedora-coreos/aws.md index 6f28ec4f4..cd38f1961 100644 --- a/docs/fedora-coreos/aws.md +++ b/docs/fedora-coreos/aws.md @@ -49,7 +49,7 @@ Configure the AWS provider to use your access key credentials in a `providers.tf ```tf provider "aws" { - version = "2.70.0" + version = "3.1.0" region = "eu-central-1" shared_credentials_file = "/home/user/.config/aws/credentials" } diff --git a/docs/fedora-coreos/azure.md b/docs/fedora-coreos/azure.md index 4b66a7fc8..d5ec2bd5d 100644 --- a/docs/fedora-coreos/azure.md +++ b/docs/fedora-coreos/azure.md @@ -47,7 +47,7 @@ Configure the Azure provider in a `providers.tf` file. ```tf provider "azurerm" { - version = "2.19.0" + version = "2.22.0" } provider "ct" { diff --git a/docs/fedora-coreos/bare-metal.md b/docs/fedora-coreos/bare-metal.md index 2a257d8dc..c866e8764 100644 --- a/docs/fedora-coreos/bare-metal.md +++ b/docs/fedora-coreos/bare-metal.md @@ -142,7 +142,7 @@ Configure the Matchbox provider to use your Matchbox API endpoint and client cer ```tf provider "matchbox" { - version = "0.3.0" + version = "0.4.0" endpoint = "matchbox.example.com:8081" client_cert = file("~/.config/matchbox/client.crt") client_key = file("~/.config/matchbox/client.key") diff --git a/docs/fedora-coreos/google-cloud.md b/docs/fedora-coreos/google-cloud.md index d11077038..25d648d0d 100644 --- a/docs/fedora-coreos/google-cloud.md +++ b/docs/fedora-coreos/google-cloud.md @@ -49,7 +49,7 @@ Configure the Google Cloud provider to use your service account key, project-id, ```tf provider "google" { - version = "3.30.0" + version = "3.33.0" project = "project-id" region = "us-central1" credentials = file("~/.config/google-cloud/terraform.json") diff --git a/docs/flatcar-linux/aws.md b/docs/flatcar-linux/aws.md index 82d0846ae..d9e363f82 100644 --- a/docs/flatcar-linux/aws.md +++ b/docs/flatcar-linux/aws.md @@ -49,7 +49,7 @@ Configure the AWS provider to use your access key credentials in a `providers.tf ```tf provider "aws" { - version = "2.70.0" + version = "3.1.0" region = "eu-central-1" shared_credentials_file = "/home/user/.config/aws/credentials" } diff --git a/docs/flatcar-linux/azure.md b/docs/flatcar-linux/azure.md index bff37e2c6..d89710c0a 100644 --- a/docs/flatcar-linux/azure.md +++ b/docs/flatcar-linux/azure.md @@ -47,7 +47,7 @@ Configure the Azure provider in a `providers.tf` file. ```tf provider "azurerm" { - version = "2.19.0" + version = "2.22.0" } provider "ct" { diff --git a/docs/flatcar-linux/bare-metal.md b/docs/flatcar-linux/bare-metal.md index b41e69fbe..c71460295 100644 --- a/docs/flatcar-linux/bare-metal.md +++ b/docs/flatcar-linux/bare-metal.md @@ -142,7 +142,7 @@ Configure the Matchbox provider to use your Matchbox API endpoint and client cer ```tf provider "matchbox" { - version = "0.3.0" + version = "0.4.0" endpoint = "matchbox.example.com:8081" client_cert = file("~/.config/matchbox/client.crt") client_key = file("~/.config/matchbox/client.key") diff --git a/docs/flatcar-linux/google-cloud.md b/docs/flatcar-linux/google-cloud.md index 92afabfaf..42cd6a19b 100644 --- a/docs/flatcar-linux/google-cloud.md +++ b/docs/flatcar-linux/google-cloud.md @@ -49,7 +49,7 @@ Configure the Google Cloud provider to use your service account key, project-id, ```tf provider "google" { - version = "3.30.0" + version = "3.33.0" project = "project-id" region = "us-central1" credentials = file("~/.config/google-cloud/terraform.json") From 6fa413500645fb8c44b0b54e399f074753aedc0c Mon Sep 17 00:00:00 2001 From: Bo Huang Date: Mon, 10 Aug 2020 14:15:08 -0700 Subject: [PATCH 511/523] fix ssl cert mounts --- aws/fedora-coreos/kubernetes/fcc/controller.yaml | 5 ++++- aws/fedora-coreos/kubernetes/workers/fcc/worker.yaml | 3 ++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/aws/fedora-coreos/kubernetes/fcc/controller.yaml b/aws/fedora-coreos/kubernetes/fcc/controller.yaml index 5d4a36eec..2072aa302 100644 --- a/aws/fedora-coreos/kubernetes/fcc/controller.yaml +++ b/aws/fedora-coreos/kubernetes/fcc/controller.yaml @@ -28,6 +28,8 @@ systemd: --network host \ --volume /var/lib/etcd:/var/lib/etcd:rw,Z \ --volume /etc/ssl/etcd:/etc/ssl/certs:ro,Z \ + --volume /etc/pki/tls/certs:/etc/pki/tls/certs:ro \ + --volume /etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro \ quay.io/coreos/etcd:v3.4.10 ExecStop=/usr/bin/podman stop etcd [Install] @@ -74,7 +76,8 @@ systemd: --volume /run:/run \ --volume /sys/fs/cgroup:/sys/fs/cgroup:ro \ --volume /sys/fs/cgroup/systemd:/sys/fs/cgroup/systemd \ - --volume /etc/pki/tls/certs:/usr/share/ca-certificates:ro \ + --volume /etc/pki/tls/certs:/etc/pki/tls/certs:ro \ + --volume /etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro \ --volume /var/lib/calico:/var/lib/calico:ro \ --volume /var/lib/docker:/var/lib/docker \ --volume /var/lib/kubelet:/var/lib/kubelet:rshared,z \ diff --git a/aws/fedora-coreos/kubernetes/workers/fcc/worker.yaml b/aws/fedora-coreos/kubernetes/workers/fcc/worker.yaml index 6d22b3626..2cfc6dd3b 100644 --- a/aws/fedora-coreos/kubernetes/workers/fcc/worker.yaml +++ b/aws/fedora-coreos/kubernetes/workers/fcc/worker.yaml @@ -44,7 +44,8 @@ systemd: --volume /run:/run \ --volume /sys/fs/cgroup:/sys/fs/cgroup:ro \ --volume /sys/fs/cgroup/systemd:/sys/fs/cgroup/systemd \ - --volume /etc/pki/tls/certs:/usr/share/ca-certificates:ro \ + --volume /etc/pki/tls/certs:/etc/pki/tls/certs:ro \ + --volume /etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro \ --volume /var/lib/calico:/var/lib/calico:ro \ --volume /var/lib/docker:/var/lib/docker \ --volume /var/lib/kubelet:/var/lib/kubelet:rshared,z \ From 5e70d7e2c8c71c37d9a0896a0945188f009011f9 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Mon, 10 Aug 2020 21:02:56 -0700 Subject: [PATCH 512/523] Migrate from Terraform v0.12.x to v0.13.x * Recommend Terraform v0.13.x * Support automatic install of poseidon's provider plugins * Update tutorial docs for Terraform v0.13.x * Add migration guide for Terraform v0.13.x (best-effort) * Require Terraform v0.12.26+ (migration compatibility) * Require `terraform-provider-ct` v0.6.1 * Require `terraform-provider-matchbox` v0.4.1 * Require `terraform-provider-digitalocean` v1.20+ Related: * https://www.hashicorp.com/blog/announcing-hashicorp-terraform-0-13/ * https://www.terraform.io/upgrade-guides/0-13.html * https://registry.terraform.io/providers/poseidon/ct/latest * https://registry.terraform.io/providers/poseidon/matchbox/latest --- CHANGES.md | 13 +++ aws/container-linux/kubernetes/bootstrap.tf | 2 +- aws/container-linux/kubernetes/versions.tf | 8 +- .../kubernetes/workers/versions.tf | 9 +- aws/fedora-coreos/kubernetes/bootstrap.tf | 2 +- aws/fedora-coreos/kubernetes/versions.tf | 8 +- .../kubernetes/workers/versions.tf | 9 +- azure/container-linux/kubernetes/bootstrap.tf | 2 +- azure/container-linux/kubernetes/versions.tf | 8 +- .../kubernetes/workers/versions.tf | 12 ++- azure/fedora-coreos/kubernetes/bootstrap.tf | 2 +- azure/fedora-coreos/kubernetes/versions.tf | 8 +- .../kubernetes/workers/versions.tf | 12 ++- .../container-linux/kubernetes/bootstrap.tf | 2 +- .../container-linux/kubernetes/versions.tf | 14 ++- .../fedora-coreos/kubernetes/bootstrap.tf | 2 +- .../fedora-coreos/kubernetes/versions.tf | 14 ++- .../container-linux/kubernetes/bootstrap.tf | 2 +- .../container-linux/kubernetes/versions.tf | 18 +++- .../fedora-coreos/kubernetes/bootstrap.tf | 2 +- .../fedora-coreos/kubernetes/versions.tf | 18 +++- docs/fedora-coreos/aws.md | 30 +++--- docs/fedora-coreos/azure.md | 31 +++--- docs/fedora-coreos/bare-metal.md | 38 ++++---- docs/fedora-coreos/digitalocean.md | 30 +++--- docs/fedora-coreos/google-cloud.md | 30 +++--- docs/flatcar-linux/aws.md | 30 +++--- docs/flatcar-linux/azure.md | 31 +++--- docs/flatcar-linux/bare-metal.md | 38 ++++---- docs/flatcar-linux/digitalocean.md | 30 +++--- docs/flatcar-linux/google-cloud.md | 30 +++--- docs/topics/maintenance.md | 95 +++++++++++++++++-- docs/topics/security.md | 9 ++ .../container-linux/kubernetes/bootstrap.tf | 2 +- .../container-linux/kubernetes/versions.tf | 8 +- .../kubernetes/workers/versions.tf | 12 ++- .../fedora-coreos/kubernetes/bootstrap.tf | 2 +- .../fedora-coreos/kubernetes/versions.tf | 8 +- .../kubernetes/workers/versions.tf | 12 ++- 39 files changed, 421 insertions(+), 212 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 0798ad870..e76bae1bb 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -4,6 +4,14 @@ Notable changes between versions. ## Latest +* Migrate from Terraform v0.12.x to v0.13.x (action required) + * Recommend Terraform v0.13.x ([migration guide](https://github.com/poseidon/typhoon/blob/260033e978f03d068b79e553e7c428a64b22a475/docs/topics/maintenance.md#terraform-versions)) + * Support automatic install of poseidon's provider plugins + * [poseidon/ct](https://registry.terraform.io/providers/poseidon/ct/latest) + * [poseidon/matchbox](https://registry.terraform.io/providers/poseidon/matchbox/latest) + * Require Terraform v0.12.26+ (migration compatibility) + * Require `terraform-provider-ct` v0.6.1 + * Require `terraform-provider-matchbox` v0.4.1 * Update etcd from v3.4.9 to [v3.4.10](https://github.com/etcd-io/etcd/releases/tag/v3.4.10) * Update CoreDNS from v1.6.7 to [v1.7.0](https://coredns.io/2020/06/15/coredns-1.7.0-release/) * Update Cilium from v1.8.1 to [v1.8.2](https://github.com/cilium/cilium/releases/tag/v1.8.2) @@ -17,6 +25,11 @@ Notable changes between versions. * Recommend updating `terraform-provider-aws` to v3.0+ * Continue to allow v2.23+, no v3.x specific features are used +### DigitalOcean + +* Require `terraform-provider-digitalocean` v1.21+ for Terraform v0.13.x (unenforced) +* Require `terraform-provider-digitalocean` v1.20+ for Terraform v0.12.x + ### Fedora CoreOS * Fix support for Flannel with Fedora CoreOS ([#795](https://github.com/poseidon/typhoon/pull/795)) diff --git a/aws/container-linux/kubernetes/bootstrap.tf b/aws/container-linux/kubernetes/bootstrap.tf index 68d5f8192..937a2d05e 100644 --- a/aws/container-linux/kubernetes/bootstrap.tf +++ b/aws/container-linux/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=3675b3a539efd8d341277f0c03322883f97fd992" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=60540868e070ea95da05fc5a6a5c9a55ce68c58d" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/aws/container-linux/kubernetes/versions.tf b/aws/container-linux/kubernetes/versions.tf index b07833503..75b52097f 100644 --- a/aws/container-linux/kubernetes/versions.tf +++ b/aws/container-linux/kubernetes/versions.tf @@ -1,11 +1,15 @@ # Terraform version and plugin versions terraform { - required_version = "~> 0.12.6" + required_version = ">= 0.12.26, < 0.14.0" required_providers { aws = ">= 2.23, <= 4.0" - ct = "~> 0.4" template = "~> 2.1" null = "~> 2.1" + + ct = { + source = "poseidon/ct" + version = "~> 0.6.1" + } } } diff --git a/aws/container-linux/kubernetes/workers/versions.tf b/aws/container-linux/kubernetes/workers/versions.tf index c9c25e95b..564a6ff38 100644 --- a/aws/container-linux/kubernetes/workers/versions.tf +++ b/aws/container-linux/kubernetes/workers/versions.tf @@ -1,9 +1,14 @@ +# Terraform version and plugin versions terraform { - required_version = ">= 0.12" + required_version = ">= 0.12.26, < 0.14.0" required_providers { aws = ">= 2.23, <= 4.0" - ct = "~> 0.4" template = "~> 2.1" + + ct = { + source = "poseidon/ct" + version = "~> 0.6.1" + } } } diff --git a/aws/fedora-coreos/kubernetes/bootstrap.tf b/aws/fedora-coreos/kubernetes/bootstrap.tf index 04d39b05f..033c57eff 100644 --- a/aws/fedora-coreos/kubernetes/bootstrap.tf +++ b/aws/fedora-coreos/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=3675b3a539efd8d341277f0c03322883f97fd992" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=60540868e070ea95da05fc5a6a5c9a55ce68c58d" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/aws/fedora-coreos/kubernetes/versions.tf b/aws/fedora-coreos/kubernetes/versions.tf index b07833503..75b52097f 100644 --- a/aws/fedora-coreos/kubernetes/versions.tf +++ b/aws/fedora-coreos/kubernetes/versions.tf @@ -1,11 +1,15 @@ # Terraform version and plugin versions terraform { - required_version = "~> 0.12.6" + required_version = ">= 0.12.26, < 0.14.0" required_providers { aws = ">= 2.23, <= 4.0" - ct = "~> 0.4" template = "~> 2.1" null = "~> 2.1" + + ct = { + source = "poseidon/ct" + version = "~> 0.6.1" + } } } diff --git a/aws/fedora-coreos/kubernetes/workers/versions.tf b/aws/fedora-coreos/kubernetes/workers/versions.tf index c9c25e95b..564a6ff38 100644 --- a/aws/fedora-coreos/kubernetes/workers/versions.tf +++ b/aws/fedora-coreos/kubernetes/workers/versions.tf @@ -1,9 +1,14 @@ +# Terraform version and plugin versions terraform { - required_version = ">= 0.12" + required_version = ">= 0.12.26, < 0.14.0" required_providers { aws = ">= 2.23, <= 4.0" - ct = "~> 0.4" template = "~> 2.1" + + ct = { + source = "poseidon/ct" + version = "~> 0.6.1" + } } } diff --git a/azure/container-linux/kubernetes/bootstrap.tf b/azure/container-linux/kubernetes/bootstrap.tf index b51496e3d..f4f826865 100644 --- a/azure/container-linux/kubernetes/bootstrap.tf +++ b/azure/container-linux/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=3675b3a539efd8d341277f0c03322883f97fd992" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=60540868e070ea95da05fc5a6a5c9a55ce68c58d" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/azure/container-linux/kubernetes/versions.tf b/azure/container-linux/kubernetes/versions.tf index 5da5fff00..e90c976c6 100644 --- a/azure/container-linux/kubernetes/versions.tf +++ b/azure/container-linux/kubernetes/versions.tf @@ -1,12 +1,16 @@ # Terraform version and plugin versions terraform { - required_version = "~> 0.12.6" + required_version = ">= 0.12.26, < 0.14.0" required_providers { azurerm = "~> 2.8" - ct = "~> 0.4" template = "~> 2.1" null = "~> 2.1" + + ct = { + source = "poseidon/ct" + version = "~> 0.6.1" + } } } diff --git a/azure/container-linux/kubernetes/workers/versions.tf b/azure/container-linux/kubernetes/workers/versions.tf index ac97c6ac8..b8f7c72ea 100644 --- a/azure/container-linux/kubernetes/workers/versions.tf +++ b/azure/container-linux/kubernetes/workers/versions.tf @@ -1,4 +1,14 @@ +# Terraform version and plugin versions terraform { - required_version = ">= 0.12" + required_version = ">= 0.12.26, < 0.14.0" + required_providers { + azurerm = "~> 2.8" + template = "~> 2.1" + + ct = { + source = "poseidon/ct" + version = "~> 0.6.1" + } + } } diff --git a/azure/fedora-coreos/kubernetes/bootstrap.tf b/azure/fedora-coreos/kubernetes/bootstrap.tf index d3f2d5ff3..67d872f51 100644 --- a/azure/fedora-coreos/kubernetes/bootstrap.tf +++ b/azure/fedora-coreos/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=3675b3a539efd8d341277f0c03322883f97fd992" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=60540868e070ea95da05fc5a6a5c9a55ce68c58d" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/azure/fedora-coreos/kubernetes/versions.tf b/azure/fedora-coreos/kubernetes/versions.tf index 5da5fff00..e90c976c6 100644 --- a/azure/fedora-coreos/kubernetes/versions.tf +++ b/azure/fedora-coreos/kubernetes/versions.tf @@ -1,12 +1,16 @@ # Terraform version and plugin versions terraform { - required_version = "~> 0.12.6" + required_version = ">= 0.12.26, < 0.14.0" required_providers { azurerm = "~> 2.8" - ct = "~> 0.4" template = "~> 2.1" null = "~> 2.1" + + ct = { + source = "poseidon/ct" + version = "~> 0.6.1" + } } } diff --git a/azure/fedora-coreos/kubernetes/workers/versions.tf b/azure/fedora-coreos/kubernetes/workers/versions.tf index ac97c6ac8..b8f7c72ea 100644 --- a/azure/fedora-coreos/kubernetes/workers/versions.tf +++ b/azure/fedora-coreos/kubernetes/workers/versions.tf @@ -1,4 +1,14 @@ +# Terraform version and plugin versions terraform { - required_version = ">= 0.12" + required_version = ">= 0.12.26, < 0.14.0" + required_providers { + azurerm = "~> 2.8" + template = "~> 2.1" + + ct = { + source = "poseidon/ct" + version = "~> 0.6.1" + } + } } diff --git a/bare-metal/container-linux/kubernetes/bootstrap.tf b/bare-metal/container-linux/kubernetes/bootstrap.tf index ea5fed8e5..069bf5fbf 100644 --- a/bare-metal/container-linux/kubernetes/bootstrap.tf +++ b/bare-metal/container-linux/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=3675b3a539efd8d341277f0c03322883f97fd992" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=60540868e070ea95da05fc5a6a5c9a55ce68c58d" cluster_name = var.cluster_name api_servers = [var.k8s_domain_name] diff --git a/bare-metal/container-linux/kubernetes/versions.tf b/bare-metal/container-linux/kubernetes/versions.tf index 6a31221ee..1efd6a18c 100644 --- a/bare-metal/container-linux/kubernetes/versions.tf +++ b/bare-metal/container-linux/kubernetes/versions.tf @@ -1,12 +1,20 @@ # Terraform version and plugin versions terraform { - required_version = "~> 0.12.6" + required_version = ">= 0.12.26, < 0.14.0" required_providers { - matchbox = "~> 0.3" - ct = "~> 0.4" template = "~> 2.1" null = "~> 2.1" + + ct = { + source = "poseidon/ct" + version = "~> 0.6.1" + } + + matchbox = { + source = "poseidon/matchbox" + version = "~> 0.4.1" + } } } diff --git a/bare-metal/fedora-coreos/kubernetes/bootstrap.tf b/bare-metal/fedora-coreos/kubernetes/bootstrap.tf index 837c489b6..f59a30622 100644 --- a/bare-metal/fedora-coreos/kubernetes/bootstrap.tf +++ b/bare-metal/fedora-coreos/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=3675b3a539efd8d341277f0c03322883f97fd992" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=60540868e070ea95da05fc5a6a5c9a55ce68c58d" cluster_name = var.cluster_name api_servers = [var.k8s_domain_name] diff --git a/bare-metal/fedora-coreos/kubernetes/versions.tf b/bare-metal/fedora-coreos/kubernetes/versions.tf index cf3f8224c..3a493e596 100644 --- a/bare-metal/fedora-coreos/kubernetes/versions.tf +++ b/bare-metal/fedora-coreos/kubernetes/versions.tf @@ -1,11 +1,19 @@ # Terraform version and plugin versions terraform { - required_version = "~> 0.12.6" + required_version = ">= 0.12.26, < 0.14.0" required_providers { - matchbox = "~> 0.3" - ct = "~> 0.4" template = "~> 2.1" null = "~> 2.1" + + ct = { + source = "poseidon/ct" + version = "~> 0.6.1" + } + + matchbox = { + source = "poseidon/matchbox" + version = "~> 0.4.1" + } } } diff --git a/digital-ocean/container-linux/kubernetes/bootstrap.tf b/digital-ocean/container-linux/kubernetes/bootstrap.tf index 0dd4ac823..4405d4a06 100644 --- a/digital-ocean/container-linux/kubernetes/bootstrap.tf +++ b/digital-ocean/container-linux/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=3675b3a539efd8d341277f0c03322883f97fd992" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=60540868e070ea95da05fc5a6a5c9a55ce68c58d" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/digital-ocean/container-linux/kubernetes/versions.tf b/digital-ocean/container-linux/kubernetes/versions.tf index 14ab23b30..807b39006 100644 --- a/digital-ocean/container-linux/kubernetes/versions.tf +++ b/digital-ocean/container-linux/kubernetes/versions.tf @@ -1,12 +1,20 @@ # Terraform version and plugin versions terraform { - required_version = "~> 0.12.6" + required_version = ">= 0.12.26, < 0.14.0" required_providers { - digitalocean = "~> 1.16" - ct = "~> 0.4" - template = "~> 2.1" - null = "~> 2.1" + template = "~> 2.1" + null = "~> 2.1" + + ct = { + source = "poseidon/ct" + version = "~> 0.6.1" + } + + digitalocean = { + source = "digitalocean/digitalocean" + version = "~> 1.20" + } } } diff --git a/digital-ocean/fedora-coreos/kubernetes/bootstrap.tf b/digital-ocean/fedora-coreos/kubernetes/bootstrap.tf index 2d9d825af..bad1275d5 100644 --- a/digital-ocean/fedora-coreos/kubernetes/bootstrap.tf +++ b/digital-ocean/fedora-coreos/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=3675b3a539efd8d341277f0c03322883f97fd992" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=60540868e070ea95da05fc5a6a5c9a55ce68c58d" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/digital-ocean/fedora-coreos/kubernetes/versions.tf b/digital-ocean/fedora-coreos/kubernetes/versions.tf index 14ab23b30..807b39006 100644 --- a/digital-ocean/fedora-coreos/kubernetes/versions.tf +++ b/digital-ocean/fedora-coreos/kubernetes/versions.tf @@ -1,12 +1,20 @@ # Terraform version and plugin versions terraform { - required_version = "~> 0.12.6" + required_version = ">= 0.12.26, < 0.14.0" required_providers { - digitalocean = "~> 1.16" - ct = "~> 0.4" - template = "~> 2.1" - null = "~> 2.1" + template = "~> 2.1" + null = "~> 2.1" + + ct = { + source = "poseidon/ct" + version = "~> 0.6.1" + } + + digitalocean = { + source = "digitalocean/digitalocean" + version = "~> 1.20" + } } } diff --git a/docs/fedora-coreos/aws.md b/docs/fedora-coreos/aws.md index cd38f1961..79595bcf6 100644 --- a/docs/fedora-coreos/aws.md +++ b/docs/fedora-coreos/aws.md @@ -10,23 +10,15 @@ Controller hosts are provisioned to run an `etcd-member` peer and a `kubelet` se * AWS Account and IAM credentials * AWS Route53 DNS Zone (registered Domain Name or delegated subdomain) -* Terraform v0.12.6+ and [terraform-provider-ct](https://github.com/poseidon/terraform-provider-ct) installed locally +* Terraform v0.13.0+ ## Terraform Setup -Install [Terraform](https://www.terraform.io/downloads.html) v0.12.6+ on your system. +Install [Terraform](https://www.terraform.io/downloads.html) v0.13.0+ on your system. ```sh $ terraform version -Terraform v0.12.21 -``` - -Add the [terraform-provider-ct](https://github.com/poseidon/terraform-provider-ct) plugin binary for your system to `~/.terraform.d/plugins/`, noting the final name. - -```sh -wget https://github.com/poseidon/terraform-provider-ct/releases/download/v0.5.0/terraform-provider-ct-v0.5.0-linux-amd64.tar.gz -tar xzf terraform-provider-ct-v0.5.0-linux-amd64.tar.gz -mv terraform-provider-ct-v0.5.0-linux-amd64/terraform-provider-ct ~/.terraform.d/plugins/terraform-provider-ct_v0.5.0 +Terraform v0.13.0 ``` Read [concepts](/architecture/concepts/) to learn about Terraform, modules, and organizing resources. Change to your infrastructure repository (e.g. `infra`). @@ -49,13 +41,23 @@ Configure the AWS provider to use your access key credentials in a `providers.tf ```tf provider "aws" { - version = "3.1.0" region = "eu-central-1" shared_credentials_file = "/home/user/.config/aws/credentials" } -provider "ct" { - version = "0.5.0" +provider "ct" {} + +terraform { + required_providers { + ct = { + source = "poseidon/ct" + version = "0.6.1" + } + aws = { + source = "hashicorp/aws" + version = "3.1.0" + } + } } ``` diff --git a/docs/fedora-coreos/azure.md b/docs/fedora-coreos/azure.md index d5ec2bd5d..eb800ffbb 100644 --- a/docs/fedora-coreos/azure.md +++ b/docs/fedora-coreos/azure.md @@ -10,23 +10,15 @@ Controller hosts are provisioned to run an `etcd-member` peer and a `kubelet` se * Azure account * Azure DNS Zone (registered Domain Name or delegated subdomain) -* Terraform v0.12.6+ and [terraform-provider-ct](https://github.com/poseidon/terraform-provider-ct) installed locally +* Terraform v0.13.0+ ## Terraform Setup -Install [Terraform](https://www.terraform.io/downloads.html) v0.12.6+ on your system. +Install [Terraform](https://www.terraform.io/downloads.html) v0.13.0+ on your system. ```sh $ terraform version -Terraform v0.12.21 -``` - -Add the [terraform-provider-ct](https://github.com/poseidon/terraform-provider-ct) plugin binary for your system to `~/.terraform.d/plugins/`, noting the final name. - -```sh -wget https://github.com/poseidon/terraform-provider-ct/releases/download/v0.5.0/terraform-provider-ct-v0.5.0-linux-amd64.tar.gz -tar xzf terraform-provider-ct-v0.5.0-linux-amd64.tar.gz -mv terraform-provider-ct-v0.5.0-linux-amd64/terraform-provider-ct ~/.terraform.d/plugins/terraform-provider-ct_v0.5.0 +Terraform v0.13.0 ``` Read [concepts](/architecture/concepts/) to learn about Terraform, modules, and organizing resources. Change to your infrastructure repository (e.g. `infra`). @@ -47,11 +39,22 @@ Configure the Azure provider in a `providers.tf` file. ```tf provider "azurerm" { - version = "2.22.0" + features {} } -provider "ct" { - version = "0.5.0" +provider "ct" {} + +terraform { + required_providers { + ct = { + source = "poseidon/ct" + version = "0.6.1" + } + azurerm = { + source = "hashicorp/azurerm" + version = "2.22.0" + } + } } ``` diff --git a/docs/fedora-coreos/bare-metal.md b/docs/fedora-coreos/bare-metal.md index c866e8764..e5d4b1151 100644 --- a/docs/fedora-coreos/bare-metal.md +++ b/docs/fedora-coreos/bare-metal.md @@ -12,7 +12,7 @@ Controller hosts are provisioned to run an `etcd-member` peer and a `kubelet` se * PXE-enabled [network boot](https://coreos.com/matchbox/docs/latest/network-setup.html) environment (with HTTPS support) * Matchbox v0.6+ deployment with API enabled * Matchbox credentials `client.crt`, `client.key`, `ca.crt` -* Terraform v0.12.6+, [terraform-provider-matchbox](https://github.com/poseidon/terraform-provider-matchbox), and [terraform-provider-ct](https://github.com/poseidon/terraform-provider-ct) installed locally +* Terraform v0.13.0+ ## Machines @@ -107,27 +107,11 @@ Read about the [many ways](https://coreos.com/matchbox/docs/latest/network-setup ## Terraform Setup -Install [Terraform](https://www.terraform.io/downloads.html) v0.12.6+ on your system. +Install [Terraform](https://www.terraform.io/downloads.html) v0.13.0+ on your system. ```sh $ terraform version -Terraform v0.12.21 -``` - -Add the [terraform-provider-matchbox](https://github.com/poseidon/terraform-provider-matchbox) plugin binary for your system to `~/.terraform.d/plugins/`, noting the final name. - -```sh -wget https://github.com/poseidon/terraform-provider-matchbox/releases/download/v0.3.0/terraform-provider-matchbox-v0.3.0-linux-amd64.tar.gz -tar xzf terraform-provider-matchbox-v0.3.0-linux-amd64.tar.gz -mv terraform-provider-matchbox-v0.3.0-linux-amd64/terraform-provider-matchbox ~/.terraform.d/plugins/terraform-provider-matchbox_v0.3.0 -``` - -Add the [terraform-provider-ct](https://github.com/poseidon/terraform-provider-ct) plugin binary for your system to `~/.terraform.d/plugins/`, noting the final name. - -```sh -wget https://github.com/poseidon/terraform-provider-ct/releases/download/v0.5.0/terraform-provider-ct-v0.5.0-linux-amd64.tar.gz -tar xzf terraform-provider-ct-v0.5.0-linux-amd64.tar.gz -mv terraform-provider-ct-v0.5.0-linux-amd64/terraform-provider-ct ~/.terraform.d/plugins/terraform-provider-ct_v0.5.0 +Terraform v0.13.0 ``` Read [concepts](/architecture/concepts/) to learn about Terraform, modules, and organizing resources. Change to your infrastructure repository (e.g. `infra`). @@ -142,15 +126,25 @@ Configure the Matchbox provider to use your Matchbox API endpoint and client cer ```tf provider "matchbox" { - version = "0.4.0" endpoint = "matchbox.example.com:8081" client_cert = file("~/.config/matchbox/client.crt") client_key = file("~/.config/matchbox/client.key") ca = file("~/.config/matchbox/ca.crt") } -provider "ct" { - version = "0.5.0" +provider "ct" {} + +terraform { + required_providers { + ct = { + source = "poseidon/ct" + version = "0.6.1" + } + matchbox = { + source = "poseidon/matchbox" + version = "0.4.1" + } + } } ``` diff --git a/docs/fedora-coreos/digitalocean.md b/docs/fedora-coreos/digitalocean.md index e134cce3a..0091f1f65 100644 --- a/docs/fedora-coreos/digitalocean.md +++ b/docs/fedora-coreos/digitalocean.md @@ -10,23 +10,15 @@ Controller hosts are provisioned to run an `etcd-member` peer and a `kubelet` se * Digital Ocean Account and Token * Digital Ocean Domain (registered Domain Name or delegated subdomain) -* Terraform v0.12.6+ and [terraform-provider-ct](https://github.com/poseidon/terraform-provider-ct) installed locally +* Terraform v0.13.0+ ## Terraform Setup -Install [Terraform](https://www.terraform.io/downloads.html) v0.12.6+ on your system. +Install [Terraform](https://www.terraform.io/downloads.html) v0.13.0+ on your system. ```sh $ terraform version -Terraform v0.12.21 -``` - -Add the [terraform-provider-ct](https://github.com/poseidon/terraform-provider-ct) plugin binary for your system to `~/.terraform.d/plugins/`, noting the final name. - -```sh -wget https://github.com/poseidon/terraform-provider-ct/releases/download/v0.5.0/terraform-provider-ct-v0.5.0-linux-amd64.tar.gz -tar xzf terraform-provider-ct-v0.5.0-linux-amd64.tar.gz -mv terraform-provider-ct-v0.5.0-linux-amd64/terraform-provider-ct ~/.terraform.d/plugins/terraform-provider-ct_v0.5.0 +Terraform v0.13.0 ``` Read [concepts](/architecture/concepts/) to learn about Terraform, modules, and organizing resources. Change to your infrastructure repository (e.g. `infra`). @@ -50,12 +42,22 @@ Configure the DigitalOcean provider to use your token in a `providers.tf` file. ```tf provider "digitalocean" { - version = "1.20.0" token = "${chomp(file("~/.config/digital-ocean/token"))}" } -provider "ct" { - version = "0.5.0" +provider "ct" {} + +terraform { + required_providers { + ct = { + source = "poseidon/ct" + version = "0.6.1" + } + digitalocean = { + source = "digitalocean/digitalocean" + version = "1.22.1" + } + } } ``` diff --git a/docs/fedora-coreos/google-cloud.md b/docs/fedora-coreos/google-cloud.md index 25d648d0d..22636246d 100644 --- a/docs/fedora-coreos/google-cloud.md +++ b/docs/fedora-coreos/google-cloud.md @@ -10,23 +10,15 @@ Controller hosts are provisioned to run an `etcd-member` peer and a `kubelet` se * Google Cloud Account and Service Account * Google Cloud DNS Zone (registered Domain Name or delegated subdomain) -* Terraform v0.12.6+ and [terraform-provider-ct](https://github.com/poseidon/terraform-provider-ct) installed locally +* Terraform v0.13.0+ ## Terraform Setup -Install [Terraform](https://www.terraform.io/downloads.html) v0.12.6+ on your system. +Install [Terraform](https://www.terraform.io/downloads.html) v0.13.0+ on your system. ```sh $ terraform version -Terraform v0.12.21 -``` - -Add the [terraform-provider-ct](https://github.com/poseidon/terraform-provider-ct) plugin binary for your system to `~/.terraform.d/plugins/`, noting the final name. - -```sh -wget https://github.com/poseidon/terraform-provider-ct/releases/download/v0.5.0/terraform-provider-ct-v0.5.0-linux-amd64.tar.gz -tar xzf terraform-provider-ct-v0.5.0-linux-amd64.tar.gz -mv terraform-provider-ct-v0.5.0-linux-amd64/terraform-provider-ct ~/.terraform.d/plugins/terraform-provider-ct_v0.5.0 +Terraform v0.13.0 ``` Read [concepts](/architecture/concepts/) to learn about Terraform, modules, and organizing resources. Change to your infrastructure repository (e.g. `infra`). @@ -49,14 +41,24 @@ Configure the Google Cloud provider to use your service account key, project-id, ```tf provider "google" { - version = "3.33.0" project = "project-id" region = "us-central1" credentials = file("~/.config/google-cloud/terraform.json") } -provider "ct" { - version = "0.5.0" +provider "ct" {} + +terraform { + required_providers { + ct = { + source = "poseidon/ct" + version = "0.6.1" + } + google = { + source = "hashicorp/google" + version = "3.33.0" + } + } } ``` diff --git a/docs/flatcar-linux/aws.md b/docs/flatcar-linux/aws.md index d9e363f82..712d49479 100644 --- a/docs/flatcar-linux/aws.md +++ b/docs/flatcar-linux/aws.md @@ -10,23 +10,15 @@ Controller hosts are provisioned to run an `etcd-member` peer and a `kubelet` se * AWS Account and IAM credentials * AWS Route53 DNS Zone (registered Domain Name or delegated subdomain) -* Terraform v0.12.6+ and [terraform-provider-ct](https://github.com/poseidon/terraform-provider-ct) installed locally +* Terraform v0.13.0+ ## Terraform Setup -Install [Terraform](https://www.terraform.io/downloads.html) v0.12.6+ on your system. +Install [Terraform](https://www.terraform.io/downloads.html) v0.13.0+ on your system. ```sh $ terraform version -Terraform v0.12.21 -``` - -Add the [terraform-provider-ct](https://github.com/poseidon/terraform-provider-ct) plugin binary for your system to `~/.terraform.d/plugins/`, noting the final name. - -```sh -wget https://github.com/poseidon/terraform-provider-ct/releases/download/v0.5.0/terraform-provider-ct-v0.5.0-linux-amd64.tar.gz -tar xzf terraform-provider-ct-v0.5.0-linux-amd64.tar.gz -mv terraform-provider-ct-v0.5.0-linux-amd64/terraform-provider-ct ~/.terraform.d/plugins/terraform-provider-ct_v0.5.0 +Terraform v0.13.0 ``` Read [concepts](/architecture/concepts/) to learn about Terraform, modules, and organizing resources. Change to your infrastructure repository (e.g. `infra`). @@ -49,13 +41,23 @@ Configure the AWS provider to use your access key credentials in a `providers.tf ```tf provider "aws" { - version = "3.1.0" region = "eu-central-1" shared_credentials_file = "/home/user/.config/aws/credentials" } -provider "ct" { - version = "0.5.0" +provider "ct" {} + +terraform { + required_providers { + ct = { + source = "poseidon/ct" + version = "0.6.1" + } + aws = { + source = "hashicorp/aws" + version = "3.1.0" + } + } } ``` diff --git a/docs/flatcar-linux/azure.md b/docs/flatcar-linux/azure.md index d89710c0a..027feb719 100644 --- a/docs/flatcar-linux/azure.md +++ b/docs/flatcar-linux/azure.md @@ -10,23 +10,15 @@ Controller hosts are provisioned to run an `etcd-member` peer and a `kubelet` se * Azure account * Azure DNS Zone (registered Domain Name or delegated subdomain) -* Terraform v0.12.6+ and [terraform-provider-ct](https://github.com/poseidon/terraform-provider-ct) installed locally +* Terraform v0.13.0+ ## Terraform Setup -Install [Terraform](https://www.terraform.io/downloads.html) v0.12.6+ on your system. +Install [Terraform](https://www.terraform.io/downloads.html) v0.13.0+ on your system. ```sh $ terraform version -Terraform v0.12.21 -``` - -Add the [terraform-provider-ct](https://github.com/poseidon/terraform-provider-ct) plugin binary for your system to `~/.terraform.d/plugins/`, noting the final name. - -```sh -wget https://github.com/poseidon/terraform-provider-ct/releases/download/v0.5.0/terraform-provider-ct-v0.5.0-linux-amd64.tar.gz -tar xzf terraform-provider-ct-v0.5.0-linux-amd64.tar.gz -mv terraform-provider-ct-v0.5.0-linux-amd64/terraform-provider-ct ~/.terraform.d/plugins/terraform-provider-ct_v0.5.0 +Terraform v0.13.0 ``` Read [concepts](/architecture/concepts/) to learn about Terraform, modules, and organizing resources. Change to your infrastructure repository (e.g. `infra`). @@ -47,11 +39,22 @@ Configure the Azure provider in a `providers.tf` file. ```tf provider "azurerm" { - version = "2.22.0" + features {} } -provider "ct" { - version = "0.5.0" +provider "ct" {} + +terraform { + required_providers { + ct = { + source = "poseidon/ct" + version = "0.6.1" + } + azurerm = { + source = "hashicorp/azurerm" + version = "2.22.0" + } + } } ``` diff --git a/docs/flatcar-linux/bare-metal.md b/docs/flatcar-linux/bare-metal.md index c71460295..f387f6056 100644 --- a/docs/flatcar-linux/bare-metal.md +++ b/docs/flatcar-linux/bare-metal.md @@ -12,7 +12,7 @@ Controller hosts are provisioned to run an `etcd-member` peer and a `kubelet` se * PXE-enabled [network boot](https://coreos.com/matchbox/docs/latest/network-setup.html) environment (with HTTPS support) * Matchbox v0.6+ deployment with API enabled * Matchbox credentials `client.crt`, `client.key`, `ca.crt` -* Terraform v0.12.6+, [terraform-provider-matchbox](https://github.com/poseidon/terraform-provider-matchbox), and [terraform-provider-ct](https://github.com/poseidon/terraform-provider-ct) installed locally +* Terraform v0.13.0+ ## Machines @@ -107,27 +107,11 @@ Read about the [many ways](https://coreos.com/matchbox/docs/latest/network-setup ## Terraform Setup -Install [Terraform](https://www.terraform.io/downloads.html) v0.12.6+ on your system. +Install [Terraform](https://www.terraform.io/downloads.html) v0.13.0+ on your system. ```sh $ terraform version -Terraform v0.12.21 -``` - -Add the [terraform-provider-matchbox](https://github.com/poseidon/terraform-provider-matchbox) plugin binary for your system to `~/.terraform.d/plugins/`, noting the final name. - -```sh -wget https://github.com/poseidon/terraform-provider-matchbox/releases/download/v0.3.0/terraform-provider-matchbox-v0.3.0-linux-amd64.tar.gz -tar xzf terraform-provider-matchbox-v0.3.0-linux-amd64.tar.gz -mv terraform-provider-matchbox-v0.3.0-linux-amd64/terraform-provider-matchbox ~/.terraform.d/plugins/terraform-provider-matchbox_v0.3.0 -``` - -Add the [terraform-provider-ct](https://github.com/poseidon/terraform-provider-ct) plugin binary for your system to `~/.terraform.d/plugins/`, noting the final name. - -```sh -wget https://github.com/poseidon/terraform-provider-ct/releases/download/v0.5.0/terraform-provider-ct-v0.5.0-linux-amd64.tar.gz -tar xzf terraform-provider-ct-v0.5.0-linux-amd64.tar.gz -mv terraform-provider-ct-v0.5.0-linux-amd64/terraform-provider-ct ~/.terraform.d/plugins/terraform-provider-ct_v0.5.0 +Terraform v0.13.0 ``` Read [concepts](/architecture/concepts/) to learn about Terraform, modules, and organizing resources. Change to your infrastructure repository (e.g. `infra`). @@ -142,15 +126,25 @@ Configure the Matchbox provider to use your Matchbox API endpoint and client cer ```tf provider "matchbox" { - version = "0.4.0" endpoint = "matchbox.example.com:8081" client_cert = file("~/.config/matchbox/client.crt") client_key = file("~/.config/matchbox/client.key") ca = file("~/.config/matchbox/ca.crt") } -provider "ct" { - version = "0.5.0" +provider "ct" {} + +terraform { + required_providers { + ct = { + source = "poseidon/ct" + version = "0.6.1" + } + matchbox = { + source = "poseidon/matchbox" + version = "0.4.1" + } + } } ``` diff --git a/docs/flatcar-linux/digitalocean.md b/docs/flatcar-linux/digitalocean.md index 52dc04240..1e0ca422c 100644 --- a/docs/flatcar-linux/digitalocean.md +++ b/docs/flatcar-linux/digitalocean.md @@ -10,23 +10,15 @@ Controller hosts are provisioned to run an `etcd-member` peer and a `kubelet` se * Digital Ocean Account and Token * Digital Ocean Domain (registered Domain Name or delegated subdomain) -* Terraform v0.12.6+ and [terraform-provider-ct](https://github.com/poseidon/terraform-provider-ct) installed locally +* Terraform v0.13.0+ ## Terraform Setup -Install [Terraform](https://www.terraform.io/downloads.html) v0.12.6+ on your system. +Install [Terraform](https://www.terraform.io/downloads.html) v0.13.0+ on your system. ```sh $ terraform version -Terraform v0.12.21 -``` - -Add the [terraform-provider-ct](https://github.com/poseidon/terraform-provider-ct) plugin binary for your system to `~/.terraform.d/plugins/`, noting the final name. - -```sh -wget https://github.com/poseidon/terraform-provider-ct/releases/download/v0.5.0/terraform-provider-ct-v0.5.0-linux-amd64.tar.gz -tar xzf terraform-provider-ct-v0.5.0-linux-amd64.tar.gz -mv terraform-provider-ct-v0.5.0-linux-amd64/terraform-provider-ct ~/.terraform.d/plugins/terraform-provider-ct_v0.5.0 +Terraform v0.13.0 ``` Read [concepts](/architecture/concepts/) to learn about Terraform, modules, and organizing resources. Change to your infrastructure repository (e.g. `infra`). @@ -50,12 +42,22 @@ Configure the DigitalOcean provider to use your token in a `providers.tf` file. ```tf provider "digitalocean" { - version = "1.20.0" token = "${chomp(file("~/.config/digital-ocean/token"))}" } -provider "ct" { - version = "0.5.0" +provider "ct" {} + +terraform { + required_providers { + ct = { + source = "poseidon/ct" + version = "0.6.1" + } + digitalocean = { + source = "digitalocean/digitalocean" + version = "1.22.1" + } + } } ``` diff --git a/docs/flatcar-linux/google-cloud.md b/docs/flatcar-linux/google-cloud.md index 42cd6a19b..51c1daf02 100644 --- a/docs/flatcar-linux/google-cloud.md +++ b/docs/flatcar-linux/google-cloud.md @@ -10,23 +10,15 @@ Controller hosts are provisioned to run an `etcd-member` peer and a `kubelet` se * Google Cloud Account and Service Account * Google Cloud DNS Zone (registered Domain Name or delegated subdomain) -* Terraform v0.12.6+ and [terraform-provider-ct](https://github.com/poseidon/terraform-provider-ct) installed locally +* Terraform v0.13.0+ ## Terraform Setup -Install [Terraform](https://www.terraform.io/downloads.html) v0.12.6+ on your system. +Install [Terraform](https://www.terraform.io/downloads.html) v0.13.0+ on your system. ```sh $ terraform version -Terraform v0.12.21 -``` - -Add the [terraform-provider-ct](https://github.com/poseidon/terraform-provider-ct) plugin binary for your system to `~/.terraform.d/plugins/`, noting the final name. - -```sh -wget https://github.com/poseidon/terraform-provider-ct/releases/download/v0.5.0/terraform-provider-ct-v0.5.0-linux-amd64.tar.gz -tar xzf terraform-provider-ct-v0.5.0-linux-amd64.tar.gz -mv terraform-provider-ct-v0.5.0-linux-amd64/terraform-provider-ct ~/.terraform.d/plugins/terraform-provider-ct_v0.5.0 +Terraform v0.13.0 ``` Read [concepts](/architecture/concepts/) to learn about Terraform, modules, and organizing resources. Change to your infrastructure repository (e.g. `infra`). @@ -49,14 +41,24 @@ Configure the Google Cloud provider to use your service account key, project-id, ```tf provider "google" { - version = "3.33.0" project = "project-id" region = "us-central1" credentials = file("~/.config/google-cloud/terraform.json") } -provider "ct" { - version = "0.5.0" +provider "ct" {} + +terraform { + required_providers { + ct = { + source = "poseidon/ct" + version = "0.6.1" + } + google = { + source = "hashicorp/google" + version = "3.33.0" + } + } } ``` diff --git a/docs/topics/maintenance.md b/docs/topics/maintenance.md index 65b650a13..945023060 100644 --- a/docs/topics/maintenance.md +++ b/docs/topics/maintenance.md @@ -134,9 +134,9 @@ The [terraform-provider-ct](https://github.com/poseidon/terraform-provider-ct) p Add the [terraform-provider-ct](https://github.com/poseidon/terraform-provider-ct) plugin binary for your system to `~/.terraform.d/plugins/`, noting the final name. ```sh -wget https://github.com/poseidon/terraform-provider-ct/releases/download/v0.5.0/terraform-provider-ct-v0.5.0-linux-amd64.tar.gz -tar xzf terraform-provider-ct-v0.5.0-linux-amd64.tar.gz -mv terraform-provider-ct-v0.5.0-linux-amd64/terraform-provider-ct ~/.terraform.d/plugins/terraform-provider-ct_v0.5.0 +wget https://github.com/poseidon/terraform-provider-ct/releases/download/v0.5.0/terraform-provider-ct-v0.6.1-linux-amd64.tar.gz +tar xzf terraform-provider-ct-v0.6.1-linux-amd64.tar.gz +mv terraform-provider-ct-v0.6.1-linux-amd64/terraform-provider-ct ~/.terraform.d/plugins/terraform-provider-ct_v0.6.1 ``` Binary names are versioned. This enables the ability to upgrade different plugins and have clusters pin different versions. @@ -147,8 +147,8 @@ $ tree ~/.terraform.d/ └── plugins ├── terraform-provider-ct_v0.2.1 ├── terraform-provider-ct_v0.3.0 - ├── terraform-provider-ct_v0.5.0 - └── terraform-provider-matchbox_v0.3.0 + ├── terraform-provider-ct_v0.6.1 + └── terraform-provider-matchbox_v0.4.1 ``` @@ -157,7 +157,7 @@ Update the version of the `ct` plugin in each Terraform working directory. Typho ``` # providers.tf provider "ct" { - version = "0.5.0" + version = "0.6.1" } ``` @@ -193,7 +193,7 @@ terraform apply # add kubeconfig to new workers terraform state list | grep null_resource -terraform taint -module digital-ocean-nemo null_resource.copy-worker-secrets[N] +terraform taint module.nemo.null_resource.copy-worker-secrets[N] terraform apply ``` @@ -203,17 +203,92 @@ Expect downtime. Google Cloud creates a new worker template and edits the worker instance group instantly. Manually terminate workers and replacement workers will use the user-data. -## Terraform v0.12.x +## Terraform Versions -Terraform [v0.12](https://www.hashicorp.com/blog/announcing-terraform-0-12) introduced major changes to the provider plugin protocol and HCL language (first-class expressions, formal list and map types, nullable variables, variable constraints, and short-circuiting ternary operators). +Terraform [v0.13](https://www.hashicorp.com/blog/announcing-hashicorp-terraform-0-13) introduced major changes to the provider plugin system. Terraform `init` can automatically install both `hashicorp` and `poseidon` provider plugins, eliminating the need to manually install plugin binaries. -Typhoon modules have been adapted for Terraform v0.12. Provider plugins requirements now enforce v0.12 compatibility. However, some HCL language changes were breaking (list [type hint](https://www.terraform.io/upgrade-guides/0-12.html#referring-to-list-variables) workarounds in v0.11 now have new meaning). Typhoon cannot offer both v0.11 and v0.12 compatibility in the same release. Upcoming releases require upgrading Terraform to v0.12. +Typhoon modules have been updated for v0.13.x, but retain compatibility with v0.12.26+ to ease migration. Poseidon publishes [providers](/topics/security/#terraform-providers) to the Terraform Provider Registry for usage with v0.13+. | Typhoon Release | Terraform version | |-------------------|---------------------| +| ? | v0.13.x | +| ? | v0.12.26+, v0.13.x | | v1.15.0 - ? | v0.12.x | | v1.10.3 - v1.15.0 | v0.11.x | | v1.9.2 - v1.10.2 | v0.10.4+ or v0.11.x | | v1.7.3 - v1.9.1 | v0.10.x | | v1.6.4 - v1.7.2 | v0.9.x | +### New Workspace + +With a new Terraform workspace, use Terraform v0.13.x and the updated Typhoon [tutorials](/fedora-coreos/aws/#provider). + +### Existing Workspace + +An existing Terraform workspace may already manage earlier Typhoon clusters created with Terraform v0.12.x. + +First, upgrade `terraform-provider-ct` to v0.6.1 following the [guide](#upgrade-terraform-provider-ct) above. As usual, read about how `apply` affects existing cluster nodes when `ct` is upgraded. But `terraform-provider-ct` v0.6.1 is compatible with both Terraform v0.12 and v0.13, so do this first. + +``` +provider "ct" { + version = "0.6.1" +} +``` + +Next, create Typhoon clusters using the `ref` that introduced Terraform v0.13 forward compatibility (SHA) or later. You will see a compatibility warning. Use blue/green cluster replacement to shift to these new clusters, then eliminate older clusters. + +``` +module "nemo" { + source = "git::https://github.com/poseidon/typhoon//digital-ocean/fedora-coreos/kubernetes?ref=SHA" + ... +} +``` + +Install Terraform v0.13. Once all clusters in a workspace are on SHA or above, you are ready to start using Terraform v0.13. + +``` +terraform version +v0.13.0 +``` + +Update `providers.tf` to match the Typhoon [tutorials](/fedora-coreos/aws/#provider) and use new `required_providers` block. + +``` +terraform init +terraform 0.13upgrade # sometimes helpful +``` + +!!! note + You will see `Could not retrieve the list of available versions for provider -/ct: provider` + +In state files, existing clusters use Terraform v0.12 providers (e.g. `-/aws`). Pivot to Terraform v0.13 providers (e.g. `hashicorp/aws`) with the following commands, as applicable. Repeat until `terraform init` no longer shows old-style providers. + +``` +terraform state replace-provider -- -/aws hashicorp/aws +terraform state replace-provider -- -/azurerm hashicorp/azurerm +terraform state replace-provider -- -/google hashicorp/google + +terraform state replace-provider -- -/digitalocean digitalocean/digitalocean +terraform state replace-provider -- -/ct poseidon/ct +terraform state replace-provider -- -/matchbox poseidon/matchbox + +terraform state replace-provider -- -/local hashicorp/local +terraform state replace-provider -- -/null hashicorp/null +terraform state replace-provider -- -/random hashicorp/random +terraform state replace-provider -- -/template hashicorp/template +terraform state replace-provider -- -/tls hashicorp/tls +``` + +Finally, verify Terraform v0.13 plan shows no diff. + +``` +terraform plan +No changes. Infrastructure is up-to-date. +``` + +### v0.12.x + +Terraform [v0.12](https://www.hashicorp.com/blog/announcing-terraform-0-12) introduced major changes to the provider plugin protocol and HCL language (first-class expressions, formal list and map types, nullable variables, variable constraints, and short-circuiting ternary operators). + +Typhoon modules have been adapted for Terraform v0.12. Provider plugins requirements now enforce v0.12 compatibility. However, some HCL language changes were breaking (list [type hint](https://www.terraform.io/upgrade-guides/0-12.html#referring-to-list-variables) workarounds in v0.11 now have new meaning). Typhoon cannot offer both v0.11 and v0.12 compatibility in the same release. Upcoming releases require upgrading Terraform to v0.12. + diff --git a/docs/topics/security.md b/docs/topics/security.md index 77a64748d..2c1400030 100644 --- a/docs/topics/security.md +++ b/docs/topics/security.md @@ -72,6 +72,15 @@ Typhoon packages the [flannel-cni](https://github.com/poseidon/flannel-cni) cont * [quay.io/poseidon/flannel-cni](https://quay.io/repository/poseidon/flannel-cni) (official) +## Terraform Providers + +Typhoon publishes Terraform providers to the Terraform Registry, GPG signed by 0x8F515AD1602065C8. + +| Name | Source | Registry | +|----------|--------|----------| +| ct | [github](https://github.com/poseidon/terraform-provider-ct) | [poseidon/ct](https://registry.terraform.io/providers/poseidon/ct/latest) | +| matchbox | [github](https://github.com/poseidon/terraform-provider-matchbox) | [poseidon/matchbox](https://registry.terraform.io/providers/poseidon/matchbox/latest) | + ## Disclosures If you find security issues, please email `security@psdn.io`. If the issue lies in upstream Kubernetes, please inform upstream Kubernetes as well. diff --git a/google-cloud/container-linux/kubernetes/bootstrap.tf b/google-cloud/container-linux/kubernetes/bootstrap.tf index 62670ff2e..7db6057c2 100644 --- a/google-cloud/container-linux/kubernetes/bootstrap.tf +++ b/google-cloud/container-linux/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=3675b3a539efd8d341277f0c03322883f97fd992" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=60540868e070ea95da05fc5a6a5c9a55ce68c58d" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/google-cloud/container-linux/kubernetes/versions.tf b/google-cloud/container-linux/kubernetes/versions.tf index 7210a4b33..178e248e3 100644 --- a/google-cloud/container-linux/kubernetes/versions.tf +++ b/google-cloud/container-linux/kubernetes/versions.tf @@ -1,11 +1,15 @@ # Terraform version and plugin versions terraform { - required_version = "~> 0.12.6" + required_version = ">= 0.12.26, < 0.14.0" required_providers { google = ">= 2.19, < 4.0" - ct = "~> 0.4" template = "~> 2.1" null = "~> 2.1" + + ct = { + source = "poseidon/ct" + version = "~> 0.6.1" + } } } diff --git a/google-cloud/container-linux/kubernetes/workers/versions.tf b/google-cloud/container-linux/kubernetes/workers/versions.tf index ac97c6ac8..c0b899ee0 100644 --- a/google-cloud/container-linux/kubernetes/workers/versions.tf +++ b/google-cloud/container-linux/kubernetes/workers/versions.tf @@ -1,4 +1,14 @@ +# Terraform version and plugin versions terraform { - required_version = ">= 0.12" + required_version = ">= 0.12.26, < 0.14.0" + required_providers { + google = ">= 2.19, < 4.0" + template = "~> 2.1" + + ct = { + source = "poseidon/ct" + version = "~> 0.6.1" + } + } } diff --git a/google-cloud/fedora-coreos/kubernetes/bootstrap.tf b/google-cloud/fedora-coreos/kubernetes/bootstrap.tf index cc86ce0d0..8ed4d7c70 100644 --- a/google-cloud/fedora-coreos/kubernetes/bootstrap.tf +++ b/google-cloud/fedora-coreos/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=3675b3a539efd8d341277f0c03322883f97fd992" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=60540868e070ea95da05fc5a6a5c9a55ce68c58d" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/google-cloud/fedora-coreos/kubernetes/versions.tf b/google-cloud/fedora-coreos/kubernetes/versions.tf index 7210a4b33..178e248e3 100644 --- a/google-cloud/fedora-coreos/kubernetes/versions.tf +++ b/google-cloud/fedora-coreos/kubernetes/versions.tf @@ -1,11 +1,15 @@ # Terraform version and plugin versions terraform { - required_version = "~> 0.12.6" + required_version = ">= 0.12.26, < 0.14.0" required_providers { google = ">= 2.19, < 4.0" - ct = "~> 0.4" template = "~> 2.1" null = "~> 2.1" + + ct = { + source = "poseidon/ct" + version = "~> 0.6.1" + } } } diff --git a/google-cloud/fedora-coreos/kubernetes/workers/versions.tf b/google-cloud/fedora-coreos/kubernetes/workers/versions.tf index ac97c6ac8..c0b899ee0 100644 --- a/google-cloud/fedora-coreos/kubernetes/workers/versions.tf +++ b/google-cloud/fedora-coreos/kubernetes/workers/versions.tf @@ -1,4 +1,14 @@ +# Terraform version and plugin versions terraform { - required_version = ">= 0.12" + required_version = ">= 0.12.26, < 0.14.0" + required_providers { + google = ">= 2.19, < 4.0" + template = "~> 2.1" + + ct = { + source = "poseidon/ct" + version = "~> 0.6.1" + } + } } From 9b7e268646d42f01ed48715f133bd651a48b6154 Mon Sep 17 00:00:00 2001 From: Bo Huang Date: Wed, 12 Aug 2020 14:41:50 -0700 Subject: [PATCH 513/523] apiserver nlb should be internal --- aws/fedora-coreos/kubernetes/nlb.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aws/fedora-coreos/kubernetes/nlb.tf b/aws/fedora-coreos/kubernetes/nlb.tf index 2e8b2cbab..57f91823c 100644 --- a/aws/fedora-coreos/kubernetes/nlb.tf +++ b/aws/fedora-coreos/kubernetes/nlb.tf @@ -17,7 +17,7 @@ resource "aws_route53_record" "apiserver" { resource "aws_lb" "nlb" { name = "${var.cluster_name}-nlb" load_balancer_type = "network" - internal = false + internal = true subnets = aws_subnet.private.*.id From 0dea2b39e62923e07d84ab56939960e1d823fcd7 Mon Sep 17 00:00:00 2001 From: Bo Huang Date: Wed, 12 Aug 2020 19:02:15 -0700 Subject: [PATCH 514/523] update terraform-render-bootstrap with latest upstream --- aws/fedora-coreos/kubernetes/bootstrap.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aws/fedora-coreos/kubernetes/bootstrap.tf b/aws/fedora-coreos/kubernetes/bootstrap.tf index a206e8c7e..ff8bcf0eb 100644 --- a/aws/fedora-coreos/kubernetes/bootstrap.tf +++ b/aws/fedora-coreos/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/takescoop/terraform-render-bootstrap.git?ref=a7ad7d74586fd698be22b75b4cc314c2d3dc4bcd" + source = "git::https://github.com/takescoop/terraform-render-bootstrap.git?ref=692be6bfbea688fc28a6de20178fccb6491f6ebf" cluster_name = var.cluster_name api_servers = concat(list(format("%s.%s", var.cluster_name, var.dns_zone)), var.apiserver_aliases) From 342380cfa45bc55c8b26a004d464b7d0c859d88e Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Thu, 13 Aug 2020 00:29:25 -0700 Subject: [PATCH 515/523] Update Terraform migration guide SHA * Mention the first master branch SHA that introduced Terraform v0.13 forward compatibility * Link the migration guide on Github until a release is available and website docs are published --- CHANGES.md | 10 ++++------ docs/topics/maintenance.md | 11 +++++------ 2 files changed, 9 insertions(+), 12 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index e76bae1bb..81218c244 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -4,11 +4,9 @@ Notable changes between versions. ## Latest -* Migrate from Terraform v0.12.x to v0.13.x (action required) - * Recommend Terraform v0.13.x ([migration guide](https://github.com/poseidon/typhoon/blob/260033e978f03d068b79e553e7c428a64b22a475/docs/topics/maintenance.md#terraform-versions)) - * Support automatic install of poseidon's provider plugins - * [poseidon/ct](https://registry.terraform.io/providers/poseidon/ct/latest) - * [poseidon/matchbox](https://registry.terraform.io/providers/poseidon/matchbox/latest) +* Migrate from Terraform v0.12.x to v0.13.x ([#804](https://github.com/poseidon/typhoon/pull/804)) (**action required**) + * Recommend Terraform v0.13.x ([migration guide](https://github.com/poseidon/typhoon/blob/5e70d7e2c8c71c37d9a0896a0945188f009011f9/docs/topics/maintenance.md#terraform-versions)) + * Support automatic install of poseidon's provider plugins ([poseidon/ct](https://registry.terraform.io/providers/poseidon/ct/latest), [poseidon/matchbox](https://registry.terraform.io/providers/poseidon/matchbox/latest)) * Require Terraform v0.12.26+ (migration compatibility) * Require `terraform-provider-ct` v0.6.1 * Require `terraform-provider-matchbox` v0.4.1 @@ -16,7 +14,7 @@ Notable changes between versions. * Update CoreDNS from v1.6.7 to [v1.7.0](https://coredns.io/2020/06/15/coredns-1.7.0-release/) * Update Cilium from v1.8.1 to [v1.8.2](https://github.com/cilium/cilium/releases/tag/v1.8.2) * Update [coreos/flannel-cni](https://github.com/coreos/flannel-cni) to [poseidon/flannel-cni](https://github.com/poseidon/flannel-cni) ([#798](https://github.com/poseidon/typhoon/pull/798)) - * Update CNI plugins and fix CVEs with Flannel (non-default) + * Update CNI plugins and fix CVEs with Flannel CNI (non-default) * Transition to a poseidon maintained container image ### AWS diff --git a/docs/topics/maintenance.md b/docs/topics/maintenance.md index 945023060..d165a76d8 100644 --- a/docs/topics/maintenance.md +++ b/docs/topics/maintenance.md @@ -211,9 +211,8 @@ Typhoon modules have been updated for v0.13.x, but retain compatibility with v0. | Typhoon Release | Terraform version | |-------------------|---------------------| -| ? | v0.13.x | -| ? | v0.12.26+, v0.13.x | -| v1.15.0 - ? | v0.12.x | +| v1.18.7 - ? | v0.12.26+, v0.13.x | +| v1.15.0 - v1.18.6 | v0.12.x | | v1.10.3 - v1.15.0 | v0.11.x | | v1.9.2 - v1.10.2 | v0.10.4+ or v0.11.x | | v1.7.3 - v1.9.1 | v0.10.x | @@ -235,16 +234,16 @@ provider "ct" { } ``` -Next, create Typhoon clusters using the `ref` that introduced Terraform v0.13 forward compatibility (SHA) or later. You will see a compatibility warning. Use blue/green cluster replacement to shift to these new clusters, then eliminate older clusters. +Next, create Typhoon clusters using the `ref` that introduced Terraform v0.13 forward compatibility (`5e70d7e2c8c71c37d9a0896a0945188f009011f9`) or later. You will see a compatibility warning. Use blue/green cluster replacement to shift to these new clusters, then eliminate older clusters. ``` module "nemo" { - source = "git::https://github.com/poseidon/typhoon//digital-ocean/fedora-coreos/kubernetes?ref=SHA" + source = "git::https://github.com/poseidon/typhoon//digital-ocean/fedora-coreos/kubernetes?ref=5e70d7e2c8c71c37d9a0896a0945188f009011f9" ... } ``` -Install Terraform v0.13. Once all clusters in a workspace are on SHA or above, you are ready to start using Terraform v0.13. +Install Terraform v0.13. Once all clusters in a workspace are on `5e70d7e2c8c71c37d9a0896a0945188f009011f9` or above, you are ready to start using Terraform v0.13. ``` terraform version From c87db3ef379fcb0d435ca3b55f95f797193815ed Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Thu, 13 Aug 2020 20:46:42 -0700 Subject: [PATCH 516/523] Update Kubernetes from v1.18.6 to v1.18.8 * https://github.com/kubernetes/kubernetes/blob/master/CHANGELOG/CHANGELOG-1.18.md#v1188 --- CHANGES.md | 3 +++ README.md | 10 +++++----- aws/container-linux/kubernetes/README.md | 2 +- aws/container-linux/kubernetes/bootstrap.tf | 2 +- aws/container-linux/kubernetes/cl/controller.yaml | 4 ++-- .../kubernetes/workers/cl/worker.yaml | 4 ++-- aws/fedora-coreos/kubernetes/README.md | 2 +- aws/fedora-coreos/kubernetes/bootstrap.tf | 2 +- aws/fedora-coreos/kubernetes/fcc/controller.yaml | 4 ++-- .../kubernetes/workers/fcc/worker.yaml | 4 ++-- azure/container-linux/kubernetes/README.md | 2 +- azure/container-linux/kubernetes/bootstrap.tf | 2 +- .../container-linux/kubernetes/cl/controller.yaml | 4 ++-- .../kubernetes/workers/cl/worker.yaml | 4 ++-- azure/fedora-coreos/kubernetes/README.md | 2 +- azure/fedora-coreos/kubernetes/bootstrap.tf | 2 +- azure/fedora-coreos/kubernetes/fcc/controller.yaml | 4 ++-- .../kubernetes/workers/fcc/worker.yaml | 4 ++-- bare-metal/container-linux/kubernetes/README.md | 2 +- bare-metal/container-linux/kubernetes/bootstrap.tf | 2 +- .../container-linux/kubernetes/cl/controller.yaml | 4 ++-- .../container-linux/kubernetes/cl/worker.yaml | 2 +- bare-metal/fedora-coreos/kubernetes/README.md | 2 +- bare-metal/fedora-coreos/kubernetes/bootstrap.tf | 2 +- .../fedora-coreos/kubernetes/fcc/controller.yaml | 4 ++-- .../fedora-coreos/kubernetes/fcc/worker.yaml | 2 +- digital-ocean/container-linux/kubernetes/README.md | 2 +- .../container-linux/kubernetes/bootstrap.tf | 2 +- .../container-linux/kubernetes/cl/controller.yaml | 4 ++-- .../container-linux/kubernetes/cl/worker.yaml | 4 ++-- digital-ocean/fedora-coreos/kubernetes/README.md | 2 +- .../fedora-coreos/kubernetes/bootstrap.tf | 2 +- .../fedora-coreos/kubernetes/fcc/controller.yaml | 4 ++-- .../fedora-coreos/kubernetes/fcc/worker.yaml | 4 ++-- docs/advanced/worker-pools.md | 14 +++++++------- docs/fedora-coreos/aws.md | 10 +++++----- docs/fedora-coreos/azure.md | 10 +++++----- docs/fedora-coreos/bare-metal.md | 10 +++++----- docs/fedora-coreos/digitalocean.md | 10 +++++----- docs/fedora-coreos/google-cloud.md | 8 ++++---- docs/flatcar-linux/aws.md | 10 +++++----- docs/flatcar-linux/azure.md | 10 +++++----- docs/flatcar-linux/bare-metal.md | 10 +++++----- docs/flatcar-linux/digitalocean.md | 10 +++++----- docs/flatcar-linux/google-cloud.md | 10 +++++----- docs/index.md | 10 +++++----- docs/topics/maintenance.md | 12 ++++++------ google-cloud/container-linux/kubernetes/README.md | 2 +- .../container-linux/kubernetes/bootstrap.tf | 2 +- .../container-linux/kubernetes/cl/controller.yaml | 4 ++-- .../kubernetes/workers/cl/worker.yaml | 4 ++-- google-cloud/fedora-coreos/kubernetes/README.md | 2 +- google-cloud/fedora-coreos/kubernetes/bootstrap.tf | 2 +- .../fedora-coreos/kubernetes/fcc/controller.yaml | 4 ++-- .../kubernetes/workers/fcc/worker.yaml | 4 ++-- 55 files changed, 133 insertions(+), 130 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 81218c244..673546bca 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -4,6 +4,9 @@ Notable changes between versions. ## Latest +### v1.18.8 + +* Kubernetes [v1.18.8](https://github.com/kubernetes/kubernetes/blob/master/CHANGELOG/CHANGELOG-1.18.md#v1188) * Migrate from Terraform v0.12.x to v0.13.x ([#804](https://github.com/poseidon/typhoon/pull/804)) (**action required**) * Recommend Terraform v0.13.x ([migration guide](https://github.com/poseidon/typhoon/blob/5e70d7e2c8c71c37d9a0896a0945188f009011f9/docs/topics/maintenance.md#terraform-versions)) * Support automatic install of poseidon's provider plugins ([poseidon/ct](https://registry.terraform.io/providers/poseidon/ct/latest), [poseidon/matchbox](https://registry.terraform.io/providers/poseidon/matchbox/latest)) diff --git a/README.md b/README.md index 0f9fba272..60e831f3c 100644 --- a/README.md +++ b/README.md @@ -11,7 +11,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features -* Kubernetes v1.18.6 (upstream) +* Kubernetes v1.18.8 (upstream) * Single or multi-master, [Calico](https://www.projectcalico.org/) or [Cilium](https://github.com/cilium/cilium) or [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/), SELinux enforcing * Advanced features like [worker pools](https://typhoon.psdn.io/advanced/worker-pools/), [preemptible](https://typhoon.psdn.io/cl/google-cloud/#preemption) workers, and [snippets](https://typhoon.psdn.io/advanced/customization/#container-linux) customization @@ -54,7 +54,7 @@ Define a Kubernetes cluster by using the Terraform module for your chosen platfo ```tf module "yavin" { - source = "git::https://github.com/poseidon/typhoon//google-cloud/fedora-coreos/kubernetes?ref=v1.18.6" + source = "git::https://github.com/poseidon/typhoon//google-cloud/fedora-coreos/kubernetes?ref=v1.18.8" # Google Cloud cluster_name = "yavin" @@ -93,9 +93,9 @@ In 4-8 minutes (varies by platform), the cluster will be ready. This Google Clou $ export KUBECONFIG=/home/user/.kube/configs/yavin-config $ kubectl get nodes NAME ROLES STATUS AGE VERSION -yavin-controller-0.c.example-com.internal Ready 6m v1.18.6 -yavin-worker-jrbf.c.example-com.internal Ready 5m v1.18.6 -yavin-worker-mzdm.c.example-com.internal Ready 5m v1.18.6 +yavin-controller-0.c.example-com.internal Ready 6m v1.18.8 +yavin-worker-jrbf.c.example-com.internal Ready 5m v1.18.8 +yavin-worker-mzdm.c.example-com.internal Ready 5m v1.18.8 ``` List the pods. diff --git a/aws/container-linux/kubernetes/README.md b/aws/container-linux/kubernetes/README.md index 2eab62dc5..9c8fd0142 100644 --- a/aws/container-linux/kubernetes/README.md +++ b/aws/container-linux/kubernetes/README.md @@ -11,7 +11,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features -* Kubernetes v1.18.6 (upstream) +* Kubernetes v1.18.8 (upstream) * Single or multi-master, [Calico](https://www.projectcalico.org/) or [Cilium](https://github.com/cilium/cilium) or [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/) * Advanced features like [worker pools](https://typhoon.psdn.io/advanced/worker-pools/), [spot](https://typhoon.psdn.io/cl/aws/#spot) workers, and [snippets](https://typhoon.psdn.io/advanced/customization/#container-linux) customization diff --git a/aws/container-linux/kubernetes/bootstrap.tf b/aws/container-linux/kubernetes/bootstrap.tf index 937a2d05e..3b0a24986 100644 --- a/aws/container-linux/kubernetes/bootstrap.tf +++ b/aws/container-linux/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=60540868e070ea95da05fc5a6a5c9a55ce68c58d" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=8ef2fe7c992a8c15d696bd3e3a97be713b025e64" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/aws/container-linux/kubernetes/cl/controller.yaml b/aws/container-linux/kubernetes/cl/controller.yaml index ff15c3c6e..28d1d49b6 100644 --- a/aws/container-linux/kubernetes/cl/controller.yaml +++ b/aws/container-linux/kubernetes/cl/controller.yaml @@ -52,7 +52,7 @@ systemd: Description=Kubelet Wants=rpc-statd.service [Service] - Environment=KUBELET_IMAGE=docker://quay.io/poseidon/kubelet:v1.18.6 + Environment=KUBELET_IMAGE=docker://quay.io/poseidon/kubelet:v1.18.8 Environment=KUBELET_CGROUP_DRIVER=${cgroup_driver} ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests @@ -134,7 +134,7 @@ systemd: --volume script,kind=host,source=/opt/bootstrap/apply \ --mount volume=script,target=/apply \ --insecure-options=image \ - docker://quay.io/poseidon/kubelet:v1.18.6 \ + docker://quay.io/poseidon/kubelet:v1.18.8 \ --net=host \ --dns=host \ --exec=/apply diff --git a/aws/container-linux/kubernetes/workers/cl/worker.yaml b/aws/container-linux/kubernetes/workers/cl/worker.yaml index fc8e85c6f..7e10ddfd1 100644 --- a/aws/container-linux/kubernetes/workers/cl/worker.yaml +++ b/aws/container-linux/kubernetes/workers/cl/worker.yaml @@ -25,7 +25,7 @@ systemd: Description=Kubelet Wants=rpc-statd.service [Service] - Environment=KUBELET_IMAGE=docker://quay.io/poseidon/kubelet:v1.18.6 + Environment=KUBELET_IMAGE=docker://quay.io/poseidon/kubelet:v1.18.8 Environment=KUBELET_CGROUP_DRIVER=${cgroup_driver} ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests @@ -129,7 +129,7 @@ storage: --volume config,kind=host,source=/etc/kubernetes \ --mount volume=config,target=/etc/kubernetes \ --insecure-options=image \ - docker://quay.io/poseidon/kubelet:v1.18.6 \ + docker://quay.io/poseidon/kubelet:v1.18.8 \ --net=host \ --dns=host \ --exec=/usr/local/bin/kubectl -- --kubeconfig=/etc/kubernetes/kubeconfig delete node $(hostname) diff --git a/aws/fedora-coreos/kubernetes/README.md b/aws/fedora-coreos/kubernetes/README.md index f0a2502f6..96e06aee8 100644 --- a/aws/fedora-coreos/kubernetes/README.md +++ b/aws/fedora-coreos/kubernetes/README.md @@ -11,7 +11,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features -* Kubernetes v1.18.6 (upstream) +* Kubernetes v1.18.8 (upstream) * Single or multi-master, [Calico](https://www.projectcalico.org/) or [Cilium](https://github.com/cilium/cilium) or [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/), SELinux enforcing * Advanced features like [worker pools](https://typhoon.psdn.io/advanced/worker-pools/), [spot](https://typhoon.psdn.io/cl/aws/#spot) workers, and [snippets](https://typhoon.psdn.io/advanced/customization/#container-linux) customization diff --git a/aws/fedora-coreos/kubernetes/bootstrap.tf b/aws/fedora-coreos/kubernetes/bootstrap.tf index 033c57eff..2273b444e 100644 --- a/aws/fedora-coreos/kubernetes/bootstrap.tf +++ b/aws/fedora-coreos/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=60540868e070ea95da05fc5a6a5c9a55ce68c58d" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=8ef2fe7c992a8c15d696bd3e3a97be713b025e64" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/aws/fedora-coreos/kubernetes/fcc/controller.yaml b/aws/fedora-coreos/kubernetes/fcc/controller.yaml index 0bec7c812..eff6fff79 100644 --- a/aws/fedora-coreos/kubernetes/fcc/controller.yaml +++ b/aws/fedora-coreos/kubernetes/fcc/controller.yaml @@ -55,7 +55,7 @@ systemd: Description=Kubelet (System Container) Wants=rpc-statd.service [Service] - Environment=KUBELET_IMAGE=quay.io/poseidon/kubelet:v1.18.6 + Environment=KUBELET_IMAGE=quay.io/poseidon/kubelet:v1.18.8 ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests ExecStartPre=/bin/mkdir -p /opt/cni/bin @@ -124,7 +124,7 @@ systemd: --volume /opt/bootstrap/assets:/assets:ro,Z \ --volume /opt/bootstrap/apply:/apply:ro,Z \ --entrypoint=/apply \ - quay.io/poseidon/kubelet:v1.18.6 + quay.io/poseidon/kubelet:v1.18.8 ExecStartPost=/bin/touch /opt/bootstrap/bootstrap.done ExecStartPost=-/usr/bin/podman stop bootstrap storage: diff --git a/aws/fedora-coreos/kubernetes/workers/fcc/worker.yaml b/aws/fedora-coreos/kubernetes/workers/fcc/worker.yaml index 5fdeca2af..d8fb33d2f 100644 --- a/aws/fedora-coreos/kubernetes/workers/fcc/worker.yaml +++ b/aws/fedora-coreos/kubernetes/workers/fcc/worker.yaml @@ -25,7 +25,7 @@ systemd: Description=Kubelet (System Container) Wants=rpc-statd.service [Service] - Environment=KUBELET_IMAGE=quay.io/poseidon/kubelet:v1.18.6 + Environment=KUBELET_IMAGE=quay.io/poseidon/kubelet:v1.18.8 ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests ExecStartPre=/bin/mkdir -p /opt/cni/bin @@ -89,7 +89,7 @@ systemd: Type=oneshot RemainAfterExit=true ExecStart=/bin/true - ExecStop=/bin/bash -c '/usr/bin/podman run --volume /etc/kubernetes:/etc/kubernetes:ro,z --entrypoint /usr/local/bin/kubectl quay.io/poseidon/kubelet:v1.18.6 --kubeconfig=/etc/kubernetes/kubeconfig delete node $HOSTNAME' + ExecStop=/bin/bash -c '/usr/bin/podman run --volume /etc/kubernetes:/etc/kubernetes:ro,z --entrypoint /usr/local/bin/kubectl quay.io/poseidon/kubelet:v1.18.8 --kubeconfig=/etc/kubernetes/kubeconfig delete node $HOSTNAME' [Install] WantedBy=multi-user.target storage: diff --git a/azure/container-linux/kubernetes/README.md b/azure/container-linux/kubernetes/README.md index 64ce749ab..8ac228b8e 100644 --- a/azure/container-linux/kubernetes/README.md +++ b/azure/container-linux/kubernetes/README.md @@ -11,7 +11,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features -* Kubernetes v1.18.6 (upstream) +* Kubernetes v1.18.8 (upstream) * Single or multi-master, [Calico](https://www.projectcalico.org/) or [Cilium](https://github.com/cilium/cilium) or [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/) * Advanced features like [worker pools](https://typhoon.psdn.io/advanced/worker-pools/), [low-priority](https://typhoon.psdn.io/cl/azure/#low-priority) workers, and [snippets](https://typhoon.psdn.io/advanced/customization/#container-linux) customization diff --git a/azure/container-linux/kubernetes/bootstrap.tf b/azure/container-linux/kubernetes/bootstrap.tf index f4f826865..60eb2fff0 100644 --- a/azure/container-linux/kubernetes/bootstrap.tf +++ b/azure/container-linux/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=60540868e070ea95da05fc5a6a5c9a55ce68c58d" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=8ef2fe7c992a8c15d696bd3e3a97be713b025e64" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/azure/container-linux/kubernetes/cl/controller.yaml b/azure/container-linux/kubernetes/cl/controller.yaml index ff15c3c6e..28d1d49b6 100644 --- a/azure/container-linux/kubernetes/cl/controller.yaml +++ b/azure/container-linux/kubernetes/cl/controller.yaml @@ -52,7 +52,7 @@ systemd: Description=Kubelet Wants=rpc-statd.service [Service] - Environment=KUBELET_IMAGE=docker://quay.io/poseidon/kubelet:v1.18.6 + Environment=KUBELET_IMAGE=docker://quay.io/poseidon/kubelet:v1.18.8 Environment=KUBELET_CGROUP_DRIVER=${cgroup_driver} ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests @@ -134,7 +134,7 @@ systemd: --volume script,kind=host,source=/opt/bootstrap/apply \ --mount volume=script,target=/apply \ --insecure-options=image \ - docker://quay.io/poseidon/kubelet:v1.18.6 \ + docker://quay.io/poseidon/kubelet:v1.18.8 \ --net=host \ --dns=host \ --exec=/apply diff --git a/azure/container-linux/kubernetes/workers/cl/worker.yaml b/azure/container-linux/kubernetes/workers/cl/worker.yaml index 35f7b6798..14eff0740 100644 --- a/azure/container-linux/kubernetes/workers/cl/worker.yaml +++ b/azure/container-linux/kubernetes/workers/cl/worker.yaml @@ -25,7 +25,7 @@ systemd: Description=Kubelet Wants=rpc-statd.service [Service] - Environment=KUBELET_IMAGE=docker://quay.io/poseidon/kubelet:v1.18.6 + Environment=KUBELET_IMAGE=docker://quay.io/poseidon/kubelet:v1.18.8 Environment=KUBELET_CGROUP_DRIVER=${cgroup_driver} ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests @@ -129,7 +129,7 @@ storage: --volume config,kind=host,source=/etc/kubernetes \ --mount volume=config,target=/etc/kubernetes \ --insecure-options=image \ - docker://quay.io/poseidon/kubelet:v1.18.6 \ + docker://quay.io/poseidon/kubelet:v1.18.8 \ --net=host \ --dns=host \ --exec=/usr/local/bin/kubectl -- --kubeconfig=/etc/kubernetes/kubeconfig delete node $(hostname | tr '[:upper:]' '[:lower:]') diff --git a/azure/fedora-coreos/kubernetes/README.md b/azure/fedora-coreos/kubernetes/README.md index 29ed3f649..8859fc9ed 100644 --- a/azure/fedora-coreos/kubernetes/README.md +++ b/azure/fedora-coreos/kubernetes/README.md @@ -11,7 +11,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features -* Kubernetes v1.18.6 (upstream) +* Kubernetes v1.18.8 (upstream) * Single or multi-master, [Calico](https://www.projectcalico.org/) or [Cilium](https://github.com/cilium/cilium) or [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/), SELinux enforcing * Advanced features like [worker pools](https://typhoon.psdn.io/advanced/worker-pools/), [spot priority](https://typhoon.psdn.io/fedora-coreos/azure/#low-priority) workers, and [snippets](https://typhoon.psdn.io/advanced/customization/) customization diff --git a/azure/fedora-coreos/kubernetes/bootstrap.tf b/azure/fedora-coreos/kubernetes/bootstrap.tf index 67d872f51..c882c2948 100644 --- a/azure/fedora-coreos/kubernetes/bootstrap.tf +++ b/azure/fedora-coreos/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=60540868e070ea95da05fc5a6a5c9a55ce68c58d" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=8ef2fe7c992a8c15d696bd3e3a97be713b025e64" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/azure/fedora-coreos/kubernetes/fcc/controller.yaml b/azure/fedora-coreos/kubernetes/fcc/controller.yaml index cdfa5f2a5..6949ba04e 100644 --- a/azure/fedora-coreos/kubernetes/fcc/controller.yaml +++ b/azure/fedora-coreos/kubernetes/fcc/controller.yaml @@ -54,7 +54,7 @@ systemd: Description=Kubelet (System Container) Wants=rpc-statd.service [Service] - Environment=KUBELET_IMAGE=quay.io/poseidon/kubelet:v1.18.6 + Environment=KUBELET_IMAGE=quay.io/poseidon/kubelet:v1.18.8 ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests ExecStartPre=/bin/mkdir -p /opt/cni/bin @@ -123,7 +123,7 @@ systemd: --volume /opt/bootstrap/assets:/assets:ro,Z \ --volume /opt/bootstrap/apply:/apply:ro,Z \ --entrypoint=/apply \ - quay.io/poseidon/kubelet:v1.18.6 + quay.io/poseidon/kubelet:v1.18.8 ExecStartPost=/bin/touch /opt/bootstrap/bootstrap.done ExecStartPost=-/usr/bin/podman stop bootstrap storage: diff --git a/azure/fedora-coreos/kubernetes/workers/fcc/worker.yaml b/azure/fedora-coreos/kubernetes/workers/fcc/worker.yaml index e392486a5..e9e382786 100644 --- a/azure/fedora-coreos/kubernetes/workers/fcc/worker.yaml +++ b/azure/fedora-coreos/kubernetes/workers/fcc/worker.yaml @@ -24,7 +24,7 @@ systemd: Description=Kubelet (System Container) Wants=rpc-statd.service [Service] - Environment=KUBELET_IMAGE=quay.io/poseidon/kubelet:v1.18.6 + Environment=KUBELET_IMAGE=quay.io/poseidon/kubelet:v1.18.8 ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests ExecStartPre=/bin/mkdir -p /opt/cni/bin @@ -88,7 +88,7 @@ systemd: Type=oneshot RemainAfterExit=true ExecStart=/bin/true - ExecStop=/bin/bash -c '/usr/bin/podman run --volume /etc/kubernetes:/etc/kubernetes:ro,z --entrypoint /usr/local/bin/kubectl quay.io/poseidon/kubelet:v1.18.6 --kubeconfig=/etc/kubernetes/kubeconfig delete node $HOSTNAME' + ExecStop=/bin/bash -c '/usr/bin/podman run --volume /etc/kubernetes:/etc/kubernetes:ro,z --entrypoint /usr/local/bin/kubectl quay.io/poseidon/kubelet:v1.18.8 --kubeconfig=/etc/kubernetes/kubeconfig delete node $HOSTNAME' [Install] WantedBy=multi-user.target storage: diff --git a/bare-metal/container-linux/kubernetes/README.md b/bare-metal/container-linux/kubernetes/README.md index 8a60c63a6..7a10f4ddc 100644 --- a/bare-metal/container-linux/kubernetes/README.md +++ b/bare-metal/container-linux/kubernetes/README.md @@ -11,7 +11,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features -* Kubernetes v1.18.6 (upstream) +* Kubernetes v1.18.8 (upstream) * Single or multi-master, [Calico](https://www.projectcalico.org/) or [Cilium](https://github.com/cilium/cilium) or [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/) * Advanced features like [snippets](https://typhoon.psdn.io/advanced/customization/#container-linux) customization diff --git a/bare-metal/container-linux/kubernetes/bootstrap.tf b/bare-metal/container-linux/kubernetes/bootstrap.tf index 069bf5fbf..b71284888 100644 --- a/bare-metal/container-linux/kubernetes/bootstrap.tf +++ b/bare-metal/container-linux/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=60540868e070ea95da05fc5a6a5c9a55ce68c58d" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=8ef2fe7c992a8c15d696bd3e3a97be713b025e64" cluster_name = var.cluster_name api_servers = [var.k8s_domain_name] diff --git a/bare-metal/container-linux/kubernetes/cl/controller.yaml b/bare-metal/container-linux/kubernetes/cl/controller.yaml index 7e72223dc..566509f02 100644 --- a/bare-metal/container-linux/kubernetes/cl/controller.yaml +++ b/bare-metal/container-linux/kubernetes/cl/controller.yaml @@ -60,7 +60,7 @@ systemd: Description=Kubelet Wants=rpc-statd.service [Service] - Environment=KUBELET_IMAGE=docker://quay.io/poseidon/kubelet:v1.18.6 + Environment=KUBELET_IMAGE=docker://quay.io/poseidon/kubelet:v1.18.8 Environment=KUBELET_CGROUP_DRIVER=${cgroup_driver} ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests @@ -147,7 +147,7 @@ systemd: --volume script,kind=host,source=/opt/bootstrap/apply \ --mount volume=script,target=/apply \ --insecure-options=image \ - docker://quay.io/poseidon/kubelet:v1.18.6 \ + docker://quay.io/poseidon/kubelet:v1.18.8 \ --net=host \ --dns=host \ --exec=/apply diff --git a/bare-metal/container-linux/kubernetes/cl/worker.yaml b/bare-metal/container-linux/kubernetes/cl/worker.yaml index 91424a39a..a581195b3 100644 --- a/bare-metal/container-linux/kubernetes/cl/worker.yaml +++ b/bare-metal/container-linux/kubernetes/cl/worker.yaml @@ -33,7 +33,7 @@ systemd: Description=Kubelet Wants=rpc-statd.service [Service] - Environment=KUBELET_IMAGE=docker://quay.io/poseidon/kubelet:v1.18.6 + Environment=KUBELET_IMAGE=docker://quay.io/poseidon/kubelet:v1.18.8 Environment=KUBELET_CGROUP_DRIVER=${cgroup_driver} ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests diff --git a/bare-metal/fedora-coreos/kubernetes/README.md b/bare-metal/fedora-coreos/kubernetes/README.md index 64fd3def2..cb448d84b 100644 --- a/bare-metal/fedora-coreos/kubernetes/README.md +++ b/bare-metal/fedora-coreos/kubernetes/README.md @@ -11,7 +11,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features -* Kubernetes v1.18.6 (upstream) +* Kubernetes v1.18.8 (upstream) * Single or multi-master, [Calico](https://www.projectcalico.org/) or [Cilium](https://github.com/cilium/cilium) or [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/), SELinux enforcing * Advanced features like [snippets](https://typhoon.psdn.io/advanced/customization/#container-linux) customization diff --git a/bare-metal/fedora-coreos/kubernetes/bootstrap.tf b/bare-metal/fedora-coreos/kubernetes/bootstrap.tf index f59a30622..fdc3da51e 100644 --- a/bare-metal/fedora-coreos/kubernetes/bootstrap.tf +++ b/bare-metal/fedora-coreos/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=60540868e070ea95da05fc5a6a5c9a55ce68c58d" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=8ef2fe7c992a8c15d696bd3e3a97be713b025e64" cluster_name = var.cluster_name api_servers = [var.k8s_domain_name] diff --git a/bare-metal/fedora-coreos/kubernetes/fcc/controller.yaml b/bare-metal/fedora-coreos/kubernetes/fcc/controller.yaml index 4700a9c40..4fa1342fa 100644 --- a/bare-metal/fedora-coreos/kubernetes/fcc/controller.yaml +++ b/bare-metal/fedora-coreos/kubernetes/fcc/controller.yaml @@ -53,7 +53,7 @@ systemd: Description=Kubelet (System Container) Wants=rpc-statd.service [Service] - Environment=KUBELET_IMAGE=quay.io/poseidon/kubelet:v1.18.6 + Environment=KUBELET_IMAGE=quay.io/poseidon/kubelet:v1.18.8 ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests ExecStartPre=/bin/mkdir -p /opt/cni/bin @@ -134,7 +134,7 @@ systemd: --volume /opt/bootstrap/assets:/assets:ro,Z \ --volume /opt/bootstrap/apply:/apply:ro,Z \ --entrypoint=/apply \ - quay.io/poseidon/kubelet:v1.18.6 + quay.io/poseidon/kubelet:v1.18.8 ExecStartPost=/bin/touch /opt/bootstrap/bootstrap.done ExecStartPost=-/usr/bin/podman stop bootstrap storage: diff --git a/bare-metal/fedora-coreos/kubernetes/fcc/worker.yaml b/bare-metal/fedora-coreos/kubernetes/fcc/worker.yaml index 2b0645a12..d0aee9a60 100644 --- a/bare-metal/fedora-coreos/kubernetes/fcc/worker.yaml +++ b/bare-metal/fedora-coreos/kubernetes/fcc/worker.yaml @@ -23,7 +23,7 @@ systemd: Description=Kubelet (System Container) Wants=rpc-statd.service [Service] - Environment=KUBELET_IMAGE=quay.io/poseidon/kubelet:v1.18.6 + Environment=KUBELET_IMAGE=quay.io/poseidon/kubelet:v1.18.8 ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests ExecStartPre=/bin/mkdir -p /opt/cni/bin diff --git a/digital-ocean/container-linux/kubernetes/README.md b/digital-ocean/container-linux/kubernetes/README.md index 6b1a1e49f..87321693b 100644 --- a/digital-ocean/container-linux/kubernetes/README.md +++ b/digital-ocean/container-linux/kubernetes/README.md @@ -11,7 +11,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features -* Kubernetes v1.18.6 (upstream) +* Kubernetes v1.18.8 (upstream) * Single or multi-master, [Calico](https://www.projectcalico.org/) or [Cilium](https://github.com/cilium/cilium) or [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/) * Advanced features like [snippets](https://typhoon.psdn.io/advanced/customization/#container-linux) customization diff --git a/digital-ocean/container-linux/kubernetes/bootstrap.tf b/digital-ocean/container-linux/kubernetes/bootstrap.tf index 4405d4a06..3b024cb58 100644 --- a/digital-ocean/container-linux/kubernetes/bootstrap.tf +++ b/digital-ocean/container-linux/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=60540868e070ea95da05fc5a6a5c9a55ce68c58d" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=8ef2fe7c992a8c15d696bd3e3a97be713b025e64" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/digital-ocean/container-linux/kubernetes/cl/controller.yaml b/digital-ocean/container-linux/kubernetes/cl/controller.yaml index 415f3305f..e03c3603f 100644 --- a/digital-ocean/container-linux/kubernetes/cl/controller.yaml +++ b/digital-ocean/container-linux/kubernetes/cl/controller.yaml @@ -62,7 +62,7 @@ systemd: After=coreos-metadata.service Wants=rpc-statd.service [Service] - Environment=KUBELET_IMAGE=docker://quay.io/poseidon/kubelet:v1.18.6 + Environment=KUBELET_IMAGE=docker://quay.io/poseidon/kubelet:v1.18.8 EnvironmentFile=/run/metadata/coreos ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests @@ -144,7 +144,7 @@ systemd: --volume script,kind=host,source=/opt/bootstrap/apply \ --mount volume=script,target=/apply \ --insecure-options=image \ - docker://quay.io/poseidon/kubelet:v1.18.6 \ + docker://quay.io/poseidon/kubelet:v1.18.8 \ --net=host \ --dns=host \ --exec=/apply diff --git a/digital-ocean/container-linux/kubernetes/cl/worker.yaml b/digital-ocean/container-linux/kubernetes/cl/worker.yaml index 1d638d8aa..4bb07e1cb 100644 --- a/digital-ocean/container-linux/kubernetes/cl/worker.yaml +++ b/digital-ocean/container-linux/kubernetes/cl/worker.yaml @@ -35,7 +35,7 @@ systemd: After=coreos-metadata.service Wants=rpc-statd.service [Service] - Environment=KUBELET_IMAGE=docker://quay.io/poseidon/kubelet:v1.18.6 + Environment=KUBELET_IMAGE=docker://quay.io/poseidon/kubelet:v1.18.8 EnvironmentFile=/run/metadata/coreos ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests @@ -134,7 +134,7 @@ storage: --volume config,kind=host,source=/etc/kubernetes \ --mount volume=config,target=/etc/kubernetes \ --insecure-options=image \ - docker://quay.io/poseidon/kubelet:v1.18.6 \ + docker://quay.io/poseidon/kubelet:v1.18.8 \ --net=host \ --dns=host \ --exec=/usr/local/bin/kubectl -- --kubeconfig=/etc/kubernetes/kubeconfig delete node $(hostname) diff --git a/digital-ocean/fedora-coreos/kubernetes/README.md b/digital-ocean/fedora-coreos/kubernetes/README.md index 76fc2db7f..6000d0aac 100644 --- a/digital-ocean/fedora-coreos/kubernetes/README.md +++ b/digital-ocean/fedora-coreos/kubernetes/README.md @@ -11,7 +11,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features -* Kubernetes v1.18.6 (upstream) +* Kubernetes v1.18.8 (upstream) * Single or multi-master, [Calico](https://www.projectcalico.org/) or [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/), SELinux enforcing * Advanced features like [snippets](https://typhoon.psdn.io/advanced/customization/) customization diff --git a/digital-ocean/fedora-coreos/kubernetes/bootstrap.tf b/digital-ocean/fedora-coreos/kubernetes/bootstrap.tf index bad1275d5..331905b00 100644 --- a/digital-ocean/fedora-coreos/kubernetes/bootstrap.tf +++ b/digital-ocean/fedora-coreos/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=60540868e070ea95da05fc5a6a5c9a55ce68c58d" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=8ef2fe7c992a8c15d696bd3e3a97be713b025e64" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/digital-ocean/fedora-coreos/kubernetes/fcc/controller.yaml b/digital-ocean/fedora-coreos/kubernetes/fcc/controller.yaml index 6ce76a4d2..7e404fc20 100644 --- a/digital-ocean/fedora-coreos/kubernetes/fcc/controller.yaml +++ b/digital-ocean/fedora-coreos/kubernetes/fcc/controller.yaml @@ -55,7 +55,7 @@ systemd: After=afterburn.service Wants=rpc-statd.service [Service] - Environment=KUBELET_IMAGE=quay.io/poseidon/kubelet:v1.18.6 + Environment=KUBELET_IMAGE=quay.io/poseidon/kubelet:v1.18.8 EnvironmentFile=/run/metadata/afterburn ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests @@ -135,7 +135,7 @@ systemd: --volume /opt/bootstrap/assets:/assets:ro,Z \ --volume /opt/bootstrap/apply:/apply:ro,Z \ --entrypoint=/apply \ - quay.io/poseidon/kubelet:v1.18.6 + quay.io/poseidon/kubelet:v1.18.8 ExecStartPost=/bin/touch /opt/bootstrap/bootstrap.done ExecStartPost=-/usr/bin/podman stop bootstrap storage: diff --git a/digital-ocean/fedora-coreos/kubernetes/fcc/worker.yaml b/digital-ocean/fedora-coreos/kubernetes/fcc/worker.yaml index fbff7b339..d099fe29d 100644 --- a/digital-ocean/fedora-coreos/kubernetes/fcc/worker.yaml +++ b/digital-ocean/fedora-coreos/kubernetes/fcc/worker.yaml @@ -26,7 +26,7 @@ systemd: After=afterburn.service Wants=rpc-statd.service [Service] - Environment=KUBELET_IMAGE=quay.io/poseidon/kubelet:v1.18.6 + Environment=KUBELET_IMAGE=quay.io/poseidon/kubelet:v1.18.8 EnvironmentFile=/run/metadata/afterburn ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests @@ -98,7 +98,7 @@ systemd: Type=oneshot RemainAfterExit=true ExecStart=/bin/true - ExecStop=/bin/bash -c '/usr/bin/podman run --volume /etc/kubernetes:/etc/kubernetes:ro,z --entrypoint /usr/local/bin/kubectl quay.io/poseidon/kubelet:v1.18.6 --kubeconfig=/etc/kubernetes/kubeconfig delete node $HOSTNAME' + ExecStop=/bin/bash -c '/usr/bin/podman run --volume /etc/kubernetes:/etc/kubernetes:ro,z --entrypoint /usr/local/bin/kubectl quay.io/poseidon/kubelet:v1.18.8 --kubeconfig=/etc/kubernetes/kubeconfig delete node $HOSTNAME' [Install] WantedBy=multi-user.target storage: diff --git a/docs/advanced/worker-pools.md b/docs/advanced/worker-pools.md index 7caed3f58..4fce2d390 100644 --- a/docs/advanced/worker-pools.md +++ b/docs/advanced/worker-pools.md @@ -83,7 +83,7 @@ Create a cluster following the Azure [tutorial](../flatcar-linux/azure.md#cluste ```tf module "ramius-worker-pool" { - source = "git::https://github.com/poseidon/typhoon//azure/container-linux/kubernetes/workers?ref=v1.18.6" + source = "git::https://github.com/poseidon/typhoon//azure/container-linux/kubernetes/workers?ref=v1.18.8" # Azure region = module.ramius.region @@ -149,7 +149,7 @@ Create a cluster following the Google Cloud [tutorial](../flatcar-linux/google-c ```tf module "yavin-worker-pool" { - source = "git::https://github.com/poseidon/typhoon//google-cloud/container-linux/kubernetes/workers?ref=v1.18.6" + source = "git::https://github.com/poseidon/typhoon//google-cloud/container-linux/kubernetes/workers?ref=v1.18.8" # Google Cloud region = "europe-west2" @@ -180,11 +180,11 @@ Verify a managed instance group of workers joins the cluster within a few minute ``` $ kubectl get nodes NAME STATUS AGE VERSION -yavin-controller-0.c.example-com.internal Ready 6m v1.18.6 -yavin-worker-jrbf.c.example-com.internal Ready 5m v1.18.6 -yavin-worker-mzdm.c.example-com.internal Ready 5m v1.18.6 -yavin-16x-worker-jrbf.c.example-com.internal Ready 3m v1.18.6 -yavin-16x-worker-mzdm.c.example-com.internal Ready 3m v1.18.6 +yavin-controller-0.c.example-com.internal Ready 6m v1.18.8 +yavin-worker-jrbf.c.example-com.internal Ready 5m v1.18.8 +yavin-worker-mzdm.c.example-com.internal Ready 5m v1.18.8 +yavin-16x-worker-jrbf.c.example-com.internal Ready 3m v1.18.8 +yavin-16x-worker-mzdm.c.example-com.internal Ready 3m v1.18.8 ``` ### Variables diff --git a/docs/fedora-coreos/aws.md b/docs/fedora-coreos/aws.md index 79595bcf6..cd2e9f336 100644 --- a/docs/fedora-coreos/aws.md +++ b/docs/fedora-coreos/aws.md @@ -1,6 +1,6 @@ # AWS -In this tutorial, we'll create a Kubernetes v1.18.6 cluster on AWS with Fedora CoreOS. +In this tutorial, we'll create a Kubernetes v1.18.8 cluster on AWS with Fedora CoreOS. We'll declare a Kubernetes cluster using the Typhoon Terraform module. Then apply the changes to create a VPC, gateway, subnets, security groups, controller instances, worker auto-scaling group, network load balancer, and TLS assets. @@ -72,7 +72,7 @@ Define a Kubernetes cluster using the module `aws/fedora-coreos/kubernetes`. ```tf module "tempest" { - source = "git::https://github.com/poseidon/typhoon//aws/fedora-coreos/kubernetes?ref=v1.18.6" + source = "git::https://github.com/poseidon/typhoon//aws/fedora-coreos/kubernetes?ref=v1.18.8" # AWS cluster_name = "tempest" @@ -145,9 +145,9 @@ List nodes in the cluster. $ export KUBECONFIG=/home/user/.kube/configs/tempest-config $ kubectl get nodes NAME STATUS ROLES AGE VERSION -ip-10-0-3-155 Ready 10m v1.18.6 -ip-10-0-26-65 Ready 10m v1.18.6 -ip-10-0-41-21 Ready 10m v1.18.6 +ip-10-0-3-155 Ready 10m v1.18.8 +ip-10-0-26-65 Ready 10m v1.18.8 +ip-10-0-41-21 Ready 10m v1.18.8 ``` List the pods. diff --git a/docs/fedora-coreos/azure.md b/docs/fedora-coreos/azure.md index eb800ffbb..40c6bd41c 100644 --- a/docs/fedora-coreos/azure.md +++ b/docs/fedora-coreos/azure.md @@ -1,6 +1,6 @@ # Azure -In this tutorial, we'll create a Kubernetes v1.18.6 cluster on Azure with Fedora CoreOS. +In this tutorial, we'll create a Kubernetes v1.18.8 cluster on Azure with Fedora CoreOS. We'll declare a Kubernetes cluster using the Typhoon Terraform module. Then apply the changes to create a resource group, virtual network, subnets, security groups, controller availability set, worker scale set, load balancer, and TLS assets. @@ -86,7 +86,7 @@ Define a Kubernetes cluster using the module `azure/fedora-coreos/kubernetes`. ```tf module "ramius" { - source = "git::https://github.com/poseidon/typhoon//azure/fedora-coreos/kubernetes?ref=v1.18.6" + source = "git::https://github.com/poseidon/typhoon//azure/fedora-coreos/kubernetes?ref=v1.18.8" # Azure cluster_name = "ramius" @@ -161,9 +161,9 @@ List nodes in the cluster. $ export KUBECONFIG=/home/user/.kube/configs/ramius-config $ kubectl get nodes NAME STATUS ROLES AGE VERSION -ramius-controller-0 Ready 24m v1.18.6 -ramius-worker-000001 Ready 25m v1.18.6 -ramius-worker-000002 Ready 24m v1.18.6 +ramius-controller-0 Ready 24m v1.18.8 +ramius-worker-000001 Ready 25m v1.18.8 +ramius-worker-000002 Ready 24m v1.18.8 ``` List the pods. diff --git a/docs/fedora-coreos/bare-metal.md b/docs/fedora-coreos/bare-metal.md index e5d4b1151..68e887021 100644 --- a/docs/fedora-coreos/bare-metal.md +++ b/docs/fedora-coreos/bare-metal.md @@ -1,6 +1,6 @@ # Bare-Metal -In this tutorial, we'll network boot and provision a Kubernetes v1.18.6 cluster on bare-metal with Fedora CoreOS. +In this tutorial, we'll network boot and provision a Kubernetes v1.18.8 cluster on bare-metal with Fedora CoreOS. First, we'll deploy a [Matchbox](https://github.com/poseidon/matchbox) service and setup a network boot environment. Then, we'll declare a Kubernetes cluster using the Typhoon Terraform module and power on machines. On PXE boot, machines will install Fedora CoreOS to disk, reboot into the disk install, and provision themselves as Kubernetes controllers or workers via Ignition. @@ -154,7 +154,7 @@ Define a Kubernetes cluster using the module `bare-metal/fedora-coreos/kubernete ```tf module "mercury" { - source = "git::https://github.com/poseidon/typhoon//bare-metal/fedora-coreos/kubernetes?ref=v1.18.6" + source = "git::https://github.com/poseidon/typhoon//bare-metal/fedora-coreos/kubernetes?ref=v1.18.8" # bare-metal cluster_name = "mercury" @@ -283,9 +283,9 @@ List nodes in the cluster. $ export KUBECONFIG=/home/user/.kube/configs/mercury-config $ kubectl get nodes NAME STATUS ROLES AGE VERSION -node1.example.com Ready 10m v1.18.6 -node2.example.com Ready 10m v1.18.6 -node3.example.com Ready 10m v1.18.6 +node1.example.com Ready 10m v1.18.8 +node2.example.com Ready 10m v1.18.8 +node3.example.com Ready 10m v1.18.8 ``` List the pods. diff --git a/docs/fedora-coreos/digitalocean.md b/docs/fedora-coreos/digitalocean.md index 0091f1f65..76bdb5fd4 100644 --- a/docs/fedora-coreos/digitalocean.md +++ b/docs/fedora-coreos/digitalocean.md @@ -1,6 +1,6 @@ # DigitalOcean -In this tutorial, we'll create a Kubernetes v1.18.6 cluster on DigitalOcean with Fedora CoreOS. +In this tutorial, we'll create a Kubernetes v1.18.8 cluster on DigitalOcean with Fedora CoreOS. We'll declare a Kubernetes cluster using the Typhoon Terraform module. Then apply the changes to create controller droplets, worker droplets, DNS records, tags, and TLS assets. @@ -81,7 +81,7 @@ Define a Kubernetes cluster using the module `digital-ocean/fedora-coreos/kubern ```tf module "nemo" { - source = "git::https://github.com/poseidon/typhoon//digital-ocean/fedora-coreos/kubernetes?ref=v1.18.6" + source = "git::https://github.com/poseidon/typhoon//digital-ocean/fedora-coreos/kubernetes?ref=v1.18.8" # Digital Ocean cluster_name = "nemo" @@ -155,9 +155,9 @@ List nodes in the cluster. $ export KUBECONFIG=/home/user/.kube/configs/nemo-config $ kubectl get nodes NAME STATUS ROLES AGE VERSION -10.132.110.130 Ready 10m v1.18.6 -10.132.115.81 Ready 10m v1.18.6 -10.132.124.107 Ready 10m v1.18.6 +10.132.110.130 Ready 10m v1.18.8 +10.132.115.81 Ready 10m v1.18.8 +10.132.124.107 Ready 10m v1.18.8 ``` List the pods. diff --git a/docs/fedora-coreos/google-cloud.md b/docs/fedora-coreos/google-cloud.md index 22636246d..21e08afe9 100644 --- a/docs/fedora-coreos/google-cloud.md +++ b/docs/fedora-coreos/google-cloud.md @@ -1,6 +1,6 @@ # Google Cloud -In this tutorial, we'll create a Kubernetes v1.18.6 cluster on Google Compute Engine with Fedora CoreOS. +In this tutorial, we'll create a Kubernetes v1.18.8 cluster on Google Compute Engine with Fedora CoreOS. We'll declare a Kubernetes cluster using the Typhoon Terraform module. Then apply the changes to create a network, firewall rules, health checks, controller instances, worker managed instance group, load balancers, and TLS assets. @@ -147,9 +147,9 @@ List nodes in the cluster. $ export KUBECONFIG=/home/user/.kube/configs/yavin-config $ kubectl get nodes NAME ROLES STATUS AGE VERSION -yavin-controller-0.c.example-com.internal Ready 6m v1.18.6 -yavin-worker-jrbf.c.example-com.internal Ready 5m v1.18.6 -yavin-worker-mzdm.c.example-com.internal Ready 5m v1.18.6 +yavin-controller-0.c.example-com.internal Ready 6m v1.18.8 +yavin-worker-jrbf.c.example-com.internal Ready 5m v1.18.8 +yavin-worker-mzdm.c.example-com.internal Ready 5m v1.18.8 ``` List the pods. diff --git a/docs/flatcar-linux/aws.md b/docs/flatcar-linux/aws.md index 712d49479..95cd96033 100644 --- a/docs/flatcar-linux/aws.md +++ b/docs/flatcar-linux/aws.md @@ -1,6 +1,6 @@ # AWS -In this tutorial, we'll create a Kubernetes v1.18.6 cluster on AWS with CoreOS Container Linux or Flatcar Linux. +In this tutorial, we'll create a Kubernetes v1.18.8 cluster on AWS with CoreOS Container Linux or Flatcar Linux. We'll declare a Kubernetes cluster using the Typhoon Terraform module. Then apply the changes to create a VPC, gateway, subnets, security groups, controller instances, worker auto-scaling group, network load balancer, and TLS assets. @@ -72,7 +72,7 @@ Define a Kubernetes cluster using the module `aws/container-linux/kubernetes`. ```tf module "tempest" { - source = "git::https://github.com/poseidon/typhoon//aws/container-linux/kubernetes?ref=v1.18.6" + source = "git::https://github.com/poseidon/typhoon//aws/container-linux/kubernetes?ref=v1.18.8" # AWS cluster_name = "tempest" @@ -145,9 +145,9 @@ List nodes in the cluster. $ export KUBECONFIG=/home/user/.kube/configs/tempest-config $ kubectl get nodes NAME STATUS ROLES AGE VERSION -ip-10-0-3-155 Ready 10m v1.18.6 -ip-10-0-26-65 Ready 10m v1.18.6 -ip-10-0-41-21 Ready 10m v1.18.6 +ip-10-0-3-155 Ready 10m v1.18.8 +ip-10-0-26-65 Ready 10m v1.18.8 +ip-10-0-41-21 Ready 10m v1.18.8 ``` List the pods. diff --git a/docs/flatcar-linux/azure.md b/docs/flatcar-linux/azure.md index 027feb719..01b07fd34 100644 --- a/docs/flatcar-linux/azure.md +++ b/docs/flatcar-linux/azure.md @@ -1,6 +1,6 @@ # Azure -In this tutorial, we'll create a Kubernetes v1.18.6 cluster on Azure with CoreOS Container Linux or Flatcar Linux. +In this tutorial, we'll create a Kubernetes v1.18.8 cluster on Azure with CoreOS Container Linux or Flatcar Linux. We'll declare a Kubernetes cluster using the Typhoon Terraform module. Then apply the changes to create a resource group, virtual network, subnets, security groups, controller availability set, worker scale set, load balancer, and TLS assets. @@ -75,7 +75,7 @@ Define a Kubernetes cluster using the module `azure/container-linux/kubernetes`. ```tf module "ramius" { - source = "git::https://github.com/poseidon/typhoon//azure/container-linux/kubernetes?ref=v1.18.6" + source = "git::https://github.com/poseidon/typhoon//azure/container-linux/kubernetes?ref=v1.18.8" # Azure cluster_name = "ramius" @@ -149,9 +149,9 @@ List nodes in the cluster. $ export KUBECONFIG=/home/user/.kube/configs/ramius-config $ kubectl get nodes NAME STATUS ROLES AGE VERSION -ramius-controller-0 Ready 24m v1.18.6 -ramius-worker-000001 Ready 25m v1.18.6 -ramius-worker-000002 Ready 24m v1.18.6 +ramius-controller-0 Ready 24m v1.18.8 +ramius-worker-000001 Ready 25m v1.18.8 +ramius-worker-000002 Ready 24m v1.18.8 ``` List the pods. diff --git a/docs/flatcar-linux/bare-metal.md b/docs/flatcar-linux/bare-metal.md index f387f6056..2a5dbda98 100644 --- a/docs/flatcar-linux/bare-metal.md +++ b/docs/flatcar-linux/bare-metal.md @@ -1,6 +1,6 @@ # Bare-Metal -In this tutorial, we'll network boot and provision a Kubernetes v1.18.6 cluster on bare-metal with CoreOS Container Linux or Flatcar Linux. +In this tutorial, we'll network boot and provision a Kubernetes v1.18.8 cluster on bare-metal with CoreOS Container Linux or Flatcar Linux. First, we'll deploy a [Matchbox](https://github.com/poseidon/matchbox) service and setup a network boot environment. Then, we'll declare a Kubernetes cluster using the Typhoon Terraform module and power on machines. On PXE boot, machines will install Container Linux to disk, reboot into the disk install, and provision themselves as Kubernetes controllers or workers via Ignition. @@ -154,7 +154,7 @@ Define a Kubernetes cluster using the module `bare-metal/container-linux/kuberne ```tf module "mercury" { - source = "git::https://github.com/poseidon/typhoon//bare-metal/container-linux/kubernetes?ref=v1.18.6" + source = "git::https://github.com/poseidon/typhoon//bare-metal/container-linux/kubernetes?ref=v1.18.8" # bare-metal cluster_name = "mercury" @@ -293,9 +293,9 @@ List nodes in the cluster. $ export KUBECONFIG=/home/user/.kube/configs/mercury-config $ kubectl get nodes NAME STATUS ROLES AGE VERSION -node1.example.com Ready 10m v1.18.6 -node2.example.com Ready 10m v1.18.6 -node3.example.com Ready 10m v1.18.6 +node1.example.com Ready 10m v1.18.8 +node2.example.com Ready 10m v1.18.8 +node3.example.com Ready 10m v1.18.8 ``` List the pods. diff --git a/docs/flatcar-linux/digitalocean.md b/docs/flatcar-linux/digitalocean.md index 1e0ca422c..341868c9e 100644 --- a/docs/flatcar-linux/digitalocean.md +++ b/docs/flatcar-linux/digitalocean.md @@ -1,6 +1,6 @@ # DigitalOcean -In this tutorial, we'll create a Kubernetes v1.18.6 cluster on DigitalOcean with CoreOS Container Linux or Flatcar Linux. +In this tutorial, we'll create a Kubernetes v1.18.8 cluster on DigitalOcean with CoreOS Container Linux or Flatcar Linux. We'll declare a Kubernetes cluster using the Typhoon Terraform module. Then apply the changes to create controller droplets, worker droplets, DNS records, tags, and TLS assets. @@ -81,7 +81,7 @@ Define a Kubernetes cluster using the module `digital-ocean/container-linux/kube ```tf module "nemo" { - source = "git::https://github.com/poseidon/typhoon//digital-ocean/container-linux/kubernetes?ref=v1.18.6" + source = "git::https://github.com/poseidon/typhoon//digital-ocean/container-linux/kubernetes?ref=v1.18.8" # Digital Ocean cluster_name = "nemo" @@ -155,9 +155,9 @@ List nodes in the cluster. $ export KUBECONFIG=/home/user/.kube/configs/nemo-config $ kubectl get nodes NAME STATUS ROLES AGE VERSION -10.132.110.130 Ready 10m v1.18.6 -10.132.115.81 Ready 10m v1.18.6 -10.132.124.107 Ready 10m v1.18.6 +10.132.110.130 Ready 10m v1.18.8 +10.132.115.81 Ready 10m v1.18.8 +10.132.124.107 Ready 10m v1.18.8 ``` List the pods. diff --git a/docs/flatcar-linux/google-cloud.md b/docs/flatcar-linux/google-cloud.md index 51c1daf02..e800e00d7 100644 --- a/docs/flatcar-linux/google-cloud.md +++ b/docs/flatcar-linux/google-cloud.md @@ -1,6 +1,6 @@ # Google Cloud -In this tutorial, we'll create a Kubernetes v1.18.6 cluster on Google Compute Engine with CoreOS Container Linux or Flatcar Linux. +In this tutorial, we'll create a Kubernetes v1.18.8 cluster on Google Compute Engine with CoreOS Container Linux or Flatcar Linux. We'll declare a Kubernetes cluster using the Typhoon Terraform module. Then apply the changes to create a network, firewall rules, health checks, controller instances, worker managed instance group, load balancers, and TLS assets. @@ -92,7 +92,7 @@ Define a Kubernetes cluster using the module `google-cloud/container-linux/kuber ```tf module "yavin" { - source = "git::https://github.com/poseidon/typhoon//google-cloud/container-linux/kubernetes?ref=v1.18.6" + source = "git::https://github.com/poseidon/typhoon//google-cloud/container-linux/kubernetes?ref=v1.18.8" # Google Cloud cluster_name = "yavin" @@ -167,9 +167,9 @@ List nodes in the cluster. $ export KUBECONFIG=/home/user/.kube/configs/yavin-config $ kubectl get nodes NAME ROLES STATUS AGE VERSION -yavin-controller-0.c.example-com.internal Ready 6m v1.18.6 -yavin-worker-jrbf.c.example-com.internal Ready 5m v1.18.6 -yavin-worker-mzdm.c.example-com.internal Ready 5m v1.18.6 +yavin-controller-0.c.example-com.internal Ready 6m v1.18.8 +yavin-worker-jrbf.c.example-com.internal Ready 5m v1.18.8 +yavin-worker-mzdm.c.example-com.internal Ready 5m v1.18.8 ``` List the pods. diff --git a/docs/index.md b/docs/index.md index 2966ed2fd..96a20f557 100644 --- a/docs/index.md +++ b/docs/index.md @@ -11,7 +11,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features -* Kubernetes v1.18.6 (upstream) +* Kubernetes v1.18.8 (upstream) * Single or multi-master, [Calico](https://www.projectcalico.org/) or [Cilium](https://github.com/cilium/cilium) or [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/), SELinux enforcing * Advanced features like [worker pools](advanced/worker-pools/), [preemptible](fedora-coreos/google-cloud/#preemption) workers, and [snippets](advanced/customization/#container-linux) customization @@ -53,7 +53,7 @@ Define a Kubernetes cluster by using the Terraform module for your chosen platfo ```tf module "yavin" { - source = "git::https://github.com/poseidon/typhoon//google-cloud/fedora-coreos/kubernetes?ref=v1.18.6" + source = "git::https://github.com/poseidon/typhoon//google-cloud/fedora-coreos/kubernetes?ref=v1.18.8" # Google Cloud cluster_name = "yavin" @@ -91,9 +91,9 @@ In 4-8 minutes (varies by platform), the cluster will be ready. This Google Clou $ export KUBECONFIG=/home/user/.kube/configs/yavin-config $ kubectl get nodes NAME ROLES STATUS AGE VERSION -yavin-controller-0.c.example-com.internal Ready 6m v1.18.6 -yavin-worker-jrbf.c.example-com.internal Ready 5m v1.18.6 -yavin-worker-mzdm.c.example-com.internal Ready 5m v1.18.6 +yavin-controller-0.c.example-com.internal Ready 6m v1.18.8 +yavin-worker-jrbf.c.example-com.internal Ready 5m v1.18.8 +yavin-worker-mzdm.c.example-com.internal Ready 5m v1.18.8 ``` List the pods. diff --git a/docs/topics/maintenance.md b/docs/topics/maintenance.md index d165a76d8..0c782d2d7 100644 --- a/docs/topics/maintenance.md +++ b/docs/topics/maintenance.md @@ -13,12 +13,12 @@ Typhoon provides tagged releases to allow clusters to be versioned using ordinar ``` module "yavin" { - source = "git::https://github.com/poseidon/typhoon//google-cloud/fedora-coreos/kubernetes?ref=v1.18.6" + source = "git::https://github.com/poseidon/typhoon//google-cloud/fedora-coreos/kubernetes?ref=v1.18.8" ... } module "mercury" { - source = "git::https://github.com/poseidon/typhoon//bare-metal/container-linux/kubernetes?ref=v1.18.6" + source = "git::https://github.com/poseidon/typhoon//bare-metal/container-linux/kubernetes?ref=v1.18.8" ... } ``` @@ -212,7 +212,7 @@ Typhoon modules have been updated for v0.13.x, but retain compatibility with v0. | Typhoon Release | Terraform version | |-------------------|---------------------| | v1.18.7 - ? | v0.12.26+, v0.13.x | -| v1.15.0 - v1.18.6 | v0.12.x | +| v1.15.0 - v1.18.8 | v0.12.x | | v1.10.3 - v1.15.0 | v0.11.x | | v1.9.2 - v1.10.2 | v0.10.4+ or v0.11.x | | v1.7.3 - v1.9.1 | v0.10.x | @@ -234,16 +234,16 @@ provider "ct" { } ``` -Next, create Typhoon clusters using the `ref` that introduced Terraform v0.13 forward compatibility (`5e70d7e2c8c71c37d9a0896a0945188f009011f9`) or later. You will see a compatibility warning. Use blue/green cluster replacement to shift to these new clusters, then eliminate older clusters. +Next, create Typhoon clusters using the `ref` that introduced Terraform v0.13 forward compatibility (`v1.18.8`) or later. You will see a compatibility warning. Use blue/green cluster replacement to shift to these new clusters, then eliminate older clusters. ``` module "nemo" { - source = "git::https://github.com/poseidon/typhoon//digital-ocean/fedora-coreos/kubernetes?ref=5e70d7e2c8c71c37d9a0896a0945188f009011f9" + source = "git::https://github.com/poseidon/typhoon//digital-ocean/fedora-coreos/kubernetes?ref=v1.18.8" ... } ``` -Install Terraform v0.13. Once all clusters in a workspace are on `5e70d7e2c8c71c37d9a0896a0945188f009011f9` or above, you are ready to start using Terraform v0.13. +Install Terraform v0.13. Once all clusters in a workspace are on `v1.18.8` or above, you are ready to start using Terraform v0.13. ``` terraform version diff --git a/google-cloud/container-linux/kubernetes/README.md b/google-cloud/container-linux/kubernetes/README.md index f5d3fbe60..0aec57ecc 100644 --- a/google-cloud/container-linux/kubernetes/README.md +++ b/google-cloud/container-linux/kubernetes/README.md @@ -11,7 +11,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features -* Kubernetes v1.18.6 (upstream) +* Kubernetes v1.18.8 (upstream) * Single or multi-master, [Calico](https://www.projectcalico.org/) or [Cilium](https://github.com/cilium/cilium) or [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/) * Advanced features like [worker pools](https://typhoon.psdn.io/advanced/worker-pools/), [preemptible](https://typhoon.psdn.io/cl/google-cloud/#preemption) workers, and [snippets](https://typhoon.psdn.io/advanced/customization/#container-linux) customization diff --git a/google-cloud/container-linux/kubernetes/bootstrap.tf b/google-cloud/container-linux/kubernetes/bootstrap.tf index 7db6057c2..e5dc239c6 100644 --- a/google-cloud/container-linux/kubernetes/bootstrap.tf +++ b/google-cloud/container-linux/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=60540868e070ea95da05fc5a6a5c9a55ce68c58d" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=8ef2fe7c992a8c15d696bd3e3a97be713b025e64" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/google-cloud/container-linux/kubernetes/cl/controller.yaml b/google-cloud/container-linux/kubernetes/cl/controller.yaml index 176529490..a37a75f6f 100644 --- a/google-cloud/container-linux/kubernetes/cl/controller.yaml +++ b/google-cloud/container-linux/kubernetes/cl/controller.yaml @@ -52,7 +52,7 @@ systemd: Description=Kubelet Wants=rpc-statd.service [Service] - Environment=KUBELET_IMAGE=docker://quay.io/poseidon/kubelet:v1.18.6 + Environment=KUBELET_IMAGE=docker://quay.io/poseidon/kubelet:v1.18.8 ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests ExecStartPre=/bin/mkdir -p /opt/cni/bin @@ -132,7 +132,7 @@ systemd: --volume script,kind=host,source=/opt/bootstrap/apply \ --mount volume=script,target=/apply \ --insecure-options=image \ - docker://quay.io/poseidon/kubelet:v1.18.6 \ + docker://quay.io/poseidon/kubelet:v1.18.8 \ --net=host \ --dns=host \ --exec=/apply diff --git a/google-cloud/container-linux/kubernetes/workers/cl/worker.yaml b/google-cloud/container-linux/kubernetes/workers/cl/worker.yaml index 41e375539..9137924fa 100644 --- a/google-cloud/container-linux/kubernetes/workers/cl/worker.yaml +++ b/google-cloud/container-linux/kubernetes/workers/cl/worker.yaml @@ -25,7 +25,7 @@ systemd: Description=Kubelet Wants=rpc-statd.service [Service] - Environment=KUBELET_IMAGE=docker://quay.io/poseidon/kubelet:v1.18.6 + Environment=KUBELET_IMAGE=docker://quay.io/poseidon/kubelet:v1.18.8 ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests ExecStartPre=/bin/mkdir -p /opt/cni/bin @@ -127,7 +127,7 @@ storage: --volume config,kind=host,source=/etc/kubernetes \ --mount volume=config,target=/etc/kubernetes \ --insecure-options=image \ - docker://quay.io/poseidon/kubelet:v1.18.6 \ + docker://quay.io/poseidon/kubelet:v1.18.8 \ --net=host \ --dns=host \ --exec=/usr/local/bin/kubectl -- --kubeconfig=/etc/kubernetes/kubeconfig delete node $(hostname) diff --git a/google-cloud/fedora-coreos/kubernetes/README.md b/google-cloud/fedora-coreos/kubernetes/README.md index da99d075e..d1bfae76a 100644 --- a/google-cloud/fedora-coreos/kubernetes/README.md +++ b/google-cloud/fedora-coreos/kubernetes/README.md @@ -11,7 +11,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features -* Kubernetes v1.18.6 (upstream) +* Kubernetes v1.18.8 (upstream) * Single or multi-master, [Calico](https://www.projectcalico.org/) or [Cilium](https://github.com/cilium/cilium) or [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/), SELinux enforcing * Advanced features like [worker pools](https://typhoon.psdn.io/advanced/worker-pools/), [preemptible](https://typhoon.psdn.io/fedora-coreos/google-cloud/#preemption) workers, and [snippets](https://typhoon.psdn.io/advanced/customization/) customization diff --git a/google-cloud/fedora-coreos/kubernetes/bootstrap.tf b/google-cloud/fedora-coreos/kubernetes/bootstrap.tf index 8ed4d7c70..c0bea1cd3 100644 --- a/google-cloud/fedora-coreos/kubernetes/bootstrap.tf +++ b/google-cloud/fedora-coreos/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=60540868e070ea95da05fc5a6a5c9a55ce68c58d" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=8ef2fe7c992a8c15d696bd3e3a97be713b025e64" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/google-cloud/fedora-coreos/kubernetes/fcc/controller.yaml b/google-cloud/fedora-coreos/kubernetes/fcc/controller.yaml index b9fdcef24..5dbf6a4ee 100644 --- a/google-cloud/fedora-coreos/kubernetes/fcc/controller.yaml +++ b/google-cloud/fedora-coreos/kubernetes/fcc/controller.yaml @@ -54,7 +54,7 @@ systemd: Description=Kubelet (System Container) Wants=rpc-statd.service [Service] - Environment=KUBELET_IMAGE=quay.io/poseidon/kubelet:v1.18.6 + Environment=KUBELET_IMAGE=quay.io/poseidon/kubelet:v1.18.8 ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests ExecStartPre=/bin/mkdir -p /opt/cni/bin @@ -123,7 +123,7 @@ systemd: --volume /opt/bootstrap/assets:/assets:ro,Z \ --volume /opt/bootstrap/apply:/apply:ro,Z \ --entrypoint=/apply \ - quay.io/poseidon/kubelet:v1.18.6 + quay.io/poseidon/kubelet:v1.18.8 ExecStartPost=/bin/touch /opt/bootstrap/bootstrap.done ExecStartPost=-/usr/bin/podman stop bootstrap storage: diff --git a/google-cloud/fedora-coreos/kubernetes/workers/fcc/worker.yaml b/google-cloud/fedora-coreos/kubernetes/workers/fcc/worker.yaml index 5db86926c..efab8cde3 100644 --- a/google-cloud/fedora-coreos/kubernetes/workers/fcc/worker.yaml +++ b/google-cloud/fedora-coreos/kubernetes/workers/fcc/worker.yaml @@ -24,7 +24,7 @@ systemd: Description=Kubelet (System Container) Wants=rpc-statd.service [Service] - Environment=KUBELET_IMAGE=quay.io/poseidon/kubelet:v1.18.6 + Environment=KUBELET_IMAGE=quay.io/poseidon/kubelet:v1.18.8 ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests ExecStartPre=/bin/mkdir -p /opt/cni/bin @@ -88,7 +88,7 @@ systemd: Type=oneshot RemainAfterExit=true ExecStart=/bin/true - ExecStop=/bin/bash -c '/usr/bin/podman run --volume /etc/kubernetes:/etc/kubernetes:ro,z --entrypoint /usr/local/bin/kubectl quay.io/poseidon/kubelet:v1.18.6 --kubeconfig=/etc/kubernetes/kubeconfig delete node $HOSTNAME' + ExecStop=/bin/bash -c '/usr/bin/podman run --volume /etc/kubernetes:/etc/kubernetes:ro,z --entrypoint /usr/local/bin/kubectl quay.io/poseidon/kubelet:v1.18.8 --kubeconfig=/etc/kubernetes/kubeconfig delete node $HOSTNAME' [Install] WantedBy=multi-user.target storage: From 9a07f1d30b4b0a0fb2580b43620af3be0cf4d71a Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Fri, 14 Aug 2020 10:02:16 -0700 Subject: [PATCH 517/523] Update recommended Terraform provider versions * Sync Terraform provider plugin versions to those used internally * Update mkdocs-material from v5.5.1 to v5.5.6 * Fix minor details in docs --- CHANGES.md | 2 +- docs/fedora-coreos/aws.md | 2 +- docs/fedora-coreos/azure.md | 2 +- docs/fedora-coreos/google-cloud.md | 2 +- docs/flatcar-linux/aws.md | 2 +- docs/flatcar-linux/azure.md | 2 +- docs/flatcar-linux/google-cloud.md | 2 +- docs/topics/maintenance.md | 7 +++---- requirements.txt | 2 +- 9 files changed, 11 insertions(+), 12 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 673546bca..3b7952e25 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -8,7 +8,7 @@ Notable changes between versions. * Kubernetes [v1.18.8](https://github.com/kubernetes/kubernetes/blob/master/CHANGELOG/CHANGELOG-1.18.md#v1188) * Migrate from Terraform v0.12.x to v0.13.x ([#804](https://github.com/poseidon/typhoon/pull/804)) (**action required**) - * Recommend Terraform v0.13.x ([migration guide](https://github.com/poseidon/typhoon/blob/5e70d7e2c8c71c37d9a0896a0945188f009011f9/docs/topics/maintenance.md#terraform-versions)) + * Recommend Terraform v0.13.x ([migration guide](https://typhoon.psdn.io/topics/maintenance/#terraform-versions)) * Support automatic install of poseidon's provider plugins ([poseidon/ct](https://registry.terraform.io/providers/poseidon/ct/latest), [poseidon/matchbox](https://registry.terraform.io/providers/poseidon/matchbox/latest)) * Require Terraform v0.12.26+ (migration compatibility) * Require `terraform-provider-ct` v0.6.1 diff --git a/docs/fedora-coreos/aws.md b/docs/fedora-coreos/aws.md index cd2e9f336..cff2cac7e 100644 --- a/docs/fedora-coreos/aws.md +++ b/docs/fedora-coreos/aws.md @@ -55,7 +55,7 @@ terraform { } aws = { source = "hashicorp/aws" - version = "3.1.0" + version = "3.2.0" } } } diff --git a/docs/fedora-coreos/azure.md b/docs/fedora-coreos/azure.md index 40c6bd41c..0e091ea8c 100644 --- a/docs/fedora-coreos/azure.md +++ b/docs/fedora-coreos/azure.md @@ -52,7 +52,7 @@ terraform { } azurerm = { source = "hashicorp/azurerm" - version = "2.22.0" + version = "2.23.0" } } } diff --git a/docs/fedora-coreos/google-cloud.md b/docs/fedora-coreos/google-cloud.md index 21e08afe9..52dbf7a06 100644 --- a/docs/fedora-coreos/google-cloud.md +++ b/docs/fedora-coreos/google-cloud.md @@ -56,7 +56,7 @@ terraform { } google = { source = "hashicorp/google" - version = "3.33.0" + version = "3.34.0" } } } diff --git a/docs/flatcar-linux/aws.md b/docs/flatcar-linux/aws.md index 95cd96033..1c371b9b2 100644 --- a/docs/flatcar-linux/aws.md +++ b/docs/flatcar-linux/aws.md @@ -55,7 +55,7 @@ terraform { } aws = { source = "hashicorp/aws" - version = "3.1.0" + version = "3.2.0" } } } diff --git a/docs/flatcar-linux/azure.md b/docs/flatcar-linux/azure.md index 01b07fd34..cc377dc88 100644 --- a/docs/flatcar-linux/azure.md +++ b/docs/flatcar-linux/azure.md @@ -52,7 +52,7 @@ terraform { } azurerm = { source = "hashicorp/azurerm" - version = "2.22.0" + version = "2.23.0" } } } diff --git a/docs/flatcar-linux/google-cloud.md b/docs/flatcar-linux/google-cloud.md index e800e00d7..9f3a97c14 100644 --- a/docs/flatcar-linux/google-cloud.md +++ b/docs/flatcar-linux/google-cloud.md @@ -56,7 +56,7 @@ terraform { } google = { source = "hashicorp/google" - version = "3.33.0" + version = "3.34.0" } } } diff --git a/docs/topics/maintenance.md b/docs/topics/maintenance.md index 0c782d2d7..6b6547993 100644 --- a/docs/topics/maintenance.md +++ b/docs/topics/maintenance.md @@ -154,8 +154,7 @@ $ tree ~/.terraform.d/ Update the version of the `ct` plugin in each Terraform working directory. Typhoon clusters managed in the working directory **must** be v1.12.2 or higher. -``` -# providers.tf +```tf provider "ct" { version = "0.6.1" } @@ -211,7 +210,7 @@ Typhoon modules have been updated for v0.13.x, but retain compatibility with v0. | Typhoon Release | Terraform version | |-------------------|---------------------| -| v1.18.7 - ? | v0.12.26+, v0.13.x | +| v1.18.8 - ? | v0.12.26+, v0.13.x | | v1.15.0 - v1.18.8 | v0.12.x | | v1.10.3 - v1.15.0 | v0.11.x | | v1.9.2 - v1.10.2 | v0.10.4+ or v0.11.x | @@ -228,7 +227,7 @@ An existing Terraform workspace may already manage earlier Typhoon clusters crea First, upgrade `terraform-provider-ct` to v0.6.1 following the [guide](#upgrade-terraform-provider-ct) above. As usual, read about how `apply` affects existing cluster nodes when `ct` is upgraded. But `terraform-provider-ct` v0.6.1 is compatible with both Terraform v0.12 and v0.13, so do this first. -``` +```tf provider "ct" { version = "0.6.1" } diff --git a/requirements.txt b/requirements.txt index 3548568da..5afa3a9a8 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ mkdocs==1.1.2 -mkdocs-material==5.5.1 +mkdocs-material==5.5.6 pygments==2.6.1 pymdown-extensions==7.1.0 From e39ffcd4e6c0e044c40ed24310dff13baf4c7994 Mon Sep 17 00:00:00 2001 From: Bo Huang Date: Tue, 18 Aug 2020 09:16:03 -0700 Subject: [PATCH 518/523] try relabeling /etc/kubernetes/bootstrap-secrets by explicitly mounting to kubelet --- aws/fedora-coreos/kubernetes/fcc/controller.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/aws/fedora-coreos/kubernetes/fcc/controller.yaml b/aws/fedora-coreos/kubernetes/fcc/controller.yaml index 2072aa302..97125c968 100644 --- a/aws/fedora-coreos/kubernetes/fcc/controller.yaml +++ b/aws/fedora-coreos/kubernetes/fcc/controller.yaml @@ -70,6 +70,7 @@ systemd: --pid host \ --network host \ --volume /etc/kubernetes:/etc/kubernetes:ro,z \ + --volume /etc/kubernetes/bootstrap-secrets:/etc/kubernetes/secrets:ro,z \ --volume /usr/lib/os-release:/etc/os-release:ro \ --volume /etc/ssl/certs:/etc/ssl/certs:ro \ --volume /lib/modules:/lib/modules:ro \ From 4b478f4814d5197db285fd9660897fe3daad77e5 Mon Sep 17 00:00:00 2001 From: Bo Huang Date: Tue, 18 Aug 2020 12:23:15 -0700 Subject: [PATCH 519/523] relabeling does not need explicitly mounting to kubelet --- aws/fedora-coreos/kubernetes/fcc/controller.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aws/fedora-coreos/kubernetes/fcc/controller.yaml b/aws/fedora-coreos/kubernetes/fcc/controller.yaml index 97125c968..15545440b 100644 --- a/aws/fedora-coreos/kubernetes/fcc/controller.yaml +++ b/aws/fedora-coreos/kubernetes/fcc/controller.yaml @@ -70,7 +70,6 @@ systemd: --pid host \ --network host \ --volume /etc/kubernetes:/etc/kubernetes:ro,z \ - --volume /etc/kubernetes/bootstrap-secrets:/etc/kubernetes/secrets:ro,z \ --volume /usr/lib/os-release:/etc/os-release:ro \ --volume /etc/ssl/certs:/etc/ssl/certs:ro \ --volume /lib/modules:/lib/modules:ro \ @@ -166,6 +165,7 @@ storage: mv manifests /opt/bootstrap/assets/manifests mv manifests-networking/* /opt/bootstrap/assets/manifests/ rm -rf assets auth static-manifests tls manifests-networking + chcon -R -u system_u /etc/kubernetes/bootstrap-secrets - path: /opt/bootstrap/apply mode: 0544 contents: From b91b9937839b60cf5d1fd6e6860a2aa16397d052 Mon Sep 17 00:00:00 2001 From: Bo Huang Date: Wed, 19 Aug 2020 19:01:28 -0700 Subject: [PATCH 520/523] need to update the type label of bootstrap-secret in the newest typhoon --- aws/fedora-coreos/kubernetes/fcc/controller.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aws/fedora-coreos/kubernetes/fcc/controller.yaml b/aws/fedora-coreos/kubernetes/fcc/controller.yaml index 167c794f8..fee6d957e 100644 --- a/aws/fedora-coreos/kubernetes/fcc/controller.yaml +++ b/aws/fedora-coreos/kubernetes/fcc/controller.yaml @@ -165,7 +165,7 @@ storage: mv manifests /opt/bootstrap/assets/manifests mv manifests-networking/* /opt/bootstrap/assets/manifests/ rm -rf assets auth static-manifests tls manifests-networking - chcon -R -u system_u /etc/kubernetes/bootstrap-secrets + chcon -R -u system_u -t container_file_t /etc/kubernetes/bootstrap-secrets - path: /opt/bootstrap/apply mode: 0544 contents: From 00244cc9c971dbc70923288c5c5cfee68089e0b4 Mon Sep 17 00:00:00 2001 From: Bo Huang Date: Wed, 19 Aug 2020 19:27:28 -0700 Subject: [PATCH 521/523] update terraform-render-bootstrap with latest upstream --- aws/container-linux/kubernetes/bootstrap.tf | 2 +- aws/fedora-coreos/kubernetes/bootstrap.tf | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/aws/container-linux/kubernetes/bootstrap.tf b/aws/container-linux/kubernetes/bootstrap.tf index 3583fa963..5e39beafa 100644 --- a/aws/container-linux/kubernetes/bootstrap.tf +++ b/aws/container-linux/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/takescoop/terraform-render-bootstrap.git?ref=d1366357e8dbfb2836d7bc7fec0ba69b4f97928b" + source = "git::https://github.com/takescoop/terraform-render-bootstrap.git?ref=d3132edba9f84ad210376f0632d435c08d6ce3e4" cluster_name = var.cluster_name api_servers = concat(list(format("%s.%s", var.cluster_name, var.dns_zone)), var.apiserver_aliases) diff --git a/aws/fedora-coreos/kubernetes/bootstrap.tf b/aws/fedora-coreos/kubernetes/bootstrap.tf index b346ef5dc..f1d42504c 100644 --- a/aws/fedora-coreos/kubernetes/bootstrap.tf +++ b/aws/fedora-coreos/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/takescoop/terraform-render-bootstrap.git?ref=d1366357e8dbfb2836d7bc7fec0ba69b4f97928b" + source = "git::https://github.com/takescoop/terraform-render-bootstrap.git?ref=d3132edba9f84ad210376f0632d435c08d6ce3e4" cluster_name = var.cluster_name api_servers = concat(list(format("%s.%s", var.cluster_name, var.dns_zone)), var.apiserver_aliases) From 62b91bea6ca3baee532a88230fe64ce03ac77ca8 Mon Sep 17 00:00:00 2001 From: Bo Huang Date: Thu, 20 Aug 2020 13:20:20 -0700 Subject: [PATCH 522/523] rm unnecessary volume mounts on etcd --- aws/fedora-coreos/kubernetes/fcc/controller.yaml | 2 -- 1 file changed, 2 deletions(-) diff --git a/aws/fedora-coreos/kubernetes/fcc/controller.yaml b/aws/fedora-coreos/kubernetes/fcc/controller.yaml index fee6d957e..7fe991b2f 100644 --- a/aws/fedora-coreos/kubernetes/fcc/controller.yaml +++ b/aws/fedora-coreos/kubernetes/fcc/controller.yaml @@ -28,8 +28,6 @@ systemd: --network host \ --volume /var/lib/etcd:/var/lib/etcd:rw,Z \ --volume /etc/ssl/etcd:/etc/ssl/certs:ro,Z \ - --volume /etc/pki/tls/certs:/etc/pki/tls/certs:ro \ - --volume /etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro \ quay.io/coreos/etcd:v3.4.10 ExecStop=/usr/bin/podman stop etcd [Install] From bfc03e195d538fc779505259e74922aa34f7b2d4 Mon Sep 17 00:00:00 2001 From: Bo Huang Date: Thu, 20 Aug 2020 21:04:13 -0700 Subject: [PATCH 523/523] rm output/ --- .../bootstrap-apiserver.yaml | 59 ------- .../bootstrap-controller-manager.yaml | 36 ---- .../bootstrap-scheduler.yaml | 23 --- .../bgpconfigurations-crd.yaml | 13 -- output/manifests-networking/bgppeers-crd.yaml | 13 -- .../calico-cluster-role-binding.yaml | 12 -- .../calico-cluster-role.yaml | 68 -------- .../manifests-networking/calico-config.yaml | 39 ----- .../calico-service-account.yaml | 5 - output/manifests-networking/calico.yaml | 146 ----------------- .../clusterinformations-crd.yaml | 13 -- .../felixconfigurations-crd.yaml | 13 -- .../globalnetworkpolicies-crd.yaml | 13 -- .../globalnetworksets-crd.yaml | 13 -- output/manifests-networking/ippools-crd.yaml | 13 -- .../networkpolicies-crd.yaml | 13 -- output/manifests/kube-apiserver-secret.yaml | 14 -- output/manifests/kube-apiserver.yaml | 88 ---------- .../kube-controller-manager-disruption.yaml | 11 -- .../kube-controller-manager-role-binding.yaml | 12 -- .../manifests/kube-controller-manager-sa.yaml | 5 - .../kube-controller-manager-secret.yaml | 9 - output/manifests/kube-controller-manager.yaml | 82 ---------- output/manifests/kube-dns-deployment.yaml | 154 ------------------ output/manifests/kube-dns-sa.yaml | 5 - output/manifests/kube-dns-svc.yaml | 20 --- output/manifests/kube-proxy-role-binding.yaml | 12 -- output/manifests/kube-proxy-sa.yaml | 5 - output/manifests/kube-proxy.yaml | 67 -------- .../manifests/kube-scheduler-disruption.yaml | 11 -- output/manifests/kube-scheduler.yaml | 58 ------- .../kube-system-rbac-role-binding.yaml | 12 -- output/manifests/kubeconfig-in-cluster.yaml | 22 --- .../pod-checkpointer-role-binding.yaml | 13 -- output/manifests/pod-checkpointer-role.yaml | 12 -- output/manifests/pod-checkpointer-sa.yaml | 5 - output/manifests/pod-checkpointer.yaml | 72 -------- 37 files changed, 1181 deletions(-) delete mode 100644 output/bootstrap-manifests/bootstrap-apiserver.yaml delete mode 100644 output/bootstrap-manifests/bootstrap-controller-manager.yaml delete mode 100644 output/bootstrap-manifests/bootstrap-scheduler.yaml delete mode 100644 output/manifests-networking/bgpconfigurations-crd.yaml delete mode 100644 output/manifests-networking/bgppeers-crd.yaml delete mode 100644 output/manifests-networking/calico-cluster-role-binding.yaml delete mode 100644 output/manifests-networking/calico-cluster-role.yaml delete mode 100644 output/manifests-networking/calico-config.yaml delete mode 100644 output/manifests-networking/calico-service-account.yaml delete mode 100644 output/manifests-networking/calico.yaml delete mode 100644 output/manifests-networking/clusterinformations-crd.yaml delete mode 100644 output/manifests-networking/felixconfigurations-crd.yaml delete mode 100644 output/manifests-networking/globalnetworkpolicies-crd.yaml delete mode 100644 output/manifests-networking/globalnetworksets-crd.yaml delete mode 100644 output/manifests-networking/ippools-crd.yaml delete mode 100644 output/manifests-networking/networkpolicies-crd.yaml delete mode 100644 output/manifests/kube-apiserver-secret.yaml delete mode 100644 output/manifests/kube-apiserver.yaml delete mode 100644 output/manifests/kube-controller-manager-disruption.yaml delete mode 100644 output/manifests/kube-controller-manager-role-binding.yaml delete mode 100644 output/manifests/kube-controller-manager-sa.yaml delete mode 100644 output/manifests/kube-controller-manager-secret.yaml delete mode 100644 output/manifests/kube-controller-manager.yaml delete mode 100644 output/manifests/kube-dns-deployment.yaml delete mode 100644 output/manifests/kube-dns-sa.yaml delete mode 100644 output/manifests/kube-dns-svc.yaml delete mode 100644 output/manifests/kube-proxy-role-binding.yaml delete mode 100644 output/manifests/kube-proxy-sa.yaml delete mode 100644 output/manifests/kube-proxy.yaml delete mode 100644 output/manifests/kube-scheduler-disruption.yaml delete mode 100644 output/manifests/kube-scheduler.yaml delete mode 100644 output/manifests/kube-system-rbac-role-binding.yaml delete mode 100644 output/manifests/kubeconfig-in-cluster.yaml delete mode 100644 output/manifests/pod-checkpointer-role-binding.yaml delete mode 100644 output/manifests/pod-checkpointer-role.yaml delete mode 100644 output/manifests/pod-checkpointer-sa.yaml delete mode 100644 output/manifests/pod-checkpointer.yaml diff --git a/output/bootstrap-manifests/bootstrap-apiserver.yaml b/output/bootstrap-manifests/bootstrap-apiserver.yaml deleted file mode 100644 index 422c7b529..000000000 --- a/output/bootstrap-manifests/bootstrap-apiserver.yaml +++ /dev/null @@ -1,59 +0,0 @@ -apiVersion: v1 -kind: Pod -metadata: - name: bootstrap-kube-apiserver - namespace: kube-system -spec: - containers: - - name: kube-apiserver - image: gcr.io/google_containers/hyperkube:v1.9.3 - command: - - /hyperkube - - apiserver - - --admission-control=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,ValidatingAdmissionWebhook,ResourceQuota,DefaultTolerationSeconds,MutatingAdmissionWebhook - - --advertise-address=$(POD_IP) - - --allow-privileged=true - - --authorization-mode=RBAC - - --bind-address=0.0.0.0 - - --client-ca-file=/etc/kubernetes/secrets/ca.crt - - --etcd-cafile=/etc/kubernetes/secrets/etcd-client-ca.crt - - --etcd-certfile=/etc/kubernetes/secrets/etcd-client.crt - - --etcd-keyfile=/etc/kubernetes/secrets/etcd-client.key - - --etcd-servers=https://bastion-test-etcd0.k8s-playground.takescoop.com:2379,https://bastion-test-etcd1.k8s-playground.takescoop.com:2379 - - --insecure-port=0 - - --kubelet-client-certificate=/etc/kubernetes/secrets/apiserver.crt - - --kubelet-client-key=/etc/kubernetes/secrets/apiserver.key - - --secure-port=443 - - --service-account-key-file=/etc/kubernetes/secrets/service-account.pub - - --service-cluster-ip-range=10.3.0.0/16 - - --cloud-provider= - - --storage-backend=etcd3 - - --tls-ca-file=/etc/kubernetes/secrets/ca.crt - - --tls-cert-file=/etc/kubernetes/secrets/apiserver.crt - - --tls-private-key-file=/etc/kubernetes/secrets/apiserver.key - env: - - name: POD_IP - valueFrom: - fieldRef: - fieldPath: status.podIP - volumeMounts: - - mountPath: /etc/ssl/certs - name: ssl-certs-host - readOnly: true - - mountPath: /etc/kubernetes/secrets - name: secrets - readOnly: true - - mountPath: /var/lock - name: var-lock - readOnly: false - hostNetwork: true - volumes: - - name: secrets - hostPath: - path: /etc/kubernetes/bootstrap-secrets - - name: ssl-certs-host - hostPath: - path: /usr/share/ca-certificates - - name: var-lock - hostPath: - path: /var/lock diff --git a/output/bootstrap-manifests/bootstrap-controller-manager.yaml b/output/bootstrap-manifests/bootstrap-controller-manager.yaml deleted file mode 100644 index 705e75528..000000000 --- a/output/bootstrap-manifests/bootstrap-controller-manager.yaml +++ /dev/null @@ -1,36 +0,0 @@ -apiVersion: v1 -kind: Pod -metadata: - name: bootstrap-kube-controller-manager - namespace: kube-system -spec: - containers: - - name: kube-controller-manager - image: gcr.io/google_containers/hyperkube:v1.9.3 - command: - - ./hyperkube - - controller-manager - - --allocate-node-cidrs=true - - --cluster-cidr=10.2.0.0/16 - - --service-cluster-ip-range=10.3.0.0/16 - - --cloud-provider= - - --configure-cloud-routes=false - - --kubeconfig=/etc/kubernetes/kubeconfig - - --leader-elect=true - - --root-ca-file=/etc/kubernetes/bootstrap-secrets/ca.crt - - --service-account-private-key-file=/etc/kubernetes/bootstrap-secrets/service-account.key - volumeMounts: - - name: kubernetes - mountPath: /etc/kubernetes - readOnly: true - - name: ssl-host - mountPath: /etc/ssl/certs - readOnly: true - hostNetwork: true - volumes: - - name: kubernetes - hostPath: - path: /etc/kubernetes - - name: ssl-host - hostPath: - path: /usr/share/ca-certificates diff --git a/output/bootstrap-manifests/bootstrap-scheduler.yaml b/output/bootstrap-manifests/bootstrap-scheduler.yaml deleted file mode 100644 index 75481b1b7..000000000 --- a/output/bootstrap-manifests/bootstrap-scheduler.yaml +++ /dev/null @@ -1,23 +0,0 @@ -apiVersion: v1 -kind: Pod -metadata: - name: bootstrap-kube-scheduler - namespace: kube-system -spec: - containers: - - name: kube-scheduler - image: gcr.io/google_containers/hyperkube:v1.9.3 - command: - - ./hyperkube - - scheduler - - --kubeconfig=/etc/kubernetes/kubeconfig - - --leader-elect=true - volumeMounts: - - name: kubernetes - mountPath: /etc/kubernetes - readOnly: true - hostNetwork: true - volumes: - - name: kubernetes - hostPath: - path: /etc/kubernetes diff --git a/output/manifests-networking/bgpconfigurations-crd.yaml b/output/manifests-networking/bgpconfigurations-crd.yaml deleted file mode 100644 index c48ff4886..000000000 --- a/output/manifests-networking/bgpconfigurations-crd.yaml +++ /dev/null @@ -1,13 +0,0 @@ -apiVersion: apiextensions.k8s.io/v1beta1 -description: Calico BGP Configuration -kind: CustomResourceDefinition -metadata: - name: bgpconfigurations.crd.projectcalico.org -spec: - scope: Cluster - group: crd.projectcalico.org - version: v1 - names: - kind: BGPConfiguration - plural: bgpconfigurations - singular: bgpconfiguration diff --git a/output/manifests-networking/bgppeers-crd.yaml b/output/manifests-networking/bgppeers-crd.yaml deleted file mode 100644 index d10f528ae..000000000 --- a/output/manifests-networking/bgppeers-crd.yaml +++ /dev/null @@ -1,13 +0,0 @@ -apiVersion: apiextensions.k8s.io/v1beta1 -description: Calico BGP Peers -kind: CustomResourceDefinition -metadata: - name: bgppeers.crd.projectcalico.org -spec: - scope: Cluster - group: crd.projectcalico.org - version: v1 - names: - kind: BGPPeer - plural: bgppeers - singular: bgppeer diff --git a/output/manifests-networking/calico-cluster-role-binding.yaml b/output/manifests-networking/calico-cluster-role-binding.yaml deleted file mode 100644 index f76449264..000000000 --- a/output/manifests-networking/calico-cluster-role-binding.yaml +++ /dev/null @@ -1,12 +0,0 @@ -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: calico-node -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: calico-node -subjects: -- kind: ServiceAccount - name: calico-node - namespace: kube-system diff --git a/output/manifests-networking/calico-cluster-role.yaml b/output/manifests-networking/calico-cluster-role.yaml deleted file mode 100644 index 2d317b500..000000000 --- a/output/manifests-networking/calico-cluster-role.yaml +++ /dev/null @@ -1,68 +0,0 @@ -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: calico-node -rules: - - apiGroups: [""] - resources: - - namespaces - verbs: - - get - - list - - watch - - apiGroups: [""] - resources: - - pods/status - verbs: - - update - - apiGroups: [""] - resources: - - pods - verbs: - - get - - list - - watch - - patch - - apiGroups: [""] - resources: - - services - verbs: - - get - - apiGroups: [""] - resources: - - endpoints - verbs: - - get - - apiGroups: [""] - resources: - - nodes - verbs: - - get - - list - - update - - watch - - apiGroups: ["extensions"] - resources: - - networkpolicies - verbs: - - get - - list - - watch - - apiGroups: ["crd.projectcalico.org"] - resources: - - globalfelixconfigs - - felixconfigurations - - bgppeers - - globalbgpconfigs - - bgpconfigurations - - ippools - - globalnetworkpolicies - - globalnetworksets - - networkpolicies - - clusterinformations - verbs: - - create - - get - - list - - update - - watch diff --git a/output/manifests-networking/calico-config.yaml b/output/manifests-networking/calico-config.yaml deleted file mode 100644 index 66be5c2af..000000000 --- a/output/manifests-networking/calico-config.yaml +++ /dev/null @@ -1,39 +0,0 @@ -apiVersion: v1 -kind: ConfigMap -metadata: - name: calico-config - namespace: kube-system -data: - typha_service_name: "none" - # The CNI network configuration to install on each node. - cni_network_config: |- - { - "name": "k8s-pod-network", - "cniVersion": "0.3.1", - "plugins": [ - { - "type": "calico", - "log_level": "info", - "datastore_type": "kubernetes", - "nodename": "__KUBERNETES_NODE_NAME__", - "mtu": 1480, - "ipam": { - "type": "host-local", - "subnet": "usePodCidr" - }, - "policy": { - "type": "k8s", - "k8s_auth_token": "__SERVICEACCOUNT_TOKEN__" - }, - "kubernetes": { - "k8s_api_root": "https://__KUBERNETES_SERVICE_HOST__:__KUBERNETES_SERVICE_PORT__", - "kubeconfig": "__KUBECONFIG_FILEPATH__" - } - }, - { - "type": "portmap", - "snat": true, - "capabilities": {"portMappings": true} - } - ] - } diff --git a/output/manifests-networking/calico-service-account.yaml b/output/manifests-networking/calico-service-account.yaml deleted file mode 100644 index f16b4b0e0..000000000 --- a/output/manifests-networking/calico-service-account.yaml +++ /dev/null @@ -1,5 +0,0 @@ -apiVersion: v1 -kind: ServiceAccount -metadata: - name: calico-node - namespace: kube-system diff --git a/output/manifests-networking/calico.yaml b/output/manifests-networking/calico.yaml deleted file mode 100644 index 50b45c38f..000000000 --- a/output/manifests-networking/calico.yaml +++ /dev/null @@ -1,146 +0,0 @@ -apiVersion: apps/v1 -kind: DaemonSet -metadata: - name: calico-node - namespace: kube-system - labels: - k8s-app: calico-node -spec: - selector: - matchLabels: - k8s-app: calico-node - updateStrategy: - type: RollingUpdate - rollingUpdate: - maxUnavailable: 1 - template: - metadata: - labels: - k8s-app: calico-node - spec: - hostNetwork: true - serviceAccountName: calico-node - tolerations: - - effect: NoSchedule - operator: Exists - - effect: NoExecute - operator: Exists - containers: - - name: calico-node - image: quay.io/calico/node:v3.0.2 - env: - # Use Kubernetes API as the backing datastore. - - name: DATASTORE_TYPE - value: "kubernetes" - # Enable felix info logging. - - name: FELIX_LOGSEVERITYSCREEN - value: "info" - # Cluster type to identify the deployment type - - name: CLUSTER_TYPE - value: "k8s,bgp" - # Disable file logging so `kubectl logs` works. - - name: CALICO_DISABLE_FILE_LOGGING - value: "true" - # Set Felix endpoint to host default action to ACCEPT. - - name: FELIX_DEFAULTENDPOINTTOHOSTACTION - value: "ACCEPT" - # Disable IPV6 on Kubernetes. - - name: FELIX_IPV6SUPPORT - value: "false" - # Set MTU for tunnel device used if ipip is enabled - - name: FELIX_IPINIPMTU - value: "1480" - # Wait for the datastore. - - name: WAIT_FOR_DATASTORE - value: "true" - # The Calico IPv4 pool CIDR (should match `--cluster-cidr`). - - name: CALICO_IPV4POOL_CIDR - value: "10.2.0.0/16" - # Enable IPIP - - name: CALICO_IPV4POOL_IPIP - value: "Always" - # Enable IP-in-IP within Felix. - - name: FELIX_IPINIPENABLED - value: "true" - # Typha support: controlled by the ConfigMap. - - name: FELIX_TYPHAK8SSERVICENAME - valueFrom: - configMapKeyRef: - name: calico-config - key: typha_service_name - # Set node name based on k8s nodeName. - - name: NODENAME - valueFrom: - fieldRef: - fieldPath: spec.nodeName - # Auto-detect the BGP IP address. - - name: IP - value: "autodetect" - - name: FELIX_HEALTHENABLED - value: "true" - securityContext: - privileged: true - resources: - requests: - cpu: 250m - livenessProbe: - httpGet: - path: /liveness - port: 9099 - periodSeconds: 10 - initialDelaySeconds: 10 - failureThreshold: 6 - readinessProbe: - httpGet: - path: /readiness - port: 9099 - periodSeconds: 10 - volumeMounts: - - mountPath: /lib/modules - name: lib-modules - readOnly: true - - mountPath: /var/run/calico - name: var-run-calico - readOnly: false - # Install Calico CNI binaries and CNI network config file on nodes - - name: install-cni - image: quay.io/calico/cni:v2.0.0 - command: ["/install-cni.sh"] - env: - # Name of the CNI config file to create on each node. - - name: CNI_CONF_NAME - value: "10-calico.conflist" - # Contents of the CNI config to create on each node. - - name: CNI_NETWORK_CONFIG - valueFrom: - configMapKeyRef: - name: calico-config - key: cni_network_config - # Set node name based on k8s nodeName - - name: KUBERNETES_NODE_NAME - valueFrom: - fieldRef: - fieldPath: spec.nodeName - - name: CNI_NET_DIR - value: "/etc/kubernetes/cni/net.d" - volumeMounts: - - mountPath: /host/opt/cni/bin - name: cni-bin-dir - - mountPath: /host/etc/cni/net.d - name: cni-net-dir - terminationGracePeriodSeconds: 0 - volumes: - # Used by calico/node - - name: lib-modules - hostPath: - path: /lib/modules - - name: var-run-calico - hostPath: - path: /var/run/calico - # Used by install-cni - - name: cni-bin-dir - hostPath: - path: /opt/cni/bin - - name: cni-net-dir - hostPath: - path: /etc/kubernetes/cni/net.d diff --git a/output/manifests-networking/clusterinformations-crd.yaml b/output/manifests-networking/clusterinformations-crd.yaml deleted file mode 100644 index 3fbc7d8dd..000000000 --- a/output/manifests-networking/clusterinformations-crd.yaml +++ /dev/null @@ -1,13 +0,0 @@ -apiVersion: apiextensions.k8s.io/v1beta1 -description: Calico Cluster Information -kind: CustomResourceDefinition -metadata: - name: clusterinformations.crd.projectcalico.org -spec: - scope: Cluster - group: crd.projectcalico.org - version: v1 - names: - kind: ClusterInformation - plural: clusterinformations - singular: clusterinformation diff --git a/output/manifests-networking/felixconfigurations-crd.yaml b/output/manifests-networking/felixconfigurations-crd.yaml deleted file mode 100644 index e9f1385a0..000000000 --- a/output/manifests-networking/felixconfigurations-crd.yaml +++ /dev/null @@ -1,13 +0,0 @@ -apiVersion: apiextensions.k8s.io/v1beta1 -description: Calico Felix Configuration -kind: CustomResourceDefinition -metadata: - name: felixconfigurations.crd.projectcalico.org -spec: - scope: Cluster - group: crd.projectcalico.org - version: v1 - names: - kind: FelixConfiguration - plural: felixconfigurations - singular: felixconfiguration diff --git a/output/manifests-networking/globalnetworkpolicies-crd.yaml b/output/manifests-networking/globalnetworkpolicies-crd.yaml deleted file mode 100644 index b28cc3544..000000000 --- a/output/manifests-networking/globalnetworkpolicies-crd.yaml +++ /dev/null @@ -1,13 +0,0 @@ -apiVersion: apiextensions.k8s.io/v1beta1 -description: Calico Global Network Policies -kind: CustomResourceDefinition -metadata: - name: globalnetworkpolicies.crd.projectcalico.org -spec: - scope: Cluster - group: crd.projectcalico.org - version: v1 - names: - kind: GlobalNetworkPolicy - plural: globalnetworkpolicies - singular: globalnetworkpolicy diff --git a/output/manifests-networking/globalnetworksets-crd.yaml b/output/manifests-networking/globalnetworksets-crd.yaml deleted file mode 100644 index cef0fe57c..000000000 --- a/output/manifests-networking/globalnetworksets-crd.yaml +++ /dev/null @@ -1,13 +0,0 @@ -apiVersion: apiextensions.k8s.io/v1beta1 -description: Calico Global Network Sets -kind: CustomResourceDefinition -metadata: - name: globalnetworksets.crd.projectcalico.org -spec: - scope: Cluster - group: crd.projectcalico.org - version: v1 - names: - kind: GlobalNetworkSet - plural: globalnetworksets - singular: globalnetworkset diff --git a/output/manifests-networking/ippools-crd.yaml b/output/manifests-networking/ippools-crd.yaml deleted file mode 100644 index 3bb6804b1..000000000 --- a/output/manifests-networking/ippools-crd.yaml +++ /dev/null @@ -1,13 +0,0 @@ -apiVersion: apiextensions.k8s.io/v1beta1 -description: Calico IP Pools -kind: CustomResourceDefinition -metadata: - name: ippools.crd.projectcalico.org -spec: - scope: Cluster - group: crd.projectcalico.org - version: v1 - names: - kind: IPPool - plural: ippools - singular: ippool diff --git a/output/manifests-networking/networkpolicies-crd.yaml b/output/manifests-networking/networkpolicies-crd.yaml deleted file mode 100644 index 4d34ad01b..000000000 --- a/output/manifests-networking/networkpolicies-crd.yaml +++ /dev/null @@ -1,13 +0,0 @@ -apiVersion: apiextensions.k8s.io/v1beta1 -description: Calico Network Policies -kind: CustomResourceDefinition -metadata: - name: networkpolicies.crd.projectcalico.org -spec: - scope: Namespaced - group: crd.projectcalico.org - version: v1 - names: - kind: NetworkPolicy - plural: networkpolicies - singular: networkpolicy diff --git a/output/manifests/kube-apiserver-secret.yaml b/output/manifests/kube-apiserver-secret.yaml deleted file mode 100644 index 81abce2da..000000000 --- a/output/manifests/kube-apiserver-secret.yaml +++ /dev/null @@ -1,14 +0,0 @@ -apiVersion: v1 -kind: Secret -metadata: - name: kube-apiserver - namespace: kube-system -type: Opaque -data: - apiserver.key: LS0tLS1CRUdJTiBSU0EgUFJJVkFURSBLRVktLS0tLQpNSUlFcEFJQkFBS0NBUUVBMUFsTTZKRG9pTk5RT2ZCRXpEOVZVWmpRU1RFZEdLMkpmSFBRclJROGMyaGlMWm5rCmIyTUo5OGsyNVNybWppbWhPdC9FMHZKL1E3enErb3JmekNKd1AvZ2sxMGFQTmFyT1VQaUdSNzZrQU4vWitDTHgKS1J1Tm9EWWpZaGdZaG51bjB1djd3cXRjMEgxY282c05zbnlOTjJBUGhER1VuY2M5cncxUUdCRUN1YVRyQVVqYgorRnJLaTNLQ3RNVmdyemVCNGFUWkdqTE5nTGxOQStxbGVmbXVjcFIvdnh4Rzh5aGxwNU14cmFoLzVyS0ZBellGCnVQM05hamwxN1oxb0JYazFqaUFZS3E3SzkyYVp1aFYvbUo2WWVCbFVzMFRvb3NxNCtFRUFsM0w2ZW1OaWpMVGsKajNhNk5ZRHVRSGxlK2JNaW9Kc3UvQzhGS3I5cmhrekwvV1hMbVFJREFRQUJBb0lCQURXWnNFWnVNMG83V09GOAptbmVqWHZjRWtVcWZUc0twUThNaEo5Ukk5RXNjVFExSUJOWWZqQ3FHUkFsRWdnblgvamo2emkraE80aXRIaDE5CnM3dFB6VjV1WlNuQ1hYdHNsVUVrd2hVcTNSeVhlZXRmTWVWNVlLRHFicUZpZy9pakU3YWZEd0tUL1I5N1FVcmkKZDlEeDZXVGhOS3J2T2FsMDcyUHNFcDR5MXFTRTFVWDBrQUxRWXlVSFhVWlNvWU9Nckl3Y21HNXdteVM2TGZqcwo3UUFoUkdvakF1OGpXUGFEQSsvWDhKSzZsKzBWN3pKZHlBMGVKWUsrQitTLzhQNG5UMmJ1R042OUl4dWJOWTdyClNJMEgzS2RTSTlwaURZRVJCZWJzZVZ4ZTRKbzVkVXVLRUZBR01NME8vSHBRRkE0anNWWVlMUWpRT01hK1g4U2EKTVd3VkVRRUNnWUVBOEZWVFRISGRuQ3BzYmwyZ1ZZaWFpdnFCb1hsa2YzcnZNTWcxVFNxNTNUMXFJVnJRV1hoLwozMkpuZVVCQlBnN2dwY3Z1QURVYnh4alhtclczc2FPR0Z3SGlQOXNacEd5RTJUQStnM2JTc2hldm5jZ2hmakQ4CjcrTURDUHRwa2ppZ0RZNUlyTHlJejlsQ3B4SGVtT28xOXJ6OEJDZk1GOVpBN2kvbzNFWUFmWGtDZ1lFQTRkdkIKakdINEExSjVwSytvVkdoc0NtZzVpYUl1T1d0ZVhZU2RTV1dxcFpCWDg4U1RnR3NSTVRFQjNtNFFzcnBTeFlPUgorR1hMUk43MjJxSnh0eGVLcXYyemJPZGJKUVJGWXZ6UzFoMC9RY3BMeDlDN1RKcW00SGFhRXhvTUpJcjV2MUptCnFGRHJwWXdDaTlVY2NjK0ZGMWM2amdEYXdGbEFTU2V4bllGZDF5RUNnWUVBdGx3SUdMbE5ybkdDVlR3MXJMRTYKa3JvQ0lzUTV2WUZLZlhscytHQ3pKMnl5V3h6TmV4WXo4UXg1OTBjS09reVBxVDVVR1ZReS81K1orWXBwR0NFOApYYmpRTkNQTUVUZEdsb1pFNlB3QVk2SVZYMk84QmtTbHFHQllyVGdYb3h2VVZuVGdNREhlbmRmOCswaFQzelBZClBxQ25tWCtaSFgwMVI0YVM2cEV6VGdFQ2dZQUw0RGcvSDIraERSY2tWN0FzTUFsdVNxaXIyZ0ZBTjZzUWs4YUoKYzNVVG01RmtXZk8vanVHcWluOGtxUGpyek94SlFtL01kZDNJVTBqN21nc01xNG81RDNuOXdmU0M3OFNPUGVrQQpKUzJNVWd6R0J1MnlTM2QyMmdXajkzeW45ejdHbHBpYlJSWCs4R3U1Mm96U1Z5MFNXeDNURmF4cTdNWjZra0crCm5Hekl3UUtCZ1FEdWEyOTg4MVY5ZkxQU2lRM01xcUR2OGgxTHdOSlNVTHVtT1RoMlpIWnRJMjBxZ3pTSWt0dFYKQ0dGbmllc3I1ekJnTUtEM3I2QUFrNWMzUVcrUEk4L0Y0UnRqczBRR3ltZlNIS1JxL2xDMVd3WUJtL2NNWUQzagpJSlNXSG1VdjRjWVBNb1J0VkYrbkJQMDBIS2dTc1BIbDI1MFdtdEZ6VWlvYmk1ZVVCMmszTWc9PQotLS0tLUVORCBSU0EgUFJJVkFURSBLRVktLS0tLQo= - apiserver.crt: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUQzakNDQXNhZ0F3SUJBZ0lSQVBGcVlYbmNrNnlPTUlkbm5laVdzYkF3RFFZSktvWklodmNOQVFFTEJRQXcKSlRFUk1BOEdBMVVFQ2hNSVltOXZkR3QxWW1VeEVEQU9CZ05WQkFNVEIydDFZbVV0WTJFd0hoY05NVGd3TXpFMQpNRFF4TmpFMVdoY05NVGt3TXpFMU1EUXhOakUxV2pBdk1SUXdFZ1lEVlFRS0V3dHJkV0psTFcxaGMzUmxjakVYCk1CVUdBMVVFQXhNT2EzVmlaUzFoY0dselpYSjJaWEl3Z2dFaU1BMEdDU3FHU0liM0RRRUJBUVVBQTRJQkR3QXcKZ2dFS0FvSUJBUURVQ1V6b2tPaUkwMUE1OEVUTVAxVlJtTkJKTVIwWXJZbDhjOUN0RkR4emFHSXRtZVJ2WXduMwp5VGJsS3VhT0thRTYzOFRTOG45RHZPcjZpdC9NSW5BLytDVFhSbzgxcXM1UStJWkh2cVFBMzluNEl2RXBHNDJnCk5pTmlHQmlHZTZmUzYvdkNxMXpRZlZ5anF3MnlmSTAzWUErRU1aU2R4ejJ2RFZBWUVRSzVwT3NCU052NFdzcUwKY29LMHhXQ3ZONEhocE5rYU1zMkF1VTBENnFWNSthNXlsSCsvSEViektHV25rekd0cUgvbXNvVUROZ1c0L2MxcQpPWFh0bldnRmVUV09JQmdxcnNyM1pwbTZGWCtZbnBoNEdWU3pST2lpeXJqNFFRQ1hjdnA2WTJLTXRPU1Bkcm8xCmdPNUFlVjc1c3lLZ215NzhMd1VxdjJ1R1RNdjlaY3VaQWdNQkFBR2pnZjR3Z2Zzd0RnWURWUjBQQVFIL0JBUUQKQWdXZ01CMEdBMVVkSlFRV01CUUdDQ3NHQVFVRkJ3TUJCZ2dyQmdFRkJRY0RBakFNQmdOVkhSTUJBZjhFQWpBQQpNQjhHQTFVZEl3UVlNQmFBRkprNTNjODZoazVRM0ttMVIyN1F2VnJUYjRvR01JR2FCZ05WSFJFRWdaSXdnWStDCktXSmhjM1JwYjI0dGRHVnpkQzVyT0hNdGNHeGhlV2R5YjNWdVpDNTBZV3RsYzJOdmIzQXVZMjl0Z2dwcmRXSmwKY201bGRHVnpnaEpyZFdKbGNtNWxkR1Z6TG1SbFptRjFiSFNDRm10MVltVnlibVYwWlhNdVpHVm1ZWFZzZEM1egpkbU9DSkd0MVltVnlibVYwWlhNdVpHVm1ZWFZzZEM1emRtTXVZMngxYzNSbGNpNXNiMk5oYkljRUNnTUFBVEFOCkJna3Foa2lHOXcwQkFRc0ZBQU9DQVFFQWVrdndDa0x6VWYyaCtHTzg1K2w4MXljRGcvcU4xczR5aU9saE1WdzIKdncxcnVRUHBGY3c1U3loRDVzeXpJQVZFdkhlby8reXRiZHB3azFGclFsS2dwdmo4dS9YTlRDcUliQVVsNDMvZQpFQStlMVpGcXBTK0o5TTVCSGJnNVJVNFRsaG1qVGZNRXpWS3hSanJLYnJMM2I0K29FMi80d3VKQWw5MU1UYk1mCjVVYXZoOWV3ZGNEUEJKb2FCK1c2MERSTlEwbisvYXQ0NlIyTUN4WXYvRDNxZlBZTDZPbkp1VTJ5OURzZDdHcWMKbkpJZTNqczhLUGczaXdxdEVlMDA3TUgrTng1eTJ5VFgwZGlCcUlpM3IxQk8xVGdxK25YSmtxdEdHV05wMERTNApZR1VkUCs1QVppSWtnZDd5STRONmZNVW03S0dhbDRPNFNQME5pNVk5Y3UwcjRBPT0KLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQo= - service-account.pub: LS0tLS1CRUdJTiBQVUJMSUMgS0VZLS0tLS0KTUlJQklqQU5CZ2txaGtpRzl3MEJBUUVGQUFPQ0FROEFNSUlCQ2dLQ0FRRUEwaytNd3RPK2lKQzcyQWlaek0yRQpxQ1VZbEhPblpPMDNOMWJPS2pBQWZUaWN1bHdFQmFKV2tna0ZwdmRjci8wR1JDN0pMY0Zob0lwUnlzZ3p2dmE5CjFtVXBGRnV3WWJsT2VCMlhHTDQ4V0toZktaWTFDZ3JFNHpiRkpUdmQ4ZUdhVXdMdVdPOTlobEFSKzBMR2dsTC8KUDFtdnNNMmk3WFhhbUVkcWluekJRQWZWWHZROEUxRHZ0bmhZVndIdFNxWWhSN1lvMXozVlJFbFpwZEhjYjlvMApIUjE4aGppak9aT2xpSXowejJRdXhxcWtkaDlLTjNLRldRVUtTVVFFcjJ2UFV2R3VXdFh1TUU5bXYrSWV4TEs1CmRVeitsT1VPS0RnTEgvaStqeUQ2K1g2aGR3c0ZoYTVBT2V3bnVPd2tnTHZTQnllSUZNVG9KSEVtMStNZFBrTlkKQXdJREFRQUIKLS0tLS1FTkQgUFVCTElDIEtFWS0tLS0tCg== - ca.crt: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSURGakNDQWY2Z0F3SUJBZ0lRRTRlK0cySTZTL0VtTlZ3d05oRWJiVEFOQmdrcWhraUc5dzBCQVFzRkFEQWwKTVJFd0R3WURWUVFLRXdoaWIyOTBhM1ZpWlRFUU1BNEdBMVVFQXhNSGEzVmlaUzFqWVRBZUZ3MHhPREF6TVRVdwpOREUyTVRWYUZ3MHhPVEF6TVRVd05ERTJNVFZhTUNVeEVUQVBCZ05WQkFvVENHSnZiM1JyZFdKbE1SQXdEZ1lEClZRUURFd2RyZFdKbExXTmhNSUlCSWpBTkJna3Foa2lHOXcwQkFRRUZBQU9DQVE4QU1JSUJDZ0tDQVFFQXZpVGsKbjNBSDJ6VE1WL3pGL1E4V0N2NWNBYWVYUmNacXBxd3ZQeDJZRGlFVGZPY1N0NFIyaHJNeXFQRDVVa1dEOTluOApteWJFeDdmMW9jM1IxdXM0Zm9NR1d2OXR2SUpKa0Q5ckkya0MrU1lGcWtCOEtlZCtBVktJRmoza3JUL3p6NEhwCnpJckdvMzhSQWpDWER2UHNRVWNhWDV0WWZpWk5OR3ptQjB1YW9ZYkNnUWM3aDB5MWRWbU5KQ0tHVmR6TDRWd1UKbEZLK0lSSW9NRGtYN0NXN2VQYkhORXdKck81VEwrczg1T2kyM2xkRFl2UXNkMmtReXB0aDFSMU1aSzIrdW1UcgpHRlpZOEE2RmpHZDVLS0gwQkZmc1pBRFRXTXJ3K3JwSnpXcmtSQTR4VHBNU3V5TTZVZDc2R29KNXNUampsbU1mCmw4ZFdUSGFUZlM4T2lUVmVOd0lEQVFBQm8wSXdRREFPQmdOVkhROEJBZjhFQkFNQ0FxUXdEd1lEVlIwVEFRSC8KQkFVd0F3RUIvekFkQmdOVkhRNEVGZ1FVbVRuZHp6cUdUbERjcWJWSGJ0QzlXdE52aWdZd0RRWUpLb1pJaHZjTgpBUUVMQlFBRGdnRUJBRzZETU5GbFlVZjVTOW1TVWVyOERENVYzMENZWXdkNWpXeTZFcGl4WmhTdzJsc0VkMGNPCmtKSnA4enpIUTZrdXAwMkRWNGRqbHZGWUlGaW9GbTFpYkl5WkIvNEQyN1Mzc0JMT3lwTExnR3FmczZsMXBjdFIKTEpHUnRDSmRGZGx1Y2pWZG0veCtRc1NsRUM1c21ucm85OTFDdEV3WTFvd3VSQ1VFdFd6Rk9VaGRmYnBXSXJSMQpXcyt2dkNOdkhFQlBjMmxzY2R3OU00S0hScUNtWmtOVlRuM0tXbzFYUkNnSERPdWVwREdDWXdHc1NZZy9LMmpTCmVSWU9HWHpGKyt5d0g1cGNtMXhBeVZaVXJTQ25HYlJxS0lYeTlDN1lHcWtIb21hRUtJNWhNaVZDRDVrRGNMZm4KQ3JNaDAvUHpqQ3lwUWxGYXd3eW9BdWUyRU81YSsyc1drcW89Ci0tLS0tRU5EIENFUlRJRklDQVRFLS0tLS0K - etcd-client-ca.crt: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUREakNDQWZhZ0F3SUJBZ0lRU0hBVURjN3pmWFo4S3E0b3lOK3VRakFOQmdrcWhraUc5dzBCQVFzRkFEQWgKTVEwd0N3WURWUVFLRXdSbGRHTmtNUkF3RGdZRFZRUURFd2RsZEdOa0xXTmhNQjRYRFRFNE1ETXhOVEEwTVRZeApOVm9YRFRFNU1ETXhOVEEwTVRZeE5Wb3dJVEVOTUFzR0ExVUVDaE1FWlhSalpERVFNQTRHQTFVRUF4TUhaWFJqClpDMWpZVENDQVNJd0RRWUpLb1pJaHZjTkFRRUJCUUFEZ2dFUEFEQ0NBUW9DZ2dFQkFORENPaVloV250cWNtaHQKZE5BME90VTlLR09IL05UZHRxT2pUOHBkTjZXUzNiZFRHK3JKVEJlQU55SjRHRjlQRmtEVDlRLzA2clRaUVAzYwpOTkpDbUxzT2pLa1I2Nm9lTkY2Z2dtYVJnRm9jUjJWTTR2R0xpUXFqV2ZCK0tIQmdvU0NYdUU3TDM4dmw2QTR0CjBDREc0UzBnbWV1TVA3eG55Qjg1WWMwb3VNdndSVlE0SkQvQ3VUT2RGMTRsTXBuWTlaU21WYXdHd0gxOER1cEcKV2F3REE0czlTL3c0M1NscWVadFM4SG1HUjMxblZ3a20vMEJINVdhYlBqMng0cGZRWlJOYzJ6bGxzRDRvSENwSwpRNlR1cHUrNTN6aG16UVZpNTdDWjhvWEVOVHBML2dBc01GdTFndlRNTThMamNKM1BCNGZsRUdua2ZOMXowK2pLCjBwdGI0UHNDQXdFQUFhTkNNRUF3RGdZRFZSMFBBUUgvQkFRREFnS2tNQThHQTFVZEV3RUIvd1FGTUFNQkFmOHcKSFFZRFZSME9CQllFRkFGMG9CNHF5Sk42WDFpYmZzVU81SFlrbWxaOU1BMEdDU3FHU0liM0RRRUJDd1VBQTRJQgpBUUF3c1oyV3hYQzYrWmdmU1VFTG1FR2gxV3Z5cHVtMHBmUTMweVhncnZXK0pMVW8rY1QxSmFNaGpINDdTR3g0ClZ6ekl2dmliSGtSUEduSjQvQ3VEKzdKY05OTldDdGNSbTVXUXYySzJ5VTVvOGI2V0NSQWVhUGVWNGt4NW1NSWYKT2JqcTFlZWZEcUN2UU1nUU1xMXlXa3RXMHVJSmQzVGtxTVBZcFRNR3V5aHl1Q3JjaFlKU25CS3V2eldJampYYwp5bzl5WUJEVG53VHJFWG53ZEhnNjB5alUvUlB2YVE3MUNEZFlvZFQ4ZGFuekU5dCtHYWQ1bWRwNkVvbHZEdXJlCmp4b1Qzd1dlMjNTRG9kUlFidFpPd1VOUzhPeThLVWdOZFR4UWZ1YjNldjRCbEtXM0hIVmlrbkYvNGZ3SFErSzUKdWQvQlU5YUVHaWhPOXl2bm1jMHM0WC83Ci0tLS0tRU5EIENFUlRJRklDQVRFLS0tLS0K - etcd-client.crt: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSURzVENDQXBtZ0F3SUJBZ0lSQU5GSXB0OE1oZXFOckNDUlE3VkVBZVl3RFFZSktvWklodmNOQVFFTEJRQXcKSVRFTk1Bc0dBMVVFQ2hNRVpYUmpaREVRTUE0R0ExVUVBeE1IWlhSalpDMWpZVEFlRncweE9EQXpNVFV3TkRFNApNVEJhRncweE9UQXpNVFV3TkRFNE1UQmFNQ1V4RFRBTEJnTlZCQW9UQkdWMFkyUXhGREFTQmdOVkJBTVRDMlYwClkyUXRZMnhwWlc1ME1JSUJJakFOQmdrcWhraUc5dzBCQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBNkd3MDBBRVUKUWIrSFlPY1BwRTZXZ1NDRlk2eTM0d1BWcHNDMzdYbWUrdjg2Mk5qc2srMDBZNG44QkVIU0V5MDR6YTkrTmFWUwovc2g2R0tQS1ZlYjlDM041VFhuYVpJVThxanIrL0IyVnhOSkRtODMvNjIwUitFbDRjUHVjdGdtS0xuNmZnNnNXCnZQcFF5dTRBOHZXK0pzaUlIT09xQkhobWJORk9yNE1mRnlSOFpRbm9uakRxS0JpNUgvU0FFb0l0WWIzcmJxMEcKS21GKzhPek1DQ1BpazU5bzc4R0tMaVN2RFVDVzc1c2czdVNWNmpFNTlzSzY2bEdOYWFtZ2V1RFZyOVFUUHpMaApONlVSQmhhRHM4c1ExTW1nZVJtUmNVSTJ3dWdxd3Fydkhld3RVSlp6dkNsTUdnSjdzemlHRXRBMSs3UlpJUkRUCnF3MzFvMXZNeU1VNm93SURBUUFCbzRIZk1JSGNNQTRHQTFVZER3RUIvd1FFQXdJRm9EQWRCZ05WSFNVRUZqQVUKQmdnckJnRUZCUWNEQVFZSUt3WUJCUVVIQXdJd0RBWURWUjBUQVFIL0JBSXdBREFmQmdOVkhTTUVHREFXZ0JRQgpkS0FlS3NpVGVsOVltMzdGRHVSMkpKcFdmVEI4QmdOVkhSRUVkVEJ6Z2k5aVlYTjBhVzl1TFhSbGMzUXRaWFJqClpEQXVhemh6TFhCc1lYbG5jbTkxYm1RdWRHRnJaWE5qYjI5d0xtTnZiWUl2WW1GemRHbHZiaTEwWlhOMExXVjAKWTJReExtczRjeTF3YkdGNVozSnZkVzVrTG5SaGEyVnpZMjl2Y0M1amIyMkNDV3h2WTJGc2FHOXpkSWNFZndBQQpBVEFOQmdrcWhraUc5dzBCQVFzRkFBT0NBUUVBZTh6T1YwYnZhWngwSUU2QUFXeVBzMS9LaFBQbXBqTFovU3Q5CmFMdkVQY3JrSm5Cck1MK0NRS2ZyWEgydjRib3RuWlpOZHA0bXg0UlNxdUlyaHArM2M1RGNQVnVBaVlja1J0bUUKR0Vya204RGtQYitYR2dyL3RGTkloenp0dkV0T2l6ZmJFTndZZlFFUjZVdTFBYi8wWE9QNWFyY2lHQ0tRa3BvMAp2RHp0VGpBd2NXMXIyVW91OWt2aitBSnFiUHoveWdlRS8vUDV1c2lPcHQzU3FubHdYZUhha2Vpck9QQTA4b0hECmFUQjdDMUc1bmJMRlB4QTN6ZEZzbHorNmNuRXdLTGFLVHo4WTJLSHJLeDlHalVsUllRcFhvanBwY0l1bXhjaHkKT1dRZlIyd0RHVTE1L2xtL2gxazFxRzNDakhpaFVqWVBMTzBTVTZ3cGtUZCtpNXh6T1E9PQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0tCg== - etcd-client.key: LS0tLS1CRUdJTiBSU0EgUFJJVkFURSBLRVktLS0tLQpNSUlFcEFJQkFBS0NBUUVBNkd3MDBBRVVRYitIWU9jUHBFNldnU0NGWTZ5MzR3UFZwc0MzN1htZSt2ODYyTmpzCmsrMDBZNG44QkVIU0V5MDR6YTkrTmFWUy9zaDZHS1BLVmViOUMzTjVUWG5hWklVOHFqcisvQjJWeE5KRG04My8KNjIwUitFbDRjUHVjdGdtS0xuNmZnNnNXdlBwUXl1NEE4dlcrSnNpSUhPT3FCSGhtYk5GT3I0TWZGeVI4WlFubwpuakRxS0JpNUgvU0FFb0l0WWIzcmJxMEdLbUYrOE96TUNDUGlrNTlvNzhHS0xpU3ZEVUNXNzVzZzN1U1Y2akU1CjlzSzY2bEdOYWFtZ2V1RFZyOVFUUHpMaE42VVJCaGFEczhzUTFNbWdlUm1SY1VJMnd1Z3F3cXJ2SGV3dFVKWnoKdkNsTUdnSjdzemlHRXRBMSs3UlpJUkRUcXczMW8xdk15TVU2b3dJREFRQUJBb0lCQUh3eU5oWmdQVDdVNWJaMgpRZmwrdFJYVEZ2UW9PeXJueGFjUm5EY2Rva0psV0VDL3ljdFNHWWlIRjAvL0RBNkxQNnRKZDV1YStEcUZUaGtVCmpPNVNQQzErU3ZlSGdaZnRTbmw4aFB5Ym9vaEdBektpWlhxY0Vkb25DR0QzVXNwRFZyOTVraXQ5cE96ZXBZV0sKb0o4emlhU1h5NFFFYzdsbnpQT2c5UGI4amdTQ3liRm5KK3VtQlhUT2gzRTQ4eGsrYnRNUnNpaEx1VzNWczVMRgplVGE4T1dvMit5bEdHa2FEV29tNEFZMUQzb09tSmx2cE1sY0w5bHFOSzI0SjYxczRpK0g5Q2NBenV6S3dJU2RnClNOcDV1RGhUMjc4MnEyM3UrY1hOQmVMUDdSSU1ubFR1cFo5Mk9DREY1b1d1Y0tNVVJkN2loY0oxQ2YxTmJaZmEKTUlBRWw4RUNnWUVBNlN6Ny9CQ1JWSjNPc3hqbm82YzlKdjliZFNENUN3VWhraGhJRnNRaGVyLzdsdzBSZVRZVgpweFQvZnV2SVI4WG1xaGIrRkxpWjJJTzFzSHJrUG5oRHRxOUUyalpSVS9ZajBpd2JBUHUwTGU1NURyU2M3SktqClBVcG9xU2Rhb1NuakNkU2MxcU05R2VNbUltdVpJUjVtUjBOc3NJT3RjT2ZsNnI2ckprTVVzRXNDZ1lFQS95eGEKSFd3M2MyellHZHE0Q0R3QmdoNXRFVU5saTZ0cFFzZkxFbFlnd1l3N1hPd0JsNGRwbzduSE1oNEt1bVlRbDFXVwpES1piRkhpblZ3dnA4OUxzWG9YckdSL1M3NXBmR0FZWWcvVE5qNVFtQmlhM2tzdHI4N3ZnZUVlN05WajlsZnA5Cnd3RzhjeE95NjVLZUNFZmIvaFBGTmwyQnRIRm5pQmRzZlFzTUdBa0NnWUJ1RUh6VlM2QytGMHRWUU1FK2Y1ZWYKQzlSSTRvcUx5QjFEajlDZlptOERPUkh5Q0FvaWRBUWVmUXZwQmpUZ3BDcXdTUEFnS2M3ODQ1Ymt1ZTE1QzEyegpJdUpXT21PRFJXRTlPUEo2TVZXb2hMT0IzSUZpTGdsOXlkekRVNzgwNmNld2dUcVRHalNpUHBWbWsvR1JMMzlKCnppckUyek1JWTM0a28yRzRTdHUrSndLQmdRRExuVkYvSHVZVWRhcnUzb2R4RXFqRmNvL25jWmNxM3ltTVB5NzgKdjdzOWxpK2NVenBsOW9qR082MEdnZEJmc3FmVWlsZkVXazVkUkhXTFVSZHJGMGpEbUNya0RtL2IvNXVYNk8xUgpCbHV0RVROU1B6ekdwd25LSUlYYWxLcCt4RGI5b1RjUEQyaVhqd1Y3VXJCRnZVbC9NYmx4U3lYL25XcFd2eEl6CnFVZ0tPUUtCZ1FDQWJpcEhITmRkTEo1S01vdFd6ZW9hR2hTTHBKbG5MaWtvS1h2QURQck1Vd1MwZWFNL2xRZ1kKRUp1OC9SbE5CdW9GcWloNHJQblhUWXRYUVhsbGVhRWNaSFVBRFgySXVUbHRpNHRET0QySWRNdHUxbnRIZ2VwKwpUdy8rT3loRmpua2dLS2UzNDM4UzJGU2hUOTFzWWcwNk9DNTFxaXQ4MkY3K1h5aEQybTNnVWc9PQotLS0tLUVORCBSU0EgUFJJVkFURSBLRVktLS0tLQo= diff --git a/output/manifests/kube-apiserver.yaml b/output/manifests/kube-apiserver.yaml deleted file mode 100644 index 96bf48176..000000000 --- a/output/manifests/kube-apiserver.yaml +++ /dev/null @@ -1,88 +0,0 @@ -apiVersion: apps/v1 -kind: DaemonSet -metadata: - name: kube-apiserver - namespace: kube-system - labels: - tier: control-plane - k8s-app: kube-apiserver -spec: - selector: - matchLabels: - tier: control-plane - k8s-app: kube-apiserver - template: - metadata: - labels: - tier: control-plane - k8s-app: kube-apiserver - annotations: - checkpointer.alpha.coreos.com/checkpoint: "true" - spec: - containers: - - name: kube-apiserver - image: gcr.io/google_containers/hyperkube:v1.9.3 - command: - - /hyperkube - - apiserver - - --admission-control=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,ValidatingAdmissionWebhook,ResourceQuota,DefaultTolerationSeconds,MutatingAdmissionWebhook - - --advertise-address=$(POD_IP) - - --allow-privileged=true - - --anonymous-auth=false - - --authorization-mode=RBAC - - --bind-address=0.0.0.0 - - --client-ca-file=/etc/kubernetes/secrets/ca.crt - - --cloud-provider= - - --etcd-cafile=/etc/kubernetes/secrets/etcd-client-ca.crt - - --etcd-certfile=/etc/kubernetes/secrets/etcd-client.crt - - --etcd-keyfile=/etc/kubernetes/secrets/etcd-client.key - - --etcd-servers=https://bastion-test-etcd0.k8s-playground.takescoop.com:2379,https://bastion-test-etcd1.k8s-playground.takescoop.com:2379 - - --insecure-port=0 - - --kubelet-client-certificate=/etc/kubernetes/secrets/apiserver.crt - - --kubelet-client-key=/etc/kubernetes/secrets/apiserver.key - - --secure-port=443 - - --service-account-key-file=/etc/kubernetes/secrets/service-account.pub - - --service-cluster-ip-range=10.3.0.0/16 - - --storage-backend=etcd3 - - --tls-ca-file=/etc/kubernetes/secrets/ca.crt - - --tls-cert-file=/etc/kubernetes/secrets/apiserver.crt - - --tls-private-key-file=/etc/kubernetes/secrets/apiserver.key - - --oidc-issuer-url=https://accounts.google.com - - --oidc-client-id=727680134807-s74j3n1h3nhmkpvg0j8v3ovnl04p27oo.apps.googleusercontent.com - - --oidc-username-claim=email - env: - - name: POD_IP - valueFrom: - fieldRef: - fieldPath: status.podIP - volumeMounts: - - mountPath: /etc/ssl/certs - name: ssl-certs-host - readOnly: true - - mountPath: /etc/kubernetes/secrets - name: secrets - readOnly: true - - mountPath: /var/lock - name: var-lock - readOnly: false - hostNetwork: true - nodeSelector: - node-role.kubernetes.io/master: "" - tolerations: - - key: node-role.kubernetes.io/master - operator: Exists - effect: NoSchedule - volumes: - - name: ssl-certs-host - hostPath: - path: /usr/share/ca-certificates - - name: secrets - secret: - secretName: kube-apiserver - - name: var-lock - hostPath: - path: /var/lock - updateStrategy: - rollingUpdate: - maxUnavailable: 1 - type: RollingUpdate diff --git a/output/manifests/kube-controller-manager-disruption.yaml b/output/manifests/kube-controller-manager-disruption.yaml deleted file mode 100644 index 1d1d02359..000000000 --- a/output/manifests/kube-controller-manager-disruption.yaml +++ /dev/null @@ -1,11 +0,0 @@ -apiVersion: policy/v1beta1 -kind: PodDisruptionBudget -metadata: - name: kube-controller-manager - namespace: kube-system -spec: - minAvailable: 1 - selector: - matchLabels: - tier: control-plane - k8s-app: kube-controller-manager diff --git a/output/manifests/kube-controller-manager-role-binding.yaml b/output/manifests/kube-controller-manager-role-binding.yaml deleted file mode 100644 index 267856a9a..000000000 --- a/output/manifests/kube-controller-manager-role-binding.yaml +++ /dev/null @@ -1,12 +0,0 @@ -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: controller-manager -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: system:kube-controller-manager -subjects: -- kind: ServiceAccount - name: kube-controller-manager - namespace: kube-system diff --git a/output/manifests/kube-controller-manager-sa.yaml b/output/manifests/kube-controller-manager-sa.yaml deleted file mode 100644 index bb8f0aab9..000000000 --- a/output/manifests/kube-controller-manager-sa.yaml +++ /dev/null @@ -1,5 +0,0 @@ -apiVersion: v1 -kind: ServiceAccount -metadata: - namespace: kube-system - name: kube-controller-manager diff --git a/output/manifests/kube-controller-manager-secret.yaml b/output/manifests/kube-controller-manager-secret.yaml deleted file mode 100644 index 5d331ce9e..000000000 --- a/output/manifests/kube-controller-manager-secret.yaml +++ /dev/null @@ -1,9 +0,0 @@ -apiVersion: v1 -kind: Secret -metadata: - name: kube-controller-manager - namespace: kube-system -type: Opaque -data: - service-account.key: LS0tLS1CRUdJTiBSU0EgUFJJVkFURSBLRVktLS0tLQpNSUlFb2dJQkFBS0NBUUVBMGsrTXd0TytpSkM3MkFpWnpNMkVxQ1VZbEhPblpPMDNOMWJPS2pBQWZUaWN1bHdFCkJhSldrZ2tGcHZkY3IvMEdSQzdKTGNGaG9JcFJ5c2d6dnZhOTFtVXBGRnV3WWJsT2VCMlhHTDQ4V0toZktaWTEKQ2dyRTR6YkZKVHZkOGVHYVV3THVXTzk5aGxBUiswTEdnbEwvUDFtdnNNMmk3WFhhbUVkcWluekJRQWZWWHZROApFMUR2dG5oWVZ3SHRTcVloUjdZbzF6M1ZSRWxacGRIY2I5bzBIUjE4aGppak9aT2xpSXowejJRdXhxcWtkaDlLCk4zS0ZXUVVLU1VRRXIydlBVdkd1V3RYdU1FOW12K0lleExLNWRVeitsT1VPS0RnTEgvaStqeUQ2K1g2aGR3c0YKaGE1QU9ld251T3drZ0x2U0J5ZUlGTVRvSkhFbTErTWRQa05ZQXdJREFRQUJBb0lCQURUbDFWM2JySHpsQ3BwWAo3M2RYNmhudzJySGNOU3Bwa0EzWFE1dlEzdzZnQXF2TklTWFpvelN3R0QvYXovRmtEd052VVNLMUZUMHdEVXFYCitJdjd1OXdGTGNQMUcvUTRpOGdpaVRLc0JybTEvOW1SOGwxSVFDVjJUVGdFU3RyZ0I5VUJVN29DNHV1NWtBeEcKeTI5VU9PZFNRNktRMW40cnVvTzYwczFxZTZFQzRtdlMyS2lBekprODZOWitIdGRZeTdCV1pzQnBUdDVoMGhERgpQV1JsdFJ4UHNtTytMQm1Ed09VYytqTGN5UXlLR094V0R5NFdQSjZ3UEJzc1JUR05aSkp6WG9VdmRRQlJHMUZ2ClIxd3JFcEpKeDhIdER6aW1ldUh4VmNaSU1sSXRDL0lQcTJPMDdleVhGQW5hUlhncXkwMi9qa0Znc09WOTNIbWMKRzBQdWM0RUNnWUVBMlhmdUdtVDBKaFFTNTFwKysreVRWcWdqQXpDVnN5bkNmc1JHNXZpZHVFcjFneE5wRERybAp0WHdveUdmeFRxcWFHS29NN0FJbVVMY3l2VE9iUlJLN2FNK1BJRE9KS1AvdVZCdWZvcExNOUJ6d3crb1pEYXlxCnVPUlZXOURLUlpNOEk0blM1TmdIbmliZHhVRkNVRjRkYm8rQkhFYUk5bmpHOFBqSVpOZExJWEVDZ1lFQTk1TDAKYTIyNzhJMXdKL0VKQ29EdHlFNnBOdENXWUxuckRJbVhLYjcvOUYyNnFYSU9oVXVXVzBGcDFrNmR2WHpjdHU3dgo5ZWV2emk0emlmamtNVVBRNnpTdHhML0duZzV0c0dPZEcydFNVTWtLd2NqWnhsK0NWVVV6a3BVZk9hNFV6TlYrCnZJeVpBTmpsUURseHptSGFoNTBBbnRxOG40Tk9QYlQ2TjYvdlZyTUNnWUFyelF3WUpOMUlEaU1BbGltZGREajQKNjBTaUQ5Y1hEd0l0cGpyaHFwR1ozUDgyTjJLaEkvdkFZaEdVeTlxK2pYNGNHYVFncFE0eWs3T1VpQ0J0K1NmbQpKR2dmaEVITUVFQmdrRy9HdnVxcEFHcytDcGloT0hYcVo1TUp1elFDYjNWZGN4VVhJcXZtSHMzc1BRaXVSMGFHClRrRWpBTkgxVXI0L0t0eXg4dXNmQVFLQmdDQjY0U0l1OVZjcjF5a0dVRjlXWnR1K3BpaVEyUW03bW9DOGxGNWYKdG9qQ3V6aDd4RGZzb0w4OEo3eDc5K25pTmJxeVFqMEt0bC9nWTlhWUZxZjM4N0xINkh4RmhMTTd4Vnc4MVdIQgpoTDBnZ3c3RllQekxqdmZNNm1VeXR6UUVDS3FPMzkrd3Vtb0lDcHVRYmNQYnhxWEFEVkxKODdFaHN1UVptREl5CkhMNU5Bb0dBZnNBemdlQjVHU3ZYRlp5NXZrS2JDYUpMMXNqMFlJakVpZS9YNW1LdmlMbmVpMXdLOGhVdVJodE0KNXQ0WnVlTlVhUndVWWErQzFjakgxeCtYaGE1K1c4NnVyYTVjT2ZDU3BqdWUrV1BOUTZJZXhZaGVsWHVvWTEwMgp4Sk5uSFhRdUQ0anF0QWlPQWVLS2diWG4rS0x1aDNkRnI3MVVIT2UwQS9NZVI4dFdxVFk9Ci0tLS0tRU5EIFJTQSBQUklWQVRFIEtFWS0tLS0tCg== - ca.crt: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSURGakNDQWY2Z0F3SUJBZ0lRRTRlK0cySTZTL0VtTlZ3d05oRWJiVEFOQmdrcWhraUc5dzBCQVFzRkFEQWwKTVJFd0R3WURWUVFLRXdoaWIyOTBhM1ZpWlRFUU1BNEdBMVVFQXhNSGEzVmlaUzFqWVRBZUZ3MHhPREF6TVRVdwpOREUyTVRWYUZ3MHhPVEF6TVRVd05ERTJNVFZhTUNVeEVUQVBCZ05WQkFvVENHSnZiM1JyZFdKbE1SQXdEZ1lEClZRUURFd2RyZFdKbExXTmhNSUlCSWpBTkJna3Foa2lHOXcwQkFRRUZBQU9DQVE4QU1JSUJDZ0tDQVFFQXZpVGsKbjNBSDJ6VE1WL3pGL1E4V0N2NWNBYWVYUmNacXBxd3ZQeDJZRGlFVGZPY1N0NFIyaHJNeXFQRDVVa1dEOTluOApteWJFeDdmMW9jM1IxdXM0Zm9NR1d2OXR2SUpKa0Q5ckkya0MrU1lGcWtCOEtlZCtBVktJRmoza3JUL3p6NEhwCnpJckdvMzhSQWpDWER2UHNRVWNhWDV0WWZpWk5OR3ptQjB1YW9ZYkNnUWM3aDB5MWRWbU5KQ0tHVmR6TDRWd1UKbEZLK0lSSW9NRGtYN0NXN2VQYkhORXdKck81VEwrczg1T2kyM2xkRFl2UXNkMmtReXB0aDFSMU1aSzIrdW1UcgpHRlpZOEE2RmpHZDVLS0gwQkZmc1pBRFRXTXJ3K3JwSnpXcmtSQTR4VHBNU3V5TTZVZDc2R29KNXNUampsbU1mCmw4ZFdUSGFUZlM4T2lUVmVOd0lEQVFBQm8wSXdRREFPQmdOVkhROEJBZjhFQkFNQ0FxUXdEd1lEVlIwVEFRSC8KQkFVd0F3RUIvekFkQmdOVkhRNEVGZ1FVbVRuZHp6cUdUbERjcWJWSGJ0QzlXdE52aWdZd0RRWUpLb1pJaHZjTgpBUUVMQlFBRGdnRUJBRzZETU5GbFlVZjVTOW1TVWVyOERENVYzMENZWXdkNWpXeTZFcGl4WmhTdzJsc0VkMGNPCmtKSnA4enpIUTZrdXAwMkRWNGRqbHZGWUlGaW9GbTFpYkl5WkIvNEQyN1Mzc0JMT3lwTExnR3FmczZsMXBjdFIKTEpHUnRDSmRGZGx1Y2pWZG0veCtRc1NsRUM1c21ucm85OTFDdEV3WTFvd3VSQ1VFdFd6Rk9VaGRmYnBXSXJSMQpXcyt2dkNOdkhFQlBjMmxzY2R3OU00S0hScUNtWmtOVlRuM0tXbzFYUkNnSERPdWVwREdDWXdHc1NZZy9LMmpTCmVSWU9HWHpGKyt5d0g1cGNtMXhBeVZaVXJTQ25HYlJxS0lYeTlDN1lHcWtIb21hRUtJNWhNaVZDRDVrRGNMZm4KQ3JNaDAvUHpqQ3lwUWxGYXd3eW9BdWUyRU81YSsyc1drcW89Ci0tLS0tRU5EIENFUlRJRklDQVRFLS0tLS0K diff --git a/output/manifests/kube-controller-manager.yaml b/output/manifests/kube-controller-manager.yaml deleted file mode 100644 index 251e9fd9e..000000000 --- a/output/manifests/kube-controller-manager.yaml +++ /dev/null @@ -1,82 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: kube-controller-manager - namespace: kube-system - labels: - tier: control-plane - k8s-app: kube-controller-manager -spec: - replicas: 2 - selector: - matchLabels: - tier: control-plane - k8s-app: kube-controller-manager - template: - metadata: - labels: - tier: control-plane - k8s-app: kube-controller-manager - spec: - affinity: - podAntiAffinity: - preferredDuringSchedulingIgnoredDuringExecution: - - weight: 100 - podAffinityTerm: - labelSelector: - matchExpressions: - - key: tier - operator: In - values: - - control-plane - - key: k8s-app - operator: In - values: - - kube-controller-manager - topologyKey: kubernetes.io/hostname - containers: - - name: kube-controller-manager - image: gcr.io/google_containers/hyperkube:v1.9.3 - command: - - ./hyperkube - - controller-manager - - --use-service-account-credentials - - --allocate-node-cidrs=true - - --cloud-provider= - - --cluster-cidr=10.2.0.0/16 - - --service-cluster-ip-range=10.3.0.0/16 - - --configure-cloud-routes=false - - --leader-elect=true - - --root-ca-file=/etc/kubernetes/secrets/ca.crt - - --service-account-private-key-file=/etc/kubernetes/secrets/service-account.key - livenessProbe: - httpGet: - path: /healthz - port: 10252 # Note: Using default port. Update if --port option is set differently. - initialDelaySeconds: 15 - timeoutSeconds: 15 - volumeMounts: - - name: secrets - mountPath: /etc/kubernetes/secrets - readOnly: true - - name: ssl-host - mountPath: /etc/ssl/certs - readOnly: true - nodeSelector: - node-role.kubernetes.io/master: "" - securityContext: - runAsNonRoot: true - runAsUser: 65534 - serviceAccountName: kube-controller-manager - tolerations: - - key: node-role.kubernetes.io/master - operator: Exists - effect: NoSchedule - volumes: - - name: secrets - secret: - secretName: kube-controller-manager - - name: ssl-host - hostPath: - path: /usr/share/ca-certificates - dnsPolicy: Default # Don't use cluster DNS. diff --git a/output/manifests/kube-dns-deployment.yaml b/output/manifests/kube-dns-deployment.yaml deleted file mode 100644 index 9874d51ba..000000000 --- a/output/manifests/kube-dns-deployment.yaml +++ /dev/null @@ -1,154 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: kube-dns - namespace: kube-system - labels: - k8s-app: kube-dns - kubernetes.io/cluster-service: "true" - addonmanager.kubernetes.io/mode: Reconcile -spec: - # replicas: not specified here: - # 1. In order to make Addon Manager do not reconcile this replicas parameter. - # 2. Default is 1. - # 3. Will be tuned in real time if DNS horizontal auto-scaling is turned on. - strategy: - rollingUpdate: - maxSurge: 10% - maxUnavailable: 0 - selector: - matchLabels: - k8s-app: kube-dns - template: - metadata: - labels: - k8s-app: kube-dns - spec: - nodeSelector: - node-role.kubernetes.io/master: "" - tolerations: - - key: node-role.kubernetes.io/master - operator: Exists - effect: NoSchedule - volumes: - - name: kube-dns-config - configMap: - name: kube-dns - optional: true - containers: - - name: kubedns - image: gcr.io/google_containers/k8s-dns-kube-dns-amd64:1.14.8 - resources: - # TODO: Set memory limits when we've profiled the container for large - # clusters, then set request = limit to keep this container in - # guaranteed class. Currently, this container falls into the - # "burstable" category so the kubelet doesn't backoff from restarting it. - limits: - memory: 170Mi - requests: - cpu: 100m - memory: 70Mi - livenessProbe: - httpGet: - path: /healthcheck/kubedns - port: 10054 - scheme: HTTP - initialDelaySeconds: 60 - timeoutSeconds: 5 - successThreshold: 1 - failureThreshold: 5 - readinessProbe: - httpGet: - path: /readiness - port: 8081 - scheme: HTTP - # we poll on pod startup for the Kubernetes master service and - # only setup the /readiness HTTP server once that's available. - initialDelaySeconds: 3 - timeoutSeconds: 5 - args: - - --domain=cluster.local. - - --dns-port=10053 - - --config-dir=/kube-dns-config - - --v=2 - env: - - name: PROMETHEUS_PORT - value: "10055" - ports: - - containerPort: 10053 - name: dns-local - protocol: UDP - - containerPort: 10053 - name: dns-tcp-local - protocol: TCP - - containerPort: 10055 - name: metrics - protocol: TCP - volumeMounts: - - name: kube-dns-config - mountPath: /kube-dns-config - - name: dnsmasq - image: gcr.io/google_containers/k8s-dns-dnsmasq-nanny-amd64:1.14.8 - livenessProbe: - httpGet: - path: /healthcheck/dnsmasq - port: 10054 - scheme: HTTP - initialDelaySeconds: 60 - timeoutSeconds: 5 - successThreshold: 1 - failureThreshold: 5 - args: - - -v=2 - - -logtostderr - - -configDir=/etc/k8s/dns/dnsmasq-nanny - - -restartDnsmasq=true - - -- - - -k - - --cache-size=1000 - - --no-negcache - - --log-facility=- - - --server=/cluster.local/127.0.0.1#10053 - - --server=/in-addr.arpa/127.0.0.1#10053 - - --server=/ip6.arpa/127.0.0.1#10053 - ports: - - containerPort: 53 - name: dns - protocol: UDP - - containerPort: 53 - name: dns-tcp - protocol: TCP - # see: https://github.com/kubernetes/kubernetes/issues/29055 for details - resources: - requests: - cpu: 150m - memory: 20Mi - volumeMounts: - - name: kube-dns-config - mountPath: /etc/k8s/dns/dnsmasq-nanny - - name: sidecar - image: gcr.io/google_containers/k8s-dns-sidecar-amd64:1.14.8 - livenessProbe: - httpGet: - path: /metrics - port: 10054 - scheme: HTTP - initialDelaySeconds: 60 - timeoutSeconds: 5 - successThreshold: 1 - failureThreshold: 5 - args: - - --v=2 - - --logtostderr - - --probe=kubedns,127.0.0.1:10053,kubernetes.default.svc.cluster.local,5,SRV - - --probe=dnsmasq,127.0.0.1:53,kubernetes.default.svc.cluster.local,5,SRV - ports: - - containerPort: 10054 - name: metrics - protocol: TCP - resources: - requests: - memory: 20Mi - cpu: 10m - dnsPolicy: Default # Don't use cluster DNS. - serviceAccountName: kube-dns diff --git a/output/manifests/kube-dns-sa.yaml b/output/manifests/kube-dns-sa.yaml deleted file mode 100644 index 4e5a85660..000000000 --- a/output/manifests/kube-dns-sa.yaml +++ /dev/null @@ -1,5 +0,0 @@ -apiVersion: v1 -kind: ServiceAccount -metadata: - name: kube-dns - namespace: kube-system diff --git a/output/manifests/kube-dns-svc.yaml b/output/manifests/kube-dns-svc.yaml deleted file mode 100644 index bbbebe22d..000000000 --- a/output/manifests/kube-dns-svc.yaml +++ /dev/null @@ -1,20 +0,0 @@ -apiVersion: v1 -kind: Service -metadata: - name: kube-dns - namespace: kube-system - labels: - k8s-app: kube-dns - kubernetes.io/cluster-service: "true" - kubernetes.io/name: "KubeDNS" -spec: - selector: - k8s-app: kube-dns - clusterIP: 10.3.0.10 - ports: - - name: dns - port: 53 - protocol: UDP - - name: dns-tcp - port: 53 - protocol: TCP diff --git a/output/manifests/kube-proxy-role-binding.yaml b/output/manifests/kube-proxy-role-binding.yaml deleted file mode 100644 index 0ae0e12a3..000000000 --- a/output/manifests/kube-proxy-role-binding.yaml +++ /dev/null @@ -1,12 +0,0 @@ -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: kube-proxy -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: system:node-proxier # Automatically created system role. -subjects: -- kind: ServiceAccount - name: kube-proxy - namespace: kube-system diff --git a/output/manifests/kube-proxy-sa.yaml b/output/manifests/kube-proxy-sa.yaml deleted file mode 100644 index 651d76f30..000000000 --- a/output/manifests/kube-proxy-sa.yaml +++ /dev/null @@ -1,5 +0,0 @@ -apiVersion: v1 -kind: ServiceAccount -metadata: - namespace: kube-system - name: kube-proxy diff --git a/output/manifests/kube-proxy.yaml b/output/manifests/kube-proxy.yaml deleted file mode 100644 index c6a6cd408..000000000 --- a/output/manifests/kube-proxy.yaml +++ /dev/null @@ -1,67 +0,0 @@ -apiVersion: apps/v1 -kind: DaemonSet -metadata: - name: kube-proxy - namespace: kube-system - labels: - tier: node - k8s-app: kube-proxy -spec: - selector: - matchLabels: - tier: node - k8s-app: kube-proxy - template: - metadata: - labels: - tier: node - k8s-app: kube-proxy - spec: - containers: - - name: kube-proxy - image: gcr.io/google_containers/hyperkube:v1.9.3 - command: - - ./hyperkube - - proxy - - --cluster-cidr=10.2.0.0/16 - - --hostname-override=$(NODE_NAME) - - --kubeconfig=/etc/kubernetes/kubeconfig - - --proxy-mode=iptables - env: - - name: NODE_NAME - valueFrom: - fieldRef: - fieldPath: spec.nodeName - securityContext: - privileged: true - volumeMounts: - - mountPath: /lib/modules - name: lib-modules - readOnly: true - - mountPath: /etc/ssl/certs - name: ssl-certs-host - readOnly: true - - name: kubeconfig - mountPath: /etc/kubernetes - readOnly: true - hostNetwork: true - serviceAccountName: kube-proxy - tolerations: - - effect: NoSchedule - operator: Exists - - effect: NoExecute - operator: Exists - volumes: - - name: lib-modules - hostPath: - path: /lib/modules - - name: ssl-certs-host - hostPath: - path: /usr/share/ca-certificates - - name: kubeconfig - configMap: - name: kubeconfig-in-cluster - updateStrategy: - rollingUpdate: - maxUnavailable: 1 - type: RollingUpdate diff --git a/output/manifests/kube-scheduler-disruption.yaml b/output/manifests/kube-scheduler-disruption.yaml deleted file mode 100644 index 11af3faa2..000000000 --- a/output/manifests/kube-scheduler-disruption.yaml +++ /dev/null @@ -1,11 +0,0 @@ -apiVersion: policy/v1beta1 -kind: PodDisruptionBudget -metadata: - name: kube-scheduler - namespace: kube-system -spec: - minAvailable: 1 - selector: - matchLabels: - tier: control-plane - k8s-app: kube-scheduler diff --git a/output/manifests/kube-scheduler.yaml b/output/manifests/kube-scheduler.yaml deleted file mode 100644 index 780d61df5..000000000 --- a/output/manifests/kube-scheduler.yaml +++ /dev/null @@ -1,58 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: kube-scheduler - namespace: kube-system - labels: - tier: control-plane - k8s-app: kube-scheduler -spec: - replicas: 2 - selector: - matchLabels: - tier: control-plane - k8s-app: kube-scheduler - template: - metadata: - labels: - tier: control-plane - k8s-app: kube-scheduler - spec: - affinity: - podAntiAffinity: - preferredDuringSchedulingIgnoredDuringExecution: - - weight: 100 - podAffinityTerm: - labelSelector: - matchExpressions: - - key: tier - operator: In - values: - - control-plane - - key: k8s-app - operator: In - values: - - kube-scheduler - topologyKey: kubernetes.io/hostname - containers: - - name: kube-scheduler - image: gcr.io/google_containers/hyperkube:v1.9.3 - command: - - ./hyperkube - - scheduler - - --leader-elect=true - livenessProbe: - httpGet: - path: /healthz - port: 10251 # Note: Using default port. Update if --port option is set differently. - initialDelaySeconds: 15 - timeoutSeconds: 15 - nodeSelector: - node-role.kubernetes.io/master: "" - securityContext: - runAsNonRoot: true - runAsUser: 65534 - tolerations: - - key: node-role.kubernetes.io/master - operator: Exists - effect: NoSchedule diff --git a/output/manifests/kube-system-rbac-role-binding.yaml b/output/manifests/kube-system-rbac-role-binding.yaml deleted file mode 100644 index 47623a36f..000000000 --- a/output/manifests/kube-system-rbac-role-binding.yaml +++ /dev/null @@ -1,12 +0,0 @@ -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: system:default-sa -subjects: - - kind: ServiceAccount - name: default - namespace: kube-system -roleRef: - kind: ClusterRole - name: cluster-admin - apiGroup: rbac.authorization.k8s.io diff --git a/output/manifests/kubeconfig-in-cluster.yaml b/output/manifests/kubeconfig-in-cluster.yaml deleted file mode 100644 index 8e9ed3fb2..000000000 --- a/output/manifests/kubeconfig-in-cluster.yaml +++ /dev/null @@ -1,22 +0,0 @@ -apiVersion: v1 -kind: ConfigMap -metadata: - name: kubeconfig-in-cluster - namespace: kube-system -data: - kubeconfig: | - apiVersion: v1 - clusters: - - name: local - cluster: - server: https://bastion-test.k8s-playground.takescoop.com:443 - certificate-authority: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt - users: - - name: service-account - user: - # Use service account token - tokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token - contexts: - - context: - cluster: local - user: service-account diff --git a/output/manifests/pod-checkpointer-role-binding.yaml b/output/manifests/pod-checkpointer-role-binding.yaml deleted file mode 100644 index 97b12bab9..000000000 --- a/output/manifests/pod-checkpointer-role-binding.yaml +++ /dev/null @@ -1,13 +0,0 @@ -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - name: pod-checkpointer - namespace: kube-system -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: Role - name: pod-checkpointer -subjects: -- kind: ServiceAccount - name: pod-checkpointer - namespace: kube-system diff --git a/output/manifests/pod-checkpointer-role.yaml b/output/manifests/pod-checkpointer-role.yaml deleted file mode 100644 index 2a295e53c..000000000 --- a/output/manifests/pod-checkpointer-role.yaml +++ /dev/null @@ -1,12 +0,0 @@ -apiVersion: rbac.authorization.k8s.io/v1 -kind: Role -metadata: - name: pod-checkpointer - namespace: kube-system -rules: -- apiGroups: [""] # "" indicates the core API group - resources: ["pods"] - verbs: ["get", "watch", "list"] -- apiGroups: [""] # "" indicates the core API group - resources: ["secrets", "configmaps"] - verbs: ["get"] diff --git a/output/manifests/pod-checkpointer-sa.yaml b/output/manifests/pod-checkpointer-sa.yaml deleted file mode 100644 index e76928007..000000000 --- a/output/manifests/pod-checkpointer-sa.yaml +++ /dev/null @@ -1,5 +0,0 @@ -apiVersion: v1 -kind: ServiceAccount -metadata: - namespace: kube-system - name: pod-checkpointer diff --git a/output/manifests/pod-checkpointer.yaml b/output/manifests/pod-checkpointer.yaml deleted file mode 100644 index 10520391f..000000000 --- a/output/manifests/pod-checkpointer.yaml +++ /dev/null @@ -1,72 +0,0 @@ -apiVersion: apps/v1 -kind: DaemonSet -metadata: - name: pod-checkpointer - namespace: kube-system - labels: - tier: control-plane - k8s-app: pod-checkpointer -spec: - selector: - matchLabels: - tier: control-plane - k8s-app: pod-checkpointer - template: - metadata: - labels: - tier: control-plane - k8s-app: pod-checkpointer - annotations: - checkpointer.alpha.coreos.com/checkpoint: "true" - spec: - containers: - - name: pod-checkpointer - image: quay.io/coreos/pod-checkpointer:3cd08279c564e95c8b42a0b97c073522d4a6b965 - command: - - /checkpoint - - --lock-file=/var/run/lock/pod-checkpointer.lock - - --kubeconfig=/etc/checkpointer/kubeconfig - env: - - name: NODE_NAME - valueFrom: - fieldRef: - fieldPath: spec.nodeName - - name: POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: POD_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - imagePullPolicy: Always - volumeMounts: - - mountPath: /etc/checkpointer - name: kubeconfig - - mountPath: /etc/kubernetes - name: etc-kubernetes - - mountPath: /var/run - name: var-run - serviceAccountName: pod-checkpointer - hostNetwork: true - nodeSelector: - node-role.kubernetes.io/master: "" - restartPolicy: Always - tolerations: - - key: node-role.kubernetes.io/master - operator: Exists - effect: NoSchedule - volumes: - - name: kubeconfig - configMap: - name: kubeconfig-in-cluster - - name: etc-kubernetes - hostPath: - path: /etc/kubernetes - - name: var-run - hostPath: - path: /var/run - updateStrategy: - rollingUpdate: - maxUnavailable: 1 - type: RollingUpdate

    tYrYdLxhlnBJx{wS}gdgvUmGCmUb158KKpd-e$xqskMo1yXb(L<06IM|wM#Dfa z@p6|b`ph(TES|mJ*+fvAkhy(%lmyvu zC8ufPBcUb*BC4{4+Rw09YtPZ=6d(_zn4yu?xf?pu)Ylga5|XD4IrMQHqzCC5`fgGw z3Q5fT{FPzYDKWAC~ z_xT(5#n#{Hic$I!8S6Jv^|vg>AJe#4XT7Ktn^?R}OeB2luS_L$(w47Hnnk3ACF(HFctc6N$rVV0Go#C<|6v; zH6E%M9UX`XY0Nm)gq}&tyzw^KudE|uI&Nkz>DD34iedc zWnzgTMkDh+eLcf^W{bzZ?2HVuX!fguw-AkAwzsrOok zmsp`KT70-8lf;npv?;k#U;j|IU9%=!S1>g6xZaLJ_NV>oxX#$DyVqjxZLh9!wz4qc zU$?r3#pb&>sJUd3-_}o^UsAc?W*rH4(|G2uFZjh->t-Y+EVsYV5>`euml7%iQ`E73 zVWH2}VOl!FI2>5Hm~-!yRR%6S8f(~`?JklR+!)oKiV>Sh9xm23u390lkx{>t9Bc9J z6)A$m_ydw7K|s7p_w|?WElQ~cS@kQJ z_GIfk#TnT3`}vt5llbWO<7fF|cpfI(hn^aD;uo{=eB1m!=Z{-op50o(W~`R8=wX@@ z?=S2dnza`VeaaX@HjPkCB2L0a%eoyt}TWhaYU%fK^Fgxz5w=*~} z+crOvl+V6#W8Flr&Q~S0KX-dSX@=U^RTGvxH7Dz+lOFq3+3uHpRNWae7W-Q^f}b6q zyop_C*IGG!h5wVrY|aF%r|zF=SdlpmG`0GoJlo99dD%}wI+7+WAxWl*Z`+(ItG93H zWW%%1K2?r~-@dn9{n}qk)c*eOS>jsCQG4+sFR?TF_4K<9PMHFYPiqH4sgv&>Vfmj1 z#V`4Fg;kUXZ487k;@J9?YlXRWMxI&+(vNIU3&EYrB`VMfjEZWo!XB`Yl`MF;y=-+0 zYae$FIc~{8PouhC@tmjf-cs5tm-Udb9@>PY3YnuUnH+foK~dVd^Ll%gCpKpn&bZ2U z#;M5#Z+DRReFOyL#m_Xb#N3Hrwtlrxz@G9ybLS`#HN~OLh^1t8Gp?NP(ng3WWOk!| za2U^Bo^yR8$}ySQDLRx1udzb#nksLynN}>Xn23CGtDaqxL1+)^6EWcxM)BxOzk_a; zAhLt@j?G?P=WBSv>I!tO-a5U<=AoCIv%ecDL*{ap$0@x`*CSzYYDObGF(VrmyQpw*Ef8XtW5{Jz?R5LjA|-SNN)dk3stsV~$m#HT9+u%I*r6It*Lc(bl&*!O+%xh>>8yHP#Er7CUtKnw=D2A{(3dar42{ zXD`oPHF-Rn>4=YBXAQe4IQtbxnm*rR1`q!zFh$U1`r>b+3+89gM1aC2T7HPmzw4v7 zNd{`(l3H}eiNX235+gQ$SIu*84gB_N^C1dfq{vYiKkSe!Uy(xsJjLUdG>OY zUnO5_Sn~If432}{$iBTcXxNU)Sal`K6{`2ojdl~-@gCd zcQA%S)>^Fho%6}x^Lw6YPxwNDhQvK30*$0B=7sDwE$~VAiw>-pVI~!w?%s~Xx|&Cf zcL;?juq_QOW~X3RjH{m(hBQto3KLf?H7)sd-SWR4Ja5n5UvUL4qhYe0ioM*d(EGfd z>v=_YH)rnm{rpj|<)Af6K31he(K7J*cs&A0v&78?++{4>V_c%|AyBa#;1*aA9&GK* zX-;WN2@*jdG60?hu2K;Gj5j`{zEb)9G<)toP-kEn?`5T)YA#ziH)_neM_Fdvu!j>Z z5r;`!V!A>+K$#(*y(#+m=ArxHgybyqI@dZgoc8S{*S#$ShAmMDzzFenBXBOy?rk=3 zDW)5F7bF3wRkT1+j#wm)9g8&}=PjweIA?Ov#6{yF=jvm*RoQSi;6{M{%5u*@m;8|T z25Z~CNTEOko&PvfMVh;7E+W-B*-ZZFCAUcT`z=r8Rl3sFp3u`~;3zj^8XhfH;6ZH^ zzjqJv@4ZgJ+Lc=-d)_8st@Vuov$e&n7a`hv^?^W!aQl2F`r*B;a(^xpsV$gd`v&$t z0(erg`uu8(r}{&Py81#b4Q3WR5#rbW|DQE7j0y(^;GSvIP!R@$!xYe35{FP+`0wg`ou!RQ6Qz;M}>nP zYeTWBg-W+dIdxxb`&9CtiL1n|jh})ytvdE{#ZgRk6AsQ*H{_wx^J&tB0_)Jf42*T@ z_3_KTn$xw?>)`Tdv9)$l<-MV{TT|QCH)^zvW=0PRBs2>Z-c`HE%K(xiulS_d8ql_! z?U)ZNVs=>vrtVtp?=(3hvP!k7HkMLbsW#=F&U~+w#N0L9oSb&HTHkWlj5X7~YqZ|b zsh__4G1`4^<8ht$Bf)MRYh^E8a@>g+fI{TsW;Hy&8K=+O4!etc=B&9Tosg~E3hjlf z@uIz!)YjbQtvKHgy{S9AIo!!51UExo)&UaR@d}Q3!;$dG-3H^#2$SUkq!B8kp%2Tm zX^XVS19JUodG6vGk_q+*r!^*FASti!1Cx96BDp)23zP(zgi}>_@P&$RB~e22G_bYz z&yldNur+f$#?9RB+=vODqz7}?te%6uWR;tTedPyLE_J_h>ad`~4e2x%9H^K!y~ND3 zAB=pNY1rLoIiz2P_AnC606n%9*G)xcG*o)CY{r@+l~vw;qtV%=_(4u?bv2qrNker* zUd!#~W|Zp3+A`vR_t*9lnu6}z3YRjh4VD04vehFrixRYj=jhm_3ec?tW$p%3&h!&NRihv<4 zM6itOe6x0*Gj`SVflXW0giWHj;lYRQ-*0SHsV)b28 zw&oRb^0dF&s{TnK%kDq+u;6Dmr}|=PuF!G2yu&T;01RZ{0>t&H3HvgfzLkH%=KKS1 zU!0jzuth{8A6lt_1KDA~4+WL#^oe0ywE-es)1iY~?$DQk5sZOX%u{Qy88RoXa7br*Z5x7XC{bNWtft-x=F*3;ItDXwV$_-EcpzORl7nfy8-5A%kqDSN+h&yS1ik_g%5nEBM=^9%QlC3rRaRa0d zxIE7#g{$b^XlhB#2T$Gh@gBxiQ>xz6K|m(0$+5$I`2-al*;;!~$C9u4Cs5o?O0(}N zD3J`!RD0Sn-gA4m*mCXmH2ewC0{}4+6t{3BB=ejWwKPw1j0E^=TYgAF28N_!Szdp) zBHq0=FMeQlg*Sb=7T=w%0EBZu??3a1amF~->$lSjY$su!Otbxu)D7-Y48hD2JDgM$6RQOKYhbX6t^`k+l zWhj{~5^$cVlA}EaFpkMkhT>i2PQ8aVj<{+#UD!&O zZA6YHpaJ3=Z=p0eXxf${iRM{X*^3xgF`5(=l3175*%q0MM{Kd9+VXN^uW^jDBA%^g z5{5s+(X7d8=^6Skb|g(`s}hwvQlWrXzB}{!mZ&DnBC`OQnuAQj!zE4v`AF$NIF1nh zvhTPknx`Y9HP;yeQPHS#l1sie4xe*z4VI7~`~~j7bRKDWuG}!~H^y->W52&OreooD zjeTEE^6^7C9aYiZJlPy(%W(WD_x{~FW8%rhP6zAN;bh6=5#ZWIhKGND`qD_^+kFW; zGs#1KcrQNn+c&IPo9^Bz(C7DzW>+z;$Eeh(y?3Oc@N9F^cum=LC-3dME}9pt3+5m4 za}cn_Zsd}v`@v!1;A}(}{Z4|>j1R_%aZypy0~rrxGXz3#aEbX;@z9C#UVtQCP!|r7 z4j=|Y4={0t{QNtg;8Tg?z$TB=3RJ%WKKVWYPnbzFr!THS0-pDXbG4P#{>NDyZ< zjVEm$sm}co*Z;lF?D?bWn20Qt0Q9juwQ!BD#+VthPJ3)byTqI(Ny!`XAEOlH+cIr@zvzeSAfF3@LT&8jT&0qVwT?j<=PdfrM?F+Q4G%d36{% z;IShdf)@F#+3cmR0~cy3Ex8e4g<}PDZW8ksms4;7W!;zZ{k-?XWRg8XV?aHOd6Nz? zpwk({dg#51GsZ83!^kQ_$tC9q*55cbUZR8 zYT@ zJ#7D+JZ&4=6Ki;JeT-n|i0IFJankw_R7FMn%6s4fsfj2l^AcI#V>R6DHnwF*v4wRS zfYZ~O8}If!VIU%MA1BR46^1?aXZ4i^Ov|mSHL5laO!XBIMrl zB$^hYBCJ}}kCA6=`|F+bZhj4zWshYV%{L%9WAGB5x;Pndy*UwJ za@lytmiA6vOroH#HJI*}*#8}liNGzUWb!!F<#uMlN>Z++RYS7jYECu=UxryWfw`9a zv3z@4U+m)w-ZOhtt$na$)^4*;F_#eN3~*v&!MYPcbYhe+x&2j}$stafj3@T7gT^l? zgqhG?=Fa*;0-3p9iHa_8?$%ptKYU_gark^SoCZhM2>$FTs(>LfJj;n*7T*Y0c$nk# zwNNMu(#MV@ErJBiD6;%{ftX-QrzGJUAK}RS01-^2bVCWs5o}xGb!?=ch5prNeP=Pr z4;>59&iWiq&<=S60|V96)DCtVuh<_W!(V#{bQ$%v?%(b@%5A^EAWgH2h$sSwfPRl* zGoz|#JTtqvX#zB*08qUqWn#i$>mSq69OYJ}$E`a)n|4~oV}kA@-frLAn3#PQkZfVn zx}?;~cgMkFEECKePPKF-f~53swstesKWLl}ZTpP3>*(R~d@BtrTbmvi{dfpg(|3jRU@H46tH^!F$9 z)m9JDcLW}}&^vKjUd~B`yo(o1(^`3w`>^#)GB0C(k7#uF{KQU(Rsr?>tm2-w*moH; zj0*@VjT?}#0VxI_%PqUC7FwFxM%FYw(R7QHZ%3w&*ase=F6#M}&1&Q8nwfcL=&%6c zOUvP7^3=i@j@%_MQqp+ZMdrb_C984ETg2yB$e`FceY487vQ_$F?Vy>=PuIqtifaCb zVW#SO|5eAbeYK8REhM>{P3Lt^?YMsf9odX59slw}<+Pj=^2b|}1{-a%UQhGF;2+g! zxoa8$1#xn*z=T;a(Vge!1#30qFzw}8nmyE*VKW~`4H|(7D;d2fl7iaK2(_K|yZU!b zbmFOqr|qtTCL1LTI}>%x^LDR>8w^NbG3BP71oN|9e>hG(s+lQbP4r9Oq*t7DF^G;3 zQm_^tp_qc(6o|_DUP|%!<5q@rN7Y5OP{(Xyy3bhhH=TPUNi0dD*G_TN)pOs#9$rB6I^8rI#|+<+CC@&Kc36 zG|f@`I#gagc90YpXu^80W`|2P$Yw!-)b(7o^eQ>;1$;7eY)n|PteZOyd=c>9mR-Ho z7g?AV_z_(0X&en7l~Ec-gY$?A!AITEis@I4Kl|;5PP3u4Pu4*yJa`uBQB&%+hpD!k zBv{(Fb^J?gKxV=BBNF;X2Bz_W8dMC6Au_wA8|{*Pkcf{MiDs}fN58Kkd2=c>;2^%< zN-XbXW*yd7r=B(Y@p}>NS(mbDM_}*N$M#~w_9L?sRh>4d$jZ0YC5P^3naxq&@Lj&d%_)_FnYQ>H=W*I zCToj+5PxxN->-hb-B*KxOW*X3a=t}@AlaWZD|L9Y$ZTAiIzl2aEpQ8sKzJM3$LsnU z_DS%%EPE*`hjhHZQfQv2v^3+Gz#+6scXy=|o+W_~0J>|i%7-aeqoev(1YE3MOG|o>qa$?A_c&zL)VLRF2srrR>-h=puDj?iR>EZl zrl#r$9W2y&7|PW?^6lnyc^Q`LOsymz3g$80txd42I*M*lR2?cmZ)wakPpgpeN= zfTshn{xa||s^^SeTmEB-ABSLe2>Pu#6_2?QBA6X# zy*zWsUgj~!H5TwB{1s(DgJ6Jo8@Wy9=NLtEB!+lSPg)HJjvfo;7$!B{_vI}%#l{*+ zL0c$g1M6m~Oe4BOc9?p)V?~u@DxF~hDxufaE-~u0AI=t>%M$j!JZ+#=w27EyV_Pa6 zuw)gcp?)%kg_(|dFXP~XQ>`$k5s_=H*lqNY&-Q61)l=ww>2zD2>^Bz6ZBJ{!@)UGc zA%0FrG45m40Yqpy-F&sgAor`e4qrMIy<-^;vEAI%MN|-R8W*YLRX8eX7(P_KK9;k)Zmgd*2#@`l6RAsAu7U?H?%A);S5{zu|Pu{&TQ z9B&_yRqfg%8SNCmSLt4=DSq<#Ui{i9-EH1*N`9oF1X^*^HDtb6EX^3fS?xdcghc{T zAiKR(34C_M`I!n7c6)`^4(MIDQo-Doq}CopqAz3U&>K*efC}?_80ov0*dB4r9a92=R7;bLr3ilb-!)uQ_>Lfl&-&d4km+r4) zs!i}AO-2>V&`OyO)HRkFY%C=PNnN{oWP!rMGK~&p#dy(es^n65;dUyP$ z_cKmrFd6e|f#R>pKUp$}4CecRy~Nbz6;>lpJL&kld=HL}j%1I`(Z-bWp&-M40d&XQ zb9emU%KLoI6$3E;0yF;msa?@BYM6LSX9~~maYF=?hl8TET^@Qgk^tUhJ0c=K zQvLeRuN2fw$-aR6AVir}R9g@tWPwkc5bk?9Plo}YetH%6;f!g086m13SByGvmo;wC z>3|n`eI>xJzgYmWvi8lH6dz1CF`5=+wJex*TvbX@MsUt?vt?ccrVm!40+ z7=Q?VBydsVQeOhsq;&ot!!w(jp?8Lf^tU zCNYxFQcGKg^Z+wgY`bkf!xS+AQh@;OZfrK5b@zOve~$8Wuu;OrO{^I{ludytWvPK&va#H{>D1hahu(tE z`%r&6&zg(3s7y^PKh2){_vhAqS7Q_(XPppzrk!30!+?9#SzX9(zu#2C8h*}51G?F8v9k0A!&wB`cwNWF znb9&P5hYy>CK?+2%UzU}nAfv+das=GU~oF~u=9@pb4hImc_lS-dk)yswF=^io{$Ll zgog2@rdx{+*%}kmmRxh&Pa+Wv39#uYfn0m$p6el%OFPZS;+H&cMz@7UM7{^;f)ExH zbI_ieHwNd}zF0U{z3_Ho&fMoR4JS&Ojvmidl(I_xUM|dF?rqPLOj}anhtt1M_^0M| zSz#bl{hi*bk78!B^~4Xe0xE9crc$DYi7pQAcp?wBjY7XN$FTO6{^mRu+4E1U3$@D)blsWIYl2YA>^7sNw_M3fI5PfR$oP2 z1d?!)!0tuw>R$Zw4@clust(6s73pbw+Ed!E5$Tw3#$JsyT1l!J>eqRV8C85=wY#^O zZzWdOoF3(NodG23~d>aH&g0n)jQz!6az5oh5{BE!JcpI#RB92A(Ci;`mO7u&)a}mGn=FUW) zR14+=ODp!~k%cF6a;DZcIRr-Qq6rjmQ(rf=4GC`HeY*F6ar%h<{VCCUPf;u(b*&j~ z7p`=0%#%EiWK6JbPX@3s<+}9#`2Gl^zUv0I)bKA=iNtpJy>iVa(qEB+sRb#Je=@KL zQsksyjy^&ug|w2SrRa&T>^IX+*>+5QyZ!0RT*h$k!CxyBKJXDdtgpWJA>46NCS0>z zIz@cW()Zp6tsJh&i(Sq~sRB8Yu)2-hGA8`<3^^vg98pfyZvV_!a4BMA3tq?cH|tj5 zf}zkqHZoB3$FNEJ=&$kI#i9nm&yp{gQ&s4jg9>KQ&x>+QI)<{{MH6nioq;RIdAXMW z8z~)>rj8;k%m#e@phw@AD22ogUSo7L+eiEnzTJzW-J2mlq@*MJW~4FHWE}nZd175) zz`ao61f?ES%$kVaG#E(_y%rH!M!pa0z$HQbR$T|SHapP_iy?9kGC-U3HZo>hmW8hG9Ur+{EzEJ9_8<|1p^ViH~{lzT*Myru)kwJo{dXw9r&x&t|K$2BPx5IJVHGIoEyf z>17zmj`OkhA(+(_5Yt+HN(vUqFKuD=7BFsp5O2yQhgtPbjKNEM|HIA0lY%=Ux4n5vPLJCiR#E+bn@?N}BK3a?q`n+6WxEds zL7@MsCfhJs0tkS}ajc6UT-vaN9YoT5pQx#!rIFpcY6G5DHEZ1(;I4 zvOY-u5-wIAu=6QkpR1luy8OIJ2xX81GmJXc?Y&{Hy~k3<9F8LUh=;mWzOCnqOCz+07D zg2mKu0Hz}tmFnL7dM+y`=MB2pbT&)UCrX^(_AHm%z0)}rFIbJpCMj)8*b#n1XD$N^ zt8Z6cQIS$w3J0+U)2Afk+0j73N;lbsNJk*d$y@QsrLck?j1uI2*ccEZB!0OK(?JR>+?}#{*wjYSiDVg2z}Sb8nB*K2Xf@S$NhtQr<&}T~n9y38A0~5zWuf zugdCuig5nj&dck=(EYTvN&LKAHN@K45RQ2#IHvk*ECZH@_kXw!48r?RdHGqzBS^(e zyUh=UVPMlPYg!W#vSG`8c=VR3dH-DhaH;KLp$RedoRG`r!Ro4Qk^2>U)&m@e@lo(< z=w2r_W-xi>bZ~xo`Gn#zYJ>#5bTW6auO}B$<9z>L$@K$TeE3_Lf}TCMDj{juIMfS( zuGJ(_-5Jdw1Fvafj=ON3@V41_R5 z;Kh|nHhnerm*8@D&#CAgVN!Alf*pl>BW_Z+aBo}i5o15R){}*=qzwRZD{yLsP zr^AcoP~ENwnu)5z>?}h-rFWnCuuc>vVaRE>!F16(S4r_-H1AGUf6aTx^3z0YihIe4 z`0J=lkk)!)gX7m`ACz_ymhV(dTZ+AZPebgvlS~@jU2Jj1PRQp%*lx6(GF`5g56lD` zL}zaXIWJ%Bb56NZD(GccH8D3gki7>k?1&bak?u{Wh_5Vdq-wbf+nk%NOwTISu_2(% zBNB%qN@rpxCKAb_Y*kIGf4H-$yI4X2wF%5ZV?)Dc)7=oK-6U7M<)+&kh*yri(!`A@ zkwZfR-0}Lnv&C}|5r~M;1VsLU2Yj9W7sSK0cJv>N^V+|H@CdtIEQ$s3$^G{8wEqAj zK=lM5NDm$MJ>b9qJGg>e;HOSPJwE1_t1q6qe2>C=yrwUF8lE*L8$74Q^FRnKck~lD z8vqt!)EgH>a<}q6Ppce(KN;gQE<@{As2G^CgQu3FoLgLB${_7#0~?=|9pp+F{v1yO z`c%QwyuVe?l`3kG^TL`AfI%LAV+j1F62rG-pUCf_>IexOR0Pa;)1>C({u_EdzI6GZ zxHu`N^ZbNoXS=Q>7zG+Zy#lj@<#qoTP{@n(;h~{+$n)mkfbTPM@=w8k z17(P*S*5=RBLhQQj#B7HA2qqhy6o{}U*E{q7vqD5f>zLyw`}=8fiNVn-NzY}aDM{8 zG|Zpf-NL=4Y^o~d=*Nv*B$e|X1fV|uVSdOE;`OVaAG%$*|LO8yNYn4rD7iS@zv!C? z@IgT1Xdq4C^sAx*QzFR1&qa#VV#8%F_pQgkw82Aw$8oy2!hnM)(>DsN{URbF0in!FM}_6hC|fez{R5;k{*@mZllkN`%Bx5B#>M1|c(^;{UYP$}t^Yn=r2Lmmk9`!Q`X?eDM;-86aEK3v2t%@hRtWIzScku%>OId} zO#9mH119@BpWVV6d>kAcx&8cEHI!(Qy40f6M#u(GqaEFV6*T?q{Qwu3033YRDyKyt0akbsT|Im^CZ zan@VlS_wFMYabegJOAAO8(3!NvBfg!_b2fkIPqF@CvNQQc&W*X4KZgw4X68WWc9B76<-tgMCcu3+?6`NP%CV8tvC^p1T%hd( zH9n-I0*w&SWv+-`dT8))hVCdu?$_|}&!;lik4QW&PM^f&nJ62{1H z1rFKza=RO@pu<&#T!KTK=*R2K;l)jo8zgnJ%9ia@aE=YhZPqw4@^NwMFL97Qh*}jA z`K0)R`T;m>2L=KT(P{;OC@Yu&1DD(%@cJ|E63`?-uS8*x%^g)$FUNh(&*d%EKhio| z$*+!4oqSwODWM$2J{b_EJdyM`A~xIqRcccRQLRpK%;Ai!dXe>_Ps}!emY8-Bu|h3C zm>xk04uGX*q-oxpwMr>ty&cxkx8LV>wh_{k37g3H?qw0@B>MfsL8PeAP>b4sw36j@ zwbRJK%Gkd&zmG1+U!`ZA{wwr^vQ~% z92_7J5Y2~xL5-=!#X;^{_FliE7Z8H9&;1__dwp?@qs9C8Z`Dk~`zA8((KHNk8nbV2 zo@LFc1`7wEFC8pGF}p?N9za;xZaBKSj9%rAl2cH?am=#f8TX3>)bvXAPE_kZkC1rX znzt_WKN)y_uW+CnVKEi8k#anL(L^g$)tHqLYk&rOly|(J>*sYxRZ)tfo zQ^`7zI_7!y^!B2|QSJ}g>f5H*%wyFUtm1LVa_zCuI5!_!Q889EeceXa*qU2jxYXi@ z3nY7Fw*U@u!j4a3ulv zpla6AJU_oJBhENQk=FXx$a});ZlQ}Ck3ae!5%oKB75NQs&Mz#tp|}43Q7G_B|E-vp zntGJ&A6$U{{3req$liO`oUYhx-wQ2oL#Dg1*rp4{Z3670!RKG+4)(>A&6W5Ieaw zYx_6%NgeX=rCp3xSo{w3i6dtrr0fis)mkoDc&D=v%`NN-66qEOLraxEP zhs(x%PERl?CxWQ1w2$91ESWTtex7TxTp{ok zsKVrmh6(0|xVg(Wp6VsaR9HDJmpP|+Nx9e~%)h0w+At0u#i)Glu&`xwrCahV%os@{ z`oK(QKs;6P4QT)Yfyc`{sJM8j!0i(-rrsx*Ao-J8RC02F>s~$Hy*~g|*AO-S|3U%0 zfq$vKT;1#*vmg%5yfeB_)~JJj2S{mV>@&#H@v{NM>jM>L1L9xJ!pOBF$lfk`wVMfL z6!Pr3u6S^;xNG`as;}<*<%|=bx`fUm#zPUSHHbN$&<%H{C~Ba6tc}2ipX?x2%E4?d69lS#4Rr+)8=gAlsohGX?Snv~Uwvo;GF)5I{+P@rrT-SHOJ ziD~KLS8Ba_hDS$TC0}=Mxb5hh%~C$t3+>r-xXn9(u)%zWb)PCmYH&H((9m$P^LcV> zW^&R;YILh_=pa8B%~ghjto7wLP34qlsj1_rX=!PNw6}{Q{LXO3VeVr`)-_ao5|&jJ zVUl#h4|J3-;YE~CXmPNpalCUH9l|kosrJKs8bwg}83nY6gGd79L1~4c3Zb|_?9O!b zRw)6RzpxXIY)#4>ZgZD2ve0mZ%3|WCY~b1`8g*`F!mAl}cD9t(`MG=S0w>2(CMCz7 znNM24W#^Ej7q3k+ruQ({47&Q+ds2_hl!oSk_L~{u3);c6K(a zU@j0$PrdG4YJ6(vO1GySa_QLDky|(>T9*_V37^Y$)E(%$r};(Tx`PA)QQFS2?q=YC z?mFo2tGgb8nLe7(cfZvj=eHW@?^%6n5LK2|-f;Ar$ElQOp%8tY+C_piP-8=t#$!cZ zV~9PlDZBWs6Xo-0=v%7&tS`0MiWgrqqjkRth0uim_=%bD^WZe?t*Kxr6mP1rbBkbT zd9TxGK!Vxu8zIL^S6J@xtL0MIK+U$vMzwNJrLA&BYlK{RKG_tN9t4PibEBxk{y6iy z`YMB?w9%~&%2LNBYqnTMghIy&dx1$pv)+amTuCAuwr>lP@E#o=AAGq3BescMOeg`X z$g5jhyJ5|OLA40-Hp&e}#=;FBL{rk4&;m`KR9RY({N-?%qvq2g!5$r>=&&XUPxi3P za9ZFi^OfKOUbgC8E5_5`#x4EnAL%(+8;)71!B;!2#;lYdT04Lmg=&ngqdjoA)Wf3+ zT4x?7^<|=XNnW?>fWoXfX%q?#n4^RDX)b1?ID64Vafx7&(sJyt@f667!IfmpLUkbh z{+|PDo3U|KSFya+3vnUg%_e8i%B@Vt)Cri>Fy~dobxI>mmM@H}slw#{&xLo&O z-?7LZjh(36cjv9Lp=7_avmH6|_kSsy`|@jIB3kL>ms(o7!zC0Z2*jM70L+$x1qqyL zku}VH+Q$%I**YWh_ALQi!UKI*D%|OCU~}2;Whnwza@HfJhR?(l9>P&(`n)Wsi8E($ zIS>IX!MLY+A`QPIV zX+BeZedr*k!HXneLtZ5AH(>e0^~gv*Y7q>;C$4J)j*$w%<-J@9zc?8VfAm^-=HJRfD#UmMw7H}B zF4yM&1XApf6nRghHQ)IcK=z%stR({Ao6~i%rw=GkRqRe%pKN@15+Cf_5Pe=C!aCQM z^vfl}-qG=UjSB_;#X>(^LFV{pKxz@5TKjYrz1)VU?IlU|*@=&i4n^6_vDNH+ruGTq z**+pfF+aCuk7KLo_>?F{;I2`(Lr*_iinCl|~+GH3)iD&@!^@ zXZEBt3&ZhBc(U-`$nzoP5cWn z@FEqaB(kg`Y@Cwg@M!K>)B3UQV45GN%>@!9W%U&bD4+ot zrY%TDT4(qr+lSEA0vno?plhnaTovR{4iBg3^fMtc84tmXz5|Mv`V00oE|k`;*p-EI z{7rLsCIppYK9*ObMY}eY4`O3!B?kskL1C zDr6&tHisfYr~ad1gF)l8&cQ~UArxMPNVa!&u4@YZRt7KKJYiQSX| z$*U>d_z(ow5)3bprDFmpDbw?W!aRQ#xsc-*N!}l%p?M==!Tt-t89=4sB|ajFiy`X| zb5**pZ>-OHo^mEiLz+^31DL^h=zk4g&?D|AoR?dnrHIc}i9`z(!8|8qsn>)e!-;+F z8b?vuZ@S|w5eETjn?Us9UN`W}VoxWM#Igko*?G#ORc>3&=0aQcUu6`!9^k+jT)3tl z!S`tokHb4_FT};G(!Esl87qtnpDyrWE=0kTgk83Tzcs< z(%^usLWRx-T3D%&E}H!8C0Pe|%&jN3O(Q|vTmj7du>N`Tnobsn;^D78#cc%!P24v# zYa)s4n!4(u8%1b?FiaSnqS2ueGW-sI;Yb3=x$&07sj%^yr!2JG$ zEw4GPlMdMOQ$-y!3Zvhyq86HN;V5!8+*YPJ1eY3kk;UxYk(Ff?^osbf&CXrF^JM7B z$l(1DCUkiA->d1B)0&A*Hwf&$JtKRBnU3}s7^@V6<*JnRtJoX4`rDkO>DXns5@>k71IU23wx>@#ZMmJ~WjfPCzu zi_$8!9aw$x#oIzdP zD#k`s_psOXd#7&~56~<$__KQu%lRMaY_$q9=lT)YT#$xeq?~D^aS)C1JMjr8QwY+C z_!kNB-Sl*Yu~aRL_|6z z83Gg9yn51__xQBBKK0W?Gu4yu&U9Hr_>kA(;JBa%ng|BS?-eI#O=iNx^+?R<4__@G zbg3*zsH;&x<<(UjTledI%U;V~TtZa(A6B_&6`Qm^yb0+<5tGp#acQR21Y3G&w5o|N zsgV>pU~x8aV`52gYhavBFB3uD|LDfhiQY%Y7m7KX@jPB%pz~$e5-eAlX=l;AWB!sR z)O$+z+4}f=-O_{~{Na?{k2p2HA}-Oh{X7HBa1qzM3ReIg!+=6m%^nRr*JI33;o}#x zgGSUtf`xMJ`NR(p3P~S$J=}n@o;2(3`t~k)^{zY#KVVl-B%>393)b?Imu(tc{@At z#Wc5fqvStFx2jzZVlXGHJ@}N38cY}qHSb?UCg~`=d*T6o+Gs_}7{FsWz&eJ2uYwtv zX8=duL4HzGZ#|Kgo8q5#=M*G7c;|worZn>@zxdH9_37M2q+P_5-B^T)6XdFEmMXX6 zI~CRl+_ZO&M8ynf%3i^V_Gbof;-i1i&8jKcq)=+Jd*Sq-3 z{fH~*creJEcaF-gr6v1Wz`ZdaiDaL#cHYSmmMUbc;p&>@M0`cXBWl4$_4S-=$3DRq z7AQ7{i3HnXq&XyItd=bL0e3%YBO-U_K!-dk-dvxsr_9P}(G||YL<n*HQ9DdC1m6a zcpR7OryC!UMLiK*7#Jsd&SYbnuYV%vyHu?W~1V?ar4k$wT@i8m^Ts! z`wUMO{8d6QPVdkb>xi{C_LcOJ5zi}X^5lv3f!b%l5yCgK`B9RK7}i6e&f=sr=L((V^s14#Zw6> z!wdP}I}AQXU36v$3M_EEPkT3NBGHx73+VL6jV|C%;sV$w!Q6IAnI!7A7G%n(*@90< z+l;YwRR+i(efZD?;?xRp;U9E7?|Jz~j`v;2H(0Klb>70u50pY#lT9Npu##Xd)OKVFT?C@SP@fs$krwfHrjI5O|A36!| zKKFyi6j!1Bz8*;}t3le)ONneUIw)hBWNxFYB~F^x-$-$kybhk(ZEDBAcd?_BF4@~* zxao=t_?XatkVKIl2sbP}S)A+iz;23o+?58a77q>ZO+H;5pcx-%=QBsJKG~Ys2HOc< zynlfB&p6bKil|Q(5QH0eUoiLhD2KJ8QZdExI_;Z~cw8B~cJU^WpuWE=mPhhS>fT4< z6+e0(PWN~QT^XJLRsOAyQUT?s7^U9l@frYC)YA&Waz=P+$q1eii?YGdQHL{)xR!}r zw*TcD#Qcy3gChjkc_R+~daPdv@P(^&dhild-QNpWOMY7T?cWtooe(L925z|g zN_HL!(k-gS1hGdT_k^q@$+zl zg^O4-D(LQd!u!s)jOjggZ!7Y98TRUf_oW0Jcy!GB5`iwZ;yM6tTztR^pktp0lzvAr zP(`pv8Mqky77ojWQhQmpqewtnU^ybG3p5;oLw;|PeB-Ag&Qb0>FoVUJ;Z3XOJAPFE zWSm7IY+;RV88=!oN|@pe;*0=bpj!PcvJCZfy_SxU>I* z+B5d;!%lay{t7ot@-yVA6M`@F)@M;oHWV5veCp1rI?s^<^Q&`hqv!?i37nT!0ErX> z@d>Bf3pzPGCF4*v=d2_YB0a42PklLFM-k`{^#Bg<(iJ#&J3!F zgvGNDFk~KiFgTv8Ox zeoj2#4|^BhYdNvQ5A^tZyL>NpdR4ZF_tGopFSINCkXX#oj>YK*%sOuf`qrvk>I8K- zy>*n}ql!pP$o4@Oj9y~pt0Wch6bh5V|`7QsZ` zOuA`6&y5b=<;-XnFZ}i*bqGiwySp~Fi}~%qy3KX}O>|+c_0G4+fZ6@Y;wwH^k#Y-W zr{L2kx0zCmKv(poGe&3dW0>Ej4ncRzQho@A@cN=UBfRUeKz}^cj{QWs$@}ioIOI+U zuye8hl7qJ2q|{g+AVa|u?s)Wv7*-nVoR;(qaAw@8(kuvsavxfGIy~bQ0S~RJn%dg( z=}X|i-Y^y39M4v3svPkvK?+Db@z5f$br9IarKn1m1r;gZL8td84!r0mg<8EFKV=oE z#cm@PAwEho&D%z{7YFYZbo5GZm43c{?aYktN>M>_jkTEZP0)l#b(IBpvzot5evuFL zj%678M7vP12+B+f`ao~zYt@7IXY&yr#SCnIqj@y-J61jZIOkfjX*z*xWj+-D3FrPc z(~u2E9>@ZHr;eo$jS<`u5O7$KbtgejlrRrF$B3lO(~5eSuZ9d1s^)wrb*m>W?c<*5 z42rZrj+sxHbK57(;CcLOEN>VP)NB69H$ae|BF$b~Tu?y(_Uu|e-OJDs2?Fk;)7rjX zM~T)e@P+tQt;vy~tD*80>A#sXJbnwC<~1b6TE?88O(oMxF)C?aVh*-+>QMr=G)*k+ zL#vh?w2EPKqq2o{51N1RkKFb}jU2w|c&*8gBOTH2G>iOGT60zDi}$l3IkkqXELn)l1T(IC*>bTgS=)cPC3j zI#1ehz84ih>*n2L{jhm!?kyD)W2~DAhx3b0QsE>r>xBQOweyZ@YK!u4^eNAVs0b)X z5wKBhfOIUB&>=!3G@mpH0s_*jJ{1)KL7E@|B+^4f2%%#^Kza+2E=`0`q(*9H$MR;@ znm=aVthozWS#oplx#yg_&))l-^ZVWoPbue49C^w`VmO7Eh%*;*pEdqD;oHg>=r0de zH^TYyGa9v;5RjCX~RIkv$hK*q**CJil=pfBy7SHi+0QD$$1e~)Nh!4= z7MGIup(_`#<8tZ*KNoHU0DaE>01Dt*hlWzabf(7R1>nr$!av;_mzc%#eGg39K{QrNwL_O{&MP}(yN1;b#P}*V6{)R9SA9qq~>y;X*C~nYqF&Cd2=Gx>ZuA^ zx?A^5Zk{`E7f{_gI0eM+yL>Mr6kEYJz2G_5|4Gj&Q^=^xL?Jp!GM0sr^$}2lZkn@b82ta zZhuho)ChygN?IHF!NQ4*a#zp-avisnPn3s$o)+-V7QCjawU z%06iIw~^JuGBPqW^pxkLrYji%z+|K?38i<6k8B#=i!x&}oU8D{;--aWymDTZA=zar zUrdmi8XBO`U~q8AUTnzzIZ!mW`0Ut(GQb^4XJNaBRk()DQ5i);@#p8}C?`w)>Z09m zWp$q@MNy}?I4CMfey=SUbaZ9ge6wO;-|8woT~s9NHt0g(=f|39>Hez|qp=o|!hoRr zO0Ntw1V}6@sdV-F=j7Y*i)%SM#!yLtn>KIb;|x*#v#ov9wOAOLz$i8W?uh}WdyHA9 zY3KiDf$pjs((h;f*I#?Fz2|h#*y>Uzxh%wbg1b3>^g2^MSq_zxJ8I83ls^QHk+6;R zAwwXc@8{(ueEW9OeI_7eH=e)DiShnmFv)s$A8Z$ZH}gET9DqE&(J>{_2?^Q81k=pq z&o3m*isl;PEBsfnAS2REa)^x)vv`qfURrl8@4M%1#Ao%0hzLxXbsi{%A7Rp23_O;Q zt|EwzH?T%ATH&o2Rp)c>(m_N$wlR~II6CQqk;B#{v`o_zs}e5Qdpv(E!x+%&bzO9#Q@fz0Hn#0ZAW6Y+{PLrdZvsS z8|%^Wkr)OLCwVOAv!A&+c#oJMY*pZD8wP&`kh&+(U_2X=bN81a zu1dQTz^MNG<6x$VLy6sQ%d6sl>3_1Zx1Xr!3FrsRCs=0FdCP4`-38wBN6S|&v&)<+ zp%tz}yG*p^2j_5xNIQ3#73#UpX|Nu+87>jXn2P*m8mZIoHsecX#hiPK^00s;f`vIE zNbYmjlUdmK#6naxb-FSr*=TAVrr{0$Y)&)v*jo5=tioK@Y%O7%?TCq0gjJ|&faD=LESOr&ybtB0H5_zCs3qVCJMR~&<`}Vz1b(S!M9*R9g#lM?U-_3tQehHTLqVvBr zp7rTC&~-z4-F`Nq`?XC%7_Va5f7;HSpC_=EIz!$@)Liq9a&^zdW?4x9t%tN!m zWI>@+VX~<%KzH`@@TB+l_NqvT3t;!@Az`I?`>>GO^$(AtZF(a2J&nQzsOYAnKowf=V`=BTIMFIZq?TrUal3YG_1S7KXFwlEYg}>lhHCR<~w!#W-Xr;eO|NSEmYOS z%;I*2(65#gy>rmJ^UBB-s|d@zH)=Dhg^1kYy_iZ$y}SdFY5fb*ac%Fm?j#m}4lL@M z@(`7wzmuxbeRkR?TyZ6irB7LdU`mb=7Ppv-vDHpWi4aQCPgV2i3uLU5N^mOKx(e*< z?9p*icz0T-kz9qTQ)Oe{ONC5QlJZGuedd^V(RUhJRHDMppmobgQ}lN`_H( z41g0a`>dQN%{;-w10f@)%V@eljWT@p>=~&rP35#4&b06bfl5v9+RssJo!9NxEpuf~ z^q;~iSLvh0au{?->vA!Zu2Wa9l|)+QFwW-);g3&{yedKCH{0} zV+WY}%WBn!-6uaGTU%Rqu(6@()8wxzD{nEDrUKuEaEr?;cNr0=P<0C_zG9BIj};um zTyyejiQA-qHv(Z`V)Tr|3_SMYgnKE5>_lK3IHwU=K+h%?C@Yq~n*w0Tc_x%>1o<>N zCMFH4bG%Wx{4m*c1^DH+K-3V~_Gq+T%5M5l*9}e^+YK#%$+3Roc0lP2jU$0?(Qir8 zB$mmcoyZI0E39=*N(Q2|?Pmk4+Q)Bo_R!cj)PKFE##_$JLDBN61uF$VJ^58+Qb7LM z5a(fId*rCEj{(Njq_s74SNuLiha;Dy#S;qWM0>>u8u2ZWxE$TP15uIc0d7Uc$x4jl z;{2)K{Nt`P&LWbV4p(jt0_%tP%wO?^`UDfV6Vc(| z+2|2ED-&?_$^SC&_6HEFy>N+RmiukOGMCM7DyNU=`k*}edH;~MV zxBN&z=*wcE_QLDtvJZQ9S`0!pKNKc!vf8B5XPtk~SV$`pNSLRMnu#OEh>Tq%<^$HO zVMzy2 zPXQ(yp`Y2-whvGx$H_h7=DT}&oqSd zn8*LJ@BKVk`%R}n_r~+yX>Qgbk5~}}g?Gs(HU)WKQdPB|D9Bvtn>l*C+KCFB-OY+e zD(&0w4?F!{VRi?j{$%6ZwQviOtoi7X+3U-5jU{M<(#m+ERJPU()?GK zmPZeD1VfMJb3lg}9VRLBA~F#F&jD&?=NYIZZ-G8iRR z2HGvzkLh$gJ7XtWzB8pUy!&f!@gaiwszF!b?LTHv)5|;gLg?9JukYQ>(2AAt8KG7J zXR_6IhH(0p4wU-aBnZ(`y4`8QNFW8GEc&;W&o`Au;`8mAd=v{``w%6%3mr?PZ+y~U zDk&{3fdV5- zWP?^7*op`F9F|{=@|e_kD$J|kr6Fo8fd)cof%AY8P=!#H>r+$?9{h^-@dFU zR-OXa(H;nZ7rmYfFU~N|tPc$aQ;J-khw&+9k^5Je5yKPVaN7D|XDg_(xW1H&2W=Ss zCyF(^y_Z+~#>Q-V6ex!LennqOe442%AJI$%?Wa7es17A0=nS^jMT?ToIHn8FkTBsm zMX8C05!n4azJ2DIxau+tYU35ldI~DNqV%XzTXc=Uce$CnRa+Dzp67VT6ryySz=@kJ zgqkarz(FA13iUhTryI5bv{>)8Lvo(ecyD4u`)YPSmDjUBGfjsZxSLb)%7M4>S(Gf? z#5}#<)JCJ%V_LhtDX~ok@R7?qzc|#kyj`IMG8$=PvUbz;*#5G6eCv&=k@$#@LT9aO z_pnw3KB%FwsI0%?db}xv>%zqwj~+aZ>T{y{)B8k`ay|6MUSb4Gm=`aGU5EfC@IsHBzLEsZ|lmE(V{z1?B_%M4ia8031-R4DxMtqMyXw zv`Hk>+xp8lyq8V}eSVinLQLFCVcSJ%`SZSJBs>%JQO&7O5ou6)XV;&N15~$Y$BrEZ zgY9~hRCQ0%jV`7S=PI|;$jtBs`!QXO$k%$-G+5eJGE#z{oO~LM(~4xa#<*onmPx*C zBpxIOOyQl0xHTFUq!v2qvr#;D>#P2KSZ#c(UcAqJ#{Z9n>A>=1<8@~!r%&$|6&ilU zvg<%{f4MKxiCi-b*6c7~$6EFzK$x7|C0Ta7v`+%*TD(Ism%NrwdiU;~LKqALdnSDr zOZu$DrL0yO93~W<7q7YFbbM$8a+ZDjQ6HOrQQ+7SN%ZV;ppzvz%}}&>P$)$ltvbQ! zvb8mSqim#E?$+^(^6eg3fsBplFz<82uka2IEsu8bNNa;`v#77)*r;rGrE*`ow>LKa z#(*J)u;270hJx~Uy3`wRT!vmYxpV;8?=S;C@`>e#$2v;UlIE`FhLG69%Jvs`M2>%erOWp0Nibe`T2S<&NigGnXPR|47k}56?T3absZ;= zHw^8?{FV>p=H}x^>dxG{c){_X-uprdSkKRb$<1eE=yxiXS9lgDC|dXKNkm6QU0*Gn zGLqv8DtX8>_k%D9O${XFPEOufoa#ZdG3NeiOpw>>D;;=Ykcw-0x0NV*;}PTIgjfz?LOt9gG2TEfHm?YTa(rHR=L+wL$n$cWj=Q~l5povh()ve zhF_R)CSVM0+?H$_IK_wj(9f}HmdsNC@2QT2@H2#&3deg=U(vBU;e_ccxhHy464)YIW0lrTwGRHWi=G2ZI z&sPk33QLB=`lWEqWAe&q=r5pBM}S4ZM&U|JJR}Ftzm>%4I%RB1=d{W667!EAg(yG+ zhpyL0bCEgHi&w5#PzSW3HY5s5%r0M+jv>+(rCFd_US;R7h+57VrCXF`dC%KUbmiN1 zB~?{jf+5tDN$hS8+;AgdDVXgKx5u9wM}GJqwd9lAtUPcey**PsVEq9k!P0{Rmy>M9 zW~I}J0q3aGiauuZ6=XNR*$-TG&CTdJb|Ff>o!*gS;T=^-X7?H@@iHED`&N~Kp=1>G zdGrugl;87rxHT(hgGs`yGID!U%(_VVjx=f5@F1$BWC6Ydp_3cYp$Lv~hdh7IvI>O! zo$o9CuJn){sWtqJi*z==9LNWsnD5a>8n(~yo9IU{mcPw_x6%6bRH}nQEbO{$j8nQO z5vzqt*P-YIcB8@Zq1@MFm?sC&`|Lxxr8JeNR@n&~Jy^%y@6V{ig^qn`HuW*~U;nsB zx;p5`jI@2mU&fA#h%I=R_gBM^zs)0>a&?qeKG6|^UL1FewTM&9o@`DHUsr>4NV9^X z`Lefnyn;<+~N#Ny?SPtI4TJWF|!kNGPot0sJ%eGGOH63|` zm)q>nRujh*r70~)*_{l@Y)=z8dQ=T^$=en#BRT}bN!gTY7=C^nq)g<~qudJCuYgy- z8+=|CA322f-Cr{zD&iBp#ZUQsCk+6qjs@lO+qm5b_Q4;K=r$h(7G@;y_D+q!&r2bK zK7!5s_4)ri*Yd9drhOUg%4^>wJII#Gz)gKQ^(h6~`A@6}6e6e@Q}~#w_yNKW6Q?^G<>%ePBPLitbTsRCiu$v7z% zYO2eFm!}@2p4x0u%7QoP$waC9&o>5uW(qRWW2XD1Q(uXO$- zWIe;{*E8TDI`UA*c$EVb_AsFyh+TWRxQ-#~YR}z! zTEfT_R&J4k^Y5{VRIX$5k8VxgxIIu!56-G)$3=(R^rW#;+W0s>AKXf=eQ=y($%*fY zo6N2Vr$-Bl1It4_?@n^;=aIuVKYlzKCN#!+uqL5@yGwwwy3Url?p=51Dj7Ud+fX%L z1A*ZybNSYJm{EyPLhB)y50V=Ts>$ihs*JN}IqTr8dp%_^Wx%I@P~GUzt5j9PoovpW z0gQAx*Ox~r3YT{y(?$ETyW=xh+;&FASFuw?`0IOBbn#BdBlg!;s^2^)Z}vjff;UkGoH5`NF;Y1LB{fKEiwC=)9LNM zmw(VWe~;Bn^*s4!O^rSqI+^HdvRNVZGv~p*DHqUJOBA>yqpt@V<5H|VoUG8diW*x^ z`P*2Fl}eRa*PiR26yHW)mBk7^q1p;Wfd<;m;I9wf(`lb#lU{;5`gO9)O)6d}cSWKF zro|BmZAmD5($u-+W{>alyQ2qIs1W2GOJ7JRYl*y|J$edWLdFK)0g zA-{izc(LrwZlpIq>LzJ^mp!>V?<|YUub&^EckQ#NylW{3wXzL)>HK9ZmBEG_(<2?O TJp$Yp1Oln5dnxzA)jR(IC7)Bc literal 0 HcmV?d00001 diff --git a/docs/img/grafana-node.png b/docs/img/grafana-node.png deleted file mode 100644 index 75086ce668239b7dbf0e970f92d478d6a35db551..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 245965 zcmY(p18^t7)A$|Rwr$(CZQHhO8y6=R+qP}z8(c27llPwAgMYnKvsGJLyW73fJv-e! zpJ*ioNq86>7$6`Zcxfpy6(Atc{(lD&6vV$PH@}yPfPj$jebltvRE#}|oLn3&tnJK+ z+`OI4iOjvMEr5W$w&JsGGj=-QC4OwtwLuDr8u5~d<|h%FKJ_ZMJhm+jiaGU_vCQUj}6KdX7&X>d}Iq&tjRxJTsoc+3hWcQynRIy z7H$Rz?)OnlUL*zwd@7w1f~L<)#zS+>T|M}3tP7<#??b*j6Y)se^ao^}J)D1x2eGf} zS+>3`zY56Y)!t;k-7eb9Fc?Z>Bmb<=IvZD+i9WnMIL0}OT%JA6ufvP!?1z1-7pS~_ zea?ri8=e)`U)a8Xr+7Yo7R>I?eI*_X23q|5kpJkfwd-Mg;PK~uUqO!D-3;ztJnTIf zEYv`^QW&Z~n%)%#cQ_$S(iJzSJ@?U$7nXeYg{L{3Vj14#VmQk4p1HXm<`W_usW%_z zLl)BOUf*<1-lRItee~VrNK%)jrzl$M1#)=`6!Fogyk~p%uo7AnoI5F-He}2T`U?I$ z4)y!&T*6jI+b8zxjZqwbwOT&OJAHS??i(K8zj-Uu*#zj`}SG1E=wRuusC0e|7(W}zX9mjV&+mGaS zDKYZaSBA6`ff_&ORwy>@r?1PZ-*m65Y(}a2> zK4s!C-2Aj;&PX9J8vE4SR>h*lqe?S)YpCC6v3JDteyv_EPzH%n>ah=3haN6)n*ZQ5 z%0Zi2Y#1C09Bo+_VHVGNe>+`2FLvQFhVUiWvpOr7NYHBjr!e%&uQ1F z+pao);&P#Yzo))qRBM8nXcbJY4s~kD>5$KO%gJ2db0KF+B<=9jE=Tu=pG5uHhOvlG zkv+NbMZLu(Yc1dJQa_7KjhCHexw*P>Pq(1jY5tG4fYx?H(qm>a-|p#CJ^aTapp>V- zBbiX&axmL&X*QD3G>ateIkf0wg$kBjclmanw{@C+kFC^@R2^<2MdUHF6ica`Pj0l` zr2xKG^rl+&&S2>_sw6nIzj96XY1Pa+nKDF?G^EO{D4#N)E+;&O<-6yJ6ERCt)gz}d z>>f_!wjK+>Bi_qoROUz&Q-b$U$}h{AKG$x1zC?Q%?w+wrN3MUb8o#3! zwijJzp}X*P>1J?dQxB15)rEs9CWuH<%_Z+(p)Kik5nbXI$CzLD+Ff3G8Z z&owTw{Eylw)o1N27i4HNlwZ^{`!re!JVQf2rs&cc*4iF+SjO?b_3k~Svr!Bw6JV?s z@>PR+!TeUZg}s}7dbpB=G$y{8x5>$bXbq7j=;Mr6roW12J#B6s^`|`yDy^BnI@_u(nf#- z#fV&B_JL4DL8XkYGyF1mvb`G9iCO~A&qt7sRFhJ+4aOJ|qNdE@zS*UkCd-xx5l^xq zL#^!P_r;;3xxuPs4_TNBM!R1K8WO&vn4(7NY(!~mt|1%$xVqRx^3-_ci4D57#}JyN zrn$w`bFa&_e@4UBqwm^Km_?Iy-g=9{K#B$9OWRVwbpUD{6B;rzuOOo+`i&?qIcS$N zI2fz~P0-hqk-VZxtz$b_3nv!-5*w7906ohi!lF!NXY*?ez5LhfRxCURm?XL{2JM2VxytN*E+4 zS)}}Q?a^JSfn5%@?Rr)Q!ossR?OV0+h<{ z_}AN!!9dhenNL1GsuUwckm#iL9-B)xb3ET)4P?l8swt50Fdn*`)vE2qW)`AMirckz zN1jC>jo2d$ug0q?$`~^KV3snwfy{hv^r=0D1X!+o@eA?9N?UE^L;N}0crHkNxrUFO zGRTaFVnktr-;Z*IM%z3}*G=t}@!ztDo@<<$L_d(xG9l z$1G(eIXW>e~fq{pxW%O%pj@pgf{witTRm3FNbIw2aOQmUbgT1W;bsyMo%OMG*TkdoL zlIIQA1f|f;lS-V1J5=(Ky>&j}F<6LN_&6;iPIWbzvH_eATMu4lE?ld6fvWek5>siv z0wL#fla+a@ROlXPi{n)mk;D=d??x7ztf#}k?~b>j7nIXt097$-J-W8xD?D4dANU*g zY1U(poyvKj3YDQNJey2nYCpuH6$$g$fmy$M9w7#SUY@a}h|I*mJ@-^%wQsdpnHzjo9O#UHxeTa+c$)8(B?wzQRZ|HV1o$b)Cd5v{ zUo^1Kn1f1a9$2Z1TmpV-7Ve!cidW1?xDA|8Cq2}! z2?gYE>9fV3e*^RT+52}z)1A*RE}m#jUKG?T%Ed(bG-N7Bfz4K}KB6$n_j;tNM}9N~ z(qQ@mX*vlBbd~%CgVh329xo)ae2~vSQCBp$S>lZgJ>a8N{=d|)N?KaFbg9J>V!B#8RK`7)0a1RZQFfSLq8Ma6uv4Y-&wOhRN zNaoRp875Hk2klcxbRpIIhMAWB{6cAJKh$nQk0ZR)2u9qpm#_dapkG-T(r6PUNMi9~ zx+Nn zb|Z-tCqAdl19#H?^1-O-k8s1zlD|RfL_vZTZLFo}F=Ejc=zv8pMFX%3oUhXGUxoOA z+n5^bh#JXnq&-b(kUc zhjkE135nz$gxxWmNuL=HtPrwVGH1ol@}zG=y?_g_z_5Rqh~45dXYE) zS{<9#!#aMN{TCQdUPMat14MPO44895UA>vqMH7UhzzQ}B+TJE?V6(9LxF-o0Qyh5v zMUpfhggK^Fu>1p1K{Q(AYY98@_^M9*AJVk%4%FkL?#Rt6X-S;0ykBH(1|E#Uz7ya+ zpnrxZWcZ$h<}1gVNoEg}`TT&_A|)dh{_yJbLh(9%p>qI?)T;*3M~Y!BnVK*oF34rz z(IbG&J`l^BLzp{?z?+FB5FJ1s6@jDG9lX%EcNbGJ>Bb>&+-Qx=0R`OuqnTL|-g{`ML%7+_BBF&N1fItu@B3RuhsX9FD|AVB!5xMgok{@xbX=!|KW_OmmAV zW5Om-{Td=e7a&U_#3{`>_((xlfF*N6-CjqH0Z8SJR5+2~qRp|DKlGG_EFz&c3x{A5 z9k@#hRu^b<&@9uyz*_7+EpB4aX7K6Dck99Tzy*I}fQx8oKw~wQo<0vWXsKhGyaP*{ z8oN87M=&@krLcdph-2_4iF{z7iRB(qd2H@pBNMW+}c-xA7zo}2jys^gkKeq|9}PX)n?flmS24|ZWkYm-Kj^N`MR zC2Sy}#ldK8)nIl7|Jc8R5?s3h3k}vRjY-EWjauo)(vb89VFpDhh>ci*{jSB1G|nyh zYdJPtytu7!IgRy;BpoH`+-eu+Nl2Az!sbpXHkVTzXj{pWyTJ07C@99gC?y%3wkqE+ zOj+9+w3bk53HxbC13CJe&p7` z^Rn(|Fj!D&GntWC14($V1eJF&AyJr~nLulC_$HT_gC&=rxe<%AO3(R7GY=6cgNcn_ zJek;#Mh+4R>y+-a+k8iFg^N@yAPKRqo=hpZ+;l%&U)2Wj4yRgRVPKRU4RS)+fH(Ze1Hw#cT*J2psCJ`73H2UR`jwfTG^G z9Eej40wLKn@8Sx;1K#1Ja)@$3>7NB+LJUrcci?x@xXfC!ME-qsRaXfP=N@4ZOh>mA zQ-Oui5Km-;BX2EOGUVz#xXmAV9^OWz%thE3FgSsmh(smCN`uJbu>c7U_7NfoLAj*N zTR@Z(ErLjfV{?1YwPu)ZaJ5t+(Wz_AWX0x3;W0?o1*a5IhbO^>ay znQk&We{8nj6HKtPa|&rRCo=1Mi<0u4+svm=dD-+n#lG7BBvxO1pYfoSLHq5Wg^E#! zn`^hUwkm6ANYG)BLWR6^Y5ySW=;~J0*811y7Fp+%|5x7%tTf@XTHf3g=1Cu-V}KW` zI$DREOz`)4$n-ckH?95O+kB;|!C>|uu()kjgl>Hg+Q2UffS`#Hjw#sXm)ZWi+jAD2 z;ddNpLItP!BDlH#ZTydb{}S`;fGh$RaNAvC$yqoL=(=_Yc3>vT5F+)af0CP|c-XsLAzxcm@d`zCWm^Ev=bm9pkA%OvxBN34=F;P)d6QjW-OOZuMlmPKS zt^JR1`DKKxFNA<_q}M)0p8WFe?p!sR@H1~m@BKRWQrAvAphc^+`+FA}8XBl@k&kV( z|I=E2yW|Vz-*a3EqMDljB1Zm>y6%q)-hIf-#2OO=&6JF|P-5_aUlSJ1TyK0|{{C?I z9C2bKje@g_|0lVHKjq_Hm>8^6FX8#wqs?wJ8m{|s_YIeeG@f9Zk-1J1+4^XL40vs15%lY&Df zr~)lgeXJwk9tkF4oVgV`z-z+k?-0hfP-As;G!uEbaFNol+S>TNy*=R#`v3Q7zQY1# z{2zUa|2}N{&8$ewz#>`lSa`3kq0-C^+aS6(n2)90pv<`tn&OnFPx1ruBbK zz4bB4`(IXge12|nztAXE0UM}thYoNTBQ48_CjP|^P%@Fh0T_pYm4fYC>JARW8i`nk zO^V74-U(BWWrRViF{ArPCc1FpjWyK_?gM&v710XLO8g>2C>!L1FW#^$pte(>h$k#l zUs=KqF>g!5tNi?oQJ^D*&O77&&lC3;pLt%@WsI>P-uw!#udk2hRgl19rjkZ0r?@)f zLdn5Zr%AkE!q8eO9{3je@<)5M*D4;4{lwKLj+5@O&Q>yA#O$e2Wl7-@w_48hN|FBg zppUO_RduzmqE8?=jN+!6nq;Fa4z?knsc72ROPO1g@G%$6fwf5JJHC1$2Wgs(SOG9 zMlZ+`zB~6O#z7AqQ1j=umR6}z(4KQ~)V>+I^)e~$wEuos@Xk5)jXnDdv zbDlF&F^{A0u$GFYOK7g@i#QyyzP#R+rm}rnCT{C=1$89PKd{7HLnESxY*KQ^Gz%gA|()N;#hCA zC;il!#yUaC4UXG15*uu~Khh7$n4Ap^>H}LR&@**yr~D53+oPQiIb6PC+-Hc4luAV6 z&I|`~6s!xRuMQ}`_&{u-#_>Io!fP+6#c-=`b*-Fjt*N9_IHsuD@pw`(NH2<8JnV`b zUYUG2Ysguip(s{!ynJ87F@yv*WffUsMJ_qe(^BP^3Y}vFdq`u1r2me~8=J|)&BN}! z(d?0m2nlEo)YVun2}inY6roZ#gSd3^W%m&@dEjRT9!}5TStn0B!tQJ@B$$vv6(K3(8+h?Fv833; z#rAew@AGW(33h?)kjN_%oxZ#OaGC#L)L;ZRU=ZJGuEb@Ih(@JkHg*Gr)+WlH3QhC% z-+`VUAAxp!+v#CUt1UQ?@y~J9HYhR^tITY~(G02Ap3gc1 zRT*=a5D;5m?q<=i{A^6v|D7Jvg3}QFTSh(vR0y&R`uS0S`y5r&ku5Jy<5$Ealx#hk zhy}x-qi5Od>rMPbv15IQ*wn_Y?QP4t0?;N;TeCMlox)y8Y4OB3mp1n&w<^XlXqE-xG$XAp>1`q9YI z=n?Xjhj4EmpX$O$7Bhen)BY_{)eu3>xXdg6LHU)%3*ZePB@J{Xp0C zyV1;l{muP6+DlJkPiyek#o;x(b6icY`_KCQ5j27%kexR_qFR4Ut4T1K(!W3DNa%yl zXhyov(W`pMgAj}YRdmZwXdT%gZ)&Mjqjo#Szcb)pgcL(WwZ{t6=sTdNr=j<9{Sp8I zlGy_whKT8;A#1ZIMv|gd5^jW4hzcgDpBjGvFSi!D<>++<r_A!s<$eTy1( z;oECLxD9b;hzaBMI6$%M{7F_7psG6R=*-7jyFI28@jL${96})*>twZ}utHIS36k0L z2p_yZTh7Arwi{puJ&*<>heRySMfdiwW#7|AACESQ_9v9KWar_KIj%Y`gfNI$7^ZJQ zb$Z?K<{)8pyTe=@b9_CMiOpg$gTa9`Uje{{aOi|ZrJ=d^1Oo4uD4e*RHo^wl=HZDd z!=_Zbww#7L7M3zwbH!+G(mEZ2*$ovgv)SYht3E7|>|i})lNYwwm2x>j9arT?aPkHv zdx^pIHe_^sXnB85YMHX*2Z~6)pMcNogjXr}2h2hOeZIc|Za#rD6r@uwBLtUd7HQE6 zOSSUo8wR*fmM3=@5yWr0JV`#8-fL9QrubEoE_Lz<%t*3W%pSE&?Jn}LZ;4C$5d^pSwS5!u9Ar#M@#jsD{ z;A8)rx!zrL#jJm>8wsU8sl?$O-0!@9zBF(YC1lYwdUT5W_0{!tCrvh+WeHH<{vBqi z`D7X%FRb|AanV(%3>q)xJproX8=B$Uqjy|Y*h-d;97s>#+u{*_K@9xirLDkZvZ&wt z1H`JkvO(vC27%r}1Yw8S1A+vV1RsvbFQqlLE&JOhQD04u=N6$f0*Rk z9!5$$ojA9<}=i%4)YTPO+XM7@06mg_ar!E9#Cztmv_ zZ(|YRkA-eN20}s0K*I2e<$^BWMj;VK@yU@e%-LJXiYdas4Z1#-pr$FD^y9q9<4=%T zdUeldJG3%7Wr6f|IApGBPE_Mm_q{3)eStk{;-cu1&WxSckmg4^Ck>2$2o1nXCD zFP{9<(6oxMSe%@`j)qZ*;o;uqpW9!Pj2d{R!jQle5+)jk-!9_~Padn#-P&tRMW-cZ znvC&y9{kGX3M4YMCB0u_xSi|gkw5hXYgzp^YnRN)aaD6it? zey%KhDBP6(JobpGtVwr9cGf`G9AoEPS_*aN=RYtGyzXcb)UUJm4g}UWmR>`9*q>c> zvPiF^d-DUNxiTrP5CIxUF_cl8p?qWNRD6gLD{!Qjo=bXILmjUiYtNaiZFI9a45@b| z5sj~_WM!h|sXFZ_&CVzmA2cq*6iR));o$k(O3%3@Y-K_^V7t}c&ie3_dTz+HoZl%H z^H^Rc=@eCXNfbl#szg&Aq6ns9?s0aCw6q!=$&Vso^tUhw#ico-h`Dm|Gv>QZ?i~d< zUPqKrdXVOtEME(Ns@OL-j#@MGZ4bL`qBE$7c-}Z?_;e^nSx-q^`m5YY$eVlAUqnGl zX<2_vl^G*(O*02WCc=(UdR@`pgkt=ykyz|f449}%!1Ce;IR1< zkmE_C5pY9TM8!&lhaUlzPw*rzynw&9+3h{JQ5ZkQ*gd`L|FuG2fl3qIPM*dD-#iue zCLu#diYM;Oc{=8ct|6FyR~s0X#k_>P+0XP|V4U(`A5WxqT*r|9vD%j7VZvs;qAF+4 zSlsRj8MW29V`4IC#q<9hRGSXapwp3(EiA1whG{LMh|bY*!M8o15y$5B;F`;z$f&g* zO(!nBrvihv(7%JGa16{V{4x@F+rVP@nlYbwR^1V(kBvr^Z1#VkXnXotiO-MQ?T)@Z zJac(EBrfp&3vFPyCgwOefc9h7p9aDjmTrHj*uEnswL9@9NjODW{{izi+t>^*y5IF_Vitl-H)g3+5tn&Yy7h2WHKBM=P&Qj23?JNv1t7tfU+r zoR71(Nu`yO!}wt$0`yzs_;Ydl*2shwJ7#=LOQ-icX7(t(ojKvlCZ;3IN^bzJ7jFfN2gapz-LqGx&uHc zm;bas1JgUTk70lITHPmgKBk$~zIT|_Y^lm{eUMy6P0}W6_9t4d`uaez-z(3dBt&U^ zdgp6_?fWIIN~=CQ&-=QQuHy!it(_;&@7*qf@r@t$3aQ8{&57!9K6FptlM)Ka0<7N> zy#Hg;llYay5}!F=G2m`Gits0lJGQ&Q4uHQq4+~s?sU+~EHkbG3exUvOIQLFqr^Ju#QW020mLjf4*u zf&$Wp9cc}RL?)!%-^asM4Ip9Y6nJeh*_jX@duOeXA9+_@B_vOh6&klMvEwW)Qxpy8 z1?UHtOGBoTW7_F6cgN)cb|qPuj?%$|1Dt1`O*NF_Pf1K?a#WVKYh|og*PnKGm_kKN zjcO?gb!4WAok5C$?=0n+<{K;d5;$WA36p9@S_|vD;>**B4=XWfcN7z|0D zT)Zx>JDx@gc7+=XbmTZ`Ly5i~>kqcDGFrGilRJDKx%)i@3XFRa?u|g76^fEOBXA#L zeU5tG1sZloJAU=w+&=K0Eu3s;`Tp6YOCaAoxd3wa&mp5?gk5enK@fc1dwCn)Cp=h` zBsjXWG%()H!F+vd@0(4Xevcl#`jYEL!U8k%ixsu_A#DG;Az#T5o3z&%z;FM_^S$|@ z6?}U#)Y?rRs>2?p?7Q>r^Vsv!t6-`W$Nry$8W{v+e~=+}j9fEc(Ntt{yDN(~(3i0m zJT|KL#gs3c&B+CvqQHe2a{6S4(-Yoe6$yd=h^Vc{iL?K8l=1bU2R<&_2P}0=(L_(1 z@#XtdBOqX#-;QP6Wup}0vw$Om&M$`drZ00;e*2HL?m&L~{-i7}ESxn~u(DGO{%l$L zgc&2Xpv(pkEe>UWFQVcYH9Jwd0^rStXp~D)U~_m(9(SYR)0`lmH8Y_mzEu;rArYH? z)YfodH?S+Synrv+IMVgpycrAoU$$`5Q<+)sXEPRGVYV%i(b6Vdnlt7Mu&#^|Js=Tb z?nwxzJ7=@5A(W9O8+dqG;R+Wqf=OLjOW*N)A=EaPJ6+OVhBkq8Jwc~jx|`Yey6sdy z{#TG{gKx3BHUkV_Tz=u#DY5PT+`cE+_e5a{ntP^Fz{UvU6N%n<-(FkUu)%?D}-@EHjl3YGhz? zx3Jp@{Z)!Kruq9{9{vu~_qh!D=T08(1eGZHk?E$6&}lfc4zQ)@7gGOyOTZ%ropBG@ ziu(6oEUkjEFOZZlw+QbX;Jrr`*YL}T$`AGk9Rpq2^=vd30R*OrOJfz0NC}2hDJ$X^ zFQBo5H@1=~n}scgI0g{usrM3L3^Yo#(L}Ff_zuhYz2Dw(Ww{TC`0<3aeMiec3*E-0 z7NQO&*N<9PrPa~i$TPOWTdT9C&{K|lgu!YPAOAZ_HXV6r=aoMz0q)V1uTJkTX6hV9 zc&T#z7am5O3#H8SJangj0Wc&xSCqq7hXTcSPcm2;ZF+8a&o<4Y`Y*(4(y#WuKVgti zBoNh}DK`F8;^_Hgnca;^5zD%=H8gRS?cx#`RA4ezAswnY&U&aj?yu_$JBLqCTT3)` z(a(8fWGuS;L;Lorj>%Lf(d6#RIY$853rn|5fd+H+*(A^XdNLMHWca6;GuL?6w!X27 zRI#`xZz`uKt)q75yJUOAZcRnq=E(OW@Vq*>qG{?uDT13Rg6NmhZ88-eV>yF=JV`yodGY2t>`Vyui-hA!97BY%&XgvYz*DY`vF2qOuAoWLQTK)`4j9w&vwbOb_=QXUhx3pngD zlGoI5I)??>r^I-IagvSx>~wwKz!12s_Et+ly!^D*{X2+(mof2nt<7Ik=bA8~N+3z;-;!~FU!fH$ zsGQ0&K~qi6CnOkkJ5p;;%x2DhPe%K({UrzMOmL-Xyn)V)T$Le9Ma1_iMRrOUNXdA6 zShNDtD7V|;yKq%4T;6un-$H}rq9HJ75ols0D0EUI@P6XXF+`0Y&oQeB#Cj?+`!VCM z=Z@Ew&6wm4v4d5g-LS+1v)pQmdF=Bd;RklXxf{_)N%&hR) zhjFk;c)TCKx!oWjF3!Hf4R_zbilhmTWDUEs=MCK8l-rehuwo^Fs6p=QW`!&CCx83_ z7Jc_+^-d47KhvY*oJZYzzcQ>cBZzi)2R|{-Qd>+mL@<2wkdqWm*yPSg&*sS(@cdf) zx7ER9keK;2_&jJ0yWSjfK9yD=jd*hHi_tTxTSs8V& z|JkqF+E$e@$_1`2{Knt^n%uXZ59dA}&U`qrWweOGKlshRhuD?h) zaPWj6?yZgIV!hRNn(cKvI3gA%4LLtB)6!t^MWHq;Rvs6Fip>&1{bangy`|g;MhvPq zKi)tZMjw6lCN(|A5euGPYMr|Mi8Jkk%k^`FsGJb0?2DCIz@2|89(%rBFdUzztcf$o zP;Gg%v6z-c@a7jRC60cWG`sf*2}7)`Kv3dL5?C+`2+TkpKP*gyT&+)5G#F8PMCfOFn#X-rjH8(>?Rqrima z#m>qYnAFANzQC#D`Z6LxNmGKdMwOul%x(Y3dCnU9J==1+K4GUOfew?+I;DBh&CFdB z19xy6be?HA^n6!;z80+JIh~)#(MYE}iiXi4l7mH{NF5@-Dm3a^{F+F=@bV%UF>xL$ z7UBUIZ#kPBaz=PzR1|}hhWQo5(Lza@iqS{kpx?NAj@U$4lcXhLy%w75N?D^Mj75zO zjgVkWeg}kDk>B;1f8HS?Hq^C7e*}C`c2y@H#d7Y>oUl?7=kdv08u}AagFCF5&rA{m zF805OIrrUAIRnmZ$~AO5WQ?VvHE*$pAsd?a?3l~ zcW)mL9IEOL&gX~eSjec1fJ53s#eB8t{MfceXWigcU01yqE82bT_JU4ZRJ5L;am}X( z9od{ia|~9AaVWe@)Cw{#3%9&X4m4jT|FWo2ldMx*c{Q4W22(N-q#p0Tg@eUtTE?H( z78&h?V~y^{Ik}w5{3c|(wwfl11rW3KN&?OhYvm_ylD3snFALgHO1TqFeVY}=GfXw3 zxRbeu`~vzL%ESs!bcfd;m`+!nnWi=FQobn5vnki|$b>qcOKpMe#?ue- z_3xP_D`3X<)h6jQFz}V>P3g5TAsrI@d3o)WAVk? z?2zLq`c^>QS0l(e`?1zHhcZKdL|oHwbZU+8_Vf2_=`N6UPTC6o4ExWQvdz23-SnI6 zL_{$eL_#l&-@m%d*O&C=K*fCFz~Xqk@fBnLKy9MKQ5qxuMfx+Qw#@844;)oUK^u`6*S#a&u`l-Z7r^xP7UB&hqG$=32Wa zgES#qB)mP{UiU#u1Mc&PfEpd`Vwp<)@&J~q+>Exhx%jb8RS#S^NNKbNG#uv-liO`A zQnJ!O8+|S{t|uC4HNDO!P#N5%Qj}j?Vu6~Ve>PlI-RkGlT7N2SRq^lQxRewsn`x4O zyFf*zzmN0I59DVJZYQ2v*)($6aTv`|VIbO17yfP4+i#CK)vt*-QgDoiMM)IYcGRzLWj7?+dtgntPqZ(d*~J z)F49{Y_kv12SF?Y`2^jBrX(Ah{zdaQn}<)qR~B|PzGT7^f!TRI)L@aSOj&otXM-0XSFPyKqoD!J5B2mfO4GoPknv{l*H`uX==F~pcS z9ES?T-uU3*_9k$mL{o}A$Y(u@bpa`##k$9cc!=h6S}0|nV-}D zCz9{^k7~_dDU)=Bla6Ie=92PE-acPkSRFi*MUyWq;h?BBr_0X5F|79ddzQf%%hSfk zbd9;5&uPDliCQ>p_1Jrz+hO>;z}Yu7%&y6B+>ItX-c@r9?qy)~Cp4d&Mme)pfYo!)kot$>K;+u;2LgD8`62$iXD z?HUGtCcPDy@u*FXN+5E&BXVasBB;?QNqV#tD?Wt@x@SC6`{Fy4;vza(1KqhvOT8&e z5y(UMd?|Ge zGK%eBP3FA1*4T{<^o_B;{PNv-Y^)4;hZuT&?y-CicuqA%i?=cR4Runk%Sux#IQzqb z!mm8ab&DtokD+ADdR^d^`feQsk&eF4It5kT8$c2}Nni>CQI_Zt{JLdY=TSlljT z^2wBn@`{v|)^eUW;Yi8BHZK*gev3HqE@OJt$*RUqJX+#JvQTwTxUn5suE zVL+p$=ane1pgAcI*+&-Il^AS_F=@_6teBZj^vZp)dTmX^#@6;{HCf_7`PN=SE^8{Q z$UJIb-#ecw<{Y_-UY<^WHG6q?OtG&G$KWBedbuvukgmt1j04&);z$= z5p`IC{Ey8x%;AIsYO(EqK(;IV;1SZB?UfY@DTnhRcDuOVnIkljYk}1xA?0%Z*^#?b zVr!YIVDr&mkA?pIwz~U#zwOuAwmEwnwmDw@3g|=K!gxu9Y`LV?>icIE`OCTouu}gz z@Jd@%1k`M1ve=#urb)_$5KWJ1p53?mUIX6BYNMR2EEiC~Vq(Uk;|L8x=?5FY_-CiJ z+7`{14p}SV%e0#PNJrqiwwG5F3q(xyu>*d`qRVDB(i0M0!cY*pF|7tr)-&qL@-QJy z{c191GnIrRwt=v_*%-KI&vk;ss;#eF`rlTk4KcwjowHr6*Y9V)!x>5cBk>LIQWN73 z9?lR+Q{AG&S(P!e*3f3%``-T0!fa;pK$u@fvf$S?^?G)jEf-97ew_&>Z`hzd`n(x~0^2HoWtW=;r(E$a}g zu>p$;!5k?%f*5EzgKLw{dtDTZV^Isu*2t#Rjv}7c?r4!p1?}R-E1a)0MD!O}IX(pr zJSOjO*uZSXi7HhGRuNQ&nwJ8wB6gSv2FdLNrrwMb)**&dn)FBp z+C6v}{tyQljngEi-=!0l>-bmeg{h7)ohb1PQU)>94&J#OFm{3}O5<%lUbY>9q4cFx zS`<0eG0U+&q4CCuiLxBGS9FpLHOTxZELsFQ1xlek$w63Z4}Qfk+gSOq)O$xah+9 zgJ=_G=5V@lU(BV)V-+Z3=BuOQB^ls|;4-xE6%QAfQe%uWNCYgc`e1w69lMylI1pl{ zQsp>$&_p>GUTbiuXzNW!)=i&*Z8EUSNE00;Dh&~?=lT5-2=4X3vmDTH>usQpaJKL~ z94bE}6Ir)|>i=L$0W=syO?I}HiRXcIWlC6D7@;_?BF-GxtauRvawLl;j{2}7bm-SmG?3YlB!6!vlEggZGRLj`L2gB*xR-yEksOTX zmb&y~BTt2QUXu!M3-E?RcVjifh=!~8&%6|=Mg9GOXD)$6OtyG9B4S{TFraY4!~j!l zqS@fRLy;f=q}Vh)LEiT8%k|9&3wWPlk5gIN%Gw%Av={_A`cb|LjnEcHRYE$Si(v%& zyzb+p_dm$=e0xXHPl~+vV!eSM>aO{wuJm^u48+tlCBf9fYU-HlGAR1u-c~?kg>|@^ zvjAt~Gzfdc%#Jd%Y%lkfInf_~)7fr6K_-q7rac`eStvNu%;oQbA7TNg!7BrMe@mIf z5hf3Y&-w3h>Hp-6!jBvPw zdmoG;qk(O+!J3}X#gJ>w73&__eZG>jb%p!>`rW6VeVzWkHaGnaXWyze--*yLxLaWbpz^Ue=cpqsPvw%Hl`(RujWM@72u!{84%Siz4( zqNBV#e1vh&8D^~t@DL1gGi1o^!T`}5zWVw$quY=Egy*-psh>7!Q%VDz0Cl`uD zt7>XS*k)h&Zg;K6@pM7=Z=QcHy<{3M=5I{4|Fbdm9~<_z&HsCUXulQPR1`AlKUTh| zyBl|%8V%Bd_V%+KB2ES4tQUkyK2)qY?e}jQl<1K;cboqQ-y3b!A7JiL5$fye%351f z_xASwL!a@+iLFv=|&o)yL&{sLAp!21*N3ByCjE@?i{*1 zzZcIr_ufC?;|C>X-r0NYz1LdLvtrYr9TW<~No&brMT?VbCP`RX;f0B>KTa+DJ=}jE z;Jtd_dV*wkF)c}s9*ja1etf@q?hd5xw;dldY=Lm$>OdG#l9dcdv|FFE|0cftML9Jv zMoRuGL@uzE!A58?{bBtZ2xQA=S$dF>iT|Ax3`i+pMQ`_&#@~4R%lkarV~?F-P&kOZ z{_(h8xYDysDc{|^%Zwk@o?#UJHhqu2LVs3ag9%Ku+TO3i0+;~`b;>} zd;NIbU=<)sTg$#GpQmD4J$Kz8@yshXNtr6wHZwBz?*I;k5>yZ;IR+r-2xKe)q@>A1 zfbR^Pp>i~1i&qCtOwk)HGxB#aWO;x3b^aY-&2^+oI4QyjKdK)McFW!Oe1UF)qIyw; z%;0;b$^Y}Ji8A}|#IXNYhP$i(JIVWw$?o z`JeRn8dU!C!)4MyQsNkswC#HzI!ZFwB2~nGd!X_`{D-o@(kQ76Rs2m`c}&rNPk(lK zGW*9JvR&}0pRYD>8D9ID&{f62FyyfUfAGH_s(9ZYJ3>WuYcKE-w7jQAe=l?&Z zv8=D*`9vBY?570f!=i+zEsaczPUGJ}?e?kDlJ@OS%tF!*RDXf8Ptq`99+UgfR?R0pKM3HQC`eag6(7~NAe?kZ=HPg#>(*b(hTEIaTzV$Yo>+&*+KV; zY@wayuUJn?Jt%}14k4L#OWh7+a6ZeG#J!*6=0|7XlhML;9r()7-xF7cFI=2mYE%(VSZ1p617@Ao!*4QWr4D04(i%DL>mI-dS{vkyL0ug*2up z4IAKhswY^^(zO{V@ng5i{_^FEM$NJGT6OMV9$U%o4}Xt9&|BX$N7>3-&~T&)D@OC+ z72GCAY&PGsVb74`^yK&@74Je}c zG~MDKp=bgjImk%RIJ<5%^nq;eVPUfE5fO~uX;L9e-cZ4cx^FJAgk;@+48G(Cn;>Z` zi5gBRtc4dV@^%Yn)XU4woG22)$g?>-zimvJi7 zo-S7ySmIn;4hsV(#EoKT^_Cl0Ff#}qos^D>&GX8MOW@u$$puDc% zo;1kmAGma_?j(#;IvPdu9kVgroa97W7j}Ljf@YU7Q=|1O zG~Z5zjPI0h#T6t?x-DuA<6^&;j)eHjj&3wipcVj$9wQ%AqXm-Y3L;HTH)1BZ58TX0 zGQOI{Y_HEzS8N4^X!VMl8@Tc3*hULW1Jy@C$Ut=9+rSGE>QIIs2(VQDp75)uYF~kN z(cBioTw4YKNQOKa;^1M<*;+fsFM+VpIcj09nBUyWp8a|^Qr^)cpU5&Cm8}TCx_u|v zE2tO>`TO5~&_3P=Y$`nC^8R?WhKc3+kfOQ7Ft3;yNJdn$rKs{~%dMh$V0TugVyG1! z1%&hkvby8!33konexR7w%=Um3M+)X_%8Ce0RR9;AuoB1r z%=VMzOM=AX4ee(7Hmk*vEQj`w+u89?sUER0wYQwaY4>#M7Lj-wqo$(08xOu$M|=up zdxAI5kPMxCkR4qy6J=>2K9P8RC2iL%oNXFw-}`8c{#yZ;jP-_I#w&~k`>a`?vz{;v z92PDpxmv5PuM$$yUxnW~gT}8?Gd+fGA6fZ|m!=S5waKg?_dcz#&8`+l%e;J!;K${2 zft7Qw9DmK&ALd66*m7lk%$OB=>B}JIDl0o;ofw5pY*7TVKs3=%lZBk92f83c#FRbr zs(2ICl_ziW=OJH}ajVJetcWkf_c+n_+t*>L84B05@fBX_!nLfs*=@gTJde7|`21{t zNEl7OsW_*-XaN%T`MVCveWHUwQBHad$cg8t1mBNV+C|OSPL=-O-3+sP-UUAY0PeP^7Q~O8Ynb7%=5u=bYO7Mq3ijQ)Xha8|$QA zxtZa*X6ycyFKampB*q2#QKOczT>`SjLIVjVCQ_@N5C0&jActzMuU%WPXCVE9QaOzM z`FX3<7PK^o0X|y&S^S)J{M=o@LSuPN%$}!G4=XlgaH9vphN_*LL~wVGnV_!Z&bcwD6}AtFXMFB-+rsrL%V~E$H#q z)br@ET5ZS{>H={E22?zU#w{D8CjB$&s~Wvurg264`{wGREthkYh@^r#d0H>X7hJ}a z2tTTalad%G1hFAv(b*EcyMyu(R7AsOp%SJIVk1j3Lr<&FOrT3Fc)Gc^KWv7a{00-7 zbWULB%hjdu{Rq~Gr}e_>RK}}{)6(_bfIrD@&g`#4L5pba`CrVZ{sE&pAJrjFsC>AC+IySJ^{=3%@$fE5-^iSwG0$Fh&TYRaHJC9CY zYsF6GmK1mMOmr>}{#MtY$qI{lW2wC>kp}5Tqeh(LW~=;;;>)Q)t%HBwX5K`=g_ih6 z;yv3l(=AJX{{%afY{Y^K8WZ6mp7fSt5laL*J=oLMv`N+-iAG15x`~s3{Xf1|KRipR zv!u1Ym#jM;v0r;A+WZWjs50bgyqYpy^1!Z9jjQ}*yRtmeQmvt)CKRxZhmK9m9d?8g zk+L#iqj%W?381Ite{}RYYj)n*`|xX|!bGVTea>HPBzLQNXQehI%a5VF)cYO#;X6V8 znXS2gR~4nt_B}X3%7(9BH}QQaGFTSnxW`TW1c^s*soa{vw8(Bskvw)`b4E&icev%M zxB9_QRE!cm?|!TK_H1Q2sLuACJradq`9HhM82ZB|Z#%4(sxgY*j8~FvVhr6u{?%4; z=xc1EnTOZqr8L6Us8RI~u!jVpxCmr&G{(#zaXk;LwW0deVC*Z3_V;&w#j{tnuDPNy zgDX$$ocbL;L&Q%_LV)TP< zP*(%IcD`-3OyF*iFh5cbyp3@fvhz_?kqGZM6b>Qgb^SnJ-V_M&y=iwZa(_*tcZ)*R ze{QSqOi)~(2UP50dIhkO_0i;(8AImPN-|v;4Ib9ojURl7A4f0_7OieJk}z)nxIP8N ze&xJ=%2*P~Na1VXs(@Osugp|omtLg@^ct|?Qd0h8x-MIlwLf`STU#Ue$IM&*J#6#t?synGII z0#j&hnJc=rbFfFI#58V+iM2goj#Oxg6?KWVdEc}n>p2^TtE>?R_M|CQhD)fcU^qE@ z?L--Vbsp%FP)Iepw|H--?ul9sVJYoqOwMTbrQ& z_mnHfe`JIvRjtCmFg4YCFz483l4DUd#<$ z^H()}4up}(xgZUaL606vm8-K-;^NlGgx?0H_A}ugs zcb_5gwse{(^-;4c3F}~+eyTye@T;=d#WA~@YGg8H|IQ)fS7oajn`h#RXvRD1Vvk3w zbL&)p*hNDCL4!7UF>Ao`^CZpDs4=W^+>~#QKf3$)6m^MqdD!vmxo!NoE1@3Q9Gi^ zyGRiuVw9NmgE2#eATyzdv*UY3X57NRqRm|-oL7-rXN-NN`mg$1 z{tPeY$rJ@W^q}zXAkq4p0`PAWUupjPik`}Y?~_?lLj_#-fv$eNh(Nz@4W@ynIitTTxnRvB1Ap-ZBqL?W&eI6!l|TW$+?b<5%E*&f4{OX znM9^X8yt0%1iiTCSHT)gilFbx^d&436A2g~MU=ka7lv|; zPhv0gC_gs5(|(l>Qv%UJ^s1c(s@?+-efr5W8v%F9Auplp+PiA*0P0@e1i5VgU6kaT z?h}xpDDb}Z+5B3n+zG$7th194dY)ny4!A7CS_cWy; z;UfYgtNlsU5fN)T#bEb40Ac{nY9cNZv4@#!J;|BXs{+nx{{9n zIzOU>vK)E8MvbHan(G__wm4dBR8>`tEIH%vi+_)RZJa?HG#%htp0Ipf`!fE)p$a)U zvYwj?Jg#pxTGL~wG6n=D9Vs~p7H`~wcvOGg+_FK5SW4P*>qpn1##v&j{fYJ_#H^RE0H!xje$O0=>n0_RLn|7GMM=cA^ zSB>TQrRF>xtkUNB#h$ihOeF<%p}}sbR_f7;GAm9ah4ti0BQU&-MtzemRnAxoTLdju zCU5g|JlcjrPNbBt9W$NEM~Of93>jqz-LLA6m>4fBHflkEpR_HDepiC-iO*&&hxg%r z%FuU-CYjpdGsA*-d+6xCbRjRnSUo7^Bv8GRjSZDj$`|%^CHbH<+uObaTt$jThr4 zkHutS;q|hgr2JK4WiR7K-M@1va43*tyLUg?H*^f4TjTT_FG3sUVK$mF^KK|<;Yks# zM<=QT`Usc!z#*mO9xPWy$rZ*ryb>jB%5nAy(Qqbi8~u*CjAooc6NDmAHHxXO?hvo%Mq{P<4>ZScY6_)P3k-Typau_?V%QY zcZq8JcPh+GudQ)11`??i*9R7k&FTxjTy#eBzMP_v_|9_H&`vbKp;1SxW1CL1dECBEmkaLpb#WIqT7ZUqs1DUG=} zUJ!W?s%XuYq0XwOK5P2mva6UjZBR0E#YvZq*3QAY--?i|Aht*loDz-Mi9j}5IGEl{ z7hjAe#XXOaU}&OOPWE*ltmz$miv60J*FJdrG_hEycfN$m_+vIhPtF7`d%~cup5DmC z?qn(#2tP3~G0;f!VXp9%CeVUr88N7f!a{Lvy;x|l8D}Q$n%|3r9E-w+pJt zK+CR&SC24Sq5!~wGLR^=$q?5I(mD7lUkTkjfykU*V03Ht(0%)Hf z;K5sWRDg7ow5KKg;)%!6?=bM*-Pd&;rTmsE$<%xbH>#tpxXhj+nl^4sN3RU9fXVVEBAiEgD)s80 zz#x&KNYWlQMvxw3Y`%?HzAx~)>T|_#u`{)BcsJNGXs8-d8sSX%da9uJu8BdfQ6Mha} zIS+U~mz{PE9u_)xcsh&6`4mKsU2|6L`^0oza%g;-m%Vu`=#2O4(9FfOIL!SKH#L~1 zr(wKX%K858Znn*vkI&_RG*RY|{_A^*vOruZ#3B1ie}EYtAI(VWqqY>e~MTKQ1({Bn`d>ta6%DAfL`|3*29q-j>^X{ZMqX5gm3K*vTTDP_3 zd+E`JujM}1cJ7dt4{|B%yYL$Er5XL<@dPxpjJjS93DhLMX>-ZT9S#;S2G7IPx9g}r zy-8On`fk>5`R1G5Zm4}nAZ?zgt9^*YUO&sluhbvHZY|kVFES?I?55E z@}sHYGAm@R<$bqzyJy<9F2PcLk0!3M#GWRL-MdQsn`+-1E|**=ldZyV4db9k+_y>5 zXlhmyuSr0>mZi#rT@z_6G~em5&mb#L7FDN`vsbU4@6x9N6RC3^zoG?HF_b6;ya|~o z0*rX*4}Vqwi3os~29w)b(uc}h!@=GV}%2kse%^!gg+xDVx@ie{9}*8B>$69qSMh*i!J*h3(^Sb z#F_i0>+{&WufQGaJWt<)tEPmWmGK@5XrQ4 zZeSJ~j;V##i>4K{7n%?fs6>-qCF3&Pwh?>1o(#iqJu{8`DF9eKp0q&BGg>YoZ`2jH z1C;IdXPt+K>Y1v&IH{YZ=ZCmNZrvVg^WM4EBN6?*usH2Yu_vcT3f;5L{l>N9_U=Uz zuki9xPM-PIEl0p7hc?N9t8-Xr_sy0sqESf6Z>|m4!$>wddu`j_7saEg{X;a}vgiFe zW-`e9A{PG+N zPA=HN3`4TNubgh$$0l}Hh{3(lbx&6d1{@cTT0Z9&+lIL*Znw@iY6bZF_{o!2E{A^gV zOUh%QIgvr|-di`t!=I>JaJcvi;&$CgJ^plplFzIq+D}59Pyf}DKfDhu(2y5hR|*3T zDyJ$;T$e#$9Sth^4zJ&FR01cqsN_cl#EeVmmM2wKXf%q%lf?D75b`5g&iIfKJgHn* zP}F6i$Mn;E4RKU=I?`j1YUZAQ|3kr9XR)BP&L-BVmb>6cB@wJ;BL!m*Pi0x-c3Sze zeLoQp=mA2dK=UytXXiLD#BZI_2ME^%tbPqD{kfWm1}{Qeb4sh}%wQmx?Kh%$HXKo2 zQ95KXaGUI4F8fv|+dBv<)A(?YdO(pc)9e(5*w^}Py8e=|uQAlIzB{W$Mpa<=uuGO+ zeVFbJrqGWMrVv?hmB32R93&fD^`3FK&~f(Ug5n?wY_U6x%gkSu_SwXQHh>jzfv&=N zDZ&&V+o6*WY<_#nzN_QP_Y7G%GB39?kqu5n3y}l5x1|AirK4>EH(2(w!=q zq_Q@;8_Q+SdEYu4Wx)fEuwB8jdP*gp+0WO66pM#MmR zJ}H>F)b>xhF86#QVpZ35qJ?CzqzlBX^eBFI&uFvd*6Vayb{>$ZQN_`0H>a4Zh=&jx zjNIwHPSUOT=E@LSX@5=yH^Hm#b7$4WU$MDJ+LOXCM2_5z9#G$HRSIZ(Ag3eusO4P2 z@$quz6A(v7gB7!Rur6ioGt|O9bt5cSa+B>Wk9>#3`i=aj(n*JdJt)Y4X7x23h*l2Q zzv_z~jz52p@J18L;0@*(v7}9n{}_#KKc3nvil4Dy4BVhRXZyAag5IwFbJ2s)2x4bS z+-$F90^KAd7Mf`BiacK|V_i(hvleCi0L)dQ5$-o>$_mE)uSE4|W{P+ZUc^GGmU;)w ziCeLSg_NxByw58df)T#g8aIO@;5Ev1*nmf)N-&4zLT@Fq{xStiua#sp2;I%S=Q`P?6A zh#uA_b&L=oM)_fSBjP?Pdfx7o7f>5L$~`n0(#RGkI%Zm!Dzy0-?wr2&mDeh6yXI}~ z;13A|Qg{KJO?I5U1T6}bYz;9gB!on#4aNpm7*X@RO$3r_4RM|Q1Y;=U2=|wX^I>St zxNX#tP}J|tHq8S>-s;0gdW7`>y25z$X(0%0!{maq9&H`O(n7&!SiqHQB9P*b1Me}s zFQ9>F^`gSKB6s+_N$BzOE(dgO*XI{@j*s#TIimIUX8PLV_?o~(&8Wn4+|gdcPuM?#nE^+X(pW^_PlB5aIk=*)W{3*oPRq@o3FXj`*L zXaOWy&wji|*}rdjFH(-SZb*dHS*{Y!f*e98pX`<;1#HO}+ENKZWE~e;1pl1=SNDf6 z>_-Ysf(pf5i^{VW4JJ#zrj<(xsGG!Y*f4Ld(=73YY7n~6G>N>(10GHFlY;nxF7Nl2 z1JE{+JPubzhqaHzY!LHTaq*k(SW<*BZT6R+e*HT2Ciyxz2y|Vf3(3NW==~`QPz9lH zs1cygZu!-wro=?bAnrZutZ4=E-S&k3q#f7Z#+BX+UMnQ_K#3qIUp^h^UDBLYosL^-fG z>f2ibLfkrH4?-5|0fNB#n5m}`3?U9Yql_zy*LqO0N z;rmUxZsQUeF7vw&{fbYSQ3Tv}*Ai^t7uAIcEBPp0|vG-@-V7WEIjab7uVM&TGMfYKPkF!1I)Bk)zro(CqLC4!_(ABfIy6o zkB=t$vB?pn2!%yOWZB^Op{>Cy0f34P6NfJ4958hd&8CQCb#iu%|5z}tO>!E6ZZfRY z{CwovzCppg9QsBqcO2oR?dH@+3A6X5o-XXpdzkBCJUVr@WcX2Wa#TS7R-%l4n-mT! zw({1xpr$(y7u%Il+JcP1tZ8*~aF#qjvBki*yc>~@kh1oyeq!$g0naPs*PU(0pENY2 zK%m3Wd4l&MXi8dI8Nms}w#^P$uuy7hY5);(1%m25m(tTC0tPvJt-woLE75peN;;$4 zooJjJ9*zh!!5qqE=@*F3=2BlsxNuU4F0|zX)6E&)V9=U3Lp)AY+m%ohS`wYI(VB<(O$C+4}Qms`!Di znHlaJ<{f>-U1%Zfi1GRP@C_cR2mfPfmOe~&qORCXK3C%f7p*!N7LIGX>!T$z&q|dK zQh`R&bctyvA}f9g^Eqv8#52xTHa6)ekkGZ>>}YIloAuM1w%g_$p+9yWSs`@?IlS%K zHvD#9%F3Zvt}7UjR<6giQAc3vh|(>x-P72?{74H`9}6Zm>*Q_riIbRg`pWd0zk1^o z0qY7_)yL@43*uA`t9M4YKxb6mwG|3@73+wBcxs>@Az-@97O<6UVCqovqT_8p_6>Iv?Z29?f zepS`CxoV5kF)`H1Kbi;BHa0d^wzj5AO|^hd#rUdWu+))ikv$@YD$sKpfm7{y{aO3^ zz-a1M$ZZF_PzNBYZY@ZLfK#_k$#@9&WOmZNr}(@v<*wpYT-~fxStIbD+dUzc$)2`N zRVs38s{l`#tc?!6O_p!8qVG04gMLfk%l4E{=Q`UfE%}ajKokL04!gPs03z`dTkJ12JMT6Rh>*3)6bmdjm&^R9pKt+q{zYQt(&LrY< zsj%-Z=709GryEahKT`u_?|wEw%?rtJ)T%IqvYF;rbaln?ea>TE<3WiLxTC3b%h;mM z=%@oN%vHH#z!YIpXSlsf=LXT;*T}Kj)0A$uG6V`QxzFfrch49o0$jKV+JVM*puo+z z5?4RxBneTUNcW@&q1$jLx=tlrG>1j?p&Lk?voS!NX#@3+>w(X|=($>EioRO#*$@GJ z^jwyj$cm7$0>@J;n)gQ&nR&y!M;0?5%^1iM@EjC8bf2%W`gWyjZUx?uu!*W;_%^T! zB_x)2`=`!~NEP3dHmF8*mEy$vkD;&DU2_8Qr+F?f%9(RS%>deoJV9>c?wo#)toq|w zREOo%p904eR$z)m6>p2tjE|n4_vo^hjHZa1Wl@KkQU`X*))89O=84{jJO{F#SsYd> za|Vuy6ItnLX@hNy%M2#S(fjMSPfDfH<3jbI^s+K0K)EfLnk3u zmdCA5AbfE+5flDK%)|`JHr)RyBQS5aC+SXoS%K1M5;vP2@%z_in(bE1Q z%))f7fb=g2Fj~!ep$`A~ODS%cfjK!=J_2xsR%N+OmG~IAgKob{(ahxi{|Pd(R?;8@ z#QtiM0ZO4K>x;8(eWUZ;FQx8&1Dt;BzYdwPsIm+?CbYGM>P8P?oltFroMJF-1T}gv z+rGyC`6#6X6<8nX#{>Mu6tM;R%%9C{s*0t#zsmeskVRCogJ}5P0iAqbrupXmg_tpx z@?ODA{374i?DO5_adpO_V1NHEM>CWDZxT5vV@iB^;lJ_`o9eIpRW{j)y$@;k^3^bk!_nb#RK2lt zhow*c5g8S70I(u`yq7;c*i1doElbB_#!s!O)#B+2Eh919*dFM`8{>f9-Y+AS!=>V| z)S3J6}Jlo~hwow{uElt*xPq^E4MkC&(z!Gg)qWl|_0=%gJI=J>kXRJRXg*m>8 z@0cM!O4fB(y?zO=Wcj2;+jKa9BI6`t0%@rKM+_o42ry#Z86JpQ*`}U|Lld|#gDP%X z*{$iGDJ>1?+((AcfFS;ReLo8j3o|p)n*I%{)04X-l%TC$*FQxd2x|h-88+Ln|uu)7Mp~@)xf2Cu;;TT95bQym8AfflMAr+C1rkNhep= z$(0p@mX4V;IVv#?3u+h`m_j3sD;ZkEAm}AMJs@fOE$I08*n%{XjtLzD_Fh0S)FkN5 z+e-1srd>``k1Er9M;ci_C}hoOs#mMsM>N~sTA(}t0lJH&=bS%5#Y{`i(oF|eLEy+51s}H$$^QZ>`$$?nj{He0AVsncG`=(?Zy^QG$cSVG0&N^M zALo~qCAs&HY!p#v_nadRr`Kncl_?Ort@>y$XgBT!XjC5cI=7q+daPISP-i-^_2EvJ z`=<8g=XZMJNcl#+ki*86vnfy&#}5t25&z`^+yizDBbZDC&2V80XoZLP<;D5*&SpU# zs$-P#C;k5GS2)14%)+!)kei;_%e&!`@O>%#*z$h5d+Mj%}m)=IDPrg_VSpfCT$N-56wDT-fD}H?X?gR@@(d^IO|BjZUU~( zT46-u1d$iOadpQ^Q?i*4I`PtAAv+SnmpW_Jzh>cxS+pkvUZ%a@{qBS|58)x!GD`Mw z_`?To7ZWQa0fxiIo;tFnI0>T^Hs5!8%-2SZcNOsMd38YP@{BVouxsgMI_-8_jkMI~ zd=y^3CFC1l5Ty+|zBf%ECnS9B(@a>~d3?f0oEI-qB*xl@h9B^zI|oCGmQmKG)u4dk zMaNMBe%r!i_tcEOEZ_&WUG%m?n)U^IByjZPn3D5=az9fx2?k>Lw{x~|rJO*e1b}|V zmX_qtYs?w?bDY7%pW6?)k{I=Zk8dgI=x?S*xa|*}Npj2Dvh?^|QC>3^sfM?;wQj-H|QNYd{*+iH~(5}K*0&rwMT|AELsi7u8W<`Ol~+cx{hzF%}2~HNl5zb39IH9fFIVM?tQ^6uaO$g22s~NiHnS^z!i2{ zdE+wPpRG1$ob?F%X(J-)$AjyGRDP8#Kwn`USAskXPv(u%&uuV2ZOiAzp8mertRQOU zNp^Lbq*tQ){y3$e(6H`iT`*ncDoY*h>GrH(XQ4xgBty}jcSV!@)sA6n9$RXilUVH^ zD#V^J#HPG8sjrU;Q0b@Q87mKWGk}^($dgS5xUDQVV@~E0y`kGt{DKv9-hQ2R-&H-{KNMKhPli$4v*b@tjPkk?bvaMk=yZ%w&3jSF4_Gm z)gn}!T$};l z`1S%|fM>GBbjp(lZKaQBnU1Ges-s(VKD?1{@cqs|x+zc@&N@ zZii%N_waq3G>b_evOmjeqeb=)cWw4L-6V~f_O3Zd1`Hy=FUiZwA}!Q`2=;2CSUEU2 z4w8SVHGKyP@hOO7P~};erSfDVhraz>v zv5Zf(aL}G`Z+}0;2d>v?8z>;d4F)6!oNyytg`M`w6rONM;nzE@TEMZjQ#^YMIkwhA z_VIMNV7&fAz3s_MxDlQrsVKAu6Xg+R>U_sWpJn8=PN445= z?X+gt4v5trnT>Zz=(-KO2j~t~s|iY#0m_ldQ6QcSkR9xpErgbOT@yiFe$z}0I6oG0 zbeTOOSz$$3F0%02+RPf`z`+?5!Sn@=J(WqPtm<4{JD_fZPrLlC-ywuM5K+QfOjv#4 zjR%hhFTf3?O!{H)?8xs%r&yOimXvGnBEV;`7yxSRJP$NfW{IyzcKaw|{y&dJ3k z>G8K62mX6t#(*!F1kednHhMFR5x|^cuNVSpUhHHO$7mUaP%QOQVrNuqC+{Gx?F{)` z68S>5#Y7ieUFn=35#A5sBsAkL9=ZU7m22P%5demcZq+?EVwG|NH`1W$J7Yy}j47>j z3^BKgv(1ID27gizh=M0+$ZiUT8`ZU^5<&a2a(qN&Z_hOieV&IzuuvEBt2l8Zh!Jzr zH@LoAyuCsoR^>!TyS!=r4M>TeBLN-W>!_RJZ5&nQg>Nv33znKC`V zH>q~)SI~Di6KEqri;{HTj8g;TIyOBmx$0?J%?q#zl9+pL>{GNuajL2G_?h^cWrwd0 zW>B_d1aTxBqR~VNqY{F@qbafHeG}J<9A#YUr31sWd)Wx*?99)14}Z%4w4f_C6Wv+s z_mWs9`oTJ2MKq`5?vfecj9L0t?*#$QAd3{CfgBStbg>2_q~pQu-gT+7re1XQV90(Gk_}naWxf zcPxo&DKS5GbdsYWYXm)HEX7!ap)mo|xQjPb3teh-$2zgQI1+`Me=PGimh@lHCLSkh z1T9wbK?VReAL=jA+}6_d%YFw2U%l1)BUetPjRk-|%q2^e?g|ByyEUvqt;qP*Qf*#H6CS7PA8Lev$7ECtQ90!Qu#<@bE!fUM7US{oBb)ae zxBiN2yEm>>f?g!a7$uHeb3?HPYr7a%SI}66S!g6 z^_|c$dt+YQSn|FW8C9+rqE8}F#;FA;NiD$fEZbjiVv-r4sQ}#CQx_!}qewh$xQd#- z-JRnQdPXG8ySbh883zg-!Vv(W>q|?R_=g9>F6nr(lC>1EfxF+9U8gHz;qDA83H?ZN zbGB{I;|xOUZ>uNFqsybgL@d#)^haGeK^J^jBPc8^T>UetU@})__H%MKz`#`ZWzKfm zMa(XQApmDQokohl(g%>FuMqqV#eEO%1uHd?hpK?g<-VAk*gqy?Vx*j`GiKVSvsoYdDiJAI^|))u66l z0`w!Uh-f;FAE^u+L zT*xwMHGTtZ@1Loe^I4Oa(@hLTJA`*5>-_GnYNUoxHv;rejNU&0~|*Ltp}qe#oW z&CD&K9jFzcD3yT9uPT95J=ZY)EU@80WnC;lk?#2cJNU#8RX@ng%ApY%EAb7`%al7ptO^Ao-=KxfuZ7hAShl z`t5gic5stdXP9B-3>G6X9Cv2tMKm& ztJL$7t>zxhEH9fYgLVuSsUD#xs>GmxH{voUEm0pxydzRSwdm%KT^AMAsUX4h@;z3x3p; z8!Jx}yk+dR<78)@j<&Dmf3QP<0B}o-)Y2k8+y{2bYZOD4n=B!7g5u8SO~l@8-ny>o zuPP~DOv4!RFv)k`dJ@hg>2HKvuX`Eek&`9%zx$rLRo~oGdDTa8YrX z9BI&=h&4D!M3R0DF&K;=#AYX5M-O6U@#>g$VW?_#zgvK_)MpFhbhXCZ_;j&%Id6PZ zWwNy??X3_#WcC^`^C`YZy06Y2-OuKr_4|IrUj2T%`SIE!ocm)W?Nj?jtsTp};oL@Y z76T-*zMo9P^JwqDxbp$Ki^X)hWUgm}6}WR+;_mK#xl9fUP^r1LOU@9xkCg9#%8nf6 zzURhEZh(>c+o+-1t%HM)%e1G+1I-(@jvZH8LX*i#%9Jt>iR7dPG`%Jy1r`~e&!FLJ%vLa=5u6{ zW!4dI6N*tbw**qGhiYGm32KohkH(Xn{4O}#oP3RF6>|ta{OKtL9vi{2x8?a6XE0QC z^LTs&>~pXPZLjzBvaFgj)}+O-=~iCy*}dnp@6|G&Oxb?h`NQJ9%GT%`zu&xQqzK>0 zduf!(-_R*n(G1j?TJbNB-((1|?`PtbySDIUIbI!jxC&$?_5sMU^1ZuMf@JD931#z9 zpq@q?t_i(~Ak%T{7tYa=kiHsF;!aDB20M4`sE{Ly^J}^Ja&PK!dFA-}Q9zQPIHs)! z{qNT?^d*8wrDlfnXh{Lh3`uc4kAFqoAWNjjbvYKg{Rk=P7jJD1 z4)ULdXT!+B&PJ7*IBnczHMbVc6Ax#XY$107pLHzGrLupY@YTcfd)9Pq*s0FC&cE(p zgH3``A2w{V3tmn7dR5CPha%Uyb+3EHG({deuuYjyn@T)j%sZ`C%B>d)>qoxy&3G!L z{Ks=G6p(EWaIXik1o=JTTDWz7nKc#0*y>%BNmUYM#TbBf=$ri+3Z z`*t(x&Cv=rVm&11*=2m>!_EOT$Rp^8AUl{=Zczs3t?j@1S`u}M9=BP^B zsPCDq2~(48+pfvBG1>NHyC!q8ZJU#QvTa-6{k-q@p4B?5ziQQW?tSh3(-vzZnJ~U& z^~VS*Ej^TfWrdKs6-Q#9PKpBF<{hQIxa_rS6^)B^ z^>kcr;`*qPRIAYx`&1%c- zY4gn;haM?M+^{o(Gumx?QR`XZI*x6JzwfJ=?~t-Q|H$WEvF~Ve*L|}wHZx|(^@Gn% z_t(uws+6%rDYCMKlZtLl2 zR|H+4@ob+ilzcasP3GmB)>GJm@Kfp_h@WtKWWRsGq7)h9K(&~}#?VD-UdOTFdRE1n zeOxYbcKQa{=r2uz$qlb=KA|<{WR^ZY+fH7 zRV@cJMt|AXThM@xZWe2WhuhK{@)qISH=;s-UB9RNVgGQZ>9tu5Y=Bjv9_;D)(senv zF(X8up?ekGPeS~^FMdYzY5yWCU5aa&sN;{b43Qk7Tm_JqduKm^5AOu>Iig$8`~AY=GV2b_C5?rDj^jgV;xf*xwO7_ z6*d$>dipp-Sx*+~sKVsd%A$bCGa6?hS+(@lIxMyo#_+?6jc8YEgLaI=xAS*3QvOs` zBn{PklR`Cl;CJ-P=(X-*pnS5t4=nIM z`Y^8fV)@$L5~K?0slaFo3p8kNO67e+TuwpEtegYWSIFMp-n@6yKb@Ujfjz(gCbW*r z?-izt%Z}&@w1ui=E z%r$b|^d!p6j)lkooA7!0k-9f%vo+lY$)><}lQ7Bphs8JQzO zhSrgbZb^*5my?BAT3u=Udbk8cu{=~yU! z`!(h$2>24ED7Zn zS$NBnKa#_BrnV{iU4R=mLQh#|*3Y7l2{-V~-oG;zcdLlO?(n~oaQHE2OI-*TLX|HX zZuMu()@6Qrx@UeK4Up1bT%ZDK^di%uaIO?v=fcLK+FE~t&!gdv>meWBMWgO~Z3FEH>M@6P2 zh0qT(*N8!R7~2o+Vfd;xzb=RI!uh3sviAORKo|}Db_(Tw*MKOCWaC8VS?I&oYkJUI zWZ*SCr%c25(9+e%0w;eR75DkoRwV~{DAQaK8#(=Rdk)L`=hY?C(I#O<%g<4-yY@_=vLNaLs^&1|cV< z8dJ&Ob8VdGw;RmT9x{EkaTP*s7|iVJBfN4)!|C0|3%KG?X>+^?{o(JJN;wTN1B<*3 zN~tk}=}6T%oSKFmX5TGz(y#vNZYf~1m@=1Fm>3VrO3|8ZND4D;d~2sm%vf1DI8Y&a1NiU>aU+wH(fuG# zm-Gb0fw=|{r2)cc_=i*lskjki)8I?!XI!9JbaaeYDh*1!0dLPnL0RM_M(e`4)a#F7 zur=BHhiUh=M>(o2;dIeX1(8UdUhT^}Sb1vwardejQUtEVBxH{wpnt2JXM{drBa~al zwWMA6nPRl|_AI*xEi;tG*BfnRxYocvRF%QZ>#Pkf+)#h1l9UpYxa?Nf127`ZHZu0F zLzrm=LZx|Bo;8o|{#};XugQf`hUN1{nr;wX+dj;a|S-x;03RDK2?!Qpc0; zxn}24QPx_R1elz&mt*jF?Vw9ile?dbq!Dy0{nD5B@0E;Iq9VjX)hG&^&C3OmOV8BS z1(U6|bKYohBz52v@pkw3mL|OtL(~r^5%q5(P#7?IIHr4V3JgT+uZ&i9HwUJCxB`6N z0&x%gJ9vw0%UQaz5xP47&B+;w>+bV3{%&H@1(y$CbyANJmG<8Ph*KJq`VVt9Y6HDzq^i7H0KB5q^kD zz4kc&v}Jpn4gn8$zehec2Rxebuq)%I$j37R&4V_e(^?&j||)=i($g3g&={of<+BW=_e>YnAKy6aqY|N?ER%$JOX@tto5;War7^Lx#tCA#3XxM+h?W3CSl=L8>V58C1QdF z84E=i_m@^rYlOa5u6|p$dh+pOh13!~K3W7zq4UZ1ez5Slw9?-_rpPun^x|dh#m4Sy zI+xC3g%I+8>;?NjEkI8^nRMSsGZZqefKMjdDX89X5KifsbrDkz9PBav~*@=Ub$D>RLhhM>wU15f5tuwSH z|5b)e2r*y-E=~c9CX}btU}u?=5se!iz($A+a%WI1@$`XHHR7!bWupcfa9u^id&l znbIv_v}Oa&X>e`z&TD)eD09V3(I`TA<)UAJi34*&69FHa1X+CPWtzN+w6tovhG!>rB(va97T(wKIj^^m*K3`WVSLXh!&(V#Pn?O$%C{Hw?mh=x z*Yg!o`t0J@0a)B&A#pXU{~Z_qE1$wJiE8EqO&i^DrvmNaU3}oUAA_(#P)zbqaDDU2 z!$ATbC13R|x}M#$1cPJ#skh+SX`otTXL@yIJBz2wwu9pFs1$%jt5Ptt5^sVmy!&$k z7e@c4qso}5tejT;LddW{8}$(*sG4H(GKO8?DF&6jH3(~U!ynnWCcDGxPVYZ)YQLiw z>z$Aoz$5e%Y9qK;QvBLHhV#8e;&QEWcv^aZXs(F8EzM9ykb8NvcJntwJmBQoV~)Eo zDX;zwPeygH!ig0k7@_sftfThtCLMWRs^;e4lo2m*V;vk*?MR#G%Six-UqtaWxU7@X z?1E)o$;hu)sYvZb&I)a7MM@e|sR(N>vJ29kqs4Ct^Sl3)$Z4#}IXyyD3 zsqDUV)qETM#*-ARA0WX{a&?c zXlmT{#Fv$waCUXMR^NOhXPKMhOd<4Z1OfZ6d_In*OqDa~#)y-o9Cn_7gnnKazgOe52Qt4Htoi{wu09oWE@L2utt2`2!_^cGWrwN*)?tTu3>()^~+-Dgt z0(Ex{TmP#H--LiC;oxKvyYEe>MF>Oglz3OMKGIsu1DP zCWx3H{JP5J-xjbfu&tj5e#6a?K|RJ0f7``_HWy}*3%~gh>bopUrf#zPZWU&N8^h=D zed=3{HMn#zT?%#9L200gPiObQC}QL}4W6wI>h38L>rE&Ud+n=8(jn4e+GHqML~9+n zwgMV;gf&%;#*EAX@8}PLe&oaXHoe)Jg@FblNHJOCMdk2Kn;_(;D)}~=!C4%17BB0} zK6CszrfKZA-$CjY=*^hcdM7mw;e#iL3HOmQZCbKo#b7a+E*^#Lqos=33HkiM2q*L< zttBUaCa1m^rW%J#I$hRwHPhIkNO+c4^uq|T7^FMA97jzt%nLO{gDI>~L2Vil;iVZz z1Em6v8cnGGmHI=6^yT}oE^~KKjp`BqfLfB-@}si zH=V6luR9}x{Mu3!0hln|o#Mhqa{SnAr*6GSi`56N*ss9#mk4AZ10g>>eQZbJ|NrFN z^@)TRbt+joyne%LJU_2(Kn$k+LYxX(hg1@OpD!h?bp^|L1)D~aaf45uCu4U=UHsIEHQ z^ppzNB!1>4<{h@`Q9o1T@ONQT*f?Ds4m*tD7vX7XOMivsQIY+r%uf-@_um0-vZtMC z*xYVSqJ286!&enEM%jK^2` z_^L=c0<2GU6O4Z~x+0Ie+8+FMnq8PiK%CG%UYJynCK%J{Aq9KQO~b8jR+&vvq!%zd z$vJjwhw9LyxM#PRS&HLm;BY3z`3PWlFMpXrw1MnipuO{+NO#P&O zmN_2x?%`c=a9OixQgA<+Gajn$lN=zb3HtI}#m7B$=J^A2Ty?1MF3fURmEf#zPY!?U zyO!qzP-V!;w?M#_VSvfX#=i6VVbXaKJaDSrn3ejuCUGa$p+9}ZSV$?Q@IR-jku$Gd zn!6n`WovT#ZguSQf-wBKO*;FBEwIQb*ls!4MVH+7?HGXpxB9AooovU|obt{#KWV5z zcBqYl>)w!{+)jUJTrk-`Fkge|wgY>%2Gro>O+#bb9-}<<$h^SOpGWrhsBiG|$*K}X ziLfB&^WLBgGG(G2cUWkQ_CVn^R%mNZF zmV^4ohB1G=lzub5A~-l9N>BmB7&0qqzqk7D$#YG{MG7rs&+Z5zx-Q(22h(}D`9OlFsH`{UDFjOvKVG`b~YLpa?k*e;KZsyu@s{mB(_8meGduUr*S zTSF3!?g$ly( zFn5en0yD7Q=x!Ij(@Wj{YYMs9_xa=WD?W}LD~T&`3GS8xWw$JUwXGUu3Dm}PD z5OzO1=k2H_^IwamGc9qS3JITzFoT7cHaG=;V$fF2zQ2pi{lxU+ih{ z$Wb^f^)g(BTjefL4oHCRYN#hyQD!yu^} zMJRA3K_n8K@u_t5+>q*D=&%ylbkT8=wY;85EIwA{*=Jey98BkT?e&wCQR8QbB%9EF zVH?m@m;~L|7~Cn0i`g?2g)A^a+Rm6APtY)Be9!cHLt9OO;rI4LCD5}nz>3$uV7Q&V zhT|jQe|B6lTW0PH+q3>{YqQ3XsH(tGmEJ8IV1Fi)&%Y7SmGa_X9T<&NVdAArlhDa> zhlgjjIkILgGVf@Oo$*afTugFEb_adH%E+#xa`B*JsxV8g0+@;_&Kfk^mv&-g9lCC2 zpv1_%e~0MC+^M?18>e;hnS_v2Z-2n~TQJhRoF32ob``ki+9jYzfZ}oLk2&8lO(;f6e%kHEf^a_SbWQ5&PDy&OPA8w@e|Oec zEf>zH3%_;%WPBi#7CvziF9iU{Q}=2Ex!B*@*_mz1Bh9?Nd~xl2i7vwUubslkqbCI1 zhkzSvjuk-J0jv^t7?6{VmnWe;b&2OH^sv&!m!g6FJG_;k0%#HlA;Cs1zQa#ZT7)2& z{D>Z|$?om~HJh=< zuJ-j-wm_=EoKmEKk8#L#Cb0#4K)F{59B?5HXLAltGT7|D$XA;^x*;OF3$?s+M{(P> z*?lwZ+Z&LV3l;v9+JCZSv`ILVi0cH#$PiEU+THDm0-kthfz?Rj3e>9DZ7(;2RCQ@B zWm!8Z%}Q9g!kiPKW!3ugRQlwEe*X5@6v;u-(o|!~jFA{s#OA8lS|e2`20!2Hkp6=|YODUGNLlAS;u|!Y>vp)@?>qG?BYr7xdYx zx|MsZbeaUr#W7hfuTI9 zhWXnlC=PYEq>S8#vdqE%ILKmVInT6hYpPREHf~BkXj&>iQ4A7WSNW*OCI`90r$dC> zCN4r+Yc9TW%!a>#f6~l$;%X#=1wzWnbYTCv-$ajJEyu0`j2qW%%rDhw8ibioW;C75 zWNyU>d|BB2Y2Zj_Ek;22If)gzFr+s+%{al>LI!uxcd`-_RGE% zJ$>oxO-n_C85J2o3m@{DYJ#AJ%$(3n;Tj1_ zAAgeoK<$S7`~-zjEhfjh7mGL|fho}avso?HS-g>js54WnrZ zUSFK*Uoxw-HmQCa-ynvq^v$9?n)~~mYRRoPymRM=ThVyjaC_@^Gkg8^hAQsMAS{CP zPO?|>W=HCvHV^0BEh^ejgoXp^CQ{WFySdD8Px%w>+aRo3)DKOkMJZ-HD#VXBh=qX2 zZA`n?7JmJLPR*K$v`ERXP<{>;V1@V&h>50cC8__^J$7Z1KJQLxBektF zacO(loFYF{QjVLB@^+63nZ>7+dlF2{truxJC2!)RUPAXFb8nJ%_P19~qoLBG&_xur z{6c4PCAQi$&n!YmKdx_vcrcv#pTYnfRcUBvBUGh|>`};mj$B`#X8Tx>hn`6&?vI|0#)%huZXMia!53~Nc-`aV^`z&$>qBPbpy zWECh$Zj{3-TboCzs(P-aTxMBEHh9%A?9BhG4UX%FrzOQW>=Duv*8X&z20&xbY( z$#iP~iZZSG9a}tB^{gs2p%rW;2>G>Xm-e4tDsZQT3g+-lyu7N z?ZFv*JdxNoTmEZnB8g$ZF4|sg`R_~RZ_Pdre94pKuET*n#6Lq3ofdW70G?rUqmwjV z@_RUh{>M!Um-`VRAY27pAg-n%Y*rw!^wKn}WLiC8U<<+PLuZW5YJKUvf_$Wk9rH1Y zFaQxrQ30YTAwL65cqLB#F(bvxWqM0tq37HY6~2fTuU92*j%Jc?l4R<=G|w$$NXciGubP#3*`V(rt$bnU0nA&N;5sK zNCRw2JkX~U(G~cWm&7oc{)>I))EZiGe(#pHyxB9rbK%ldaCl9HsQEXL8u2M-y1%fV z^0&pG*=H#;Jj{9Nf&;hs#4?8%CMR!%0S<(J`d^O4*jP79_j}h>g-e{(TVaOQYcQNk z?XijO=IB5HzM=VL+#(Vyl!?a&Of=%Ctwm*by3#ay2wu;DV!L0N$x(wlrh)r-`S=#DkY>b3K)(sLqWV zXK83}Pm5fLSyf68tRtxv3k;$3Q+5KhPI+46?Gs$hn_nAM+zqe{w8`Y19-<8<@iB?k zlXDy#LMHERD&C%4oG;D@IZ}=Go*<0)P~8{47-J?i8={XSySrP+##_f271L%0MJF@> zlX#L&@Qz5h2KmrEl3k(8IWk4|Wj>>#Nc#6xIN4#nQ~KMt?e8&E8nc#l1%rDpNlYs^ zq{GlTJTH2dW{+8cn3jxFqWp3n`{b{fe#91i2vbpChszfmCp|p z6sE(67IRtXj0^3tt!vgK_JEq?KqRMW!GTybQKkTB*Znq&*s63E4+I&0xGkgDP zhU9Gdmb%n9;xEU=KppKvM_Wcn2-JVXqkXxXM-77lh%~6@wqyAMK2657S1kH4R+KOJ zEgch{X;JQ9^^1979F=4n+ejUrw1#6p@y;=30OM$ZmnuB_gmgyblpLhHNvva&aG^qB zy4!U*|K!_|9I#P@>|R96Aw~&cmr2O(cxT4`eK;tk)o6#J85A^Ys1_JT@UUyV?Zy_E z+chkCGjI8NH1zZ&lH*5Ca9!wl_b)j){5tzbYBQFx?7oQ|kwB)l;fmLiUSExnATwk} z@hD}5I}8k}C=e&35S=G|)c7Q;>RbQh+m~Ccz7f2vAR@lMJBT65{u-<(*HY(_erIX> zVnBhBP$Yw}5GPE6HA(eS#LD2J?m&yNN}CV^pWLinanhhs|L>saUh-mzxNqJTO*qJ+ zG4xd@eKsD+Mq=l806;z?kXP;g>vi}&X&zsSX))jtvv8>KV~UUc(@BxqTDs5@qjH+M z<&4ZFU@z4P>AP;f; z5qwRYP&_mei%un*a$WwFS^?&#{rbZo?d!A>H;8rS-I}34lwyW|KxzAWnpluZ7PJs` zZ-I!=vSiwFg+?ixBSaWC3$v<{3vB#WHn7T0h9K42akjNC1RfX>BibkxD@%(zuZ-|Rhi(I<6%f%E4(0)Zq_QxJ4tRXh`FM~cSpoW_FTsoOM*|==lp!4sr#Tbla2n$5L+p+WBTwd*55RXGJ0hc=5}VClu*W69oa_yK<$PMB!YNeAL|R zYybfS05`%rcYMMkq-^EoL;nJ;o|PKnq~DOffXQVR{&WWKYxM?A$L$2 z?!WSvt{A%Wt+XPSMs5Pu0Az^EV~u$dgQID$5jSw_;UX6+)Cn4F3!OBgdR9q`wgKZy zg!lJAv=C`Zfhc60B0+Ab5E3Rl@O~ynYj6^BCOp(RCP2m!KID$s>j3^1A@7hU;>szn z@C}9^M9F??+os$ew;|A zpptwavp1JmJdNK>Fz{vpbN$z!h@o6vnR_#IttC9RQ3O$#M*P-1`& z>_=uZrTz23ZjMw0iNLw(H65Py=b5RPx}5#NU_FF?U(E*hx0hLDyV;1}dWel`*g=A$ zZO;i=j~&wjfCplGUfXOs~>KqoYz)<4_N^83J1c(;?}av@e{* z2v&qPCX6EP8lta;{u2{zoE{m5s1`S4O(v@kRx*!7$kHKmZ-QJ4p4vySc1L;Qx4TES z)MP)yE6hw3Bcx$CVPm3JxNs^y-(LoW--S|R=@6$EtK&E4Tp{}BW_g|n(GXbJV}o?z zVOZvUZDzUs4#--8h{D&^LYz!NfBP&Y67*K2PTJ7pY0?!no~ovzs;{K@^(OPRuIz8| z%gO{Bh-H4&61GYhakCFiGa5dTRXxCgOtAF0V){Q{Ki}B04vi!va}~C>8NA}$dHt2* zj-ta#@_8o?ph0tOMQH%}B;rsP^tZ-=hF_fEbQ`qU?N8UvUh~@B@v$Ke4QLIjJWk}Q z9=-NRDm1zRd~f8tCS!2yAo_oKIYSIP{M)}-iV6G^;Mf|aln(F@C#vOXX=%CTVuyl;0?c?+WB{Xle0&UF#OIk6dgwt}h>3%v zUGqV3Uwh?EFm;=nn%V-amoMr)(R!~E!;Y7+0+()fsWV2DlRhvGqq%Ebq zPoBsYl!W#q)*7fnm%*9tTeyK#0absS$WLVVh#q)hhtps~Fe(MDnbwko<5DWGO>t~T z_V0LYtX5^V8oQpSzeB6n()M0V@Ixn|-pYrianBx0-6LAQ71B(n+^q{r2yIaZPNi!!`AZk77)(E=Sv1t7z~2=S-^{~o%%f3f2RnXM9sfQqgSuyioE#lz zAzZ|a`7QdNGx&hju33K~tlDx0k9^4X^A!)h$Qy#f-dGhuyGs|NKSfo|6dtg!EB=8?c7 z$e1}C`9Eve#?m7b5Tdf=$ndvs{2k#pAfxTQG+9MJM7Y{swqy-@UYo{&#`ad zjfKi}_nLDqC@Z|8;n0d~eo^3_ zwFLj*kYpXUIFd>@mYAvGj81SZ8CT(a7A6366=)PvemzlG^*f6VbTX@5B`faAek^vF zCn$#DdA(N}ZnutblKnHAK}~=dq%Hd+epp{(h^ShrQ#%;>q5%vfjw~#>Zc3dh23gv^=OdTn;VX=P zN?Dl~!eoj@xTqwI8L=}gHahTWY?bL=Rc`PJnRXFjutw)sFrW6RxnU$&hwOL0OSe6> zVo~P@?!{gfbF;_LALb|_w&QjO+RSSiX`t`NF*W?DS9~lVG+BJRSo1&yNNYfbc9zE^B)reAl?1W$isKw#IC#e0*G{Te78d!J||U z!|TI@=teI})I0tx$!oTfirvfj8;z}AL{dX4C);3nu^iI|OYgWKzx7YUkmHzSqe&LAe8F6Q4wqVH7m^OU`NFds{V z3r4RnMME=#xNSPZNs&iVD5~_qkL#A0Way0w&77Bx!P8o(8sxRkAt!r#Ni2wn=ebi{ zRSk#=(&-wGbJX0)jVvr$CRZBC;m2RviBQE%ow!s=MT|m4PiHZ0q!Yq&`rM~T6UeY+ADC}=DUZhCN)D*R62*~Wx$8xVqt zf&vfs)gPzkqHCd{(#qtL1e(%}=8(0ex;Wp%B-SYQ(SB^vwdOn_!~K}2I(Vw}Npu+~ zcLdMkO#i0^fDB4>bd7HSqIUqBPLfou+oe^SL8HBB$`UST>U^R2r9&zjfREU;`mIRmqg1eNCNc9Qi*W0tXW zCJV#;T(@uPjSM8PRz$I_1=4=B!zbnAH#f~e0y~Fb?DXW za=9jj#Ds!@EyK5|xOZSf2>R3LpV)Y(1CA1DWSq{3ogVj1_MeQ?xhp*L99q|gcAL~f z-Pbig{}$0E7E!}W(;0<}Jmu_QWDO=%8e2mO>xvdw>=X^Mt{cz2S_-^ZpZ}zhT|Hq>_HlkZ78xtmnT zqz&!ImA+2SUEd5s#848s?}Uv9#uUKu?z+*C@Jg^GTB}e{TN?-9*&i1?o8DzpFjAA zcB)0-e^|FXb6TcNE|3CgI#HYmhfxkdmH$Od06UOv{K!iRAJU4#O*9RpiY#W=)^Npu zOA|{6-h4(u!XFOJcDc|>7FyfY=GV+Uo=j%?4!~tGuBSkL4f)Z8A)HO&YV|qq5^`!q ztv9i+8vHU=3&DhHm<(g5?fB4S1Pic;WONoXQUHLP992MPGab)vJ_Um-z4L;-xTx|@ znpIp1e1{Z#&hVJ>e4NERCmL)5!+c$1cliUOQ@u6isDlF$niFQaE7Vc|Y(XOO% z5p0}EOo7XXya9p54sJ90f%|s3?1%z=1X_%?Zw`Vvi|zl zsYhvFdYq%AC6-m)$uOD~hzO1}uW3#DtDG83gyCqVg)0xZ1k#aCy^;`D&q?XhtF4A= zNc9Gbd3AFcHTxRA6;Lr8f0>38p^~@q!hDnvFw0&FSKSjd*sYEy~$rGJ@@CX%>AK8lM`(D<#JE_Uz!=I?E-7KT{5{& z=6^t88HMl6q~)RYY^~S~(NgO7bjj=L;P2LQdOdf$`A?BxHubRU!TV3{4j0WQ6~O4XV$lZ5Agz~?^@Y6GLn`8ZLYh}cb~oS zZ>bd45%u{>{FlGX9&nA(|9vhLbD7Z=x8hs%Ch9oPzV$sU-kD z9~`G6@CBmcgmJlLsesF7Ix%qoPpA*Qz%amACf>1cwKNrwZ4B%h=o+?G0Pys3JRF1Q z;Id@4wWK1YRh>z2gL0G%+P@#(aH@EE)NoV~4m=%#e4Wp5#6O7&`C_@sH_Jsj>0&^s z-zG2@EBnlKgLJw=LR6rZ=R;;@{b$->5CVT0V(O0P!*Qv&!ughPR4~zDC>9d8=?JF4 zmnhXHXK4$kM6waxAO@Xv1jmfMV84Yp8oAdurB&BALb2Eqr?5@el?6|l$O1~GD0~95 zy(P@xT0=~%Oop<<%K7oGWMrpelCNdh)3vpMlML3oSJLY$p{9&Ru1{{tph?JEIRLCa zgbY)|{)*W&K-YrnyF-dvT-0(ALO%+bQj5AP^JvolqBAF`m3s_=-hT7&*inZmA}0?Y zH2fcSQ#WF8r(&?%B3-3?!EJDao_J?PG#_lq{t7`n|HdX7WQ6U; zK3?VeHUyq$FZaQJdOWgmjYNfoX{4H0qRa>FH}1935MF^R$x?Yu^}B&55^y8Z2z(*> z^PjcBAk0aoV=^{G?-YjWs*FE`NYkO;#rk<33LArMy4&qf!qnd)PSw&h^Mr)WOlWi*b zmkbu6dKJJJkE0ig!SH1YNx491W+9F^<5p3~&mxF^DjHT(y3hv$OjkJQfG*-9D`WA{b?A$IJJJr)eW*)u7JOIa zLr3ess>@)i+WUDOswv5dAEhDEmMM3(Q{$VU$VmGclOyM(k!{J9DXE=EJhip@G!r)q z)0cWc;guU-y`$QZK9Zc}lmz0YbyKrsuhfP?My%*?gICVS2R0la42w*eWT8O&j0+-x8FNL;pXPCu(H^`+U~dB zZ2G(S&^6>V#gCJwFG`CEEMBaVKP+=)ztocdJPkx3z#nv`-^%g8*Rqls{`F#=j5?KR zI_GVr>EezxoXr#bw;^w%8z|zJRM^oLX%M!8jVgmtX}PIE<(6uW{K7C`0)*@FI}ZXa zX+?$6#-WaT;*op9aN`JOR5IwL=sJK3L8?otP)As6sDdzAjE~y+`Ea>y*P-)AR7s%nEM?UgPmqA< zs>cf4+19q$#sn3R?t~hQY3e6h_72jil%ngFi6S-cDblq)k;V3?t3;||w=t7q<4tzV z$ErKm?#!$m|NTBGBnu|MDpcOw-=@&C^ZIw}@>&ODuFf7e zv~*I<2}>@D z$enIPSeQG~A0;I!6%CJJmx?wQ$WG4}EPi%5TV4#y=--=aR8^Mlf-pkJlrpP3$(zeU zb!;Qs`nbb#8U@_(v>PiUA)bpljd_`sN+W6|Xk~j8aKLHr4NC*Fo3XePM3)IHp*bQkb znK#H}hk9Byn0#L3xaHR>T!zhrp9uE}#D$l^klN{ zgynw~Hd#FQm*%-flR2q5O=Gv;>gLJZu{udkF50_d%hYPMxc{x)OYSs#0%VCzM`<MkpZIbnyF4l@#<9Lw_F$6VCmHkR(RiRqwI@7@zUnT2;SJI74 zuvC>4mwIk|r4pSAl?e@+8QhWJ)V*@0ZX2kv{=8n>LSseZbDIhA^MrKpnz%HGG^xoR zLC8w1uCCws-7tY{B6nJL5JE|~t6l+%D>LsffijsCYn!OpBeg;#5>+G$Y9+&0QQpf?Xk-A z*Ct~ibieaqq#obguO+DETwXvwF>(OCSbMM}6(g`1=us=o4fHoVQVLeG4yKoyL+un6 zdNU%!FYMSUtoE;qc15hZ2jk`5iQ;_ZlQN2QEq`G0xm@zoX{dru`?#;bZJ*}V)Hb^U z-UT=ZmabuUqH8q+i+kX&bh90+MxA|Ko4OVozQxZafkv9r}iIxRX~jx)`c2UVHf5xptK*k-Bc8 zS2{o8IgEQ;=s2aJa!)^Xfd#Z~%qJ9nW_Lcu%BB&hbbvhr?jM{K6q%vx;U;L}dFkwOc^Bt;?o3e{79 z`6Yl5miUk{kn?(CUDw4|#jX32C-p0;;P|hX*EF{_AnSt%NNrfG&MMA|@MS2F2(s(b ze4d}l+T3UexBCRyU4t_0W@l!4_fzyI1>WBOTb?2*uo=V@I2++iKXLP#ypzP|TD2be zhQ^>-$=YP(a8#M^JZ@ultd4H)H6{6*+jD}mH~fp7ghb01i80FJT^>1yvp`hiVKeiz zS()Me-Ly_e@>1uBjeKvJO}3#?Xj`M5DR5fEf3dEdV=zH3`$zH zJtCM^N$1$*4d+^%RnZAee_~{-ModrkSnDJ%GQ0MX2`o}&1M`0hwwalN}=L!YJfme)(Bk4roQ6a44FL>tUSKi?`Wtl6&e{j-Y_ zRo3gWyr$2BDv+?BAywG7>HB%f>9_~vD)^MC>7lX;4O}c};b7!Z<}Wftb_E`%B!*oC zO?@YV@EI}Nu&Gbuv;)iVy~7W__%3 H9d1U%b@yn`7}~Wr6nfhWO*yZbKOQBmgQ! zQbIz)UbtMQam&H$RZ{MPe_}G_f!Sozn7-2yE`>gOsDeSRcu99}YbYPrr#c~PlD~kD za`uKoPy%2+>eXcq%c>bnK2JlJnzsqnEK_>|`f*3=$dZ)>qNLWDPS8P-x-24p*wtnk zU6pbsQd@+~|4J=?H;@LJ-C2kw+Ysl`p<2io$h5>?I%GkP=rZQB zdpZQ7MCfB#0XZdcT5~us56=DRNH=E!DE1W`J-2KNJtBa7CH%}<8%!H+h$Fn^^SlJn z8RwfK2?PXxnUY`|)+~I=+}*x@U#m0wM(+CE+XBHh-cm54S5U?vZbh9$~{1ZT&M)#Ii)IqrUSro zgkb=$PSq)SKMk1|xrje{(Rju{-|B(m-e0-hi%foK?{&7Nn8BV}ULMee$?+mOquDS* zQ)dG_#rttqZ4mI1|G8&NN$YRKlhaG%aYrzIdy#dn;v75zPd>}POzGw zfrG4?RjeT=p|ye6VvbL4$l;>17!5MoXbaF0o8C(|nXnM@j1+YmB0uD8Feao#uUTdw|>(qH>K|7cWc;P### zkSCo#hZlKup){gHH(*gLB8?*pc7^g)fxCS$*DbDxd+tkhc&{FR_UbEnOwIocP7q#_ zBmRy0{d>sXTq|EF5W^F4c_{p(*XqSZqgK%aCB`bWFHSAT^{*aw+@2UxWk1&nh{hzdbB4L#n3w0Wep~Ex1`35tuDXqz24!T_B+t= z%js{G)YiVwUhCYBsVj2o?yVEp{d|U_F(H~UM~P7Vac{J6ZO&L$)oq0-(|Xa45M9&L7-Sk*)BuaHzp_2ggc7H#{l^s|ne2!G%K7leb6E}tK~3{; zakqJpwZ0IGf?6Vtd%rs!DxK$bP|}hu#;9M6PnVUw@MqM`21!R zp`d5hF$E1-U!DC5k3EK7G>b3RMyDr=Fr(*#HMz5MjrS`e z5IiWso{`&J9EwU2GY0~Jwm!~0HvSA>^CE3N0&i2dLltAr$Pn&Xp}U+8TG4}}`py%I zSwNzDHX>L9w6txS{P5bfwU;scbAku$qVV5-PCH<7V%Q2~<9Xhx)|*&(WZ2*T{BWX9 zI`Q~HC{@adM;sejE2boddD7`nkZ6<_RYPkfbMP!LYb^U?%h+aGfm3(qFGZ7sr7r`# zZD*#o&m%XxKv&W`4pJW3(nBcHE;$5{ZP&!s&-OW{3iB1Hm;641!wz%-f^+)p{m#ZZ zBvQKCTPw3+%Fwvl)0uJq9hW2eZhbt6T*!0MP^aG5vAlrIXd4onVfyLo^}PDfjdq!( z+&73j1WGv!G(HbeeZ?7Vl{<8{cQPZ7Dm8Dd)s>RpU*EyOVTcr6*5&$%bmQ_&TU~^O z@HFjmZKdca7fOos_QWdt!ARW8e>A8Mje1yv`BaPEihrJ(&mhjy%~j4 zcD1FCf@Dh!@wYj{tP{fbGXMPfb48SidhJO+qh++p-tKmpr$X&;OqQZVpCnPJ^It)L-p62-ebMXIQj22W{oRf|K|oiSlieFqR9eh_?t99Q zdSw|)7W9>ah5|B@kwQp1&F~ELt$x9g7No;5(~@j2$(aSbn)12)ZsV>;4B46z|9C^f z^yDZLP`p^qs6zjU#uO}RQOEz|=_{kM>bkBGX;d1dr8}g%q`Nx=q`SFD>Fx&UM!FjT z>FyQ*DUt5}Hup2$@BCme7{GPT*=w)8Vy-!(%d3-N23W8S%`bL>#6Q_QHaC-{bQ8&$ z!A*se5qzr1uTjnwYqA8%<5Z(F>gT4&-n2HD``VTQ(@_$3t5=Q>q7UN%Hz8MwUI!-r zWcMdAM-o_YKU^dCTs}?oir!F?lY5DZf4d1MD+=aZ14?-d7#UVY9I%!ald zM^m{ILWrLz75GlMZvFT;e>r0S-hSD1wlozggUwn%w*kt0HVtdH*2+S7+he@J5a?4czS zaq;~8b#np%_Bww|XD>vaSb$K2exiX(OV8_{KtH=Jp0CylkEzE` z)BzE*lbX>`yuf0oP*A#*15C?y^zCZId8a zgP^c*IrG&njoir1O1Y{rPgPM{wXLx3W2^}8;;htkDiQ74gJP#eDHG7?@k!Y83X@c2 zc;y>mzgG}`x8ds<`waFZ|Gu(lC1RXK&w5wD=7?af$7iatt^uzylfbR?r*=68sf(SL z^L;cFv#ay?g`Mr-$WycKj{D_o`!l3+^ZDTZ8;^H7Ab>h$^4)cF&`SZlZQ&*($sawh zwh8z>0$;#ue0^^gIxwxF?dKkVc2 z?y`2T60&;1q+WYz#=G=1%}AR>>pJ|7cMsY;u}bSMsj$8(b;xtV>&J~3anrBKo-#T; zY`yf&i?5rLQrX#^Dw}hMSa;z?Rw4zt`A~{AZ4TBoST*pK#>mRiHLK!gYacXm8IqS` zh7~{3z)%w(-seC&&qB0ezX}y7i+?TBEmuV%z=@O2*tUg>lPAjnKwZx6Fw&=CA zkf@Iyz!Q8$pV4T^UjL;2<9l8^ZL4Zd>w#Wb+pl~Na}-rNbCZ7Z{#rD7r2V3>$NquO z3khI#od!bDyG}=NZq|GstM*g5(CRmxvhVruIof;T>5`{bS7oi7kbyi!L2+@0GtU@~ zM&%zWAjAPu5U>4KCZ!5>$lv{9u(BnChnAs8R&0^+i9i8-b3(JPwmi7LIhC%l7fpNCLL`)nFz?tnsvweCbU!u`jSQuNz0=PB4RxVC0o74HA>!ho8b7Iaabi!64FWzV z3^d(8UwoKXrK)AZZ!})_6;@z7nH8Myt$ki%TjO9z&w$lNK7-&`47uwq&0GWtXL}&y z8ECz+KG<2zl^gAFpmaLc7CI<*1&Vt_YZ9$Bg!qB|RXzSXJLf?eOoo=%K1ZnrMt%H* z^(GW{hoxuh27~8M8??MJchqk!BtGjjVBmX2GY4)ojEA~U4V85#uP#dCr@Dz-x3Dt~ zs$2116_gGZaeS%7jk6KwX!^h+C&?KlX%@FnPl_Su6zYr}XCuNzquPnWJ{iJzk$d(C z^SD-0`5NZ2;|m35o5B_jTu!1yQh13Y+T4Ook23GOp2n!(4TP3cSq+FjN#N+TfA?N# zlhb2}r*Vwz`26|JqAF`)Fs)?N@GTOPX@cU~T~X6rS}8010Flfj|f7f+*8t zma5ZUTwm&4B!>k~5Mfa(+lFsnGj-E>{_}D$uz47m*u0TWaI|^Xu_iUpj!56>Nu<{n z?eurjg5dh$@>xz&^1T)U$%%LN>EYq3!$79&x5e(Egktx*{bFitB(H5c1C#EaAOZpB z&=&_G-4a<(4>(nW@om5Kg=1lci}{=cNSyG;R0*CyfvIzF;B71 zJT`a&?pN`>mS#Div)xl zWIcLdXdhqSNRu$|x<(|Y5jap<{DUQ=*n`NnO=M9sMJ5g268HbUcs~+ZSl=AGz?t{c zE^t0wug&#U5rqzd-$Coe#RUM(Cnq-Vvn40O2>$?4kOKic4Fh1}2uZBX?c*WY(EU*G zgXxaG#YKS7X&JJ6T7fB_WY2PCM$6-}7~AE2XZha>pxA@bb>8+IetW)y0rM(F-<4+lHec{hitn~Y^HVUs``;m|A$InU zHp^`4`fh~Jr-BcjFW)q+`S41_5Nrfg5XX*eYj%2Z15?|s;9QSn82&73!b7oG+coOu zyZos%uM{Pp|(Q7pO%OVT4- z)5^nlgAvEpYZj2`iG+-FCUmsdH;vxu5gikj^cyemB;&5A=bt_xFFracw$KIrK!Nrb z!hk^%R2+12v<+}YZ-E+^*JZ_;;``hX*_-8zSnERIs0%+{AX8nW8_e$lH~(j3GASOd z{_HGeqd?}?-In$JB*@arFlZs^h=T=6rVwX&PujAsyqUUzH6U-Df@R4oX(<-)cu7b9nTF z0~RVxwrHbmzSWR#gF3tpHBWwTzOki*Rj$WOo~`pwVoQRS`z~K}=Strc!+`yCcEt&- zER50!sc+ff6*zgfnHWueUDmm~U2=Sj^hUcoq#!4ML-{ukkY~@FbmG6?GwUDKBH;E& z1dOcji1`L1_gRO&)Vb>?y+qP>>-u0ojTg}C_o-<(o&0zOh***=g9>G;a*B&XOf(pK z8*Q$R)-XJj{217aWf+eMt@^J6Sn*c;g{A=9cskrm-)OGXxX{Y@5KcOwR_>(4S~^w+ z*|9Qg_?rt%g$>8dPQ%oyZOv85&TiF}eYMI4j|HVdU-RNTx+m;_mjt6!AJaWANeO2z!!w|fyT_>uWEUPh@vCRJEt3gI$JKZh2}Q4J?QVd zIfHd9lhnaHzL6B5v-p->LYV&=2)o$4ROshhbGxm}a~t|yt!oWmZ~p$!frGKrDML^^ zc-mXsWBWFH&57Bl?^uzkVME{fas*>4!(JeCHM8nwSmeR-M|6WH+F4z>7kgeqzq}R! z#KyJ2_>dwzeq@bUzE_c%i5< zpx%_#2v#s}YTR;Y)A(D)br%_i-EyHjLk00&4CHBIm@E2;lbQdDH(ic@7v1R_{ z5CZk0eT(mtn4zk@Vhw*b@+joPT$hxY{BX3@=Ii zas8vp-#`pu$^J;Qv^t0ss>i2s_cuQ{v0e1SgN*hH*<4`_60w6s3`gM-B14VP0jh|^ zpt%HgJp?VxfQCG%M=T2)uYiIYOU7{Na!KGdnfe1Ic_e0FU{1NF{Q+Q;-upb0f2{J4 z(a-?lyWJxM@fDtH<%-PxsDrM{y0ZRVRn;aW3z02hZ#^(DBApuP`p|9FPov^&Vf~qx zHVNt0?-gUqALX+kQVc5j0%Zo3%d$v(!7MU~=#$$CU%!VRP+J2r#hSDrO}K44eq?C+ z<$=F8Zn4}SwUs%Rk;ZO?oo1a36yHzW#ko?3u^xo7b8)=|KV(*>8&1>C!HpbOFQ7g} z1b*bm$#u*d{^jy0E+`{hbAhI39D;$~+q`dCaS(i>w1Fz%LQE68{|)hEcGkKmjS^#z zW+iZx!*DwtwZ-&HBka;_Jj>?b|E+gUDBjJ80h2vs9@~jqrdMx!?T9kC zzg+*ljk9oi?|TB@-e!Vq*M16j{k&=7b=|gJ2$(JQP<#g;Ajm+n692Z!_x5?(r949L zVcacNT@A(SQR7e1+;wG28*J}S~`B?_LH^0vv zUN`Ouv-Z3TA;e?cQF`B}8Az4az%oDlqe|>!(RiMs*;K`PQnhvY&go#0=@%LNS_naH zpI7<(DdnaEZIt~?Xkvu@j3no45M~egABPC%z-=N7JWU1Myyh=(5h`rT7#8Y6d{AY` zr!x)BR+#1~^=ZWdW@+G%3dIsOHLQwxo1Z^+c$2Jnl_&!v@(hB9n>` zkFPKJm^{5HqHp~2YJ-IlUa(IbV`o(iJymz0#+MR|G#4c9x3%nS!`~e@mPy}X_Nv%5 zduQ<%#^O<$F(!Dex&+jPKoq3CZJW^Ne&v}?o;D~k-YEBT9*5x7`xk3%YDxh)cG(;) zXee`~$p%vq`F;z-pi-hwKRuk;XM5qZyUHitfDo-T+HPxV;5>SJ-*ggde-OA|gnKJN z!1qOgi>TsObxgB5jUcbIFI1H`$bM}SeI?l1mc6I3Wz0)^IhJ7yb^l))<5;UH?7TkK z+IoQR1NBSx2sg^McPyqc98li>$~?&6vRYf`q=R9X?lw)7r2$EbE~C@lB*}m9#s8)DfI>w%yf*}; zZ$;b~R^$8J&HbtIK()>Nze+a{3S%k77MKeVUGL54mLQTBLL)%epP@;UWvtj2$%-{9 z^!!Z?UlTy|+{7#&O2Hkmq4rlR4Q0d{Ew=7Yn%gGrF5PmMxXvFjQFVX{hgEgR+2PS0 z#K45l(IxuBp?e>y8M^@g6N#aU2GMr;4P8NC&Ez{AlG}s=DV8G}5fPCdDpIsAM8r=Z zR=b8S8qfNfr*_VDaitX9>Gp<>y(f^#kYcfh`WtP{;q4mz^Bp?&I!Cr!)>62v@69nN zK8sVuL2-1KSdNgQ{cE-Dy)OAkk0($qpcsz|4RWyK%gXV$p_60ku&0W=JDf;*mpPj` z0Y5Gr8vgOb-5??iq+>-#utI3ZXK@!(Mtib2FqUThWzEo*C`B7#_4aysTVsadGZc$s zFfi|?0IW${*i#Lm_;jGkUqcLDSgv}$1r?cO%)Lo&%nYZZw<4mvN@;!Lr6UlCCj4UC zaA+;O#jBVj#|GgBZx6^XZNeFWFpLS3=pkMF4 zc<2}w^qv7NPI{44fno?TJxF*-rgL3bw@C}52-c%xIdCnu>LWjKG#NG2lPd4Wn7x|+ zn(%V-o`2uWq%>#~w&uzK6Djt%N!e5nm} zj@E66q}`e7gKlN}TB$Dh^s#}I%R6#-+)b%i0o9q9Y)?RH(tG;iyusQSHyT<1F|4|x z|04umVmiFblJ%D|uoRJ=)c@|#1>{+D0*5mLHJ|5Xqiy#$2a_2w_$&jF4cH`gl>af* zo!_qMUjD$MZ#H4H6Kn`l1*ReZ3w@~OP$W;~bQzTxLm-rLR`R6(kPojEB=FEl zG)=AAnQdgiBm!$b5lQo1(2!V7k0D=L%n7bBpxOol!kvlBW9iUw$w705{&K;^g@V(R zC-2olv4LRF4~>7iZ8)so`u7mBN|X9sUlw;fkXu5_{=gc9J;J-ujNdK>n{PyrZg+ZP7jN@ zPIe5j*STX#kttOw+$nF=UY7a4tLHr#a0|vnBrs)o%-1q?X|-R_S2T``rZvR09Yx<+ zUBpd(7!`Ews?RpE3!^*MlWpXVI-4uOxCH8bPh@VMyft+O1J-Wbn}h(=ipuUevA)gd4f#P+_v&FXCQ&$N#P=ao|Dv zDWM<2Cw_w4cf9tB?(~ctcZ<>cX1rlpO%UN|5Og?kTy!{%oxIutbts$%9M1oJG}2X& zfS*zyDv$6tn{5g4g-1J0HV@QK-h_;D&b{iNDu~a{1YObt%_xK{K&g?w&F!O4!_Luz zMMF*rq3NVj2!Ezwm6g5E4O>c8Qopng!MacU>ndW?Be}Z0RiAIY$8MVihY<=VRFUrl z99up*Kk?bMa6kzD_vwt%;z>72%A|#&G(pOgMm86ER#A!ldYAtgzq?@38Tu)jlWfJ5 z8C1cKm9!oQpyaBpc|9t)^g6hNfOBzV)I+?;-8n%vMFGXp4Vg z*RR@8_@I?-5(cX;9_&!a223MgO_E;%tTHY7|G!3)Jb$7Qw< zXLrJEnYH~{Mhba&B~GS7q2Q4`3x47j&swN`q^k9u~dF*r3f+S3#7iaN`KA?JBWJ?2^dtSPF8A5(Po{v9LTXyjeBCvEUILeuUg-{)N2 zpWd4xo+*|UO48mx-7NmMRrZW$LjgfHs2c(H&x2ObNZrGFvzY zmB^jrLLHjGB##iKtT=B)#E5PYoSe!($r~HlaUmu4RWEoTqAX zfAVLo&sxwj>ET5P#|q9Ze*Quh4^R(K?LsP-Y*-2^r@5PcyH;Mr53OIvE|p3fDjQ%# zZ5Xr{$lGI~yP5{AVme1~o|n`{wb(-zI-6>0PAMNUk)V;bq%W@jAo}N!-2d#^dn3iG z>AW9&%X*e=t2tTi^69%!dx`6xX5o@~%h_syJFQ-)kLGTlGBrM_grKT^C(=h%y!4_~ zLm4S!*WYUvBU@I}%a5p3Al6GUby;BIGQP!i6PYb63mwk4kJ)gsQ0l4(9SOM^bh)22 zWYL;subuUVN<=P&>FgdD4L>&Q{J!i%5hzryLM;}qUth%hrV z12!ItgrV3m(^7T1L}?S<1&U006~>fT!9FRVy#=mFBk>@hJ*!x%+WhzLcdgC`+N*y{ zW5F8whMRweydM4FUxo#B)_t=lypNo~T$t;>MWyR$>>RJ*I$MdNWg!ScMJ$hyr@)|M=`*d=J3da-vtDT zHpWuO8=8_bMp9_%Unr&Z2nJ4el+Bni!c+S4Of7`2Fn4IP%qfllT#6~A-JeJ1EO zL_GGf! z_tc^?;2tWKV>2u5W-+Vi;VVuOfqkVSPrKu5%jiVQSSXFfK%tG$vqJ*?ilM^hV%H2K z@U4ipWSu^;{|Sa&6~!y`3W`_n@pBOl14VK|uk4CwziEhkXb`td5FDHJD!JbiN`phh z^6`ffJUMMIgJdO%z`QDf%7va0$x*!_D6wyWqv+X{p8cu1X#PR;PyS~jJ{mT^GYcE5 zJbq&p+!S^T1|jVY{^~LQ$z-SzVI6F3ZF9rif}s>V*2~^{Wo=YyLp=V?7#Kpxr}xO; znbUd;#D0@KKplFTPe_#g@X9meMTiytibYM&q+5xVd={r|y#GV?bx=S*79Mg?A$#)t z;&8b{swZ*awkS80iI%%|-2ORQDQ1cciiJ$Q2?;$uJ-qeGs0^~j(qdy&B6f0hoJf4F zN&&&b7b!!Mnplw!)glqCq6o$+*diYakb*klN~c7~3Ph0KD(AwQmySu8*P9giAqb_5 z!(buV7m6TOewf6o2(-|g&6_`Y4|M3Lh3BppJ!`Sb3Oazd0gW@gdT|cZ?Q8|l{N|C_kU4BL@*kYPhlb^5^OSq zPXFbC!Nj@My#3=_PUTWnpha5P)|RoY!DyD#l}(q-uyxCBg%@0kCiDT5E>&7yR$Pk7 zc^Ly?p!v@&t&nDpHLCvC!+`SC@Z8YGXxaj3dYc`w=GA|4A?k_g_)M3ZNs1C_nSJVp z;nSg{3X}5lh4xR{aYJyE;xHhf(^VCc4Jy zSUWScEOK|VdRWmEp;q8L*(>+uP}akU$?5mns;c=C`BUfqP$CZGB1+_u1fdn1pZi?s z$V83klCV$p6qi+FRlWv@%J|764-PK7h+->XeGXb6(6jK07Ri^VF4Id-R^@dut$BZF zkzQaDXhE%6|D(BopogKJjRXPnjcQI6fh9X(tGfi;M>*^1ne=MIPXoAQ(q@SLNZ35c zlDJwC2j%_AH5|wcLVg!f2pDRyETbW$=v_+v7~&s9smR$p_0N^0w>ZXr#IcOxv)@;U zEAFd*p&$9CJ83Rg-r^jgeK(N3{P=w-@ZhMTBs;wI{>yYfo8p`!H82Pc{a3j+-7Jh5 z!-%pTWhLFz7bl_ZF!kbaO1<%L43_G6NTSRVR`A9EDX<4euJZ=CUd8SHB`@Wla?iqV}HJ zVm3Y3a!@o&nZr54Oz$ZEDeWf*{GtT(T!|0}_U4G9 z;gSj`O74Qg;tAX`0VS@Z=%scKs84d=-%e0j)f0$5hoOX@}U`VARx>IX;hXsw>K2k_U;Pvn2Y?+zT3sM== zGHkY<1`R!v=|A{elxBI~67cZ0xjzNrq;grHJME74oqK0N?r1u%8a!KemaS;8!vSeJ zK%0apvp1<$z&Q2avuzLG_!;ctS)k^6)`e0S=<2=U40SFKt*EQvHBhB0``$ONVH%Z0 zRo*oJPSf`yJrt=VEK-t1!Zs@IZ1!t4{`E{X@9BmcyK7we%JBycsYwShmUwn_EaZmt z*|6{D;`XKXX>i5|uXX4GE7e-JY(CKy#)>bBs+;0^lB9>bZqu2DY=P|bcg>GDf!yl7@JYURmu)63Ym*Oa$U8bzWKMeCosp)d)gWdA^R~E zkZkq-g4Lr|&V9O`d$Ycg z&ARVhzC3@kt+EsXSSBp>*1c3TZyoZbA5iC9-#;eatPxf;m6qP$*^)Z&pb8Su@^KO- z=R4ra>IFoUvHwh6PpylLXXGiq6VGfO6+>?fnq7Ny@>;BseXPKbzSA-k`ePZI$ZrUd zhJM30ocZ#N1`W`A^@pX`X?#OMLF`o>HDWfg8O^IcHMb<@>B*bqh&yq8aCdKIZIMvR zDygPJlp82+j$B%}v{U}r&13{ay&_DFT-vuNAAw52egiF05NWn3cK*fNYqLIyBCa0|bQ-l*yYj!>x1tvHWF#Z>1~`B7yz`4Lv%17G2}X-=u+WVO&wnpw zJ2Fep9C!1!uLiAyG;5x=q`_DWd#)cT*ZhVk;`|PZK7A1iiwVJMfK;~W?lP{BP9Q|_ z^Iq$g81`o}TG_&d=Fj4JOrZ>GhCtO2AsR+Fvw^0bFYH9ua)e!(mLB)BmN=A*OaKf6 ztY(e~rxJyYm(7-Oh{$};0*7BULOoV%{QOQ5x6GD$xR4d@I!aV1gK6MVCQG@!gbK+R zOJEIw7^z0M_>@(CiC|s@mY>5NY(=#*DiLd$UIumt7qg~bBP-*X5;Mi$qS?C!acK#7 zQFS@AmLcPxRW*4u2yD^^%?2&gMNBPMRy`W9xnC7)<#py@L}oWN>jwI{BPC;Jt706X zJtlqdd~`4<*$;edGVUbon&BqWD;yDO*kxfw-Yi?5xK6Q)LAad%MIX1JH8ljsnDf## zvZM#&qcs)G5*BYjVt#t)-C|T$NsW{U_p+J(yl$?vMl*b^qq;LYO!61g;IZ~@o6YsO z5@({uHKww#GFQ4J-JDogU2K68uc~z84jxgzWL;pD!xk-kyeG%#IHvVsev(5`$Bd}S zJVRSQ%=i1jwVoNI>b|mm=Ohf%yEW9RSP>`9gwkXsam1uQ3SWZ6;D^f$0k>Cb5}gJ* zj6jkjELGBG#De}vG$F71*5v{K;EM`VoO@G5tH$kom_FT|CMjXfwRRyV5XQvbYh} zqu@)QSF=0llsLcC&@{7%a-E;7oLbzQGAFd;i-A)r!S7noiABJF>-;I5?N?|W|Dwla zL-xnXgi4vM?jG82c7x$AQM3(mFg0_X8|rT}YsU58z3ps8xwYx_uX}b)?e&J6g}Jnc zbe=xF>$1K%CnR;seEZSdv^kJhX9v${E>;Ic`#rlNs={Hn)dV%~-?h*QdKoxk`x5Wj zDx!zgcaU_m)tZe)tJQ^*Fycpfi>Roda@;scfOW~g^XXN3dOEkmCd}5#fYaF@DGro>T5msTMGK~$vH+-cL2;L@59 z8YQq)Y3u88Gh=zf8 zk_gGfER6<`9qSFgTy-5F^g8@?zu4qfa@4fNHvQp}$5&{E%r4Jsn!xfAvT*>*scJ*~Mq41gq${|D#i`L5%Bez0TYP~KMz~i%ob2+*b262~44DPB zQ0-Ab;2$Sh+MOr<+H<#sN(^F$Lc72)u`~_+NI3qeI0o@`OK8)=G7KGJ6{J1|<>WLg zV(*uPaBQ$nGV|G_cEZRuc=+&RoUaG1r+0gL+S%}VXWcM+$m@k0$6{->p?55fCv~v- z_ga3e*(n>za~S+&QwH1&uTOPPpcnf{B_@|uScQ`R7*IpzP z!5V6Zo-7~C7_p_`eoD&yG}lWS{M|{yWKi+)Dln$M1g&fGh9&BKt&w(!%}R5nOoKM) zjD8MMHyUuDP8zw%0zY*yOmNgA80bFd8{TF~HUWLnQdOi6(QN$7L_#zstIUe@*m25P zaU3sFI^+ocH@OKeSYGSbgQiYUK!TBEZA1KZnk*&JIg906M?Fq?e30#PI|~zTcgs=+ zm}mj|;X>t>*p%}yhZ)A`Lq*!86K}z*v-jUP6c3o*ubqR%#S!2IFlS&ks4)4JN`JNl zaFnHW7sRA3pikAlJEWmAoVU}G)#IFK0fdPp36OL|AOTpt&h#<@JPU|MYDZmYz(C;7+VbVvRx^?^1@nrb zFSVM5 z$&=aNXam}=+m5*4rtgAxAs`;#y7&3FjR*5ePEHOC;#j2z#!Qr;cM*qB=dXEB0IV|~ z?m#2tD$R^*ZEbAfs3WCF7sKxiaseOXcy~bq0164kvW9bIYLUe2(HYV;9%GhQhf5=N z&sWdl!!AR+=jA&0&^k@FQ96ARJX10|I~jHWNCNp%tL5{wY-R0nz@Wb7f}>I_*DF_e zGo;|t6MeXH>YXi1iP6;hII@1L_wL{N#C7L&)2j3NSh+5Q5DK58eD?*VMP?b}T#yJ( z?n=$ZNUN%jWLb`pPVf7h{kn;>TP82RLJeKvWx4Zt9lP2m08;jM)MegAh9&aYGc=XW z9(U_JmsoYV=3jN*in_SCvOoDv0_4{AZ-NA-!_InVOI2F7#?{!^IpZdFA^)a`Ke{Zd z+IDVZud0TVNyMDZCMYN~rj)4j?~G@TR6X9=O|GnbYEj;}U%w}!q3V0=x!Gr=v+u1A|91CGq6C!b$Fo)`xUT_UvXtyvLtDNMWROg?ee zl)YK}p6*9xg2vm!NuwEEv56T*2lvM{wujnqXOIHa(Qn)dRX&IM1OtZaogI8ni^iiF zoH1%{YrrlH59Y5SK4X9X^zjXY`21b~gNLX2F`5^5eLi^i{^2A%g3N@0;Yhb3{3Q!MewZR=fed5%9A7`Z1@jw4s_YL_U?T`CE~bLVVlLzuF-M-!KJz z57qQNFV#97k2^+F1qeAacM5R~j7E4fXp*E=>9B~NzzC4_64u4#WsTD=W0w2F$D_7q zAKTV_(`X|8o-3Obtvb`o5;|6(mk9L9c-1TNjC)|}UoaGrMx{T!uh$efGK3|EK&h!3 zOcI&Djoz{!sc5Qk-bbe&c>&cL0pep|b}3rbp9PXlPPeFhM?1Yew<_rni3tz7;fWu{ zRTTJLf>R@C&y89)XVM6nHnC=Ddj@ON2sBI9@ zezU0!VuVt=dHI`zy!COU{?wWnhqPL2hnID#-jJnsDXz z%I=ZldBVUe<@%R*N|oTv+h1OfyEu2#0xawc{O;#toSMKC4W92_zo2>_@2}5?N2=^^ zCSQSo%C*sEr9T`iyH~1^%Xtr#bIlo}&P*@*SOfBSxu6AEb7{ZW z6#migRZn2Hi+Fblz@J-M(!?UL>wKRDfD9TB06_vp_?=R!>_2f5M7coP8cYdQw*Iwz z+68hOS>9K;JiNRPYf}!7ik^&#^X;d12kp(ageU?xAMUS@%>dbJx&9*oKR^Gf`_@?d z{SkI$%MKp&`tG=zTaCpQ<nzrI^TMML{V63D z9Kw>R`Ya*=pP8BYqB8VvwIu<_S)Py1D6qN=W11QWWB{RUR^M*ur+*61fi6}CIhwRl zWzzJ4E`%gPRv;$Do~Y95L3YKaT=Nas^9t zx6^dwB&6#fU)Ry5)0 zXeNILGWGS4>*DgFdZ%TV3-XuT;P!6^FuN@9oPT$-^1pwNsPcV0Xg_&4U-f-vvj6@2 zmAZaYd~vo%`0?Y|y8X?GA^pf^+4o8#;FN9-p0Cz>t`GLy1xPnpW*xZfe?z@2oYU^O zaepMP_3{mNo9p3bDe-e^5Hb!6P)v6{UhXCQepFCZl>+P|M({B~@a4{UH7fFe5$f2s z08)D(^$Q}V1-)mAU%tm3!AGx+(*^@RH#g(PX>2t+Y>l70W8%b?G;IGFN&tL=reo=S zM|!-cd@qD;U}7(o%K;t!-M@iG!d2OHFM$b@6uo}my9wX!IXd;~r|pzD5oVLWm`}HY z&o~eBc8}PEgoL~NABMOk5ahG?0NFOZc5`R|{5xgZq`m$9?nVaF(bOR@(E~91&Xi!1 z*r;z;K*HBR5ZDnRyg2Z)Z{EC7WYpL@J~Y`Z7|I`pU9F`DM{UByZW^Z@5kOpmv91>x z+AIx|ozW%WiKiA8WH>VV&iRTIvd8OX7&mGLh@x~XUmfYX;5-j>JvQn3_P(vjxH12L z-g)blb(CcbIt1d38C$=*U#kwAz`?<#x(-oD%1qDANl_-SS5|7*nV%Z@9HTsag(q^4 zwWgF|OaZHiywbSG=MmW0=wK$QMZEuw&%ysgXja7QhTR^ARKRQ|5luje06m?{Daw(u z5o~vGc)EFB*pzv^-7wY1 z^gAgdBbi8sbkUF4^h(0amWi}t;q)#vBUc?Owp|Jt>%?xywLknn50kBjTRM8-J-LSG>ypMsqOEnRR_)e2i^*qMUS-@sW|TmRfW z2QwDye;TX2Gu6JUqrQPVg>>w-%KPXOd0roNZ=EXE6#FaXiyNm?`U#PVApEtoqM@Y~ zC?#lN%g&tTZ&RcewUVLZM?xMtxi78yPC*nivpOrPr1X)Sn_Kl?>5tvZO$X1=0VB+m zMEmE*E|cL|3wmEqT>Yw;)o%nx&F2|jpi?t6Hg>3f^?G@m1P07jijtziy?HN0PC!ZU zZf1E}gc{SbiNO%m6i)6RCHG^~(uM*NF|_K<(QFtP068@hGV)S`6O}+$b~wN1-g=Wk zleSI)_Ug+&%2$HBU?RhOgB~SnZec-*iS*HVPjPv9+1%E)fA@Nei-WtRB@&-AT5Li< zqPpU3ujB{RNIdp%;CP&7^6mxsNTX#{>BuS25u5+y<0sRf0_=Ygq#9@pb znd|W4WksnErF4Y8?&-rC#h;V*H#=aw^hta@{to@|o|6n5*%=p8Q&q3h4ztj1zwO6w z)@95xs7EMVOEqBj2sre2iBnBzj5sQ%-dR-ZxM3Tl^{J#q7r>njCCzPLs&zY0iv$x z0Qm2V?GPu4z<~$4%7U`iV4D?z?rp~!_tO#gcKx#N-yt^}A-JrjZ&JoGM*_O;K~Qn= zA=!e(WtnYfyyF{HwpZTD;uK~n9YzbJMb~=_I+~Ue9<-Y0QB|tpEe@$=Ol%Zju6*+R) zzvWtY5}cuVb%jnkB{?vpdQE_PSQ{#u$_GILN8>h#yngJSqeY3QaU41aza<{Wh!Kol zKi*yHdcta28Ixh^{5rDTb8vaKone=oo9nlW!!IE&4tEO+3zRNXK(KaoyuQw$@D<~f z>%6Kx-kz&x7>jOjWF$W?&wu^luf{jXyw*3jCwkB`v(WcA6{A$h^13-MjL+egca$y2 zQ=ER#an3$W@O{0~nC9|}K0G|a{a;KB*!5+3_GA9Av$ONnH{h%k0-ssP@40E2Ez3SP z#|EE1I5cG26`GaP*_qWJEbd%!AK2WJ>&b1bz#YU|q_7JfE2pyZ-GD>$=PPTZ@bH@% z6$P{~QTw*Fi^ByJ5*Rvxo3A}RA|4le!ut9|MTY)?tE;+R0)k5|+_Rjs`4-wuib6CY zRrU3WBTU%?@to^ldRtwfA_;kuXUshT>QWpUHn~ZBcss<$}bJ<4TOj`1S^!N2#Pv75vWKa)N4C` zNE_S_m%igTF=h{B*tL(hx*UKT7gbZk!^bbJtn5EN-#B%rVPK#jBMSiA=sll1bCCs$ z`J~WrlTB~qzpI)&W$opJ*1Og}JCdBKI2Nr%=aH*<39b+NyK(=-xfqcuA>H~HS2oQVlVZUG#4Y_kh9lapVMrk{Hiu9u{i;)wV? z?B~=LszGiy?|p$Lf*>s|-R$*?FPTxRr`3fx*8(pnC@3*8QMt3@TY9=BaBSpcWC`E? z5xi3;`Z73~`xE^m+*_ibe#5WVD9f?l4ds=Tgwj>DhI9V9;l*F5l0XoylE3@X8~nDU zOj^m|BSl|dqTPzLw;sdmk>SC?!4FKFIC={fRyKu|mC>K8vE1(arQIfTX_PZU!8pCq z-`P^WrvV8>7-+C>LEM+d-9V`^t^RtlQY>J&kPhXtZd#yYP4PRUq3cGTy7vJFaMM1uytd8sQuy^CF#Ru z+)l3-4DtbwZARzA_LpF8M46O!F)^_|C5j|@ah`Sdn^W&=!}pYa!5(=v?++HLcKb7I zH-C3aThHVbRYtodGZ6_2{ziWf{+~Ug7`<>#A8z3gg{7|Syqrq`onp^ zm+4?YznHAtZ_xILANl^Hze1xmB`aM-p-eLK_1Ud?y0o6&Gz8)^MOrkGIt~hLx4#2W zCfTBsmazq30{L>Fdj@t8*}QJ+O3mBtt8z==I*#G5x5e7s+L2>bf8Ww`=!=g^7~_ zDr~Rxy|>lS!x98H)M|f9veeyoTt5bC(u+H(e3c~c368>V0X>|Oa=x!NE2xVIwrF_SRKz5=bLYRheDP>Q3w@e&BktM-^1woO7{WM}Gh*v)){%aUl5 zZT+$po7R0Yri=$-Gcp5;+Foh_bQ+bv(_xV{yL|W?*x6T|XHsIJT@P1>tg1Sf<6uS( zU27mSroSL&5S|SzDuc}R3I9CmK!4$JzIy`O+hGQ;dRGP@^iZzffj3jzd^^%q?Rsc= zc6R3e+;Lq}Rb{rkUwj_3J(A{myZyc>tf;iT^#*J6lq+Q{EugY3cKW5yU07CDro-)3 zih9C(^gx*}9Ld;QjmZ5O5a&T>aC*yv7Yt$o8jy=cnCP^hG;Tpq>G$8{($oa1jQOpt^CM z2&MdiYL^kymsv) z@sI_hnEe@40+nV6gc}%n-(>Emmc7jgG!aA4Hfz5=Y8!g%XUi<0oI&c-m7V>#nZgE( zhX@xCC_(~)g#JvoH8B|{*6z=-nry=cPY-65T^r)qOg?F#R>ESvXm366|G0X`=t`Qf zU3g;Kw(T9;n%K5&O^k_c+nP+A33tqiZCewc{haqbYklkc*Q-~xc2`%`zU~Vke^QPS#=t`?!gWg!B7C>L{T|_FHD$WdehV6)2!ew0?(SSIjl9XsM&gG#5e^*VADVV2T&dyl}r#A&A(^*&BTaSaW|iL*-i>}nh9bx&a4)Zcshm=F_OqA~*2 zvczNyg-J(?+TD`_i#7+ZR9%@e4P^s#Wecr~>ZJ+`9U@oxGV^OC*Kd6JEFLi#zF*bV zo~b&liuaAz6#(@FMbU+1s(`&)+?e8g&Yz1xX@B{JU5URDH8(MyH$K;o3G`?P1ra@L(d=H`ffT8`d*d2pk9z5~6IO!H-n8D??M2Fp&lBeE%dB+Nz z_?NUcDf1ak+L#)bfv*1qPNqQ*RR1uX)W&IH6pjkGlx;0?hZxyik^;6TG9fH*@{im3_H30JIylLvcm^iCymi4B$^I(K z=X$g{Hr>y&>+y6vcl6ifb(fSA1v7%>{j+QKY?Kz7+Dwrawbo~DvBlHe$Dv^&)A!lnSJ{gxpbxl!lehj!a0Kt)NfbQjaB#b2 z<+W7cyl0-&TqCB!$8PH-9p6e<)iJBFo^)Tzu$pJzxzplIZtlYP;?C1KUj$r8$-87E z6pAc$|8Z3(uFP4m`0_E}G*v+@ua#2nrgwbpk?#(){ofltmYfM{Aq+eDjE0VXt_8ln zXpdCzvuw=2l8rST%r3T+2VU7WEp6h^+H@W*xbFuo=m%q%NvpbBeZbKuVMnemvaGE9 zYi(0bG4MK+7~T76sus_M6JJK7ChS$l68=5jR{nnVWJ{MvF`Lt7zEn1a`hPZFaAf%V z{=Yk9Q;ran*IpF=UxWW?Y2-aW?ezaUasc!S7XJ541{9Fh)i>AWxo9yluE~crXfYwL zPF`|_l{DKNbYD2U2v8b|EHI$f_uKtJO>Oe!a9WsDI`fC6;<-Zs@|HPjsnZC>SF|&Z z)Mj&==8ULSpY-FhS)^|#F8>Wn&SN2{bB~ZinY{~sZbd=fEZ~VWRWp0 z!fh6jF&QbEBZVnYkS~oDFNUm=h)9WIwKKm1Vwj*vMRX-7Oa_WjcUO)XXPyEHbwx;$D*)*N4x!Gh5`P={ z4o+8~cfU7Z-s^@EaSOd5vP+o`4TcS%G18%e+fyk0hhZqi|NKggO%Sw1A}>OnlX%PY zVBAY$-F3d>MU=mLaeqmS9ZuQ@yu=M++ZLA?n2u0@4p*S#W4eLVBv)W19DG4J4KEpS zv7OX~6DvYx8&vulDS0jXDsPo9fhMA$kwV}u0 z!z%W(A;wC?%1}mxM@*FM;czpcoa#`q+K758Xvhx*J`i38yy2Y=c7WOHg`j!f4gYnf$-V=9 zU%JQC>*|3yp+2@6Yd&qby=cfe8Avtyc7s1z}3OZP=10Ud;em5^I#?;1D9g)S*-wbyItI zwJmLoI5{?q288sy-Pz)X$A=E5laZu__FxGE<$#Kum4DjxB*8%Zqn(YL|4EDhn<_Rq z*}4VcsV+;q0lg)81Q}FzHQ5xvMZTCG{TI->*X6nH0q0G-uLNvjIfeK4M`BkWXCwkM zUrvZK*1D%==(@*%tpz+8t0~!92%3UArwN8K?obJuDIA8zbD;@(T+Y~VYw}+#RgI+h z@hAkxqjNW)v-5{-HX%;iq1oOrLdVAvLDrI1<6BBy>#8H?;ZL0w)?fWeGL-Q)3}vYx z90*gg-z7JtxV{V(oCa-h-3eZ6Dr+(rB}2uoi%q-^a{V#1o(|yYTEa~jM}b9TpoTV2 zj*pCPEywyX;l;t+8;|1oYuD381}J6ubX@MV(SH!)E3~1S04z(48jcz^8#1Mip=T{7 z8(_nb^eW0Lbnqo3P5+V9Qh&G+7}woWjBfDT+5suKk&@Xnf@*klNk2_ulXOweNvqS)A^^*?Kt~i2V7}h6oy=;0Q2n(BQ&l z0fk);7)Rd_NUyAO11=p9;{t~K*2TSZX_*{M6sq=IZHNezioW9xf=`u|zz1hV&8>Zp zvzxTo@H8>f47x!>I~CLD=jMu->I`0QAeh<0BB#!S)TbUsM1dJYxeVy^4asN!Z}y8%sICTyu5CPt7E5U1=d4pfct-1&$XEPWpXm(> zRxtDeD;+LpxDJIJ+~xW}dZ-J$-$AmTwnQ6#h5x>^dKNDtLAr9`(Kki1yg&-*+0L5u zCY)&tnH#}CXQLl_eI8UC^eA_)>DcW42K*vo{;x%8r2##3klNJg43Tgx(?t!!<4Z{E z(N^Y3DXXE$VkE(WdMnK|BAVBth=0Y^32UXtE~Y9~!}IiVGQEc_6i)R-!~rmi&5c7Y zV$-UU%?TYVSfrHmNk2r;kX;~rk#j~?iX=I&UossuutOsbcBklyL~JY;B9Q*BLm?96 z$q(j#wDRJANFnxDZip3chwJ+<(cWSBjor!(XmE2J8BIY3FbIG4YhwFfGvs;|Cim4f zh(CJvz11XQ+`ivYQwb}~R4kebjLU6V3*JUW>UfWk{o80`cgaiZW5&fp z1v#b5V3GH%`82^+%FgGMJuhm7y|d~_xNoL1cYOtY{^0B!6zBE2atqmspg&@>B}ad; zTgOFi6DBRvK|2%TF4eMGqwie9KLB6E^;RGhhS%lcW(*)OVePRCN3OzcFPOeSONtMco)m_oe8u69nfarGA``DUy#BsfGgZe*)I?TB--m)f z9VM@*RU2|Xb+Unu*d3B(|7xkZ8@5GKYEqLRy45v_DQ#bA3Ai$6uj(Y;rVpBRFiv)V zCLTk{SHpr5No%+Kj zzo3a2dLF1O{C^ue0O0vuMG8JP02ewnHglRT^d9cwHS-Rd_B=^a$%GECIj-(}S&aoA zPbFPe@enM;1qDJ1iMsV3W2h3L<094s^W@RI%RU6eqQjZl&kc5Xf66f;E413x^gkKk zH6b9kiJ!4*YN={GcBYNB;a`zM21RJdM)MEQ5O6oL81GEbnck(MVOjt#)?eB9?Q}(0 zjNey}6Emzy*Wu={f2C9euY_4Jw=j1mnn3t2EFA#39o&tCBtIfl64NA=q(bu68OAQS zVh!Iuj+96CC#9t(22BuVFQC5YRLk2^r!m+`)0C&Jq^dBNMpoY3TJ2o{0t@ zOg?4@7eVt%+>$mTRTs;`3?I&!qIqRj6rBntYw>q|?jJkl883&SEXFu##q01;!yhLa zxxp-bkaa#JemR8Dhy-EQD^uln0hH=E)dC~sYkj5*Ns&w9B8;hsEM5wc#AfBLo;qYx zZ`@5LdRXyiCCj5#h}T9EwI(z<+}CAOsHuqALmgIUt>#wqYk$Kq#}-UEY*k&yvvR%8 ztmR@*c2pqC|DG(=B<3}|r@k3hms+_{x1E)>8wtY;;9)*a7QKk@K1!icK;z7+H)Sb5 zwH3CfDr=qnw~oL6qM(@>YuGo|;hs+iv@Hw+N7~H&tW*CeWFEw2PA&KO3uVm*dn0+^ zRk+T0)r;3v{o!~gv_xsq?j4*UGRl#^pGH}27XBH&-IaAAf+0Eh>SFw8$M|*g4a$s~ zV`Fo4#34=Q_gf`~CKx3m%pmjx*u>SInH(czHvRWfMTJ*huYL!8&U#pr;VOP>mrtJF z!7M?FzeDgHC(Wm=wFV)%pGb~)DA)e&#LKfU8v%p3+}$lP;~(pl)>B1{!jsZ=o@71w zX1VY1jYt<^(fismUFQjAYp`{jjgsi8N}Z_b_R*kE;bA#XPQ6%+WTf))&+E!Ur|evWCy5xvii}PfWp1Fdw&K1YTL`uMaxDr^`G5QG_COnaY`CXU_9V@fEo4d`ZsgJ zFlZ|0@nsdf@4#0I{&2csoW_HwS%P=$umVfMB55p9G}+qRHj~+b5L$6bBwBw~<#a~% z(9={U3QiC8tExts#dYJbd6&WWsx?s#jh4T*{p5vZ2;N&EbqE9Xi8fR*C`r*IY?bEr zo7V<5J9VLfHXDF*q6*JP)H(29rM-Hl&uV!4Q?Ii7yo0}wdoC4;Be~xs*M`sI_a#O> z#luer!YJ%no|f=Of@c#xw5QAZLN;1FRu_0GPpTT8i;<{Q8S;sBe(}4)Rs` zEAPlg+YThYrN6G<8tJ_xsPamilh(Z11up)Rf91HpxW^SNMG1YyoNaf)QDmE4 zG#i2dpaauo;O2QpfY3#hdA<}T3wHDRjy;~xhQNMgjd@+ZpMFk#Vb+I%cfgsWi>)a} z;n|eypi6yEhkrjlg3)^)N*WTIE-BgwR|c1+9v|@LB13Do%EYc`pju|gc!)`ZCf#3c zz@9FgMDj;rBT`~h&JMKXF2eE%8w}QTVAfre`9xs}Cx#ZyNGswQ4^q+yaD5)VI_yiO z$jxv;%Vgv0POdOlDR#5FZyOyH?GNw$_5prGgV-Bmi~FuNYVZJuJ3uy1cYpRcfSCzT z$fzp-sjF@a@p<_+VXs^@3%Qb1)uh1+J@f(yPb9iIhs1=0`2b0M;NKb0!Kx4L5Et&reLC>GGY0%-nRfyPA@F)ut`Y zQFwYP42G&Y7D2Wau2Yv@F1loyP!``MrbBNZ%`c=u_~`7_0oqmueM4c({9)r8;aksbxLH^bgT7H5%yHaV zL6_7(`$4B6r6h=wguF?cIp=quf!+!|FKAbkH=b^8BnvVW(Oyg?}BF)cPhV4HaLn1 zB2-<`;sib8=nLvv1({L1;TVoJ`irj6@4d^@EOfe1y_L}0eEkQz6!2`GA84VCwnuOE z(WD&5lP+6OT?2?;1U15sbe9Bv=e0EsVdC#w=8M`dlDu4%5R8ti(+b^V>OMQ|tb)>q zaQwV?ig;rgw58Paa3{bt(uP7r86_n%zz37OyHOM5H-uQK>G?OkWIdG8-Q9be3llrU z4TkpaDpQEz+C9eZhPec~7sY;I{CiWfJY$W^xa!EE`o)L0H8;r#Ui-Sm{6h|+e;FV?R>CqZ~w`8um&uJN1#TksVY34-C}<(b*Sf9xkG=dd#* zELn7T2qj@KmOVB3rZ*Q99Z~6WC^RIIvyTBKId#3MQwEJ`%#X?<=cxmb@BjV?XIFZK zbke(W6u{b+ZcHjFFck*yf)~~hB&Y(J@jgu)$d5c&;c2B z((`**h?yi@I7VS6=kFnBAAzwH$)Z9&G+i-d$IWo+c!x|VaOpK~L7jIeS>gaJR={hj z=(je!eXYef`*du!q^T;73N-zDoWPq%l$)a$Isp&HDJ%k?&nD?rt@hA@O)tR|zpt!_ za+y1&H1Jzzg0?#2NPIXP9(z6^Q3O&25o#IcAcGwZH%TH82>o z+OISI!3c0;e?x|Zwa}&&gJMu(wWnCKsO=J@g@;vVq!9gDIBvPAp1`}=<1Hr}GTSv+$ze9~uwv{Oj3TPi z9hZ*pOfXeGbGlDA8~X`~Rs4toZp20gC%YLag32wEA{8lv%#ib~bH0`fZ-4#W7kPbW z)HR=%SMkKBHKV?;bwov(HIywT0rShQWt9{M--V45q9VZ_LI|4kRIzc|F zDL84^Fbfka99EN0+896BTIP9K8BBHlq4Ylc;?CPyCWm$Qn+IHAtQzD@(7uo%3d&BG z@C#T=`QxFMll{SzHm?eAR5QtR(bp`S0^Zer+S~Q+DpTNRg*{>xGZPkeIC)7cG}5ch zRjJy6OzCWtX2$)95wW?!F1S>B-dHuxm6h9S+`H>8YYaQi0y>U$Ua@sJdE*{8Z)o$s zxo%>_S;9T(GOR>cjbY@l_G=a}q3G5EbcEn6R$S+iVa!Fw!*mj1@uQ*e_k)~cj@P>9 z)pl22E~RflW^mhrWlE0jke#?F1B}h;^^oM&4 zj$(FpCU))Nf+S8y(UG46c%N+gIX!X0@2`@U0dK!^IhqkaLh;Qbl#ZimF@uTNZ)f!X z{5WS3`xz-ph{nh0;A~}VACPaL+#BO?GWJ{}6B`yOdz`0q^AXv8{A`EIb^W#xFsJjN z{Ryz!HW=t!;-)r7HkWGvQ^03KGn=s6@Y=qA=p#$$=jg5k0MI$1$dgdhjruQ_oJ)u5 z1|pU}MM$M3uetJHcmm_TKPp?>A*l48@)G-;ed}!sD)YYox$&hB;U)M$OYz#})7L-H zko5E)(Cs#VL*f+(wFA=18zM_m$d9GXG2`^ATDPMkXEj`u|C$o1WNz+GeqFOQ)b6BLKk)mJ?#R=WTsx>BssJ;R$JE?1scuP*9i@MO!EB zB|!29WOSH`q)PHET-k-crDi??p=6NroGLJ}q)zYV?gId6=wrF)SwTR6y}*_*x|4^q zK;V0}sDoP6B=NAht%|x02J&xXE6k*lwt4*ESOqNSWZ61<7+Kd0oysx(Kk@D&v{4d> z3rf9Qz*c7;k6P{_(;025uX+jz}@ zQZ%`+BiiDvs26V??O%XUt}_MMX-Opjg`d{t0?=P58R$aXRBm~ZpY<>7Pcs!d%tDg2 z!Kk(=1t*HedB)98j|LU*5u^=1C*(sE0VPF!{tHTxZ*8fV?QiV#ZnRnKnce)J5<%h4 z6pxy&XIitAOr=*N$KfL%itTrDV=8*0BBj6JxcYDth{$o5$DZsirL%Wdki~3*f-c$9 z!E+@L%MR4Bw^(61xW_Lh#&HFUV^{Dns&rHa=R}9N+maOxr%~bW{WCRjlGs#>ZHdu6 zdJG9ll3RL%M4iB>6jnp)(9o0CPhmG4^;R7lsLnM#Z`GDO~vG~ zq<(F6!06w%CUHOKymDo)#dtipkUMiI!eg^9hqq}UGR(*m;14p!6x;0)maDOg=S4j_ z#@w_1E=P{i(@`x{ii@H;IEWo!C&`8_wac{`Ndk~Z%@5+OztXC!i(6;N5@i;lf(^Rw zuX4KY^Q8)!N^sfRgRE@5mFj? zt8-T}Fqywq92782Yce91-*dAH>1P$P zN=PNnHtC7>0l$`uY?GZ?!)E;tj>XL!z71bf^T-3E)-TN-Pofwnk{?E&Sv}051#`{p z@8H0Ch92zakVt*vZONC5`$Sv zU<(>$wTn{(_ym&@W#k7JoDhA~sLAoy0h9ZYrT6FKl4p9WZyC!?=$G8;Yk+y_dSGEM znV6w0nN$D~ny>1b6S|sHzPOO=t&xTei;5&3p^4Y z3fg!9xq<1!-)4X7%D5&4nwjxS;~qX?z^%^9=&guZ!-JApbZ`#uWYy@jyCB({RTM+_ z(A04Eg3=|nWf5nPTH2e#n^t`uX1N{)R3i}aByMJuTZF$YL?BXmi8JlLTB&uk$7H(i z;@l;MQl?adUA6bGGh`<&@|BZ;^x1S&?doH~Ub?Vkvz(%8g)xoCgi3h2lORu00Yjpx z?F6`#pWZ#X#}BtXAWCFL<2j-$0)q1@ec52dJO_WQ`$g$Owe`uG%U*b?MEizzL2cog z1`G41*f?`8p#~7nUxAYvH9MZzwzLyYiGU;hH&6G^z5C$rr-JV2p{_=S@%y>Bg>9imi~@hQQE@h9%G8qy7Qqhtc~saw_=7))P! zcbjo2@WGgtt@L&*B>oLC4}%Z{se}sW;hdfO5?ES%_GlBRnWmm&yx0@8{lOSbX!8TL zOe!+AOce{s3qPP^Mw7_0!AT4yu^Zqp1f+iBhg^A`q|`H7Mgt+V07r=2AYciZJp)cH#^>_K z!0G<$-20wc81xM3Z&nO>b9Pe+ypCSMm9|iIlsL2XW@oGP@A~wWTW&vsF=+N$@NI0P zi)?gcBl}8N(2L+EUs)A%)b-ccxR)@YR9(wo1z9 zwtF)FjM5?Aao$#hnoN%|6h_iJKVAS6d&@Whn)9bva_lyVoCg_WDa=&rn7DTP4U-r=j@(DV(!t8H78a|WA1i0wRiM{!*h zjR_{WP&CsvT+Nh{1-8}7IB(S*H(t+$3p9Rl=KEU{U%duE>f>;9Fs2{{=PTf=NY!o( zoSNs9rba!vFs=imCicZ-2-a0@{t3VW2h!=8GtP0BDJeUdE9+uze*cDe@ZiTas3(H- zbW6Rflu(0AS2&YtkZ!!*$B(HO%oAd%1@imyQH?NH-%sy#m6Z6~_4kGt7F_h>|8W860fm)@ z0=@|APeWNgXWIT>{*fje<%L&)7~2o>S)~p;uYLs<(_cap>Q1o%i!Reb7J$zTO;v7Z z#t(#N6P-L)Bbn$_IX_osxB{|`pojUqo=(k$n?X) zjt%LT%m1)gH}?Nlj7gMnu4X?RSU8I+fl;DZq)ae$$y8|FeE#pR9Ejb!mY>f_Iu{?t zXhG{XvnzYfpLNf!7KYpmpHHwbcF#@kp#l0nAJ2HYdI>6CKN_(PLw_4u%w{wkqoBN1 zX}`c<+t>B)KNF?Vj(`aa?A3YU>y#7sO&8hFl(eV_%(jOPH`MN}g$o}m)PzVX_bSV) z{SYl_IPzL>ZF*)pOo}55x$;^8_<(JHtoF2qL1MwwG=jpkwwdSU)Z`)KHT(_v3VodE zwVJ<*evN^w9uetbohJ{*>WULj<8QARex7tVgY|uWNZR|o;Tf(k%jK{o>Ue!+_jcfu z2z|k>>)DRIN>gP7eMH70LfWkVE=UJdR)br{mWyDpiT4*Ek@Vc5Tksf6I1%T_8-EZ8 zJk5!6{ywe@BT7Dy>hD_5(doGVw?E)a*-4J-D7+Nx(cRiAtQD~TTOWUGY$2gfrtnu< zQ>f+reI&x{`R5qP_NIKD7q1`n%rQ`yKye7ZrKTKA9Z6D*aYdnpAw9Lie&!tyakiB5 zd&2%VkamM&1j6HXUkO)Qj*)COOfG_Ex|#=P&T3aI@a4^IazRl$fQZ7Q>F~*>HM5@DXqQ~!~Okos>F(NKLF1EjK=ItPKobMYl z(^i1Rz%McRM!dRbg}}QT!tN--e6Wgw%p!JxFkTfdUkzZ1n|u}aTNtL-$_iyI>#nE?hU7{ZK<4%N$M>9 zY>0t{9ZkWNf1QgTyT9faBtjfO<}QD1@TTq#Ns?xL_knzS4_cyFXtp32(~z*sJ=z|H zLOtwT1_9d2@ZK!yeBzKI{pjPMI{og!F_lp#nWJteBiyuYjX$hpWuX0*6%;2P<{}%SOaNf>}=iW+6D4N|cA<)$UF8_5SJ4#Q7~v%)B3;ctjh=Pz)4~ za!Dp&5u^29gk&%ZjOgXr7^-B7P1eRW#Jz(*B zYj7xnIz+<{Zb6o)-SAJk&QCsKKka6xMZNBSDpd6fU$-glWiNeX zqV)9S=%&ZpJWX!^FTv6j!j|glohWmpm7^on=o3Rj|5riB-p>kwVw#nfBa5xA=1{b2 zhr@^lfyobz^{Q2aIK0ilpDgtU^-#hGpE_wnXV0qZ_C-~Z(0Qof6n`a3h{YGx2`S{% z#4(`xGF8CiE`xXmAi<|H??z=3OKnu#!(|LoVX%jjE$?t+2AkGAT9W_1-L^^T3c|+` z7@FROt#_RBBp3rBx*w0NR&;bGtH)VD+TOsK&ktyRl!(GB3T!5l;m2+NM7+mE$R|gd z#P73f6l1Q?iu>{zapEmh!FAVnz@YSn{RkCg zLPCPEl{^GjKX+TuKYW6S&2eRtj)L(*^XvK#nW@Foh)9u9+~5dm1;{q4Q{e#Oj9BIo z?vjfSpj)>A@o}y5kMgMSYIrfl$EeKEa*@WWcnST$r_RCuTk^`1k(%-^|GHf}Z$V$Et6Q_1nE7>P2Tn38id?VGvoip<^_pXJWT_*BQ6WOx&gb;u9h3) z25M&SdtN_bSD8yO+x6+&OsK^P!$VD0DjjiR%uIO44kXeYN^D;%=xtUtw)q}a2tO&( zE%soEGSW#g@rE}0db1s#zalJ<*xV^&Cc?>?6y%Ff@#8H_<%8vwJGJMV8i}A3zf$ed zs%jH~DIQh~^pNx1Y!0*TMEu0%z*fC~udRGnAE(<%czkVCvj+6(CgkcwSOkO3Djr(w zsoH$nZ7GZn^M(tHtyJ0N#ySPe%3uaz$;2Od;z1di)k=_sqqcn8bLVI0+eA(H<>XEq^ZTs ze`ojCRSAGJ(eTwG*v9Ori& z=UblPkGr`LqD--(j$gj}iSYvJ@A|e2wpV~1;y})mcoGj{@$9=9I7EmILH2dzK;?$Z zZ5&prHVg5imf1EzP7br}odzc{^6-JS&B^l{;Xb6nhWV3!xu^dOx3x4fzG~lGBK)LB z>ditqXT#;!RB$6A?ACi{G?C+zakKw4F{N6X%AoM??+YwHwOj~lV)y{sA}voot=i6C zFR`=ku_=s;?I*q_XCdSBl0;tv>-XQ3&zth7;J!)}ZK1OE*c zd&LwK`3p%6o~ZPo7cc!bSw&wM5Z*Nnkv0=y$=DRXoal@8e^$&QV&n=alDilVzcW zE=*Wr9`)*Gn{e!qpD?|61qx~&-)$&eFef-9`$vqt^-O=g!DBDd@@QKFH#td2t&Gqp`;rF#kcypNQM_lLUPv|51SG+NI*_ z%-0^zvLf#4aG_JE&e9luQRivHgu*YN4Hjt-73P5F_1zy^kfr*PHv-a9M6Gen(5tpkNWRSbFnvP_(Oab4kR5T%T=$Qt$8 z`2nG)2y-40ZC-G1jI#ieC}@LFGiw`UV_`m)e29^7r*)IftB!o4to^H{r8jA*RgpeP z#D9;yj|7|SyJvEndpXS`t2k_iw;sRP`D)igQ_AVNt+0Z}G68tdBYGD5;$Xqxx{#kZ ztjw^GC?7UB4^{WAo2RhbgDk2JtUkl;degi4iVH}9`G2pw5Q@&Xn+rSM zZ0+O#EQw34LCu2B`p8oX@C`BgMm8t($6b_+QWaic`EkOTtLL852)3_pSkT&!W;j2& zDB;In&agYaLv+=N+>0_elfJxXE3KoGTEhw}?l=fj)MpU2QzKl*3UhUhhK?JQ0A|5QX(KdAz!!loI?<1c+?4K_ zY{PE`?8v-BecTE#CJHqhwsB0@r;T$~8nG0FQr**G(@l=CcKmy7B!tQS-vRSQu(jr9^k>)<&eUCN)Pt|H_zbKn0)q)kK7nl*L_{I&O{x*=a(0+fpj_nnf^k zPjX&v8U`H{QA3T@Wc^9l1{luu8N|UeB9mP&oB+rhdZ+Vf*-M8h2>HwX>YB`|>X9q7 zqNb&v8TROz?upFL7~7J88-FBW5L{aTQxZ6Kg$MbWIrVFDDys56an=i45c=Lz8+uy` zo$Ha7l5o0wY&k9WyXuGieP5&+A9U-PHs~YDI*A99&OavOL`sV{dB~YOq+qSXdwT1y^+}D z-Nwpoubr&aZp&)=UCTyh1wbBIh4YCs=kRqDei#Qi8A|Z&+M(s`76C8ka@M7F8+SV2 zgR2BEXBcgRgpK$#3F$h10W z^0G-MTx^WgOi3sug$8NRL3@zHt-vZrV=@}YaA3#;t3A#xi|1LVt<^5<;FG2{h%{Ib zX5ws}rO^K+fM1hVdDlB+uSo6RMio%Sw)J-U-c6^`lSc~hBO?|KmoKhn1h<>}*#iF& zwUf^~rL6@tsQuTM3Pd2i>q&Lh<#YmnJS};;7ZrhvhSifv9{KAMY9DEJfedl3{qM$T zKq@akWtNM~GdijWlYeo;B&IuyFg_;PM-9eU^I!B`+ntnnVc(Y9lKR2v>E+?-=&0)3G%ub@dNJ<_=K#qB%D`$s^1y9Dn8v%9s z;-%o;mG~W5^$fEnnvY1;m|q1rV7Tw+_dJz)b~rcU5VV&|VeyN#h!Ec4CQM0w zJIal?V0H{+rD92`)FWAE&iTv5+$DO3gdL*6hO==w+v6~yl9L*Y)|uaxvT@ey$MV?Y zFGR|x_jVWsh}e{@)yTCt~<5W#4#4`mgDY?_mc-&tu2 ztukzj&5gEV|K-l(#JoO5SMUa&VI+_;kiG2VqCo*x;&tgM$7pWJ?rQYA$6?9CUV^nC zW?YrLQ@RtS4;2|o#Nu6SWvERtLNRmlTT@=yFEq$iL#Vumv42{a-IsK{A6eC^g@q0s5D?6&hwn7`^Q44Q?H3jz_> zw~hY9hmkAdY55;cOyU>J3{)7IR}c^U!!!>HM}?)6tyl;M)O=ODQ|x>F=Y?FLYWnG-Ux;^a{zK&s%o+fB}fksP+;%-w(Y&yxM;;$jQ8q6i#_&M^g)Hi?a5a5YHS`YP@>L3E(MQ1? zw-x7Q4sa~i0xhP{I)ndET*{2(2O??Zvw_oS&g5xho?^n0C$p&o&c^nOL*9;BBGlFSwlhWoF7I2SXDKuJ;xpIAJV$*!O`;vgebvGkx& zvjqMat!2g#dNTMbzqpu@zyBd4e6@m885=<*@5}oXs{vhnev{brmUwoy*OP}Dvv+Pf3sLJ zc&kJ~=ZEX;W`G=wwRd(fI{0!MtcH^L6HFR$txXk?sI&FTo0mx93jja2YzLt8plHQy6P6rypMjRnp7z`f5N_Gr;dy z0+B##Iy=KR{)FrhGyQ;qC-}ltP=wK2W_+E<$%*Q-+uycu6<1EJZFee)JHhJhfRLbC zx;^=piiKt$2;JfD$p~KwUp^{#zw~By(xk5s!>Q1OzFJsJg+E|EU#=^(OLVK0koGF! zj4YfjkP~{}mD|PvUFvugBS9IW7GQ?;ZDh0e?Ks|)di&aKVxwCL>@ygU#bn^+5#M84 zDFD7BT>8BCGTle`AQv~PhyqVMX{fdqCLPf<{W0aMzz`Rl?a7-FoR`H=7WShf{`N{| zuIKk%pCiGxwDfE~`Z_TG$5gT~)0x|67%ere?NM_NOlf5n>L_`+2mQrG6{@)ndyo44 zGYw$2sNvA|;u$YD`?XAc`wRX^>YpXanR_`!G8`k;@XZ>YB;i&Pg8a^mNIU48aE5Z$ z>tKGLUjgi1frsGf!mr<0gzssu%X4g|H^(s&P%q4#r#p6I=umzgt!x!(zfu z=T)khgj{;OT|Aq?c6$zQgcBp#NUmWKdL4zUFMfuCEDF$r31#O}&quN1-e0x68u|Q@ zpFf$FriyXd8E#wn7z!Q2{ zh7eGR+xbRjIpEYag3X=M0Bro!coVx~lQ z>3$aFx>)llIYC}4Df}BS{GC-_n|cgmmLm80_%%_^dvY7D?uSE%LZ%ByICXSC>@?ZW z1s>bE<4N9xwehjQZ=HREb=CEx0A25kw7gzoMhDqxgTDPM*drR2V-htRa#?OJ9~}|b z74oHqF(Mvc!P)@TfVSHatn?cpX6QU!1TQnPptu4QE^Hfgi2V|m^MZBKT&~+Rt>NpK zmbbq+dXF)s(N_a3K|x^V^?ONrPS7L6bUu47mlA%~*dziS2Slhy@hm(vN5Cs3zSc*| znR%vhy_nBtsT%S)qlMR%vHj^2Szd%Q{;1Pxe3bxte!pxBbD{UWQOE8U51cfWMMW&t z+2Vc5*9Y=qS;o!zb>7q{7Q$1r4jmWdr=yWxc$WJ(U25raOMQ9xE?HdNCG%XAlplyb z+M4;}H*x~$W+)7fY&5P!8tneA$Hd*$YR|@nLq$X?No|JWUacC#zhf`qnV*W#qFu2B z2zRS5BiAom*BQ>AC_#fh74fFM`=p_G0r1h(l1DH4lQDC-LLRs|+dfg7a5zp2zVym) zs23Fcm%@>M?)4`{u&8`Aa36%e$;u$n=uE`7`3e*zxkDB3?vq1v3-nJJ`G9dqt9rk4 z9oTlgOf_5^`xs+Ce|Te?WpsztofMft0vr>YXz~Y09SDN-dx$uaf6A2t&f0YlRdQYi ztFZ@mH+sB4a^COE>9MYp;T3KVQLvtC6Oe!WIBNx>i)eIiC-rJKW@BavCpT_$1>I=x zgPa-Ms)=HYOL#wcZcH7>rb}!6u)Gk+K9Gr4AXv0)bs0K6HF0IeB1SgNu zNyRI2r?prFQrIkrmAAhj&VJGQjIXb=$OPxP(&2HYu217j1|RQ4Ico0Gg_$JQu8)e( z)R)b%JKJZtaD4tux;<_`)KlI)0h7vq(o?)B zBLDBdU9&l_zzqdN7X_&df7aVa!@&RH=^LXni?(Gu>Daby+qP}nHafPy*iJf5$MzT7 zNyqHiPF~JE_uVnp|2@WDbFV#DRn02rhR24qR$4;e>zE%a8VBjJ3<6U|iAp7lTZgSw zN+=@K3C*;8HmESx5Y&tSJ1VJN zgfa3ZXDz!vINZbPST(lVx|Gy)P=9>0#khI3da0$Yzg6Ax8*>qwCW};hir;=0X~%m$ zp9Z>vY)SK9TMD`W0&f)5@$Bcp-0fgy)=aN;>}vm${PTMkrqu;n%4nI|VZGYBUh<;>xwXP7dk#H-8wBoL$ zB8nPtTBSCAWz6B$Cn^oU5Owpf_sDtb!Epbwp*i%Y{bEb22`oJ7QVpT<4t0X@DHo+q zv=^f`EjSo@XlXyj-|ylE4f}uwa@PBFN_txZ^jqRrnlhWGk{kJA{?)+4(N7Aaa@3yH z;j7eULvPuyr|?i3dgH1F)3#ZDx-{i5dHv-f2Zx&JN4Pij91}x>_DNO1+R;FDs2X&m zGb57G&NI2_#GRF;i7Gi0g7R?CmgTFwkZ)#xPFhVEYW z+DW}$S_O@C8&O2Lq2%|X;;o0e&85Ln(%FQ^50OSjnJ^p@b}-R!D0r=kG{|rGBGc4} zJ?bn(LxjE6Bg=52Y*!kL|A>2A|Dw;f;VS(N(>*q*USNxpmp)hC7^zL32u?S@%eCUrw}^ZtwB;e zy?Ydsa17#wDX3WbWtS!_5q_yMf(3vZt<=OQOp7T2!M74h`1S6(u#dMntj5tpX7MBTHYru`# zYVMEjY>=ZgKeE$mMyRWaIJaUK@mlOFBruNlIhGp-PbIy% zLhax9`&FJ*%S+Kp_Lw2^-OyL^4Ei_U0>t!BupByf>AE-r5;Q%))Jw_L)KlxdJ#oYd z)!Fk|LXo~lSM4_lTdDpB;3J=)Opf>YQ^jccXBsB>Sl7t(n^k&&7;CE%q` zRM}az&O*g84TmRV|5DF#{R!w~D3!z$H+T6zNorv~kpA`I~Y3r4$~PSxVBp{h!sWP~Zi)ryn<6%P20 zNDU2^h8s<|GpUi+3h$@;c7j2A5=1o}Kn_8ckC9?cR8pFo%`h`k{#+`2?2%H zQcOj_lvlV55%6+OJA^C4b@8}>mGOnOfdg)l9v~LBeYTONL1%wI`f!pdjUI5c5=Fh2 zXZU#TgVDKRG#*LJr8regja1lc%5*(S%oRsJN*r>6*|@rvB^D#WWHQy%VGaPKC984d zsbGIlU!(a5m0Rz-`W=r1BIEeseOrn%Z*bU+Kxbs0`*PC53KtvKYg5nv4SAS6V?krH z3V@K4k?!!PRG4OiKrhwqVGq}yC)9QWBaPXanZgN)cqIMsfOQc6Rc}0(A>-l}fF8Dc zF{F}QY-upvu=~`0*83&YJYL?_cWxc~Lxdz(W1!(;b>Lre`2|fbg2K;^TOp}Ux2z*T z|1;?;a^Nn$S)TOb9Tl$$ThcP*nzlf0KRCi&)_exHv6w9Bln(k~sdy9FerR|=IPDDH z^n&0QhhlGUwvcwHzDMMzx;R;y6vP8#k-IxR^Z&*bcnD zG7L-yZrPU9Qzgt6ptjvtC-M2taC+NV!w#DdQ!HdX<(G0hcS83%TBuf3Bjt-9Z z?7ssGiX2YEzo3#Q36t!w$29nCfSXjj)~g{=vM5f4vos5uDyF9pjZjPUDg>{ilF}+$ zOO<2g?oEE5wK1{Ig+4Zz=DI=f+`6%}F(>{QcadhRHoEk!Q|6{0XYBC;HSh0LN=XE3 z;>R2Yfd2h;l*_Ai{pu4;ap{FB~9%{-Pf}A_sDrKlDJKAx^yg7=@1K! z&xOjt6c!Gr?^pd)uTDy+2*~*|1qz9jwKg-Y3ABr(!ttK<@cTu?_TKi{KH!m1q@y{h z@#fg&0zeXufIYNWX{!CYc-Dj}jtS{~xN1Be6kzp#*e&1ppS%jqD!icE;3iwrUc}!X z17)YQkQ=^(A~A2G^Df`)-Cgsp6LQA%Z0}`q4M1cJx3gaNg+-HK_N0s0;IU6L&Y3^L zo&P0JY=Z(v&r+cfGf77iDES-h7dnuEU~uy0rR(E~oGev3qo;#}Jv1`K;DzaN1gd*R z3H?`GdTbC=QWiC9@Y3SMvvaR3_7-D5sVAdp4Kmqh z=H1JdVwDsIp?F^aG&%1$K6?ouqfFxOZ%l38Lz~Nh9f3_L0$vRM!GF0tNx#-*BV>XK z`{O-^yQ}m;{muS{{+I~LID2Zd>Dc)=COBTjt=sv8RJ6QH*3@q6#O`#XF?{~yKR~>g zi9QbaPU*?kZeti}U-3`qFK=RL)xb*ACe_aiCIDzRAOGtfb$ zq(mDzOOsHgVgsuR;kILb@;}N3pa3EgL2}X#Ctse>7{0E2GkdD?F!TL0TP(6oXS%Zt zbF>C#cPIk7t;pejLc0Dtk9pbEFNTUG2<|=yhY6qA%ieB-X*l=px$g}m5p2kL`@G7x zbG%`4Z$B1^RG~61Ag`jX+L-gKD?x(d3@dwwZWB&2>1TDt^1X=Td)^_p(|Kl_SprH! zMmR0B21aYM=X9(Pm<^V>cj@{=UM0f7trK1z zE|nYZb&d-0~LGVh}F zJfy~~epm=>_XJY(>%+D^CEQg$3CWQD$?xz~D}xivTpb#@U=KL8Lj)2DQSoucok549 zfw3h#9eu<^aq9Iu67pr&$mW|)zmh@w|M=7UXlkKJAbl4s_{U-2MusDQx;p#POiVr* zSv=TX0pxi5m7W1V{XqL}WAZ(@UpL_+Ea$Fw8hlWPnym*2ZkW{dd19E+>O|`^w8?s- zP0c4SsmOLW@)2le?LdgS7bfI|aWMrQda6jG$iZN~WeapYMZDaP4Zq;y{E}bK(jPH! z`}ve(Fm2QjZb+vs>U&Ei;Jar2e*Lx|g~y6yD~wBCJhPu|*c*G$Idj+SD=6#r#QT2p zpkXof4#({!=0alUb*jAFBo6I4q2Wsa7Mx69wSBzo=t%PW&3;dRN}D4qvc=MUgN2Ch-J3vooL6eSm_7k zSMdFzV01m#4NtCOBmUj&D+>?ftd~HqM8m*~Dz7ZrdJ~%M=XJHuhjL5nNk&S@^48^t z|1y8y;5=fOm(Mhw`ysk!Vt#(YS%*G_&+7ueyFMivjPX@*ypN$~mEb50>deNI3^wrx zT-g`r$Jlr5t_*|*MoNXgxZkt5pL6uVZuz!YJ3%wMyB16+a%oW`+(P_ZmRwRSc>^9~ zi7*>wEV?cE#p__@Eh)r>y**dn>dYiEh}X`;^3@u6rMq%?M2NKaR}fl+YPBxG@#aRI zsNpl7qod<+)Ky6>H8fO_xH$k5?At^Vs6ZY`zR;F80CkA8Os8XN4LB~6lsz?6j%?DY ztXvbGUtoT0SXfYfBk&>zv(mqG#m8l>%L)3fDRoB*mO~Q@jfbj6j_K#9RrRMP}$k#5-FINkRJJb7O_@=t7~mcmG$ z0^qU4qOeW%MMG)pa-~0ly{ju+>$7auuv>rRb|u3^7Y9E+a*b($N@5+LAE81=FuTLg z*Ql*5rZjb??s}#uoXdJOSeB&xzylJiX$}7x7ctq;G>PlU$)+TGxy$jKOPzYW;C%-% zlt4&TFFnzSZ4pZrn8mQiREhBve*clpdNsVpA$q zX;us-yX=+6gM$*OE#V%wF%zTE)>`&)c6A$Jr1EZa zPtHdj4wjEy5uZ1{MGtc6QW5cSn=TOWRKy0|QL{Ig%(h0jclm~qVUqS!3V4b{qU~zx z7@{l^bl8;E?V+wQqtFph;Ba6Voi_ihq`5zbSt?PXvrE?3r3s5Zw!f^ugQVn(Yij7T z*QWlVdO)El%L)F6#FrXLH5m#m9;u;w$ z#|Awf)nISRDr04(@@gv;`onzT+w5W1?d!`Uc;qiZRxM#){nVrvQB7;sssR`T5)K51 zJFcd3d1n)lc8r>;Pnx6N=0$=8yggtxjATn|Fbf|>J|)+eEevTjN+Sz!{Z)tC)&Q|w z^BcU<7y=Y4Yu$3EIO^xDb9!Btu}SNN(bO$ZGClN2ZkD8QLC`<;32VeO6olsv6o=$2 zdE1*%vGjEU{C&57S}CBkTcaK54+W=SpdfC|U+NT)G>kt)OrRq0vsaX^hOD6#zjg2l zSw}b;bdDswOwom>1!y(AEwd+kmS61f!pW=46nuv-zG|fkOh%Wa*AJy<&3LVtr)UMZ z5S3*Y)tl-e>2_t?C3rB!);243WMzj)2pLwOfCYtzSBl;Q9T_(HL_4eU!f8?JFaG79 z!k}fsx*rrct+bOyNTSg*_PfoXxZTlQJUDV2JZYaPiM=~N;a~kVC?hPwh=WQ-XW?RBOc%3}Zu8-25)N3aky-uC4PgtlnwO(E}X_qMU zEn00a9$Yam!-0ah_5w3&&#>O+8dj#BcrJPjrJBjGd}Rp0`zd~}R2Q&oiEtGZTNljN z*cq%q;d{`5M?Kb@U!%!*Q<3}o+YHRe&}Ns3;Z@qcb(CcM4}<@$?YJ%Es{R&_M9bv^ zT;@;0La=m7R-4)(AZujNh&obl-A^Ac8haaCzX)-Xz(uI4o2Gw}LJrnmEVhs(Wo^oS zeSh;Pvru9yKT2g2wun=7pHX%7Cn!|7Gopo(Yc@P~#=A*tYFVxnfe~s3?@@!GkTF!E z5gJI_%2U%Ekn*VAXI!XwZmox+B&L|z?9q^{WHZM5X*_xjj@cK^IqvB%KN!gTypmWs zY%}3-xS`4eXdmoPd+a@WU(YRokcK-7__kp`xr?Ogt?wQ`-oh>y;?-Mub|(EJL@pJw z(Yf)Mn_^qmZE*H$1-ajT{%fl6_EhSIywj}8GGvkc`q5Z0|y|#D90=U z{+EzWNVv@0Rt&iAl!OB0QXC2~VO_7J;T=I0N>QpBfxVxv?J_hXd1=5*q=11QQ3?S; zgJfyU3$cSf(tdscPn)!bsjOU{ar+3e(IviX@M`oZAzv}a5D5xTg&KG2p0z0A`v4!I zoMwEt9E0fn-y1j(XsrMEo}>h03>KNOgD+K=ak46S`c<1gcI~zm!BcO`UBtse8L(oWnlxG6ewtJ37pVke9S)T;xAJ)+4#aPK6@={i=~s(L$pj-!g*Oe* zB8VL2ci(h}ISUe)35t3Gv2Wq*IB7e33#_o)3t6{wbZZW(_3)G8!U_ob+F=eCDMW?A z?QszJZKxD+`#fIozT|3r(|^92{x3M(RAH4XKNphkR zz>Xg)(x?fPHzm&3_IpD1Ep>*TbnB?lc#5QGaY&>0+hzOqWONBiWD$wfh&I2R3ph4_ zj49DTDQ$##C}eoZq=?3MY=OjheqMRL%%K~wASXcL=~d<9LnV}W5m=mPZrleENFhnC z9c5|@vQMHES!lQ*-t0G*ULrlZh~WSD{BynGTkE2b>_XPBbX7I|AFb5?d{_t^`PVxu zhK?kCf-k`gnH%wRxqaKADW?f3D0q-0WqHE|eo<$^$HS{F4*VM)Y{mB>RIaUdCtLmzJ@HTfcrj3@(c+7#aJa%#!k>V) zYa@;5XV%R`DqM-ulyO9fc1ACascdUJM*ai$t0}(9Z%ck7M?h4ai6A&CTd}0BZo0-i-{yiiID6+c#wE63r+d-=qD2!_OW%e zg?v^l(rPz_M8KF3Tz-5j4BsAM|2~`|)8X|olSU|Gxvh5m^XIs->Q4cIz~hfg`ZiHd zXRuH$mha;m1-b6AYbmx7F@s2UU!sKgki-XGgAA`uEz#-r1clN{QR34?5Q+w=4~BPm zG=0Dyy(y7cZrurdh%ol7+$llv0SBlAf{|aWIqZJ}=N{~S``WJEO&bJTh+q&?pq`6M_Nk4y_JXgke4IvLj>M&8R)R`Xxm zFREIif+2-3ADpXz)G8ADaW1Ej&6g;(re%@C^=rwoM{*5%#c5w$64}vZEPY|JnkSXPLKi87pvkpm8oiSPl6o$q~F#E4V*=W#_^4*$I zz|#)Lrd_YyeZ}-wqQAfWy};TPs_U=&>bW{wwiiO);l>zK`;I&m{$*P#wZV{Qn$p7W zra#!cu*z@}9&cGLU9NH>9>Vp}1&d{Xb`*)KU~Xmw8N=D8_Z-H^(lbjQIHVSje{lQh zdsaXBqj4q0J&1z6B=N1yz1b9GoH$p`m=}Qnr4mo2k}dw&Sl615jcxbEqZ<2UfHYSo zS zhU`%De93%^BT!{WYm9-%W6m9aMZuIC=I^YsLYCP8Nx2K(u>n)=tyCzps-k&;25=tT zebk6>O)x0XSm;|BG-&bk46X^yc9<+z>T?5&&r#(xFTLlG!AzMuh>iLiDmC_?DJA01 zMYp)qqWGrEn%clFN_;k7Y}OOzoYATOKUJ7Smb>Nz-6wU^+<<=f4;V07@%6`RyopwA z(Lw2Qr!IbuY&q@F2}!4;c1va|&e6zy!HEhh)R?pt7Ml@uzn8~4C%1q4dHNa&#^?Jy zMaqXA;k9TKxV#AJCE{w^*(6F;& zJ(mqT6vipzxsAK$sk^0W8{F}~z0WwLlK18OG+J-tsPCVOuudg07?Y%SUfpST&>*Kx z;^Q8^3LzIHjSe`W@LvaWEqdd?Nwdd8Rw`s(@An?T{sYzwE-p;@8OkiJv8&4$-nkUU za>ZnW5uX{XDU`;QxB!jyiW?AJ3sdzFR? z(?^V-=MgiU_6bYL=3&y=rP)Kqx5?$Pq$=5QgSxn{c0buQUyK*aQfEA?HcN>x8IqXj z4mIxNr=H7%T^Zc8`Cv@ok)iG#_gfQns0xNSIfbKukKhcahV#@5@XyH&q*s+W*UGjN z&xV12@?qUq%95g^c@Y_A>#ifG?Wt=eqz}c;7=u zR7rgeR?9*wJ}r1S_Q33eoOPRg?--`jMHpq$nf)$fuYdBum?evGVvr>O)ygm1@XJBq zL$Nyl;r@yp0|Pt2JZiL={w`E=0>iFXRyv_t9Or9Hy+AyMR&8Z`A29e|xNHAEMFa zl0)Pr$lxA&tcm1r;k1EY1Sn`3QNc@4lKloCjwfYJGfyH@cV_>~E4-^%!9uKk2ugFu z!9UONbJw22&;|cz*4EJ+B?ilR5pHJQpQK*F&REKTCsK{N?GfbTfSCMT-n61)jQwVZ z-MH2*Gb0(%?G+LmkTwj3^`e=O$!+^k3?|+!G`-B3SlinDN$J!@2EXu^D*ZBrY`IYv zY8s5L+v7KHa&NGe8Jr2+uV?zJVM*8h&i}vS$inC-G-@2VYEmt zhod{}Z}`D-7R|U*Ys;|HY{*bl{X4n>x}`>$#fWH(iH(yTb)FQC*N2VVhLUk*#T30& zIbfuHlYJKk6dv|@>grR$okmLO68{Hmp9}a;$VW>zakS06@}mH1ayGlV!;f5PK0Glq zu5>+j9Nn)^)qv*Vk+oCj-E-6mn`hM4?aoBk&o@XtzgrW7aq6N#ZQKEA(UQ>qpF$96v5J?e1Mhks0*GAKlMU;53=N7ol9(5nDU^WQa=@I_z;}Z8?&Z zcr@nspC{_fgC96N{ar!4E5!cefjHST*~c?#|=>2owZQmx2m$o0QH-989vx;}jQrQem=naC@E zq>8^(G=(K;jud<{=R9gm5>wv3z$WakHhg%U89p`0&JfRcr|bQw6^RKSyTTMXtD@3@ z*-Qhwfd2%K035PBR##;hbOd$wzR=pyN8vaVrzD&Le;Rhise>Qigq$%RG+$gPX{ZY; z^gY22|Ag2LMVGh70p_{?T#?Td5nmwrYcO#tT+lg+ zz+_iHT@M}Aq9M2qW**kichG0wX}7|1-Z z_>E{D8X}8I(2ilVRwTe+f~x>+D~Vg|c?oz<0aMqRnoPD%_<~>4A5H5B=Bjm;^T^5h zn?95m!0qvi^hvWRRCa)-I`T3;oroGdj#w@}CC@)N3xrlLn@bS^SCge(L1w=HDnrI@ zD|h!114_k8ld*tPCPVz&#J@MV@>-sxi*JP%l4`E%?;Fpo*9d;HjwTAXR|i~>bF^xs z`Ou4z*%|gEJ}}3O;LENW>5oL3yOnuURTTEjwzrP6!9? ze?7m(K%0yhIdtxODDdv`Uk3X8><_NRW(^9L?TmKWU1M#3Sd@v>4xl&HhZ8q=ak{NZ<-0w?P#HE}PkHe=t)bIViZi`9xOn1VzKk@sYNl36izz$fsud>z zS6QexK_FG^%wr92y3OwG3E_Q(4K>O3dJp1VWqAh;aQcgl_b}Z_XBVEs;Qk{j?*D56 z=$ll8HgBKW0U$)fF&cIsBD7bMNhtru)7mdH z=nU4n#defVta9&jK^#|qUy%2T3T=(ZQ>ef@oTMToofh0NQ4DV>xBea>#g$k!(`0zq zA;%DU$A}RL#jo&RBSXy$z^k#my9%1TxB00^W45PH!W_a(>P4{kzOXpX=>Ec;g-V4h z4*d>#TbEEFj>Cv{_N?8Ta_?-=<-Fed4WU4wCDV;cdEsr<5pG#7Rfb_jFd#xC;Q+(du!+|7`{}91>q(wTAG`myfR%ni78-6fn6RXusc1w5n#=4PvZs znIDV`XTF@cD>{$)x88aD389F_;vO=)F|AY1zh&BoF3Oq+AC|3999nJk<8CfjG<7n_ z@Zt+X=`eiEk=r9lGCk?{1y&ZR$zCh%4ea8x0c#dns5KYezEnW+RJn~m=9Eq6Vl;OfvvtFsbQulC&P zF9Yp7R_KlWU#TPjSVJb%T?oC2GJ72aB8bVlYI3vDDT&C@cXvr?F(^BfO6oKEeLXzL z3|z)z=dgo%0xCwOLTAJDSjJv^UwfOM9TApoVlok`JJov)oE$0RbJopO^*CvNy!>Gm zT-jrm*__{ZP&CT+Gd-Ys^+~L`e~}?gm)0{0RnrSA2s6*kV(8=$9R>J&h5g_>L?I9i z*)`0K!*-M$b0EL`O6Yz=`16|wQ}9#2GbdnLEd$#pr3{LMhy*GazH<%@mLg58yM0S+ zfql5?P1fZ~-a*chLKD?AM&Fc?NCmLP3ukV0B**)4-ne{~z#0O;PRtgI*nIHL`}h%| zx?H_xdG&GWS>s7f2nxOUXsPuuNSJJ|N;POQT5rQK;~V)XSdj{J7@J@x6xQ@2OQNEL^Oq6o{p1puon`c|1yYFLybm9 zY-Q)OU{Rw+uHW&rZQ349(`Ro75}wBl(~xS;6f;jrOC$xBut@Nn$1EGw61t;ffHDD}4_8c%AG^UL(_yr zSMQrf*qiyXUaZN8G;oE}C8nkNJr`^4OnrfGd+*Q(H~(Z(vA@sqVJD3~d)_L;5b-EE zoBS4T5IeI-v%EE;MK@0DtT0gF0cod7Q9-xqO~PrVw750cf+_%5H(U1zZDgzj=BenT zHQ=!oVW;NSG-lWq6p>)Kqn50ITsP=+#dxQyzuQNp^a!Yo?NOBN*G*R*5si<0C1da&)p-|h=&FhfpHmBxn|;2Au5pHLd)KFyx)|kv2 zLNY1i2C2^%+_zC!sm394bAyd?xAxNW;`vT|@!FHgi0}vh5c4ONfxQ!sdD@cZ`2vfK zKEE8R{e#B0{J9yGLsr!lw_uP&b_o|lh(b#;6qDtAzD;5Z29!z#!gFjB-FcT^IQnKR zpoglU+b_=D?9>*4k?P8*$!y2bL$tI=_To%(beWXeqbihpq356$KUx@kVjTqo2|q0S zv%Y#>K+~pkCu}$+7OxTFmqLZw3BuAqHEPdxFefC;c@bd>N)n1e0h%MD?2g3f=%FPf zb5)!cEnyzWg*+CCW3-^eRr_tT=~7f8%S;Wq7knR8fyastYQ&Zq3uGy{h*X4?qu$C4c2gmORi+igFi`?Oqn3I-@yUUY1=*F8| zaixNcaz+0#PqeouITR&AEBkGSMHz=HF5c_t|1s`d>lKmD*Nd|aNBTY3Wt~;_ zy*_gNuKVi%kxodKQ>=EM|1{3)sq~czaNe^pe!Jsl9?_8)8KcM0Ww!pR)ZCiXPpw+* zbS0Nn;jR`qzP;hjFY)e^8-MH?`R$&KmD3x#rs)?vR6>;^`Lvs90-(Kmr)Z5+yxAy| z_IsShw#)jW84g;=)H<89c{&EHV|>1^LWt4%N&Tq44wIHM_x^T%Z*U-$wAGXRsTZ+Nfb(x&-D17HTb$%{pSkOCS$|88S z=~VAcI)AmKaxu(+$YtQr?@jPKGj05f=`wnTX>e_nw2r~T{*}Yw^D_6G6~gpNd+3#4 zV7GvaiYi402?1((h4HvW@=?i}x@)lPks8i8QGxvrcO4WCsNpkRK5u%aYwdCEDE0Yf zLz4b5ow=!n^U?$MKzW|L$6&GDc5^QMg|UUfBlP5))5zFI=9^L??B%n7VTSef8|29K zX=ho3=PoN>C(AUD5o;!mGf>1-7%3Whu`uN~?=jX9?%Mkfr}Q!)=!%f#N)?16d~&|U zmX-!-fC(+^fMsjbi$K%=6gH=+GKsy4Gx>owtMbCLZ0?AW2W+l1>}um8~kYVBX5 z0K?RN_4Xs0=Z6iyddm>6-Dk%D6TQm?ulC9j5}W9=M08kP%}&dN!~R_e$3mkyvn;GO z3hB(zAYm{1$mCE(V7U3&cXRz%s>gPWLGy%Qw08}Yraa>A`31QGzdrKOS$;R%Ic;Mn?w>*=@au}q$=eLv#TXn@Vi*dD96WH8=N8R7jHLA$EAmL5~bla70& zbnpWyCGID8G<3Q z(|kyc(Z>Yda$IkBshjgMr+BmUjhvtB4JWtYdHe-C((^?f3$Rq4V@hAb|s3;q4u&%zkQcdEj7a#AFrp zBM(cV&ET)^^AFUL{F620{fFV`RgIpsT(QJtN>|Kg#AU2>fp|@3Es~jRrF*zmBg{S+ zKR%`J(nH^*6ifmN2G`JNg8!`HoM9PVgNP3GJ2Nj#tx_uAUPfaw7g&8J(66753!KKs ztzkFOp*YdeacpeAllUPB$%74op z82?R)Dt3u7Ye*MNcz^xRsbU={xHndwr?N7YDf*oc*mHH&`z=2&P^;W?;=&to&-qes zKE7n=LI^ApYoA7zI8B8C$HrHe1&3KtzZdkzQ5|`tY`-PffGBf0o20?0t){f#q08b- z(613Mawp}oAD{xwq|mvw{>OP$?DEVd@f2gaP8Hpxk6Y)ga*l3|c)-Z>uBI@Wo`)&}!0kW_k&66TXS$g3<*bkvEOQmi10Y?u zwMKi2#AnZKw%7{L#J2_ac)G)lkxoXn(q31~XHF{R_}z)d&>b6EFX1c_nzTDBL3CSg zmp47FrhUMC;Elg0IzwH@;sqRtm=DYpaFpV$BcQ_2HZXfeyDkJM^Vu@GaW&^Mf{JU# zOZ+g`^!sV4=#fp=@ZSoAq%qNdo8QJ-!@;yNJKi{07z79e4C?4a8?Uc08ugFpKYO54 z%5%jPq7%f?Yp;9Sn;J5Yw2e`!64XP74rpcC^S$lysZD{E#SzBpTbY~|wXS9Xz<&O1 z(v`-W=`W8dY{2jRX}k_52OC85&|t}8w#P;he=?OP&e+iXNGHJ3ko;iVVv^;(mDl;} zuf2fzPYGu40qD^*3zju9T~CTie0Q8dpRoE6Ctl%+4kNshP>@0{)^-U^Q><_OhgNIP6n95HH1{IMi-&QI;(x$ROCHF7J_ zl6|W#+0L;3_Mp#CQ?IKZ;LFvWt~8z-zZjBBA9g)%_`|MP;sKUI8BdXDUqqg7eEc0D z6asYl7s(@+YPrz}PWN~7m*of+voTLO0z%_a&5CwAzdVtdySE+cuj3HkP{vG9Pz_0q zw!1c@)4@~LGG?OLQCvsLO#ee}GmmQW{V=K8%$cGo_)#N9g*>*X{u>fsUDy&&hav$~ zqxZ$J#bM9`ot8gdpBro#a&m_%94BP2jp^*XSRjlBrsmpB>>h zSwvG&L#scSOW7yd+j69|{`{+^DENsT9%P(VqC%~Iq5YpGe#k zcs7i#y$DR5#sWP)9aRaOr5#j8OU%T1K={u_U+djnf(P|NyMrwdw$D?loJC4iWfbu! zM(~SeIHG4J!9Y*3x+BYr>{k$3{!}8Z3@8ZdAdCsFK)W1)A~hh{D7=Cy!E~JD;=-Db zREh){$k72!!hxsmVHp;#-wl*a0fF*ruecJJ_W=m(!o!;s6Zah?d z-tw$9KD_Fg`fwz#JlB_|vkJK#HJ|^MIh$u{^%J#Rk6E}ps}QG(VGGlYqe5|t(&&mo zXIt?pTt|ZVa!y>~EdwDhfl8w#qwOe+$FAerzCU_Ut=1mycwsi>32iFDG4%zR4LS)^#C_y{PCDi~c3B z{rvb0-;UzWy$`m(2N^mmzPnTuI3j?gKsNDJngghI5D|!I@|fAR9}_6?-VN3A-10Mj z6W4}u_n2`1H!V(?PNI6B4~BblK=}bKSt=M2L+ilOt;n*GC+nE zhYSynG+v%4pF}N162(Fy$vU^3`=U{{Y z_V4TGMTnc#^;@#0I9@q% zbufSkw2}&Pq$Ub2t^R*(iqol(%XzY>x?2IGq!MruU@ccQ{YDT}b^pSQpUvhGXpg%Q ztcC6Kd8&T}UCEkBDi=s~-T9~%0SbzEB!F`ALVIs0=HV3;2tQ${JF!~4A{}UZ33e^v z#Kxh3;0|PpHVeF6OREyqWrQHCaPy+nVf7*a&2Yqjro0BilU!{Hr7B{k%m5xAUEa5I ztS1rZY`G#l2&^A9NQ#vb9qKYvDiCSn;>1Ww}CmdK+ zq-TrPuY3QJ#)@vQ{+l<)An4D0 zJB|coIMl|tyd@y;ud^_b)0l;9F)8TsO~XuDJS_g3oucFQo1EqU7Un_-Yi}SaIdTw` zdEvh0DI7U?l9;w6MES8H6HRITz?@3z#?NKs%XT!B7%h^ZvNs~*3c+}Y)az=-?UXCy^_cd=Iz0)cXZx+=cB%*(%&^t z87wUPC1>uYE59%%8NrwT*7={VZkxS z?awkuQR%46`YM$q{YrpeHB@E`XfV;vgdM3Hf*f&rjhTB}p3h6&hz&J=Nvzxr>0kI4 zMfoe7$7yHRbBZeH_M)f;+@1IlSMHcanNb>$3jW8jucL5Tp<#afVRLL**mXvPGE)hB1_^ z6q6NW4@Fn}a->5B%odV`NxvmkE3J1|jOS%^Ch5V?z3twhr-#wN6nc49Wt%##7M9A4|n$t_yMLs+4%iyjy#o zy#cWlB4A3AdciXPeaQ<=z48e#0sNIG0)Z7sB1X~i6(?u`M;A&4We|A2HOex!B?R#aAplc1r}tk4m}yjyWbH(ehi-lh4(e*;@j&kI?G-C`;2tO zK6@`YC~JeXuo}h>nUfJCFnRs~goHglJiK|Y$ez7z@j~8=<3*A%e&{0jHSVC~a6Bbc5kZX#cg(W!0{(<(+{ANK^5%6HK|gSJ#C45^p3-e={gsL%&=f#v$u;y|~s} z?aPnP-1j6pYFj$PkJkw8_Cn|lmd={sszk{P-k>0mQc!V+c%#2w4qbMYr=8xk&r{u@w$gUOy662wu8vrHM>a7{4#l{37q8mA2!K zJFm}FUn@+INd{XSPda>ev5`*^c9CDINI6_@JQj*LGH1Uvy{*;rl2XEzF)GR^HU}mW z_{S_fnJ{x%L*47%Q|UDVEpoZj^s6wL&bwlq4PFRV1oHO(cm_sISkUd}S$d+-!45bx zQ>y>(ZpJH)K>q!uAR}ZxTB0~qP#aUDI8>skWaPout96gXcA+C6P?ct0*FDmK51;X& zu-kFQ(69dn;m8Say`W?$WLeKD26x4*F)>tNez~>f_N^E`W1K2~I-qSg8#mhXr)07<32e*+z3EV%%wflF z@F$3Q(r{hi>=~cJyx+~mG%Q*}F!5-OO{=fJ;1cGYd%%z~(&Z>X3XjqpRpLD?-t>AD z()czgqFufsW`4G-?cHO;*$#+`bK%kqJg~FbLzm0s4&g~6w$*f5Z_13VfJcF&eZ2f# zo@is?>L0%KGaTgs@4dLLsp`KlqV@Xr&d-^57?3>L%G#*w{W0^{CUkFiDYaT5Y*1Fi zokz~~%zRBL(jn<`IP(oz!Gr$Z%Muc$0-K|?b3Nwt^eyk~+&gmBa=oclgAMEN%Yi27 zgj$L20@9=QTVFEU+i_w@4suXei$dqdo;DGx*J_r=F z?@R-buZtluS86hA6`sa^I~({HxTlAgjm%}Ao$)rAO{cRY_P+S1&1s%$%i5k?|Bwg{ zcK6WQ`XP#PPfbnS`vOzYvVl9h)3GC&9rwyuUju~=QLlRVN9G_SZ%Xdq^Z@JTynd*t zE~Q=U0M2W5U}(m13gQZjhn=8Cj~8NsKzFLwX36CwiLU14ekmtL%9;g}^(5ZNz3xQM z2;4jqQz%KcP5Dlc6N?#1NCrnsmU-X@D>b7KTL~9i5X}$F{?Ji72B@^%iov^A6k(|R z=9iThS!d1wV#PISQ>-1k9mxc~@ntzER)esWmY5eu@!wa8ELVXv zYofn=r)$QTO5lhmpbf9@Xg?^a=Lm1rg_KUk^_>r-2|GbXmw=usVc;y}7}r6@M8;c-=c z%viBu*3cd%o}A7k1qLa7E-Xf33lEOFgVBnztDF9VE&pc)GL|NliQu(#f43O4k*g5q z!Xdwip|T?E(ujj4R(x&@l$(FwQMhh*qB1&lCQ_DJ1gZauhK3Vj`K7;bcr<83MkC!>$SO! z_%xP!aDBZzw(RJkirq-2k(1hZ>0(=+b*4Z}xVi2c=+KX$w8-ALp&@o6WNAMSWTSsb z4hLaRy=6Wdt_R_}Ep>Xr#!Ng96EqnTXwJRi!wcq%|255CiT3QkbL|g|*h{HBx%=e8 z2D4J{A6kwsw$;So);h)#gL*$BcZlV3+8kx@T~lOd`GhF^Ou?a19vEKE=6^%xx_w2X z315G4+p(7l^KuzINLt1|9_nWk_q;Za>M2n&b^MhB!{uzhiO-buF0l zeRrktDVts1Ln#X&ebZU3u$WqF6^J#5I)(^4RFEz5T&OD@-qR-zS+TJVSf~on@LDP+ zJ3~PI2W`bphrHgMIUeayVq}@z73Lde+|DD*7MBaTNQ&*57eVt1IRa61xB5I5PjHD- z0+V+d4x{EVpc~9)Hj@R7(BO^UiKZcxeIYSjmxc($HVe~g{G<~D z4)R^gG$dE(&ZtjU!P*F0&j>J_8pls#^*v`OtjB)|8kmMJaT>+9FjC~t8XcG8vR|Jp zjbPw8Y`AQ|SLg9naNrU`*wRT)osb{7%Nq|CK~!d1i6laP&q2tr%3N^y#;9wiqq9Gg zSNV78t_bIP&TDF~zxMCMVIqWpo7A|$0E?|m^l`e*-3id%D+BA$e)tSL;iNSuF^^>$ zq!PDL8Kf5Z5JH8b>A-N<8ORsgBH@nb%6Z!JXR&KWpV;Nohp|>of+C(tj7`NMk?mk; za)e@3Q;2W%7wW$)Q;)yVus;XsMiXI}9?qd&UX8A&|1?U&=fM=MXVtYyw>>b|w@K%S znq{?8KuoYP9*e~R7evI@E0uk{AA}hVTl}epKB)MqC)X48dZ`!YN)dUNA=G6*nQiI0 zsn+~hs;NSE@W{?{aJ(fx!j&$!>s*zqp>4^)P!u07g(`Vjlt+DIqUpOz7cvehbQ?3MDAp)>FBMkXc>F8yo)R|Ii}$O=GX#Kf_7 zQnG9I^oH5in8OTxkk0xVR=})cbcglNKdVDZ)wNM}B)5i%T2%Z`%mE4O6Hi3EN||g~ z>{41aF4tT^t_7uabvOK;rJRZJF@}MMnMh@3)8mQwt9$kV0PX zv~}1biD$Z*tn3m-816a-1XX=ueY@q<8MW&{>Z-W68+V2YOfbBhu^v~Y{^D#V-hd}4 zN49Zw_>WKc$_mfc604LJ_vJxep80F8+X})*YK1I0gNEV#JqF!+WA=fikuNUSoY{J+ zn;~H&M!IU~wGxXFE{b!pfKW-wTvFTbAcn|+?PtESe~`TaqmOlX&sjr5gEZIYh|YCA zn-~23=JVUzhn5DM%3cmd@&vf5ofH2P6=}Yo?zb;zgx6})v{%bmywap=NUto9uCl0_Ze^td|J9VAe1lCln*t1l`fOuwD`krpGeda#+aZ?zGde?!+IKOvOI% z8($m?p^p>&#?u}$W9PFH@k%OnLO5nzDB37ZnymUwcLNd>*Ng|$8e!=-9FizYa?xQ7 zX;~Cv`Ez)vqNxd9gg}$w$GjaEb+Gx3eWs;0=_hv*YYNmKjvr|^N>7Jb+JYjRqsn1U zPQ64wH>*1G#(ye7IclFJ{KMFV6<*i1<^@{3$OA^b-`i67mSF~y8!PN3%3ILNu?9z2 z=}MRwuPN{^=fn1r6ULyU;Q56t7YeZtKgtu zHb5&e%Tu45WWwN4|8&HPf+)QsoNb8cH#y(BFSA;?lzPNq&=Y_iq@j~Sfe7DDtviT#O@_Cz z(KrakLoJ^*B2^OuH)lfqp$m#%%&ZzO-}GQ@hW-1{tk~@t=o7UeSxlZ)A(`%vj$#;W z_=q)K^w-y_km z1W7_98#)p_K+4rvUCB_U*b*2;%ZsyV!v$wkLwBp1a%f5#Xxz}P$>O!XB;NW_$@6z) z8u+k7qH;A9@MWgwetCsAsuI6Wh{S0%SI)m}mRr4RU_p#E#Vh4%&`VB_Y%Hlayvjg|`_MXGINbsOYG9N?w>RbJ*$Imw;;F3X`4?!Z`M^2JbP#7f##!wLHf|X8i z$$+DFqvaR_N#T6eOhfK>K&`cEliI6*;ns4`k(Gjpg5aNzo8` z(f5b^9KGB&0=rQ{Kg?g7$U}!>lgcQc@`HJL)Et2S;n*xK>IUq5pZ&r=gZ=Jw?wPZ( zv+d^^hs3xsU+|q`-TmELu;Uy|f(@z3q0v?6Ga_YtOE8(~aRryr_k3IbZcqFl4yUJ< zhof(mZTDXiprJrjhLOA$$_yRNJlBXP77GJE3e!zcZx%XPZ0dO)j2;?Lmqekf1HSLS zxM7GH;SIh4Ne;RfU5lhGPX00_rw)|Kg@UbV`N#wLVMGdBxI_~4aYclMXnYs{{+qa4 z44etQApX=ZM|bH<(#N4HXN)&!{P8@YZh-IM?L&2I$rSpP$M5OuLRDf#=U8 z->(1OU2!L1y!rqx2$jjaD49G*>D>Qix+p$Y>qQP0_}6c}yL#~@{FSorMO)>ak5T+8 z?{4e$CFCJy+dtWZugP4PJB_nO{jze<&3F-rzX|y+kc<-aU!G^KA=ldf!Uk_pR6M<% z;cLE4^>9?_DL+eP=5y>i zuVuC=WFPqZVDR`ZcRMNlZaAxoZwb#ib-h(CRt=17GR>OXVqOo+biXVsC(H4PiV$VToA}@`F!QeG@L_WQ%F}t>#{%^$S^pc0 zorTtYx9~@wsXnLm?}kg7ciR8N`ESH>bRk4=9VH;X0q{Y+M=Ly;cOUsu&lw3npWyTG zd|!REkcMJHg6v5M6Z)r4eC?>LCq=jzmI!}_J{8d>wa4m3mx+=|;#RT0gmjba7``Gy(k^KGaCcqZ(8d3rF zDYZJ$9s)`{gv`heZyyLsz&vQzk@cbZ!Cqc8zHRu^;%?#kXIG#0OBlTRcS+o)S&mZ=8UxrNC*B9k;)^e^L$`-k z(!7UlEnPnMAE}%SaDgS98h_vK2P#3rp`N63r4lTNWY9JbN?lkEF!{!b@chU1yD`Y{ zw8iA-@3(Hwq!ASWAWkh(?5CrLYB{l&wAA0_QJ~UGsW+1qzBjXwIEYIWM}Tu#X5E`F zVxx~k!ioer$NC}UhDr_GR?Z}RR_2p9TypYoEC>N>_MTtq2%?XJm9IGYVfH*-)x-#o zG^LwEgzlqEAwbVu36Hv1#e<0E3$z5i7kWG|IBOyo@VQSzGuWx_beAy1Dx+vhl&nZX zE`&8YmMcZ+$ZsemlvP1tC%_bJZ@VPmAm=zxyiort5Wkk3D5;v1CWj2!y4av4A-uB!5)~@uLb1y$U~Iadw4aL9paeFE7cM zsw3(SiQ_?VuPW(#M=E$D#>6N#SX}cwGBU+HTPifEUT6%du;hLD`-d1xeQ;bjE4#*G zZe=sIxvNf@r60`NSXar>Xs!wel^A5>+@tN+<8GfFt2zAhYl2Dj;mfli zaI?=8vM>sRW>revlp;oPqR@p|EFej7Zua-m%piH0sZ4X0a)PaOVQpzd@t{=ZYMqSn zCN4WsTys2=hoGCzU%g*&!KcQWO-ul-^+K`;^l4Y;glD7x@adV z^oYYjb_kr>(OnG?GAYV}!))26Vn^RVW4JY+%&&FGxU9&5Nv;R%ThxBntNeIa*G@hm z>J6<|XL*+~#}+2V&K`b&sb3GT^;hd3@MyWE_2mb%l41n)f+lWcclaJH+1brrAv8^A)c@<3Arp=QFh?$3tp2}Gj7O}5{ex1&;2#IY$^ePCIE4EU*+81 z^wCyE2lN{vcZ|Mr=;u#~Myn44gl48w`$;63q7Dp0iMS+Nhn@uR$UHH;pcv}a>E7uP zb@&sXD2qR91HTSIY)x^yIY6P&DEVepMi{)$k7~@k-VpF=jDBaThF%uzCes{3LiNG~ zX&j%O?ySBx7nN4YT=kpA)jCy=T+pStJZ8WXzdnkeBoN{2QC;#&8M!8cfn3OWt44H7 zfIhQ8j-m0$B@LG8H)e`2{$P~x7c%qtS}!&cdNOxr^+j!!Mt3l%z2?KvV?Y8u(jRC(=tCTa`- zW2?HzW8r&UYE7)#P=%dh|JIaCY;T=L{&7yg^GA0>{s>=&H1upoCO{@Twt6qVwtonV zp)W$I-D*TkdrM zDTTZjOYoVGJiX`b#WYjWbNj=SF`3V{Cl=FG*pI#JUk|pK`RM+ zBbblr-Lb_^t>3cT*ys<+J2X4~QEYaZDda=og*869J`oOb+RO$FIa%7+fSa`-FS^E+ zZl-`&hm)dQv{CTx759=U@>X)Ks|2i)`LEg}Glo0#7)0j`yb|TC(QchsH=2Nr+!7yf z`Bm<3aMo6_Z=p!w?HJ&A?m!G~FgcLPox7_o{7Qr>bc>Q;3`ti_G#?pBOhTHetftse_~XZ8ys2?rh^?wEq5T_RpVeHgl{--$55^A9hsX=J4on zBZNkOW;rWjRS;ZmY*u1uVA^QGIIG#mxB=yt>HA>}wRi*KXY@j|vQNm(vQD3S1EPyjwL|KnxOb9z~A zUruoLtK4|a)_9)!%hMfAcx_Ud$S}jnu9+!{b4P|hC8fGaPAeMU@g}fS$MucwI_kS! z%+C4mmH;1#;#iiWnoY_4UtR@Y?gFJtn~x5(btjfeL31|8J3c$l?<4o;Ph8cfk)MyZ z+0hO;47-L$X@El(Lid7iSKEMxF@0k7;%o_nyufU(QBXvfA=#mh4(!^d;Df;GWeWx^ z;cTv`u4{6!rvIL{uy;HRHf8&(9ZrRP|7U^B`pD6?E)A_)Q=<%%5x{$YAFA-w*}l1( z{i@XFt_x4+LO$=YQ5054*-Wz3L}%DmeycAnK&MYCEBG2A)Q&X zc&4jL1(>G7)#amWcG$h;^c+-3?|HO9DR5dhS!jfXCm<$9H`yYM)uvo37ltfTf7weD z-b$Ib!az^*VUt{lK#$V>^(9o}XE=5YCWeq?QB5Tjd_i~aAYq_okeEJ!T_yG2{KsJ9 zZ9ea%7A9pgS7@oKKu?y6tzh=4&5ALL?FzmAvPvc>3N5SQ+7AF)+ru3%=G^0LPoW2+ z&}|MDpMdr*7Z#II#g_*Q;hkdL?=sn}N$p+tOw7cKVF6}==VCl!&OPh`K(^siVebHM zpfkGnhG*h{x5-U4k_s)4MA_3cQ14=`+HbhN+zk_Hwe13|`LOS&GcM<$y;^XY$^lr|U&H$x0{tf4g}kB7 zw)m=EXWbviIoWCB(rNc7AdNfLtFNZBo?A``{5RhKexDlsm>rmH3f=>pBcX8_)cWH6 zuo?XiXkHFukkR1De2T&Z-ywBi-^Afm%S@~@r3tSpAi?3y8@_BQYf4|{qT!b5#tW2- zgdh;OL$v#V?@`r_!E|D4-FFc+-hNI~F%0{w{4i!U*(4agizRT@@>8i?QE9MR5&an4 zRfgb+CA{OxZi0Y-1JkP#Uq&D&{-3qfK0>)t>msvFTxj%F4B zNt0j?8Sv=|K4(b+jh3*A&u+Ecd8VK&PG{9bvgfn9;8BzrBsutROkxk$^9k*W%G%h4HJVl4EZOx0-4t;>R{Er z72sRyP?Mb3{aMH<=Z-ePgT1twIhFinI6YJSk3uo>nn*ggvn8>CB6U2SH;7{j3ViyU_MtDukeksC2&Ds0KFn5#&*3IY8Z^DZzau}E-jMSr8E5w4nOy#Mbr~a$*^{Mb8XJYISdI?r+l`7-jetJ z;{xFL{n{xhQjtn8xuKyFkI|-l2!6Dhp>Ax(MqdH2BhJC^ zP3q12cNmewia|H!pzS15xH)(0j|uA|_5N*f+V5AehJ-FOfT!E$$JFiT>~cPoWSL^!^PJuMM*P!Q34-mux0zq5UwJm^D*BnMMFv;_u=^cV*3!V zlO_DdX;q)-=u9l^WeXp;qDqMYFJ^uyP+oy&?#x@wyxw!7+NaxQDRd~3?7X)lFG@;C z*?NfWcYh)Awm121BvnV$-jR5>r+j_kBCi@DyY_Ia%-(H7L2^1#gLm_ zQ+U5L#57(j zT0{}=9WE~W<){Y66GzVP%6JOZ5&^bz%v`oh3hhX9?wjyP_&T!3#heY$qnBCw_)c3) zUj!-wrE|Z?tg!5yWC6$VTZbNRh~5=l_AHe?v4xMO54>BPD;7}lS3OT|#CX`4FRX}g zeTJgmBHMCN$Kct+ zYY>?NDR6huM>x)?istl(O51s65}J~KxByjf`0N$GGY8jcelHE<*8Q@34*QzRbU)Pn z60;hxTTk-NeitoPa(o*g5aS`AGo(5lZtiV8DoKTN*@k3i;iPW4$ zRNdDa(L7DfAkxk5K*%rPJsiga4w{GjGD~y82cvULO9H^N@Th?9)cqdD`>u6PYt4t$g ztmXJETIry4rA#BJFKpyv4Egr3bO>%8yF@Q1&l^Li1~vT-uMY40&B6&u!RG7f^d{5v z7Chs$G`6Mcm56o5QmVYRcuB={fjn2X!;Ttglm;gHH4STf(*BRy9`DHVFEcbyu!FC}rlM2YdXt|=X5!om?0{-Spt0v1e$FS$YzKgQXwM&pa&^}c>>r8q0p zVZ!^$%Ow{9iCAUCLztL$%gHED>cPRkHBE`IVv(6VoKI>wURBlD zRwl7HxY+PdvW2*mCp8utN#GdWAV}FX~%!G%Q^Mf z&>itQ&OZ_GTq$fyb&zO|occ%KF(a)+qYN(NUwUNqi%aGf&X3nV+xEw$M*mH#yto zl<8$yk>B&;Z0af{>>e|CEA-elu2C|l>RBN6F}LwtOLBlzQ)*^;gu(D$s;bixMT)`a zyvyNYui8ItoEob4atOTI1T5&l@-N`|?b`+W5WsY&X`y4;7i>YoBIQoqbdfCjsxim(W}83iB_U066g3YdKp* z)m}v0hPM!fgmvc$4lW?=vppH?c%4Axs>NLO9p0#o7{il+>LKv#c_o6KTY6`IkRDFV z3c68yxd4&Bi2A#xFjjZtZ5PTn6D%I5DbF=~2@oq2TyEcc2Ej3t)HDcC9*ncjMJSnq zYj1zyRaAW@-1~*s%Tu&?YIM1{3>vM+2KGpie4xQ?AdVZ_;r0DRFKQ8cDky3o>8e*y ziNN*MZHsU}^As_sJSs4I(BalYfH=-#Ol3gAU~--9JD)V?rt8AE)99)=a5yq`^LG$`?+N|_TCStp z)|MESrl^~N)1-szD32LO|)5^>x_l2QS z1V9+$B^Do=ryu{j&%fK{g7BwwUn1c=b6qAH7oF!&-&w5^ozFWrcHf@QcBK7JhB9(8 zelp5$Q*NpI-^jY1`z7)y;l)UOaXu{SI9$+=C`iwzeds6g%T~&C9x+RETFi03QWkm; zRe>vre?D)|1GXG92X$BI$DJ)wd;ez=0>DdXEkWX`(i*FT4L{(Nh0v?j)l4|x$>Ke_ zB+o07zv}`zn(d2Y&*Pfgsy=><6rSnGbv8qFM`E7O`}5;2jqu290@Xe)4((XtjAefy zB<#Vh*I!fjG75ZL3|8%*dOMk_^?w(7ds^MdpF3O_&;cuHC7^`MPO>-Kg+PnNmSApo z(W+ik)0bfBYjF{v=iB4_s+U(fwZ88LpCE^>`&@Tv!$;Hc-wl{?L-no#7MJB^!N$Y) zUH5|aZI9-LFWb1+)8A3+95xJLzLWKB#p435P2TqT{rjp6+LxN_2wwg27J+ZbYqtaZ zk1N{1NVZUcsgE9WmCepOF)TX{Z%@^1gBP;|4qu&|5<0!_YmOsm!t*^&_XXh~njAK| z_Ue5_(JCanW7zcVr`XYl*a~JW%O`l~CLS9XVrdjt4j0XRy|5&CK(u3Zx7?V6ctND3 zBo2TDDWKRoOg^pk1m)t0ie@4u+0^yIQ~l3F{fU;lpbDQ?tO6n4h0FJ(>V(-INy>)m3>JlyP{O zGk6~lbIT@#SEp{BZU1Brf9qj!*2Bcc1G2ff`9h@L#Q4P3?>Xg!VmgQS1d^@dwDo>z z;VWY)-KM6g`f!z0=ytDamFd+uP*Ecy)DkVmx=^S-j^R z*B;=5ADdUAlhib^oe1ZSZ*>d}?JEIUjmJ?@QAjY5FyDLH#O&<)c%>65$0sH(U(fx4 zjea3*4igg-TiH8c_Xj2F9W|$|IS@<4ztrpiIKKA*7j@e=%VSAa64z&Txt_IxoOg%O z>lkfZHA>3py*i26+1b_E&5^_UO7ZI5PL_Jql?CVJ{FT&Fsp{+NKb&j;{lS8IeZFt@ z8n>oC-Uoq}8*LUt?HVJD_(6JtKfw!YrKh+^5Z7}OWDm^Qo7eQa=Jrn-V*zGMllTTV zcb=XKJPlLWl#VOT@0Yj<4=8ePj0@hc^$iJKp||Crqye0$;)I{zCR-aP!i66rlt1I4 zJf^*0y>Az1IqY{mpHV2PYowHw@i=aFn-lh;`mOEg&e2AKe-yEC@Sr|)Yu7(&R{vQ> ze#i&Xay;oSkut?4g}y{!Fu@_3<@olA&GRg!{x%1!d5ne6~@Am9QjVK<^JEGIJ<{ zcyHTP{L8PqNfJ)cN?$%Sd)3%1kSGgnSZ67PG_CR|`sLv88$Z4BF*P-1SUEu24Lsl8 zu-f_BTzO{M<>CO9)PYwX0&{=$P55P6d`;MpJCPFJiLkz-V;nq~@O;blGP zS|JeTh{XSHL`gqAxS+sgu<=u(f^DO_Il4pdLU zNXirTv%Jo$>nV7&Wzp1|0=qfPh`A*A1>}u_wgvl@j;{Z4XW%c{AE`r%Axi_)enKEw z(xL^ysJ+`mrv)AB>Tqa8;ViOQ!u7*YM2f4Zms_|-YKrEF)5iz)F*4Nc#&`8q(3)?? ztMz=Ok|~m@Qh~s!34&=9B!G8P=C^bp^82sP9J+qlNl!MT!-~{%Y?Jx5kHkl^=>?O6 zUUm&pmXqqtCc>u{W%NP3Go&n9@=UhN*BEU%EA$+XRu%;aqCaSTIz>Xaw4HHbEEpsI9e_xt<2;;;;nHsm zi^8izrkn!RE^k|G1E`R)MinJDEEdfeZZaJB|a5v#OGo8M2oruR@gJLLCH=0YB%rF$Sxs zK<2ASyGC6e(+KZ;T?IP0X=o~4t<9s-eR_tmP_^xf=hmYVPLuXmrDQ>>t6VJ|7XEx(Sp@^HycSs)p*7Ucs+F}0 zR5x-XEE{OKB80$@51f&FJ$jC%J-1LX&=9)v{;6fmgN<$-1h2M!Oo1kh&hDYRxKc7z zUmcDx1X#(tX!?j9m-?_j#ZmP7*GblBqS=!8A#%R>d$a``IQQzEuBPxZWOQa1jEIXPyvoZLFFsPZQ9H7); z{<0PlbO&{c*T9f0h*%#~43mb?W>o41bI??~wL@F0fjPvAZ=H*FtET3#NcqDh^eTlY zGOc7u+PU`}2P$qaSQ+aUM=nXTpcBnc>Hmr_mw!jlo=RII7+hAcxo2>NDBeQ)fnq5^ zOvukSDR+&P4(I!2S7*MAp@RmcyxfCjtS=*tC4wAL>5;Umb|?;o(a_EwHo0X|VXU54F!|Kd6#RqGEutISGGs@J-F3y@C2$77Gt` z-TI3`z)@uQ_6k&eNxvVE^ld-O7JAmudz9j=EL>*ySX2J`5{F+RxcV9?WDpSeuu8#!#u4P4RpcWY(9W0trKaT+ zr=qlAZf_U2;@xC~Y?zSGf)1DIY=Eu6fcG*-ymv_ML@wkwv%v3KfT39H1wn<5kT^x( zZCf(9gZAe!Tuz<83Pwnfb-TU^OBiII^Izntsi_(qOU_pdSaqz#L66KYOkyaf7)q$< zU%AWm2N4oEEq&olPYKrKU$xm;bS&gq^aI0-uUui@GbpCU7XHX^HGNhyu>%64^1o7R zqr?!F>qfBAFdVlw@_>fL!?F^)QuarUc6kJ#O||Blv%U9KhO}BcT1jdORsy&-P&szG z6{-k_2@5JZO;gJ-v_g5pAipP)Zt~>O-x^kQPNQYIDe+{_N6%$t4V(FyNn9qT&!7&- zFs5VRsaDsn)ArinRWCw|^)PBfZH&~mGxQE2_QE=Uom4USq|!kx2|_;0kDrnJSR9nG zVbCL>u>hT4HuvrBHa#y1j}(KlNm{Z={TtlnJwyJG{EfW$=rz)w8QWg$2X`|OSHo~C z@}3>JneLh2_K3-mbVuj3ED>~7;St2c5oh=q{x*d0u|9k^^=9+~)o{`DdqIZX(i?9O zCi=}8?84s=%VJ64j-q6p@TJ{*>{(Q?;(;e2`gVgOs;mn}F-5e@{9Q&LYO>qsxF&Px z#eB(P^V3HMH1=M!IQrG4dvwTITd<Ybp!W^y5;Y-S#1HI4aU;zq8y@&0xP~zc2G0-A1cosV4L5(@oQ$qxX_h`Y#s^t((%J z!Bt?5zQ5jnUu*Uk!NB{{j(T|V13okst3sut`?HT-*wHAOxn1W17wDl^GYS2VjL~kq z8I=bkO&bjb&u7Ex@<>D6Isa<`K8tBdc}~;g-&=KtG$-c4+l1m?dE2v0pN6?G2Bii+ zei9vgevpDqv2+g&Rk35&Kn%T-L@zkTP=MeAbB#iW_3#@R`AV^E@6~I69QP zL7GFS1ukU!?IiJ{M9snvgvku84QU9{phhvP0~G8w&7ZRjA8UqNW|T-5Ev zlEwn&@S028T&!pMf!JRxRTuyKV&Ryf>A@4nZ`i0_Kq%|ch}PvoD-G6>=(e3*Fn2hu$bZ6L<1Y?u+6q;apdC$gnc@v&9Md_yueDQ} z(4dMX-{0R~YIi3Q)pzibpdE`G2=UD3OO0i$hTmEs6Vuc@0iXQd7lxvqDrut*_mLiy zema2`Oa_ZeL}`1Nu6bV>Nwhj;X671V%hQ|A^{jbV2oH{8e8ExLS~V3YyFNx>O!u(5 z$#0y`;Th$QgvU|tZH+z-QA{_5aeNBW@nIMpg249hbtD2Ucoh7suJ07mC`q`dMD-de zq;N}3EyKB;-pVaEJ>jgw&SDHgN8Wr^k-Esq&RJ$Rp~>6DV5d&ky^C4>y?|&|MVqmy zyC-rNQ9MW*As^ORha=fRhK?DTt~$~w;YY03GbwDMV)nEtgD8Ur2wx`P>X&cj*A5}P zUvq_$Xxx*IUisvHjv3p~ALljM9+A=a5GnfS>3-Xs6DSjR^K3J3y6HWUq|>|~qaWfv zLn$Y_Vl8b;UcjfzU9OUfVA!4p7(~SDkg1dkM`I#NW#=v#k*cJ0;t3v=A#CJ3c zUpT3^WZlOlBJr#2G96o{^W92*h*1-c=4oEau2FZg)@y&l+t1j#r|9F?YMEhhMG=#Z zLKV2R`COL4D={+kA^-SW&;v_oaNM2@jI(I>0m)<%z-Yz}$D#k;eR0bKN9jg1;-h5HVCX?WFA-$EH+v)^#FchQ3oeYtj&D4E^?T(8&zow% zuH2(I8$(*C1OPwL&ocnuvXg|Z!ClNKD&6`LQD1w_!hLTrD>i^5OWl3>6f(c zsvA(L>7!k3PR*Zcn<;(EbC0Z!(hKnRkEV>6mldt7#br6~)8NBHeO5tV#MpZ)eW%}x zWrI>BI~&cv%P`rQ{N~|>bB7mjvi#jhf1@u2su|+3`pAb}4Z!Cc)gjMjl>*M&f*ARG zt5GAmQqTpz{9u8jbC97m%uvg=Ikv1>*jBuO%5%kNFATQ+lOJJ((ypboBus@R5hz27 zQj=jqOdn1%PUo(3m8E`#D|s;aBldxdZr8D4^iG5fN2=MCt~y3Wn47dixVl;2+EniW z=kx7fcKLreMO$*kuWZ#6;eE|+c%9^uAg0fR%Df6vdPJ`145jxU5P*D?k4Nzrfe_3{ zc(0whQi~xPh-8Vp{i@YFc3sD?GnG4^U>H9wx!~CPtd7gU1VuC4FTci4(r8Dp)O=-8 z90p{NN0g#=$liuFnrKt2f0OP$Ql3Kd^Oo=$ir*~7Q1r!W>4^gTZ=|M9@%va+o(f2A z@=ZVND)CqBGzymti`KM`BL$U@t%R3%9vHRjdJQnfP=PJ9~1uHTN8^2oFKp(Pr# zIoZ7Q90htXW-?Lyfk0hn3DNUW8dUGQ?ahA0{cWT=VjBND(lI<#&P_>q6hs<#aYYvx za2F_FO;o(CP2={jXk7BQnitY3)Xu!;jCyO_ws6wgF%ob8S612B(TjI&zo-4>!?tgU zpm_B>tn1P>)m`zad`M{?#>hrGDOzYL{Vn27u;H26WJ+x`BYBatVs>$%gP9DP-?CNX z5oD0Ik$ObKUN1iPn0ZC-9$(*zxX7OWD|dTd!LQFr;oB9?v1;)$9kxtBKwW~NW1?fXU?1~~=Jw^LvU)tyXviTG$iU{w%O~We41Jjab8Qd(6MSO55O!Oc ze=>JT#TSgkml^s?@yBZ~z@{QPuuQIGZtIGniWCXL?Q~_}>gux7aEmg+%PC?~=qc%C0m z{`(qi5q2i>id3W)`;q)A!X|abW#W_--G2AdY0p-pL25qcv zI$+b3`Y_(t*FI@aYiCRkk_OTCWPkKq0IplWtejIw;pm+q0hFMwZ3 zB<k%v;gO>IJ{XY7&+GKCA{MErT<95j=%7SV9#kodgq>%Y`Hy^-ubfP zVtxw6Q#rbuwxpX)+RT`VttgW<0lMuGhh43?OR8+C(9k_73gU7U8NsW}*6^B&0cRKN8Lz%)F=Rvxo`zr;4iyST!3ykxU5 zvQj4F3B$xMAJ0dwbBpu}pGe{8TJ2Q58bROc%Gp;p`cr4GFY;YtD4W15#$b0>!LJv z&7yj(*UUf1u>?I*1aMQG>Pmn!Ej*x>)23!8qxX-*{x57@moW|(ac?tII~tF_!HJMR zg5K#lSp|&p%J1(X6_d$>Qg7-BwEyInAhJAT2m+wQK{O(u6uR8#v z=etvZ%YICOw}+j6a4(W7Vl>Dx-qj|%L+BY^>E=Z$Jk(LUt_aKYysSRWQ~Rw+I&lBm z*u}u++d*llvQai+;VD{@gnrrb)qpaLP%?IQ!Ll;z#>z=va|;W{vS86AFMSYbpVed` z^;-irh!Bj$Na0v#|F9ifZe&s*iC+mgGy<|#aI7+wM$$Q@b8-8f)>gPaGy*^%P%-%t zDBINB9OlOvfa$K>#g%}ycpV%PrhyUSud*X3H+mB!arg8T(NMg(r>0qi!G!MZ+4!w? zuVh^5^e+9pvNZ{Bru6f@LjO9s8%t4F@o}(@U*(8)8qCzHZnH;Y(t?Raip1oBe=thllUxogv?7245*FjvVjmnbA5N>QoKIaB zU3Ox)uYRJw0wrX76}iEg4WSp~D{@d{wJ%eg&>b!8-MO^H@O4mYh-V!m1)R9wuJj+- z>P6H0B<0z2RGGx!l03i2_lx8AT-Drb;rAr}QG$&JDWTg^<=UE>0BcE2t&`>Uw94%5 z(z3{9+t_IsLV<7bq;Pt|2lS&|zRykn1F27^728i%fz^$E*yy8mU<4joK=wNCSMK1U zkwP0ee4Q=mec$k>+A~+Z)bwyTK}*`NybK}`%HP_JNKu^N2gzR zSWsl%J+L#_c7@scHnFX8=yM!uHqJAoj;x1g(Z4N-^kl>dHA6$Q1-UV9{fyiMvaq-D zQhB?;<}}Li*5Y`4L6Rr``R;DvBq;y1tl40?AarbNroQm9?>Wg}cuW81uF3u4GR_yu zbrhVK0_O+_`F!HP!=j+CVX$#?59OBR4dV4_1Fx|EJf6AN`)>%Hz3y|jp45$@u|bK7 zlqAJkP~o;5Y)6%jCMq?LIeNTID8D{0R6TN%hY#!RHlIQ|nA)tendqG_4ho^`M&&my zH!C7+X*^k7K@mSLjJ^tU^w|yJs?bdP=o&-+)-StpuAFWC{<#KNFVO#p`;UALn;g0gPB1j>npTZa@@1q@AVs&0;biZYZ zer}>3c$Cmk$+4M%hE(Q<<1HUpm9nK}g4OS?)&E5sJ;Tw5TRT}(1- z(%%TxM`k)v9n}9017~@;AP=IdTSG@RJKk=qOSswlNc>)Qd@f`8J;v-DB+=fEYydl3 zVZ?g!@a^~d_a+rv2IqdLlB#G@Wd1i1%0?A?GtwjjyhG~k?$=fBz%1HGkCO!Hmgh5; zxvY$yR%`ay_5Wf@Pd_mI$nd?{b=dxx9D3*!ISQiID3AW|<(>knouO>C4axpcrKmXe zu1WGnjOdXm#d=0%lAzZ>vs37i5!=2bIw(9ax95!E(AOiQN8f9=TH(lbJqcvcbCq)u|4hJQ63vV zql^bf+AZ9NqWQUqfa0FgA?ot7OuzX?Yc(i8aPc?>ZkjX6IB6j?k>J73qF>4r|F;$Z zp0#N9HA(ZNClPF((i02jf6z3Ej{Srg`%@x6Y;Gfw z8UrfrziqZXlQk2knrzo(o@{fnIoY=DI=Lp>wojNQ+rH<$_x|yzK0QzE=ZCe|T6=FY zZdIljO}U_R(i$W593O3R+lWV4c^X-TFB$-s>chUe4|JPH&VPgQ8x#bWKhv~t4-?Fu z#_$aVCl7sq<}Vng9E3P8DpJ`bqEYUQv3> zy;a9oxY?q;x5S+vb+WJ3QLp^`&w?DH@jrCAJw7M(xo_Gzif#K*W)N{G(FB+u(!j79;!K# z*<~1!di_VMl4XUXQju7eL;)>Tw$!GJ{X&?aq?{Af82U``e#@|WhnOJeM@J8V^bL~7 ze0hnllo;0d7{UI*m|n~c{TSy^id0$`rqUe%J}TR0miTiOAensso#R*z`bQ5XEPaderDgq322LgfODFwt0j8sImR{;O3KWFps&&l->P2oX9MRelYAL;%+KbE3^FMOoVbT!%>&G1#OB-my|#`*Dk{GnVUpvH;r7y zHnXC-%17#R);Cl)bI1l(pPXh=Xy+#D??(~M&7PqRE67`WB{u=xr*f$R2%v|^uw$Dk z?6W!#XV1tQ8Qoy#`OKQqjeSK zcb)QWRh!4fJmIc?y8k6m()nFuWQ#Wv2YU*q zR+L6B=eu_W)tgmF$yscxLiZ!UR7Sh$cqo>GRgObGP``9go~!b$BGC`D{kveqawTIb zY`X%9QAS5;dko8Ch_F#gbV(36`6gVIg_DELxozFOG@b2ap_Et9S|9{)$XtT^5@UMC z>ay#c_52WqV5+nmB09F?@0ez2<249$ijnU5B$JLBPez2iwKv_XW+Y~B8lFO}h1Xi( zkEcLacB}=OYmNcPJ~x%MXwxDs(AEUIZ6+G0;JFaG|7EeZR$O$k-4?S21l{Y6%VU2= z3T%0Z3x){6R?>&hK{3zsO3a3J^Cpg#3i^ zAEtt)m^2WTyDCEVZmuvF&GNb>tKOJc^QX0aCdu| z8BLjw3+ME=YSSe%tu-!)@725NFpN$CgJ(L|3Lndi(Qq^d(c)q&YRM`rH86%FHV*Z( zdBg<9%tOrjDNSLPrc78k)F=Tw*LylA>;7TATH`jFjcaPKP=t9j>y^iUnrjfNo=YPZ zc4c6)e~1IPeT;*fLOvZV!8>w(cVTc8N3n`s+dz<<7`P}6Q;gJeL93|iQ4Wm}2dV;m zgw;cNh6=Pb$|~gA?Z-8?E9wlaJ}CG0YO&w_nn7P@3UjtCnjUBCK-u3_0FrMPI1Xq4 z_1r=ZqCmQ#wLMC5#zC!q*~KA(W0ta-%~+etHBvLyCFXUhHRi`BNZ!*DXqgD}f$HdwB)yTQ4hQ^AI3L z734@!QWCg9Zi(}3UdDh)QPQhX*thvqfnn%?o0M$rLx}MUW1Pm$u7WV?MNI>k$uM>t z)Qq%%IYl@88*2=y8obGKf1Y`Gto;mAP5ud#;h3;rWL2GYMpI9==4i9(>;xZ16+AA` z0|r&ZlHIn%cW42H)6wEK{!x@lN<_xdhRg&uS_=w7y|&emKPl(l=3ox4^ZrpE7%6+h2Dfl2sM)aTA*rdAq)YIUyflc>bYNCg zJ!Z3S+k@tZrl(le;b93cBg#e)I8h{PS_z~c&8?pMNPlv1TD9uIiR?skZGe_TPAo|)MX2N{Y+LWR(`gx;~qiN$?WdrYmj6qyx3iq4O_NNkYHEz4Hmw@u!zw6ajLg8+zxIQpr56`N2TYe3O34_5 zvvPEGLVY*+X)bc*(t~rbW(7{T`JL8oq$ZzwZTJM3cGH}At}2~kwoXx$FzmrD$^f#J z4G#Ma@{u<4(q}n|paO<)w^IWb)ASu?>!vjOsaYu%9pHGr&r8gMwM?7lW{dO`w>hC^ z&b+>+ZIeDA`Ha`q-r(b1Ee3!Zt$%?3SIOY$64M65#v|iJ0&L8?C+6jHd}J0MUw(xX z?g9JePB$A425bRjq*WA4jdKT?!8l`wsJgei7Cvna&_=y{DT(hpNQ;paAwVfZw&E^} zF=~RD8glpVUpw5yUN?zy7?eb_VXge9g0^jidqB}&6HPiuYN~xs;Dw1Srm^^S6upgw z-iN-Kx%lw1)x~Wt^}bIz>aw&2%a%UcV)T|Ar9|jgASD*1k@b`ipPGSwn+`J5=2uLy9gu}3LNnn>^6aug)u>pJm2ep2+MbYWAGe!=z&f5rq<~J%90`vX!qZ=^j$^DPs9q!oC>%I;Edo5n?#e z2SN4nO3kM_>y+KtM8b``)2cWQ`|r*XY<^n6%|mtJWz6$dl6^l8`ZHxnUnyu;@n~zt zv%%G*_{EHmLra@#cp{AJV5#$cMR+RPpmR9oyWFA^Vae!_<>g7^3gQ;9G}5vT+nbO} z*S$N3q-0QD!_Ij9|2%=({Mj_EiX%VI3hKz7#ON9zLkpc==w@pYW)&tB&rp`|sWRYn z7(HRBefhpVdxku%V}4g%K;SX9!ekGY6Uy!{r`XftP>F^PN9~msKWM1D_47oltKwx+ z?zwc)J?)3xNi;oyxh#nQF^q#5A2vCbAw?Wmj~r;QjnmMbZBQA^es##VCa^>HUIHR63LY)%YStHa-OgKxKVIbi-P7!d9;&E(kK1y=>k7r#fTrL z><}5Uqs*U+{?vRgDs|yT`p~Dc-p0tKxUui zs!`DTNMqA4%etFJvO6BoG12%TJFVlXStnGDve~;cuB%r4x+hovt&Me&5Y>vWxbs8U zCB3MLe((uG!pJ9D9hWbf<3C;Y=uRk92odpwjAfpmAQuP87oUVXtCRe-n0<;ncnqEU z!ydPkvd6X(oSw1z?Lc4{ZRqQpv)uWP>V1E|4yG76LC<*i=bQa= zbC?xl4sUnMH*Q-G0=@@szM=cY|2XUk&_W3?|LL0p<}l4%Q~W-dK8Ou*t`r0xg?1O{y zeHX$t*#IURv#5Yw~$}wX5vlWy3c=i zpSHvx`*QI|96#)pu4pTd+2b5cMxUsy#Bg&gQTF-^jGMVd@egO$B2GbVRotw~h7Bz~ zJUcPnH*T_LdHL5SuoT+na#b0Adtl|&bGVOqbO3KL-ADoJsWO+7NulnLU=W27f(ceG z_Hw+=cc4GENiP=?6_is3X9EBfMcHUzUo%CF@z&Lp+jYirM~dL%8#>G318qcGy>(8k z`|WJ#YNr<`=Y~Bre&=QIhTq3qkyL#OuOJahv$JDa1ziC6BlOraue{od%0n004A^sR zSh31SIPG;Ri))=Nn>7o{sYn#A0d2eEZI%`k0`&`NM4HYIY-{dy0E=4P8E{wDRn#cp zWz23%cH%l3A_{ImqfMM2Av>OyO!Y@x0M}TJ<6PVJ*1Uf$4L5{l!B{P_l3yJlo)SG0 z(qF!EOe2LAGF{8$p_$?`W{T|d9$0{j0!k9GGN|`Y_Ai_J?ewGdkH-5GnwF9s{6<~T)|9vjC^^G-mXDIWh(TfM~O1n9_A%QXuEXeECLGNa5 z+2Nw|;kv#1Z5`n<0GVmmE0!P0zbU1-nEJf!=G*g|fV#d9`?8)BRf@7L!*`^QT|u#^dr`|*0<{_)}$;rBl6m)`ku z@36S2esv`H#tK$6BPg!<4hkamMws5`K12wDsigYO{N6u#va+^@M~1VQ$qd>xgg9Q$ z{zV2C6PtstlkTxrjo!N0=I@MWE^|QX_VubTqJ!mTkiVQZVOmHREEoX$+DlY z%AcHPGqZJ`6DSSIyHEy^m;(i83>0nN^k0D?jwM!gYl~(Mv9%3u{(=jakeJ z0y8%BLHdLRYF0bz0XQp5`C95hIR$JMn2?lIpb>)2w1OMF3J0%7gOIVyMdCiM;aL!4 zjHyio7^wV>A>!^E7I?-4e}8bf!OQ4ZaCw#}JbJzPG|3#BuC{q=Ticl3$GzX)ZX{DL z!H%{53!Q@C9nO-L>&2E_M#*cY-`z)Ck)Y`!Q~n@bha6wn}n3yicdtIdc31maDXwmS?J|EgHH$(q$- zH3Le;j(aac{P`j7YzK!Aj&6EiSuv(Q1oCVI<c`A%8f(v}WV6)`6VWXhZeZcXqo!k$dS@3$>IH+f;C7HXeK_9)g3+mN(aFG<(B5 zcITj!<8|gCPH@1aH%2ieD5w~4kj*G5%9giYR`B>&%%;;Ff)94QI%%N0 zcCd<^f=9ifh4$~*zit@&YglHf52&GhUyYx#>0wJcY?eAbg5Bi|dbtPuMTF%;2vIH| z{JNb?z=*s%m*wZFI?E>RsP*WCgp7j{9C|F9gS|tFd$}YvE*Z*T9^GG1;^@7)Hw8os15i_y!XjQMblR@`lE|-f1IgAr*^%7eVo3$-mGibBhn{B&FgOv z&%yG)-f>8cWJM-GD|X-kFXpmH7tC8(MDH`F0xi2Jk^}YYEX;0R`Ed!RzUznx^hrz% zW276pDlcDhXYh-p;CvCdo!6w6uDB56FZvj?Pf2RN#n70x|0O69Pkq zRxwMqb{@5A#e4m#>%vE5l!X4M3a`)i)5=f$VK7np{*Ka*51Bw15yZsHwauNr^C$a+To70i&&zV4& z<7MMpB8F~Y_Q*z_u1XiGn&V;=d*P=t1dV!< zVbcZ=A1mTym`nU%HTUwtrNW8ejM?E=mVD73tR~tNcB!zpCy`=A2@2uWi}tu+!Ms5h z={J+NZzsNkmk)D4%#lLMnRxC3abLVr+1cro>z^o{b|OyF{_PvCmQ-fqpskn%*A zxe_fLb#oh=*ls0J1ycbMRGoN_U~4%N0aS_jqvj66n3hM0w3``3W2+_A%gt5tK-R$H zE-(7nXPfzQ^@1b51iF~)uNYKES+CI3eyQKAvOy6tF1Xzc}sftJMLjSxoEQIypW*ADu6r1HB% zCdB3=(->3yF7uLFx+EEqZDh;&{s$CKn*DnOOmZpjXPE$V)c6ZSs&2PbK)xb#lC+4< z{nZ=M3H#)bH6@1kb+tO6xEn@lP}JPX^_FM!_IO@D#YR5i(LIN|apxE_nWws zkVR^asUR+1BN@!PQD5fQ+1HGOT$u%}%Z}HmP&8Gt%ixQSg+B?&kR`@fRq=fZ1v?Ro zkbXDr*}>utQx`iJzqpiAP&{9^Uf?}Hn<@dIK$1}yobaPn8W(@Kv5+tq2;VyK_TtLu zx(;?cL>jo$?qbfH6yeFWSJeAv;!0o>7N+;wv17U5K z=CRAj7SkH}iKx;RCu$T=bJv8~jV`-@UC&3M_Op>~m#x};R1=k@K2V&Sjj_{HkK;vQ z{+IWbc5Ut?ovCFQkha7u@|B~q+oY-K&@NdOwvxxCiXOR~E=-7kE_!Gil0#^eXm_A5 z)vo4h;3g3VIUrjmrWS&hEf#t9{cVj9F^*>N#d=|yE{V{D@XP@cHm%|Pa)>^OP`KNMJbdU7|(GC_(E z*@?ERAhTSXhnMBWm7$_y9pv_2amlm7q`lLM>MyWpjunHtu3VP=W@~GNQJ{zHtZ{$T zG?6t!f97@4%5dhaE=dXHCy%lej5C)g;#ju~gchdt^CN0t8Aa0J z9(G$&0`J6uKZfEaJo#Uv7IwS{jkh}3^44rtQK0wa_8Wr*zD_)7v{07_CTS81IL$o) z$S#96jZ;8RyIVQmsf|MCJvi5gXd(Y6w~ ziL(jAvZtJi%!iPm18a#M(Cup*muMNC@3snEG6C_Y-aj)M%~C2d#lWZvSEPzN=Hnr9bN>2DD~mh@eF5MCcmFF zLmcdeL*-Q9;NdNYqk9#28qz`zN!Npl;U zR|MOo#$Ch)30`Cy1N*q0ot@<*6&wjSS&U#!HqWVkIXnE1UAZstjSM0Ax}A&bJ0p0w zV@YG;A+D(VYQWX50T|sRWgvT6R@TUy*mo4lgx)k=Ue~{Rz|jtmGJu4H*^+bORiNB>)EHeLIz?7S_QWJ+eEP#6Nj6$pM}s%!afZYQbML6@9-Qy&c4rAKJy7n^r0! zp)y4e@zvAs6g^?yscNrM&HPA;R15Nc`u*ij)tr&n>L*XPEzE0eODS3Tb}-yD=VN}Z zws`?I6Epwv`VQE49=erO<*`^9=5Kk#c6Vtk_<%jiUA#P#r*oMVc zjC*+2=|WJlT6Iq_QaNd*|3UewlAAwR)JQl|qOOuO{tbV=p^&ei(&6TBCUg= zhz3K%U4htxTUb{Rv=KLaFR9sFGu+URh&_qW&8`Dup{1dPrS^b4eXi7@`TgeU-ap1) zPhq1809mJdSNk@`YO8rOA)xzLALt}5pH}Hfz3d1jduL+ReLeABKLwlaM>3Kab1-+8 zOYhj{{ORV~jn3?e&gV)X{%F>QdwT2{XIhNYJj8^(^ra29?vWvUXO+J=@a;2fbdZ6J zNUbXtq1nOq2)tTDnN?1$&}eAx>5mm%8BOKyY}%336&TNbgY+UKq!mh+LC$zXDrn18 z#Q%@bVsf3ZiX9|iAYTefs~e5d8-q@aGY)WO3PvTJ0X^0l1=>S~I!?LqSCJtWl_3&2 zikDG_ZqOXjn%S)lsvu|`D`Amb zs#};R_5mvM><(2w;z~bkcBYv3;FHXh)eFsI4zsd4$ws{5?ZesGVhYgrH81tp?4^I4FQ`N4+eXchuvy@p%Y7x#mU#*cHpl!JSg ze*x=ZLrgMAUV|U0OkL8EwPl+xfG@u1^B zRFhoVOLRC$U?Q$l?`s4_#Uh=y zt+Uyz4>peBPZ5vAGmJ7^uF>si_2uun;DulL|NMsK!GgQ~$bXb*lupw`{TEqet*0pM z{REUNUV%ZVcn=$6eSSE>!NU|STV?ge4U1UH-%~<@ir%d@Eaxa{Sk+(^{Kr>&71@@} zjvB7+A%ZVozO-%!AakFbHmug0vR-&xcs<>>bPwL!t|RovvB~{@&Tw-&Wf{I4E0IH) z$Q(9l8oNB-RHefWAlq#_*&4sM)n4uJK)&eu`}w?W5AD4B!U#|FV6o;+~Vux132xPFe@PF_5R-qVNQn0@+y1r_-i`AFK& zlbjy*u9sB)>b__}0R@xt))_-{ra;j`JN9cy>Bu>;J@SJf&1-te)aXZ5`xI<)FI1n%j5M3 zOfj8qG(}2u%IoWs(yLXO3|`~fthXg^_V@EWZU=U}UXNGlHDr(>C@xxh1@<$a4$%^^ zSY}|sK;~RvhU{d8$-)cIICT=ESu+X~$c|=!lLv%0xg|?@&SujEjYvx+eLYiEsHnr8x7Sbl^D|m|M z`HmYV*)qk)&7Q#5TkxB0O4y%&F1D@I6HyfQyr}gH-r`vnCu!EaV&HkaxwEdBryFy%TV> zIMkTU=Y(pFk0sNuHd-|ny z)B>+4HAbm~Kduyr=~erVpFj_+q2H%HjpqQkeHN>uiPqu`ME+G(Gk{bPZF+@(&bE za2d_9ndS-jA9$U9+7jJ1uRQuK@z43tnX((#`1Np5IX+)k^mlXt0n31ttd+{=J2-4? z?BBURBn-tGR!t73vVRi;Z96&kzE~y4$H#9-_(l^XunA+_pROjJgj1P3sOkHW{vu`O z3&h80Z1&Mg;YWnD}?ufn{_Wk$-xoDn$%}&K8r}aN$r?R#BHksK0ku=1;fT zB9;{5U&_um1*k7+J=5WW8w*YK1tTj}s=!uygASe!at^>m>CUpiGYg6Pwr`Hly(S)` zUKC4+BD{ts2G=<}0Ug&_>Qe2l? z8OX*xH#dHEn%1S8+EZqeC}$)3df+H!Op27ASMz}ZVkWzbU3so{X441jiJ~!p3iQcV zMdJtlQ(B)dW1tvw-R^~KnXZmVC&PZL`4kY+l4hq8F=KidxQ8&Z*O^UYLnNiChiE~i znb8dkvyU2BVl8i%S#$OzH=krwHzJO&EAm?h+~26Hzu9)31_!B(u1PD|@4eLT4|uu# z-Mr|61MHWV=9o<9@bleF%tyt>-t^r}H~c*;)a>+R*XFR>S<jmRF$sh>41aXq%K(>{G z4Wu-wNR|Lv>5GS#&odcjFFw$z*L z=UbhbOg=}*DzsHfIC9)IT^FnZ0#gTq57Qdfr4j1jZro=rAaKVY?~8WMc&gEbix$d0 zkG8OXqX*3VyqgX+?)8|bD#!+E;c>Y2gC8I2&O)%b}G@P^!sEcr`?*Dt>mhfjGx~} z*8j|hwEDj_kktxk)(O#m9l-vagEBWhb{ zVRG_DyJk%RN9e)7eyj+T>k@kDZa%KMnxVVA8lc&}fVj;6#av!F2_rRAXfVBaXql+@ zuE!g^@Y!g#Wj=VoAdoS!vvTvL-t#Sw$_$!AP>Rgc?Xf(X@9)eH07aO1LJbW|kZ1UFj5=@s+0bWqJJnpCAuiV?F)%Qr{S zt!ha#s@*Rrds8;@dKTkFbTKnwVXb!xH97hI8y5YfAj5N>$xD8t=ZBV?e16Aku-cdt z)fe~aUAKU%2PKY*8N&m|dk4d`(D%!KL40TINjm;YOj&aibzy$yLOi9~o6ekw zRkTS<)W-5Dl}8U@c2dh}6VZwst)UF*ZBeTpPXnJY(KK~Tiuq0oUES*Y|8VMkg?SV! z1qN~)kah~^)pTYI;*i=`CE~&6x)SIe71S%(g_{%n{K{gN6Te7W*KoZ2Z33<)6n28&OpR|t;%v>hjT15sr{7L}6kW(mm1^g|0uez`bLUk<}pigz?8^FiuzrMO{P@vj1 zCi(hJW&mmtcnmsboY>W_8hH%~62$;pVDal5Zs~UnlSO}}UgQiL_dl3=Te^dfU&hIo zVNg@9QpOd`n7^-3Y6&SaLq>J=DUj$ZFvd?n;sq9=4rP$g$NEu9GHHvIV~DI77@35j zPV-O4NYSK1-8K;!u&lI7fLIP3uC5~M^_q_e6gEE~+B5Xjk$F15a7%f?36WKcy2z%9 zvCx#83{F)yBLQ8GDzpES`Z*`6;hLhFXo&$J=<~vx?_u2>8usGJpyYzhDrc<}Ua5QC z6$w@3Kej*m%-^?tP>R@RUcx#M~^;$P=URzLk4reZT}gz4!a#P5r&a<$+>O-t%KnyomkW zRR4>Z*N7K_nU?Rs#BP@UKa$Ts`=@X{%%nPyX9PS9fNwNwk7T)BM93&lYMRKNC8UT$ z{%9cbsR_LUwZR;DJu?EccBxPcB-4~$4^)WBt4JI9O0wEtGcYH6PwYFXsxL=wQ$U%h z0UIc6BfF_R1bm(Pm@KW~4P6>rHuicdar~6vGx_Gd@%y^IZy>S559&W;FUjVJfJ294rYq0W(iOp$|Z5b;%S=+GhzME19<(6 zRBX{aZZKRTc3=_d{Ooi)xa{;s{S`|=p3K+3^qzf@0;;^}HaQ|FCzZSD3ze14o}ceI z@Nt&ag-$#XxB6HtnW5X0vrsPZE3KvC-`mJes)IH|qk@-{|JvP|BVD&D{$T~Cc?f>pYS zC1ye}`@!!r_4o0s3b)k`M2_4i;?<#>SpE^+mW?6N=}r+!8m#oHXXdM0*VbLPZ)!9F zlK#SV2m#F%(#t2PF+L}fzu$Kge`K)e-NANU6$7sp?pdDk7z}Fd{6~bEc!qL13m=OG zq<4hx$v>dtt<)^48hU<{vDrod&em+fb8`N(ywWC~iVc-0(->BNC;zO}NNjSuG8&w6 zTxh};pB2LB#zoAARoDRED){n#pyAf%i6>e4rE;9FzxN6C|J0;PXgap#a)$#K;iMVL zC2l1#bLz<8qBCJ`^5TQI!h-nhIv(J->o03E#B`KDa( zUpP%l3J(4*tuows6&`~d`ub3RH(2`nBqh_s$U55S4{00zm26^QSOWy&7}TB;reOi# zsH5||P^A6dLmjLo|L5RKorRHY)^)ZLEQHdi6^Eou3!z5sP|L7oD#BSnfM1PX&9-AD zk+RLYzEZ!g)A!|7DwNP-sC(N1-t1opMx@Zik+g}_mx&jPl`}=YenH|;8oNnRB9Sdq z;Go5k4$7D*Tej2+%S64z0}tzdr^NmIdZd0%jS7`8WX5qO&1Ki~l>-u5M*aJbL_|R$ zLPQxgh9Xr&CBi_?-lbGGB?Y>Un-Kkjg2G5PmE2=i&g*Hx&woYz?%iI8#O=t0ScJv4 zH8Y|KIuD6;?wcE))&)O+8_%x<#RVV8_>j#NNn!|MwG}ESp9Po;q>0y?h#@>yH2p_< zIsIEe(~jL<-OB$)=N&&`s30xr)-;7Rg_PrzIi`~{%oZ~|#Vaf{!N`*Vy*0BStak;@ z#qQI^(;~XVGevQV#ADN)_g5X4ag*uh zkrMrSWu7p-5z_~jl!Zj%Bue&C=P_Tweo*Shf+~fLl(`r&%R!*NPrXk-&LcZGkmp4*yZOviF3W;Q%I8qGwlk>`(5&on9wsUM`&#MbB~ zZ>2y9rbIKS*!J8jJY38VyH{*U{lga;{E}424{nLk?<LC*!l(Rsf|2uXFmvEi&qmr@-Bhd@so7B-J}kl^q~>lOJ_>pPSC z!x1!_QCLg$9P+iF9;56eRF*C{A)Hm4z(f;ITg+K&Mnr$jD$R0YqtgL19dTQqX<}Je8)|%n*JLf zv7ig@7j1A4G-%KREnej7JLxOA0^d8A63&hX3Fp=$5X`Ja|2Ne>9A=q}yA`T2cyH?E zn?6F+<;rKXWLa#U86k|ki6z4P#js#$IGNKss-*_4d`Ve)l-_$ES${8`CAZ6Qsq65P zCW71`;SPOc)M?E_451InN4R+y2bvZLvmQnZAj`Vf>;#*s<~=)yw)Q&=qeeTkY;~JWCo4Pt z-_9iCnY`3h`kU-T=M%JTXGYd~6?geEx3k?OZfD#(^M99DfaIrh^-}%rJj!Pm%~=JZqVz3 zu#)m2ZEy?9*X6Zvr)Sn0)}ls2Ql*1kFsmCWBdFbfzqkSUzEa{b@kTuPU}nhHdmW%j zd^s-K$9}8^Jz{*lr4NU(>A2H>{HqHt`hz<>0UFc?%utQe7#E&Vmy2O&*VE6+xGJMi?+k~`@_!^CVk<2MF_9~J>{G3*vI>kLW9W_7RS$xk>|UUa3t8OrZT#mD*_sHBd>Hy zAQ{~cl)z++Qj z30Td{rZi>MkhqO;IywsmClR0&WAQeA_*n49I8UjKn$!78l^J`9^65zvRj8!!+;8KC zT`dt>hJ47Ub1#834#ve%x}mgJGx*OlbnbGa9Q^Wku8`%8d=o4$ zYm7c-fhl~hzu+-v3_|G95hbv!qy+8wPS|2Hn5aYpFvnR^y#tu^ZnfvT<#DelpF4S2 zNOH_ARbRlPY+f1F*_8VV_LsV!1)oaW^@l3rv@*Ge@Q{)rBi`T9iUa(`AGq}rq6?{J z%UMd)5T)P5jM6vSqh~jfjKT`A3=bXVc!38QPU@JYc%OXJn7r>oPnQrBpvKHVR=BRv|2V$@A3dPg0lTZ_mhNoQ_icC-G!%&Rzz&DKA-$|V zwlxV3rodzSvg52cmv2Wd-$oi0$c}wP z3nx0Yn!vsEWb5G;!2U3yOuSSin(PCOI3%*xWCj1UUPJy{Tor{&2OAk!U+* zfKY^&Swa&b7d#h0eLMPVJsZH-3~oeaMT7n=FF=vzOA=Jo)V@Av0eH|V*;cSLbz1ZYGV^~;t!db-E^y=1DoC+Xg= zk)JL@L8Jb*d3xEI-B;r0o6N$Nd9qFM|3lSRz|<8j(Uu~`-HW?>aVhRl+}+(>TijdR zDehjlxVyVc(Tlt5KYj1N{QRT|$)(}kGkf;Tnzd)IiFb__X4S{u&GDA4o3uVmr=ZFX z;nB)Ii&RY#H2ecVhFG zs@0^R)$9v64M9+=IlN&|WQ7U(`zR>QXg&;=a(OuE`=j3q3qvhcQ<-X#rQ#nf@cwmJKiieRAtj2M3`&hN#x zt(||QC>vuBsb_oLWLMXf_4DAEUejvotg( z;}$D3oPCdPd4bcQwTmqS%4iY&{yqstDjbe2AB+GB2I;>`v%2q>ms?b>DXOLj$=KIF|!B@;VuC}O7{(D<&PkG~GG|aM!=atRtM`8yuh9p=7 z!Ga2I@m)xkeE zOv2egVb~f2W%hdzwFeQ>RW$FqDj2ZmNZ7@fI4~>nKcUA^c~Tx88B6NF6yJGIPHz)` zJ6y(D3Ng!&hxX8JS#sNZ6M`DEC+ z`Pflr99QA~JV^lzomas^{~qLD4L%($amX3Wov@=iOguI8KAqE}&U)WB%$Y_)E=p)T z!_;TT0C)j%e0KK!?yCE(fv@1soSu=MS1#6QWdYLpDAt4K>AmYZl4Tl89uxI$4yVMUJN(R8;oMEFC=ZPBlzU$Zcr)btnZ-#XF z?KHo3qKNVIcv+|h_jKKQCx_RuKY5+qcflO4fhvmMP=R%|HI>tT5opUMKuGddbA)bz;-|uOX>8fiYr>?LEbt4e8UDR2H4b9*^~J zvz|_MJOj)m0=V?7NaDi1_o5o znZoMx=4&6C;zpf;yW4M->Q)^1AAnvE{z(&K;~(br5nwM>x+JS|+5`*?jvd^g%zKg{ zZ}hmhIE+Eo5Rvc3wcI->?Gxu0emOFQBI!lZF)_ew8$bt}E~zN*z$%@Ml~p*N;-N#b zS+^UvcXyW}M&ch*PZ`zIqD4o;kAZ2>Mp`N^zVrBOQslZ`N^OD&1hsJ3J?AtEXmn=ySM%neV%6O>>O{N|NT!DD4alDk*w!n2-XM_O?#ky*_Mrm6@u&`ThbsPz`cWcfc9EHUv_a?v% zR0LD^^3KkHBd~Y7LjCmbnZ@>_{;)c&PqVX22Q=f{5@Iw-GMMntKrex*DWZ)EaJX-T zIsU+1V>R;N&spw*=&><-3SJl*8k&2*zrQCZ1J+1>9tP;kWv3%&ajTvyQ>!@oW1(KZ z429eVDjKj!S7u6q>t*qAIM<(9Sg5``pXA8mq=)@Tmo$>7Kd=3Rr9hWSk$yt0QoVHE z8ekBczXVuLqV)hD(~Wz=Z$ogFYf{i*7LB(?_HQRvIj_j414B%Qf&`D=5 zSL&K{t!9(CnQ7Y>(=CLVW`}iBlO87RtWrTo0_}~k?@4s)L!#q%GaYxAckLXv@v%9x zenRDH6b!#kJOkg~sB>~34>z}G>f=b|GUkqvs3h-M%(2jdaG=Br1tW87zsqUO?B2dH%jUD32z5AFg*#vL zJ0vG7!Zllrf~aRd(oYi(`c^`glf2Ki3W9rplKh(gab_2_iI`eV-yVz()upLOL zd{Ida=y0&Z(d%O!B$7p^UaPlf)M>N@@Mcxh6`gve_V7-e(1b~xAg>9=NTU(#k3jmW zmPoA8lzJDif_>*HC(w=De57}7yz!TWxyoSD?|vP)*2;m8v_F|MiKU@G+!GRoi;us3 z)_PW;U3rxZY74(Z>fJ`MnS4ie zxB>l3plDok8YGO4jh3r~f9{y(J2YO$^kAVx(c`VzFIu1sbmX;EP zf8A9Nw`G}(#-}COIeAAaL2EzGpLN|I_Zam5gafxBliX|;Kg#KSvSBnvByQV+A@V;U zZM0t>`S)e7u;`ZrIQj@dn}YcWe5b>YfB%wb=(}MdU^Dy;!O{QQ;d8!s@WqDv{^dF% z?_(VQ70V?5>rG;`FzUDRdCdmvFW_tsCa(AA&IAg$TkouSo7nx6`iX5xp%yP|bb*8f z4z)etlIHFMhQe>9YL{b1(3B-d({>rXmj4`lXsEjrn9$>ApHt(vjmWPY8S)$Vqn%^2 z%pF6(kSlS6!k&i3j<9~&l#2^dl*=0xMM4&(Co8MfK;_cD!W6xMnL=5wy?Kr1=LtTo z7FSw_j=#WW5zO}5w*uZU*UH&Ch%VWUeLbPE@z#4f5~L|>d5@8*+NfapbKcp z;sIWZHbsX2VM>($dQPS7@XxI0tW|ODg7ajHo9#v$&>tl|o0uQR6DrHpj+s$}c#6Lr zCiDVfgzLQiyg3=)AEnjm&iI+jHg#IsyLj9ijHOZIyeIQdT=*2UKG4F zPGO4vHek#CYkpt^S|q&h-=|}(CPz}M^sD>~hksm@O;vL!2@LcRBC|&_(aO#1hc3j{A|NR{IOx%={qOYll8rtVHR%a%8X+zxn5e@hA zXYljpGt*DTqHNczVS<1cxV5Kz;)ckKlSS~B#qU!nUDwNg0>3fV=(rdsU_t)k_5L$v ze822cog0G9c?VyHCCGNIwW>W?ja~ytEB9;lvEXd4&~C5m=Bu)DJY0gm{|52(-yU1M zY#b5eS_*_=Ujf}h&gX!*tM#q*gwOROW*%9bz#WaEz$29K&qKHFmq*?ATog1IQou*f z1I}CLj@_!-fZbN1Jv@N)x#_**+239KjM@g!B=%Wz%JjQi7E|hrI|n8+XK-FFfZAhB z;&@``ivI0YpqV6lMuIHUnBRf%Qz|J8ud~qSyGx#mLouGu|0UX}aI=DaPi@7^ebf`XCqQI8iY> ztPQQS0%SpToQJ*>bQCtDaqj@q+VhUjrNsr$)WR{aXSm-1$I+lmLkf7Znh-eWSKz&* zVCjLp-<*tM?%=*fyz)Fe~X&bIwi(^UA;J_pSX8xCwzScW# zoPaB{$EN$|xK1|VTICNusDm1a@e4Hw6T4jRy@Nb%!oS4OiC#v)O^0u%i86(d@J^-L zZ8Ui;w-9enasD~-i=8=96M|{^*i0nxiI=tOZ8taLFBhwue<#8?G1T)Q1x=`?845o+ zPciEa(Fvqr?)$v`DO-Xmft)OoUL4Hu0Aw8Ahvm7}UBC7fRFas;ahBQ?juYINI>$?H zAgNbr_m@|<{Q!Hl-P$>TBdCcihPMm(-+QhzJ$vAQdm^boJ0#F^79Nf&j`q)^0GKk$ z)S20O)|Oxcc&XPl*y;Hc#Lg01Sv`AhB-*&Log|HPXO$8NJC@a z%mhV&B%94^eVKk>B5>6nDBu~i++v2%>VCF+HM%(*1c1SO6n-_zNU~%GtsaC)p~(@o za_+Za49~3)98H!0!GC`YNB;pk-Dzy9N)Hhk8GVp7#ybm;RgU&(+i~V>Z;Lox1=agL zI%oUs4<;pMPq*=(%ot&5J5gE0n8eZ6hK=sUz46vsOr@YS4(Vx8wDdUn8>)&%#C9DF}nb;lFkHF%O*UWJO9y7_|9 zVXLfsqaP53{x>_Z`Aw9I@~>u8RNwOqU6mI17xZe6PRX*m6<@Z@*iNxSv*~GcG5a4!7%^adP6; z#Q1av8}mv_iNKN$%ZtEo?Nc=mzwU2jtlhv+{8S);GC>^RTr!ky++|ulT0gQYJgryp z53sGZwJk%W=zQg$6vRJ08_V_rw^{59fPuK~V7@$K2B>JnSSw(Mreo;55xHG4_TiBNKvpp!PKMJS*eBO^^j%#Xh5ywyHOA>prBD=>1 z|C&8lu|gKUdb0!>TAgH!$Rfwl(UB(5nW(3ycM$r065M-!?J|E=URqMoh0l&$ESDzY z?#{`4wkGZ1z#J451c>OYp6+MI`DLntGB5_?^2IQrj2;FnPEM3m)zsL`$8d|}vn0gL zpz+ekODec~<0&9^Vw#zL^j@)v3cH=c(-}_e6Q!*qC44B&m1S<4E+3Xey z|o*YU)o$%DJ7kJw-r(slzE`RMk~$oMx$z2|Tm->^m_!Hzn=t7>~JZ|1M4E z!=>zQkqDYMtEf2f>Sgk{q(5GR6XN4VR7}Xb4a3>7ikF+L*agj(-1|4LQY9i@;*SdP{cSUJrCPA}YH$b{RwmpeH3JkGQ(uw_Wm9S$r^#U2^*L=<#8gDd zi$_LT2h}S%?N`M5(`M%u7Ld!QKT``rI3G+2kI2ZDR+W*>pBe*va@~w5CJxT@>T1+5 zMM6?i|E&%P(5p}>mN(fBj%aH;7RRKN1NQuEk$hoYUHsUX+|$!js2fY%{d`A9N8j!7 z`C*i!&lj2$=Fd(>cqcZ28%JoLn!Pe5S(!B0@aEH1CO zHDe++wom5Lv0>hnQ$ z8;m*Aa9N!xfN=OBI)mkV9;En*pBV~7PTWhaNmxUti0jzZ^J^jjOpZ{_d(8^P5@9(rVn!n2Wg;ViP{ges1+BN(?A{M=K9L#GZ!MlIZ)#>Xy}6l^imKJ= zL>@Lel1CqWb>Xe@)pbl6GbD(zKar4EOw>={OqEMOKp>HbSIWtWRX+zsDSl}D^Jt6P zaSb0qDkJvVnW~spUVD4iS{whseoSFUN9@gwEB|Y6wGLH6#6&D|WF+!8b?wkiaDQ67 zh@KrIWj`f>pf9dg@)T0czu_jp4PJ^}UN#$wAV^3t%#gDxCQNZsdispJMow=Z@1J{H ztq%RG-cL8QfL_T^ZN6l!W**a+{P?)3nFXu6t>FB8(m~4v;Ec4m8ejeWAzJnQBY@Gf zGqdaRMq9cP=4Q1H@BVk~PjoG1zY-FH%F24ZnSYw$DFz7jiz~GsoE~=XIXg8rB-Nms zFH~p&h-YeQsw+=C>R+VSS)vr!-a*7$U_05fBUve*rf^t&^cFgqS4r1TKU zsne^aM!W<)Exl3K7O&j?;C^5JXqJcW5oLf0FbjMdQOPVV`CV7{gT1)y@Y2B8SY(lv z0^v**hen?_J)@+&EH5wb$H)l&QHIU|9-@&;vy)~Ys0rmW$q?#E(a{(!9UUFFOHri0 zxbGPw57IAM34#j!v}Y^ z4vTZl9UW1NXNQL>7;M$)L+$?p;wl8cBKW=}0U6loxDrIE8xy)Jpg7Bstmj2JTdW8M zDq1o|#u!KLF-iNjU*UP*fKXy)M#<;V5eR6G=`z*T*H0s5VM@ZUXJrjhVSsE`2X64# zOUlbz8CVmiuQ?rC*Vo)8C6_xLh_@VZ^%m1O-RMTE^%bO@SOs%ZinF~Vzki3YZ+oQr z=P;+Ir<#@OKtR{8#)+Db?A_`*zBK}&Y{K78HCne!RA`0hHoRE7+XfJUM#X7R4cA+c zxU&=#vUtU1Wf212u5;43tk2fO+ehWqMZZaM$iq-(*Vo5$ z-X0{`*w{2{t+C!c&xpmy=;A4iI5Y5opu|)hmBwxX%w*?6YW;=nP+nG6q2C^QqLjZ3 zav-*{va(DcU|emSo}Cp1-fVA=db#73r>gl*duC>4srfoYjKEM<1p`QXIu4y#wb0NU zoSeqBZPLkc_kk4BYkrdgsv2r8z*VJ^Oo?d2VmKR56C4|MdbK?}ZTZl~kAK z10EV1i#!lVJgMig*?+)laxoa4uj)Y;q|eN`dNKG-L>I&XR4yYYJtF$Pm;DE+Sny7- zNL+>oykF+$RW&qf0nIvZA0Hm~8yZ?#TA=t2DX-FwEh__Fpn+aJ-r!poP!j-7(3{On zwo%-;?#=i5`U+q!n6TdOs8Rs z;W=1J3lBRgZN6Ni-g_K>Y`^6Dp)0RejUj0UrybE9DlW+{RQu87W7f1WAkV^rR%2)) z*SwD_0aMP*PfG)lx3I7<)6OcltS-Kyg2BG^l+zpsTUk$S{9ro2={1&b7|3onk6Udx zup)m(D(0ZYZi8ReB7HB-&3NL&yq1NRR^bp>GhOW%;VFDl`$_ufer*e06-26 zNhwzR;muNjY2Prq{??@Fqo!uIS9Ux!Jie3E?jtQLOFn;UF18Pw=$LvFjW{wQ^G`*N z7Wq^6H`J6>d2WIMTed+Fa@BuecXh9*qy#o))D6#--uhz0?S6f3{>#+dT=3|gv8g(? zdn+Vr&5A96{0P*Nk_x!rRDJ!0u%kkdGB{QA)PR&n?=}ffQ_h`|?vC z0t)E{6HO0BB_)zo7i{#EEF)ez90rhJ)98KiMqMTTRQqm(I(Yu67Bd93tu|Qa^4z}* z_DA&mS(iOX(g680)unu}$-y)jLTSLZ)-dMW0%X*n$uf-IQk2_SsipxbhS5mc>43he zFRm}TIE93k@TmK_0-HVF^0y!C6+WJmC^r$GUs{YI8tfrVJq3okFv%Nc1tb+zi(3$C ziqM?E;oS?BWqK%uwC%F903~IA+uMrGP~F(ji7k*)dUM`ysE>}Uowy00Ui?k9ii(TF zUZ3x0*XH^bKHL9pDM}wspt`=!c}+~9&a)4mS4(>E?J#m7$AluqZ{=}k7_HWxz)<8D zmXzdcsj9cUBy)Cgd7NhsNE^z-g%9~j5fK^+qwjUA#x^sX#%afEb$G~~MeBR!lGA(Y zT6Z!+PfO28csJkK6mPfE7LtBst-sVv{`B>)b3G6%-1$3$OeEvBom(%x+oOkvw_1+X z*`0T#fO~g6e}KSQRb_frFpNpa9XJU#Ic-5oOvFSf_bDQQ4HhK@V#z-uGQj7k8dyQ{ z53}YI1&;2doo~0ym)?G<+MK-noPZ+rpui)t^h%k>uXvjIx=$he|*?wm=M3T zi_3LQOZ?y}J}lI~sJJr_vokFjkdXRKv)-Ms+b~e*Dwz=NBu zALn20Y6lMw4!X?iS-H7~fFP30pf6%?PlSNOMA7*>@ac)y+1dGV3*{}PsE7*4&Wc`M zJX?;IOHW@pB4mC7DV*Kmji1QpM2VjWrKPC$<;X_Lspw+K!9R#VrynUqv_h*s`Q^pe zd?LL+)9V^l{Ciui$>rlK|IJhiUItgPR=v5?%>kNR(N^U#4m`A0?G-PF+D=@+3nakL z4zn#ftdB&P40;|G;yAUgM~KOv+Sw&0CjNuX<<;%U8`=5;Si34}YU1*4Uot?pTXOw) zEv+p8K=S#B4)y&T%TQ`2K~?h;ezxxw+{(%d3u4wI0=DwEZ(Y>^kFu)rQB~%ly#T-h zOsK4@{A1qQth)yq6AO#g{{;%L!Kxdl5#2SV>id zSbUg+ZwxhtTvY$@+iO~JX+?z?DpZkN#=u(hwdc)Zr?I2qhx%nX7EFLIP{`(&R&rvE zL^r>Tr?(VXmAETWdYOEGK_OBUqF1llez-KTtUYGcwL^mXh&hqg+ZTc1bu+z*3Li4% zmZfI1fEpp@c@;rj(;shmm_E?;mjsBIm??=YR#&D3?w>+j;7iNPcYaBB2m=u0>T1Bb ztc@UHnV;b__y965kETMWGdugtKHK|x2y$25Hf-@fA!y9`Jm4tC=46#Eky zijTw2EiJsw_P5smlE6PWlae$&J>3l;P#))97J$m=^^OAyMnjUWVCe)+07b?L-O;@I z`2l!jZ*TA4T&rfFzz3)UX=%80Y@_W0=~Ov${ngGKV72~ANpBfSv(}sAu$c_V5xtIA zUq1S#n6dF=BK}J;us^yI6yK1`=hH8ed4U|qwdP5lm6bKOIB)d91w3AyTa^ubHy|_P z|L7(G@RG$?Dgicw_7G4$Ts*!2g;Yl|>v3I+00IdOOsFUc|6(ZSR@;=pI-Oye^3OJy ztz~Se7!L=eJ`9-*D8%wWOb%3jI^~8Rdz3;88bPTFv_tZA#!`bd@Jd1Vy;2Oq_P#HE zpy!S-m^P3}8+bE-ID9lhvlDN^LMTt7s(-3^-IiK0UWq&9wkNcUiE_wBdsnY=tOElC z{)Y|$bG*M1Z<3LdXLW4o1H@A##W4TX5c$oG1aY17^%X44N8s{)DDXRVJy1HZbpXoG z7=|Ab6RTap+8YKqD>592#)JK;#1Jv+=vwAk7YTsI%W;Q*jG`h5Ha2!Cd3m$_F)iQi zj%OMnuXwB3qVFU>vjifw>Mg`g;yG5Psi=U(?TBM#Wfh2f zL<^s89wIkWh`htCREnqP=7vs|>y^mq44$7Br;!NN)KWRT;DJ@OU8?ytFd*^vHpWiC z;|x?~VHEKEw~NI?gprmXyn|w?zod(AKssXh@J0j5tve< zGS_rhv~|J?V;q@+P5|lAQAj|27hrYdvib1>fdhv3zkiAv zK7^Kr#&F?e88BeyH)Z9q>3m6=A?Cl2U{JmlKD)(47l34TX;mz3Eq~Y6IxQiNcoH`q zXZtKQIRRUXIt{pu4WQn)N#D>~FB2;V2aAbJ9#;>yTH}5h4!z>vh6IaMn%$YrF4D5{ zCg&UNus?>jDP{kyqe5|lUv)?NkRcyI-(ZE)&fs;WLqJ$^J!3&(aYA>q3c5Fo^@TfvdoJe&dQ1HU~`wc>|{ zhe>_rXgXf@!T=8@V`fGPn~eaVU>SXNUNI=F!DfNY=8j>1JVVh>k5Dp>h!n_q%j_1c zHg~rFY7PGvT=UtBiW1-;NNqIbx_L6-`_)x~Tsk+JFs4?gzre9EYXcBvOdO|1f!#7h z^aVa@dF<)-b1JI~3LLEe2X2$HyI!- z^IOW|mqS8uy*aElGkZjU7Y1^U{fJ_=>1;A1A|fKoU%bC9^X0>%pQ8Xe7&r{>I#mGj zl(<|r8+3xS`TN^zul+-)2&wr*_IU56XK?xv9S{YAMo-Oc{RL%H7y-pHuiZFC!~0QY zcC!&v^U3U9pmNvh@b$9YcHsn6PL0+cE`4upP+Xo=d7z=8fjZvR%?;_i&zNTD)MITp zxl)J6?TVt_Vp3w!4Vy{3ug%M)+4xWyQ#`kzDYfBvffWAZxV-#yU%1J@&y0*wA>hDg z@$S4BSe;kD*Hk$w_Y+USJI`Cvn8%pP3uEkW7CL$T@;l@6UlkKr)Wm*Hlm#cRZlqoE zQG_LvN(6{A7OXU+g)?409Q0*~o+~E}j$j(p3U*;yAa;Ic7=8|_I(Bdj^Jl(S zM}x)XokeZ#-TsO#zOAS8=>(R)460;Cb?kiGBYxnV1(kTICl~eh_I9a49YECj)xfw& z^Cxc~)?r|_eB8u@0wSWfw|AbIylKQ)mHl0R9VOS7FUBCgZO7N8g|rq^ECzd2M%Ovxz#fl>hR4~b}8 zVHFb$N=nKUN|U@{Q;}I$Ya1JujC?J=PcSf+Ha7X>*a&#AP?io3gDk&6#nf99`0&tH zc6R;e_p?h&;p6t?nDAFuReC9A664nO(~GN;K05TA8At$`zX>h|I2c2cK#ItlyIY`? zlAPxNYCN=HO9_FvK~Wu@&+#<>*d7fF9QBN(x3@Pj-@Tf~dF%Y#+-yUZaEj5J+dDQE zZYe!^q&$+}pLq#SPlJ1nd<6}9O6J|Ns;U$K-~V1RQC=paq4BA`y*;JS^=H3zKlm1) za)Q^*O{R|f!PnDMOI?4vJ~&qd%X@fmwb)P1&St@4;SvxO-*g% zjc(uB)-Q9f&h!Kh2KCz(RfBM217xYK>=PCksDWHI3gAks`G96Jl0!}IfALzO-5{B5tU510Jc<+ z|CQ+IIM3;_tY~^;Ar?P^VnmEbZ~9>;FON)nG8!PDJR>fZHpX?awPR=ik1fRJCi-?abKYSidnk13~;fq|mdGHFkwh^z~OmDS;m ztHQV;5(4ql7V9}#s*0CPLD zwk8HVNQOehTN}T4L{Z6qyRSxJX#nFUB2P+Q?MpmEekq;Z>1paX)tEVOn4$-uJE0>s zTh7_pRZ%@X4yBa9t|~p|+%meDn3(B!$1VBa2#fR@jkC8ucXW?TdLr6Kle`qj@jg>RC5C*Oc&0 zNotgc<{dN(P8bU@JgJ-IbO?^t2Wf7cmsAeV1l_JFLBco$PNvJpRbes1Z;M(?8O_7C zfFK3hDm^(WhbxiB$D7|K{y2rP8xfcE{2;&9|Fp`ar$-i9MT9kbwv{uCF4=ODIj~7} zrhO2+k$rIxX+tg~fjt0f-_cZgs-DPrN5AkenEB$URvbQp*uJiV`)5`}!myo~ zTboV#%cR7wBtOG*BL^`gjK72-=?z-fQwpY-rF|v&AV%!G(|{J#ZJ76Ge6J)8PO16H zw?nKHNeQ!7lOecwD_ECOTBa{hn-RM}6OLSd78oyGJ|9_iJiXIpRx0^bt$ zBzCMaU=p)~p&x35iW~+!5Dsr({bkJj+RW=9#-P>H_g`2`HcNEmPLBCDXv#+Y6f$lu zi`(h6tizBj)~@$KQU-q8>;Ur8yXx}TqCuP0H7gvSvC|$G#=RAZTf>~9Y7?Lw6NZB+ zq@g}SkVXFB7uyW|+_{<4)DdcASik%YlM-!|9QGqEEpm?$B_~`V<&o9O>0Mu0^X9o^ z)cj*R!EHVB7jw9uB?&~t_1lH9W9AtC0VKZyhwMR}TPvCj$=9xqO)EelJrPqF`hDJ7 zzVGwr(C;7MddYZ8jZhIKZ0J{Q8?TMO?Qnrd3w6?qw(cJ%KGzp4rJA9Bu4ljo4i+Nw z3~M#f?-{j{YgbPrQl}r5Jb0`-47|r8V=o=;$7XI15&~Yfp!1%HiF-eO|C(2Hx)+&$ z`{)ZkRMG`)SN(OugZ>D?fGwNX;nRjM`qDuxG|_LpjPfZ3zkd515>6nZHEr;`q1%u} z$$((LuX-_A3z`sPL?`_>xB!{?R>*P%9n-g91AWI%iUn0^Lj0 z&P+|s#B6#)M=-?mWB0@Bo^jNN%jnxWiuP8q4D@hF-i}Izo^nE|AYO?vbQaU!l9ab$ zL}ZIAa20MO4>8yEH;AX4`z6!#w^q18yv{*C4x4_=6_N(u92`|?b%jsD`Ca4|Nxu!V zB0>sdqJ=L9I^@421m;=!4sZWuDu+~L$ghWKLi40V-_QL^0zbf4b%C|GEOpZ1mOC+U z+K`=11P?zz8M6s8J~)zF`-V%W+RK7IVE=L+vPqo2cGNydpi3YeE{NL~aBf^W#w@3b z_Wm|C8QS>nOlcbN7MnPg1R`JLcG4QK(B9-~MvszhlG52ZjRM&mbbu%BUu;}>OzC0C z@@Odh!6JA_cJ!Im88{=4%I8Sb=c=G>#P)sSVEgtoUFH)=(bYHqgJj(D9h~!OoY{mX zISi9vnhsJTI{(bkv=2L3(OQ>{x-`8}ggeedoTQ7OQk`i|Nrz3G^FTGMNEZpmrpN^qbr1AtOShGj&V?Y1xZTxN{++qb10cW?D< zdKNDSZJzKmaZRVaTzm1I;?a$wC|BY5ocA;$z6Lb0^BExaPBmXxY*hI&xvw*O7ILQQD?!P}8Ta7P{?_SA6l2{G+57JZZm&Jhhf{unwk@~r12YHy?-bbC5TW8-s>H&F z^V4;v!QsDHRH21Q_}8P`wS~`ABTR>CQwG=Qq&#RN=16+Y1p~<%(r7N!-xQ{gTj7tUxq^?%s?r^nVeykOcB+(BE;EN$bVId|2 zZlOyjvdiGNN^dwYX`X&D#!!WEVkSca3^rmaO1>h$(-B|}g?|Gpc?*jy>}k+;l`!D& zzefSjJf~)bTvE;d9UYgnkKX+=@3Q%ad|wT$ru#Fi*2ke+I+4CEmcg~2JxzjY2;DdF zCkp8etd^=ERQcpD`ah*44ZjA48=;b>dsNCUTFtx7KyMMY@^^lY|No!Cl^0k_C*oN7 zodymTg0NY(*A+n0s1SI4^>pz>ZV8-nJRJ@ z$#kNJ+Qz|pJcqwfG%jx$avnuT7@gZY0d7lZY&bKxC3FQbbcQ`kzoc5e^H~t=xJVjG^h%|Zox?%a@T_xJkUg!Qx`%!@ ztlM#4Mtkc|5rO`yf}~t`mBw$6X@1GrYWrUYxN2mvB@313zE{l#@W%@R`s6Uobk!Lu z)Qm~!`SY+)^Cf+zEDIIlvVy8y>)-1>pPrwuAyFsnH`fnmqGdHm!`649-s<4jfUVjv z|I?mLR8&eha*lAT5xS0tF3BO5kcfj`n>oF!wU8*kH}hx5Uz1_B1~=ZB&$@|XJwc@A zYmUVCRcWDUz!g_yp8aOdsQ6%P_;?-{0TjPiSz9g6m{J6;3kQQ|u10-61W?KE{*}){ zZ-rHhX2MNkLYNx`#2v$ex=oYf)zILq#gNyFacgAcB&C}J7*cujy+NYw{eCM%D30CW z#cs+8Nz7?-2wM!UizLPBUTkEOaEBA{c{V8i5ZdoF>#Vo%d89wKS{#$$aq(ig+COPk zV{(u%gTiyiDNMhdi-`TdyVDDj_uQb%B%Xp8-8HXyW#il{8zt_NUW~XzT>2|!$ie1= zEG8-?+U3VZP0=V+bq-Xq=y3E^?5PIWGJBHWc{J8HTWU(*+`ekS(aX3J%ac}!89&pA zxuPN>QQl?e($bX01`3P8VW_)7LHv=Tc=}6^TT*+f88tNzn{H16tHh-&znFi@ng=$* z?Tan`O>St3Mpgz9nrvBKRa&y5;{h#=`LmEKAd8w!R1|1S_z{>Z2^*HQe6AyW}bEv_D;m8^_cYy$?D z7wt<&y3KDZqZf!zF4c4edymAIh3_lEXLTWe1aJM`UI!;agdYE( zr}mAE_Xlhlx`M5w5RUP4-)^zQ9vLpL3^!&{q2X^jG}k=VlIhbW+7H%Z5l3IwKC<7G zAR-DZbPC5D_BB^qmeg|HJiOt|=Jq|p9vrP6m)Fs>^+#PJis?oDrosP#`-Q((v@zdL zT2cuU@w=D~k^nCe?B(&#rP#Pn60XEhhj-hswv`Mrx6F3 zb)$U`ihcL_0`g|df7{T7lIW}ETf zn}j9~ON)<@5|HSlQ8dkONj^$Y)KH4`{9qNWs_peuYw&F*rJwQMFEnXZ<ZBqLdQvCQRP6#m!p5EuP1SA&rW7^6hD`Z6kXkIc~EzvNV4>30wbRUePnxgDt1i z0xx;`kUI!jxk2USsJ6KDgrolU8pT-XhW<@?MZS+M{`1*dsO~Ffoc+1!PO(t;Lx*Ua zrqNcTj{}L2`0vXhp@(IY{;`%r_O)hf-n`Dx;($klyFabXuWyPqzR_P@D6(!YgjJac zujz4NAO#t%qjisDVU|wNIkm(tVQ?Ef1-Kk^Lk}+F)8TF#2sPKBF-<=$z4lcvJINUB zWuesE9?@D_#9jAI28ah8;&1!;8Oum?5m9uT5MKb4NIm4Ac#p(Tb zyifC#zIRn>1^B0J534yf`HV3cjkZQJG;n4$NqKcd*cE!x-&OVw?l!MeG)AT?+@W57 zSgAuq*ARcf96%M@;Krd0%q35?9pa?V`@ItBpTonzT_;j{20>`;Bd9q(Dv@$WL=o@0g&LLvXMeEIu#mKfZ7$<(0c)7%!ohLp9 zCeLY!GTUKO`sl`g8$_&})*7M6L=?t1DeX}|Al00``F4kg4|!<&{89{i`a~K~{jKHz z-vq~DYeGkzjcl!lUGXI|?}Ck}|^Y9kQP zq9xwrqpu%=UA%fWc>Ph=A!q-xYoRH?p4BlxM)cfITrxZ0v>J3D94YVZm;GM?Q+=8K!ds)!W0v4uN%j z&C!1AlBYPtz9TcUz@yDwLu=FCnO*d}$U!MOsbXLqVW;V!6FZ3CQz&0sfB$|m%g#1rK8cTbGqcblbgr2rVPfm$@>-sT(j@Hh!2b zDGK&-5kF!No?JJ%=Y@RRXGo6bDk=LcmMpHSM?v`oGDTh_26rjTzC!(zN=k*9+#>SX z^8?uPrCC&peZlD(9pnA}u+rDWPe>R zO!7MzHnWdQZa7|*3g8|3H97h3uz|UOCy}H2Z?EMadfl!_LUD}1+!JlY-9xJ#~9ZFC9 zXJ!*w(#yjr%Ec@s8P{4HJjj-caXlnDl1Y1*C@Z4)4)+ssa@{+z;#GjqBl{q|V*N2l zXg_YR)c0mU>*Qska_u#NT-R^MTw>MrDuqFU`x>R=?vyavpdKdwhE>q*9LKBWgdugT zjkHD$l?9W~rJG-3-It4p%a$BW4$69OG5X8{S`%N$lkTJcWrc<%Il(>StM>A6jG{uN zXyJdP)$ttS;Fu`Qkj5Kku9$rHW^_U>{uzdwduNU5glUa#iCizNrx#MU+NY zZ@gpaTf_Z2`$GwzZV+x1*l)g{tYzhEZH|9GA-97es{ijtLWtFxK(|7&A*%4#MGj?& zXp29d@#jJ}Ut+C77)I^4TMfL!^JfUEvdk&{wj|WSnR_-MyFCv^lP0fq2HVp7;V;&Y zkBgn%`kPS6`(6*zI2}E2-;iaCV<|n3oH?2PTpe_-kCIPvxU3^(47+Kk@mnB&R|x}@0rLF_gDNtn8$W*GYGG;LIeMNW|2OkvL z>e+)OJq#xA@1eAsy`WvNt4*YjoCMvjEBTMp2C^0>DNazx+hY3c{|of~gV#}rA4aN4 zU@apf{Z1^g*IH8%NyT=NSnFUyEk*7cF1NW*rWPuH(2&!<&dn`M5jLV)+5Po2;Ddy? zKtlYjt&1r)Vs`UT>wDpHOnCz7s<9>f4PsB!S-kMWOWGM2@d}F%r&m4haT+ouL6jNbX`3(z%y%NI}BH zjo&^M$N9h>=)>>_JxFrnB?&~bda*w&X}kr^rPVvBF2<*C7ne3>Pq<_8EQxr9#JmYY zhk0bRl z21v<@9?mB-}iBx z^j)Y`-cQNH+lde#f1vv6iPjXM)4oE4b=(A4M#AR7(JqOr=?T0>CVi7-9E54P{N&aK zsjrL7^~l4sD<`{Wv)LYbm@r$)$Nq78QCdw6A@%hS2N!CUop92t>pjf>;pr`-;%K_I z?cl*RxP;*DZoz}QLvVKp?(Xg$Ah^rm?oM#`V8LAm`8wDAJ?|gZf@ylHs!MinIZl=+ zJXzwypU)#o9ra?bZ$Y@q%A) zD!N0B7tTI5*+6!EuuLkcHC$le{v~@VK}Uw*RQE%}@mCY3@Hb&4;g7dUYD&HS2tGNT zN7!6@lrg032K?{$8)aCs;{$Vf)3->NDZ}#Utk$YTyy9M|q2!%2rI^8`?U(5ZcD^07 zn2ZO+wjU_bJ>gm}TCsBx1GLas=7q}@<_jZySKBkotgOGLyx!izjxa0Z=4FX{ZsH1p ztLzgv9WiES$kM^EWxR8;zbi2A=N&o0=X)0v!4TbXiA(!CBO+5BPeSIWS~S zqj+o)wt4T^u{6$7)+8pY*}4)3#n&Mha^&VloY_+Y#n!l2%2M{`4zmC6gl4Vu-X3E5 zP~eXq&HQ^;I9|kJsC7W{11@YRHQ7_Ag*N|qbbui)Zejp+ATCoHSpc<|MqJ0+=Nt~n zr!$3z&qcp}y7P{1I3Tw%et|6b82FhC;tK{OgeO05Mj`7*S68x$NlMZ1?FIJeZ_vH; z2AnHLf?Zh$bb_NL&e|TP<_!7a(F=rn`-$|}f3`n#{Uk|ZLoOllo-BH~te{G7dKMG} zXT)8BaR6db+3w2LTcFvXtov8fnH*GB9)iQ{#--*8GXl4m(Dw&fF(`7SK9i(lf!8lvFMaZ<>vzl~uSa*@?|#HZ<0Ijw z5rR}2!q_vWjbEJe z@;K!Rt1Z#Kx+~Ltc#^}Fm~X)jr~H74jIM-+7LC?P98g)Q>V1~8Y!*$ahPboff+W{N zjyO^-yH=-h)P9H}NW)e4S|evMGGYYnHxhS*8z6%;yRj%Oy$SN(?}{OOy;@r7*Y0wA z#}(=N;3UCm*E>DK2&3Ui*=A=D{w2}kbEJ`K7^Gp_3ChiYf*UFC*<(ALmgl?lte*Pw zs+UyfV#o%o`=~>R#Y>%poWjSFt08aVP>2HhpZAiu^P%Kz?5apgl-f6>tGT7#1&&Wp zk)b8|$y@J#YuqOlF#f7(x{p%{qZKkKiA38X1_l@e<6Xi^;qhSjIQYjWOBW+U`ZARp zd`7@fPYYS>594m%<>aiX{q1wAq@9eVOT!C6hM!kG>=|7wi6+KINg9m-ziU~w+e+mp zQJ5Wxd&zu!;AdWY#o-H{)5GPY3oQP^Q-zDOaQuI7s({B2j&-0u2jS5C+NYcO2h_|L zOdEcg;~H?Cwpet&r78qc!KmE}1S}i}xmJ(j*CuXfneyN~tll4MG&ea;#4uw=vr3fu#hqR(p0SFppj_ zqdTwFn{+e8yHlsWDj>^+MRB*B3RidEF=_`vZYE7P+<>5TU$>F@uSp6&#&#wWN=i*A zb}Ab9lYhB=dOF`TA8kTRqn9%l`|)=T>68*eoC|G|lMn^IR}p=ZQ`rU`S`?}wX95ZF zj;W(BgN}H+IBrWcj&`>8UA$ZkJ!~%ar{%|bn6Q5QHO{0=yFTX^yx2;QbS9sZtXp16 zgX%R&%g&)mJjO2tv6L&d$crlKd&Ya^SibI+oS6%&ACS<6XYqJUp&tB5vrb{0OSkUG z%FBt6Y&x^z1Rg!KWt~T+A>}^RauY63vWCK9XCb%GiI+AdYsfR1J}tJ7o+zPZK3t_q zLwQAFMpf>h^0Zi;sGFI};Fg)KuJL_5VJ)!8rcxEfL_ed2@**iIFrq3by&Bs80YwfE zYCz|p|01olH)-O$t!T=QJi!Uq6ppnf-)oD>(_~Gh;pa;Bu+D>`hosWs1J_0Y6DHNe zj6@%cWmE5OhMQi+xNeuz-|kxXcEfcAvZwD6@ZjZme^i*tL4^lZO2gVkNYYQTBvs;suD=~f7L8xq&DIbRpeq`$>FBQ2(31(FEMET zcGMPYB1J0DFS}5|q{Icrwrf2@>7ZoWpB4zSsE;>dGd$v@tRQM9HMZcbUyzb$%V_ff zmvtZb3cjA%16de9yI-_ud0ncf9LVr!Gvh-!6vd@LLw=QY*Fkw2q`$a@B++Y! z-Vz+<{{N1##PR4x+qEY9x4ji}pd=JmG}i!=!cq(!h(=ahrm@Fvwa*YviVzTg+_^wR zyM%_nMfDhbLxsH+o5&`IhxW`WD-)3UT(EOtUEEe3VjwoSG$-aGO6NlaQk{1(t{;2! znwtH+F6HLM1_H?;h5qRCsZw14Z6lrpgEPy@FHC+G=BjX186;Q=w1we(uYn|$m9b^0 z+6cn+4BpKBP*^fS$>5!dN4&qAqFHfz;5Z$`kn(jx+4cMhx z)JDk>@Ex764XtvV95I9irvTS!OZssXsQ)$H_$zmd^rOyb#j4ycbq5*M;dSzXwN&?+ ztJ>(b$yWq%f78}sn}+by4~Gq+2Ng9ucG$DFaklp=`WoIbpbr*hbF0r`JcX_Tkq%G*^47GA-3kjpZ5U zc~At$311lnde8=eX1gzeAqGo}4-^5X+ibfj2OSqOnhA%fp3ZVhQBv%k^-66~mQ}&g zR(&prNMIgAX6rz$@`Ui$GAOp_#M6N-^6p_{U=r1ivh2)YvfSksWkHehS38x-4vXZT ztI&Y0pH^HFYOVSnh3R6Iro4h2dK&DJCMX>ayM|)w`IezOj^O;6&nHNrQoAiVSJ61K z^M}p)#Pb53*3yiq34PJ#WTjqrO2C=5R#!11*NM{kM&D2@aT`~a;Zw07zmVC=Mzh1X z&-k)Zq^tgKgd@hmAITU^@|Vp;$-ng)EK90j&a6Utx$0f})yE=?+`(pxHmAP``xOs8 z7j2G>Tig)Gm+VBJ@T{QLj+g60VkZeS8@hswFxb}DdRJX^sI!9et0yZr<_8V*sm3*I zdLAKs=-&J8+!*sUqn6w3KIwqRaVw}%T?@^!C$2(FUv7s!wP-PQem%61-Z(zgtzJ0V zYzaly0mD1y^Y;ARw=Vk&aNb3ShfsK%E6tu(89SmSTWgZ;2%4Ey*(_5-&_u$eM)89n zbe9HkMX|&&bN95ZuY80i6Il;U-w1#7Cvd1)33;-Xv$9(7E6ifP;NEFju0fqmZnQKoAlP|CsTifHq$VTG&6Pd_MXM zCiF7*Mm4k?rT%9JBv2;l2n<@0%K78Utt&(5XUw0w_=83fVqh!;4}RNHbt?;@9%o21 z;%AY~$LEF>*SURNp9jxCkUTQ;9I_fGzyhDHHt2cN*Xqs*6^j-{Xx=6MT-X0GZGi}=_*kJ< zNf3XFl#>a)^(vjUEF0^m z5UCjblzC~BvSaj#$m9|QUC51n4@p{Bw*s9mF8bnExA|GZ6=8vPh#1(!8wWEL zk5P?<+XT@=mjIJMj&cuOqhg@k*9N!s%&-ule-J?(4lDDTo3j9asO7tV8hpC*rKPU> z4qoNc{?g-<=<6RSE;$ffqEeo-W?5if@6s_ERARhQotO=~Fr%d1yf51rMb>!I)%>3K zMKGEOe@XJtw27ip2y_^?6u`-{=(M)gG)8*zCSG!M^{QtGb}y#skcA!orBW^-CE^jd zWkZ$=l=zvxy+F-vKNJAGm!_sB_#pjZ$B}&}vAb=z0s~hCm~Mp>Wy^S@O7Vq~;yTQA z`Q~Y(m|ohDYN4==^-Z2)j!py%sYARb!;lsih#J8Q`qQ@dmYs7@>C59-uhoS2z~bt& zEaTl({^BpH9_h1Fn0ihhJptOuR^Kh+O+$s<&I?&Tg=V#Uj&x7OqJ?w2I3GW0>*NS{ zTLSZrw}n%#M{2*8GYDzgC_YxF`YXKi=-s*3$@J+D^kybg+Bz4=OFerRGL)K2BX3(k zvF6og-pfP0(pnAi;eGmrVdw8O`0?d3$;-Bz?<3n}%?>B8L@gddM2xl0g|k=v{&ldw zPqRBmW1rR~m;gg}xT_=w>^pzRcYSsGbn&+udiU!7!e6Zmil&hJnacRV#gVBsg45h->skscsqVc#>zCVXqi$x^l>XT9y zCXiG~jyGof!7KLrDMlepotD>f_&}rKwW;^vrNe+>5!oUy70UYCFQ*gC`WAMDj9AJI z{pmUTsa#Wr7QABF=^&}UG!MJSbw718OQyt(T~#dgK9UXylF51Xje}T?t=Lj_NR!pM z<$1}%8!JU*S*45EYM^1vWbKF5L&HH574QhqLa};*TM`Cq-!=$Z^K$lNBugFyE;MB@ zhNth;PsQmeBiu4}CHUp81-xNL+|)jj<}y{*9p5@@7wX|mJ1WLT?a9Y#eGNe~*0QdZ z3Y5qv!Xbp)#nLq^b4dXGwD<+T0bEZV&q7-qK z6zQk;_jq!zf^vRZJ1#W>jsNRCOU81k2#YE;Oo}oML$M0+P=SLU4yJe<*xMLs)$uZDTWi&b zH22U(tS)^@5KBYaKk%2B$=p^YGRZsF8gt_PdU}a6O$dt_xdq1%K*``%=JfmlI-I## z0?LLI;C+Ep7I^Hx669UF!C@zO@tAVX(3S_rRqBP>Rk636@T9rI3O8JQt9!9eL^%io z=%B2%|N1Q=E3M!#`)o3oK`=hbQ;RlTD{z|yg9{%w90#_2;RxKBQTZ&ef%EBq7X!~; z4u82Q%e??n6B*+V!vCIY@ba3)GT!MqXQ#*fD{H96zw!LnnoYBb((S9xPK=}fcOlNH z^4U{Fq%NNEEW&imRnKu>J8~+8KlXpO_X0}XCKRR|L(=(kPk*l`Y(HEn=SjxF+d+?F zcE&?@|96p*{eu`@yn?ItC(?n2y-_oUc1^@~cx?iq)8gXdAEq(;$@s@tFhjR*@{Pp* zYsWoo?4BZH8o+|Ygl!qZb!sJS)r}>W$_-+`5{bSS>t31B{Yx`Z7?XRbK@%c{(Z`(a zka0C0N7Y;NuNT3~1e(@Y-tX`#)G5lrAbcgPQCVsc{xJORm8+}M^On5}+#4w3g*n8Z zjS{_#$1~T3IRCX197yH$eGM0Z`mk4ozd3iz-a)pP&iZWR6aB(B+yC_!xbEJG@t3h{ z5;FHnPGpjfD-_3v@1139=>=OIf2rxhzHUOstD44Cf zshj#h1A~%X0i`51M*-+FZUba9t~f-X<6c}LuI#u$8MU18eU9vlWlsU(s_EMkEte zx7oW(XhQZ@s%Z9`rnPp3E&X+nFqx$XAYa=*Y3n z7yXvb6a3))cQ4!R2j*aA?+t&pV}%y0d6cA^|hK7*o9^ zBFl$73xWY5vtQdOpj4o?(`+&mDChg(>dMN_&R+jBWTW`&cnS*Xz)h~}Pu`DFQ-?An zK3q31zOZC5DB!0#rmkTQKkZ16))=4XFObp`ee6ri9Z%k^wzxMlDPLW=YfCYtOy>;Zr;U4 z@wm)j)uw7sJNY^eep;`qC)sk_cL$!$E?yy#xGDASPn_< zF6e1pK=qM_1%VV&)xmC{$DOYn63&N&sGAbH>v}kTCqUVX#AuZu<$;A9Fn#CwT$;7 zjgW&TBHM%Sz{Y-vao+m3`)N-iPn@bNI*JJlS>^3t1JL(+yBL0?&<4XdGI1wxb^L)| zT@PoFc^_kl@}fq@DI2u3zj|q4+O6c#d%22xU^e0JDaiIOK7g!Vr2Znpk(QR@$n7K3 z)pvcQD0meS=UCnlOxOKN)@J1)&;O1gc-b1K?C}8CS~48(h;`7qsqk+@(c=_`V~{mu z(*WcuDyujg*TM-h>X21c&6@7)WoO#rD52c`tQ4Z@#dm`JFq3niGk6b;KAQIn2vUo64`t zEPH>HoR)Lnj@9w7m~>rF<;i|Yoa4U!dF;N=@bZ+(r#YuLNS=BRLv^XZ#GA{>WU>L} zc2z<_FfzrH>AdBM?;MCBbKaE!)_B-{Tw!0pzpR&IhY=)8NlKQnEUNG8;5mrqZi6;b8MHxhg*vT@8j!j^lht-HE<>%AtST!^UD?~3v;#oal<@a zRic>##!;D)w$66nn7{tiEMxde}S?{Abc{h}UsT8MMWY*INGA|J1i8vJP<>Wv1KQ~4dJ zgdSh0T3ie`Yq-JeP9Vq{zjwduNHXnOxQ@s;zejy;W|D*~-;cw@yh+@w?zsiZNIWkw z7yJEbU0C*|SqLcFMUIk2WR)~?t#&M2ug}9Te?z@+KO4;Oq+OSxzVi)Fxqd`(ry+To z0CSsViXMTS24=z%Dn42A-N$%e*CS>WgkGWY2ci1DHDtT32TbrBSzyb;JFH<}#m4!B z&zVzUy+XC#){nbiK}-hkOnDM64(uUlXeSO}EYO;u`Z18SpJE&wesN;Mc0A2z`)JM5 zfGUt0q>W7B2XWDI%KF{$N2>I_*xQZq`;?&^-*N&^Y!s6Z!`v~24R7v)->wjMK0`sG zQ09a8r2wH2lMKje=$B1Y$&_vMlbW;=W7_fQ=>V=MZYY`aJ790PBP0(Y)|+IpaxbJ$ znT~&x-;67cR;cTa>dL2e*IF%URPixjSbBYdvpHkJfOB>zFzg6n+VB4EiZx#I!qjy= z7k0IV&|(}rh){(g4Wj(+dX0F=_tuxkwWR5Ak43xiozqUnB5PJm7(DNKTUS+)Ci<|3 zI!y01QNp?gwaZ?M9$QnW*Tq@?{T$G*p$1Dim$_v#vt#kUuY@?i6B%6s z{SmsUj^M#K`jRIY{q5T_it3JML3QRHn!~QR8-UZjj~F$4X9m=1QV4VIGd%ADr1Cf?ys+@~JcLM^QBM4c0Y%=2 z#w6Q8d0^6woA3SRjP25jQPZ$E*0W>#3$Q8EKa*3a@ZAzAKjWyaxD` zWkwtHD#a5f!($w$;q+Qtz0_eMYe?(m{e76zJ9kV1*|raFnOP(!{db*>V{8V=gRA6c zub@zB+1iLf#Oq|@H=exbMHhPaJ*w8faQeG>@|&m|p8OZ!W%I}8sHy$nLYxFtBL^s2 zYrCD$l+38^TRq12YjLp(>;^OI2?)3*j3GP}p8b@L=jeV-uM>_JQ#qmU+fv(L_Lz*n zBV`)GH~(a=hv>OICu*R(L_XFLEXEJ8IAWlOh91M^OoZ5Tr*b@OO8T*g(23&eKFtY z&=CHuI%tyQ+{Mm3(tN>v-Q=I*xe#X`BwhHuQ#+^M=ZNS-{I({Yd?;N=^UqE(uOf{# zPUL=dJ1@hzZtHL0f4Ri)w1LeD9)Nd;pzq$1-h4`JcGw?V(e=DKS)MNYdB)7;3+{i} zy7th$&u^Y1Mp0(LubREae_AY)wGK$}d;XNB@4~kt8f)}MAGeb@p4|#&E20=0pZN0= zN`TXzE8De~V5GVwHLmNT@a2Bm`tzP>m8||2XECuW?py<7z6*_eYVW4)fT#Zn<2yUa z<233`QO!FOmrw^$m7XF3(E{%2S^n$q!t{k?ib@JuzQ4F;;NMFE{(zt9e)!^M3ZQ3K zEKcp^F}cH(bUmtmuSN5H*?G7sqO<$SBpbx6LS)jSo$tKiWY_*&;eDjCWoqd*@YM}n zZ*9oF>&{T{vN!HJrN%doPyNn758gMA(e3s^w^I-c`BSc_n!w78rt{2e^5lPWYUhF`3OrtMa&)a8QmoE5q3Uk>4(# zkYFUL-YtAJzYCO)VJg&toZ*6f_b7)9?8sA>ZU+`{&PCU&@3av!z~%M6zt=G_$LI6~ zw%S%czoI@qO^#n8P|eH=hSU>a5(1^O;O(fDdP7W=2Mv1t7hE4K3wt}$lUT{t7lN(* zd42DI=`J^hG@l2%UD>ai_dFL+c?ZbDCM&aQMh9;y-=#WkFt@(6 zZ{r?3tWUl(exHnnT5{Z}VBC({`u>gm5W3@6UXw^)klRaU*!MTXuIp~+75x3e{xyy#O-ATOC|ldo%$0Loafr0~X(E>^ zkIXDvy6TL2eN6rQal>BI`xUkIHG29gg3h-uQBB+ogmiO4*FCMTe*QVjnML{bFG=y6 z!-mWKKZoMt;^HlWT;k7Ud>YsNHb&s3w^n`4mifh9xIWtga0uWe2a*RbK4m-g&B)r{ z*GYZbYT}^xu%O{Hf;%(QbXYGt)zBUiuny(<^Jk9dhbv<7@~G&;Ta1ejVETQR&A(AP z_i79}cpARiO)749yce3&l2=?rYa|)4q>|m60jIIur0;!OWIC9M33|kXtISttnc#ha zj@4pM>xeyCX^e(Tt)!S2tR};W4nf0Z0z9=-EKFHbdo=90UGm@+3X{XF-VP?qrQ`_;$*CI^EP ziCgzE<-$SY$^PZ<0m=ablIHVLcYTMk|4$c`;XGej=U38H{i-(X|`(m53V zz@%8dXl#Lr$gp$cm1pAYdyj1QYpDrqMC$v>{d_QSVSDCJkcg$YYFR>7Yy6%9cOsrg z*c(HI`(>{r9xJi%jg36-{$s+!?e;5btq1jXZIb-KQIOeO!?it7A#91SDe3;Z!hP;| zPV>Hp3~2~2*5{5Qo70QWG;_8*tWqz#``Wq+XPaglTk{LWIZAeT7QrZ`% z5PGFD=nVc=rJIN17^euQB8kAh8|Oba5N(3;Qh@j3TV@5j}X<@aKky(6m-ctWNU_p@$u@ zK0$dAMv+x`8biDzMrbI9{mUt8%b%ZB-Zy;RhGAS`4O=hjW>rt{dF?M|V|HIn-CMel zwGf`@r>gab4?6`gUO3BzbgUf-!)!QZ$p5*L_*se^8iSN$_Vl$c3mvKFSA`8{Vb>gg z-x*hNz#7kiYhjJ)rL7y4)5%xxT4-Ym)wJUktxJXMCnzmS1V07B==i;wJL3Fvt4RlL z35e3fxU!c!?zqV6{Q50l_zF2ZL*^L90>4O0C}}JgSQo=NX_S~_t5xV@qpI07q*$dD z+r5o}Qp4W|@@vWaMFZ#PBV8b|7A?0Xn}S{3*i^@uBe-axa$;-Sre`)`9p@2;$xw(D zOZ=X@ymCMgRAnAT5%Sb=tyc5r2&%-ek5wmav)`q8Q`GMdTFWsnu|2_n3aal$g!=DC z4M+{Z`i`umklNi4w2`g9aZNNB63RXGUdI<@J-=$)R(mz8A1}NUPt?n!`WHEospKHP zdc?N-Gs9x5S1+SuHoVTI1UQSpxX_W^@3#c&6|7j9tfkEOox<(5S#vHKS8nxLl&Wt9su;Whzi=q&!s&q-G zu@ee)@FUzQu+(V8|G8t!3@R!9^Cd*=rSI$*egJIV>Z~r^GSxnO(Jr8zz)Y`9RZ%)& z)T9&;Jy8nQXX4!k3kC)8)lw31=S`Z8ny|7mYW@2{Bc|eHBYF~E(E~gWvV045Ut+dryxSm6jtU?d{dMGeF0%Oq5&mI5>ZOb%jLqjtIK|we{7zb zr;;Ve)b5vkCIEk69K8cmk}blRw3F*~-s?!1tQ>4Oq9jQqhZRDv*OvUxP`BUY=d@Y= z=H*48(`2{5P~(Bzb-2f!`H#-+xj?JCMC5##RQEr}WzyWEGp8{UfSfPa=!~4q;?rz( zf)Pe@xT1IuJZv3Mny6~t_y;zi%69)EN$5yku~fNx@gNkhK3qePXUIJpq^nOUm6%{7 zVm^NZnCDDh^k*xMgaGwjxC;AvvAHXSD2UH}#iqJ{tabUdnqQaKRR@mpY zmXeVN$LWmIwN@tI#TZhX!mv@xDxQUEG}w=p$9w^hE+watr@bE zq9E-eO?vsUSDIk=1A(*cCGDs~4}c1p`2%OjDwb^WpkS=09_>c+AJF|vqEqvKMDV^^Rwv_TaxK7z=!Lmtj%oTu{!e~ z+ncz%>Q(BJ=gx#-LK`kP)rW?%fl?USYpdZ_fNo7mcX!;B?V9@>jJ**cX|e9RR@K-H za(>*Ato2@D?PE3pm^->o$K99zCNlu>qXyV}n20#U0~-Lt^M2~A!#qU~mYBt3lz;En;@Ih+0F8z7;ipnyuiX|ug?;ft!%Y(ttiUI;4I1k$a5 zpp0Bg^_)CkOm#IQFE3}(q}fpPyM>*d;f^re%=&t?;VA^D4xlabQ&BMv;j(5R0dZZM zjW(CGAE%=x0C?19t?8_FGa8-)|9Pdsd~Rt;5)f^02z)wJP9$gN;7FK01Uwhn$;cMQ zA{TgUX0@i{sZ!0T1ko&~&0D?zA4qNtvx8j9!qT$$!3UtlP@S2dfdy5HWerajtCBc# zrnScbhA8D(9hDSDv1|t9S9|-5Z!wXhfG>nGtq_k9V?nh&K(;R zf$8gZ1|s}%*{l0wX6>o(P;gkwJZCULl6Z1EmNYXfv-KR+&tUTJg0a;U5M9u}*X*re zMtWM{SYp25u5r=>V-{(5u*I@VW^d0X%2Zse3Z zvwp7{f)5$ffMkXv;hqO^g?iO)n8urb{>~Snw!iP1KSoH`{kDAmIJGA9cse*-@>mW4 zT@&x3ype+UKlA|H@q$ZPEcrlyX!_>p_&8s?O1`a!K?*~<2t!UzE=llFO?qZ!g-2Q3 znN>+TfgHZ;Vh;HAIB;tyB;5)mjRu|c+<+4J!59f(;w9bp>5zTf>hb~qTI(8eESXW^ z(PH|6Ydmszqgt^b1@?iKU^V9D z9gj{f`td~>vx*7}p=vf^wT(3aNeiK8r39Q@Pg?W58`kRQos^_|O<)T#JFAmfW8%M4 z)z;Py?~7k12Y7M;P^h{+0C!5u{dcd8W^=BsPV+8U5Jv zxX)CgUgmIQ80L7f^(LWVPKBfEKx*%?CQj&lTBqyxdUxDbIXEMq`N;w)cJKGrkt7rzKA60{z5Ttm z)_>N#1t`6>0fwiE|;1PQ4@-zK5Y{&-&G@`yG>wDFIN-VzG+J+iNcz z%bh<+xy&GjG;7=u&?~w;DvUFV-Z{{*`x7sUvR_PKZ*rE&K{jW2fBVC^A9)EA8UoM; z+bPbn`>ao0Yj>vY+1Bnzm{xp18D01CSWY4UXV(8K{cX*O( zBb);k@!y*KK>%2dU!;W@8apm8&9tvG3C>AApIFZ)9l-$6(yIxJY$!X;ZsH;=VBIY& z43hZ1gA?Ze>N3LhMDEjRzWg>~&6xNWpt9tG2b}4ni4=dFc%H#vfNuUJYjPI&hbG%K z2g0y_d&LJ}qvP@O?!wLceWi-YKjbI=hps_k+1PUWNhCm|ld73zUlGk@#AG&T?uex; z=Eua$Ls4h9R!=PSD6kP24<$D?HO0~kc?xpE(B|GaOlSpk+GK6HxyJPgv!`0 z{&^FlC14Hqnhrf0L7VVUBH$md-Qj@WuK zc%1#s7K;;bI+uvhgd2^Y(m+hY#4!gbbb7!p{C3*tgu-&t4g})F?kF)NeVj8qTVjXF zG5f=)I=Fs1Q=|`0g-}W=?=pXTs~{`p)4$Gp8XzF7of;wFpU@>EE~;tj8$yL#f2spn z6e%K*!!VougGA9GY?IRI)cGmt;*dyXZC$`e zRaGG#hfiX#C0jqinDhR^7vCIUk=jg@J_XS(L77z55^tB*_13yiNdad5@}Xr>>7YXo zdyHk)C{z&Mo5Q4M@yiKe*WHv>$IV)5H$Ue$74;<9^}2sKC*kH}N4Gl?(oQy{<#6pP zvpt*{<#m&pzcxrWrw3(k6xEUhH*VCPp`BG+FUh^7X;FKk40^y zOi}ZmjRt+ms|A?8JCZ#%p_5%ioasMn7ny^6?@LbaW2D!3lXk2p|6W?^s5k<9P0Ud% z?q)x8u5OX>)Az_a*20LhPWCo(TObA>*V~!6!KPq`8<;R(>~$U_84kZ9S}N-4hq=st zDq9v&Pa?!eBPpw8RAfKOg%*h4PDb7wLvynYj-0dAFgpa>L@QuVqx{P;0kgx9!)iNa z(#&OPS($BfMPD**60~uQ}=g3{eRe4VKss0>pnCe^!-!u zo{e0CDF57Gqjdop+*zO?$XoS$zELgZ84*P=3pxPv+bHz1pekQpcW*)0kJSmoCvjL@ zDm*PM`%GDmyWt))^B&t)wX1|k8!|>c(JPoD3RMYVFB;y!B4Qs7uf90@aJJx8m*ZGN zb&vc1k7RI!k+xuOp7RGEkpJJR?l0UAkO<~E3NKM~%j;Q}GkN#d3sEJGPwnP6HCyXr znYN~Z5%^j_Yeo2ZBH0b(zN*b(nPw>k!-2)9Ov24u$riJ%Kn{N`QtX$WNbIdjP}&L*)d7J+@J z0&Y51TA!+{M1U$9xobG?jADZ_Wxg5l+W}ZC3Ub~+WI3l_Ppe;FS*Aid?PM?5heG(0 zm=bVs7;P2zXbyI`!pt4ili3wtr)4>oO#V7Py}|1OQ#x(vtSlpwJV`O>ce&H1b$fB9 zd0P{lG^5N=Lev6N_-cM(`1-G*=xk*K>U{)`r>gwK*YAZL?6^P13Y9a4n_KsS1d$&t ze_DY=)G&VJ^{ymfrz-bTz7nvO?m+9wLs^yB%;x^N{k09~G+&%vfr!cF@aImyHb_U= zaadDZL>GRpsh!i6Z3y~;oEdRiZR}2ka3C60t$m3#3PC)|pEica^TI*en@HQn*1Lgi z?1LK#q$}7bwSts^ z;3lnN>4v1x;v%2@r_$S|Sm6gPReRk$9d1IYQO@eYMs9L8qEpqhUs4GZjuD5Rzbgr> zr7L8q?lyOZ1g&0pj={DQB&RKrkKz80k4T;}!3|%&fDJyGTO)n%97&ZxFcCohUym1Q zS$Pegvt8S=GiogCbWvC!>%B`+-gqM25w@x^Z_ zgYE+G=ZUoPl7X?pDHy7Q1tN;o@)dOLD2=5o6>3c^Bk5H1A94hRW3-EXbpo95gWQK1 zs~L_m2lcWH(j>QOfn&oNAxUk3jVLk5J28XNH8N=Qm_4Um{epB+misOt(*L(nNn5bF zC;2cmx3y?sJ6+2Le7Vh>54rHAkQi`84hT*{21ut1xFN`FtkmAA+hqTIK^hd=l6{+Z zw%~by1S*u`2jJy+!orrqy#JI4c<)r5dGDX`2wn!+_2FI>f}w(dWpM4;wv{n(3|B&%bPkUjF(}~+2Dkq{ zDDLd6Bg&fMZ`0t0CDO7W^UvK|H?542IKWyp znC~&9+Ga6mw-pS+K5HWv<-={e!e3?dWLoKuD|s1tB^M4YuH5bA;1ImJf#|xOR(bpb z1M{{A?ng=!;a3>t?*XQB?iN0}fsGRUfC|UXC{JG@)aehdGYGqhRzBXUR`k;2#`?Mi zY_$gWxCMtr32p&+dC6cl3#DA?;da=&`TeVtbR>?5eVfD+mngsStr^B{XqdXvUM!gk@9uC^^zIc`bMY;vXwP-jc2v5Tdk?ob zQ<%O3DGlFSbDS9Z+^{6@asL@^k}-cL6DTqc&~mU8`8Z`}t7qU#XQ4Yg;_oM{01KrY z|A#_gd!xib0Ik*XPde-(b`wGynP;qpmgNpEQ0rckT7Hj4yrbgKZjC~%wNXb{g~@fM zd#sGR_tzi(dH1Q3$1ougktStMncYMr8Ki>bJ$gzMUS4?SI*E(N*D1e5gdoa{X@i%| zX|N*d2$|_bH4I6Mo!O*XgPqnJF;Y(wkWiI7_8y?&2xWH~oxer)Z*$`!wtgViG=E$r zspxpTM9Kpn;yuQGiDb}*g#b=CcR|bOlPb;1!qV)FqDT8G)u{<%9riYfRcD^AlC)dF zERE%XT|rF>FWZCzdzAS0u3$C;$KR!ah#olrA94#}7hhljNVQc>3JMy+YlI&Ef*?dl z{@zby@}??rN10x^+Dan%z_`RjTbe0O6CO9ldVB6ldx-t5n735fr(~37%uZ?=I3e;< ziUkO;C#Guao<7SiL6H}Ip-qA@?jb+Y&Ak?Vh!;BBXYV|9NygriQKrr3$4SD?M9kzP zu43la+`GY<{HY4^-lbY&ju5y01qT=w16>M9#ViCe?xhO*`yPp{#Juu|3J}Inj7e|Z zcb^tDvjd>tducZ_(aepbjj=dOl%AB1+rY}TV0Zir6`EvyzxRMI4GTS?UwUoQjy|Ma zCevTp;1qoto39Y(y04=of+EsR4qcUTo{znSKd(r9(xD(yIF+Hj6$7%zMNR?VGgdw0 zoO$z&TS?9@a{|uF{DxK*+W=5%uh93&TU4KrJ{ogY_c{9>8PvF;uPW-!CyA_2HY6k- zqI^YGeK>xtL^!{-If|ix;3w%lLyX9i0}4enG?l>BG{TpV}XR{ZAa?H@}YiObN>FEDa5DKhu+S991)g!L`VxC_n;_rkx4v#`1bL=UsumeI&MMa zAAt9>x@ik&5IVJWSx$ETeIntN4HvRe1twO#@~=4*kL{|c%k!FTAx&|bkZX2mUcD0N zPE2d)(k^*7KclOqfyau_aMa82*l9&RmLR8vDXhOr0==~IZsj>=N^QSj-p^TAgwzy8 zVY5=ofB)}PlG3K>8(^&=8k@sbCZ(ip^fo?(lIm=s@!RE{Nlf&G+EH{6%*!Oa{4n!oBJDzh(*UCP zX@L&aZK`!zEvoitFpHLejdu z;^PNQnE~RGIoc_88tD>qc>UaAGmbX(INjlgdtL(pho&Nd&J4IY505MMa`B=))j^@^ zTrGL^!Wh67b@XHZ=G~(3G{`fdOOh-uewvA(S4ZB!ABoevy`e(tXAZB|XvuVW!~J2? zxVHwBN*-J1f6{jJtolU>3u4=Y_=8lU!QUhk&U0(#At1EuVV6j5QN=Q}&5hznXnaCd zRechhb0hPSRDuPYHP&b~+clI`1>TF(xS#eBuNJd$Xy|8<|Bg-##X}&Q1@9q9wd-LY z{w4!T;i^Fhd9O86q016~A#*>o9SqMe<@ax9rOjH}*Nj?4)-a)w;Qh4K)^bmC?DW6- zWW`K+bBF+#Y@R(|7llnv5M{R857VDpwzHuk2EEvSd5C{I8$jBj(E==iv3eOL`cJFu zUa$WjO3g9Ym7%2hR4B|W~Vy3ChgFcv4R)vwXI6}C6SUtb9Qr69!1owGL6shD=<*q8|0In z9o?^4m|0v*nP@CUSXt2Dstlyh;ch<6-rYSQ4>4+vTFDB9 z)g}f|_}^y|GF$RDEqpmrhdW0WDT$F@TO7EoT%5W7*UaE{c$a#Xk>|7Ke|s!d1bX@F z0)yy;lFxP8iS0pTt==h2Tg)JPNKF~QPp8IMyum8cEQkm>Z5?l)R3! zUS_W6J{*gp89$#mm%HSwZyz#!h%%ZyUs0_SQ9_cWRv;mbFD4p*MF0=OX~WLDIYIX= zwfJxYT_+F$)en4RRmgFCc6vBjyjXuIe^!}qKCkqoTp|mf9}t_idpE`=#eU0~w|5Z_ zT5J!x7|w625TkIC#$#QTTR`y$7T$m0R^Tt&`DpoCC7`8kue-%g>r<2nF8Ie8&`9nx zns#6K3=IvvZr)$q7uz=m=2xo(C3+@sx!r+i6}KVUW}xvwf-GSl$SnnIKEg&!m?kDB zk5i$(k!e5UE_@F0p91B5;t3$#u7c5z&d>9d>8_6X-wk8%I3sqs5>2Fd>^{7~1@*ye zdi33JYeVq(d-S-5-7KhuYo)-482zJDouCQZEkqjwP~} zbta}cvxmRXfj+Z4KtF=@Dh{_xqMA#GOs(<`|NGF#AArbG2V@?-*Ir-XO?)5de!bwV z@x|x&dbKAbBWpQtJ9&D%@CpdU^QhT=E2>RE1P8_MIK<j`q;_Cr&B$z7(m+)6BE1SsmA?YNk9Gfhb8Y&$Z~f{T(G{DX7#R>=Zux%N;Ppu zSLGoLs7P%tfAvVU`h?qFo;)rD;pWm!IV;Ea>3)o&A^gH!Ltofc8=|#4A>rc*A+AMr7bPZ|nEVa}W#4pSp8X*j5)#s~J&4_U z-*%FknP~**N*q&L2; zt6t3ld0Dd9>{qq`a|<}D+=2XRAmvho83irKQkZA^GNl)$JoYq{X(+a z$5Dg?IPtoYk)&A1C8l?XS83TQv?PnFYATy;UZ*v8Ilcpzbmk&Yta+mRi&rwfVaRs# zq4w-{KTuf;P#=MwMM4p(Aw09FT_BL)EN50Cp5*glKP!C6!~}^XpklaXhZ<{?T3J@I zHdzDles9!6c!c2d7N1!}b2_`!*~R1xx>kzq+~N(iffX$4zNRcm*ANaYgfHrt4@J4# z_AG=G$9YAz-Y`E!53Z>0{r&!hLLY%*$6dCq{Hd!@#O9&OsT+ASgfcY^o6yC?+j37^ z4@Q`?U|p!t?K^Yz^7Hk21cFM$9;e}bhq*VNuIEl35XA#?W1tj5$oS|e{-+Omt`I^f<3=Mj$NqOD zT=%5EOBhlshskkL5|9=~U9SA+rhG$le>}*1AAhL_e|K`=3JfJRC@x-hrs%V^z9?GE zI+$N9zS4P0uKd=pa2B1PmzOtZ)6sa%4GRxnGSRirGK+Kw)X>t7>LewA0|!JVfDo4} ziWcBj+@G3Kpn{OMOi+RBDg99rmQUgALMADsriPv-69SBAEUBw(TmZ~MD5AC;_YLRX z!VxtBiLBeIDJ$!oPTlJWU8ZD>7GI^^C+9BlEA>%PAJKZ-m_%j*gad0x)wl4%Yd=iV zSFAPiAB=iX?LyGiMX=T>DZR^(Km3uFP5(tB9GR4sk0t>nUn?>sI)lyrVg2E<^TI1s zIP^Q9tc?JV%J*h~-eJ{qb9Bbv7`UHWqhlqYriJac#GSRW5=0FC^j@T{zI=t51oC0L zvpZkzn|aGBb!;fHHFB8K#x&4x$xffv)FGRS?zrfJnsg!AdIw;5=di4Pe*N0+Hm?QM zC!e1W+2M$D-SB0xpy`COtX+@n#osf{{k?>w5n3qw-i87$o0h|Y>SslwU7tq^e&_Ei z-nTZTUi3)n&dS5cQOdDfCBxqAO&*xMO7wu}ka~T83iAML-m(WYglxZMV+Rlxif@Gh zV}Ya{dJBk6B1UH+6imJBoek2&%fsq%Z-T*(3L6VeVrFY$MM=vNSaXVUt&u4wD z|K$x^t^s{KLY|{<<&+3GjS%M2(lbSd#Y~RA;kD#EY~1VWdR#$73=jc*?C05dd{3qP zy?5GxvqR(lY&G=o#iWK61J-fM&$4GG$(yH$+rSE=h(#&Pv09Xblna*PZ7bvUAJmsdT6d0?Ifrf`RoDa}AIV<8f(XAFP z<~U|@xdcPV4thB~d`Q9znP)n7W(dh5Mi3L{Aofi8KCb>pU%TTTEnH;v;e>?Wz%&x! zmxs!u+j;;ME~DH0YL(0Oo?Y)%XjQsAQAlb8NZ49xH57kwU2S?gq4%l9MS_TqP2-5U zdV>WvKgJ*gKr6rcSEbX@W{IvO<#N6jZY=qJl#j~@gKUFitRkif|IKTQj5-4-rckg z!3dCqHP+aR&a2@+afZyvZS8PeCoBOm-Fns9U=`BQh6Dvk2xtNT!fo|tfoqu%fs{!6 zx&Je?HpUt5N{foEFKN>7exXgVS5Wxw7@t5pWul!gq)aV?Gy#hOu8P*}$xdJb-3rI< zcc?6V{pZ?EK#kTN>^;URWzQOhI0}406@By8pvs{j2s8;&vP`$&`qKDq;}_Llx2}*J zwMdT`3o|q}8l6y}9jOu|6MWKTw~o!+LsF*Asw(dm#tydkUwrCV6r(;cue5cWn|phc zoCl88<QZ&PMZC8AfZ3jzICZW_L_p;B)NaiiBIOBNtaKUFgs_Nfj? zL^{@pwweeT1%nfk8eV;rS0481gfxbjz|7VGB?@ti6yNVY5ikZPnyQXOP#@Md(cI8y z(7u8cutclY78yAv!{YUi*by@iIoO=EoGxVTy`$iLClJzf>i7_r-CkWwEKs~~twUjE zL8~)&PmakH%}iARVn!H1tgk!{mZ*aSEM;+q)|?l0fW%nfbq>!DD{r>mmH)V9?s(z7 z0E`*|1Y6J8He2%Vz)yCwnh}PsG8bnw3*v(c)8hEk?wDC(MhEQIcwAf|rl3t_bMN12 z?%Hp{Rw@+K%w+W0!Xk)VeW5;{K(1X3yIA;&N8#a=AVo@|AAU-lmotaOK{}g+7Zii2 zmg@lPo!gdi7eJ3Z@0R8T3tXd_n^u8HeG9>~^r9lYnnFno;o7XRkaC4Y{@xmIJywu4 zvV1i2P2jiT?hBjO#*8V0C6^1fJGRMlm?f62n%@NBQ4gQXJf(>&s;D(LftFLrbqknU zxIY1@YK68W0{Caj9`{8@WEQup-RwAfy-(lQh3d^e*3DfhYA)w{SN1`fPGPb!6Ks3g3=El1*6uF4zUuoA|S3 z;DTrTKzcF0dJ-OKNtdGJi;1rgPHI{*>0lPpmPVIT+WQBA(3p3*kYI$z-1 z{p)@m$YhT}J>@{U4AWf9CR=pgN z_m05jcCGWWn(3Tw226eocbk`k+g-4_#nbz;pd}QBvn}-9adrh%(RZ z0FFOK+{esUVnfn|)JNF*Z8SP+!X5d*ijY!6x<|jo3PA{=KcL4(w#A5cU&#-P+rK7jkjx}^$?3(T z$xHw3__Ez`X<$y1P&>uQA@TahCUO0pm;-wkj0!C=4GgHnG=;k|460i7MB0rRaE1~f zmQ1>M0q`Wyr{Y2kYGE0rEXBym3XxA3l%({CnUgjH3!u=sG8M1MAxE9EMk-$3b>%V@ zp2?v)nudkw3#m`AetiUwXE3ru>UCDG2TTEff?BO=4bi3(Hx(zB=eZ)rd%ka8Wz$ZU zKrj}4ONMQP-^V^=HIVyV?;?N$JL26Lp?U?sB#xt@n(?zV5~Bdu{3~7f+IM z__YXJaXx6#eeQx*e|}=;${FHXLgf7&GgA=)(^Am$3f2}0t~Xg9%E6aCy0biHqMHv~ zQw*V#S9O)@I|{M;Yv6Vr;k1Gs>v9p8Oev0qn=kPmrUa4b(9V}8%edp)V|68g z@|$seiYa@6t9HGiRWI7svkM{C$@djBnPk>)@`Gy)jDaVb#S@=z)DEK1RvTT+?pE8m z#wZG9(XbIyOI73vSI1*-q}5tS+Apu$f{rt5ThmYQ*LyQrYd&iN^uulfdBIhNhZDGP zpp~r24L~lJ0A;YT2JoR3#Dfnxj~Fr2Xf-AF><30gN7p(X&$#4QjzUA_f8W*&{Tm7c zCs>|#a^)-H&ctC6#Fv6vT&YD=r<`7fNH+;uwUBE4CdUZ zud?@r^>gz-AOi1~n%XDbOM$bKa?J~cXL~0QAWx4Wb*GVMVr3DYjIhDkY3sHayNQB|z{DfX zq?uj#x+A@JdK8<^$m0tyxc$8=oF4yO#95UAXG>le9GdOcLp|Lk;Q3aeXiC<8$KwG; zmRV__c|M%)@2BYY-gwt7(z#;j-io^!s+gaAk81+3L3%A|;n%U&43*F#25{+Xbd zW|f>kE)Qfx2N-JrebAfWplLE_i_Y};lhq(7zoT;RZZXOV9}3?bgm&>@r?b@+-bOwdpg!Y3 zRjAepoQXQrpgwU01Gpd&QmpE&=~|A-y*SFhB~)f`S)*JNAF_#JkC^|e&kFB*CyySZ zJ?)>9^YalVlW=%272#1=*^R2yK&`)1)vt5w4H4-r@?kcqBTk*n^ywf+gmzK z5w;BL$VIVU>h?FD)us_PUl*KE-~sG%I6d4>t*<+}L+P1$?o#3ZMYc^NRo0XsSlGj( z+QtbZu*^^}?W-0Gw4<#6anPeCf)8a_6(sP(XdwQ*mOy+Qi%_47q>p)J$$&wJ430Fg zb*(G+L@g2WJ+~KnthW+Ykkc-ofu1{cL= zS63P1C&09p?Hrl)uaM$Ja_JyHTLD$=@K2h-bmenL{jcK8v)YA^COqy{U@rwCrzPV@ zr5~jT=9GPz#XEQde;>_Kd%|gggcIG2Bb`qm#4-0S+I=_slc0_&d*F%cRO;Tp zrrwt;eAnk5b|*B`CH9I1`w82YD842iaAs>n^Z6j)vc@YyQ66EP7BZ<2SBL~PhFzNl zPxyb&jBZsG+n}O|EcGuA%Fhfg@I5k*J^8k5=Yi zng;?LqbKE9q|yBnA_sBMN@0Ts=cs&e;}aZJjbiM5+l@+&-4k9LsC>RKa40l&{1w}L zYGCctDKv0#K@xvVO;Gffk7l)M!qEJ{mOdY1%_SnjKsu^7T>|e3jaS1|Gpd5-$ zAa1{bkb6Zy16I*5YagyV_aco>XYS{M~d#8Y2X-y7E97`h-;5olcw1=G=7 zX*D?{a0btaFRy(TnHN zOC#wD95G{Ag(E3@ah7XY-V;Z!!a^vWGrjIA?MW(y$&pL_T438 zV^_l1|M$qvhug4BpSXG%h%opfoOF~^V^2o6jc|%XBjL7JMI;1q$fEGm^KHH9n@sZ( zU{V_jMdqo3LhpBI@cw8p`3a9LMeR(D#9}X+{@?}$-t0hwZGtbdcG4k61>brwwCT@6(dq2ff}lLUzo)jm0ZO=&OW| zmsDSu7PY*_ec~tsL1J)`aDDSoGL4RqpP~hF-wCZkN#u@>=(i4aibEoXH8ykzAtZmN z;!)1jwg?pW*^U*U$3OnPDeZiPRTXCknvZICund{gi3=EQSC3dXZm!e8-~e2GsY)my zl5SQhdwEVnluqpx3{n;`<~ll@F2<2EdEJFO(;;BN(%dmoKkzJ6^P)ZFM+}ZCeZ9}? z9AbinN~alj*W@=Ma5a60cY*8lTjZGMgS*-_6yeKgo$f=svBud1K~aB`HO;8d-&R}t zA}fA*$m#dayUsgJ)0fX;XG{bRUTb1bP9nXaP}YAGj;yZ%DXpL$PhKH5SA62g(`^>r z0KQ*K)%Y{(-MKS*s`REP?T(Y94nC{j?W9Rx3vFJ&{?Vv^izvm|8#Ced@8?H~9%9D9 zY3aH*?f%qecK{rfx+}KNVF*3lNm?wzH&i8Vv=IM#)u!U2bxujLIJLwFGDlQ*sfzb$ z`NB=)*E`Be7b<%fDr<; zp+uKgwMAU`JhTH1;i}cT+*t5xQRHuDQ*! z10}*ZmqmT|j8ys=_o}l@Mx}XkwdjgiELYANBV)hto&ZK~TItZ%Am9d!F3&OAAWe`v8Qq<5;v z1Rkbd&_>eJ&I~!tBQ)DKhm<(8hgV4i=06+;k+D~%!}t4%x6=E4eCl`e8ck{9wdczs zAvh%EQ*R8{$MG4bhxfEDlg5|ge-WFjdsj=-u?3`}gJ`9aJnzK=w$hJPnvD^w{Fk6O ztAn-H*H1jnn^&MfDmP{k%7y30nT(uVo!66tuFo;4OUEly_Qh@+9S9IGYtRRX86g_2 z4x|rjH-ueJf53qEv%eqQ`wfUIxWAmI_l26AygZiwU>p(`@gJas1rz_`p2NY{g7gn!o$tfzicbIt@>x; z;D$wik~_CH17(lH9&ep}#uYJ`Z0h|8IlTnMq-9&!^nfZhc#@e8z1+bgJcv4X0!nbe zN+fF;oml6MJFDqy=i)1q;axw3OVIexs6=&V2+vT$UuB628??={N~d z*;7y2PHx!c$+7oC&7Gud`w5k$H>=&Nr@Mu$ulq@3Unzq-6c8R77dT@2hKC{bUjC}J zykx`YDGSl102I>o3=F+erHb3f3lP9y$JupJZ?z1PBo)lt?%K@Vo*18&h6%I@_RX(0 z*?)LH)LSl6ulc;-0u2tA*D)!z))}-~Rr#*dME6}yp~uJOR_#}aiJ#)j_0ub9cO`}UJTK!PZ!@3T(p7hV97h9z9 zo#Z6!6>-|_=8Qw+s8zh(aoU=e+KblBjex*sd4dL9IJjUe*Nf(^oV>jJ%LB{CB0zesBoxI8&8iYqz||40qrR#WWXTI3R}#H z{l7U%my3<@oVOd_iQ!9zs$B$fMt+}XbJZ%%fq(Pmt$;-S_)o2S=DTpBQr+D@>!@G9 z6w6iYaSok0;J!#}F2xAr3>bN$ZNK=uc<0$8la7P!+d6-^)22VVv0~BMbO|{gv6jni`gOU&I6R^b z;lFwv6@Q`jp#6>qZ(r}AE9`6Xi&HXwY!haZx$-1c6OHo2cpi$87e{;Id1B}=B$8l2lB35CbqBL_G+>13L1$D9w>+lM=5 z9v)b~-8P=rtw3c)!I*~cGF`EG@=N}_oIlu-sDWv=V^?2 zscBuVScNX-@Ps%EU5XNT36-S8#KrA#$ejUg-HL5=X=$m5ZMUMK7ruU(W_Nejy4pA$ zn0mqh3vlYSwYADk&L?=cJik8w=>ev-k)lmE^!WG);qHU28yrNcvo7OWQP^~dkW>q3 z`LnGR$Wti4&?!^?a@6u3c3d2zgQ8O23XYdG!{_r3jNP)tT*Q(vYMNUO6|?;4cfo0R zSD}Utj}gx2rB86*i z1q9mEnnM_YfZyi5<9O4oJ_QZa6(njaXiGys*AD;1HY32uJU0FPC_L+GpEE1c_s;^8 zV{vke_r(fd?&8>KW`7(Df|h0x%B4oEppZD%IJ<0a2h-;47VoADa!2l-!r1zcYC#Mk zRI^_H`+SG3-sS{VeY)1Ow1m<9cD31ar_BY83rMu~UV9u>CrEQA95J$av4@_UFLc=K zHaqZqPn6&Bc?8%pL`fqo0WwzpVYJXvyjP4`nC@j`0VL!>0EpP>yNLuK-lXz{+~p z2DPmOS67eB+t5eygJ+8#e09bH*K{?>1>5u^lMQ?O&e~hBA4+L;UPjH+Wg^he)GM7wok8lQcfJShX0j zHD|++O!`zR%S?Bk2T#$)J5eJ-?V(_X`#)<p~2*qP?fcXX5i^fK7sbDYn+bwiyGG8lMxm`{PVhjrt zqeYmK*U=`2e&LHl`tNf57O8bSlmj+C@^ts}Wz*F|X{Iz8Dpw%@;%iAbzK&I`ho$2P z%l;}Tk}Vz`y0}f_)U}HG+il}aAV4j9tc>Y?rqq_~%wX?!-Ah%^O*fyClV`U`ez|B( z=(DeKpA)4qW68XfW8hUR!nI2WoFsefM$DC82wO-4YfO^pX8hoOQVo8X!^J_`p^KRl z6MNA|sqzIhhzexGLG0m-7>Wz=wfr?+CCZR0T;14|DttQn`pXs%lKv@e)4elY2$E;g zGL#4~GMcKPFIzRyDggx$F!`WB2XjK62G>Ow4z{8WMd8 zNE!X%CiW@9&R1kqy9waPhGw)BKr7?f_2&VmroS53tZ_!Ys?e!S$eK+5s+do?g5$zn z(=w+;6w^X)G&B%Ao*!yhuo-b(c%Fj_w*7%d72qy4k$Jt{@Me zjm_sVguP`Zdr+kb>v1PI@7Ep16`bQ~$H+2Puzh~72`8^*mCZ~*%DebaJ zR@Wk;~y)6^bX1P!0oAl>t9ft6+`9XCreHGt7RdN zs3|mCL!b0PsSpSwNfz3+EFV*95j2XKy#B>^yO3S6kV6#1iAPgcGk*9AnrQ8SeoDL_OP3oa`QywfQi4)4afGp{$12>@jX63)5G6P2tZ0Qko&Dvf@709jCgTunio;^OY z468}>Af$Jl@pxAQ%O-WRr`cB5XvPmT;k2x5HBRf(*F_?+t)&0oGAd~9a!+-;a3%>D zqRPjSq)+(EO))su1vTsi2ZNgJg9hy*_++90lS10_;_Y)DA@H}Z=lv-C4z8>$(S@h+w zE+rRY^!9!G(xxpaT_O#-ui$8%GmwlI>G8Ud{U&>u^+`zC`l(_WtU6h%hjKW>TA;28 zg}mm|>|a6KBj{Zo6*msnkusM^i8Q8)o_JuluN-nSSVqU9m1!4txqf@HUi{JM(Bbg= zuPin%&$oCdnnrx!(ST`nESqn7U68qTy@okNP^iL#n zmn*t#3jiAG)Ih*uf)2$EU1SH{T!^u6&A}J%m)#b$usQ57e^Gp@(W2C^Rv~W3U!{g( z6zG$O>Hj92-KX(zy%kT2LJ>-Q5xo+DYTnv$_Bh)6B4#p5RJ!>4%qwQCksa9s;vTQE z&frf$^(d+0K~y5Jn#xqDfUg{wo&zv~2mzXvQmxv>N~Im=PePDE2-dZ0&+hk|rZC5I z`JmYOsBB9}jMUSp--q)c-I+kSvvP#f>cox-gU@nVjC3Lc2ZBuH2aK$zzk4Ttv54Dg z63nRZZo`?;@_$?C1XNUp4tLG=%)n9TOsWcEo_8d{Qkf5x(P5`&8TiSL<33`n=myf= z7jlg*Ys_VzoLWO@?TMGMBmrbRUOzdMU|KcDy?%L0Rg}fbar-@^&bR}HEbJx#TQJa% zq2<-{Dr;KetZpadJfnbtfdTH>Y_A3X!t`gmdHvbW|)+2^X*&pSu_ zqMVPh#t??|#r0W)7~DhS+x`fa0cl6gwMhLm0ZHOec~E1lm+$w{R=*aUO2jI8r&o;a z+Q8e67nhhUSPWXSwT2$@i0DJWVWhobRtflHC?X*H?BoKn0gE#FwUHy|`*Wn;UUm25 z;^}gID7H;Y+;~RwW$?$dZ^DR?+Y4w-NUv0S>JiEQWcbHdEv+sjr z2P_E&7uk|yTD$|!x{jtQgl zCZUHpE>B0oYTE|WbZ*oF4WTyL*T__D*IPFeV<104p9}aGRC1IB+aRepw^MMrg%VF= z@>yZgcm}~SVNihS=aR!%<(IrDqw5HeM#?!{)?24=Pn3~I!r>Nic@rZO{YFj>99A-V z9M3|;yn%&=1+D6j(zP<&f8PY?Be1iFqvyGn%%}!U*GT7 znkT)E&N_`oN1?_AZxP;K^&j06!mB$CrfRc~-b_8~*n(ok6DQ@Gl2tQqveo*4FGn^U zC69GU$sxZOvyyCBV*&~eGH-d~0->x~F7f{W2#S>DbvA)|Q}}g1Y|-|vTb9PPHzy!4 zOXj;Hbwei3$lzoVZKjaVrU~i^Y?OyQEar8+7zZ^B+pJHNzeGCi;V!^E2E{Rf)qZ*r zfVu2Sr6Rz0s!W9#5`Bd8=}Z4XgZ4DY{|eUra9RwIb3QM-_~cPP^ErSaB=I8KOC|NZ z=5PQ>@VXUuakFIjONma+?X>ojyx=#o8*s>{Ac`>ryjkeokDP)C1bYxBAKj@(V#D>i z`}$d&rclusYUAMQ$H=Lgl=Y?=LO}>e2ljr-gjio5kGE5$oL2g%{TM?0JB++T3#;_; zl5M-u>JI>D35?{B!DmJDOC>$j?vRp&%YeRRFFY*7fEpCUWFoZ6v(}p3 zR!w5LN6+7+mw4*xz6s35PJ{2A^L?K|<5zw3f(38a_CD=cm7E)74 zpy<1Ak$&+$Q}DkmPhx0y1=LP6e*O9t0~go$xUNV>MwTt7GEPZ}E(vfIpg`lw$jSku zeXhLV!}YDaqCR$gKj@?6qi*bEm>w$pM{~Cg1wB#2Th&4;zqp{K5VF|j@(vOgIGX}sWtdP_WL!s|H z$npfw{^ETu%5CLwG24$4HkH1*3isu^IZa|(LFMq$=*8pBrO~_`yiSBW7@_=ssQ+Gc z37b7|#dma#kR0aAU?2Et5s8A4KP%@Lm=Gk*$(^%D5CSm03t)vFJ}%**{(%3m0 zcWs=H&MOy#P{{M(ILt ztm=@17U_KhLGxKi?dDpAJ@xuE1lXgbJ))By#{GcBNu2=_oD14Mg>GoV=r{iQuY|i1 z`iUPnluGoWZk*_q2j+VyegW5|(7cgclr)qIJh~z)*RS79%@sZG$!~oQZb-b#mmDt< zu)o6GIxUIVM&gz<anscSo3c{bhNcT??qpV-#tEMnnICRY%sfTaQ7?GcH< zg3|ASa>{==37+@UjUCv85Cgl+>I)EIGwec^G?PC)0GJO4@x1uJxdPdUNb1^xpo$8ZDBWR9w#gqSzwZ@KYtkB?Py}sdv4wG^?zo0XhXhfHbaCwr^)15x_w;EdvNZ%(_&QEE-qjQa`~XY>*$v z!mZPaQF3*mE*x;Mo@QMNK9h(bCN1Rc&a>7GYyeLh!D>w7(i!El^ZHiFaZxcTwDMaj z?RmoV%GQg#T~34MB92Dy6uF2dq2zOK1yY&-7ad@T;+Yg?XNCeMz_F!xX1SpXZE;MT z_#`X+Ty=T50|<`iAX{WaOQ#eOT33V-n(AtBM5>Bu@CFGLQjp5KOeU3(+l1o#89!T? z@93zWCCE6-Jn1w9im{FQ?mauSVIEy>J8F=EC?TJdApw>zIVL8?CAw+ea-qigWZuci zsbQdMAsuK-0A}k+QO1)P0DBZbT}32OgMJPq)M~cb{|9ulHCbWWqv+7wzs!8eO!}_jzzgpphJ&!sRv_mY+qDfbkpBmAQ-JY&- zTFt?d%L7;8ewGV+EwL09pZHhX3~FOp=+5fb`1>mM+pE}z=ZJaLdBXSbzY$^*KVa1K z;Rn=9eqPS7G8Q&zCvK}1I9s&EE^Ms=Fa@fw2z(RCdbQB6E#Lumsbyh+|BH*_XB!#t zwp492MQho1MoSg-wlnd4TsBq%lk(7P+MY~`$;bfL;pdb&0Bq$`{Cv4Tvs!LT`Z%e6 zkJxeFjXj1kN4pyFJ6yOaHr(GwxYzTkCti;!)u zDlnA%mLM!T;d!Az9b}A*!!2N75@0mDy5xLh-()+yrJhT7nR&gxw2%_usNSOjx-@8ULJ?nKhtZF)akJx%&%pHz2eXP@JJ z>YjMMw~Fs~ru?7p1i;VuE|N? zduE{X@z}Sxy84%zo(c=G?g}OZ{Q&s1hb<*(vat>m_e5}Vw0Yq?youhBauT^&{C$&6 zR+dzKbR~r1=%8M6IZa`IGvVguq=-#yTIb6M8V^@83deQiM^5KMO#EKUgqJTn5%XwM z*7@;OYR?}e5GZ-1Tx8KH?rBL95@?cr)pjoj>dM;&TeZV1@0BE%2g$>cA3G6p?bJ%y z1fohLkZ}YcUkE^BF;aWn_^perTgRfp9>k4Q)gEp+^s#>U{o^j7s-ICYT6$Uc+#M7Y z35WmgfY<3IQn=of(3-excmu1$GCVyLLtp%W<6%abED3)RiaaDy?E}%??M&X>Z6D^} z^~JIA7E{A72i_P?_6}M{d5X2R%t@8ZN?*51@vJ*nee`lkB(stt?1%yu$ zeUJEWI*48~Ne+e!)p=r@`8g>m%I~;b!~1I5J=A(nQfxL${cEQ*4{MI_-ItNw!>gO` z-`}6k+m~A%$$?{r51wuLfGqo3Sm(Ki&q=dSB#Z3v#l>#AWz{x}-Xo6zhW*PK|KRH8 zm0W-T0J}EHAz!I63^A~ zVq)STgQXgnad z*$_UyPGMwr!(_|eTd-um8r}$^;BejdFx@naI+ik4jIlZQtY`%AHVA^Kq8b%smLp1} zXxY4lr<@V{JulJBo%gTC8`P98(jbfyA|o%h^MpI8!W}?ECVP;}?R8X< zg{F(9T>pUkYU4)B9l4r{u{k6v+-utH5au5-m|0M~$tj+vpl@Xr<^6K+Toel%**ohG z_N|T1+r7Kq zVqk2wo!D+&ZufOOv-1A|gZIiSX?>9A>?Tst6&jX;iHlPMPzX#fA8~!TG?j zxro&@<>a8whp({K5vraiy2PTJNS`+ByTNGE7(R!eH5{K_gx9A|mO0agQ9ju{m)%tv$p(#3O`GJr{6TBP)H+dEQ zq{i#k$0uXlx#c%kPYMfJ=BHva^)no^*2i9a4JU`fSnWuIaWNLkd3`@dK&5JG1@YNA zDr0YWvUz{(IkvV}M&H`GAjDPGzIs4@-!@yXt)*&+x?#!&tHfD-Mm?r#f2m%{++>l( z#Kqmevvorb3l+Gulw*X!l|rBt=NVJ9^$%{&I5UoKB! zYZl-{F5C{efyL)}yPXZA_>RNzhg$bas;cu|6Dyj`_pWuaX+7KXC@&P9t)ek1>YkCg zA?wNWBRdyd%F+V4ab8V|TS3Zfyg4=JH>1Z0R0~(VlB7UoupKf$Ul4rZ&^uSaF z3xS^C7@zj^@6LLRcxDtw7Q3k`U^5NgYp{PvdYv6Pnq1$Z@vEQ1 z_l|wX_rm|h*Osg$CTDj%7)zYsoG)PD;jIbJdpp$!gZzE1W-y#U+RGd8_JhYo5jc%9 zOkw}Zen4oWzC``g<1WnX3_glO@GL~A+)x5pA$F#v;;U4IvM&{Ep@(mP0^Dx-JXMlK zl2U1do4rewGF79FdD(oK-X4$R|KsT`!{X?ItccGZ?zYxzR{L6GPgwb~GS8fiuT_dt4aBxWVT zp1{67!DdHuj5`Ed+<#*;XvJ-k_hoke1$aCf4ef{q zS93>DRi1_8B0<={A$HNF=0!_QZTC8qhz#N&!>R5W1@RSI8TvpK#Y`?h$((si@DD?m z;=!W~J~vTZqQZOTH=d@@)wb%|Wz@iBkQ#h>78Y6Yb^^26F^y*8UR^-iG#N1?w-iWU;$i$M?-69}Zx13(1&{@KK@4fy<)``~Z)8I?Y$G zhHZ_KWq?GF3Bmd=|MP+~9E_{cU(>ct*gbd=zjZSb1>j?qf5|85s-_B?C@%wyC;z6? zI3r*La%+cK;NFXF;>uM(KI_<&P%@ONH5NUC^^CckO*jl08~HcIBEmOSedkNl;(RHe zRMtk<=C3mvBRvZH$L?815%{QTk)mu?)-5SJ&CKXQ>i`Aq19Q@Dy zwvnRPw6P8bvpdJ1Mkady){}k;?%YS2%>ua!2qE6>bJ^PxuJQ(HyId|?*(HfbSzM*G zl+&zszvCIV%&_rib_&-kFZ|y)QpLoCaTJv(C|^7{gq0OF>o50TDU3nXzyL#Ss^+6Z zmsu0=>bt@czD&DG0APyxcYYYxYHJ)PZU5Jw-Q*!k*v0Oj5ht~lqG0~ko>@Iy6EnNd zKD`3x#TOK_{%}nH0WD)nDuiV=hq%fTV3G0#O~rCFU)s{-WaB$klU2pQ7W|)KNKs2+ zqFbn99B0v03&CgvKFQt~)#1#AeV>5A>A~i$WqlvZV!%0kq-s8xnw8=ItXUT>foLSo z-V?_6du*ex7uDg*(QuzlP4}k*9V8nD%jY>p7;~-YA5x@0b_^Gj;CvcX{4@5;3&u7^ zdfcidDt}-6{r`gr=Kgow;1IGrc@3sxy%?%GJ3wI_z5y5xPR28F+g{Et8=i&;FHgqT zW9w~OS0)eV6^E&J}$#ZBllqnmI0`{Fn6$3nQY3xS% z!n8rZQ=mc79Ofif5kEs}JbfEYw#)Hmp$_H90KH)I|9JZAK&dJ*#YRWm8baU15tF|n z@!f$A!gHrd1Zz%#rP z7h&RvbukBp|LZCFa7~|ti}8Lwi_>g?&tCcUd~Tr!J2|9?u~U~B?-x9D&9$vg;}P}G z$}GnDV%flnMj`6-lp%Qu&b(HQKNYfod0Ee(?9@XF4M*D6FvM zP7)4K%;H8i%bM4~uBfD0C!PB|? zx!fN$dNWkPmHn&)NnetM;=C%pHY{)r&zROp9O-!j06Vy1|?J?(d zMlyx&c$i(Ql0?9+;DRqJ`ahj6luL>zTs$ksh$!1VwLtH3`ZrSCv~o7D5bLEi3N9ig zNWL{e90s=_Vv?v`oKdE7yaYu7VI``KO9*KR>~3C-i+%*L(yP)- zN4kQ>f0C6>DZP9MTk%F9$7sIL!#bF7iP1&oa1l)tnEUV0mPsH(gj#^tjgrbQ!Xa{} zQ@Iu9P!Z(|D=If`-rG#~Aa82bu21fiOU^)Ng2GZ_?i<%V9fnq++jR*HAvY%C!ZdRb zS4Pnuyu=&bbhxEyariRW+EQp135S}|;&KJ%^Ygq~xxWzp%qejw`N@*=c^|E}HAFL^ zh*U%YEF}W&&@-^M zhvTk)Tm_P@N5nv8f}wkhgM6BRdEQ%Ilm4?W<%&_F!5nP&c#zx=^1iV&AUI|yk*l00 z2A|0EfgT@Ku+tf=J)2s2XEQkEj@P{?uR1S$D9oca7#}z9sXd6}=^z*-VSWWf5?BAt zy<$T2xu!1op#O^~_~@y?(Yj7-F6YR3wxZ>bryIDXPJKNc7`#zWB{q){;9(Iq%0Abi z%dva6RSGv))l}BU47>S}F2C!Q+gQMGZ?7@v>MDi~7tkA&^Iv|q6^GS4f+wQo*5DT} zgu8|vl`+3RC^)!$} zaK^BcJ8;7OJB%cE)td){G?GpR`Hqo3W`QW9tzsnpUT0%eCg7Q}=kicz4xi56c0MgY zU*J7ybGs|0cN!bcQX4Um3MgkY93A5nz~Z^_u5H}nk7Eb_0>niDLgmnKo}o-BsW@A& z{CK_iDLk@5DiBYfj)_1)7e#p3wjLPWZp0>8os{2t*3g_@_-FROzhKW7VtDg5d5g1_ z=+E9)v;L{1-oIlAiH{*t-IyJCD0g>LDkMjr3tk8UZ&$X9i(-ybC~~~S^Q+Xhp6dhB z3Igvefwg>p;}Y&#`|FBApnu|u&K%G5J%9hSr}LoSXC2{Mn8XOwszc-N|G#7xv&%ZO z%WWdZ&Joq7bjm?z!$HjmkFVWKXZ*t7YL(5z<9Kk|Zags4{m9-M2!RLV=H@N0qojwd zeNZ){i2NEn!gn3%`XTtQMU~mD8L1Ew9=-Y~GW_9WY`- zUc4}jjB=Pi=isCX+#<5p>?^cvLq_(8fAOmI$csKD0a1Gp;v(+ADP1FMqFVdH>mpC8qjp%&xyD1djE4Z%=oG z!6l~R$UH0}SD~guPg1UXgJl9qY+jK+-mXYxGK#6A5z9i4{00xNUdcrKK>RfEDn;K& z4?W-SuI>)Re|?>=)lWURLxQruG~1dd4PL>jE&sbn`9FzDM;x<;1Mot>1mC90_rU6m z%@Jq;Rd&diaO;On9EoSE;qI~VM1{ZYgT?cH%|-n066;|zF!4w{hHL@R#V6SjtxA_- z^PZrY7H_rn{afAIKg&VZUwn5Py(*A!bR7~liCE?SHP%F&nJaz?$v`@0!$U6kjG68( zIflU+F2yzc<^*r7+$?^&lsHeBu%Iz^wp4L?gqIjuW}Z(UX;`jJ)`~krB`L*hfPbYK z??=LEP4Lb^n0BrebRRL4?noxc+Oq3ZjcTr46 zjG+6e_gPMyW#c;^;eWdrcjY;45MuXixkz;J;pakCdc+10%vUC1xcNDK>&BC1fq!KM z{ydaZjjDGRJdEjd2y4AFNux}A9a6o&CDYpPC`BG!&xQUn_}x@yzBhf!R&PxR4u;BB zb0NBA06GuX;@PAtlZ1w6Xj$62JlvwZLY4KDgm zGN;)WKNq4}VgsI3?-OdR5MXHKvz)N7V!I7w`;B;q%>)lHp=b>uAsh&SPbyhY-r0D zgSGPocHOuyETd;%K3=FVT%1f3ZRG>CzUv&B`$1@WUr7lF za{R8qZG^&SW7Vi6Vkf9_TNsf-eo=o1V@I;g>=Q$O;~4z2+KM5bgTZiHiTJ?2K~PsK zSJ#$F24KsqZ*Y(tC*|1FGFV+l?|fJuxu1fD{-yfvuzSpV=lk%b7XjCU*)Y)pLW0>( z+7p$ZdOXAprVdf}b_({yOzaCk*VwfWs$wQJ6~s;CfHE1Z!gjp8S>M02AC9N)jz^`M zYbfOlE*%_ehVZZLJh7k`zrGru73PtkBf{Yy9`&MR^x^I@0-p7F42!i?=(J*p*Kcrz z9|ZbAuhad*POk@NMf2xgMOH9ScJnYLv8jN2-3mOCguu6y*Y_*Gqzu2mJpU$E66S1M zm2;G!hnYZK7c1ZI6h^I?r(lpt0GVFbIIk}z9|Q>IVR&J_JD)cmd(B^0w%_5H>yeks zIrk_sj9m!Nq;XZI=_dySx4Xe0L6>Y{u@(wr-LHR-FmB(cOMr{wVN%KF3Adde_i!|Z zQ5E`T7i}@XHhrp*0Gcmb6Gb>^&+rj+tNd|!+vZoVFCA?Mg__?MM>6IME(hg9p|kJt z)=lnAw?4WUWDVHF>}WsEQUWL5!yvjTI`8oca=KWi~h%KXuqp?r?gRx+7(HCL{6nJoUJ!qm0TUH}74)ZfP zTgXCyund@(nmRyvTu<q8Il^Q4rW3yiZK0E)* zx6m5qXCJyfri7lpWawQt~W8Wx;@4N`!|7tL-nRk5|Rcub)gS2y1}w^x0ezaNJJB(u#5Sb`+!V zoH@&q8y!ZCrb*tguASIB~|(gaGv^FBSj4=yk+RQ&UL4`)EF1Gf2T_zS*hK(b~FSEh?fF zNV+mvO>$@)2@p}2^$qz)mU5-iNGj1Q(NQ(D(h287C!lK}3g&Nf7;HyGnX+Cxu7NZ7 zU$TlfRwA0=lQ=JPk%M@Z0__Qv+LeA4hypK!^z690^Rv7`n`ylk9Mb-o$F3gw%=;Yq*fM;~ zH3|`3iu5EB9fVA8p$qZw9)A~qGm0uvN2bH)G$jK_NRfw%_o0P~e;OH2*oaFnBNVV@ zQs8jrG`Aj2m$uI|i0p;m7Gnr+bV0V1FnT?;9QD~>VM6|HPEXNEvD^ewCm|`r29oQ3KCP#7ebLY-PA3$Hi9%$UJ z!j12af8no7itllbxnyLn&YjWU{=#4Sn^Ue*6D5-?2dONI0p(5xzoC+PE~fg>9hbP` zmrRlH^V@#zs>o1VhQm?Qis97iA*G=E%9n2mQ{KhIRb_Q^@=45$m=^(|WA%N})h3*y zj~FD_RpESXu{$#WVJ8hX&HxGUWq zlLXr)qFpa))uTlfmT4K!C^-W@X{TV9bu0TCZ=J4 z+4m!an&KhWaHS)YETTabB#eh9pTj(jZXHKdl|z znE35}n%BBTknNJV%i$qD;Id}=PjJ12w@!PitU|=z=@OL(-yb}OoU3l!us^)+!KeM4HlyMx1T$5(`@%Hfx zU-*Nt@8PZFSX~6>@Vi!IxCqF>1gxqm(KzF z+gouQ@U%n@R^G3THpRf|^ZUmUj>$8#dQFgBp_VK_ikFchaTL#7RQ%IGC!W%N^JXy;3RV#`b+}LSIL7D77pW>aqw{C`h0_4Xvm>6^_yg;mJv0#$2P zRP|XHKC`@`AZO65at=@+%>QKouWu!K!s39K!Dn5?}jOvRNlAsOokdGPC`&1^Tl$7i151*c0( z^0Jf5nD6-yQrm?R>YBBajufkmpvzed6A~LArWU%8?U$-1A89~mzRqH1{e~!U_hP-Q zp3R};bbs;9B91K*I$3{E!(?q_cu~Xf?}E~X-%IPc{|+X?7eUz`%n}rz8;D56H2lL2 zg4b<$;B#uD*lNRE=!y+d zqnh5yki*o)G@?Vr)ik30`qiTWWg%Av^1CB|X8%E{`z1NdU%W(U3biM?*7{Dr?f%3P z8=S7)=z|`y;#S$aF&C`~_m2}fDyM$F6LM{A zB$5;S*-Sqg)0IJjjMNi(2Uc{1x6nFuvExj1APt9DYTO9=UvATw;k5OKnID33e81&( z)=wq?W}q}J66z-F!&wT~Ho3al2yuWa6Z~H-LA_c4?wjRwdK~$R6H&3@Qq5;(cEjJO z+PD&0Bj2^ybm#x30n$Q>Rke4BX*`B2OVa*13lzS&f2zLYKTBfgh!;y)*I8lxVvbKUYHQIf~nf@{b?eKzVJbc@AOMDZI4I%Frw7(XT~- zr;wP-zfZL{B~)YR96Wr=M6|IHSW4CT?wFqz{ z#P2BZl@Q}Rr$|t6bUd6*9*gRZgII`Pxdx;ey*v451)_aY%dSp5R9q2R+&85%Emv1# zLefJvUav0Ri+A^Iz~egR%WT<2PJkpwO(vBh8|e%c1xr;dhZKeM>W!aRmALw}ja1Yh zACwun(blmcb#`E$wO+D}I*MdL#aHiHc{4!4e`c82smj>M{%yH;syE&0b+1dO&q6MqLgV%BCpK(SN_jCwL|MMhSF}=202N&+`G== zr!LhoUKwp3ErU{aM=mWB^#^$0^orRc&NtZm>YGxi(K2lF)>;I!xSBeg1jd~L;-=%f z(QjiS+m~XBnQT4u*Hz}%_^q6m8?~Qah-2EVbPk-lN{OZDhWT2Q)Mg6h=4WIl z(mB=GKeu&&^Jq0e&CSaw#XnD%<#`GU=WK%Q^lWUb?qBV=Wxe8(Hkjm$0NuqelK33+ zL<)`S!*Pz~POqlu1=b6>PDO5yEL-S$1%Tq(ZgSpwX%UDac&K8rM?%p~7dcphC} zC!8gb_~#tX>|DO`$kh&9iMX>J%6O;|5*Gj2mT@l>_bya)_Rp$NFITT!&;jm7TPo`A zNfN6yS>j1TxLO#cj0D;aJKbHCBrw8`cu>7$@Q7QM~Q@zNk1c#^cNT;p*;;an5!A*@rY7>A4`xKj+}d7KjTzPf&bYG_n9gY-=9!J8I3rMVcYv*H)@i71Ci(@< zvtkp2Kj$&ZvGO+yWgoYOOPkqM+c#8BY??M(BuGp|xjMyB z8EHncRC_Qykm}pjBPx!nF6_1+e7Z4zJXReq`%LLEg?3#wIuZ-AB;IJqADSSub%RM$ zmvx^-*T>kNetkD8O&OD;O)$qhB124K6-OS?a5sOyTF5+FOqln2?cO2oa>5i63?b5& zFx}^_$}mzoS>~A&VGqv|3g%-A&VmHJznBwSv{_#Nu#O%`|4tr&#kPBGF~3(zjLgiw zUGN?XM4;Av$8Ynz=8q=kJiSTu67>^z%d@Ye0Ozu}p5pYt-# zNSwAHx;80^YyaC7+VVR@s}-^lOt*y2fJtqJWxgLo$2-I`0=O&h4wKGWep1{Ij%Z>9)rleXLEWc+N_q}R%hqZ!Q{LG_IJjIic|25*I``(w4Gx1NK& zGLMo>D&~Hfi7}W!{ANXR66F=^sOE<=FF2B-KB}H+3Df`dvyx-cbSO`MOK5Q=G68OK zRiq}atXAYaR;45+fGoO46&v@vX=o^1Byyp$W0Nq3P}KP5RS#Omir5kldOc#+g>Tu^r1Ye_P_C>qn zZg#uTz^WzXd7l*vN(x$tcn~u{ZZyH{L3m)wATXGcr;qen19g~Bvgsksw}q3j?r0)Y zJ(^Pv(Ox`t3HXNw?i-!0Ou=n58~egHjtu5AvZTmzNdhAZHMrPlKW~A=sf%i(Ht9dn zyjKTgvV~;Wr7Sw*m2Kc?4!lKsGa`4zl6YkeqPylyADA4H=31J%-@mv%uWG~KkF9ou z#Tcl@nDR^4h~-{+YW9s#Fgsi^WK?$LmI1`6A+)>-6WsHY-jDWaN*9;~r1~hqi2T=h z@$Y@u!FYSxDZn}F3&d2<#ozj`y~tNkINlhUteTR+#hexMYs5OVlR3`(fb2Yj73U4| zjwyY)oE_Pm((bm2HVHaiTmXFEEoCh_P2X_msefG7x%y2nmDAkcvuOO4%cVg3(L`qrTDF=HU$G7^(uTQ932f_|i7(DPeE!wK^B!#$ZwCU-Scf)_mAXhBEV<8DYa0?i63N7hAb>n!u!3AX$>5h&rRQPF z(CzMo$`itncXDI{7VWS$F-i}Vd_kVD5 z8_0=@G0VH}H-_;&F;w?!2ji9LYuGIwj8mK)f8#_3WSj!I%39<=!N@V0=G)+ElNXvr zu{0>gpV3UUsb$(f&bICYwp@m2vZwXjR0n4SN4DP&^AzRNsyK9Mt!riN(Bg@CL%c`Q zO3^^aM`)=Ga8orw4g=TR+^Dv(&62hJ`(qx?jLg?CP9M%&KAVWOnhgjaQ3)T@k6thJ zZ_FRhOf2Gr3YfTb($U%_%4YS4M>$SriytqfPbmVV67o`i&U+PV`Yg`NA~Bm}eMm4P zSkaizsB0w31$p^tILAUim`Zs6>A({N)hmW4)h~ltdLL+jfG0tSy;j; zTf)%IeD#Y~NI|Da3%L)SEJOOK{m2;R8qZ2Ux4klZaNyj6K4YD?K=0{2)B2nx8r4pr zP(Gl;s4rz>0{*kCzJL4FoNw5=c|Kl$HE#ZO`WP5D5yhcdb$I@!IXQl3z;(y3NSEDn|ioNN6_{XO0o<385 z+-KMsH72|)K9@OzNaB3iR$OV&ox@Yo%r?3jR45iU?0bm>qJ#=(2qsdKt0}D703;3I zS5ea!Bd;}*iHE&+ks;D)PQt=yS-1fO@Si`BC^ZQzXVG6>p+5>BdW7X2!zpH;$2vN~ zl(1D&Q}CGMo#kSq-q=!FGUYUsIX_dSj5!l}>;6$RGPt4HGl~Y4VWV3g^UJ$WvACjV zFiwjh4UTve`WcAY4T_cw1(7Wss}}wEL}Kk$hy)!68{B>zmOo`fvZLA7@PpB(GSRmc zu_NAp_@jW0n{qA7Q>m?B`z#HPMg!Txz-q2@?~~KTO4gnzUkV-4n>$o~g$*MVUDV=K z85WmL;chgiR*q;5dGt0Mnivh&DMhO>JuOsAdIOn+twb z%x>O)7~8hPviq)bG4e$qNT?&`;1_c;4o(2Q$}(+?x@^($V%pnJ4sb_!hpRCiy0BB4 zQ*$v)fr^|Wzk9CR9$E;{PMcbwt(C{-iD|5j@0<{6kM9g2bTYpz@l(r> zvgKDr^y6e5d~4sE-a;~Dsma&e{nPeW&i>mNk7f!nhO9czDv1uwka9l9Ysxmdhy3*Pizt~uL#5^j zHE=v~sP`u9{@7za0^#3mJ@#eA{V}aA?))Ab@&07`T<80l)ccA7aX%DTbrU7>j(Sus z@Nc5sNJ3C3Y$2Uw`x!6cG7p@b(U)zwY*=n;XlU@-;zNX-*2A=1 zX5B*Z;AK%5Q*ITJ&&D&YWT)t{&k^Cmll}CBI}LS>CyQ216AtO{);{1N1%0J%YWrt? z(m3+tsSYs`uM~QIjjBykOe@>U^$=UIg!!5gaXtA1^X5)DhFUYX&}~&Hx(Zbo*;GY5XSjQKR9vddzOp&%A{V!CfPoX`%@FBdr$NT%ayqut;GoJ4wXKfrFPlzhp zy%B5XTto@}5l57dZDD64e1}yIkge$XUJ5ZOEajIb?PozI)Lh?DDmG>tv64lt5{XYv zD?iHcM%(UHee0p;v$^cXc-c5$^eO2@Tf)gi-P#JU;Nwj;|xa;0w*i<DdhpRZ%CrrxprVm_H|Ekv!V5#UQCF~Dn=6ks8<__m z;b5f9aH^c$=KAj5KR!a}D(`*#Cy2qhsGibI_fS!{LFO^h=&p68?|k7Pqz2c%H%rW4 z4|Fx~z2h*Z0&6!e}t>^?K^!qR#(tfPl zRW7Npnp~Ri85A6}e_UyI3qdg8m^kBtpAbve!4pcv!F z#iSsrj)hCy#2hni3H8!w=g7A?INLESeVOe&_qCE?6e_}fKUe~8Vj!LT`YPDI;lj}V zv6^u6FP@}^kZ@N3LulB@{rIUH)zP}E*BSG&5Fr}t(Hd8sow3_+xZ25eAz4pA-(A7AZC=b)8Sj0axic>moEkla3&4UUMF|#fv&ch1`K8YIJ;kcby*z+s`Ha z-{iG`(L&RzRo~l!iTaY-`)}agh!@?&%jgB)59@C?H&bB4pf-|2+6c6Yb=OC%-mftT z&OZ18uB+T5&-Tl8OhPEgl}RSsyiThFXPn-4S2|uD`bWD>y0TUYERMv4AASZV#P-Ei zv(&0qubiCa9H4CXL~Jf%o$;WtJq{afy!?CVyiIo}psne?w6sUQ+-<-aR95+cf{G9- zUlI8=-McvIhqs+vt`jEU=1rLOt$ z*%i15fpmu_@Q&vxc%>)Yvme|G!S41#u)Bk-!^7c4ac_p+XC4dMK2R;6(aX5hG{3e8 zH!NA!fmG=;2x~D6k1c8tCpd2*8L^{vREBsZ*q!SC)dE~1;~rDwy26@@Vur?Yg?F{p zNQIhJ4qNvYID&?_b4xVoIQ|ga;^m?btn0~O+B}a|5pOzgplHwDLq}%a|1of9m-^@J z)8dMb3)I}nRu_>*NkCwLu9kC>F z{yF<@<1u&WM|#hTOIW0c5o0<*ZGw<1(?#b_X%kulz23 zFtI6SG0b$=XctG#?|_Hs=Q|W9cP2n|>=J@DRh~SGibz92^SKkE1Mpl^eRxMkY+v&myRAvZqCeh!G(T=R zs~hR`U3nI$cDgFNeaa?=gO(#p{3I77ie$;)CD=y02^o<`)BpGqz%#xnf_+Y-`vM#f zY1He<&5R5op`XaHq!Lv{sl+@bz?Y|!yP@h*jJV(KmAM79T9<#O)#|R zOm;~6H#YGFA?#^GDVeIef_d11E~a6I++^>s8WUACX#hE~JL+_)emdDCyo@{W)F&p`ii@71k%z|V8-YIFF3!j9tNUs%5 z_xCrB$c{NE$!4^AB=d3Pmq@~wJ(q|i2VNEq zN0+qDG?CcMwWTfI$>qL<+uYw79F0g_^I6Mi+8TfQyCS!#k*u@n`Y!!XX5hqbdkhDh zOjhD6xx+5>8IYCe$b984&0kYHVznrQ9a@EX8e@vCK#O3kg#`Ztf-$xd&nfDu$t@}( zXI!V@gy_cTr>3Qu6WFL%0@}Bo5zdTtK}kaqPj9pS5Cq{nuaJ-sqe){j{z|Roz+jmD zc3aq1{#LHD{~BSRUIiIWAwkx0=2TX_Sumk$BUi9(0zcwARTAY&x8nLOF*yKtfo*Cb z%I~PYe*gw0M^>^}SH18h8dJjQZZq(WM#T7odr&7d0)xj73ppB>R*O5VY-OrkRsl;X zXr1AhuOT7Nw`&e&ME2}_zHFgwenYq>`0?L~=k}qEmxU!Iedym7Ai zHm;kB@%Pi7KE<2Q^DF&uU10WKP9Xjg;3QoVpv|FRFN;f&GGhqELN?vvvYm1O%j0}i z86uM9x#`aX5fUQ*lF;lnp$-x4JwE4^wyWNMQu?`=6&VR?sHG|lB5G3I<8G4WJESZ^ zF3m9%esSVZPQdQ4exnAknp5>U_#LtIK}<#z8DYK(^os5YMXv#Gzuy@2{C!~U6J-$h~2ZbCx z>~c|~#a!<7ovpJia%Vdy(thy{nQ1{upitvN!%u{cm=~4+WQ1i+<}aXNb`43$e-m)d zLS;0NHxxyhq{}ygqdgTF9AWZ)i~8KZqZlGdAx81#XL)&ee7yP6%F5kdmwWtRD*8*a zDFN&cG01lVndt^DJPOnqiCn~V3Wm)OmdjFd4nP*EdFb9q+_(6!Nl9#XkB5=sCSVNG z<&`yn;LQZM%FqpoY6j{n_lm+GUqbbu*A&K#^%w{wlJGG zx!?Q2o);i92fsZrNC1MRtfOYJ(Fki)&OxcCEf$nXB$xZF+x3J#$9i;ui|2Dz&y2pahx z)4*D+IUO}~MofObqjBJyd*AKvdK{#L>%ST*e8bv=;b2u@s<|_I#1LPJhu8PS+gq2* z?hhz8qr;s67>KmMo<}!}$!tBm{9JhI1%O*gBK=w(u^}c6G&E4qT5ujqtFNXi<9!uY z#ZSFHBg+N=M&D&sRatoRmpYg$L^K*d!UFGoG27y%QvKw|D25jvfansIORZAnRk<1n zuS@Gl04`q<h}!t6P3Amno9cI8A=FFn=>E4;9&gGTjdQhzDc1vJ zY>4(pT{89`E2WV%`6SH3Dx#CixAPO(!fNouNTQ`TJ|5c;XhA?^=pHAUl__l8K;>@8 z#C~+)ts@E1AQ?FxQwIQ8m-g(u?(8Mgu<(|H9+6rrCwCZ*40q7Hzx|^sOI0SmqXnQC zCl)CW|2p1woQC%-Rm#%I7;Ew!-W|e8@^O=QRcJ#rVS&x0u{;ibyjwPS`L}?-ExpJI zA;Qa3E@;L3{+|^@|M#b#P!*7-W4s0tLosNy{|w@E?ljabkD3L81K&XX7LlzRerG0q zStv#CGyQ>|rPRFkZb*x(=ub3>4D`X8wznyv4PsGi@sG^rDYBwKXuYys*cM~?VC9!> z_+)I;kn5c#oObn4%1T3Ey{eXa5m{rY^CTREd{;&W5#Btq<#g#tddJ20=7qC(h-OjS zFP!CVq}EH^lu>Jp0bB6yB&*vk0>3MfrK_$(aI|bF4_zWNn)WvdQ&;b5)%!M zkPu3vwIMfrcps%ukM7lB<@VPP5RkoCQcaZ$|KVCd@cHUvV5=dAy+7LKk1}8Hyi#%$ zZ!^;i07mgsIyD`gqos=-mJMapCxDjK^|gT8&qd&%|9g^IbVydDD%XqfmmUY^QdG|B z1QX$B;8iGBG&D8{9yiAN(e!zbUdaPG8dOnPF=~k#RBW0x(!!YzgRPYm8ZK~50Gkmp zMJ6E9$PC|P)`G~TF>_3%3Dv9XwvJb~?J?_wx8_qBZknsUkER@M49Q7B9K!vpnBoTj z$F{ay)zl3-&4=FRK$D|L&8GX~(o}DZ3BL7Kk@Emr5z3(|Kj*{G{{Fmc-@q;^T3RY? zFBNUQ1YB=3%PUqXFZQ$PZ?E<%9lGq_)-Mb%>Dce-o8SDqHlBvpY+3T1OcLIwM;yoQ zDhZ$D?(f%*enp(*Yq$qzn`+GV$AYVUrAZC*6TXZ`#DaZhF>>yHCd~Wo`nk^{Jl>ZX zSP9F9GfF?US_*J^vsjuG6#0G^-t|}u{Wt!4OrU6ai(1C zui|9(QH*%2wan@cM1y5JJ{|F)PfWkL(zTiLk@sj5mDOb7V!R?PsvEnRQiermqu-7mwVt2feqTIA$TXsk91+zD zr7Qs+b)EO;PFaq9CxqN`5-&Im%~g(B$xfUNQJHJFWuN=ANkt^a^JG|G0V!sH~!<4O9@3lJ4&A1_>pkkra^b?ruRsy1ToiyGy#Hr1K>N zq`U9t`~Ls_cipvEESAD~&)H}8%skJ`o+q_XN`QEvfw4`9#HKM#gE7rig(eZUULh zP-?t{;{~76V|MJI!1!TV>WRNa)7wHsp}D9LpA=QGD(7VQGpLE9prAX~vgUroTfMt* z6OeD<3_{=ha)r&j_PpG;^J^*@-g=przBy0k_Ch);d2LlkkhRn9yNEgSsMvO zl^i4mo;58Xy-gO>W+v)f!>bGt1DTQE(sobUuJg`?4}N)=K#I$7Lr8pVOiT7F`OJZzpF1d7#l+3-tBn17EWFXnqN400%SEbXVOJ zDP{mC+XXvzg~yM+(AucReCZU1e&w~*i9#OFNvsw4bHT#|PkC`YJi1b9md%`rRq-wa zgIsc;JkIv+^$06kD6|QFeOt;l*v_GL`_W6O1h)TppYMOe>Kt&I*_4YW)5xSqndiO1 zPj1Y<6h%@Au|{kY$M){@p~MjJX0QAM>-Ek#{xvmverV}0&?0J>N^UxsWm!bqLyfQ2 zyrYa_e?Eht#GNdmyYnni?U%gcjHD*vX6IgYA60eQr>4$*{-y3Woa*_XlL%j{?TiXe zh^ekr+QhFJXxi>Wf!odZ+2*_0Q{Ex#Za#06F*7FnWW{%SiSM0!?Vjv4@H&$iWT*3U za{TC&i#8pa=i4oD0AiC(r}5u+7zU9p4!nZzCZiSDDT+$ChAD7s?rWdA@)2e*Cvu&rnGztR!7Hkt}a%NdUHd zxn`rkwc}V%;LOa-)Y{rW#hmU$y;K9T9)E_(2E)L{K9cQSn$4EX{N@p_um(YuHi0j$#Pc_lf1Sk3c_YjJrwB$OeMA_n|I zoPrspt+#b5b$xxkYzFtcsDa*0K~LV**}edH4BWNVNb}Tx%gu4W2jNK( z6&M>6dOy?Jw%4O1$qY1P9J6S*gb$ZWTWQuk-T#9=@6uTW~d`pWJ%r{c%UH-}P$m z!6kasfJU7g+X_v~-oJl?OKzNUacpvOvZ?HESD?jI-Q3 zpgi@@#)V>LQ3_ zKD)jE=w`hI`6Nd$MJ(LhoD4st%6SC)!ABUeLgRzIIec$#?~DDJ#uq5lm0DV8xS!Ma zY@oh({YRQ2tE8;ln^5)z10kZydL*&UhS++^OcpQp+}d?k z=UbC1DwqpDXUWURFrW{u3VQzC+Z!5#T54!^nKZDlu$Two#G}RY_X_i7H7;a{MnO?$ zJX5gs+#>&(S_HxGk0~GXugNqoy6!TASu{11> zf4|U!t93o*%kd|SJ}_0wg~Uq7ZKKsn;B_=xE}7@Xpdd22hU?0x=a1G(6k@8y=f;(P zEX$^46QMA-lCcvSSQCK@H}a2_VsJajv^-754#nL`iW2N!)xofhxsD&iSc2`GFqv2! zxaq#(7V^E96=VN&5H22`@n)(HwFQUEJ$&Hla>?pge%&EMAitduIn;{IUZ!}Gb997-U-w=^Y3tn4v?Dw}QtiDfPCfaPlY=)^C9R^gv$V3MHqbZEq4e@ro z={TRixWs8`YEA&%HfLw&pE~XM;;M8km3nq|-`vl49Q)FnlM8`oYK}YH<1l#{ImrR! zn*ws}j@z&GQ(@n|d-t==9jjWMk%ira0(t-7K+3oc)^*jHhqy8+G*sHZo0O@%oKeF? z^vE5!yA;`}ndMP!*R|fBz`0miYI8Fya^fh+xkvLsMXTe6-);0HD^h8xrGjdLBxO%| zd=9{y{xUv`_8qYV`?H^=)f`knD>TXKnV(e@qx0ba7>wg7Rxj++>09WZ8jdtAGDW#Hv4Fj#{+!ZNABJtH@ui{?BLoIA z5Vf0me`I*H+zcSn(eupfUl{O=Bb(Em*PhKv1URATMuY0uS8({Am*w4v_d7eaEvp*mMf1W-NM^mX>aESWO5I55UF!J3-W+kdVLuY-71u z&Eax(ZY%daGo}VbQ3W`hK`W-Jiq*L4LQq<&q^Q`vzE0{`k=)yMdk&`giYxwh&Tv29 z^@~sEwTGUz>j%XEg*&zyTZ~{e@a)md3_ZYNO5m-F{HOfU4EQ&8G`%jrNp~iFv!g zt*SSjd?k;OT>XQmJy%jvZtUikyt9M-0EFa6(ye@y}lctxbkw+PH-uy+tWv?21@l zKm`VHq+S~|JS?rETF}_|7InGWyIo|hy3fUPZ$pl*4)Ve|pj=Xo7Jp$6yW`zTzFtzDu+H*HrZYy@ zjBC3-!+7|&En4iuGGA0atz^Jp)7B5~b7!l&J>vVUYvoz_eM2E!jPa9QRh%G;a5vA> z8H<|}Y@Tk`Z$iv(f%zJWiZMmJ^O&U6RKeEM@WCBF954ksN&FZIHZpd{wsA|zlZXQ0R>s}Ex%Q+|YbbB+S%`^=NsI4VT1TfdQ-xkm8#t|k0)e|h ziqT}gxVOY!Z~&ztmbFGVjC%ZbJznes&K3YOuUiIP@mC=0`~>((nIvYzqc(=q`^RPH z>y@9Ht%+dz#+M4Mp#~;R+M)RM#cFSY@87?JRQ4pv_ttmYy4%%4rxj@Vq28~0eK(`* zNoUwm`0{l)bEQ>^a&u;O_PlxGnG8^uX*OTSs|_U<+T1y=fzrS+37-fjdjGaVj5L*o z_ego$^^s42-PzysYWLwFBpbI|<2w(}5b-?5uXR5F6=3?)cTXHh9++8L=H=%HG&FF( zq+`$L?Z<5H!z$FDGk=X7%o%>AjsSU-Syq=}luwosyOgi5uTF2Ks;cqQA8VNJy)$#O z=cc|-yZOrIIxB4{{72pt| zL>IDkv}c%Rxd-9QRTKb`(4n{@)RX%Zc4Gv{($Ye^Cr^l2EM%bd@7@tLCc?(~t(*Pf zR}YUCItC{2AZ)gGG-$s9qH<9)`1VeXH)ct8KMdhqhAS_;6#-&mYMyKqw~tGUz)^cR@@|4F}}0&D>22 zx14emoa{XY)c8BGqE+ch3fZ&p$HV)yqqS#At1jeNQu zkfYehA&b9q>Dge_&d6|b%^KWG_ZmI;8Pj18`2;LOqe>5o;mgmh(#nhay}O5p32l13dOzRcO<0fX5*zn#DS zBUD;eT2fJ=Z{~*E%L)W)JztRPbm7?g5RLCphqvw2^73f>P~1Yb;oCBs=0NZJ)zOG^ zM%R&JPRbu-1fi-9)mhwwI6omd%tfFp_tlyU;`bpl$ zX7?`S2bf^dlz!J^@LL40`ZqGcU)v&uAZsxtaw`-UCnn!3&&uo{=Ol*Ek~D^Hg3K@{ z<1Noy8)It{KxXCxJzn3jBHKHR6}4a0IL~ntR!>g@*#hXK)=Hri8j7=wji{Udjt}P@sKr6T3)7Af@E!Xcxr5FH!^^RNFeka4R9<^cO^!JVkojy4Feq z+i^!{2*6(Vhf@;1%r2EsEQT-Z4ts=N$AVry!hmLY6>d)4`%qTde{dcnU%=~fK?~ij z)_LU&kkIp8xh>e5XASF!P=)4`^Ygzz2HC%eqtgXB1@_f+Hq4O1st+CI^Na=rq1|@& zih#G(>NJ8Eeq8vR!sZQIx;b-)7!|Id^fk1!^p^o!XlN*a-tL!A9^OL4ZtEXitkymW z5Q{3Rd`(T$sBwZP;I;zvU6sw@*Px&vb>|T#tJSs?H^%U^? zzrXK(H`m!;! z?+AeBb88nWluzp~oh?(uO2~TJDgsn2;9P&UrLUjZF{AM6RxySF$7sKuaDFrN*+OeRpT%}GxE9l zPNJZtl`gRY!4-djHyVyO2Y<>Pxi?BMYb%n+>ddEaH&A@LexIO%f2JZtaJ+cTt7~j@ zd{HU7`Ax=cHQ%+_7v_F8q|0u2C!6myh}W)TyJ{3aRib~l{vD;(^7<%TXWGonY(0p; z*6835Qvd}T(0~#T3J&H2uJ91D)O>jLp7q`L0bT+vR zUsxr?a_xp7=zDM5$phjIpj9I!rt{0oBhnHp*MV64z`^AdNXh*zvDfWAC5x26T;KB^ zL>5->Mb2<1{HVEOH&w>8nlp!>pkO){mbXwKx_!u|jl=QU2$0rTQP$ELEfTtX1;U9e zeL`+^bqx5xyoQd*Yvg~x#?C;a5Qx%~T){@=#l-<;l+yP0bV2sXp~KQg&DTPIIxmz~ zMe%DV+}DbP#6fca5fKrHGA}4~kw$hsqTLf}@1Lg#88nn|wK|w>g78U1zfFf^PPa2a z^eyt#qfTADK>lP(F zS!aD!I(0exes!V}_#&5brw$4w-l0B(b*#aF2uodrN$VUUsXS-g$c=J@-S_k6TOX>PDnyHhWxx_3R#J=vzO3@o15_F zQC?@cH;!i8^$xYbnl%s%Q4-8X`7CFw#23>dXtUMCu#LP(x* zlPU~lhF%~tfo)rNOFfIi*Ds$;O=+05X{_>jiW2YQ>%C8zG z0IooSc(~MfI$As70IlsK!WTvY#A zFazIGQwIXrF2|nwvQl-xDG0m$Q;Z}Ql$Mf$Kkj@M;#qd104VU2G8cH^U3IcPcX3(S z2mCR1xYKnf_ujt@s@vT4td^)#WUJ10W?zKfM`(OQ<{;vVsy3bH^4!+TgO-&xg#a48 zfVor1K&{<|2t5Nm;G=SYF@h~#?Rm}og53dg2sa^gk5o`l;7-*a@+ogL%@=D{RaM~b zQKG6KLagR4Q4{a)0R%_*nGdAoAiv)@zt?c-eFqQ*tK|;kB8~id$NjNsPe2G|96UCo zmZb7Du7{`ROI8SIz3Dwp;;7&>wep=noJ`Z>)ph{J@l#XP>u&fvz<8{ou5XZqzsJYB zkzAI^Le~6?uTj{5KZJ#St8w3Re&KX|l1K6s%~xBw-u>A(bu5i%*TIgZPO#P-sd3Hc zZ7?$N7TBs3>(`L0+osEgCE&C_pZ2e|$0sJ7KE+Usc5EW-Omrf2P7EZojm0M>o`D>g z)$~ZKQR(Y2n2S0JvdEAf^Ef2rl$p(g3W;e>pNn1+-tASZG%nZGA;0UZ^HGio zg~Un0mWo=#dlwz8Mr7^j+y3VVP5d-``_Pr8a3kbMgH!d@w$c)#*w`thL>Ap~7T;S$ zFtYs{0AqmJb_8JB*LU|l-WkK|URFibgk#||xE#A=P0)xO1PJZ{Id`Fc{VaHNcP2n_WuXYOuS9I9?b=tP|C=Ip>gi*J{Uy^bZ#6{&c zVw460$NSXQg$8Ft0v@aEwzg#rQ!VBf0|IKz;`UPTx{r5r=738DVcNbIcGdgZ`e`q1 zEw{OuFDclQjNRmX#nQs!MYG=RK|(mUtPD|3Ifc!U8?rQTp5gn z^j`rA=W4;U8FKYrz5Win<8sWGI#ZI>=AU{1xg<5lH?dijpkUY(eq)4gz2AIDmmVC! z+3un1)r?AeoQ=yK48VH|JX#i^pb!awUC}Z8bD{QjndE`&_;Bs>K(!$!H1JK z_jp@Lrnln*%~`qoIPv_Cep{`fp+1X&!cT74#Vrh`j{KB*yX5JI(W(pxse=s(qg!iP zos4c!!MLyZ{o76O{??DOx9PTN!lt^1dAEczpsl~Ys8viuo4osW&e1-ZHYkH;ONGE3?p-3h~H zp-zxyGoLb|mP`G$MWU)&o@{z7;kUOuWD)&VbI9+)G<~xdq71*ywkGpBLsxk2LWQ! z&0is)3N^hd1l7yx27v6j!Gnn)zV5?W(xfxaF^P~YxH^>YHx{405kfvMIEswIpSjSg zicLp%2Md>X2Ud$sA+8`n+sq}u^SF1r-M$2bM%eJa1%U5jj7QO3TNIQ`ONqV!Ui|{r z1xj%50f(GiIHdbDt4kKJ4j?Qi4-cprsMi?53%0ilT#s_Jfpw(BkXmNO`TTrG_q30w zn_XX@0L0SSY?gUNauilYLgR)g8Ekd>1g}>HEYoyzDmsX#S>4wCf|G^3`9Zl$YH7Y& ztVe%mA}b0+v=W*uqvzIRu+i=D2~uzRCVcx7=X4B@GX(>B6(4*^SUQ@<^-tSA;1Stc z(COYT&i(jgc;VuztgI=cs>Y&K7nMBGT_U@K_TZ1bo~8W1H2U&34b~zbD-h0;tUuLl`kq&zvsIuv@3Ue zPFP?TIy!b&??GgIQ`z2+E0m_4R?WO-O30MQU1_yZhrE;<)IJ zoJLYmNCz=8nlfLmRi$a|us4r3 zkK*Z+%#D)Bw9`1(H{8>ECVh-}Jux8xD|`(_lp9G-5i>QlpOu&QgS-1n+J;9?8}WOP z6_lcHynH~dgqDTD)XwZj%OXG6Pp2O3KXsqcA0Ho`T%46uRmFibtZb_Zsi+ulvr4yG zDl9F<*PaUzr>HYh55ZCeAP>Bk;<7RdUID=T{>sqTO=qInV zk9Y+}gOYT|1)l>O@-Qk?<0%^Pr!e<3{|XGe}De~ zbTxv^R8dhwO4pVpM`4i8IzjrSVFBztIXO8XA*-aO7QA-63sS9S2ddkyZ@ON#5_q@yLkO{=1bBxNY%ci7fFA<}Zr(xwkSDY1lXKN`5eoaHt8rWB7B*nZDFf)m z*;t1fHLy1xK7jbq1>gpsmulPJV{pC2B9Y&ORZs8?;B?FGU`#KFR2weM?nlk0ikJzDi(i3}l-Qbe?Or;7 z7KJv{BV)XqfpEOulM_@>72%`4DlMhQCm>J^?76#S0bJtWs=KK0ldC0LWw1lQioPVa zm5n%b#x~=?bby}WBSH6-mtutOD{)e!kd=IWHA}s9+)ng-iI0N%*+g6lS_n0A6np~_ zY;$rBB}u=aQTEvQ6F=Km82U-R?-YN-8Qt=M(&~ao>F@j4CLS6)TiI;q4vxHGOk}LD-G)S;l78GhEVL^z_85=tEXg9;$X=awza`GDtHT_rkv7g%GOfB5JetYIorLTd#VTUVjpG@YwqTlG$GbU!k2FNmiZ@q+k$8ksg zEP-CZ{OLWybCZsP2Y==7PEMU30-0JXYgSLjq^P7ZlIy2#4xa+_aZ_3fi2+mkN*<5k z_UlO?uZ9fU`_JA&Bqb{Qn8&V)ifDUI2&%IVhGPxZN+l}HpV_=gUFNGUO5=OGruLZ&_(5 zD0^=UnOg_so!3YztwoWNS zOqVWjaOC%l@cIMDbV;nxt_G{R^doy|BiwJ-t5W`!4jXiE zuh-7JgbqEk?ANF14ddw*p5VbFaYuzqcPtt@JUix6jM!5Q=NcJDN{DQiko{s%CCzs) z8jLkSO9D7CDc@*%&q*8>!yUf-FicaOYo1#4lY==4F=<4c{Ai0|zvUCD0$BI_SSiqj znX-6_rlLBs2OO>(qv7(~v$ks$(2SPiuDOns$l|2ZvbNBGk``z<;G>BiO?p{Kw_W^*ej`#&530n zq2p5yMk;)HS|>DrYm?B<@ZgXL_RtB9%+Yi%*|-~x9V-`M9$sEaOG{0_jHwludLU33 zHyHi?n|*1O68^Jhq9IH$^ou~N`#3)Ntv@Pxl3}tS)LZ!&8ioP0C~hNO40hwsgcjEv z+a0*?Xc^BoL+Q{4-2$Zo(mF#!!@pUHDmGm;r(xZisp?IMDvb>;zQ>av4zb<>lG=nmn0u{V4d33lkF!g<7G`CKpc|;3HQ#Jb&A9YrA;_3>B!GI{d53+rkwj5t5wH z3U}Ty*oejLMEH?R>0TdvlbXvhSk4Gl@azcZSeSOB-A7AtHN^gD-ObI}4>Q}PP%+4= z?z{O#=_zykwSb*a+n%KQkED*S1vkv>%4eY^&@^uM;@Bse5yw&g z4y?uDd!^y9KaqA%-hy8%;;cP)9Ct8w=^+dK8^J>vEDt*umnrO-+DFR9gexXEo*b?n z-j8X;o5dd*BS+Q!f2*9W4yDh93qI*#C-?BZzRG6MMsq&qMu$GH;mhX9_-F% zBZCaa5G-%uXdn5XYW~kTl>eMaNkT}`?&Q|{_$vsX5J&xr18Pr$_+bs@V`qrjA4OQ% znF`LuH1j$kM6?D}sevDfP}?)nJ3V}&P)X={6eRhhM6ptY-1}BALkTUBRO#o*R#pj! zAB(K3mZTv#!&H1_bhw@Wckty6rX3v_)SJ5NY23X1N9i%L_L?#*MyR8A_MBNCd|4tS z2N@yJkiB=&W!K;ylPIi4pmRuLMpd2Z14K~M;^XIe(=A^sIgB>YJ>rs0ki2~ZX#@wF zKq&j~H)bwOKTF@#8FUg6<*r!T@7Q;m_A{d9cJMQ=%{lj2c_H|R`gwX6*mesP1Qo|3 zCxm|NpB}QA$EBev7zr!Kbo-EF_5GHK4H(fU^r@ zO@szz(B`U8okI$K-=_?G`THxwDX~PVR9rc9ovX)Xb9}?kK9S(Ofv62D8_u$@i3J^2 zXHWdU_q)c;o6j42zR`+5#{Q`J?hPxIxG>UtXOB0M;=Hos%<}kcov-B-yKH$V*L%rq z!YJTn(w?OUZc*?Cm|*aeY?5`K68c%#e`(PEV_9eqq|A_wZ7?dNmoGFLcV|ZbzeP_X zfQgo;mc%>Ddql#M_!f-~;>v-mIZ6I|XSV|8!O3-OkyOMRrtf1?H4Hq^DA1j#Nz1s8 z5!^n33JRES#8nm9VP1vi=9VHze;Pr^$u1rD)aYXf@TaGo(jF0fhEz5;aEp&BD4cJ- za!DRzKT9W1i!GZiDn&@#&eeZ_NtIqv|0d&_LkCUpE~z5d1O`3y94XY{ECo}1NP8E< zNekzHPer{+$4bPUiAWQZNg(Kj5m)87$J-9~r!GC}DfqTcpXWWrEA8ZmiBOuGO>1GZ zmKVN=VtoB&pYHe(Rz&IK1pDOu_sQPJI9^Ee&v4zGA5mW09f-+$ypv|(X1roO(RxK8 zpF{#g&^FC6$hsu^;DUPsL}HeQ@?t{K5CY*O;>>iOpC?wm2ZmJtgpoW86v<|-EF`WX zlEFud6f24IBaq|3DPg@~acRHG@=&d~T&&P3kXD3IEagOl%b}#C^sAFwi#>Gqc)u$! zaU8X<@UIX`$KuMpQzve8G?VZA-vYz&Df*w8XQz1RKJi-VR!VeuQ3z(bWM7+ z!A|5K$1!kAL3@9JI%Bk4Q6eGV$^Z5OfMaEpv8;w0(EWEX)!(u0rZzYR8@amUDfl4C zdxK6S6#-t<;^dj633S;oO?`wVDyg#6a_lLE3OWSx)0}eCoQLJ);Ztv(9#?|2Vq6ot zn-&oH+|uPH?dgnSL2I^B3FCHYn0`}!t$w87BPTUX*aemWqdISrMJHvNe4MoDV18Ua z2Oj$Z-N^{Si$?e%025zzo-S7Vk$2 z&Q%{l6eDbhu}wUy@?(Pw{pl1Pj>JM7l0s39fU-h|esQ&P+gdGZSzOWMWEA_p5UkO3 zd!EL}y*E2LPgLU?oQVxM$B@QG!I1z%;a?>*vf|i1;pOtDe=~eH3y7q=pb)7pjYYvS z%1y5tL94I@t2te3n<|ld`5-NQB31lAO4w^lkMN0T*ZWLbp0_yC?VKG97k_I;r~bp= zpvV8(@l=(hSBo=7lSDhpoZi8Os7Tw6I#>=-FUXSapZe#dV!gX=4#{$l0AZoFfF@sh zlBanrFFl)gmomq@pHT#4`jHIjO9&LX;Ppx^-cqQI=%_ASZg4vj~6b}YGtmA3i#*j|*?(IlAz`;*&q8yrcp@D?{q&;70h z-(?jlM(?^bV&1%Hg#kYza)t47_zE;R@Q7b(=_c~b)N12@!)3kj2B*Z6P_h$3Wwxc-VV>MWgBnOrPu30`z`kJ zx)cuec0RHYFB5K!azy2c7~9Rhf8L$>k1fBV_|OdPxolM$`95UdFtX?_86#*dQphhA z5=q0}0h^Bqm+xOajclSK$$B2nz*lZUAbfvytwFWxI7>CKZj5l_6`ELRZKZQffBn2C zpSikeH<&YhD%G7&5oM}%&F*u%)Twmt*GCxZ5lQlVr~iq_lmG3k3fu8(oSJV*Hy6%x zaT)gfri4NvUi`v8i?R#N7Zr}(Mk-OEE7wueIWn%jcW5ofpd-;+sqzqv=aYxN5O+R_ z^Lq#a#Z;-2v#BIEmj#!9fko`r!(vv~vdrl`ff~wXcm&A5&E+=Ym80@pyol&c6*wG` zb0TL2wAbWv9OxL*cJxyEi7{m1p@vm)(S);}XE23$8!ohk{@ttig>|uqEnlRWbUgdZ zkgDiCj$9Jj526vOqUj&TMWLp1@LFgk76v@=$ zmZ6U!-^8Rf!Eh>Kj7;VaoW!7SzR6P^&EN{VXu(evL#bq51MTsM3yH{Wllss;Qc7nH z6>$Y%@0&0B$&&_z9jLPI`&V*YpXHsPYXtB;__v-3gb&jz`92|3K5a(1k;B5J8>CtZ z_2-1qYcyVH;Cs9u5LfDoFXz%$aZ6l5&NozJKub9wGA@F@(Ma2 zK!pACr>)a?wl84l{hOkhtA4J)E~`+Z(F8s9s=WT z4k~Smw&TsiN#59u_aR7Bo!NWzaQkH|rYhR8;J{<)(GC`9O+iU|{$ry{EQ6VCpge|G zK<~vN?XY3GJ!jrc{sWu%a2^H3BoTHcTuJeQE96F~L@zrMYz;41QMIcgd~&K)(yI9^SkVqFZmgIL zdyF(cm%6gNPiW$`Y>V}F5B$d!SGyJ7SFIb%gjRR|YDCgevgvdPgupY$JDJq_}* zz5!;uuPVYeP?XWQ*e<7SQ!%gE897E*TR5vv8E7K)L< z1(X$s?lj+Lu{e$ouN3ZLP$t{m+P!|}cL zyFaEug+tO*XiJA92m-H3a$?EU;lWovHSmKQ^FjsY=@n*41?EsBStIxkes0oC)k&Ee z{jJ0N@qRqRsXq*+T;M&oZC_b>Pj{T9VhQt~O1Bv6p;sIxk%iuU$^x|&DNjd-Ki8&2 zXY==9;tt$5hqgKEoa5|~uDHd1y}G+2+}9*(aQV0WN5@E}QAQrYN)bOQ+-}d-QvW7Q+Z&vcp9;%WqB+sxKHH;*Tx>dk@}%Hd%-pj3ZX(ifL2Sh@M|$pyZGSTJ%6 z-NO*v_Y!)YAdwp_fCFzCAwv9L)ApoyVro9PT5^;NO69X&3Iqecky_Zk7eJ=4XX6mD zG<0Z1-B@crq3OtB14jFr>~TRmoDwg7eLvsqP86~CYPu*Y2eK{UzMcYFpI^O}DKxV< zB2!Wep016!Bya(AuDGr(4igiR(HU~H3e9`hh8Q}2qx)8~W2-#-1CvEWMU$~iX~_b! zktWVzG4#k_r0Kx9eq1;OM(pQ;Ewy$(8QaRo?Ht(3EqEv! zGKr6i0pGEu6AIft7sYOc5`Q6;HKzPPmO-klaN`eU)%%MC;RBf{d4hO~&qISB5gdeL zMrcVs(CsiGAbydnMGWpETU;)wmly@qvR0_{g5>H-p?imv= zreVjwNnT42btA;QLTE;sLO?%D+nR=-(s;{(T1bvBhEy^$BoZXn@=X|5?3GU^N>Q}L z{72{p^0Xnto9{3`$*!$G3T03xF!}El<&yhP+|6lv%Pql9e8k1B*o_%7V!eu$l%azU z?o}^ro)~T@G`AI;u0b1qqrd4!ywL7tGIDDL2zTWoc?M_CRbBtm_{C$Km9)JG%3*20 zZ4^Vq!1#iK*aUsF^3nn&xcGQCCA89?Wb9S8f%Mur=Yb`7SVV*BG82c#8Nt*ImDkrN z;`H#Or&=Lc#o6@5CjxvwM-e` zl(JtP!1amoz41PN=I%^TqAiKe6eZK`V!~V)5GV$_kAOB% z#D7<5aGQ-To|?rcWA(&p;BL)#iduf8dTKY_i*<+_2 zuohenjhlNTkMnuDPEb-{t`BJ*#-9w8IdXfzly^Raabc>iw{p+9@!_b0Bjc6je=l;( zRj}coAm$ZnB_u&@94kXzRaJeCr*(NpKM9^eFkUJ7iB**?&5HUnb%sb`T+_$bR8yF3 zemSjeSJ8p(4G;So2QMk3+;rXRhz}bZc{L`O^$#?(5Gu!~fhd=i!PNHv2Ecg8K?cHtG+R zwz~Mj!@R#cSN9$Tn2FYg$vaC5U*)Fb(`aI7sT%v7&|fLa#;WNpaaX#Q=XKjmY!`$F zpcXjWzK#7tM>S;Rz}mdhUhirfa$e=XzoqscyO;BvLr0Q7q{l2)dYokqZ@00=g}wF4 z6v;VhNK^u~XKyVKeqsGkPrr*|Or&lI)jWHc{#fGQl$gRU`z-bK@w)cCX#}zYAp?q_3B}bFet>Oh`B| zDOmm=Amgkgt$*+JV@}dzNSeA)IS>;^8b6PMNdD~4rQA|CS7Nra_axF zjsJM_%Boi|t^I_325kB9ph_FsXJF1ypzsaX?0@a^#qO`s5kDyIpJ{lqBX~*7=?%=+XfdH^^-) z?1M<#75B5Wdo*%LCNaCX?Hc5$ufpM2XydR~fwM#W<>@ihVnK2$L5h^8a&#v5!ODrs$SKaG8#F>DOGkAk9SvQ z$do&6fJqmE23u%ukh-I1$Ajc|fO*PVB@0>s*}~6Y910Fc9%+%L^Dij;^y9A#kxMY+ zp{u%BnPjz$+yqE&|LyXcTTu1+YkI8Wp1EU^u8@OkTQRHA`&N7-toeBB9Tc&PQ&1DX z=6h`|H!VgR>@OIDSo!EGpM^nj>~DOuD6xl7gUjf;=iDD%8RmDbKy0!t%Y46V%4;DK zFQP9T0*YX%SE~hDHG7yTUiNA$ihNr&d|G_J)9kXTT+|4$aG}!;(CDP?f3dps%`17a zU6DwPwGSwYVzccd?@UeT4xrWhghu}2EbqpBmz^&v6h{^}7CjViv!E53I}(jJA_~c9 zmJobZ&0xQpCm?$8PSn7}{nO2v4=kiupWMzKx34*?C9-xWs!;vpLvvGX_aklN4@k&{ zb2!qM^VmBK zi*uwb+=v-<8`W_l=U_nm_fIxfhDLIiFo4Vs5pVNRija!cZio_+kBFcwd6vT4-ri0* z#kbXkwJgFdM}53$6nAmiZ~r=v_ck|-&&vvP&kd6U{!ZB)6S@}@6Ewg^w#PrW?-JW| zY<#k5-oExa3|0QILCVG&ZKP}u&e%yABgR`GLFWjt4?X!^PmrVGip0t#HOLhr>V4}m zxd?z)c}w*lGK|;lO-piQ+~8LC|HIQ+Kt&aGZ(jig>F)0CZb7=cyQQQXq`SL2rIGF& zLb|29JBR$P@B4qhSuEF5nVEafIrqfg&;C8kSJyZ|A<%NXL19_69_6>(=J2)m(RwwfDa#Yd+Iq+XuQYi@)b`WpHrKwO&BMosQfY=xy zvbZwEMDizr8nnr<=?c>sp4wOojFQ{?6Unr>nT9lZGhF;6RTPO`(nI4Om2qR_>${lS zayO*Y3V6=9fENHw|IqtZ02$fsEd>pHM~RS{FpU^VVqSN1>UtN{Ck;|~e`m)cHm&Pr zcArjxfyCk#T6q71-LQwdnUzHaS{0~#YSS50M~|5+LC@!>5gtN2K{^37Xp)qw<|dC;S8TN^0;fAztu9~+21`X%2dBRmhY|nMCEX~tL>P*qV+l| zo)9zA!Xu%0<~%(JCGTmF;X=Uk^t8>bAD?xybBQKz;+&sjR?}px{rPADVbxE9NX-Rw zGeUQ8v#r;By&Xedh4AAr9%TG*+LP%Hktjwsw&22I)Eke_UW{l^m)Xcvli$iFGdK{a z#XOqVj=Z^vlSu7qYqodHVCtAK+@giwY{-d0Dj_i|zF>1|Hu_Q+zUy@r%@=lMlq&a2 zeFiUU5(jQ5@%#EfhD+vf+)gr8Zr;=?C`~kWukiiN!`!u7a_#Y8sbzjIK*1-W6`Wd@ z*^;GZtv)+zBJM0A0lhIf;e(tnLPPpMe@TAFdPa( zZ$@f-W0uw7__A5DypFwn@3Y?rv)FUXU5wg>CXI%Qe0Mt(9CDtaC*F^J@LkujzxFje{U8KC*-h3S^@zz(dJ})caNCI~tN9E@Q zS*m{{#>iI^VY|ZWeYx78>3DLt`Z$SB^g86$dYd-=+ihUPg~?+ha0BA$S zSI~XFxYGM>qO!AF@ZVn_Y?VHAz%WgklzJsFWU~ z$@)7ErY)HB;N4k(O7DuH%h@x&Cfk=YcCIV8#`i?`y!9S6t3h!5p;{oOYY3B;%3y>h zcCF{Z*Oy2|;T_w6pHkL@RK(?lrMkQI<1FW}g}{E+6^`%ip2zCVtNo_?CLFfub5s3p zezGPTZR=tGjypIQzIt3Nw6VjiR5BuH1@vEOb1Vjmizcv#^V;tU~e&2OZRo{VjBEB`9pLT9x+=v)JooT z;q>Sh!*D-H>081qkBvOliN4lYe5!B!WJh<**@`FkN&mrRAA`{DLa7(`JD<>Qg$K#; zqAeA>m0f#CP%B(H!aaZny_n+(yn`_rp0MuD*#D$XAk=At5ELsVR(O-6R^kAEoUaoWR^iM=x z31|{tx5aReg@T0dy=va4&<_a-nCJ~b2U#F4K_}KRzgG<3yG0l5k9hfc?1zhi5m)$5 zFGt?Df9bfrqk1E#4EnM>Ah2?}!b<$!!ZYnU{STgp9M?nTjsBkQaK8u9@L$hbgLhpF zHlc-DvnRqq^^cV%Ryx-fc3pQ5{A=nXtytNUA|GRR*T+xZiT;B;rC-F~x#?@e}cfKAi; zT2o!%R-uipYSrl`7Cdi%VX}9fjICn9YV`z9iCaalgJ&_idi00il7BD+%e~$UHdWsd zlfds;aMAbL#QW$#hWj>ZFpGP(fNe_|1)2Y${p4{e)jn^nHXh7h7}FY1qt6og{B|R= zdap|6+9M(r%!56A~WO|0ox0(3VEsx=bC95DO&|4^9OtxpM~@8M#6rcS5W|OI*mo`L>^ZYhrPgN$cfmVw&e=U6$Xs zBrm^58o{gL6Ax24uC1Tol5xT+s|$+2wH!BtV*oG}txC(WRKdqRImxG{Ea%lO84fe> z!&Pv6+z$*R^nOU?{t{AE6p1SK)1i-^JYi&6B zo0IG~;&%$PnuNN;xj&1aXg9fK2Tsnl*8IyI$pjFH-_sv7H+S~P?lcnu;0pJC;_oBSveZw!JG%iKt#>u?lmkT&n^AB zg*;`~4(Cjx%UqCCrLPC{=>2q~IRe44rMMjaxV*~D@-^*PDhElRU{9~wPRr$NYTVj2 z@??!wuYpo{xP(1~4c_-Mf<@Lyyex;<{gU-P$ zf0Z;mC!3x9ZVL%kxs_fS2L3Z;Gb$vS=@;gz5BYhy1y(&hJBB@^?B=X2lYMTFtuG*5 zxUui1BM}6TrGNUr(;`Dl{Z9*EZD#F6)mLY!*&XxN@2yEq5ZQ=gQM=NB46)J#sNytJ z{#!4NXvgjRr^XIp0nt)=n-|ePE|$ZuJ-#&dnNh<)F?5vjDoBQ3MYy>6I`sX)N`05~ z+Y4t1weh@c|2M_qeTSc2`{GT`*c}(5k%t>E%#%WM{i`SM*(xPcoEy#QUG8yFVJx=Z zVW*?;04k`2R5`DGu~&H8LyW!~NrAekwDvWjnCmrcB3R?PbyB~taicA|QcY}1x!BIU zFJxlxWphl(ouGe54ns0P4nh61Hfle?ezbZvd8m2kpv^)MPEdF&2&!8Zx73nLb|pRN`f9k#bP6r@=+Sr zhONK9`w{1@;XRwlXB%GdwLyYLeW*6BJwce_O4okro8k+z}txE&p)&yQ|X;;-S)kFqpE%r4iBt`JahpolN6%y&CGIaeOB37s= z$6D~2!TY9}br$0e;|{0ob}rdR@zM}37H9vO=qluKv7v`6(s>POR({Z z%6rgY_jb{(sQ0$lnd-NMm)G7>?)ylM>)G6>LO}x4%IS$*KQ}TY!PfL#-D@6S{0zBz zyN|QKL>_$>tlt^_6RZPTQ9Vwx5;b=+)}ZNXapios>>bQQ6{g`aU{QCI?xl4fe{nsS z`<4KvYfEFEn4Iyl-&$U<{ms4TcxKT(`nx#Raq+CCo`Ya(X7)Syh#=F_CT4SP;$Z0# z)z0Qm0Wa6hVhpF$}e@ zLeiepbizQI?W4?=lvrMUL>hbmYnb{Xv4|i)Rqy=}I#K&0ZkH}!ve$)s2mv1-_g%li zme;-Pxvx7-vycmB?(3gem~qc9CNJEmOacHc<;^X}PAF5sdj}-z)8vmsHBqSSVsfmn z`4`5NQNQD{9VosQ0vJ2qf(k$`>dGB|k%6HE2z@u%k_p(?dflEoCsYz3e@;DOi$8MZ zoU^A2lq>la+%!Sh=DN^eeujwp8V#JT;aHU=^paz9LhdS;YqP&G%J-+Zo@$~u*91Sw z=^WB>7rAj>Z_gZ9O&*NwM%*rR52*a&A~Ewnv(}hSx@T!)l%2hXZaJHvI#R(D#PiH- zu659Xo?-4~&Pyvcdsc6{So#yw#6IsLh|6h5^-KEXF+GKNc6PW;jsP8v!N%sC1ihMg z`I1#eUsu$A0+cq*QRWN>R0qYxRAcIEHTXvlBtz6PHfO(mr)}Wk6B<`CMt;E-+}4--3ee^Hh2IzCIuw4tOyJY%_9PhB zv$gSdfzvNaF{}hP84B6~9wgsE%*u3c07Y-#>q{RacS+5{P5@aO7+qkxu`mwM< zWA}E7mj9%r(czWpF&|nRUOU6jNvk?bVf-EI$r~8+B|iy@k_3@vO2@gt7t~36f0m?D zpv~^RLN6g8M*U)vxw*?$G~-bls5(R%Clf!+3suM>(nOpaljtf(TZlrh^7DGQL}eG2 zF@%5|7r`VmLYG%;y`j7eCWZ2M0^*S=E}8CEa>E1#ieYpFT4N@F+^t0Y)J-}|=RV2w+rcII=bVlZ5Z#AfrK-5E<){ zop_nj5ls9b0%BZ*9kx%wz=UCeS_NZ5-9r~Ay;oa^5oG(7GM)HhS#_unhQrhxk4b`5 zjT|`~we}M+aK$a+VG*@8sx`@sN2Un@>@dLr)Qi2aVzp*D-~N*5q72#tDR$BWE!A({ zrYcg`G2gKNDNIOmiBqMO6@6(7zW%5uXZ`dNFD-((tNaC)R9I|)l0bnJLaCs4CdjDV z6nC;V9N0~m#h9{22gXK>Qu~S~NKuSn2r$zJY{BIJ45r_gqMUUPiOt*SbD`Lhgi<+l z=>F>k=zdlX`W^J@)8U8Ss`pQszzpy98I-+-$P!fDe<~PQ$*gXI_Rvz+an9Gq9;w{jUUxnHkVH zYICoxUC&Bi*4uON-ZTv}_Ir@uXShkr(P1E+HXjtFw>=!)(?8G%6cydR&ETcMn;PQn z#<3!TUmxm(C}a||sd+BnGF*Soj?{3Heu11tmz>1w7`94Nr?K;3ls=<~K^*9I#HRtp zUGH3u zMgYBdF=eOLDCrony7V455MlP4VudKNEd4BYB~#LRgEPGB7{t+*R4{hf3bSG=cGM1} z3knB&$c1N3?;;JyrTgZ^je0g@#yzJ;;m8Fc0dhTqqLMO5|6z){u*0OnGa~dNZ~Z1(75K`&A2g{5p_lXWuQc~<2h|qKBrw=H?F-M zO^)`D+!yto&cEQf?VPh)Il}=xpyaywWtT)NdKC@v9{pb```=EO-7~EHzxi3>S^vI3ynu`0pJPrxWLWU=$ z25pe`8UhSLp|nlU90|r$GIZGdB*1NB=LSrT83{c3tVi%MDo{Q3wte}`p6ePw+Vo8L zzJ#J%g2;!xP&{_X1z)(4mL|I{v|`=0QWd%2RU3EU_c=PpeW)mPKZI_qk6f~EH!ZEh|Y=8IW z0&cIj$tUp3rM1g6t-GC*S$#DW3Lad(F-YcFd8_ z5kFEiV#}2Pk1L zpWr-hrzQT2_)g9qmjm9e8xafvkD~%#-0ju6a=U&wIr@XQDJg04lO7VDyq~`GEx68& ztejE(O#m*qZ{_X54~SX)AP=z0$Z~i}|J~FywLINh^5ltWaIF+%bh-KZFgrUd3Q!YT zbrFlvsj{6>v$M1Le{A?biIM>=Gu|jE;`Z1nhb7 zSW43j#SqA7Xe@?thu@d377#`bTH4!#>MK#6#mItsT=Xh_{rU*hEaOds3jZ*}i$~*% zd3imDwQaX>DN6%sAnPd5 zVW*du%O{Z*&F0FKcK?Zu=lEtI9_&Tvl{@|qV6B?i%$z?Lmp!D(adD-XcE@^Cx1{g#_+hh{B51y8tnmw+o#YPmI`7J?p0F!Q z0wgPKUNTtJK+o6Hx3%*EE| zVJCiNyKeD4d^JD89ZA_P=8@Hs?eBz4O~{mh>y z;e33PBxypkiO9C_J-A-S5&DU9C)ofH6)hc|6lG#m#V?coCl`QByU8tfEFN8;Nna0e z>w!~F^@;H^CxHJ*X6hVRVljDqoL~HLK+p_80TdxIuF&Lp)58;~SRUd;fmr1;AqFGK zfS^g${~wu2lL(VH;@;OfF$_;P^#@?1l_q&#=#b{5*bzx7-;MB%D>*6OF}XWFw{w7_ zm7=VLQwV@t|AF?%VtHYmDtc!X-U3U8hI)&mn{1W?>MMR1D-Q9wO6i?iz1wu?- zGobD4(j-!JcowI9AoOIiTyF+o(=z~Qhr?q58L)u7+^u&{t*yxsBLlhse!_#+;z{7i z8K1Gp{R9QfZ{0#K%m6-%LMDaT{1h7)E0)%_d*yzDQq<^wcLJ`u-u9mwShHdn`j4DU zxL-YRg~3S6a2ll)WQUa9DxozxK+aCGu(QUHvoRS z=y!z35eJzWE8D+!?s}>uZmdBbyd^BGwKE~fWjr)7$>CiM^l;;Rs<=gxX1ScatPT$} zVI91DKAb?yxX)>22OfA9@y-^TeP3v9u=PL- zVOi4rpQev2MBbO}9fYh6GQ4s(r^<05+Ka zf@dfqRNWc(n5YG}vB<{E`N`4+pW7&8%7tlVbAzpLf!c5*9PARq(@GXuzYWt0szG64 z!E}oom6fB_xj<)%95N;4?N&grTGVdMpJxOcWJL^xUE(ivyi+tu)SncLc4T!rE7uJS ziGQOatngn0{erpF4cGY*pVL>*@M0w7+B1Mve zXqwMUp0e=S&;JHJ@bljeW5`oX>)V}TJ7Bh&=4S2GLXSJ2K(-%`JDW_D*&i8n+C>ZQ zwHq6yV?a{R$k#GlrhKcl*KmAhGW{h@OP?!MPbRnW#E8i20Lz!ckubLF$`Dxe@56x~ zXzuKn?|G)l zqv8qQ6T2Ugnz$N~vH>NKftqZnl<37sDMnWtk@E&V4TvCP8-7mfJ(ao8ajB}QSa-C4 zYf+nDV-rZLVwoL#&bpD;(9r1iyy$v3@9chle(maZ5%=LV2Ta)BP0F4 zSCin-n&8i{Fow{dQ>jJIDyE-jWeGFC!uTgT*Y0q-{uywVl?CXN{<2G1lD-!<5I68xa?_b))?6wGuG=7#*f*0jQdq1~=b)af?6Ob;U6M z{@bvwiD=X{sJO4g%ZQF00LP?X;AVi-kA#Ri*H3-R^S_HS) zPf%eB26<@6eJ-e6t&FLgbE3yz@af^Mx&jFwcwb$))psfw(Gx0S%Wh8DY-3~W?F!Yz zuz#HNVXs1M#YOlY9e!j28# zEpjp|VHm`sQrJPm!l<7UG*^3Xd_;t7aC@H``jD|x!mVthf=B-I;e4W|FXG=S*;rw4 zylaWv%Eis9BaX1!RHr`+(=gjWHcuHoWozBP%`rYbJg~aiee9o#nAMm^^=0aJH_1}} z_HBTZ@)A_8GaeS+PkSspnkDEdQl9tZYIg@jSPtWk9s^S8NF zv2WkL*`Sf`aFH5k1O95C6RNlH2_UUXV#c=@eT}1NUtHO=YS+P-29Jr>p133ARVrCloBn~7~Z0yBDIdmXqS;|w7xx@7< z(}LlPxq^>XkoU;SQLWmIIa?>|ROh5x&6_l>Y9prKquumke8W}Cb5W?9n(8h--P z4kr;a;lfjgqsA{oh_O=wDP*@4DYhhYjxa`3Fw6vK4#m&ST3nhH)CSz7q}vmPC*?A2 zA+z{TFRw8DdG(5BS0Xvry1{*>;WN&`Q*bM<$rue`2is-BC>LQ4j|W7%y$9!e(A_un z@>imY{m&o^N`ReZKU!N8-3+FW#$q{*M!WW=2r**@@58qXvX7b=@XIQG zY1KH_ze~>}G#r#|=^Oi{Sca_su3#UZtJ&vzir|$43?v;Sqo22;Go4b~1P=Poom}Z2 zZu$@XDbtuOtlB)CB{81m6Rp^vlS-tsp0_@o`?v~`At5?k9dche(<^t!0!_2D{Y@Il zEgrZqHWsKXC5;+(T>-7*h!K#HP8BY;^0lBq&=iULDIeIqBKH`-wk@BGQsTh zXc_jn95pMXRge{Zx5WntcopVbL!$PWC?goM>onj_SB;#d#p~Jpjd0K@Ww;$8-8n-D znq9DOl10lu^rITWWF_?vYEYHatG2$QJ;eQ?gb8FueUJRlB~|`9a4SA2H9M6mNBGd+ z(ga<3$3t>!xWM|Rox5>5lm|~C{e3cT{r19mZzgbR4-5Utn*sCbF74o68EzyyUDDb_3k2s~zZ7CAJKgXP=_ zOuyzGJYDfKWjD{~X3mWVjp&B13pA+S3J1@A(v;{8S%ZjCT?~whOzod4(nh?W;ViJy z`33tSbGuT*cV<;(=A+ee82>eav0kg~<7X_eR`Up*N`9$6YFFz&1`^_HvKOC%j)G z<}x0XE1dT@9jm{coa+0HG`&du7hv0~5HDH$7`XB7=}c4^xKyi0^&=@!rRVi-thS?SEPT z5r23L$$9V*2MbDh;D@foTwU20CG7LqCdbN zvXee*Y)6e>_7@IfhgQb3_yx-s*FVNr#_=<KN$ljDzH#JOsgvWJ z5s@F`UaPaln&iKMPgl)o4fg1|h4vGE@A;7Xc55+gIujMEa2Z|W7tIWR#UEQo!1s7+ zAp{b7FMY>@K5viB1@BUU@4321dPSkZg-q`#@0UccWI(-dvg0Qj3Ip&c5gQ+yQ}Rde zsuQ2)b2t)VpgJ$p^l9c#j;?SB>R4O&C+^|{rY@o%Z=tD=Q#JVe>!tVCe^XqYb9Q!Cq)i4K zIut`*7VO_H{&n`G-%T@M!@y+UafN1F=iZ(lK-{wLKvN9UA4d%HH-FI$f`s^1l=qo$ z`MqupmX-tx4@w zOa~qw@vurOB*fos%Mk-Cb39$DBE9h z+84Gi=i->^3~i*JmB%0i=~IH}J8AF!igmW3Q+uQ}S)nOCp4Gdbg} z$xLR32gs#RfEC{Q+fRDpTaggL^TlHSV@^m>$TSu-&f5KX=^$k1ecly70f&^=hOr(L z`>H|-O}L>zDrr<&DQ9wN07pXssN?}BQ!_2D1G&Eizn@fQ*)I-_x7S2y28{(3#$7HR zu^p>+fGHD5ierbs-#(D_|12)1D1sk0XLWXa0D2c27ymondDlha$?10YJJB1X-*s}$ z|GdgPAjZ|(xiZC^cBI4n5ee};x$>^P?bVUm2~ zJ3I1`_<4&F2<=&(if4+7GZ0rU_H1{5Z= z@q&fT^1$!Eg#f6~oj>Nl1MfS_5pdgbqF9!D)6LG>s!n!A==C`IOdn`4LTH3xzR4`j zp=*rKYFFax$j+`Vn8=Jho*fKi(f4doQ&V%7&+qDh|M@DGjf+pFPLmkE?_OGiG`-+hy=?vj z-}{h6-+k*-0CACWxnTyS-TdCl#v7&#m_PS*b@F<6tH0y1pQs7ioCN9y$nV%xsx!{6 zuC5Z~R*nceT&A9zZM~p2q#pDjvmevr#VsD6sAoxU-9aQ`m3SN;obOyR3H9Ll-Ev;| zzSD;ki*{;1PZ?`hyuMdWcvAQiC4_;jZaG`ruSlDAh((tfkBH$JngEAqct}NFw?CGi z!tiz5Et!tQnYEL4axgl-fGfCJy0=X^LfKq|I0rAU&A}p2u4m6I=8cwE6)p_8SXmtn z@ezM?{5s>*7Mb(P6XSOPb>}S_>SdpBqaVo(KVsehxD1<{o7=s=EkOYD$?K8p)yrq2 zmMwlZPhF?SJp-_szR#Un%C*P8M#*`;wRj9>^7aQ{Z{@1muO6WN?cf`He)q6Ki4EZp zl49jA(|@phw}LNzY;Awa@qNPaeKHXm|F;It965VsAq4z9Iy&Nrm5>H(h^gR3OG-;G z4jWLPND3A1RrG$OXQqK_z#361Trtd0Z@NG42zo_x+hgIF4EC?DzMQt)t)pmEB^-`F z5B&0a%!xI^@wBO<-^GO+UGZPgwAR9egM(wojgIC@#~NFd4?)F|G%Yh&X^K9=USfgk zXJt0Q456t_z&wgU53y3zr=7J7k~Bbd|HvBWZue7pEZ&FCDw^TQ*>fe9q3Rikg6b{; z>>fMBPXCAalqXSE3(O2!$6H5OHG~T=~Pu}cL7kVEsy=Cb#O5`hUZoK3IW)JbZ z7q72=J-c(?o^u*)GhenLy1`6upWZLId;=AHk9oSD``lI_5=CH2EX=vx_US_r1^)CS z2|3Q+sYxHPcp(%erIjpqah>lr0KH03h_hOZYgbo-wYFO&M)QS4QG~_CCL~_sGtcEn zr!KFceB9Mu?a~k}5*urh)F~ETdPWVMqxd#8+*fBRNwJsc8I#@jMGR zg=by2=c-9luy8DM(R9_Wyw$Py0mdiXOs9x9u_!bVSp3P`QJrtifQ@l@47d6!c}}#+ zpzO>VO-Ea30Yx$?mI4mFI%yfbF&INZEA0)3f*#5>SXbf zAO0Rn7PHAl#okoe`vy+>o?C$59=ar|)MpD6>agGMHo*3l?8lNc;Ma|30bnAh6?HBA zqK{M3^XuvKv4{DCQz@3ib`z(F{XI6iW{lWjF#hx3c2!Khb3b#TLg-LSRXkfYnUy+} z{p;_|%mWw%)jiHp7?9`kNf5Kzocd;EFfWRcVUM174Q9i1Q{RI4)9hb z1A9BOa!ech9^8vMN#B4I?*4S=FF2|_X1DJj>FgRsOF7}1$?J)G*bzqA%%oK)?)jS% z%hA|=F3AEzDINwi|IOCXjYTfexNg|s8EmjwUMd7wlmBzXia5yi%efj$1#e@$!+Cmo zx$o_1aNnKN0Oss>o}E0)h+!a;m+tQ8_LtF`H-?hZ5+J9FJubCS^Kx9HOq&8EAOQ;} zKY)knF&}Uxj@F?p9WP5Ya$Ay~lTBd>wN!`Ax|_oBAUoURHvMWVm27q@K!5t&j3`z0s3)l(x-?vo1YXv};1Vi#e&uu5w&~iag&V?KYIIYdt zq|Z2i;<6kheM2pbG`Rs4i=!+Hg>oU9ymNp< zsEAYp7Krk5@)5){0d@j3I&mtyZ8JJ>uwSQc|Hx#u$bp1I&g<-4@d=H9A!ABij&aOBiU147O>U)~52j7_MyT2L z*VYfRJm?+Ucl+|%#;mW4m}{qgJA7MOhsK$6QspQQgh6cJ7bs4(VDuuJ^=lwKliTj} ztJM3MqkcK!X>SpVz|fEN*;Dcv9Nj~o>8-;<)-PTjEzir@{Z2G75Sr%HwGUmO;ro(V zzS9uZE!g-yS0qf$L%wRNVjdb=o<2mYnR+~>PPVgtb@T?|_~YIY>xaE`>2_6>Ut}7} z2aiJlUv7+e8Ry{N3bOSKohHcQm1)t^+cF`d!}l(8gyA&_C*@3jxVARF4d(wUFvUUI z3@05r&6G9La3fk6MSx?xW$j^nR3_*G#H~n!@}{TYGOOyEC~VZqRX+K?1qS`%;zt2L&S z=M71${c)2ZQzNI$G+&N&0`$X+wlMZ*KrCI9l=RC?Sx+h}NR)oZkcYDFH=$_q97hOR zwnI^ntbGEg%RTM5VC>I`2jiP*KzSnDFmccC@yrT?j%-VhAyeuv8i%<-{^aRWQ#`9* z`SHlsa*YiYh-cvijI znkK1gHVqC#ic+ITVx=r_oT||T?`dk*o#@tcWJuA58+{3)V5Tn=cx-B6?GUNd6wqt5 zE{DPW2p$vLpWHiIS(ND@i!pd;{&QS%o4IhNeLxD>CJB?+$Q%ghkPwLCIP&rGyhSHc zmm_1Y^xs4ZD$v>0yO#@FmGmuU^Suz4 zfMIY=CK&$tbF^SZ*HXXRL_;~gTUR*PvxYY5FEgtgyv&2OXTFMInx|}ZF+o62kNjV1 z^c4FoH6L7x*zMmC<<3Xs#b|LsA4{s}m^5Rg+L#8X0$-!>@l9{1kmm!&iZ!UX-*+5V zmG4ppJDNxt%=UZWgaZTW8rqn5v&^dOyH_XPoUR zChU$#7k>3z8=RByyQXzra055UHB#| z`{Q{z3Jlj(^GURJN!Ov894LTyfCEHM^3X?e&{Is1Cx@h*hsrTd&_Oce;ihbup6z+g zUC(=ql-!TtXB=O{IHwx@0ja+$78t}pC^#l>YDW>In9>91aKn)`@Z?pg)7qSeU|0OJ zq_zn8_pF~{C1WHR1}p)GnPvzfn?^OyZ+)(Jg*<`O;4f#v{I6>_@I?4|@9TtwPp2Q- z+?=nrFS?|u_o_3iW1|WKQFyLBG!xIRNScXa}irYsha;%#nk;C zjGO!^b+lte|CUQOA4HW>vu0bE;$pBWmO2rS$k@rzm2D1zf|40B{HO;VgdD_bYN$b! z-o;+p?G-bXx^#~T_o`b*tDQx!%(Q?b!2Rbmy;W|DrG;|b`|P}qQSfq+s6f3E&2vjF z-&~ruB9f6-6$s)6`G#pML@e2Lh3J$46?W!!kvhUO!B**ivM10yN&1oCe}E5zx*nRB zd4+*M_p=1u2G@Kx<$Qb*yNbugHzRdM&YD(zZ!(kiwzmQoI@5*UnS@V@4qJN;j2-L~ zues|A@130k4GyDl81&DG^Gd$mRwa6kCkLEb%OzE?IM$6$7cq9OLQRR&qJaZW4*T*G zPWxV1z)r|gl^<6PsfbfNm;|{b|Kv)$K!CjJVO7@pH};&gO9dD=yKvcM%@4SyCtw$N z5(JCkB~%|dNu@?-jq}_@bkP&;XRXabpdY^XB@=!kNj1ncJiUY&souRdWvKTB zM(x5E3XH3y!0g{ZSYF=CtXYI;miN=gx}<0WBrRN~`iF2m&6xb?ug}$lTW$FwVpZED zK|QXO_vM*Q`Hhl6)!HN(6d1uYhd4(=zY)ZVa0=}Gkfdo#Hqv*t>2_McxL?V7YGoNl z(;d~lmm`4e7JN>-QV`80En#CZu7Fp&GjAVe3kMl#6K?!fJE9~`!ny2hb4@6i?M5~A z`@l@*uZa`6>D#1zCM;|Z_c;GfpQzllnV}6zelrS+kr*hlQ0R1CP8qAV01U1#i?p^h zx1)7NxV*()!M*KfH4g|=sJ}9O-26Kg&W#_tzRJJN)2s=uG;0Bo--Sf;FI}bz0jh2R z#o?qjUsh0gDsthrS#;$ZYxJ#N)OfeY85p)0u+N3{R2pX zxZ0S|Io1R-L-~^J=1_qEkH*$SCtH+6wXN^gvb@?nhA87x7Ar;UI!5<%5D$W#p;v`y zsu5~>51L|f7(hzu83ro7nmxpJlX2cfs8N3|iO^5QA!fbRC}%xwGdw#2{?AftIU zPE;dB0W708<}NHny|N~z-1aho7ATTgs&mZo zlE30d*M7-hT3S`p&olJ86DGW*xF5(!Ig#V9yfP&LbzNls^XJR&gf|w=Alh6b+&Ned z*c~xXHESmlgILCC zm&@EpD0B)`ajK-xb2vlj2iS>dsvQxrk=RBBN!K4?2m&u8GmxftPdAgtW?NMZwmwpO z`Y!!=f@Cg1_pTg4BiK~+2!$c=>j6D`a^yy`OdOp;g;$pYiMTOZ5dm}W&hwMse+yv= zp8D?|=i%obz%)&KFKE|KY}cp0Lix1y{Ip70yca^ODAm#u*=}v^58c?lmiM{hvsdm@ z-2DEQk70ScYpiH5z0v-;@|;)yZLQC*{QY*e<@-S27psr)hEJ=tkLbDG@~BW@#VoRe zS;|voq#th0>#o4C1YXsnJX@t}tYf(Z;U=XeR+h*n$PIT=oGRhhe8NB?n;UBR`8R9> zD`w#&ZJeZy71O02g(>vn^v1G#HLB$pXb^Y--6OeXdBli_h>b+F9CbS=g;0WleSIXl z;!M9iw`EtMuS5aDG$>B%(4@h8`nmM_OtZe=?jOA;F0Mjws&*@rU(`v`i4+#T_ zX&Jh1>aCW9i`dwRmZGj8)1PAU&1u3wVL4m0p12T5Ur{gBNp%g9>_}2AxvPX2HycVS zoRmeXJIUhtrS`(t+8sw+jUQo}SOBRC>rqYxRtw@S01xnT>& zj*KvWW`WvLZCe$h7ecEaPaK_M^zbn6Jb#+)ZO z9ttfO)(Cm~o2OVjzr=x0?B|(Ze}Xr@e4O#)lh{d{R9BL&;T+liG~K&$r2Esof4gK;@KFrLY;}X zrdXUQ3eWWshCl|AYSpD(d_#!l{kQf#sHAL` zk6I4@v*Xs)`ROYVLLjB2UiC*I* z8?`ztvsK7GXasZ(chbABn|Hr+2CwSlr0h-O{k&b;3csfp+56F5^zZ4VG*{X*1GAEf zzTG{fds7_$#=FQ+Vp^L|v}+@xr1sDV(`5SXIf~O|?tOX>AN=CO?7zI5L`R%VUz&Qk zL1n%|X|BrrnMD$vaqf9)HxE2_53?s1m_56|#IY%6&(7l{9Qt?nk{!y>y)#E{sFSW8 z9au4wdeK{z#Nkyv8WoS>hXy$C>^`R6o@L_r6q9ewP{~&?Yy%+#Ud_jjS?oG8#_)rK zj65=gldvdGl&<7b;?9(q1+B34*ZaMp|R%ur<5X|i8cEd07Y@C!t{w*s)ec! z2Q)<6d#aQ|h7uVllIaAy9vx%vW4q`W$|3|bDh>Qv&^E$rC9R|y&&04|I_F&&f_P`V zZT#z%g?n|oxw?IC+4fZs?}%f?OcKrSa0@j;FC>Te0!FOwHN- zgGPXpj4}G)5Pkc47=CDw?p-++FD&uy^Jlp9#w5P)VOlFZ$2URC!yfem8w&v-6+OGU zX_Ol@%8jkPSlO(fUY)NJcp(zUab;X?yF@Tq&)_xLW#i)Eao zLo5-)N!x^e$o$zwj(_uAijyTmU#|M@&<_cHiJh?M+0(_&503JIUpU15&$vt+pW*Be zFW}V!?*HHzJ3cT>raMh2b)DkLw`MQ`gH$%nQeg?pS`MWZrXV-k!Hy4(@ZhHpaL>o~ zvUn-a#aG6eK0QZnsFRU{0}LGKCzf(>(l*rvm&FT9I*-i~WCk**P-4a`c04))O>eH0 zQrD5bQa3=moH1>4RmfxDh1AP+rrw_A;;R!ZPA(C6AxcVwVUX%fvHOFg?0#$q>AobT znF^=>@+>lhWM>Q~F8kNPM?Om-Nn<4NepvY!2RI61&DRJT!$*$xr z-x~tGfZ(}4ZoZ~-|3Zlh6lUCH;6N`sJ}}Jap+RbeI+MqyIsL;6#8NgpA04Cro*t5& z39OjO?7IuR@ya;E4-fFcUw(vrKedaaUwxaIGjsa>bF6k)`|37beLyWQc^HOK{V`$y z01p32L_t(238ocEVr(q*Ly_rA)2MiO6(3=4-7#0E7D2t@p`;?+o78C@!rq+c+{}mV z-jSoS=n{IN7D3#UC-WXOUBC5WgU}1<9?3C&bZXl;MIs_L1CjCj9tfes{S4D!;NCv^ zclOY;r<>mWT{sDwdZ|If^|<)z1eJwqQ{{OZb#Mv{Q_!e*LXB89h6=aQ^}9nTsj!-%@DoR;nLIkZ9H!e0^)*aEEFD8B zg}YRvns=FabDC6lLVNc1bkQg_n0j-Tg$sEqb5))5DwjiHS0aP}1A=p5%xW~OGj{oisDp>x4(Uo*^>(dUPu^9(%BS^asxN-Vy%bVs-9n+N}=k4=W^!4|#xUh(na?LisSya=gZ+&Mg_fIKBdAh=d7cbMZ zubbh61MK?n7zdu+2SS%aDKAvH{OSa=rx);R0ZPHt+p`Ql*w5|{kMYc}Kgq>c#yRo) zDHbmm2>d`tBD54^SM*OZ)LwEGO6-_Jq9cwKvvCXV_Ug|!0Wy?yjdkKK)pdNKi89RnE*(`0j6zRk1{Ly#TJP+hFyxtmKM^JWkRgpH8=r2-v8*-cpnG9qFt z6B)nnL91B$1D`s;!OtAPZv-q|DscHHlT_xaR2N*zb5(+RfEBm4WZKbnQCix6{`3Mn zjtn#U$PjOR>m6;#Y|xcX!!QWy0b@sokfCDc#2i+9`-H3C4O#<#J32zp&KJG1PNVj~ z0}9;5Iw!w#R(tM3*;WBHT7pX!%Q>?vzpwdIdlvK?i|3Y@dv}43;S7EId$1gfxwDHb zT`Fkf(>5_I17QliSpu^ql+}K?Mf-S5GZl_~?IelLINiH)bPS~_&sHf;l_||s0I(c$ zm7%pv%j#|ZCUg#ZWv+V7HWWg%MYd2d`|cvsZ_kq%NYS$^$Kd__9Qo@HA$^4~;Noi& zOdp>kf2p9=*&TDW=*m^M`x+C`wsGDR4RcI{Tz%}fOk4NsN;kJ|<1kcAAD?CFtryM;PJ zL1od^T3?3F4^Q``$PA>(bfw7lr%?!g@H?+i&X+OG&7onirD=*?If)n@gBkJ{3;1=P zSSGf$uT4`3t~^^IH_}12H$$~l-F)|CTTNcjv!{#cx99L&4?D3n$6dn^gnmeQwnFDv z2Zn7BHiBDE6N!k3SXUzB_ZkQT{93?0PwwO3XAW@cyXUz0;|Zi6;<*7T)JC6a8_hhx zt=ECy2-;(1vNT?#R;)32-ykQSKeM6RLk1EnZPC3a$I{di#pz14GJB^`%RjFb?zZwW z2utADcH3aqe^)oOpOs$+w2imQoJ(oyA{g52vSQZqepl{a>0@baSMJfU3}UuP&djAlIfBr$n-|VF~Q!gse6B+Q^39 zoj@Rvfnw_TEIU6qO2=@P#mh_B@tZd&*9iD*zq)FYd@Ic^Kn05INSZ`{g15f?w)QG+ z(onK3&l(2K?e5F8O-#pZ8`QV_{Ho0X2s}yOSPzf= z@?om;Ro?m2(;##Rq>?vSTXP$s4GM$l6Z0H+b}s|_`&r6`Q+WiIV?YI>LrqnUTE>jdjQMMWWs8weo;G6>1`r!j1U zYQDBT%2_Gx`E?gvOvfbCok0j=)7_h*wRka{!LJ8Y=BpUi7QTMX{-<7UP%GBy9L~TS zE5er|A|mc8k@33{0bwX{;x`7{T{>ViuU$mQWdL2tK%N)dRv?x_$^ zt9mP9Q=2)PSALa%(3f=dWf(g$$k`uU!gm8K$Jn0v+7Vli=ssSiolm`-8{W<@zM^lq zYC$Aw$Bl=AR8JD66xBsnM+0n!>m&q0BcwE2p>r^+#RE56$gt20%w1!h$Ux#2>zG?v zsk^x#Un|zBS3G)lcM%2>%iLt~>JbrJl*ssP(#q{Jguw^?+9A4k=lJpOy-9hls>639 z!}mQAh9C?iXTEOzgs4-tmJ(o}&k4BOYSQTFuTR$9OUq#rW!;2@@9Qn*w?NvXprBkMOJ zV(W5cesT1;Zytm$U6$@j(b1o!FkakN{xi``%_vWoN%bYM6Bd3wK$x2&Vo3-+H%N6P zN#&Bdo_8a}N^fn4f-nV*qKCU!qkB(|cru0`M5z=J5gUPZGk%2-7)=2Rr4+I$EYf;U zO1Ub;MF`QZtJr$2lxn}XQkAr|zxNdZ(vuwg)cx#vVh6{+@h%YouvUwh2(aT8hN+c(em1CtI zTMeWPo1ZgLO0GH10)%0BhtA~25ePyV;-nm6X@|Kp3wQgF?kGY)C?%EoDx*7wF&zuf z^)XDbIfgEwOH?I$64-H@>38OIO2Vc%_}icw?y(#br>1YaL|jBf#O<># z#;*|2Xf&wTYe=a`B;v&534$P?T-FMm>2$hTFH;*NjYb2dB#A^4&+}+B8pPvq8jS|2 zREk2O&>n~#$03zWzHfb{5YVW2?Emm?9{A*b&i(Kr7hfH3*Cvg)L6k02W5r@xA$P^1 zfmieO;>NZA6?f_CygnsW(_q$$tU~BA02&%opWzxb%Yel)4ym3bOW!TBy>cf7DpZta%Vhgg#M3dV zc^7ky5^oz3Aq3J3$n>X)XJZs5ip>&q8~@mE3kAY7sLWLnh9KRWM4gb^>(-5k*q~_R zSBT}UMeFs-zpa_CDe!}U-Fx?O|9$rni^qBQ&4ff4wO% zuB&Ku1N&d0{Hq-t`ga+u9!R~Z%G&b+LlF7_o!vQvX@F`9Wo?CPd^-q%y594$T@Wqt zw5UB+v(4ILy`G2bjG=3OW(~mtDMLDPIi7mF@7HDoK8E zk(XZj(ek3@jhuG#Sk?P_ZwSC3+ndEs*i`0RT{32a-G{CIQ#JEj+x^Jx9;Ogrm^3N@ zjaoo%sEhn~p}F6U)ds5&0)){_p*P6%WH20)%DlTg_HN+Azg-kyS|-JrDu!i}9mo>; z>JB_Fn}ij8N7NoGi0+SD=SId_{XulqF?Aao+8~KJ4*7hZ%a<NPY?~M(&wYb~Iv5~f@ zJ?`2j02zjKb#>u+9%s*<<#Rvxb97`oICA(1M~)of?3vR%{mirM+_{rRy-pAWsaU}CJUp+#v(G+@ANZU*dzObEd4vys=rQVbEiia<$EV)6b63rB zUY!FSxcR2e!v?$;Tm7fFvNYRzoWCh(R2tM4T{?#{dbb diff --git a/docs/img/grafana-resources-cluster.png b/docs/img/grafana-resources-cluster.png new file mode 100644 index 0000000000000000000000000000000000000000..7d64286bacbd55444fa13625651f068b5d1cd06b GIT binary patch literal 88285 zcma&ObySsG*FKCQAW{lShe#>i9f~vp3eqiIo9bZWo90`b zb9|ojynp=0caI?s*mtaZt-0o$*EO%X1K-L^Vm%;!fP{pEB`qbUh=hbfiG*~I_C6~3 z%ix)tKltySy{Pp2`{3}nZ}Z|vNy0ZGJ@EfT06jy z8-Ej<)+W|qA1JE-owt`+cNX2Xf7!UY(!1-5{d26oF+y%S8nq{M96YM3mzEq! z^x(v5P5i*<5+bd?{^3;5B*!Yh^se8DVtU^&CNrFw*Os zdO!MeWH)foqN{)N`23TiUFX|#ZS%mxh-ZI}F}n_G%6cY>enlqApi2LvHFDYO$A7Mj zFc&b03`gOU(p`Skm}TVH&s-{GnB4i|&&75%#beG$S)uCmejAB09h`OWk#D6bw8?+2 ze*_+#LqokL^i}4GJDFOpq>``1*DwDZ-!}WCMPs@I@e^@skK)2nSgyQ&cANcqGqIeN zVDNEVP^R9y;x`7xw$FOEYMA~!_}Q3!M=-is0(sM26cKuZx8I6Jzl8tyL!*?Mm~d6c zugg*ItiC)bNy(9m`SCPgZJ;XL=(q#_&*&l&_-L3Y3%4G(lf2frvp?q|pyVx2J+~zM zB=O|M8+zgf)lX&Cj5<12RN}XzNwXB;KqT9^D{hH)RdN;@C!&6jjI4zNIl=y-C(W_m z%sZ3E)s)zX?pcsd{NpEzE${w%eREY)Db};IP#?-PV${~6hs3yVjGcyJr~L6v)Y(*& zU(duYo0sTPHe)$AiV^2_*ob`Ycx2K|xGji&W&@j4)GjLAdEEZVk}gpRuI433@Ui>g zQ5jAp2lOqzrXqanIkC0R5a(AN5<(fSzZRqGg#lp-qmWYlVwppAXwr%R{9U9W25-93s1P}xbPMS~N>^xJ#Os#mV0P5=NdV@P2b0rwP z$AQ>OIWz(0RC#uftv5H4(Pl*MWILq)S#&9@9H)OKgWE*LBYokJDciLSqf*uNZbIbR zPoJ&K1-I<+P-U}gH8lx2n!hI=pGacm)MrR-Iq&O;neJBsZG>$44>QKpa(_~j@~CjM z(&P->@I^9sF=ODAd`0?tX|xXu z{k7>bvsM}cg~Fmf0$s26rPcssdWOGDNRh5D$8#cEHH5Nbi}&_f#paBu-5n3J%No3(Im-KN$y2A6uUwV0o|Es9$_px>?>EiYf~f3+WyOU);OjJ zml=bLC8WFBEa|LTca0iimyT@4B40Dg_GFi(Ou3K5*YhL?xOE=Bg{eWd@%x3&JtX+W7l5W48RsR1yAp#UA6tc1c_^qRg^AOFi=Mj2cwO z?itz+T1qeJRYdXlmo0}-=5N4{QxZNyJ30GvlJAGys2L6Rd-!i`m_lmS#?P_E#fXbV zuCH}y-w+P2W)qA#KL>-tiqt1O+L`hl&(}04*XZi+HyHZap3zfoITZ;8SZq3k>jazE z#)*g-|0#~a@4R=fDS^uZjOm#4-54g}@6=RpotJKI5>4A0jQGZVRdL7LJmJa7+Gs;# z*sZLl69r7WGnI7g>~||?Uc0&R*E;Re($HiywBqsQi-~hlGBab>&dIB&a7%`gHt#x3 zIICrdNl4&QEcnyPB`hAeNGS|HWn+t&ot@RlZUhUFRhenCrk>I1=V3e&_y8~b-79=I zakNsw>?8TU`9%5mMSfX>fh#!!a&h%^`-xs_IdNu!A-Itf+U{75)ysB}MmRGzDenei zo~&}AWAsIoE9}bqU<#@`s`NQa-eDy3O^U~z?eTm%edvH&;L>22u+NzBT3}<$LVbgC zp-x|%Et__&6U{ThB!kY-V$%a$;fu>433yoL*U=N#=XD(z@tM-*=JWwL6ibQLjRt2d zjiW9kis{8#tyoKN=4a0oV_s_HU=rCe9~>O)Oh&xa(P6$g*`=eUO<3>Od`LtT+Hc(X zp;)hSrbvx3n26&+Vt>bg#_61DGCtSR(vlh@K061;=5#rRkXsU#izXVmK#W)rem|d= zAQAj=m;ruQgId0dd=js;v9WP^rAfJlf^<;&mL(NGe*!vQ$55Hs2rjHF!x^SSD(Dg} z9)us23j3A(R9qf;>@1JV0>Ab5kJn~nIT;If0jS^wotOlsBhoe>RjR7 zy}c_tdvnz%3AvjSl`LV`XDn4(1xkdDyScgW{mrb@h=dqaLNT6@52yP}g6CbYFA?6U zhs^;LW7AFph9R581+c-|FJGjrHo*LfT<%kKhEdQxd**||%o_&wMy%%ihhkQ{RU1Zp zUort)dHwWmX-Nkix!P5l(N zXRg^0^*T4x$?o6@c~v#ns!O-Zk)9mDQZeOXt8SA&Hdh!OgfL!(XTf~ct11Vw&>q!A zJyA&Dyp!1PUPp$Fh8^HrRK)7z<3q@6gT$m!w%AGG-L{h57(qt$tRDJ2xiRWuX0>+pVy3R}tDh12^ zmGcEK8yPTVA;Lo{F>Gm)OE$b}<>rp@uu8MJgc(S^ADCO0qe1V3iwy7fwOHFL#|b;I z&Ih~B_VUVIAw53gL6_If2i-f~@UU3j<>k$h(}k?mF%6DGr3^|h?15E_D#{d`N00cf z{62pUPfeAyv0)|9cA$)icobb=K!{pgT%7fk-y0bX`zd3ez~0qjGxW56qQO&O0X~o{ zP-+il#-=N0!F(Q{XJN^4gf_^0Y?cYaJI@L0=F6eI@*1u>M;sOWD` zpUZ5Bp~f>dWc*S!Ef}F@n``ML@i~Q}l-)}rs93v-W+Wo0CG z&m|pM?k!C#w0iUkNVvNgz7d{V5lrf0J9SA*9;}EFGLiPYkB+W?d0ISa$qoF3dY#LQ zz(BM+zTx3|s93~eu=)BgVb-mD&?KeY)U=m`{>`7!@l1!)(V1ft$8uFmt!B8jn5|W9 zgWA8V7V6wOsfY+ap&z;4SYyx8rwRf?oY?Tc|JvMez9^N6zCet|9GMxR3HHrLbPvCF z6+_Y7z$0i=HgR|U=@}diGb#JfLm)Ysw6my}83n`RFq>%WXbzq}YcWYMeg!i}#w=~K zs7Bn=7Io!RJ!+T>WUalmi_H$7FLjL(65!FFhl}wlmrJ0Q|LhI`48@m*Q}Sq#(x49D63xkM^H)Z z?^Z6H#Z-cjyIoit<8^h8OG?_8BIF)_1@kV|ZOkvpHy z^^iK%`+#P_`v#fASgYk*H^J|8X*^)QeT~AWUmUl`&yp#^HuaBlxU_7VUE4Z3B9oF@ zvQj-e6h!*r(YZaxN}AASg+PnC@YW^0{^@cp{uSJW&h~TMZ{@B9xptSi=hgHud!*ml z29gE!C&X??yd+=*p`Y)~d&6|8$@y1v3yXDHKf>xz>~L~SMa)5(Kp`wy35BH(JR(mx;&M%N?t*)<^m<-M$ zzV@!sval5P?cy`%ZdT_C!}zp{I$L;VnC_|TyD8v!C&no#Ry0qyKFQMK7q8_@vD)9P zShUkvR`KquIv3&g%W1&5xsu~1OA}hH#vHZ}qQHrQvUIg>`b&D3u>t=q*}*$$Kitxh zRzBHNii&6z_H+Y&^^J}W`GN zQ_c9Z{(Q|!bOxclfjv(QyYM#iG*IF%9SBQv)_VBP$#-f*ll8K-H67}H3en}>O*3af zM}M;R*uopBD`{4%C5Ls}QQ3QJb#;B5m(Khy8NbtFu_OfzA74D(^*OccnGsJ0Q#DiR zWZGL0#hmAD(ctTYbRr_DLurz>Fd^rt&@4l#btJ<2jW$) zoXilieW&@dfwfJ zHeuM8;-TYG8u!3!R~el&q}}O?_LYNhyJf@t)a@Mdn;g-kFSy9D9} zUY^n-o)G85xG9a3k(dnkA=0gm0S3s{cAL- zNJ?Ls89@n1b~Vb)+o#L7C#&HT4=TN%&*>d_S+|&3L{z~f#W)AkH%AE4Iye;{{K_9Tc&p5H_NKxD zYIRm`HREJwXLl)V?XRy_uAyED5dv7^P-wz>siH)0DzuX`oro#oUZI)skT@r-bcnxtFDSJtvo*5q=h{d+s$*b-FCG zots@X$?N3}o15I_zP>!gOV7%hs2V^ccZ&l{jnr^{ZZV&5jH;^Zn8AX6R!dUQXdM3b0~>o;0U^>+*tmCBZ|SGF z$NsfDCJ}pE?B~25r*Z4aml_F(dS5g=^?J7uYZB!U`{A}gQX#i*bT=?qgChqA$6p`< zB=Iv9RxAj$5DJmdG|8|M&x=sIUd|x1j)Rpox@nnE3um|-4=z4_N3!$Wip3)Y$WdL^ zC|Cd{aV|3M*C0iS`dFpKe0`dAaT}|VdSh-{G)r}|WT6AI4!u?ljW&>l24A|HlY72{ zWIGcYOjK09lH^yVg|hIiF$)(&a%I(CLar9?0JE$QlRD6{-N2J!90KYpz0R)0c=;Ga35@oY#mAc;stus zL}_|XOG`@}ukGF2Jq&0COmYuw0G#ztc5Y5_i4%FO^(P9nF-f?;Nk`G{chlXh!$JC1 z2OpqfVF_ywk{!Hma7U#0h)J!51ZY%PHUr|pbR?4t*stM2gD1$&Y?@J^QStHZ0REZH z)o@WsVA26F11UgUyw+zQzsW+lmKyh~0h0&A&v&>`L4M{#Ir$*=$FQ(DDIe_cL}$h@ML=z(5MP*zOVL7(f;{P{(|!RT<=*5~o>c^c&j zVBJS*RSVe{7D^m*S%}I^hd$gwpNtYN+lAEY#fx?h)5Vt!e23aJ7VjBpXxd{x-s>6| z(7ztUs9HYk{Sp{R$-oh%-|AoTVU3=(o@4Cp>e{;9>HZ_Pl|M)d5djsIqBAZ9Cx=t^cG|Lv%nnm8{}z9~GoPeSj76-8|r& zwy2`bsipX&W3o0E3m9kfpZs&h9PD)*T(GIq37(->wl`O>8=G38dnhPl)0VlP+SCs* zELBuubd%(>@G56^5k@EP<*yZ!4RlKPCbNte?j|3TG_4%y5u$>HS(>#+BiPQ-tSgT~ zEGRkE=8tlJjb%AY8QEN&v2LVdVj_#}6&G!hUtU=W{A#|e2`HyUyXw=;th3LbU24fq zOrVPgRkqbyh9=N&IhCv@E$vTs5jBn^ykLsDyI%mdA}UiZ5qLfckVxE8C3NyhPuUJ0 z0#eIidpsJ%a)Q111>q+OZUs8^F;^D|RTmwMSlzjW{;{k2K)EauowN3ipPo*%PL4zD zxa3zazFJkPb^MTYUYIEp`nBPQ!t5RSGnnw|bs z^F&it_`VKV$s%6XRQzo#1hARh9mZ*E6N|{i-iy^CwrRt$A3qTR&d(dK3oE9&9r+2T zn%fU7G3qs3T#RiWQt{XHYJ-!@M}-Yx<@-&hbZG)LUCtzBR$ErbYKP>C^L#kPPu71h zHW)#|cs5BaIeLXqC9vEZQ4iJa`I7<3CMxPPRp-i3>l9529i^tD3;6yNR_#=+RqoVy zKRsm`z5URi_c_C++MF7)ylChSVV`EKgpvnom+0O!($d7N1F@FODsZ!4lndGg$W2_XwtSSx@u&a^^NNkb;EE+@zx zWTmn~Z74TL)wFHhx9svtb-LHM@4Ia5fXDIiL__*}tCa0&-fQJYGkHY7F{Me$-i(#o zT#l*aYjE}ay*bKllb;X^IAQ*vyuR2vi0ivL=%k;Z1RSa|$n-X*O40cEri>b32XVJJ z5G13e&eof3$6M3q$0T1uLNc^#D?V118<=jmEl?B#+Id-)eiaI}U+sDX2w{iurZThzX=(M|zxw+y^8$X3Y^J-y*MrH0 z625$~VT@@Ji}d^U?J+*HmKd0<&AD2F>}<0<&B{pLGC;wIyS$3b z^SNiSRKbvAU?TG{Y_QaJ4v9zg*uEH+?x&u+L)t^{WXUv7U8HVfnTLNfQ1@gk6*`Rp?D-2(BjD(Z=ye^k72-H(`D;3V^CtFJ4P4X?d*!^ zRNP@zXA0l`ocv;3CagI-JTg)g04m^O4H0`nMzn_sTtPp~l&q#d8B)7EGpKGU+4G~c^jLB%HNuF8Th{@+X7lPawtH1wmQZjg6Hve@e! z%Xv?yB{liNHN1Mi^-Csu9iNz4nQYE)DO}7w)KZj2n13W9(Czyv9>lQw70agkXp>7` zwBhJb1lXofXJqI>PcO^CzAw%##zQ#j=3|ZfkOZ}xA7B~R=rxPtGtsYWzf2SU-beu8GGv{bW;+m z=`#Jvq92SJWuKD;oQI1I&_HG7zDw<|_0`o|S{%^41AzA@0y~YuTaY9+_hjAYxvX(! zC5Wwo;wI*|A%JypUnN#Q!>K5`83zNa_ zpH@cvIoCwLeubVsk4m>>;1Dhz;UDg&kgPeqNcZTd*E}9o5I-jIZ&s+a5)|R12q00R z(~#<|v}5em_Ie|S7~+Ec(E`X?x3Xe-O&hHg+sYeHCg<@PQVK1|DBBN8a@3<}aQ2g zpw(r&CsXUdD@WI-q$)z&=W5$3t})=QVzji`-5eIiyhG9GN!|U}j@E#%{poG%{iZ9# z!G-(Y5==p4g<-{x4;?GlYsb%5j*f47Krx(xMSss$|=h- z^Qp+PBZu)k?alrkIn7%T8% z(?v0@&c9bN+CX{k>f?0%fR4+75hi9w2nmJ@TFNziMPc-pJYe2A?hOz3uG#6APZw$A z1G?_prH5qowr0M{EX29qVY_t75+n&Bz{EX!0Xo8Z=Y;pL^^!YVs}OQ2R2dbBs!TPa^^%oYql93TsB{l6f2rEao(R#aS^DZhg$ z%tGz$iE!+KW7(p5-~kmDE*?$LWZrIPSKAvHw}~o_j*2PdHt4;X>D%eCQpg825H&* zD*1!)o~5+Y%kZqePAa9=zz5~uG_pKpF&N0r zJE)?lTnz2*S?iLlsUm)r?sy#wx|=cK^q3{IXEN~w=@6ts#u!xw7}YcL)}jSvRCP|M zewU1W%`lz7h-K2M`ciH0ZP@l@NsHIRmlwdEO{?BZFbLug_?%O$rxACUJV@w2=bJ? z{)9nd|GK8i*zs#0-}vJWi&^`LWG|x!CU-)=+-D)nGGqU6%9_VbA{)z`eWcYN&w*As z(~mWw4c&$4?9>NkX^Xvm8}5utI6}?B6AL(X4$F;)(;bxu(HPhdmVaVoMW&<-fH{^S zD?AJ}*j&4Y{xBQKoIij+h~WwhHtdTV+L@}2EgLQSSdf~K6p*8{5=UEVa+CP`_gm&_ z7dJrq0oeXDH8&DU#Y1;n)!?veyZ129h=T2AM(izG$nz}dDbCsQ8a)MU z-?h>{s}OmL_mVhp4e?-U4<=J@L?QC>8J$u%LPx(}hjCm!`VO2CtKLhITfVz>sSOZ5 zL`3RaL-{l6pun!ezbWNs{*dltzsiv~zsN$PN#e{d^Y}{B{xNCxyI@C$J|4)dn3tC@ zc0eoaT?tySSwcn+FDbUym80rneZPFOmywZCvD+~*$XoDegAJNKUVrw-$6^{OGgIma zC3_8OO*XZ2jA}(iErzeN}Fz{)7ObFtwhPai6c5Y(oo%% za#Mb1{|Y^Gx2&t`>rnP2y|j?P)=r3@Mbry@$Nw1JzR}UsBBYW88R0>8aXeuVTIfl5 zJ^r!w1G1i473|{SFdXw*m@Rf`S4;4$G$g-sx7-fX!+c-8RE@DO5y1g=9_n|Hfd&(E zMFO=4tIZ;k=h$tYb;ILc$(zKH6#~Xz4;K7HUZ`}@?!|}iu0d_-TFhvMZV`dGg zY`rl|EL=&NPuWms$=umAiQ-rL#5xxHFHskUTTP*?3%7j>;LdLlATk4*w7{(uWQ~q;6zAG1Sqyn z*3SqC%x%0&bJ>C^lvqyL0e;mp-$TZ(T1yaqq|}#q!{J(8eVg{iZI~MOyN#%>=U2`z z0apKTr!VX5`htSji81i{>VnPVly%<403T32xzV24TTYQtQQg4HUBYSg*)U@m)q9H5 z)9Le5qodoC_!jJk#0d`fU*nXx!toQtxRzS7m84~58A_+vn@(*F)T=+xsH+==Ehx=4 zcwV@|>Jimv>xnMAW8Os$Q+97g(t#IF;xK(k!fm;l^#RYf?rK1w?r0f9@MgBOFCH#5 z#m5&+Cg2ZP@53Vw4vTWl*Fe3%Qh#*Ue%%A7$u)`=7YGStS#DZYGjk>s8CEYqQ(Nc#Fe06&pnyV5aSU0SI;)#4Y#>vhT78h~T6(h-s*7_VV% zt6~fYxIAsmn$9+w*f$6gr;e3_{e#X@SyF0-jK-}?r_>rQ34q#2<+odZ%i&t1 zUb!0``Ba6KfF>Z_ID=@jUs$3zZ~UedKHo4vYK&h9x&Y&kqcxHr`1Z{qsdC<}vogP~ zPOx!;h$tRRKk|Hxj$ZL&IsX!SXyMLeahje8W1d>^gWNHJA=v|d`p8Yz4TOwo^BIC! zAl}5FE02{CA31mIDI3QZfyUzV53IpIE!I9&Q=(j~Ydc#+Mtc>0#<#59AdFQFXuq3k z`CusxYY(?`LT}|=qOLZb9aF|uLEg2f0^|K97P!!)CB803`{DRhf?-T;@{yn~S3q90 zQ#-S(7817cWc8UWhF|{41H=exupK};>rS=t98fe+Z;=9y1bC2ENF|8={h+Q0;627| zi9@w8#bI|EWq%%~P`sRBtLoW0aA*3D`onJo{Bmxo4Rg0JO?I%|_ z_Q>>_WMG&XK%L|v0CgW89`Wp*gYYC)TCQJ8M-g(GK83H}^v!!-0QM~t6ho5S_o_b! z1RPv93V)ZKU{24>EOF^pY)bqn!)NOgCLG^?7Da2fSMA%~-@lX!Yk>0Difd>PGpH8) zbPdACq+m;GhGMpCIvlLez*w!X2OI35atTs^2GaG0=j-mQF*WWC6H#X+?T2itjO_yY zTRugwd(j?s1Qa-93e}o{Gt8~Kt~n+F!!cg`{t1Yj60mn?O3mQ3DV}>M^558eUA44G zE9cxWU^m;y*vhto(PZf8JJa=v1Ihdx9+Pj+k2ZZfch`eh8ka-Vt(WOf2quI3-heuw z4^U-rxE(z|KiZQ!JTTPpT2oeeLXx|52_K|-{``w|!*S}PtHWNF6&Jgbg-Ms@=4LIY zO-x*;J;U9a;|BPmQ|;NH_jRuyfxOXa(GNffzz0MY3=Se`8XWO($a02{l}6xi6-erJXQr`_S$1Bx??Mf^T%H7T+!tg6GPcbk{!Y1a==bNR?DL>=rPX2 zCT~#hAAm(B3YAYjJ&l^WLzbJfBPmx{dN?4d{>|(0qllbCd2{2Kb9u%2)?Fu11BQ$j zE;Q9RZufdxE;NMX|=sV6Jh}HQV?Y8_Sz#_})Nnx0GZ*aF9EZIN9@*ifW|La`D2+{3O?A0T{K! zhY!!3+#dp+3?NPoN~eGj&3yJ1 zYHNE9hBi{?O4^^$4kU7XcB>mNf;a&82J(84%y%{W27dm0509LDPnKVR6dQ!vPT)tu zX{LUu{W1oem_P*{s9vi6Yo=3O{TPuc=66Q76tiPv*E^$5D#h$Bn(&L&7g~T0oqm^L z^6tth+4tt+SwKIUdDauv_ammT0Rk_iATrF>=P7Y*0#@$L@W>Ua$KE=M-3T4ewKe*7 zCxlQV&aXU9W>qaOqLD1Qm4xjA7W=zV9m{@DV_Y(_ei)=Cqv3Ug(vCy5KhV_@f8F?% z>1D9Q^%y@Wdi3b0w2V2`Kuhrx40nw8N1i$SKxk~W6J(YkCMg+fj`A%gM#bEC^W|8l zSA1q*=Ss&48#)$N`UTWkZ zsRbDSmNWxJzI|+MWpS>MkrmsE!w;Ee=M6Bw&n}(tzW#nVjZ?0>!hX9c5TS-@#~^n@ z_EMAEXDS0Om?*|NWm+uWd6Yd*rJ2$8>h1~}Kwq97Y8LY&c{zl25u-lkQTEGr@GFzz zZ1?DU9q$z^8te3-VJ=3%c3cq4`ebF=PFr&8I(MJXLlYniteC^Y!!lTniVGJib5;bj z_(%&6MPI#oRgNDk<+{FG%z>Erf+}G7%5`hY9s&YXwcV3iXBaIPSH8-gva(S*eg;Z! z4c+AKozMyOKp={~dpj`ay%+8$>)W)bwsUQV5Myzk{V)Q4`P_S}-BEPFVP_Y;fj|mi zCZ;r{W~q#6iRqjkpgvLvdnjfuHc|AKa>bL5!a~WDj&!?cvr1vu<_QJb`?@?820Dez zyI3o5H52rS=^WGd)+G0oy~Xk-8oHqkl?sK@rt_{)ZJ>QkZ(&dMzD7YRtUJ_*?)@XP z@7heGq^8)`$PY$GXUz!CUsod~y(iuFm^5dr5@#g~Y)tjvuFL z;bUZYe|H$9r*C8qarV9F$5b3q^eiO4-Owt^t+TPq@)K9MW_iS|R}YCLlTg^k$K%40 z&_dl=cdiA_x1iYg)^;P6rO`uIbyt^(5`;&aI}1Bbn-$aX9uj{k+;4vnr%cO=lEe3R z>FG$EU3qQ#T6tt=(~VrCA{Q9#4_48{jTSsP<1AR7PaiSp@|rUQ(=)^qaQCRu{-tGB zBzS{ItSCuuEoht#6T^BjaQFe!@=60MTP3OBf@WBa%m0FsGn7}hL(O8Y7@5neEP6Y0 zn%8D@HH>B46Su_B*)mPc+46>Y>igd0KR5K*dG$HK?*k#0$p_StqwwUYpjP#xxx63k z>MFs?W4})_suFfoZQio#|DeRO1Il}b+mjN5T8C_40l)9VX8bd*Bg3;2ZYj2Zj^n1H z->iyhe|~*tH~p{P4Y95F`EqYY7URWR_1wI_5!NV`4*!wKe;K`_#68w%yBDtGgLczeq5s1u01vA&oxrV*);#r*?$>*LG!Qc<^Rw1M0h1%HW96z)(@pZ z?*He`iUM!SKmWJ+qb7s;U-!%AC=zgb*80-xJ+Dv)AY4*!-sHhp2TWCRLxAec#(m%P zQt6*r$wuA^)t2MVMX6-xwj;Kw)lL$0ja*y|zZK#CeaI`BILLM_zi4|&*L>nsuY#*6 zj{~X1NbJvK(D>(+t+x26&x{q zB9;6<_Wtc(TSGBxAa1I-HvwK**ImGcUXhgTF!zrsOKJ0ht|OyN)i-Gy)%y64U&Q;o zmr)#enpxoZ=l!fQYYhHt_;ZT5CFJ+y5=4&(?Ah{fTMf+L0b%-7ku7pWM8tn*Qp)!y zwIMQrpcQI`j_0-j=8H~OXmaPrJIGrCnE%{40##1B3zgY$8Z)ncn+M&DSYpj@;4L({ z@IQ7cW%TW{p!kkK$H?k#nCam-+T;b!k+=0eYeL`o$6r&rxYzqD50`caNVqO!8RZP_ z*Y41_?0J4~DM4x;VWHZppAz#tCDM0Z2q1cV?_)U(>6xxq)a_*l6Y**qm^VXX> zG!i^MQc}aH>Fw<7gs*Fss$KWmUBee{ZpBte4Oe@b{x}o^Fvum@jmz2--6SBQGZ;vM zsa2U$0=SlI?dp15W;GLoMZ)v&y-fqsdVlI*H8cgIsmj*i>@b%_9o$X$bRW6K8K$y+ zaJD{UJy~S?I=d6-wNx~-0H*!xNY?=CfD&AV|9Rm|au$82`)Xf1&uaR7zO>9r@)S92 zN$nJGisQzAcW3@{-zgRnk#`lDf*Zf9h==T6<9tC;UWNrvOF?bK?2U)$IEI2g1$*%C zAY;8YPO8J^#zj7t@EwAbj{DfOmp?k2HOd2rnEF3-yWYqJ@85q$?snH%^e!LEK-+Z5 zlO>+{(4q7fk8ngjUD!OEViEQ&cNF}=Gpl%kR?JEsSRu~eSssq|Fm-6iD4v5 z0ot>m)%_$^x4G#Kay+Z;Ge7`L=y-=5Z*#M{jhfE8BZPE3HpKy_U1hW7$Ee$ogqZgX zbB9%Ts#;%QlU41M^j5uE66(c?7lyPT9;6Vq_nybz zm-_v^8ELid45u%NGdrzM9E>rj26TDp^F=!eh#5f>-haYUP|Jl3ze#(3^|1m9(|p7%gH z*_mnuDjh-|Cq@dd(>qAu2W5PGd@w1W&+nqQ#`SC%AVrPoPdq#FPGw&imIb2BSfseC zZc89HIt9E7ukA8*s^>Y8nVA{TYZn7>0I2)H@2`Bn1P8AsJI^MZPL?~)(Sw-)thf}({Q!6m)pD6vSv3_VMS75In^g%-Po$C z>!oZwl`P=Bb}|Do7*Djm&DooM(W=+h6QyPNb=C|l5gw^VN?CEWl?ZCL~K0&(0sR^*?36Yu!A8SZ~v6Y4SX>I~ybPe>EyV@nnz5 ze-68uGF;d1+~PS6pWg}DbhHLJ0YPU(tE6r1T!T?y04@=?hfDl1U?q}JE>IWQ<|gyF zfpkjR!GV)OHUFsIq*x8G91J{}%YlNR3^it);ra${x1IeQJw7#caJO=<8H-12P}_My z4(LPXU9PwQG}JIYYY|(mZ@fYF_FO*ri2I*7?kFzZrPLIa@Rf#<)Y8d@?`j@{nB6D5 zx8NB(;=*QkR-(-}?6s*>{PFwuv1<3+k7w7ht+5^unn>*3#mx*W55c=q0AAhrU z^*0ayq(L&?Lcjk2wW&y*!D?ZXFJhU@dl-V8g&{k0qBdRLYR`@`vP--2L*ygXg6h1( zsy%%sK#7@LFS_Yum_+Q3p7(2(HL+sqDi&FJXeufy-pvu&bitJjBh~Numd*9G^rste zt0*7GjFADUUH|J>rlDavJV4|Ubjx&Dd%;O`+^-xvLdXI-S6A|4$j(nm(O!E#svG{_h+#`gSaLOb{*8 zYk3cPHEL7kR&`BqIUT6g81-tJIc*Dc0yk|QI$a1&q9(oGz~Whv=l8cTxx1GZaOZvs zJJ!YiyL$==ic7+TnbI3DNGJCq=`#>`nyEULwdx-n+M&nWqQq3;-@~Gi>k4LI+bYt8 z(Zp)`%t5leZ?1z$L{b0jxC|T4);hV>3KMi)oG|1i*FNMlv-2||!+=I1b_qVu*%3Sn6Y+B&gxHBBhr~z7 zpDf(?lq(B9LWrOe8VVirR%ypccXnMZSo%nY(XIAu+rVU=dsE1#hac}mO%Cjdw_SxT z$;&^3jzQTthPy5sMGOKhUbXppHw!!5JHznGDK$Xv@kSbWEsajN$h})fzQ0?izBiMsQx5=wbB2! ztcz{M{7)LlDNw;lEG^x+xlris82}VEe;;=gW&e#Q{{JSG|0lCIP8XHlfAjvM1-OlUNtw6w&1SZ( z`0jMM0caim?B^G)NlQujsj%^y_~YhC)(EJC+4bJa8jdG)?e`a(i91mrl9PYZCnWU0 zgCc^}+Vp*&%WDur=iUBYn9@zUfto3yQ*D1`-Ri}&kWam9&saooVtch0e*6<^yEeOu z^m6lS_8KlApen+^I8yKK49N34O`y|-Z-pf<02HY1e zF0OewsKNi)kX3)COkZFBO#){iU{R!3L$pph0_;*tmK&qX{%2@_cn(d z-UuhM4qf=#e?A$m8r6{P3Ltm;0-38()_|<{#UlxxF8W5t$Vo}IklmTj7)SdilWLt8w5}x zJepR@N1%6r-D;W>bSv~SMTLhqftCrRTP+!2_t#?j2L{$bJ4HeM{(r7YPa!14#~V>v zupO=5-@lL93O4WaQ!_%;o56X{0LgcIR-}z4Bk180qFCYyU%qpqKbLz~G*tYQJN-WW zsWMdDn)yl4x;UR!Ht zXFw3X!;Tneway~!Nr;(SkyWbU{LvR6A)GIOsL5@(ft2J-{QA-0Nihy^#jei9ej3hb zow_|18TT8_O3UL7H~4mEJ;G_I+8%l|qG11+m^cTU3^LXpL~y)Dkjn6!SMBOt=m$`> zDP7zJ-`jzN^bxnC$YoDTN~>N}^tEz<79ZH1Pn=c%`V7ei<>&oCQTNpd$+fk$jMdfE z<-Oal1p!CF%g2*onWh$iVJ8Sp<4u5z|@mn`vqg+u>?|UeP zNM?6b;{>SdLJ~U%2Neb}Jv9LNp9u6(<$6n^MEIQumIOX~|J@@?3SC_$^*Sj5r@c|I zE=RzH{_;4po35}*oOeYCB?pE$)-+O5HWPE&glkBYzyH|SczwRj;(4_oJzeKoiYCJK z_u%6C4V*hA4z{MtbFYuLq)?(+3ZF2jbS(@ju+;6>LF&tWtx|A=&)*_u?ZZIUn#NZO z?OG6dS=cBn!^2%Y#638oAd=6FaC)bx80oY-n^6m^kp&o+8IjdoKCu-MKrs$>nZwDf zdb#lh|BDuv`#7T9RPD#DfMnS@^k$BHj5wK zDQN$m#;>H}Qh0>^Ki2yb%(pA7WMpM4-Sgd!`_dkSkzdVho`7OZ7HH$Bg@V$=Pat=Y zR`f;1dS79wZZ%aWjvpi*O|KkDedW0mXFirgV{L7nk(FfudV1V04?4L#PHp`B{JQNn zIk$g5h~L#x+9p0bKL;l&^V_cnQ*U0L+4GYSKaGGL=!!rq@q~h+P+jEum-J0Cb?(^Kf$-lp(M0-~Qm)8x#)-?)B{@em zY}B*f-J+vUJx8NF$L(Z;mV$!9YN}>z^KLo^E-rgmH0dI}h8B8JkzS=kC@M_|p+o4*j{1K0{pPNjxp!v1 zHFvFHEte3;Kgs!@v(Mi9SB^yzc%8fg&P_7O?kKv6-bc&9!6BC4D>%#N_^;u5GOPAn z090A=>38>UUD)uD4h!q=BP5fM)6gI}q`8gva^wdIvxyRlDk?f=z`w1r2+4alv|NcJ zOZpoPQ<$292Z2H-h1gWY>5rQYc7F1I8HqxDfFGFP1Ei&;wKO%!o3}-BtR9rsR$z>2 zBn{6}aQ1;XxnydP86tJx&l@S)HL*D1*1vweA?cOmQ9EvEm6AFmHkdVQ$ za2BiXk56L-=ITZY9{4}=f7V~3r->ge)T~eR(b**|^=4^i5W`{3?xGkf73>WZQQkE^ z%;v_2E3-+Ina~N?Sua+_JQ(xDSd*KJ;j378tAL#VZ*cAt34_|b$C(eizRZ^fm$e`< z$g!Obdi;go)2UtVpPt*pXTx?cT;@$X5@X&<&~z4B%=O$@Oe3WeWtEgPKI^BKdU}#Y z0R>gbx*t#72jaa4q;6nkvT37{*6wj%rg~tm?{@R&Ler*JY?C~lU6pnuyG|c`!id=T z57(9B$SnQB!%+gxKh@6rIu_VkI=%7s_I3d_n~$JtJg^sjey#oeZax|YoO`1^@b<^z z%ekY++FE(vw#TIy?qGS(QUC8Z4c{?{xNIW%b*Fwm)sXz$8_tHYmpOl(Q26sm2dm8Q zxp;(^T<*_I{oLxw8$8L-;^Wo0JQ;p%c5hw$=-*OlOF^KNcze{Jjk8=GD})vNBQUpYH+H%W<4; zbw$&GVC!G6sP>uM>L23zr2)oM{twfrt*x!Ai~sz2WmS&St7dN~xcch+{I<8Y6i}d% z&b6!C*8x7P4)AQ45ukwM$Y1>PL6$=wR@_hqG*$3%_Z5S`G*Cb{dk;(HF;Ih1seT9B z^0Qq@Ji-6*=kKpdT`Yn+bbb(>#7VKcV*!pHTfhAz_>Yy}l(LN#X2BOz8^MHK1h=~poI^m@v80)ta9O$?I%D=-s`*Pt33aePAR(NG*|4<)yu8ic!l`{nAEAS0b+PwH#D!<{<; z#H|bld)CAG&q90BldKT({M?x{C?@<^m9Aa&O2Bp1#9WyC9~O=(CZ8c^{~0ZJwjau( z->I4S+d}#yN7DSG&rP~yJr2GNQBv&Vt}^Q58MdNbYJw)J1V-H%h7oqS-t6*>XVK7 zwcZu@3ZOes++2e8Q(AMg0-dO{0zksCTN}qg*~}&?sxRT`Jjf7t&<%6kx(j?dhVX=h ziOMs*k1V5OV>#=8O!rlLOlK4XA`Lo*fhjN1pKnB)1`z7EPqOqrs*q9}xV}i~D!w_@ zo?Nw*!`p6{QM`TQnph|M{M+qalE)y0cJ;9*><(#SFH|(=-5iK9a#->Tl-cy&x0}4v z(3n{0<@5Q=3JsrGL1&Vbt~K@pKNZf%BamMz&AjT7p}YCQVW6x-(3qluE7cPDf+E$D z$J+;F__-n2Q3Ijt)~fv>l$h6X8&y3%sNNn~n@vIjCq3XC_0{@<$HP75s8pbs&J8lx zwYFgmf@lPLu7yZ)zG?UXTZLx{55$R#AD!|25W3Q#R^u|M)sQD;D2!vNo9(zqna?R# zllKTSjl<<&m&%QJs<)nfNKv(dV3g=<^#b+#qdSd{dR*TS@)L3Txwy;z_Q$>1$SK7k zhVosXg)IC_!zowH?1xN9OA+w2GG$?6+5v(2t;8y4qo;00Pk|9XH>bXtkfvp`iZRvF zaMi^|uSBg>IfV$$PoHj8&W?M54)AIa-EJh4<@QpWONSuFnK0%ye9-H$w{;rkp)sn& zvmYsl>{H;fMJ|55I<+7o#eU_Abmg7cjmf(bB999oY9_0*FMKqoey zPO`hMe(zo;F?0h<&Z=us0+k8@f9^HQPL{rz;Qm_IY2|ocbzqD-mgfUaJPVXWG~#9j z^ZhwFy&0zKlOa;buRL%13ckB|9{LjmUFNmMyw|wSp#M%6v>KR)#CmmT_DMB)Q)z5X zp48!Jj$Lg+YQTAPFDwvpYU-?vBwxji`9Vk`zCx5n3FS^)K_JJgD^8NmqX8MryH_h? zGgA8XSrR1@#37!9u|wF@2Qd!ycfg3|@q7sz4WZwCAIdILY`dLUR|OYgei-}Udi=il zY)3-09Vu9B==-d%yo}|y2-YuoOjKd#8CEFb2J?D>HmjBI?O}7T?iPdha`OB@9&WvQ zxOo55%Oz|L+Aqkvf6kTLxcc3a#T;wBhfkDlIET4ksE^n(cyBas%5pc$AsiNlZeXk? zj3=8^3ip_CBaBur#eP+P|F)cGmtmezyA4Mn*G+OR@Hc(nW8#N8boKs|EUb#hbX z89cO50`}cUT;_V@eCWNK=ZceH+9xQsxptHnY;F26uRi9zz{)k;a&}T;tuL&*D-`20NZ@lH;tr1#e>oY_i$4*>PW9htuQR_`A;a zPxvwPgWCf9A#X+$+`w3%ID6z7RjKft_d{f5Wzh)E3+i!FKy5IGQAf`&hh3vN_@NXS z6x9DJ4Bb32G4Z{!6}F`thxFH^OicD?CzgKt9bz*WTa|^|;{DwRmh%iigp*ctXpq{hh)2IJ?^ZR&oxP zT|Av2Fo2ccZDA<(s-@{{@$aELsuDy;91@=L<8SV87EeV!aI5uK4Ck=)3)Os3#^891 zTQRUdXP|j-P~K)_^eW=asZ-rtHnv~w>bS}}Cfn=M__5FVu}%r1E-;lZj1-zi>*N_` zfi$f@%Rn9l6>0JI;C9*S~K@iuZN5em3v%fKTOPNaQQ;Ls>??O57b|O&mA10d)9gs1(xF#h$p1vW*{S>zNTPEeM*G)>A&)4bdZ2Zs;gZ1tj%a6TG<>gwJHUSPrUEE% z(+1Ec=#|Oq;P+BL?`^??71yjyA_g1FnYS@0Eq2n^bv?wz*gtTgHzP|vWRbY)VATBP zC^OH{NZD4y3VhEad8h%UFp`=)ZH7x9W`K!l-Q}DO+@M&Et-OusNT-u+Tkwp_^qTW^ z5~rE=ud)4B0zNy=h#r8VN*-;{C?{>mZ)PW{{)rsF#ONNpGS3>CZ-PB{9N;sOxCWv4AlXWv>62fY!?Nf2|**4E-+X11IiTebw}P%Vk*Ty&KE=i>HAon0LtA75X&!?h|C?s1o!MGhDA-8nT5hRkgT`)N>V2KD zr)Bx#p=mLVp@E4%iBwjI)Xr_-cx&ej9IZBjN;(DruIPetuWvS4(vy(j<{~Ou{ByNaY-z;ku88EpGrV&ign#U2ctfdw z!IY%**stI+mC97*8lO=1zH4oFDMNuhM#O$y4<;6-ywarshLoI~7&Oi>FxWwhMK41o zQR|Q;QO*D4Ao9oa*RCgcn|9x{s-W8vRyqSCIRu@5@>gel8Df zC&Toh53!AHe`*Jw4@2BT}k^O_GZAl_dcf?#5 zw17L^hq2Nf^KR?dsnb1Bh5k-kG>@Cd_}12KZM3>9u`!>0h{5#@)E%zbxqzRfd258a zJ~A7R`B)V5QcpMA5YZ7=rUrRA6N2K+cY7mRVQVdG>)b?7KnDgA!ZegITUe@Hrp~!{L#88Euy{Yqpe(;>{6a;6Jh$S>9KI%27$tSB99F1FwW{2(l>2Ab z>|zqcuxk)-A`%nxTO-#b@xg`ks~SCJxOgf`3e`tD7k;8??W%7{N=lZ}2Hd`_)dzjW zwC?V`u13}thzyWGF8Wl=D>k2zQ@FX_Fl&MY;xlrvw#NAK!82QH$CSq4eMOKGP4%sU z!h2v|JEy^w+^bsSxodf;2YExMqh&$oky})uht@^V^`_M{U(RLs28K z!-Y+1+Lf3oB*<_tguOwTul!)HbYJ;E!J{96!+hQ|cr@F&>3y*0VKY%X_TAF$?M7%4 zs@iDwOL@U#{z=G^4Qy?pVTsGK9+n>ScCzZvb_rtF&dwYVp2%HZUVcW)_DRw6&u#dj z*b>NC|1eT8ML|a={#RG!9CZMr(SO%fiDs}cysKH^_{kO5d?0F%sj&Z^%V{jB(h;ZY zn|}xHb*!*oe!BAht#B4q?#0FTR3dQI<#oi-Y!`xt4p8P7oe_utl*9_a`H(7BN!`z{g>2=ckeoH*Lge$YRxzZXJ;%vW&kHSi=Q$9T zo3UEyyN2*6&-Fa#V=(rOcetTE-FgB#8@sr)TMpi*BLD`ucBlFSt!*{qga;}8o)Wrd zrxHNmwFDM+cEuE9zmB<$89`~k%q3Z9bR>(|ernuzFV zV_$@JwwCM733MHIe@2pb_WHVW^oZYLg$1NXVhf2J3w}qzz;y$yYoZPC1!Rk6D;Irs z<}5rY44Z?k@*90ZSQ>AlwyE1XH7CXRX7qGTedEIL79IdWLL1{R;112_*n0)7r)ZMC5d3aT*{gnpTA!npS&*UL5(9VHco zM+>iUwBcbF5&@AE_w{OOe=MVvlwXxgMCsM3{R^W!3=Fd;uZt)jKB!bllEnA$LFn57 zIMM}=*`VvBym&D?N?l#Q(m4yNiPiWFL@#YWxJzdY`iSn7L~(BS3mL%{9z?Yrc&@%1 zD(=~}!L5IB0b+T5pHc7_bMm~2M)qYULjx5+NlPH|yW*jXZba;=Llb?J{b;0FR#poi zr}3}gtC(V7SEEgAig^3@&~;i+1>{%dKi%6qR>IsQG~MEQ=0{e0IK!YwTv86R7$k%P zUQ2a3p`p)6@E9wT6ez2pUhACOj1#C(KSpxCePi$QiL>OCl(QNp#alZ&^S}vF1Lu;n z@nx?=%`R>}h z#HlD0NZqcqET%OtG&APgNVy=@1N9{nf|#dm&(bz&tvla3D1F zO*|~#oY_*9Z{Lm zOBc&$9thb|bWY8g@6jYbSq`?Frg_p9GOEGxt8a4%iJ1p0g>3bd%GO8@tFpVGrPcxk zC^Hk&tHD1@`1lr5cu;vBb6kStJUkn+8IiN%+z{-JsZ{g_Qx8FFkS z>+rbb=1@wuR?CW|%;Kv+q`8+hMt@_5)()FrZ~-Q zr_UEYQp56imr%GME>e5Vl-B?;b~x%nn0`xy-At|>ra`L;0U0Ptz;E2!+_1fX+@SUN z$!wpOo5W(2W=$@mbhIx9a@=^ZJQQEQ zW?-Q87>#{Akek(Nc6K>L>fjyjXg&~1iMXvzFLb_jc)t`?eh5}ug>(o`1a8o-fc#3a zLA86K(WYZ!r7}!DxeAjoaZSJGi25%3ob~pz-jRiNTWRm7y8 z)!|p3JUzC)ZDI0rp~49{k)H>~*E@!5=e#!ch&VlVzy>;)g>5+5xnlcIGF*lQ1s}Jo zYw&31OG)?rzCKjNOtN*FCGNfFm8qAp8T40DRUHHuEO9EmAMaP5kL90bF)Fub)TISxRuA}5k&*XnK;hV%YIk5o@O=321HRLd z;94OcL}Nruy8w^N$ggoQ+$dVQ&1~5=!__dNax1=t82PF<(kF3P^Oo&>zxBuC$HqA? zNN&B))8>9bQl*dk6eVRRLR!&{z1#K?pD(Po-B3FiKz0GuMq-I8Ef_1NwpAqC($IC_(l`wT@A9mida&2jeR%K?@72+Ml0fq|} zXaL{Jx{Z5eWb5Qz1(5&5@#C2~c~dzv`@dyPeHLpBWU%rg@q)`0-abo>vxEEi_XVSXSv2t zE6%iBXdlSxr7LZMga%qia4he~|l>M0f$&gaoCqP)yHKEJO z%wz>CT9xnL$@_rPC$YC&U%CxbKMlIX)5w<#AOBGea)|$~*tj~`0((Gx!w5Fb zMq5!4HjxRB@Qe7e?S;Y1(ak5@b%*u?S#TE;Z@U`ShlSc3L~|RF{r&e9wlyWQ&dYU}%tZ*Mz42f%+0NH%R{cCZj( z;)iZ7j#&`oda^WG1Z_sN;K7}(l#qnlILgr8r8=HL76w24pC*kG{VK+z9pc+hGoamP z^`-f+7=H~s4+S6Wees?Jqs0sWhd2_dlI8YZ4)2 z{2rx}AjZbZIwT}vvAH0FaoZbmu$d?w6EZ5)V1~Yh4M3myT6Mc-q_@Wk&`_z&W0pdmunV+ZSmVJXo)jf~6QX4X=ce{N_ zd(Q&`vJaoIFd_IY+Ka}L3k@q)vrX!1^Cmu;>o7Ajv%%M3L%5jpWAvfQjy`~0oS$6{ zCtF~ml>fpxj2Y4tBl*y@a9qg7S2!TYD?%4usE_ddrFCzT?@k3Od`(I_uK+CAv%i|& z5gYZ*(ay;(Gzrp95XBhTCM|qixDlK~&GCw*$?YPw>`FxsLF&;pTKnjXO^(ETDKhe+ za9W5T=Elax^IsuR^8~&VeZ|3Ov}w~kw7d$(s0$q?@9ZqBbzSDZ#E^8@V8&xu{$A>E zkH2dW`mVU>7j>BgpBmsVZ93tBVF&xReKwj%?IL15rhbQcz`bB&V!Ev=wHEQ=!v|}( zS|tSH>)!U+b-eJzJ4Z7b#V9BVH-091C|}{>;Q>1T&rPhIZb;(T^z3W~5hRyddi(X` zU#g! z<7$TaQ~Rn97U!*~I(d|YzLBcpb=$M0$WFk3$;*5;hjVi@8v@B~dYvA1tteCh)cX~e zV%$aeQ3QMq$l7e(D05QegDY!l6RH-L-rdX6V9toL8mvwDpf3j$rFlR+!yw*_dmTz( zY{y16<*u63^Nr`|UZ`2`u`kvhJy<*{4 z_5DT)BNlXdI}p&9hhDbe%CqxeWZgR&#A04hZCyk33>VJ|Easd}-{`_(4dC9C+w4H< z?g_R<7%+Q>Sv$WzM+KRFm@&gf0G=!!YWT6kkH=Kd{tF+;^Mf5mTiz8zy?@P=m0Ne` zfQ767py!0K@%ABM95va8XCe_UYzemhNS8Q@ey8onOFP^`E2mJNYk@W{ZTgkcZP0{Z zYRuboHvB%+r7I_-!^F{qG*Hxck72)bL0E#Yu(q>3n?xk9V)tHDlsk2beVMMVGBgIf zPhhJDy$q*!fQ7kh4m9wAR~N`MmmwlGS36F?tu*Znt4JYxOM_^Hsyp$%ADj&^$jFDq z3BmUDLBeEKVje46u=$LF`U85hAN23Y4JR5Dxxa>bM`*$v^2PP7$&eA{u6Nu1zIr9y zi(URc3unnpytgZeKYs5>TJ3?eY~*em);?%{cH(RVB)L|#W6(a{cbv#K63ZBY>?V@K z^s*b@O!jBuwjC&o7&`pYgry|x@9(cA2w!O;NOy{D&%7Xx&p7&>o*(bo;T8s1Y|O%# zY6S34OO(6fgoOuP=FZ?2hMi)?x5bnb<9B~_K+~XDLPbfbcOVbz*wLFTOM;b)4PlUq zvP&+Kd2zwGY%(;vjSmg+pQ$R52zZILi%vu~86~zuL>3Dm z^|}M+UvV?-9qc;#>=jOmeIz+?BJ1th!ELTw#YlVzpP0wh!?S5g;tn#9Ch;K7oa0P^ zVjnnLoey^ZV=1|jJt8ip%Zi^5X8W=Kg$J&%23&J~C-+fz8D{jt6YyXmkz#*sZ z$9oJ$2WC&I+frMd4AXU{$3O@Ov>t=qZ_tM@VG(Bxv@34IW}l9#zmDw!(t8`VKAYr6 zq)2uZ*W_09n~Qk-cwWCDFDEzbtA4mw++|%y|7hRQ55#3vVn%dQzU`YgZ?ZN&z_R*) z0Z(SSGQXPutW`)WH5?`DZdx36Y0Es@Z`hLP53UlA^jJOSc8=<%G<>2`Wwc(g4ny_f zt*3(@9cO-$sw%1$*ZHctyWf^{QY3fWlLj+qg3*)WP&%{Oxf{V%)N4!@jc#V=fTAb-k8Del*(hV#=_{Lo{NdWgOVh{JPY%L z-ZvFJCf9sOs5_jKIy?x0rnWq^GZPY)*zIfO={>$IMwI_)HO`2jQ@|E=C8uEGE2PPj4><6oo{>HmVOC z%=)2^B=2abi-F?PH3NwTm(vG_%_m2{XYDIS{m7Lnf3raL)=K8HCBdY!?L&lz?As*Y zSloIM@Vmd}GFTvIa;ZJ{g9h>3Ut+_dyl2l*_s)pHi>rE zm3zLXcuYh*0ya=-6QE|-@+NwRbfFuVCY;R)_iv6B=L z2|$X#eSBJH-AiQC)g4N}2?0iENJv>>mB6(43Ib;HDue6@k0veoW*Fl>swPcpj%>)D zW!2d%cNq{6gkO|4K@#9O88isZKBqq@Yb9>6O5px|Kdd;EwE7hf)9N=+=gV8)E|6*O ze+q9_`E>K*MU^aQYuF-(dxUCS@s);yR~$fezz_yWCR3JqTO9G0!O*jHd0)vO>;zxP)o9a84ftAGnsJ-tPv^v?= z1m2AkHguM(#d_m?W7oo6mrAAd8#M6>`O^gjFei=JJLT(NHku45$Z%F0Rdr}7C1KfA3d5v;@(G0X^px0*}bguT)JL?#t7zsT@{sQ6xgd^jb{JS4Y8}+)?s67 zneWXA!Rt$SQ~(?YvXA_w>!6+^Xo=-d{cAk*GRz;%E}#bk5Ot1X;R(=^obUImN&n7s zo@cpH4O?p4E4EY}x3pmd5pOE3!1S`}qg>V|2zZY;vS2cKLSj5UjZXtUh?uq2>Qsx0 zk!FbXNAgEzvx=Eg<^Bs~?MeYFm0CMpVuX#2$Olw0w5w@#=j%h0b1jD@Y6Y0hvCG9T zKO#Hegs{3X(V{xz*yk^GySpFAf|P}s&FSwgpTFcuXW%%Y2_$&uR&!Et>8<9>n2RN_ zW>_IjQYg4$an^?M!3>A;Ryyc`5(eMya)(J!3QVY=oj`p@Y(hbkb8r6`G|xGpyH;L) z6IO|`X8PTJ_`|cD(iC5SMBd-3t@uh+?eV==#8kGgPm9Rngac>>KbSZsQ5evqrC)C= za2NTFApny&ZNkqazpsxB>w@5V0Veg0V#QenKY>p{~BB@vVMCeLC%+t z*vw`WgM^<*9CEM%D=LCz?_vC5jKC@w2zoP@-_e*L;*{`G1@d~jI9 z!?2{fr0SZQ$f&4vkiA>=rc14Nq#|}Uu`K|jyY8QVBaT?ZzIxcw+}sVECyd>av0gPNyXJK@XsjthLY@`Qf^nQ?_gP)ZpciSKM>YrWQ&kzw z%q@_)>@#M7XsLCe^F|))x78D!SH)U$%UTNVnRl?d&s^l@Ey$IB6G~0ltCb(`(@R)vf~H8DQUEEMJw;-# z+PPrQ^oK1Qhb6j&L+WFHbN^l_-IYtb^uOP*sqjY$;>A@E>8Ivw|KW2Y&EezYZbf2dwngm) zdC$oPG$`p+-OGq>3uE|(oKo>^CjD4T!g_A?Sb0bx?KgVB2;H~sO=CY7w1z%dDfQV0 zU_CyXU+PHwzl|f1j1iA-3ucL1m<3#XA3xdz6}Q%hDS$7`yTEKgKc`Y zX%iMdu=ep7!Kyiwk+|z(v=+0U-r?oPoLYSwJ+L z-)RgIc>L`OTxf)=7&-yW*RQwyFweA2@T?>jnV{i0|5Hj0XeABnD!)I6y?Ym^FCZs`mct_ns~Jv2k2067W4X7dXC(#q zy(@@pKN}w>v9rIQIpu$i5v>M?vu6KIpFcEoFX5&`&mzXC$}IrmN{|a1hCHlQTCRY#awZM4jm+g!8_|7YN(R!kd8$BmESM$jVUnn zI?UIOIKln_8!Xts*%I_O=EG2_De{M|LjesB3o+AyAfV(Pbjr~|_wc--sOTg8^ISF% zkagD@8DOkE!ij?wQ~YT7Oc-nlT3z&GOo&T$iK(Dtn8w}#c^4BK)6JnY7qmf{3XHa5 zoxA$ij1L%>Hi~__-s_0yvvI*UodSjbT=)#Usi8`nDkzOr`g7#xYsYH@O=mc@^YpbP zyqv?Kc~WXUHeYar4M8l%FHvT67w|&og(wo)9Kj3@3_@X*zfVH?csbl3A4SyJ*|;@z zvR%6-3+*Zp0clE0KY@bS8hwJqoI>nUu>{HQ7j{4D)+F>VF&boyfp2ruYcEJV)_r1F zy)}3g3zp+Mp#BDarT{Q*t9pyQHs+a4Oiar1*UGJCxe*%CUQ2}%lt5H~zM2BabB4Y< z#{ItZlWUhxTcUDhreuBS)u8dh+%k96EDCV4l*}kDmnJ&5pF248V6yB>06WhE7N!h( zAn-fxHC!D0g035A3d$eU1Zij~9t*|xO_1!>$jHi+AUD`;#wt6eh5o3EDoQseZ1qG$;2U{Qj4@rG#Q_y}#P1uAkwIOeD}EfW0ZJr5%S)W+ zGyr7Dgaf9C6&~>svzCiB`wkdl%D?S%V6&F{ZB z!W{76du*uO)1!Yjj{HaepX{nCLnQqF;sX3%h|B&bg0Kgr8;PPojy#;a>*-=CV3%sJ;D@+T(Re|fm_-!ydp_p*~;^Jsvm@&laU|9HFo56#Xl4nB@9 ze?71Bgb!iv1!a;h=zp_hUvBc9vxN5LnRDm*s<)l`%YG4B7$x8ggLy#IkG3600=ttK znn3<(TnLNV{u_@$oMwiydP-vu1{By>ClIZ;cpAgy$u8B0x;YStQ#}pLd5-m`Sy6T* z37dEw>}j5-5$FzOlr^KGfV%W93eFl7-LIU|_3-e@QBS$!Ira4=kyH)Zj)#j9@O<&0 zU*bM-B3+y?>J0N>1l*JwCd-ZNen9@h^hf(f zC*_)h>Fj#_eRfzmIkmuxdjOJdj0J%np#kTCt$(?+ufZ}K{Nf_l_cdr(yE7Fh-^`%;B0s~$T4oX^oYyy>d`mBk& zF;SYURBxhzL}uR!@>GWH3krKro4y=T@2OV-bz8*kLGa-e$0~uKTj4k@fb6qU1gsP! z#vda3v_OrZgd*}BJlh{fo<7&-cd!<`xipdS&o$%18rT4yj6%8wZ<3yz>YfgYjKS~I z;x+&Y&?#aA=m^z^G0+9hc>m_k;+U7fgT3P*tMJI>(HlHLSlNWOuEhqhNQq=ER0O1( z|DfM?gNTkORAzodnFHMtfG+}x1e}i_`G~^=Brby--8|q5yGAi#9)FbqaBY3OYRM9! zt{aPtr1Y}i# z|NKKb!E@xu$Nsm!-wX1vcE1%8OYqz-0hk}Vwp2^Vhohc077AvWU?(dC7M)xEbk~=} zu=T@*F%NI8jpV~aLJVg(Bfp;dslR!z=G;%lxa=D*IbBX~zoZ%u3O8OVBkFk;S=2T3eud>^tLK29YBr zRn85q+BAZW&(l?E)Kw){q&#tXWzLN;7LRw+*<7JZ+JhPWQT;hFf+=Lj#?o*ZSKpg>w|rhuNC=4=Pz<7WDgulD6tJ_&$8KK$ zbZP@9^;Nh|L~lcW-lbsYJ@DAYhpmOUj4B1vEng-IXw~gaJ4*-?Jo5C4?-++<_>6m7 z;H)P1+jek$nswPqJPAyQqTzFSd+up8G&BMm=~q{z`4tL#u)Y=dkjA7$pBJ2L%^5^1Ld%$-ds?F-GLJlgDpq0>9oNgxi2l81s69F z%L(?4!^KGhl#ty(3JrDtJxW9Sr;S0or~$MA`k!m^kEgS>vvYI%8kOT*W@0SntJk`E z!JS*c-M)gIgJa{sYY9grSHY3e&T~^rg1&q1K3m`A^-G;&_I&I5<3JMqe%7+%oAqBO zJP%tmd~5l$7*p#SIHGe{7sO5xwb-5)qI0C>gZmJuGiT3kv?d*3#<3oLpQpiU}H>i@pn zk-|xr1u|+$>vHh4W-Y^Vg*2srN(ecX8TO!8i$OFK+K?@?Ye2d|z>3ynkY_+>I5@n9 znbo82{QDAD7{yP=Tz&E41k$+*`;vB^R_mS`m5}oEK$&=xrN5`=D#CMS zxMC-_NP|8^zV6+DlQNtV;J1%65$o{kL9jF!2CuF@&CVVnU-MR=pTEMdcA=5rw*l$a^qp)VtVIWUd66%j%{F9qG%N_JnKEZOjNbYA)AxkqQ61(2(x>7!DIa zQ#0kn+AMA*CWP(pPzoMTSvW{eOHVK6(H=4&ef*}jMS@S1iqyUUBsU6|xD#0s`c;pD zz~?B3(!~8%h~#EwQPDYQ{28G^-5ERhN0HC_@^8Kukk91Gy`gv+ge|rRAzli#zPReP z0spRb?skf`ZNR^42j_m@v}UvFU|43vuaEyFE9bvrHU5uZ^M7XDDtG-Uyv?r4G?TG$ zvwe+Vi^ipA3AfFzS#I_gmpp7ZqphutQFR)z%mAWgSmk!QK}14AUPh9KK8KF?`}uj< zvCe;8gWb4ECG2~BfK`1!kYm-@s=K&;jc4ASU>a&6<&9*2psyx+OD4g5Eb%*(+;CXG zAqeRL@hddz6r*NpL4kCTt=q1>tdIYhvuEnlvqtr^K*ywj0!er7j~_%%c1HTliQiA* zBS)KNs7pV6H3{dwI1{@3@>!DHH6Y;L__q~yqsxEl11WJ*dp~_*67UappL?DW7TcUW z9TUTDYT}z;E5NZ(RdICX>BZqlE6<%{jf{dV;mZV#Ukk6cwN+6mTJ==SRSd?(u`(WC zuM(|UMOauAFaq;)Fe#TmJHykRt{nF;RyVNj)4j3oZi!#Z`Oo6o`A^G?r{9H_HRJB& zs(gb$*DaHv?3$PLjoQ0N=gevQ2$8E!R30=D2I)QVS2sGibu3-O6rJ=Bm*O-p_7*X* zZE>n9gw^h^Ht4U_$FDCUJoWOlP}-?I3lBAW8!wi#Tw*i}?Ca}${Aq97(|gAB%<OGC!6Y;634CNOs zuW8Jjko|PBF;l2r58-DoS&+8?(%b`?HN4`>*Z{?ah#(we>e%hO;C%qhf0xht@|88sB-zr8>PiAbP_mnJih{)=$&R z=r~qBclXH4u#E2>ESWF^;SC@rY+s*}cw`H&#CP!1qPXdHr!K#&YW~aa+J|?3R>)jG zvHS>kC5q+4otx~=0ao6P2|Sl^&;C;9Z9X^V7W;K&PhdUkqhx9IyG|qjrrd!pdH;U* zE{0aw&&BSY$zG^#(+58{lF6`w2ul@pnWrgw!q#!NpzsKWkhpqsn(N z?Dcm_$m7n|yvI>x#?>fVQ+_x6KPypxLAnSX zbt~Q^-LSs**UAs*>W@3EmD&-`D*Gm=UQy$R4WAr1PG;aeV@E;C7ue6zmblJW zZV-#%S4%mjC#gM4mF+0f_qglp%<6;mnWf&L_pAO@(~mc^_Y_iA4ffaxGtJ4?kro7H zPmiaGum0vk1V{eOm&oz~95h%brIj*R-n7@Q_C<2hsT?=0@w!G=<9|u~#@mg2Ww&Rl zQwj;HW>uHYP3MxnfOyfpcKldQ@a&gI$GaN5c0C@I&`z(2KhU;aLVpjwq@9SZlOH{W z-@08wotDO&oi7=IDx=HxCfU#lGqF;kVlTTUwK)AYb7bV8qNG+mP1$(O{MC)ecW?8i z3e+cBMXz-~h!pY=EZ3XA)ZDYYa6l3LcVXr#omPtXjV=rASL2$0%Uo|MyeQKtvGU28 z@~`GopTyGQR`o|zjVn#NFge4$x!*}}TJz_BmPaj^9$a}o-n4L9FMo_%)oH82S?^ojXfm&c5D0In(mjAdx8T}bPCdfAZ+q%&<>^5 z?S%F1q#R{w@A3)`fH6R^-zA?4U>Zf-*@t zHM}!3#^lSjp4T2lCQtcjDO|N8C$(jxlu;#Vc2AjF62d77LL3tYX_DjVsc5UFQU-n7 z;%BPX$IW{XE(Hfd)0j*)Y&xA>=W`zRJ6bl{mOd#bg#~g(+! z(Ei9iaSPv!?4i)L?FsSMm=&_ov8VIr%c}W4WVRX5SlFj?VF$TkY+N%g_JH@3>B&>{ zWzPobT5jGsj`dmAc15JyshtSX)7X-f2y>)c??+~+PxvO%D7i@j27;=Yr2vX<}+_p4(3d4I@3dQtSJ7_5DyX*RS0x zNBeTSH%=R0zOK>Ot;>HgEO3w?#kfu$a?eC=j0*RJ!iDrgBa@=$bpw$Z&pMeccv5~q1PEb*KhE!W6D@^=#Eg;^igP*al8NGl|C z&HkE_69>rY?L9n>)7yIiq~iBSu+{Rb6V^mSt~yF*JOzXNsdxIfXbitlo4fHkgII@RS7G-2qLKPnYVy zoz|?TJf1AJ+|8RgIkvNSJn6})lQIixP`~niKw5d-CqI|de$wSrhLOy%uit`dw8kS| z$%_l~$s8ke<6gQy>*y9X6(aDo4cU;N6ZB8^ix&F9Q~m2-}z-zGXIO>eS#S76j6!y!Aw zN(6tk{UF=s2~$>M0q&Aloh{{e`JXfmD<<( zqDjl%Gf9Nklq!nUHri&ibzpKn*=agG@HnGLOkm9ta|8UdZ*FDP>ei{u&Z~{K#rznj zll)wB^%F|ZB2X$dcM(6<^^NpkqEf{xGR4ih$9G<`G5zf$a0RmtCNkaNiZS>MP!tzuNYNiRWV+s59P zA1|X#q17CRS6Fy5v=CM7n~|C#wszm>9XY4!vdgRIZGG3b^cxW_SS;?^!t>9$1N~NZ z99E7ya|%&LyH0^xv#dfjRHXNt1qw&In>#RVla?wYZiFufe~)Gi)daEIjaqq1CI%|x z!zW}HEmE2e^4jo=b$u6WVW5*Ngx$477r%{j6dj4G>b?_01`ohYqhp zDLQNuN(d>>{hLg+{X7I4>UJM5>vLFUl-y>{E}f3iZ;Xxi9SPTb7i+59ESNyuQfd6= z`nsqIYHreJ*;~+ZaO{6$?k%9AUf*_MW7{GYq97nGAfR-EAOa%YIR;3Fbb|#L)JQ1Z z-QB3DlynR=jC6y<5Cg+}_lWzy-|wvdd%pLab>8(I*4lgFF!PINp5OD__jO5={(XJ$!&BFPDme?@h=^MbeR!Gcw3PV{cqLsqd4_GlekgeGXb)~>#>D%r z7KfHfzw^sI5#$MSrH*8|_ePhy^lUgfzq~pntib(1;v+}-FUc5>Ra3O%bn(W@zk@o2 zTfLi6sSb6wgtp$y^`y<(_Lm#^nuIu2e^yQt{(Nm#eD_!nVk9~ASWaJK27BqD?(Ve2 zGtU#uZhW@Ac_4tz@_q;-tBSE^f>^<(1u0jKP=Wt+h7Q|Bp1v8E<6lTH4!R@j&|y%lrVXQ3@|1>r7tspJm|R= zPF0|iwD!QJ`zd#;uzNuZ);3j5nQwXR=3ZovufMKAe4%}6D^9r_ATT>a(($vk+{mDx zVev`ymv)R~`_PzY43|m*|6Nr_+RH7sc#S=)th~wRThFoV`nq!A`e3AE1`B&{OY0Ao z4#0%rO?8cXbGQKW;MCD`RNn?^RG+?Jjioew9Jx8`8{QxIF@NNI*i2Hi*CMTU_9AmR z>HOsdMmO|Dk-CsE@Ax!k#I&7@ZnM&?scro*i$qQ_Q|$~SgV&?_)^{S?cbTI8t7UPQIH0jydKx-xHKBVjOio> zzZN$eM{qF!7-`;4G$P>2@MQZzqri^CE+SllF4C)C?Gn2$C`|-PKZ$#DbB8B$Ah@E+ zQ?Z)-{EA_ZbLgCB#BA;KkhBa^V=&3 zUHU7COBr*p=jjFX>o#7kl1QEgV~&dnud-}20zB7sJpm<2BTExx|NWBb`@xSoeODC{9c8Jz|sWE z&J6pHk(<|E9`^Yfv-Y||Yjpfk|5i+J2DF&R-z}^1EKjJt{p!fnlCAu|g}p=;iN}cF zq)2lMSi8V+NW$iUyz{|8W0eI#n2a{-M7%&(#IC z{sw%BFZ_2yu>Ws7Cc`n&9{RbE7f(0Vr(8*-u zeQe{vwJ;*IR9dR27*XWeKWmp+TH3ez_w_o^zq}v9uEz(+RwqCvg+q%($OL93gB&Xh z%WL<)kI`+MR&}7^4Y7RlK^zg(?s5N_Pq9lUB^0pInwy(>#mb%UwmJXjQEaTwOz%V-*uGPLdzdNA)(oy zXQDp@;}-I*M|}fO1%F=Be*1qcUiwdBuvVW8h}fmCd^PDJmO~m`Vxn}saqtab-5vweQK-4;uy?E-g26)jD2Ha7vCSKv!wqs z-11{&YI(Bu=7@V8a9bn3m+XJ$z@vuweikl|ep{>ld==|5J;3OpwXe5Pg>Xe&GcvK03gl=SHWaeXEUu%;8d6(=-xb}>kocG<6mNZV4 z(V00pyu1PXp#9w393W-M(En`K82p)qO?dxmf{(*T0HP=Y#yC6R>{aMo5Ir^<_f;_}roK$-;+d%?ufI0ctn8|2-PL@)r(p3Thf=Nqbh(b!|ImdKp~7VYZq5?I8y`>c zQEdccPqV1$=o_zIFL0bqx2L0OsgzbjaJojhA^d4Bo}QE%C?s36kjS`XQ{zy*eKvGP z!QV*!?ogPJP1^2M(!A*g)9a5>G=}uol&r_Bm}2-_p&S$$rDyZbp}L8z-RZ5vi`33v zEtt{Bh6SMp_i=VAI&6_PPWxA?q0H;rHgYRWxa@#PtPjQ9m+2B$Bb=lF(rDzv-yZ#Ad01bv@y9hCMp+oSUVrFY-{7;*FBoJGt#hraflf`np3k^@OY92QKPg5ND(p53~dWx5laAh=N;p)-kXOzJT( zS@AP-8=y(m!H)0Ea{Q6*cKg6tmyaUOS>rbPKcd3sQ%Fl6$mf-r+zg9$&gLs@Ryg^p>3il^-umBh)}d-{L z^1-k2(a@$VPjmV%@J0l6^{uK5vYLhH=J6j}+A7E6*fcc_X&BXRpmeEL-rx(G{+E+E zgWTA??hn&u)I6#rct0IwiuldOTy);^9Gc{+%PL_@=yVY!sUK@cY|a`XWfMHOY##h| z*p01+L|CEr%lHMuzyYmi*%ue5{K=yIGS#)~I2kdD(QY*z_viv_&`rB!)28san zLmKbOv>@yrX(VFxq0!YmRfOSr&zqFi>7g>SMwG^Oj4_^O9V;&Ar=hsF7DocvBBPp zGRWeo{u#`34b9km|Cl%-W=};_acE6RBYyYd51`H^LUJloNK|TBq6nUO?~THu;Be=_8KM8CbFi%4gmXuZV8d%1w!k zgl2xMi+3&xb)DqC^J^{yruwdd=r2#XWqFz8I?HXOSgV)@J9BJX3HjNBupoA?4-9^o z26OKB4c$GHJS#7KnN4g$89XZ76?1z}g`zcss$K8WJ&L7og z8*mI>v=X2Yxx^b;=XCoHhC?sQ#v;dI;t1ZjC8C)ACIeb}uJ`I0Kc^{8?LE_hM5r|UapC7*Rw5C)Y^3fHi#xK~cG*6zy zNcruF7>K#>6)E!UM^(Ywic#hd`_21Z&KIVAuw4O5Q64#8+8XTCd3OEsn`r4jE82b9 zmkS5q$lv2JZl0_(n`>R&_K3u<-^kG}kc5KV zI9Ea;Xm0fNtX=7}x}0j?sO|t-=7;C)IFE|A*?X^Eeq>KwB)hF9huB_QMQYO#53`|Da! zzS1GvY~qcXTg*>52oCwhaHKgc4^TZFgos_8>`%@5=BptN(NF&_(Y)teK)hb~QG?+- zmj3V3$8 zmnmK=fxNmh;r3a_s;uxY5O4$e+PHs86$8H7rhle*Qa_YeE0D0`tsn$FSoKPMtY za!uO(2SfFH$&FO{pwhwo-}x7*pO}f`TxBneaqVw0o`Vt_}RiLe1MW{i{U6WoW;QoLR_Ks%H zi#rH^4La<#?a!pOc>pe{9V5MJNwW$4 z|MMdV%+O*r<2UKZjUqEv4%VK`UW@gf?6e82Th0yZydDE87;j4&aihiuWaJSuP|xoL zk%KG%neZmRpX}@R4L-YnbeX@M$jR`2z_Saef**WIXi_7l@P^0ETTSe^|qadz{ZXhLf_pGZ=_AsFJo=TYt61s0vTl z#OpNb#02qVa@vVokH)^fg-0O~C^q+;hh6;?uev2JeO!@OTchwz2%zLk&&^d-P*5O} z7=ajL`Si6rO_5{p|G@oH{X-7p0lrE}NeK&^yA4uS6o#hZzG$@VmlG614(SXTsZ;rU zmRstOn8FCc%$*@^dPux-DTHk7ow;YV!6HxQXa_gon^2;|N_fkj;U2n0oF>2jNqkQ! zBc68cEDSE7v2q5^i;hHo5O&Lk236W#I=i<+4ZsI02g|ITpU-Rap5H#aBD^$sY3`p>;qUFa&*co59r8fku{)v585G zb(uwf{wPsoZ-!JZ6z-4)Sr4-%$``Kvbq@@^>yI9(;zb4T-#4@ue6_Rmu_@IZ;>b_P z!402kdYnoec_V5s=80QNFE1!|u$PS_g4-JX91lPPoVCB6Q+qC`7#m*!Z?q~y_r;Qn z;f2npa$Xd*iOB$EudaSFM~ z@nrs4?Sq^SH|>H)ufD$%)M*zH)_dl&4vqK*p!pokfP3-nN8e%mXQ3y7?SO}_A9?#U zHuQ?*JPg0JwRNhv_76+q#3Hxa@$I{<)o=hOX~&>ToH;kuNIPwFLtTG|3VRL_^>*9a ziXf-^@0q(BWiFF;WUoABX4?NCFS1;g|Ip!d|2uHR|IK42_k)u;kNd_A(XF5OS&nt z3e}Lc$WSVO3Bjne-Jz`Lr`OF+oZO}xTtU1u?aP-8O*c==b&+Y^SBVqz?9z|?`WK2( zEi5d|Y`(8K`39Q-mduK3480**fhKbKokzBdUPip1T`Vn|X|iG;5GqA$mjToxXFir6Rz9n-IKv@3i?AtscC3Ow8yVTa9=$B7o8$!iI|mh6XOL9HG6GX ze84Fw%+PgGLv5bRW<)-<)K%xTf^mw@6TXU4_3fUHi!Mj{xwk7mi$#$;3XZ(n8_=1Hv#>_9MTwaI za<(Sio{n+O&8=QN34N8`788vj=Z2O?>q}T!S$jZYD3drd$GwQtD7ugt3lkij3vI0 zQgKxWJbe6^Md^m8O-KCgMJxWU($}Tg9ehT_^3|~ z&pD$hylnjOEAhvT65l`GFtp^Y+dej!2~6zgG^TDuxn zLX;e1QmoeIAHJy}7N?{nvQiE`uFP@(u0;xAf7?nS*CXnxGZA}szF-&DF)MuP)G5ay ztFo+u0+n`gaTyX4Yy2-il==EhXQUY3n)oPSmobReRt4NBX`Y6YQ|YA}E;?_?>uBYO zK4Pg#buzuZy&xobCC09q1+2>vUuWTvyidoz;W~qI)zsA9KWW>|VaP}v`>T8Y(f5|_ zx07>^m!AFP7c=h?<@Dur@6Uf&x!xszKVr?f&Rr!f+U|{^#|k5{2gBIdSfxzMd8z!d zZUmOBRZ!`NhGK1ZPjOXvU5*Af&1p*1Q!2PU^3p}WGomUe{k6je&gg(l@X^Wh7L~%_ z`i!gwRIe0}IOqqN4k^AF_Jy_$y1r*RdH0vtBGCGj+Uq(=UIoRaEoO9Kor~7~{jn*( zoG7i(QBhGmPkvnXoEFBYsi-7Sd;B_BvpuCp!MBj?KkixE#($k|8=b#rXY5GxMZODJ zPf}@P;`lYrbI}6P+pFb`fTwCPGtb6fKq0t$w#TXrodD$x(e(~|q`=eXec3vcPis)w zI^ALz;N{cj&L>L0#Rk~u17j1@p<@!0!YDt|=YOuAUXfB>kGrUjd+`fGu42m2jKpK* z**mg!M+%!%k%EmH@D3{BtQ#+#HP=b7a)QU_;pDp#eALtSc)cFxmaim2ax_#rSs1FB ztJaaTCaMz~71n5+|Ink!W{q7w6iM&Pn4QylOOMlRX3oa}%*Q8OJ`3i$ z+>@_MmK`+^2JBf`o>tk5nA1fEjpmC(5LA5K?J%}f1)f4zJ?X69E-Xa*k0U&>1Yb|* znF0qqPKk~>jk9WD=$5BUgPXt8jk)XLcj~7C$`puu{Oy;)SBx+we(GH5@jE9D zAEesawN2~HAs>)IM4UF9knI9NlcbN__1>diznX$fmufa=5MoQ#gFly_;kqIeJEh4D z(xDF@5O7^7`dHAivI_=b7hcXAMZ%s|^c!-r0U(FG&54N-Wacn`=(s5XW>O+BB zo(%mxg=*y0;J(G6W%o++b}DHLepv95h@BEe?PBs6!k)9^cmRJ^Z7uDXOHm&$kD0ci z_2#6OL|molIqxZlsrfj9hk-b*lecoQbO^3dp$rkVXqtb;tcCzPuP=#=u2vo@YqEuq z^OmDhR$5vb9C+n>%;;a6c%PqNt~qtJpOq{u>Z9sitjR5Q;bM`d8TZ>w`kFTu#gWb5 zIn}L~_q1gHI@DE~N|hRJmOAock$bT8W>tV>P)dZshLyCCA~TNa^}?}M>azC(L_bcuTa-K}uxSnfML2H4b<@hyoaLo9t;q*hajmy}eEEOm9M=VvEaKDn%NT6UIJ|Y9i#O*bw+h5 z-||4xD$+1QeQTk}$_N?+}hd;!l5O$qlDfj=D4elu^m|dlE9?i+f3UFMeTH@(}9UO zJ5|ejdWgMMm)5Aw=YXKs;N~QId#(lwiBVB|ZK;S7cCUiHn<|idf(E@dm&N0&mSa;_ zSM72Tb%uF$wX)PfX+wKmy_MF(<-~fgO-)Ji?=RsaM{F&5=J*~%$nVitOUZHz>Jy{4 zI+cf_d|G|c6EocVR49CTMw=9#k0dHTx*VCpL1^ea5U4=J`<|^EoKC*?y9io5g0}gO z+_HF55!oVqm6=H}1A#wJe;UOj((_G@lx>Qqw_&6tkei2>K4n5D#n zw~|ElEWY}W67`^9A=lmsQcsR5>lR9H_p7v^B zAY_l2i*{_d;((Im>fe$y52)&y^fuN0qwUpz>oAzf_TOC{21-4Y|FfnJ!JnHMN;DCce0pEw{wOR3 z?;ebO`7Ww893-op%UJ%Dacr~A%~moh23nV2zSENMP4=On2$Vf3J0|z%lH!xH7xX0? zzg{jhrb55`JL#4v_c{H)12F%G(&ay>^S^zO2t~)IH7~!%m_R~d0lG^6)GpX z@|U+aj4@p$C_~~xsGb<}~Tf%FS#0lFPyu5r&#-95R zW(PN{-Ezp?J)hmZzIUBZikhBSv^IIvUTVr;VK7Ps@5k=Pb)M>4hOV#d?CpxJ7lq)Y z1|lnuM0XqOVKtfMX$PrE=)EFpr~hMzYmo?5SW(+i}c;ShO{YK1d{cSszO! zc=e3cV7VL^eN?VCv#k$d#4@X^t6zrGG?98c1Q;ULm13OtT;&i>&iWi$&2`<`cjou$ z+n-K~A>Mk_hHo5DHTP8J5P(tVs@H!yNk&8yVe=O?ux|}=Xcy&nSPaVRhF&yvm8Z{Z zv5PiNktS_L_SlXWq(n@xqeQ$`5&FGB!_PR$NytkzRs=%n)v|4o+9Ywf~X&m<(;TRjaS^HjBP+byfx%VU^j zLu}uRSc4d^rog3u!3?s!r(v`+U*2*kUpw2e){a+%CbcP~PZP5Kk7rb?D_NuRl>S4j^4 zp)t=L2Nw|wtN14^Z(}^Il0Bq(4MxyntkbOG@Q3?sVJsFd4aksR@8kLG=Itv5VX7r3{-veXy_=-8koBGpq-VJ@7yU5 zXWzL?MmAYF8LkK6nSflEuuWYSNT$dm!#Q*^iQa+8^{?-h=H9T~1qWjGNKuQ@1t&l1 z$n|))jXB3IlomUvf!4UiB&Enl$RR{O{ey9yr1M9~ae zTU!Z>{b?m|g@Jl?kRYjXg zRIY1~*Md^lj->NeCo(y@-p>LZ`~H2>*ROY^rDfQ}F&Y+8N(p_D;Da$z)nWk zJN%R=m6W2Q;F^G51eBs5=`=v84v;Ok%sQ}g+b1nYgBI7Dckj~I*8Z$)!-3T7!3>B} zIypwh6t7MH(90*x-W)zw2)<3;K=uUFfU=`O{@GsU%8zLZMI8q%aOQ0hxaA1M%qwz) zvZ^YQSjpR7tFyM871L-xnpbo=uk2~Vd~7%q(BD9+%Mb<*nLIZ3h<(fL+39-aAZoLc zrD359!k@IpX4HwVlruD40jqk0^p@c9mlJcaHM`fZ`j;aH8GJ1iih1LnaYd^RJA_U$=W zn_MTuv}S&dUWF-6L`j8~t80!)P8G+XPg$+MEvm;Jxr$5*Is6I+i`O(zIgmwS6mHL} z>cxN1BfI#$@7%hEIK#KFI7~c6Qs!OPsfh_(9CMzXpqN-~A0AY>*L%7NAnjKu=0cDH z_(K!K$Z*}F&Ml^vy6%Ty{D{ur0>K84qq?soK}6qIi7+)yKP?4T5+ulEbX5_LX-C0X zE>X{1P7x80tzbI(q>I$lO!YUxImk{ftUD_yDXF*A>iL&1lAuTFj7$bYL(`w14uSw1 zOdlAs21(hdR@VBToBMJW@mSiIJe?z8q~yFT<~S3as2J1M=ZwK$i^y8aXn~ZkxkWhf zavCVmex2<|c&wIC&k`MZ>m ziF9oBZDV7J6g#k$vT<@MfufmtT#upq<^tAAUe;erQnGq=Er{s)0MA(%<=lxQ;OnF7 z+IHcPW?B)~M7G5}XKO^vkAjBdL;IbAi8!}!WkD8?)4Y!LZ{ z3*-Fvy39#5UAGHl+MMA&S~zJEK~VdZukV~6jV2ot)AwZ;6&01{_I5Ze49^#UtCO-y zFE_V|T}$J>3uSMwh5&zm_1m8dFs%Eb&tJOO!)C)vTo}_vu*e#))RbEd@HNs7d+mFM zD)9`JDkOTToV8npBLE)31`q7m>ma)W*-S70o6ei3!QuQK$n9C6jCd*o12i_ts#lxO z)j9&{x^io*nfbnnb-(sw8bA`(*DHWZ;(b|ka8n`_t{IKn!4Rd1)~4DbAbD3&34u6h z(m}|$Y=}`q%&5VcIn)?To|rbFZ(^`B_$VX_n#`zV+KLD+}Z2(2=vzw_Kww-TJ&XWfIRW~aNV9#m6LIX z4Q8Zpz_QkF7PfG(Mi=X`@!r&{`89TmD2{~UFbE9|P2OIcwm>OWF%Yfd-!tIb^SpDw zS!*6pxxUXY{Cd)qXz+_G4GeOS)7LUIQCRk7{BjZDwMMdP1U}E^TzLEX^~#Z4Frg75 z?AGe$KL|-~{?$7vlnLK1b|xhC>(^4^L~IDm@S-&WAt*k}MxyRUn`l`YB%eh2lv5l6 z2%;=yVBm(r>xkruiH+vwLACkx-)P(cpKuOY-xaOdby7Y{zNpjn>D56GdbR3rveJBd z1Z`E@H5;OlljSVm8!#_Ao|Nsw$K|K0u|R zIn@ao{p$ATWq8`%Tcz(Ns^nS9U;V@%`LG^ljYgp~^dXrd#dS#wri zFU40mRae&vko&md3SXd6jHiUq@U{nA7f5Qaf|?^&FQ#SP`vK;9M#qAqfT*{i7#rKy9OG=14VVp z-v>_H3f02vdB>FD=F18ei{)K(-z#C$q#V=<@J(;WhUl>B^U<0sbYATe0t?i_AIu_;YX z(gRCjJiSa{R3o|rv6GO{bQt;f=HlFulILC1(@3t_?xI z>Xh#fi3o>#3ma!a3yx1fxmf!KkSBaVk76MG=cjptj zrdz7tDCG&W0_}?zjX<_`CE(>mY_I?eh-yEHmp47XpHJW^hQ+{95{z1E-3=l^4MYKW z1t@reA}2leM=BsBv_}yM8ji_Qes&CSjlv*mhg|Pg(m{T8uq;ab!_qyFR#nt?FMNVV zvefLtJdGs9rMrKptpb8FL7HNB1{THmEQ*tpi@#ef{0qp4hKW~AF`xOOXGJ+5 z514TL3pPmoCz)pb=f$zlJj@yS>G|J<>Te!W{*yp=JiqeKNpF(>D_2(gR|0*L?Ek(O zHEiQgNuWnuci4%K4pHbHk7Vr?t)C~=ngg2n#1B&FNm1A$%*1x~w@AZD*ZD6~%?e;6d zuXs0V5}6FRFpAi01aaBnn%!o$){Z~IsmK5FO)s&9jU2p^v>+CPJf04NTWb?XK#Bz3 zOeYJ5@Eccn%|rKAe;G+j-!+GJ7qa~8P*pu{P!G5& zs0y2<=2D1$Yn(U`1R*F!MD5{s?dcZs21ng5TS$(~>jGNN(=1Tj(FsfEF82rkscG;7w2v3e>%_U{&FVZ7X59{F}5i z1>Wygb8C~zQ0WQWOj6K8&Hzsky9(06P4Hiuf4bX*$Nq48dQ-P|mK zBvLozQG`_?D2>i?hsHswFg~zIdrv7Y>oYk?A4Eis6OHE}aMe`&cIL#21?Bj=5C-h* zkPImGJq&pK?b|mNe*Uu0sIeTJ`DNck`XmN9zo07dWQNHJO%7%Th9389h+aTNkh3%X z+2PQ$h=GmyBMAQTKdlkPiZ6qiw4(|XMctBF7#Vf=(Jo11xUCED*ccM83It+vbgD`} zA53O|#L?{ADwzTk{sPYKN-8Kh-oc5;FO0>B&?j{?=IdqneikPt#bZM!W-+-y292D`Kmuln=_o$dMy z&@Z>m8d8UrRzea;wBwiA>%zNd*N+6x&aP27iqzjEM%zbDk(>22T0lK37kAS~7u+IXNs8@U;}pqrU#{3pZa45;a31lJ zfnYF_&ngn^sFG*9Q&|}qrR}GFn02Iochw;63vprP`@K1R`uIE~m`TX%&IG7oE3!bj z8at1&z|?v|FC=c%k7mwta|!%kWJk`SZO2J~b<&~^v`Egn$fS%s1^o8=9&Rc<4(IC(kR_c$15)@d{?ppw-lj`ae`3;QD&y9@IK40zeXc>BborX(pvw#L-SYV z)&1@DLu>xc^M!(EC1Bf-BQ9N8&T%8GH*WMq5b#_bjQbCF{KvDBi8=yT&tMmN)gZ=3 z*%RCpIU?Bp`TI?(sw@B6{Y`Gvf9d|a^YX(#otL0fwtu_>AO7i#{QvpI=Z6g>vZk=j z^X!nlg@IDzxu4JXN5&_d@p}K=%&w&&_QWX+JkA0|iW_SS{b%zZHn#h}Z({=l|UQe`!!`@7fE+6l%9s6E2mR zT_B$U9Qo6gi61){;^Ls3BIS3nW~=M1`4As%M@n=twP@FRZ!{3*l z9d@~fkh_QLM;un`c?uR{*a$yDB3Nvx!vqk|csz{T(c?8+O=il?cCFt2PC4gjtz-b& zmd`k#2o!yA|BU(P=ywl;D==t@hhB^~SLD<(6qIyw_kZFgww4_9big>m>@(_AWPaup z2oEHaitlRI;n6lmMFWQnKs8vP^~kYG;+N55-5x6nB?5WKXVo-xby+jGLWkdn5px(s z)x?Amko9PY`p(8CG(GRGaIY^m+km=1KYkw5J_{^@?jz*-@lcdCZ9=Ait zbo5ww8@AnBEDdU%Ao*{;OEun`B5y(^?*mtKPjAgMXTs5P`@gYl&zX zUy(&JDY(h9Pzjk1_di9l%2;}d5-b$r_9}%Oh4hF%;?#RB^tBX_I6+J6lwNKYL9r*p zo;iS0STlo4E$qF$Hn*X-6q{!D?tn$Ejkw$#IxWb{dnm_FXdEQm&*P^t?WmpY;N%qf zaPu!Ot9t6Z#o2WVi$5=e+^WRmTSVpN_*c56>y&`1kWhPsFf@(Y&Ljd}`uVwZEg!^- zZevZ%7bz~Krp3DH>Xlh{Cd-ksh}8zi|ES!(t88Dm%2-&SixdJ_7=Ntgh0s z(HMwD@n7oeu@mC4q39>wY6KTuqWSJpt^4z%q|^(-ifM{5z>ApX@%QgY3KFA(%*vavBhKnr<%t>!-V0^`!CaUvdX)TDt2uf#ZY2PolASx3 zz+oT+h}2y?4sl&>F1F7AU^8MSoJRzFLGWJHRV4N(HjV)-1!i6|@8!a55?%XI1T>Sk z0GtO8GM5lw*e&YJVGEyKEF0-fe09q48<|$R5~fyshz0^0XsJEF3h zu`*-IH=+lXZ_s5TeA~wup??Dat4X%-`Lep500xZ!+jreX?IKyQ<*+@r@&z_~q8EP0 zVm>?vncOb6b02eK=dSqTa0amU&47`HdosWeT+$MN*YARsexr36Y=ZZ^J=ayNMyjRR zv~ty#M?4+Ah0(8X=q<};R{K4`npPwx9txo1S3G=lUCT@^W3WWN2L!)|)8~r@ECB@T z`+1E@(C$tUVR0E!=gjI2x0pzHfii){5XfU}ly6jt-=BNc=VhOZQ6F~}Ml3->g3P{5f zh-tRWX&gGGaV-;dvPj(KvX%*f-N4QAd`Lu8+E=I70dC8%;!ylvpyOYi zED(l8)d5l&Os`l*4SX3Yf~&dN!>CjRk7$P3*UW?^J(B|M(6cnlr)aanG#0fqdGY1d za~8_V!O?Bna;xrXKJEYV2#{!j;GbCIu&MYmkF99~w3Y@%NI!@_mR_EI2mGFN_gd}h z)`0D601zc0>P4W9KyWEF)1{xL=O`7QX;MH%NM#BbjEw8VbR;f8?m*`&+EHv z;-Hb^ZgI7&c26L-*Bm>4SPt;gbb!cNA(aP9dlg)?gMpZ3Iu2Wkta90rreGaOl+tb4E5 z_m>M`gdn2v_dh;4H9<506mq$#H~u|P4ItvZn^UB2KcDSj!4S5$Mo){VcS6`hdGmPh zd60my;rpyuNdr;0TeP&dWdbL5Vv1ml3~%3l#=s&%H*AB4SqH{12TGLgbzx|{_0OFf ztz6B_x=4e$3!Ylo5=tQlf6QSsia~Eb(n<9VIm_sir<~aY1hmN!9tBXwNZ#7o63VNn z#m{vI@A>=tD_dKmm*HENQ2QB6)GlWa)q)x#lmUnyyI`xy#?9TG*9y}64v_5O=d|in zcQ)jR=7mfn&UrRQhEkrJi_S!n3k|`=56ZTO@O!0tVUcd(UDItj_Hj1hp7gMIK{c$7 zrJ)xZqd@#R2=|RMw>w=WiDxG9HYCc&hz( z<()n@1o+eTG$I}x5AGpvT$0N2(oTUwL_po+YIv&hAjibr<>Mowt?p{86I{kw60(w4^&BkeBSr`NP}df?*>@*cymUyq!f%>qktZj zR`A#c^>t?{3{9=N%!?{Rcv5uHSPe1Fq{WE$xkrTDu0|lwH$39$j_- zs8hSyBK5boD+$kbcp#Sr+^+}VT}sD5s9Gt+^5_6ygJ@c)RP*!?OotnFOZ2vy}+_G=d1hCdRQ( z$8ELAJ3YBvyQNMim$Pp79uccp+8TFR?5~wV_ArSVLaODqP|!ejeyjt`I*EQVz-8dA z6bCpM`pew5p%i2t1Dx$zeCg_JC!pLt&kpZ|g%l`4p%0l1q;B;nej80EC#R&HWv~Ns zn;4b32**ND$7X|h@65-vxK4mT=4#-%%W7}O%J`jOu@j2H0b74g6yl;!8C*u%)vKC+ zRi5dF?V|-mxqN9gJb9tRH8Wyzv(j8n<5Z@H>w=@S>qk1eM10%RT_+e`?$2>FyG3;L z^t`TJ6Hi2K?%YWZq|S-nqaq_qAr3Gmyf z(`lyyt3$9)(&Wc@6uE7RX>AVMmqA$@yI2-L*;-~c&IE5mTv)0d+C2Ng_k%6!WjCGA z0>nlfHS6bV*2h}@M3m2sE_-W;DshMy^jnrPQJt8@YW9>LBc+F&;?u`5C{Liaguh#i zetAWNrXsP!fO@5*SVKq}Ryj3jp{`<>08gt5umWw?3*d`NDF*1+b4QWv(xpgcs}^3% z{z=^Wd!-UkKbMFAR6x-=?8?>df{%ZTU4M8vOH`^S{`|Q6FYS2sznMY@?*Ahw|9|d9 zva2(PK<#n|z070VaX<$|XRBR803JkBSmjpYScw3WFKuhwW{KF|);NXL*!TsWhhJjM z!Fah@l)5#NSH8%Epas5qZsV<2U@y3yhhi(Mx&v5z=DT@MC#?PM*YSvffbt_i&tcEIEQB2=wdldN5>B8R$Dv*zI)~;6rGchAO`$pYkJhs3O z+bI)g+WAHLBq=G1yIm~FpsEm9tzw`&6;Nwc%#h){Y|pzFxI)42q+evnwtvnx{re4P z;-iP_i4H*Us1QvJemy9aq)PCzs&rUN1pE!UI*?Ffh~&L8*H;sThyp9~rkl@;O-JeE zRAK3kCA6>EMO0Wsa|340p;c>!B617iC)z(G_@KkCe*;wgE?yY#fjZdkAP@&K%hMan z^tF}K(;KhX@trtvqFF2-V}l0$X$CnMBdEIffNif)$p8^ETN&Aj#HsEd&_@VR)C0g( zAOeu<%iefB*s8^cM%eN&h2QcU*`$!6k&&5l@2Nw3pLBzAKd*ZqIS3+Aa6idr$dC!* zQHb1)zl-4m<^>sVAR1PvyQsM6VM84qcgh-?$fI}~?RD?&RW~-H%K?Mgis z!F3Yg9LK1aZ$E(F8h&5OKfl36{dEbCakEuwJOJU}rnqe#v_wPN5P`FTOcj|7Vo+J` zn>-K+LwK2`6gM*XVU|YI8bs%13}hVfBz5aK^7k#-Y}o0kK)gQl&B;6{Pu(x4+aKn5 zrn{0yJL2CS&44x?m=9eVS_pc%X6-6Xs2Fw82Z<%FjDIIqrk87Wa`(1}i&ouI)&=cL zoR?!M8el!Ho6W6S0w-`(2Gq7;1@2|ztfPaF8Dc^OZ_i55RY(S7_TO4V0gv}jdEm*l zm#5>2IzgbIR$@&lZe6{MjTW4Z^+M2)>UzsNFVgLD0JOo3K`7m2;!LNbl?5|N80 ziXXa$uacc1W4?r{WaksmeX2RlLRFVut*x^ZIjXEV>t)3S^pj*iK!O|j{@v>7K! z?Eb1u=JtDSKt(a3tR>SLngfIhjv#6VIU)VBLm!=F0w4=wEH#-RZ=NUp9N*Y1=*5-Gv+Lh zY7`ycDxPq-l9n=f*C(b?Wi+^RM9(JQzW;rmey^A+X=eAJQjJIPK zDb!uN1hUkgYNp{{rfd&da*T9(9MLeF3%=E2ul;ScN4t(?qJPs93WH-~sRR$3*Kf1! zhem&yPJS)k9(c?^)2M2yBc~Q_EcRZU53WwY#td4jMQ2|4 zlE)r=wY24(em5%Fch7uANX{=hAVS*^m-@rviOMeh`8pB^B3<4VN`@%#%?5 zS@l}v&t{1+2a~b^0QX27uB2w34g($FxXJ&Axwnpsat+r;?LuX#gdm{^NC*fBN@D=h z&Co2RySpr42@(<_-5o=Bs7RM|451PO3?bb!-~BFK`|R`E`G6UKEkdr9#8dRFkIw0sm3G9 zyRmhu6Wb|1f3Y#CSx9Q6Y;)s;?{t3O={AXM$UlOzx6`yIlt11a_s(QDet4#jiqg$H zLwP8Pq6-oLUqSHkv%F^mL7wWFP9DSI5lXFD)fhW_;lY7zw8^=rGo)Ugu0Mo0&FX2z zd3=k?4f0m&XiX@1*L#rsqYdTF*RK{vFYX+U{x+j;{@?;kUGiPqACC`zHn8x;6*`s$*fz}pD|84@f*_?d z)J!u7HIZa7B64OyLG5nA{kKj_ZM{HS0u{=;du44g0b9$S`Aw6Yx)9Qtui<_tQ_Ymu zPkEd0LWgef+=V?4H($D88o%6+h(zs&kuYRIHi+x(bvDh4C!t7c9ZOiR88cj_v`k4( zp0=AwP#XVrgQEJ7hb8iyEg#1%d-b` z3l9vNub8f?a48=Tk7^jB5K~TPI@IpXw{kg!i z7efZ!hx`1aYuwxrZK;5QL1O?}4y1VNd7%r>dLE|IKBMU^MNYR!OjgJr5`j~e!y#E; zzutk;%{}SE-_vlzfC!LB43Zc99--x2RiTIxnhKQzGbn_4S(aO3@>B9ts_?#gdxHQz zh$w&mE(p+xs$xFn6Yes<3SmcN-^>rQDFASl#&zRTAniR}C_cM+?l}mpFLIwd$Av&1 zI^%_^n}mafw(gBP4#iROEbyzUsF(nK1j+9h0K@>wzw>DMNcxEP;dApD@u5RF%jNc8 z*Z)X?w6JY+-lEAI;jlce`Ior$7aCo&QyXIda7Q|6@u2caj-)UdhXI#H=^0tGn1+)mDlYMI-G; zi$)egUP)?yJ&}4bAX!HL2Mvd`+vi@ zHDXgwRK4!&8nR<)zTEq5iyTGuL*)E@hQPy)xa=z;#pa~>Z0}o)1iY_%?RzAb4xXUC zP;|P)R}>Vhpb&b|fg-Q1tVN}iZ5aL2KYcgLXZw1=9O0vxKc1uD^_-~7vn#3&c8i9M zQtZNie5I5doIh~Bo?KJX7<|2CWYM3gbx2|6R250d_0t9em!)GwBYSRl;5Dw7B~QQI z+jndYu12N_QJ>$C_#H3V%yJ66p8oVpL+i-@Ovv;<5r+M%Ykqpc{!IqZQpreO7781P zY)t>?Y<3R17pehuLfV@vw-7L7>-^iiQrWkC zSus{siq3BdDG)5_dd#~br2=T0=CuSRng*!!%Oio2kDnhd$~d7XUq=IY+6f?J32}#J zXn*x~C8c+>3seNUHgLarGW6;Cr&YYX>VQlxujgHx*u|^E@2qGvi-ajq|2OZ9KJSQ| z=tRqAbj2B5eV|!l^Nv0X8g+FOu&$NuV?06Rq@)ayto4%U-xth;SX`?D%DklHq`U8l zh|ZCcQp^l7{x+_8m=e=4dSBO5@@F}SFq2MQ>r9UlrqQ(@%qny$T~^u$~R<8#-2>9-kON6ct@ zw=F@sj4LMc>m|rZ~r@S&woBb`j6fT)#Hy*27t=p@z}W& z;RE{`u%AucAJocI zLpUV>C&`F{K8^pLmt|5%^b6eZks|PqB8tZQ`WwUg!RF&tj?H=Y3jj>&r=B7(K|YvV z&rKJp<3On;4_91NbP%KgZp8PS;yXSe$4_4jfsz1Q^}o%@#I~Y-5wWV)2K(H;rXVXm-|*`@ae5UEnd_{JhS=#KNKkAf-B9pOZfR{%=EXqzrD}0)Psl zL3Gz*$3`}RSN81LRRr18M%qrA5W*z+Z}Ku7^}luF63^W*;08nk^JtxA3{?J*`reSX zeITG!%}dtXv~c*+ymHMv&5A$YfyET7g(`KG$MBubn0u&CpLq4IL2eq7B2uAOs$lq4nJka1|{LRJYBUX)xOSDjM z)AmNt>!g4=v2QrFiJO#%CWt*Xu6pg44f-{k|HZ~Q^y$(6G7btv4_l0W0Pt!d90est z5ZV|i&Vag3&cBmEsm$uybd&_V5oF}ew)1t`+B@@#b>Ik6Rs5c3Yc2KbiXU&wG+QzL zyUv;xwApoA8F$|7#>Phsv8HuL%YO$4FM+s7j1MO?>I;E-ECspeNx2lqB zWG-F8D@O-UPH3_T-{3zRBQS&f zH0cXA_f@pD$8xbHl@}OV(|`Wi6p`fkTm1Fg-NM>j6`!qhK4NN_%60|(zLcSwm`5sF z%C%OK502hPOPsc2DRI~H5%lp*7J496HhN6q^jDkLAtt0nZDV#hjt7j3kHtQbu0`)y zD_!?m_tghaNB}Q%hIOX=Q)j19ar+Vy4;tT{c=a)va6AA{EC~Ms@34)7RZSrkW%(?v z-|5o=+oOmi_3JM=?`D)0l9dJu;R*FthRYjFD+M=VFd+k&=+9$e4cm!>K7U;HqT8#C zS<;RY5Gu`g9%d!13hG9u4sK!!rlJmmy%XUUr2bC+op`U6x4pkU|MHi2245Zw|Jw2X z;~*{VlyugM{A9jWp-f3j-tC{SL(Y6CQ4#j~HEG7jX7heWQU9Bom7T^NHb`2TB#chxQ@5o`VMQ;%+T?|&>$83aABZV zVcM9U{kotEFUbwof`Xqm&=o;cNO0{Ez$a@73C>VIbpxd4j9DZwAN*)K@Pjy`POGmi zeiCM1eqJ)X#P#drEI0f^NY#O$0MtDzaL*)r-?L%*3ltj!#ad#awi*!|JH57jG(X7fv%0zS*`>JJ63iNn@79QtNpsW zaAOIfDrf(YNQ&x3*LZ0;xxueV4GfZj*Yq-s8Q%^)BEA^WY1H25H157HM04{<#j|pZ z!Br1q%I8I&_|AxG{GdAV%}(M{(J^T$DK!O~kSSaEBZ%NAIB5yhgS6+*MWJcD7*tJM zbwzBxiN0z6m0o(@7D(t+dI7*DR3N2J29%C=_x7>@3_e)el*uUy&e9MZ$(LB*w9nuz>%Z=t^H$|UjGvppsAeL&Ue>)UL?q1&yx-7&ud z{cmS|&xh%3)g)YSsXTlBJhbD4B1F~CK1cvo?Wk9J)&G1#PQ$c6RPSAGXvlKELMmT> z+JpMnmp-R&__Fj*{Z268?-uLGZ-SHmrIh#YFSfRv+~cCV%Ed+(HY;|Gc&t3)ngTt! zx^XEI!BC&jNh{QEuB=^xL|T7}gtMV0U~fLmb3>=+hux0J$5^{|CV_@%HklMB!&nSN z%zSj+0fb{Y&K!W+k@vm9rct8;#f=RY5f&Etw<5cG_r>ClH{7{vF|1H(GV+kI;j4cH ztHyD$Z*NwOFObODc0I};EjTrXMgCiXzCX2M7TZery!wR-eu#>#&(NOXH&3(?$eDGml?=lT)QH(L#7Z*tmp%Rd)NMWu zg^;j<8@>3Lx^~J<`_zCE*ME~ZI5aXc62Jb*8z@|-;`qW61(vU7`bh}W0nKNdu$~XZ zN;KG=-Z>?oA8%Gg3J6Db_0BX#y}D*?k;0l^^YEr`aQKJ?ux>pQRddu>`F5$yKgz~m zCEWB2E;w~A>AwB9Y*}@e`+xt?`;GVM%AAU+6^q}GHN7op$J#ZioNp(@z!Slva3=o4 ztBQ*IgU@69x$*gBGoFba-<>R`Lji2 z^aA7S;=n_BH0Vq zcy5;ptJ23`^?q}}@%Bf@C*KRW&9-)Qn^k%Hq=;w{#!yn$@B+!pTTOb;E|7@Z&VE$A zHu!vBkQ0B|{G(1IwYTw<44ypb*6j6q>^rTeg!nc-e7;>AkHY=<564j$I`ogX^*NRr z$7hI84?kN%tBqj9!`$gx4Z>ce70!*+^~m~Y+0c+Z*P0Qpq8_1OIZ>nv)vtIiZf+rp zWy6d_WWb+K?dIhBLmQ$Bq|66=E0)e@wl(jgL$kkzeC;JV;8N^gjcg%={#fRl>%5nd zse9-aeQCW?E+GiDKFg+loFOrDilgw9l)r%ew_I5hlQZUtCrBI*7zRZpZHy1%0=akf z?ELvpM@2>DzPS9&;=FxOeUA1$gMw31PZY5q2g-6^d}cN|(D{x`hSZaaG(bRPqw4Ft zc4s2ByD>{n426Q$b>V}`Plf&*@H1U(D_+`E8k zh8*>v=vmld#iyj;4d1dam?!SQf`nVMhYgDiHrKB^wKqPmgKTF5661^N#~V?xoCX)d zIgtAH9!qQ%O)AU2FlLe7Epow28!{hOLb5p@2R8+Sm&3aSc1knho zQ9jk<9ixCoBX^fuZ9(Ls5+D#W!%mZRwPyWu{6GP09~weIWYo8;6-R#W$FT( zYZSD7sn+`?laz8@ce6%Z+w0I_CXzif%tvZ)ukms^5+lMj-c5 zw~I%<0_$vJy;~&mS$HHmc4{(7|1^LB#o^#pLr;uEZd+rjRDoepShu@pUd!X?zvyK8 z{iPVc)1nE5gqN$0^Fgb3cPrm%Kzj^r4&56(O zgUE5UL0cE?KmQE*yGK#4PXgq#zcx0vZSMO>RQ(hksmC8yv#~0F`lM<{eG`lObrgYU zIipqc?-28w2FX8|co!0}f8=p!#4Z8m7I{6FQ2YmR{!Q`!hqoq%59DengDgNkqnt-t z?jz=+_M~65gTxJm*N@qq$2dh_ym(LyT^2doSmstQ=y=JHWz-oBX-Y!)=xSRSU@qnkwvfZIx+lR@2B?tEbIPcivuZ zC-f?5;&Sh<->w%Xb@uWPy@HH2tF9sS5j@$|S@g$j{Ka~~K-cET3-77`;HGc5kZ>!2K^CeZu<-OU;M@hd zt#-G~0ZHH`*)NuB6dOUsL`2r}&ct(71Q3K{OzwM9I zc1~KZ=?4R7J-uLIe$7eGZ9O*4l3`PVR;gLmr&p)vSDTxqLILD1#-_EE14*Fi`^#F^ zvQB(@yQ^rYv;%bJvKnUk(P@Ly)LFOXUg20lSC;PK?FGkxQ^a@>Xo>-xWHxYJgF$@S zErailVH?hM@4H!0oTYW~h<4Wt0kacPD%_dhl#_IN>$sz&T07#7179fm?mX$Z=eFl= z)76D_bgtWU@Q91wr!WU+CWN}5#WgqwMh*Cl9Nkx_cP7j`-q9mZIj z4MZ`s@1Mo8bq}RLZfu|O&NsP0%1xX2vaZ3mIokGE4=lTus48E)BJ4SPKs>^OV#{~xFc+PZ;fImXYN2uRlb}w(@7}FNJH1YIgaJ-Ix`~7@}{awK+ z>@vT7L_$~7XR}g%Pki7##u;}jr%hp<5Bl2t#GIVmvGJ$J`9vz+XxYu=D)#XZLqBix zqch;^#kO8tJNwfqXDLaxxpXhUvNGQMYhoQjj|N;O>&=^wR%*Ca*|e+>Xe^Sr{@x(b z215lhG)l1H=q2HVfb3*=SDe5tNf4`8*@WsPZezaBHHdFiTwxd8-w{T287z(q!4GJVDZq}dq-LB@`+LtTxvjTegV;ft|YZuSkVkTwAV>q9ui-Bug1Ab{I5TE6nUkE)`CHor;EF8&ep zIF!SF)R`2MU>Y8(rBeTC938I_A8vikm{aq1mfX5VTU>emo)_Mu3R$ujC>_ePRn884 zd4mShCQ=sUr&CHx7 zhtVbh93%|wLkNbDRSh1HG@z91M^^wY%3lUu^pg7O*wqUZ%Ex+@iOAWN{4P=hJp*k9 zGuv0@)5_!AM}DkwQb3G z3FE8UON!_2RIiH6(I0wQ#Rv<;l~>Gn@rT7?v0F<5u|bV6x^cn@la(U_`x_(0#zc;9 zrLms|+Ykc;O9w6~95I&1ZY%Aw! zmQp1U&8)1TfncyOTsQ^ia7DWarLN0&zJV-diRK_`P@oW5K-c4yK<i!N-uylNmp5?W+^iHMlE4_eU0m06Ef6v97N z?oR-Yo53I2WhShykvJ2DgicdW8x?Xj4?{pB z4sGfbm6V(ctp;=Xs`i&Z{az0$UiOO?`{wysQw|r$W~`1Xg+3fJ!Lf)N@7|uF5kPtB zA(CLVwHrHA!KOe&q2!+Bga?Ow0iWl@QeCyi^A0Oq%?fCXr6JZ4kmom0itP~)Ao7g@ zjYbBvUn9!P%lklDadOD^q8s%1jN5V?ui6C28Yl1(RbY6K{el}V)VB(h_Ntui+U>Hm zBD=nBj*}Dg(Q)7HM0xGEt!QfvA2|C}pKMty>$0YHGLEAba2hl9+S@;UpPGn+!#bne z8zhY?gW}d}9h!qgV(a9IuiCgL1=-FER=9CgsXU$VlaCjSFvld%A|)W3OJXf{)6)rO z&!MT6Y#vqXWDqBGt~+eej(AV9yRHUPNorOCElRHQ@Wnnk`oLx;EfSBPoq9Q{dq9mmI3}1cm7d>$Ry)oQK^F9;-5pAIPRTn|}FAB0My7VZ+tg zQeGA%B$DuQDpE&)jY3Y3=-j#f8rStrM1K_^DUooG0^YLe^+YjUdqU33HQK>(n>^gy z${>TF2t*v!G$~R<*TGQB*xyc=c@aA?BEJ%u{kW=6sVPc?sB|TEHT=tHbte6`>W2g8 zS<%r;VtZrllkBa? z#uA-*e5r^^V`JI(nPt5rj#p@V>_q2>RUgDa+Wue#qf7u5V@m*RR&Q($P0mfXDuckS z35XDN%?m^LH-p8Zrmq5dD&oD7qPVy(um?23JMP*hc0w(cTN55RLxY)g`Gh1O!skFl z#t#^dq`e?uIBL@g<-TAXkY)r~qngk2|BEJu{k)0m+(D)jUm}I6ixED18l#mf}jxhn9~G zNrQ%n&X(h+mp({Qx$bbQ{$v@OTs~=p>j;jaPTttRXv?mRkGF9gCAU0V&#H`Xb9Z;M zoxZoywPp`1N3NN!p3}||Id+%K^p?9G*>=6!t+jS%xdoG?z^C}S<}DRCxrk`3A1z)& zSJ#LQO!T#TpbqB=2ziiPmxaj46dnnp3)gJjTJd<#+qyL>&`Urikb2)BpW}h;5O{9D zM?`_-m@~iK^j`>p=i9eG42}5fA2z7&1LaLegdm9GL z0u2SEYxkE0)>c=0pg$1^H4vMD?QFNS+e#_0Uo^Ye7A8VC=rcw3(Ig73n?lAG&B|yL zeQKU(np9;{p(s0jCfG*NnIOnJDYiD=Th?)ZVk2cDo(+(60Q zRE*k9HfiyONj}0QqkZvBK^7dD9@Rf;dPj{l0kPv*M98&+(9_ zMeXt4DeVvb-y`pFBUMq+0*KH0hJKg9p708w9+E}18mPHdrF~@K_;pzY4MRf>M|_P> z_v_m%)#oLh($eF@3?je$Gshp(I&|wuZUi)kv>t5#q4ZBYZdV+D`G@50a`pd!*!|zk z!2d(?ARMJ}`p8h--Wrh@*!>?d6xYhmAbLKhm(;iv%%< z1o*uUKFoG&B@#NL8FB_Reg8(9LP)NoeOBY>FydS6y~eVRM1I^DH8Kh#_`O!0)Gpjyn$!o<2KP-ocrk4#4wSGOLj z5rezh1>V^brd|)yx}gaP32>q^7?N*Q0pVbHSRJTaGb8yD+ccC8L)XOy^YfH5rQ2T2r` z3kTg@q0mKwJV&ilv#PK&5X8Tsli);6rN5BfW9LbGw0<=ZnA#u(4u_<(xdj9B?OcCE zx`)>i(=ypD?~UKhNS6&pq+38+M!VTVXdN=I-|(j$ptH(s^@YFJkSM6a49~=Z)B66l zM)LFLM_|wPta`Mo<$-!(z7G+3TldhtAzp~mhKjpo6-=J|1*MG}B6@mIVwhU!O|#E; zg7U`DbBr~h6si%1APAx=_N9BKLp!%e80SR=2ulH)Ra zRDzCrG!+Cdp{==;$}P`M9>;#@zd-#aQ1$c32{>FHhxXTb3P{Aj1W18ssg_zMXQlcV zX<1pv>iQIMk>onhy|!nC2w~i8D8JUq!d^cq2)A6wShtsrlneu}-CM}e^XlONmEqQ~ zNd*`sQf^P*GK&!wNliS?&U3x<-bt%p5G#cqTLy!Z1xA}@LV|XwZX0vUOtYJF6S5%9 zhrrQ}6ALD8tYcwqvH20xrk9jB1YYG-0>(P@HPTK3qOK+o-GV`Jb)azaZh#o^1&Zv~ z+{P-kdl7s1{S{Ie$?u?R3ea3cg>-SGxF56jc@9*sI=QyuS zK+yaSb~tckbj$ay(U7HUxq;{=wrf7g&lg$@NPhbC324A*5;C>MC083PBK}lWCwXOQ zq*mg}ExAw-EU1<*C$7+O#0?B|&>FGSn0<)8XPsV(8#4WLaPp*;J*ZWd>m%{Rkt0mO zQh8@avXpO83wx*o(kV0*)D=~Q+%{}#tV%Uu4RIpT`BJiqD*#9|OD&S}FlEf#+}ro= zL&{Jp^qKzD*o2;&Th*YoI@-w+gRERoFM$B^7PJMC&5&LzIvMbV5l`ngZ2Qwt{-ch5mQQhT+i52T20S-b#C{13O=qg4h%1CSHJp z-;$4qso%Wow!wrTH`^mQoJoS?nA?8WHc2J3NN<3I?`^0^F)ODtLGX)c83N{@#)>y) zpu$!kR<2ywE%`w(Hd3r5m$$oP@JYOuR-nead3W89mdd9k2VJLGK1s$Vxo4sIHs!k7 zA9{Ks(5duE9KVxNWWJ6g^yz}EsllZG5;ii&Qr+bPeU@v$EG$OC><9`#j$pgSmIDqy z%Y%Fm1HDQpNy60n4TgD{hg(yu=o+p`PuPCy$cQX_t&7XAp{ri%HIycL`xN#NXfPpy zk@-~%ER7u>lAszaZHKN`WhbG$a`_68pqkeh-yq;w5%#XomUl2{AsE;_l#FBKIPovZ!F1}T||J%Q=^_)#>FT0l)XU8V=pRi&_Y$3D5ZLK{du@UaiU z2#5Ec%ll$EUZ#Uox&eJU&S{fam$3WFY#=)gV;%4wW~Izxe8K6cQXBi%e64CZ5Ekg` z9Cv5B!Kny}V48Ivl@Q(Am&(eP0#VLlGJmD6U5iOtia%zh@O*>KuI5^$fuvFdOC_2n zSF?1%#}hl50k#=8R%yTRHR%gqI6omQU>FGI8YL1fZ z1s8j5+!->cYWeC}KqO_QqkGu?p3H;PKn}W5enq6rA#xjc+P4fH!>xRO{IR_2YJ#q)efi&>VRUbY}CS%hD zMW$!;l8U9p){W(w#q^RBvx|h&-@5+UNg8kT&QefOxOl{FJY7do=l0_Xo*lg)On89O^A6LV&pr#2Xmp2BF&Vr1T!}1Xs)PwYh%V6b z%0OfW9Yjy5p3Q^r*P08a_MYo$reR%MRfYvT5UFPh$}k841j)Pem<=m{69sx~FD_CG zhUb`IJAN=o_gBkXOZpIi3>3{PBz|w)A#Ax0$9}lB&H>9}afH58FGR=*@n*_ahGF4E z&sK8Tf$AkJuv+1rZ=Tfr9*^_1N<)K#gSoi4s17;i)FBtA2t$7A!d_&yuOCk4Glk>Q zpemeAxIuIzJ>-T|LLItmx3d8?=!PA4CK~%KAxy`*N;jP<8AfvTs`{VRP@ibVio^@L z7Ns^Yk=v2cZPVUY$OQukTAYuXgJ~ntXf&_Y*j&;0ke+8=@f%~FKffJ8^wF4^nWOJ{ zzYsp0?t4ib+P%~Q{huQB7PQU$PmOt1^E)r+a<3Yba0`j+dF&e2VP+Oz4VVoic;mAm zFIH75D(lsiMzH6CWP+)Vo(-2%$iS0xr0;w#|@vFIMMng>?<=sv3Z2gR&wXLlls8Vk9 z_|f9vgJJY+I#n_-CP)-)%x+T*}uKn7*1nb6x`p25P$ zSDE|h7uiOXnYoW{8crf;p+Is7ij`cOLn+vRCKjPW`NaY8!>`yTh6LA}kV}DI|?76=8}EKtdV{ zp5eBlzaCn2_vuyN^TqeXT9Ua8BG0!@iacQq6?(N5r|(I&Pti( z_>ci{NrV|9OAtYU+}87R7T{gD?WRR;c41aP(N9tSt>;6; zZO}S|9tyHj4Zg&RfTU5ot(PQ6m`km!@92AZ1XHi+vBYwit5Wakk&_Y|Sf*2~jNEYj8g9CNzL0w+^lX2~<1 zbb1E6l7Rph_cpH9;+xE&7so=?%EUgsBz(VWz1QJgm#;dISM4gTZG6@P)#p;P88f){ z{cgNgnMIZ#jhjO0O>^ANYW>DYMuI6MW6g#Ok|w;2v!TkK26@&TsKh{&9WjX3$^bQQ zmizRN;Ss+=!4|r9tG#QQ{lN(iaP~E;iU+#76PiUXg^*=!F)FTnD;v5nn3HpE#elj1)eq?^EsqI)>-$n#hA&gfJMPzJ{2=zzmn{#K*_d`WC z34%i;;vXbMserT_sjSNuMOv!*#$b%HVKIjE@@1uOg=-jNsEusyTnRQ`8g+ED8w&$Y z=~p5SK@QmSJt@r)=cvIB_oJa+H>tCX1Kot6bot<(P=S-}$J8?H*4x31TVmp%F z(7=ERRvVl`CWqlGCt}X z>Rp)gNDv?lGdJ9sUNRhniXDPQLljv<9Tw*}4ULQzJoa4SYT5`3qP?uk1Dd?TLWn^6 zhyk)K3-EnF1lWddF=r4Y4Ysd4$EcGsn$JGHZ;6|WYjedJpVz9)$O`J18B_8Cy3+FU zecbXQDK{maA-ev+-OUyYn9bHGPEY9sZg7dwCMC77!;^7H*&vE{JUK%q;IvS>G>ZZe zK~f2p88p5m13}C~7{z)k5SvDnyl_yYqtJ@uN3p_~B1dEpm2k49Ryp&Gw*Rr?#~fsF z=Gskb>+&Mx_q$=y#eZ9+GY}Ph4smBbNS8Cw(S2SGE`d}|V{^0l`fMlAv;uCE1!N$( zq>9DcAOu%`a8O+3gv*YxuG26d$_=Aet)_bzde^o$C#Z}uOcJGQ>k}H*Eg1?W55-LV zney%gVtj6SMqLWVs$TXmWuFxm9=IxLfSZao7J;bDp*oL##L)S`YoT8fD0ZxXvcM0o z`i+)aX55vo0#jS!)TtR+NLU4O?4e=)a_tHYC?O67VU?BHs>-Z%0$m;6f~U^NfnM~c ziRqjb+=nE18{V{3Dzu3G;BHJO121o=!)ylysQJuX(;E0IJ2Z8tZ{*|vECt{W-3o=E zReOaC4$J37KMMaI4Ma1SM#zon`gKvIE$r&zcr^OfUuU!js+*7Hchnfr{uI2m9pe<%*Y=4oVw4@SPLWkwC zO6`4nlEK(9%D72ksGa7+0gT|G!C0~K7|i_v72x~hm8QkLeZ6)|8FwrhTE9+~D@Z(6 z>FcS0)-r%xh2Fp>n`_(luB=XSeHqP}vY0YT!8yGkERsV2|7R%6J!*5VXDh*^`l9COKK%l75)e-t83+*<+$)=ztxE2)x z$q%=MtPB!rYBtGeZsQ=xCS^#_xITt@bCs_5p(CXHZ&E^k>Xe^Xsj2fKb=_JO3&4y$ z-|vc-?S;CaW1lQR!N4FDBqBk{%LUXV<6ySR5IcCQNf@Mwvk}}ZQki=;eWuiETngZ& zwu{)ERefMCEx1a{mzrY4L3iG!=KZ8FI?p{%*Zq(O+G^*WJ99?CNL)OBM@i|nKg7nc z`tv`QMIy1C+J1Th`yeQ`)&xPiJf=^t0;T-fh0 z0l~di+z}>56HIFueUyxEpJ#k|^fQ-CJ8=~$RPkl<{dz@__DVE%uo(hKJ?YIyi!T25bR=VXotP?;~tOtNh>2q2sV1Xuk z22gr=l3}{`+9Nylw8pK?&CCiZ)J&Ni^z=`Fn>z*If^NFfB~12>RSf>&@cg57H0>nFRX4 z&|63+a|v49WWtgB1fCml5~S$U1CAYkWpGt|>mHr%jw$@TBm^zx74j#k@8PF+PBq3( zrkng-x{AIF5&znlG2D?!tQU@^ z+QTjt+1UG%IVjTQ$u5{lQs1X6Bdj;vH|D-q9P8CM$^{kydM7A=d~>Ysl8bG1W+qc} z-Wa^|V;UVDU4OZr9^@-@rO7y$afoS9cz${RS#*HjWBo#Pq)*kXs}Rh#ywIjfK1bH> z7RGv=gth)~ORLnR=OG-ALtxQ&1})%&(>)cdj&46cod%GK66kN%!|KnbED^P|2DHiD))>Aq#0mI zW=y{HRO}jJ$1g7y-Y{Y;r{-<{!Iq8*C2oeWU^Q-%bRp`8($w0|J|;5 zszerk1{*?(1#v==mtpFcx(`h{Q_aCB7*cvkNc}o#{8Iz6ab*H?{7W!m-0T{3e*z4* zJgftD49RTUZ))C=8PuEcAkbelh4dTA4qp)b9u6((x|7hU<_!F74q{IRQkq}i_nb@V z8S0_cNRb%8?-Zay4UL{GB9|b)v$WA}Vh8Y=6#azAkAZIUuGh^PMkQA34~*q0+1%Pm-IRP>(hyE!{KwFyABTq zh`>Y8ZM%***zz%83MHO$3JXund!4#Knfm9Y8{tQ*aVYKj z*XMGkZ$$_)quaW_gxpL^dPdB}r=%WX|B=NwUZqWrT%)o3-MPsZcIboYCzWSU$c1~v z-WHR?*#NcIbGfc+qJ433pJWSfv$58x@*$Jt@yz*nLn(SHLd0OYzZNgplG zwL>Z8Z?=c8kGvq za|QpCW1;Z;@dI|hW2v1+tQCq*7rFpd9N6UZ#~*(ru1+ez{6x0pneuSY&Qs${qBWYI7k zMK>)5b5L2A*#sFn$V-Qkkdj8=cA*I6zCmW;9}+Kqw8$>$h=yx;!kZfsQjZ#G4D3=1 zSg_Aky9Bes$?NdKHCyud-@F`gxdvS&2x?)Is;Sr6i5d~-7gg*GMKak26o?g9nvyG!b8LY{k2WFD(3=CEwf%s3Xw9^ud~~vDiJ`iXG74a;}!vaOah3U9HsJa{Hm| zx=G7i&7iSW;h9FM`PA-9rjC5wmAX(NX5FnRZ~pjg*Ddp?d{a8iASumXKkXDQk|%qI zc--oGhuW@=D8)K6@54k96&@N#qG0eMrAT?Urx2G_olHNRH5d1*u` zj0%@?5G!GK;sn2yBWUlxj|F7`M zb!tOuUs#B|un>vtam$r7*HC)5luI=YFSj>R-B15y0h?bl==PNSndS^|G~%Ry{1c;z#8CQ@U*3YDhwt zt(we=HbTXA{Y$lMw|20sBW1gXm>1TqoAq74e0fWiVPz4XUpsfnKQushsPvs;>3b#J zt4vyUFKaNHq4c{G+aAnu3HH@vVD`VO4jv7!I=A;OWNY1&Yy7Tg8MS$8nBO{Y=Fr1g zt>s8MvsSO!*B0v;()+xU?=DR9tjQYY#ottJP*ZxMI^vxB&U5=zl$!|s$=5A)?yrix zTqJziq_fqu(!FjP`deJr`XM<&))lgKz%I!Z?fgEgy*~Au&4_$c8}IfNW~Xi+{hGAg zPHDrcty3eN6Ghg#$+;&J+6yoS{nk;eWx1|>X`a`Lr}8yWvvmyy{h2oz8d=T1V~bWa z6Am>?pG55khbvaUSTruyFI>Urk%(U#fmtRi>uWZs`p#&0xRw>}svBxx=2^U$5!}|Y z8@y`vZq46IU3|&6C_yMG_atZ03SL%R<+!WJL0N8&QEUHCaf&2WLrk4!nGViynOjBt z&%Z3H2iy7W&9+?A-KeAX1H-eNbVkB{7;5oImrc$RY7;D34ePc}X^?k>WQmbSea>N> zDupc7tned4t4VnZ^aO)O==gFCo_^4aAEHi@I!l!6wnn;nSdz{{~xP2CPaCOAVGUU>LXkE&L;%u7+*S$S~Tg6WK zVW`O@u49i3F{Qc`1W#$0Eb&18wFN@K&0Be_cx;R0q~gl`&erv?Ru>*Y?jMOl8dq=)H&g)lFbJT%=>na1`z|Dd#Eyy z02TOg!SGAXb+a$`k z{by@kal=M0F+bt+kw{C{(Dv_#O-_56@SIs43Bs*~7v1=EPEV8v*>$+j)|iqHK6tbn zh%e_t;a5KM|D`ZkrrI`IDK~NG+j{1FCwn+yTeg}1XAlg@=q7vaE82+Cr}6qCgO;oY zb!AapSoz3GLlnhR9tW@9@lbknUBk>kQhl!;J~F_uKrJ-7Nwzbv8v>Ak+ z4>Krl?=Q+R;EMb_5Fhy4u44?dzV6l@m(P&GXpXaFXwCkGNatn~4c(fCTJQZ6 zK5hF6#|;GBHF;@icY|G5Lhqw=|FZw$bAQOFIZ9t(rDvsO8kZo%j_wMb8PS><$lmwi zQB(!f<>3$T`O=zMm;iZNK;?b$*-YOehIoahvF`9v z`&t^2IN=4$H7uZS%^*B+b$+Y1Ky-Y(ODv)5xS_x`j+kKdzU}OAmvKkE z;o4?o<@;$K+cKqFkK1!pop(1X1SHsXD<0jw)x0~hcwZz&%Q5UJd*Hgg70Gq`oD$SS zO43|PQ%sG_)|W7@NnWWfY3u!qSuE>9-ECf6Y8nZGHyLiV2)fiPPIynAvU<#`F!-6n z08^-${$9s$;V`6VRyNY|P%tX`P&vJ6$zXdcV)s*|;m+x{z42?kg@;_kUYh6f7{&TS zO+OM^>&#yj3A<|+8MSzHuNPfoY)(ct@1a@pqWNsgK`_S9Y+^TbIR1U;m1)Nzm)2&K z6XCk6&_+?Xq3WP_MvIq2Ld~gQFyd0ebtCBAlPtLU$S*IIM2{A66)veWG+&p#W}mEsO~k3U?&6^N?t3Z`U~?$7vZEX{edC7Wcu>@=X{@h z9DDxlzdpVgdw=8z`8I+Dn89ZMRh4f}epI7JOaY}fe|>?amigG5zfP*0c>8?Az{qO< z&~~My!{EuYV__Y;g7Vkxu^my{@%xAI+^*PI`#f}cIP+0v%Da!Q{UqA3l=wMH(@ech@m$RL2=j}14Y7Pmn?LINQbL%%U)~^w{W{*|NWow_hn^h{AB&piC zwU=bXn=LTag#V7|FC1$7Brr|0_>pttUV;t&Oui!aOW^xtx#r`}&t~&(+%=>sqxtOK zcYOHajh%i=^^?^FPP3OhX62dN)kw43UG%Lqjz$v?3_Ke?J)7=A>*6Q>&|rZ6uKDG` ze2$-Ze4UA&&(v|x)%L3~ktwBw3-P))zmm%|s0-09eP&g_IK9Xnb&(J}M?;&W5&X;;cEKbt`_-q}U}d17_)hl#+?={&geJI>RWT=twE zznXPH(*{IEs`HYW?~8Y=)PEN7I`uiyC#wG9inW)#Nmp@NZO2S%^j`-g&n$G8_u6|Z zrqEVb6Ta-Xi`ZS>F~9VI@2vTd)4{!_eQC=c|F6lJ^)!sdY!Pej5 z^?p5wFL!Ydrn~E%eGLor0L~NfRDr5Lsb_-({o|`TH^1m{t`90~d{(y*di=k-y6$K; zxUgTN2vt>T6GZL3M@5SmwMx-aTh*qjL~UxvmKs%C?Uh#A*9gUH)eLIZsx4YONJPH$ zTjzY=U-ysu+~+>$IrkaA-?{hP>GAlQso6*D=Klt*xuIdx4V(^drReHyl}^%@VJt_u zL5@>l{#ur(JSIe7SQN1j&!|EVg1{%AFJfXiq`^P>^LJcahQiGkry z25O9^w>v@Y;G`>cfd7K(CB>in-ca{)9$lE64AiIh{6q`hN^dJ+#%C>V@j-@};193u zt!D=BxS^HYxca7lFoPp1YkLy`*76$5kx0COhhb3f+!x*m`i|wRsFi0XIcnG=UZ(WK z=Pw#fkwQ+-&QDvbP80ABr!d`eg8_?sY@SxhFK-I3Jd z!LzrJf(*F(#l5t-gN;L8cgiyw8Q?NEYIdxOCwho-sV7|RYHerrw{F?RfW684)TA8t z=Tlo8s~29OG6^F>1Z3x-AXdtW%bjddRPk;tL}F;u`3wKgg*S}Q`frpTj0X$%AaYcrnyZ$ zHi|n5u*Lqjj(w)k;wg;`hRw{= zuS)3FQ(gN@S<5RAC2X9&^OI*P;PK_WGl7~))`GcXqS48d?J||zXO_jCTghU$@6ez9rV;MmswGq3`IP$)bHGw)`;uNM z=epOIE?#zwYmJkpyGp%3KV5DW`~j?aF+N#ZV#R7_U;w_u%h4pe&N(4tsTWo<76GDO z?SSjjN0z!B<$le=znZI%yOEib=_@~3rH^NBe>lD&TyyMF1M9e~|HqTd`cQE*`%X;u z2<0y%TM)*o?(#>>rg7iT&rBwjwV+cb5l;9(HR}m=8hyioMoL+4+0APF>9$d1LhR$qOXhXMHK#@s|E)Mh zp`Lni@V6Bn;~mdlu$Ay;m6yN@w=FSioWb<_Ym$?0mgcha?IS@iVsL#G;Z& z%cbictoX1~gYOlG4oN zHecGWb6&kXd3xT;+Uag21Dt&^MO|tUYNs-6me`zo zHs5^W=#z5xU}Dx5IIHnne@>@Y-=Y%bO5LTzP-WI95Ml|!8Lv_zlAFGkJN{L%o@(a$%H5NSy*yaWkCQUQ<$ z)eE#0+3}`}2~jSnzAV10^iGpp>kgAIxC8v7J;#J*wd`b!{(r znPXSFbEK7s?d##+m0id`Y0C-%@-kygE_TxW3{zX4QXz!>T?TI<3Bbz*(1FEkPvaMJUvYFHdJgw@u%@`Z z{u17Ko%6Eo5m22Q^PkuNCmCoPM75_g$Q@K9N*;u)6W0tN*u zgK5NbStgYhGB7;vPoLT>d$GQ&_x*B{g&_1ON&BUU zUzp85zI4N(0*Ne~Ik@`nFjVFOP3`!_u;{KrFWBtJl_zXc_q>ra%hI~}Cd;D}C~2`B z=#Zf$f9=dR-v);CKeP>FiXB^gFumOmkO%^_9SC3L|EX-s%uJUh%X@#rI@Q*TRgckF zp~AOYhMnI$Mgc0@Z4U~TuFXESLFtSdIdl&phyKxxKiv@A>%p5!wcXrRdLXY(ciJkG zpxQF()mOp*YCWOyrH?5QYPZ8ov!ZJ#JknMZ_Hd(u>pouVikcrn7(KS^e7wj#@l<== zpsJbVmp(Vp{*=1`(8w8k@$8C(3fB^PHl{qt-DOk~=B z{@cg#?kX5}FyjZvhPmHV*#?)r-bry=xbA&Gd5e~W=J?-#h=rbmgl+MWU#YfBwmm@v zS7-RKq$Xx!!TauYJS+_XOI_WO`+o7Naz-y^GsTnP_feD&(4qrd6cx>7cs1jMZ?=38`P!<4{cdyex2)@Im7}!K}^dq!#F{~vvKvq>u@ZUTeexp4Win$|btwk_!kJ*!{Dsw-MF970_6p(8lqlAa zb~3Vr*-svB7nN`X1~sDSKe|vilQNq;n908xD`w7 zp_@U$H(=&W*r;jrVN~T=y^vR06T>A42y;{O5x?g8BD8GMbRkg1Z_X*Gkv;C}iu0u# zP*d1pyO{N^5j;E1L-gpe-juz8)8#wh6T!T~(6@crbM`t{)-(1I+TK6N#(rp-aug+Y z`1oZ`j3O1kibDxM1UAgPG8V??`1=TqBcl?Jy+i<{iakY+^kyZ&<}8KXeYeF1$hn7k zBBDgLD)~^b4idKL)cmDDhOVP`p1l!5I8J#Qk6lF_QNiOz9_1IkCB}8?yim+$IVQFG z@eC^%sG3LZ&7hDQTHa=vEZoT>rVL=$e?XLf<^Zt)Q`M zl0q^9 z`da_hPM~2aHOY?*yS%mZAOeE1=gZj^+KABLWTm)_DMT;Au&!J5YUVv5pGyO)nV$^%~ z`m&?9ZDNN5$Nb%yUnqjRc9|U7)i^nMweN_?FE3Tt@wK4It2G0#meR6n`_=a*F@sIr zol#ssK7f20(Gx_SqJ>eQJ5lH#z=AbcBPg9 z@k?t;mC`aA$ufU zYlec8*?K=`Rui!$2K@ToDk)4z*o1P+sB5wwXysf8aSs)YdcreYWtDzMW9i=~Zguw2 z-tV_Pvpf-QN6K~1@dpeYbiPH;hCV`~AB_7JXbvH>3PXLK;Q zeqnRn<4R`eu>q&qUcKeS_Og@t>0eY}VQb}2vs&V~LKk@_FkT=PQQN7m86LYsLY>?l8mq(G8qJ zk^=DIJb;K(cw)i6a4B_Gj)`A6&NP*>yqp%ql%A)Bv&oh96lmiiu{Kp(&@S9M>S1EF zfET1!4F{MKqvjj+iJy*0mOJ)dz-4jh&gmCnYistM2UK{?^RGB1y1b`L1XA_oDmrSzrzk7QMi*Dao*L$?YVBOt07TL4Ce5 zru4Z}bZZ1e|g-JUE) zUmS+y7KI0ywRt(_4I3%Ts)bilwcUzlX_)WBe|htj7hJ*&ZCsv*OI=j-}8`8AChq7mfoCO9_% zZ}JQQzDcI^OeT^O4&i|+fm9tPB75!lj=&@b2wNwm?C8Fx=d=FyczzW2XAq?oq@!!RVvtmH;1w$}oZX;YsT3 zt+USK_G6Quh=E3ZS-57RuTqOpxi-JbTnxV0aJ%QO1x2-f;=d1qAMjUmm46l3iJ2-Z l=Wqr!eSej;Z-!eTl2rispKlxlHvYYkr4KRIZqRgy{tu`JAZ!2t literal 0 HcmV?d00001 diff --git a/docs/img/grafana-usage-cluster.png b/docs/img/grafana-usage-cluster.png new file mode 100644 index 0000000000000000000000000000000000000000..a301f79701be89c59451736cb1b3037b1e24631b GIT binary patch literal 94682 zcma&O1yogQ8!fy+S~{ddN(7}Oq#LADK)R9c?(S~smX_`Y0V$F029Yjl_+QRBp6~na zKkgWJ4F@25?{}>?pZUx=A4240#88k3kRT8UiugNW1qcL|0s?_~h5!eCGwN9O419yJ z6BJiM0GB(0K`{6~zP*U5y`q(oy_2r3A;j3y%EIuqoxZK1p{3nNEBnJI?Ys~O8AM!I zK*>36Z_&vedvXcx$k<3dR{BLiv?#03cY*w*@A2g`7<)KhG?n&TTWM9lK+~$UK09ig zhW>o^%=HDj8gw?5PxMmU{^@@%HIJ8sKdFA`Ue=a3)R7kYcuts>n^OW$dm0SZVp4j$GFq^3&o$~$==6W<)%ibzx`Vs`0$2nqh@ z`fO9Xqhnx*j``WWs7mg2?J&Vts()W3kWNJv{iH4^!Sq%N|ihpeO>6XTytKG$@- zv;Rajj|zfVQvbWjeBOd|c4L#0{GaQZE1KO2>QkZtmPCyd2LGt# zimxcfv?*nd%xk4z{I}Z^&U0tUF|h{FWYC)o4j6Ly4I9D#8I%Zd)49Z$3;W-lp%~dm z^4~t4FgnnnF-$R9=Cx|g{il)3Po;(%Jn0@^?Bx(3D-}#gQcIsgq0=jiP>oeYtmVJF z;uv45G4#4Ay5gqlcO&J%aDSRDg@HJ^)X_z#J0SB<*54`kexe6250Ro)B`3CF|S zKn;AOLQDw3bN>5IUpEg5e3eV+NF|C5(P1zSENQV5lst9KO*U&(M8#@rnv^D^T^ozk zEL(%sN~VsuVbsJph+W!P#TifXwV>EEs}B$M{06C9DK|{Myr@QoFft8c#hAiQ72vE3 zXB4~)^$q3ErBtGSijHUd_gp4nz*P3S%a@XgQ>P}fye>=o{q5)HMCGL7In)2W#UvNr zPHy~j!Llzf7#{J=+!&idFLG{8i70FZDj<+A_S5C>rsqddF7~8KtmKL0YBzITmdtng zDZ69;9d_yAqQ6H24n|iYIcf45%Y{JrGY+b(k|IxfRF%nf?R<@=eIAE_2mxFA-{T=l ziBPuwRMhNcA;p}6X0d6>Ik~Bc+Msb+ZG5$_VrPFmW~($ACSw$$iL7Y|$%~Vkcb|X2 zx;cM=4}^13%|BIU86!)LRU%QuhIEdc&Hh$G5tf)#nR?Aq^Te{z(6!7&xlApBPNkeP zA@Q#*^I7~|q{lS)ZFczinDVa3gj#LlsDT-cafD&~C!?YMOa^Qn6NQXL8#3Gs#`dN0 zS$KwXB>o09-Ha6@e#U1+10O@z_=~5cn_u$Weiu5rxTs(qUTC!YvXZaD=yX8j+~79x z>323fX^*&%{ixvT{ zSO>Eyfs6S04qMeHf>6p@G#$QLP^#bQAW>phs}(imLBGq`aqU~n>Ch27FuhB`H^}~6 zt(kSo-yOSR6=83(S^rAT!GVv^9aOaH)dd0FQZX!bjAcPudY^C#YoK#a^wiNi`~jN4tKUQTnP00B0GBq6g54~ z!C_)L3nRcUC52}H^E(uCN^li3%G#RaOrF_t8xJ`J1v+j$*1*8Pu$+Q=XJ_ZnANg+} zE8nxbzwe$8YmoPy9WJA(*INn+2xO1ngysa}zp?Bb%RQ!f`O?Jq2P>)oIyj9Y`}^NH zySQ|HWXaCTf^3gaQE+iJH-?E#6kJ|9Q!p~3BLzVB_EI&tp3{bl>f7NC1|}upc6AA^ z?(d_-4fy-|3g#6QO<%>+WnRU0pcH6OG*ksV428%$Cxt>d}sSx@FRA#b}o z;JZApUrsIZFJIr@4GY1t`Wiza7Ly!5@ygJ7KOyOP+~Cocy4Ae-<9OQtME6apKB%6Z zDpOZ+zdqJ0RlRznTzWfO)#h>YyJ5SW3j=HX2^17m|L|byQ(A_W zr%*CXS<5w>0~?6Bj(4rvoA*f9%l2EqNq$HCcHGwRdln(avuA;Cj4`vv;#b#tCBc#x zl9OX>X_<%~#6cIqKnftrq0yW8j1fJ!`OegT`R3e2H&?mjc_?{kXy|v)jkBL)#NLgy zE460$2&kxnMn=ylaii?Eq%;kf@ojBw<&~6_4%Y_^n3HPcibt7lZ*LhL&0o2Dc$8Qm zidAUw#Kgw>HZ%}_H+6_me3!$P#hY2{xPQy!e5B*=UsJDEF}`EH)+0{DT$b%9aWABMw9TgPUPH%tU*R4OxgCg#i;Gvo7`3#tFj}ub+%L!0LxllLA^*a{7nW@<(46)^<@89tJLqqAR#fDooaTnvfot>{P~k6jn9i@zOJ{i#(V@5CVSGs$zGVB zA3`mgfe;|*)6&A7#_2#XJIeu<0Ro~)RB|#2IF>qUOcX0!&q=@UsLAp(zkQ3KSSZt3 zAf0MKuv!CW8h~gOF{vl#kJz3WtAA|yK6i7Gy~o?i_~UVu1Fd=ccS^-TIo9m#Y&<^C zR~xpcS5rIo-*@bfj*h$)ih8`E?jFZ5f{?w-z_cxfl!s?BnZ<;>rL`3bqE?3L;^M-X zBK2{)oLGsL%JuAL?lx6f$RT#uS1TDP)7Iqk8bvQ0jaGVEV>%MA<3over}x!f+)TY&;%UYzaxp)m8Zmyw7Etn$t`*Lh#)8Gr4c= zs4*&9em70>tsS(iJ3c;uC0GUbw`OG}p$i}9ei_FF2Y=27OUqBW#782(YobVTWpnex zHm@%kxH@3-tJ!q;y$0K1tr}KcODtkyFcL2`o9qP?esE|s+V*=NuiB+&!lC?CBN_(7JTJ`kM@aXwUC7(-fUJS{2QnzqRg#F~l zvXy|nwU_eImYvHS&*NxlY}6e}9s|pchL*M~lDIkg(DYk>OH1mc#N#7pL<&YRT*2Npf-c`=gx~q(VhXYauVTRSk~^AMV5a{b5?}Dx6A;!n1|o z)|XP=U|{|r`S4-(F}{4US*I@X@biT}?zp?i^?LA(9x!<8@QbDG>Ni(jo)TOSud^4% zgefbC`~fc&3&{BRNM%P-!qAqqVNx^#A8xndxvd{OeohooR8>_~Zbk7OsnqhzX!czl zj?%pvyB8yM!HeQOVa#x|Y|F1rrr3n>9?KP5ol-BLU}3?Ag$;P?e#z*0YXXb(bk}l# zi<<2jm(zan4MX_va);X8kMwrvRn^rt=ilC!rinyE2)^EHap*e*O!i;!RNP^Ft$qPakQu#B_zWHuq)69XNIPEiLJ%ZjTE$-$tgTHkGgUq+0n3W_PeJd%~wr zT{DQ4D3^WZaas;o?G9WW<$I`Fs=S>UaYr?4CM8_YsMx6%2cYeI+dPa5;0qjwbZIh$ zl1agmNr70Yqmz>=sRrAF1&+Oi#=$E@)#5v@{#jd+x7H9{-E6rcQ>!rt@KAaA`IBb8 z$=9AQZ12<9&B+l8FPsEgn;drG7#J7~3=DL(M>N6wKp<9D4B>?A&ypCto~Fx0p%AjI zMsisS70M8=9v)h;p2a!D5T~Vf3XF}()6mg%U!CHb*Dbp4+FxE@cj+*KvuuXPvbQNv ztNHqkcDKd0$gPcH?9!sTXsabRNOIjLm%)u=RV z-|mr@Q$ki;CL)2Kw{KPPwoq3PyIB@HbcK1mJ~)yYqJFHpmj~4@BJp{xc+{F0to2n`b)6cdJN9HPp8bl`t{95xC(nrcbX}?;$by z`G(dP))jBkbF=%aepJErWl}+ixg2^*Dq4I?OJB8IP0fG*xX)f$=J^V-&JYCwp(E`@ zX3y4Z^{h!#Cff}J?z;)@`1p9e>4U{YmM~l|m~@$-hr1IxVfmg7ajfH2_^_jmg~I5B z1j7LaW@hM&40heSK$QOe{x3vy+^>ij+Rw=A%ohVaZZ0Ss&uF*-`g>5dA zJa$7$4z`Ib?=jw#)r|KYJSsWsYm`{5O(H(aR~a5#CVPwZq`U0q!m6rx)JyJIAYS>n z$nq)flGUv0>NXrSSzel6ov2k9N?-UJ7~jn1RpYUmQeIyiMp1}GspUUxw=dbyJ~Wvx zRpIFC>0=Ob5tK};K2X_q`5!kp?i0vla0VQ&_kAGF&7Ov19F=+a;fwplVg5J4q0dSd znS0Bn5Q49F?e0`=X(=-J{PAzT<*w$#*F71nHxS5Tvr}KHRZHaB8fK~1DSVCEYY@Lf zy&d;U51Ky#DB^ap6XEL%ohDwhPmn*kbJP{aAP^s-JYDmP&h$gMtL1LfnsRyEhokI50sv+bL!nMjur3H`#pjm*mo%&f5@sp8t#k#1 z&9u6qK{}R~(S!n#MFA|5kdSzUrQi{sQ&PgjQ7=9MmFKLiCyG)qK2(lZJc>j@g{|MRx}yMcYha&#yP%2u>m1 z1M;ri+;TzuwTJ_a%>q9LU}bc({mRr7?2<$i$xNNfOL@t8_7Q)VA5@^GBCbcGKn7I4e~1Mogap&eCS1! zrr~%@3%?pI8Q=0<2v``})8cVd*S~lcaqO2>YL4Tw_Q?X<$%vE{CLi%|f2bN!7@A+O z0M)5tprb8FaHR|r1D{N91ZBzDNrhKAt;)i1_S0>fDXY;)OY;Z|PDT9^^I~=l3I-l} zuyIBPe1Kr&^pZ0c05m8grvb1GU>b2CUtEU?+-uEW8ZFt?Q{i2$++7IJ$LSvMZBZ~V zkV3wgU66oR5esBfLpLApzs4p3RA_gaDW;rwzGp$3fcfBj|2~b!4Hfc*haPp=>*6&W z3V{KlFo$hg?(uQ|D7%%UxOi`;U(l;Q>zcInz6c>#SI(t!%%5}s;{cdX&dG^XD3e}( ztPz2a6?S%JpT_I)b#`1M*URgMSkLwA%Dz`yNbAjS3d=rJB2Ig}IRf7y$ja_+SS;qQ zc!52qX8LO`MfyTAKozh{<5}T^?|pv`)9QBKb-{Ow(6~@%;x1HyqH{oSyC%sn?TW^I zawG&`cSl;I>~q%P+o*KD2eq(K-a9!^lxyM=+4Gc z)lRkhQcPE-dvO^U8BI2{R(`cC-SspNcO&_H^;eKB&s9kABP^87?7Y8&CkxG)Y;TpK zridiwiUingiVl5N&S}}Ba1wX6M5SWw=a_SFPB1uz%x9}`s*RVQcHD~akds4$j9pWy^Whp!HFX0_r9Oa&+pIu104aE zflRN}5?oqpzTkc;4N<tzq@=*+THhglyO#^)NKtIuykb_^_*TdIR9rYJW_wvXK6y zFf}BDmY?d0<*s353gvdNb&}+zo&^5MH*ef&I>z!;#KYyh-ZLEXr3LctRDJ26YYeMn z%Q*e7aI7`6u4cx3MBP3311VHYN4uq|SB9AgZt)oU0x9wDnHGYhUL@)+I3=9ApY7{( z#ojS5pu;VU&3nJzHk`>+yCN!j{eWUD68_7)$6-JGZ7M}~aHK0yD{aoZr?=Z|H#bUY z?Zpn!Psz0iEch?@DMk!ZPC1fk$t$GzC*hpT;^H@_7lMmdOUY`l(v0H!zh!ceS4qIs zTrg+nRup};(}~rXCa+nui%Up|*sUP-CgIU5&W}C^h5i)D$VW zGG+bxd{-MAFy1x+lZWwNNyMgBbvA~0P2D+$+ue&K+{a~5WA=y2UU+fJ+vzancX&Hp zZ0N7T@Y&tpaUUI&yNiX5>k+?oM&FJd%{Cs*aIDYdcEN0H+}b*@vEho2j$V{kJ;VK>SNp^qtpYksPb5h|L@L{~+B4(L%}w2r1(ycHYx@NjUG6|seDizQ zHalF;`>Xc;{-K&#Ntrz1XFbA-QA0 z(y46vmq)r&H!MeUN}qR0tO3JuK8@A`w#_S_caZNNSyXFHSu;Md<346FJ8z7?zoCY^ zK_jZU4ifhCgaVy{=4=1;21(8CjI&rtBHrX1QL(zv#D%B*@1}{fS-`~RTo=i*?{^8F zGIDH>YJZUmmYaOITYk`C`IMzZOK9P!Z2axRU6{FzP33!zb61{l(cH--H5D05#Z0@U zFP3xVMXA-*{!hFI41CEDeomFqS+*Pp(rP-F)Oj*Ls4HYg&lBaEUjLdubLF|eJ!RO~ z#s%1sjEM}B#z+`h9>hixsCu=8f z`eU5)^76>&=m?pZn2yfR1A>C!0dlDHyc6Bn*dPPp6*Lzd`YTYskM?U1Zs{*2)=QV8 z67wfDm{((D2#r;fq#20x>DV0+oP$Zqrv`U&JC)R727 z^P0J~l7K*IX-P{-nby;2H`ynUAg88=l>fuM3?VuaG^&wSi1{_`OU|Hgr*kw zF7x5)$2Y~3!C~`0^0}f@bId1o`1N|dp_w)-I^A%r4qy?MRJ8iQlQf{A@mxYiD5=*L z8n0Pl%f(%RgPIlOV-rZ*Mr_zLYyad08D@ z?^dj>_mZxCODK73dxqe+xA5NBtPgNskM!+)T_1-W*EU!PYlS#;$K#<1!I7a|{V^1b zo(rnTgq+wFE!PCGBWg|*s#W&*%+;k21Zrch!z{cjb`q$3S-@ltFrWRlD%juBCOHe5k!}QT<&}Nnv3I(AX7#Q(m_qw7b9{hBe z*gdyaY@FQ18~J>AHCi3w9AM_ue4nb7JyWUB>fIbn z($ta>>+R{8Fhj0)jquwcr=p^gqfG!iL;&Qppg9BlPlc|K=C!CZ5s{H|Rq$j8NJx)^ zvARoZu$C5YBoT-2BPZ=kh=O8|AlT~={pzZ*IWkpb@Ay656OURVanKNf@4UGXaKS^%?4-q0elTxG%u*@8sTAR#Cx z!4H}u)+iG-*Kx;IB{=I>6II_3+Q@1@q2CHsb>#P{S)<)nW`Xz1Ke400D&zV-mDv83 zMsFJyKhTMVytN;K*fDi#D;3H%(J(5mbbp+kgW5=dAyyuYH#y*~z@x(=KI1udJYZY{ zXXnvFTPRKqZRkq&QeAnQ!#ZGZ&m17aY(Uw(li{^u&zdez=X3WMj_CbVvML4`ffac} zXMxrw?_9CunfHJk=^O3S);A(y zy3;Q)c3yOEwtQ0Gq0HMYJK0|p@X^F2aB8wvDJhwE5cvf7XVgNAkBlelvb?=9xnef2 z*9mxUHuPK0)}AI`oV{VwbR-m1zW2s3%rX2vZZTh{@0Mnvc=OeCY5x2A+iEXUH3Aa1 zTXE!{%VnDE=B5(hG2KOoo!hf>!a;QSB04?$9<%|)uY4p5_W4F1FpjbufGX{M5kweCS+zT_vrk@D_6U{ z?damP`x<4dBTmXl(@|yd2~KB&jiSBK(q#llE*Sd4AL zOOfoh3w?d9B&fR^T`BX@$)@qbQARvKgiQ1+C@RLJrmE}XnhT(VY$vNx^gXxxB?6^* z*P_!hEMPDm8My1Sq6YgNOmG+jy3j=2ZA*$A&}#-JRMVKfJ!c;e$XS|dYFL0=^qlq= z=VS8Y+PcK7fu^&##sq!==}0ieGMu6UIr3xvq_H4FNks)FI&uq*9p>XPLv^Y8rbY$r0K$OKhbScnUFPpIVPvZTaBHE4#MnYU0Sr3x5`_!4y z%FVvu?ex_Q<0fJ~_hd4$oeW_n{#9QPR_%9uAHO2V#O3}o=uUau-WBg3MEDW=+!jG1&r{f1obHl|ZTh9t$Fu zE>C=OwzMr3U7J=5g%W(qsG7r=tZLxfn&xqxJd=!ZK9`z0t0XwIy*|B&Q6x%f`8s)} zN&7Iko5XAw5Uy=M3Lv=wtK;MdcyuiQSG%gYog%aQdMS#3T`MW- znHMw^w4T1$6GpTjw{wJ6Y&5a}{?v`6Bx=L?fdrHYgjO!61EA5D*mwcV3Rt#Ri+3Bg z9632TEBpI~AGTLleMorB{akrm?Zs?Qt~mQ%SxJpd?$Hqn+$+93-)88Bl6us`eQP1z@_qB^-;tW?B&)olubRZedfk-Xrcnnief*-x zHgBg`yWo&uBO40c%SjC9|LJJ!8LY5;sY7OQ8BYm`@r?-VGdu@`Q;efbiWAN!Ht$CV zd)+zcH=xl_z6+XRTk9u7Wzgy(+Za87{C@pg0v!ShN-v<`bAa6Nc$xsN z>Gtjp@XStV zR%8wNBTCv?lS{(WNIQCIWt0X+HGiy@HE_eYc~;Xsh{bb!A}s69GuTIah%x=L^u*<% zPL!lYF}7HWivB&lO8O_|;#isDjIT@AoStbfQ>)KETYV^=Fa%N6eC7*AcOXh{{6O5R z-8a=$=-AjWvbA$nG#vN?R|VgD9WQNP~%V97vo|y!{R_f%W(enc)_RU7OSX|%E(A}RN_`rLfg@4*ZV;) z$~Z()&l)`SI`iIvxWc1fW)-=mH4}|);e5BNXkM4yneF3IagWK^{oPI1^fWGKIth@a zw6wI?_~hlMI|m0#Mh3)9C%tWs8f6NccRV!GQwC%C^bUfr%hOwce5#WI>`41Rml|Kg);1C(+Iq)JpO8ikw^_?PhtKNWUVO zvF?*TcQRwE71`Uo*-%)I`j8?YTE0&?q+MuTETW@n$RZfHzis%PkZLe|^}I;MM>$Z) zgt@T!;}ZxB#_4opHb2Emx{VQ4%6y5(GYR&*A^y{Bggu-M$S6i<9p%^=e--j5|5_#jDQb9 z3m9WEfxZe@84qQ)X;`FaP#36n)u#Bfg>gjHfeq_@t&p9M>Iv{DO9g$;V%5=>45h_y4ih6psEp81#7^xjuejm<9t?AvY-ze4E zj2rcQjbs}QByu2@N#{gicXC3;=SBc_xJ#2>2}xYt2vI*-z8j*aTJBJT?e|C^`UpD8 zpaKsBlj#Nm@V4j<{4iwe1Dtvowpao5Eq(}-?MC7(< zwGojr+#`^B9`R0*Vp>oba+uNtuYdqhq~&QJ%hwZT=2Br?_WCZFFg^DMJFZ>*aeEJ`z@1s@n#Fh z35Wfer+GimA&q$K(b!tla|CG9Uj#Cb(pk0Hv}HIE2hn(ebW!cOlz_nVI?Khtsw(Cr zX(YwP5Udwa5J2Y7v_Fu6Fq|uiYe!eTKffNGk`e-_{xzUG_YV)7)!9yM0?Lnw*L>b> zV@kbH9i>;FIpt_vmd>IrEi7l|)tgV#BIq!Wt^?afq9<@?k14?SdMncSgaiR>kYK0-lx%i&k(X*VlWZkw zd($Y(e2R^?e`Y+A_Oc=<*y9G3AGC^~$Qm_=y#sT0H`IE2ojSc~x`9_S^dyG{=<3mG zc|B=-28qlFf}5=;U2yd$O(_>VDbIzLM;8X8G+iRiz6oLb;PfpX>975IEi)bMHSW6xaeE?}-+2#bk5|H~Ui1X9cfcMe#f;eRibwL06w_X9~OG$Y@k z=4(K-?Ck9PTx`Y#ahOrB$2ye-9PVxx2i*ZraS2QvIf{ z+=?Tgw&MNi_h0tTPOE~amoZPZI)H7WbIIf8cyfA>$>E#9P?`%G0F0FeJIq5FJc9+u z+p%%S({m0D7a7eMn3%pGIds1b;W$!QUCWnr{@r->DB(~qus}%!Km*-oCH>;r++EhB zl9=0W$yDFidb|nCBV9n!@ZdR=Y(uveMJnbh8rXf~b#-p5G1%YXkGRhC@rVpigsU^h z`j@MHI5f0%#0wl?%v#+pzN$aC&u8xzj_)3?Nmi^x&RcbZ92|@v>AP6}jnnPpgD#Po zM!Tf<*Nbky^+s9>ZTc+UFdFphIJ>z~0O{Qr6YIEL+fzhAVRM5Or>}i?glmt)t^2F= zTca0vCWrU4bv&qOd}m~l#C(wpEiG!}Y-e*yqs^|fv{QwejiJUFH-L0`@RUm7gv}v) z)Sn(bKNzH5LsharX6>BsBI=L63!$x~7-tCT>OOxICc#n_7WSX#iL9Q+fOx9c5|{u= zy-wY#x2#?&NF%#rl1}Al-|3ONot*&7V7_H(oHT4D~rMaZG;1 zk)vW{T+!kUb%PYOP!fGQVZKiQ-7nbWLHAo{Xb|IlwOU=Dlanv}(5wLpB;fPcdqUy> zr^%{C0a8vS+I`C>dztOUeHAa|)aUEIwU}9osCYd5+AOsQ|1k$Js%A-*lUCYGq4TCm z`;1-1sHTyK+^f_<6E_dDxxOVE-wDz1UwM7W<)ST(?>}r;cPz119M;;43S|c27)ptY zcjikby&63V)hPpkul)h?k6m$h3J35qK;3Zsz;UrR8@91=3S5Z|)~n>#zxN})f0r91 zdcv8G`ts$=;3&RGfa6^=r{^-Ppkd^FH*S%qVtkm1ocnNjNv3d?TtuV9+UdV;zPiDh zzH~;oT+w1RiujUl^Lvzn^LF<_fl|XVO~Bm4{ccDfJ9tY#rT63*2`1Z})_IV5XuMV3 zsH%!&I;$>B&0~H$(_>9;c8ljcT+yV+q46|wp2=g5Wr6F4w>hpvZntCi(Qq2kJK8sG zTH2B*h;7(U_c?7SkLlY7rr&y-Nk@UTGR0b=VyvQ?l;XabnVHfMNZx^Y$C;&Tt`iEDQ(;XZXyWhaio z;BA#>>$@)|ly}9>5}zMGCB4Nf>>U}0E6LiJ*iA@gRe&X}bqMPhOG;SjYnb8*W0Qa1 z=;8vl?8_&rWkeMeAb7GCOkw^s`N*+p+n7&iXgA%+L|xH8I$Cd%J-6Uca#&eX_o@VM zq<`$0w=hcORSPuXh-=-W7DJ?dXBE?4yFAH}FkkYVTCsFiA>%_YuJ zMWH*QY}8(Ped|7`n_Z7j<|!`-Qrq;qE!=Nn!wDW59u8=4_bM}ykQ{tfLEqeZ*HFkjsG#%%g>Kt`bH0D z3v|9i@Xq?Y{1zXUDs~q-CXxCa2X}U4g3VW{P3=qdqK(RrA4fZdne47=e_8rX7uEKX zWVke=)ze&MlBdac72f~tMkdGz;@<@Z=u!3}-Om2O6lw&d-66gQ=^*01h@aykfiWJ# z>yS%Vq@RnuO&ka1iV<(b|F!aC)S{w171esPI{L(wh|9QZ(~SDIayL+jKSa;Az|xzT*4^CJCNL9+?hkEiONW6OWJ{q^vVUraU3@&CQGFny=a-^2GfhK(@)RWq`IY2iB5 zFA+N#7BEQ`L&OyO=kY%8y!-DK%KxW@p5?q$M=Hl2>Oc20CHe0+|L^UD=MM^A&!+Ou z)B2oK&ZWtv{(FlMYc`8NEs)z{{LgUUh5ucR@c4R?4&%Si6|9jXEvut%MNiN<^8456 zlr;(ApC)p?_9gzO@B{R$$uRGAky@NsozK5**`kEJ{;yGI{lDMFxAWbfhRJP1*m!)T z^`)e{Ut0co`d|Y1|DF`dnw`X$=)D>f>|#p>?dR_j4xd+Y{5$CKxp6bh|J4V-|7pB# zQMwrsu%f^bAb>t}*Gi`qqK4(y2`N*Ck25em$oZ#)W^5V3FGdDPc|lUtew~!rdUE1l z$5*G0movd zyOc{Se_lbtY&OCAr=|aMW&Y2S`QJr{{9nr%jNEL^{xF8d_{K~cOX_{O^YT9}#+20T z=;SRjhEXcD;pUxCP71{!OaE97?r}^1(>nn@_WxQ!C~y>-!W^EMx1P%oaq7n^0}3Qy zA-}D321yaCll-5#Qv+L1oouoOKl);Ce4#gs;8DZByScDB(EC%aRLhAhTE}lWZp1%T z;^G$*OE!`I)5^3<`ae%T3GWvp{f@~LxuyLcV@cZsO{5)aYTgnL`p|H>+jx`qv$pG# zBNnIA^AC+kITd`|mhJE)vh8waY7jZ;c4FS;a{p1Z_AdP;_jeLh5dNt`z82mqOkO&b z&eJjxbRofE7nBAJ&vq+sbUgE=heWQGYFiu#|57vQW9Oobwf&dBO?n5xZ+KjH?#3A& zeK_WtsJO*CI)AfGI{s9PG^xUC2L)kiN!k|G&(F_#<0rW#mdF|c3ogcf&iltYB1lkD0mWg2qBwgNBLs& z(TskS?$8XAYP&9vG?Y)KMmw~-E*~Gx^4@O+`nh^6VF*Uu}DZFFZGY*615p1GcATsffOtX;Wyv= zSgO{FL{t`vtpS&-WZ5eAafxY1&VftNNK4?Q{rRkM9HpLCW-iF7So^IQ0-x_28~)rdFwr z&27ZzU-eMVe=K$=$n+6mp>V`|Wf@hAp$%100!f0`70p)y_S}~plA<417`6{BcfFW| zK9O;xaYP*+5<;Wdf>8DQxg6ga42+ewbK4npxQc>&mGg50;;t3XhujD+OuBCrp$*Qf ziusbg#4wzG2Ua2?1ypTx~-utQAa{N`@`99z(5kd-~ zNss`{8(Yg|SD5qOaUY`xum^ZdFE*S;axrB+K*EXHQ9(+p{pt!@EUHHzRKH#RPPMAM zT;alaX^cfKwve03%2xEDNc)&Askn=eX?)D0UL(`vUA2?3`Z5cX_! z(P^psNVmHZID%VlmYN;MdHK9}Pfkusr}FdiR`%;3K4dCN|Fioi@`%D@YL#P^uIknz zAP5SZRdSVOw-IuzWe!M%H1JCh2+iGz;=BBO^a(9FK3+74&bLvqv69j5WcO!`KXu=6 z!@3BI*>Sunhl<50SR;Q2dlfBJKrkMP6mX?ui!Zf}DStM!&bV_fhf07%gALlk0&UT} zruDz*k|o>yF_~SJ zj!kQC|AQgYUeq4z7`(x3@z9~A_g7Lgt(K;x_0O^L`kd(rm6J(ImL%kL(y-`mDrxrC z12cb_A!z<=e#Pb?_d6-CcPtgD(AoJL_&0-57T+4+Am~s0pdtzmx>Ccp&hGcU*T@lE zTd$m7M0H!Ayj?g*Mif9_slM$CofDL!fg})SIgmZTeH-~EYS4#}dhX-n)8I-)3?!H; zde+xCLV>8YsZnDe{o7hBAv&?)(SsWwK`j?3-1+j*;W@}AkBxzcr>dV)40k3)jb&hq z?|bWKI#gJvlLtsn(DRU+!*(>g?O`E3JzlIF75pP-c2=eP}t~B9wt8|1$&{ zA)Woydfx{9erCgMAa!Kb&lM1{AY0Zi`{g8UcSqYJWiYfsH8jr-7uMD2y`w=EH_rNS zO1Pm};dkB3zT&yv%2=S`|M@q}0*`lM9}fdc(UOfa7bDVBFet^UMW%PxytxB1~w700Q#%W&Z95Z zKxgMl@DeqG`vH{uJn=X}rIEQ4L_|a=NdJZ{e_{39*7|xF(AK$c_NqZ~Lc?x(^HFas zRWLE})3^bH3svCtJl?>PW%=Cl&+ZeJj?fon!?+vs^Cs)sFL!&s`vc{hd}X4AZxB3; zfHxin>ia$CetlO#bXtZ0B?V}ai;QT8*7B*Es-Fi(u4WkVenC%i5Z;OM$7b<0kjBcu zN4?1KZFxAiZhyeQdsE&S9gm=YjRWyE{lP*L80l?#o2Rx*_6~D_)E)Zde!BZ~7B`!K zjo{i39wLpuV6=e9#EADTn_`u1-4a21X7OT5-6}vZo4?)j4xP&g84!A=aa%{(-N2XL z+l|JD3=*n?#ayQKI(*e?*965rPS*_@$Vz7riNA)W<;$y+K?Vx4EMiDjX$>|YBbDPu z-=@kM8Bqfgp2=|!4^-x?{hqfHYj(Vccn1eF-qx(WusPjSledJiStZLZI=yK_vETz_WUFTf%5$n00oF_8ey`_osG+05Wpmc36cF zlQUk;JR9q5BaC-e7TImB2Gko%I=;5BNQm>f`$O<5|h;HmWlUv<;mVsHQ~jw7pa!>J?y_A{PQFhSuiH$jV7Nu zSM3}Y@Z+PyWZk|;he7zP8aR1#LKV-@aa(uNS`meI_SJNJ9-;ha@n zg;o0&J=`CR$qHTff6Wa32eKpn+mqnneyaZi=l?*ArDS!&2quV-q7LfR`S>sH`x|3_ z-$jWI5wydNgWYHP;|1{h+-bnf0eH>qv5%jcY5wkl@EJQfKRVUB0@r;a3=AKW@q*6o z?%b~|At52E)kY`~o$Irm;Ly-wCJPgjms6!5{D3RTHKCC6|2m*koR>dW51;}BEk9^` zaLJqLFydv6nS!sW|GA~p^WbQ9g9n$>EubA{O|guPjRDOa3L=%tRyM*-K~8>jdpgRV zrz%}sQ;JPfqG9x(?5Up#5C~H^!S-zF)h2&V&EU`HE6M+zIQ`E&{7y5Gb6%FRt2c{wJ`>%Sfyn z2Q!ZUW^B-8V2oJMfHOuscKqgTb;cj)AFS36R~xSJMF$GV^S`9(KaP<@I>H#*RE_7r zvr1kE>J9#z_vJjx$$m{i4BX(KD+!&1{$*4!a^w+uql>_V$pjbh`w6hkxe)+QV6RR14!oL|V zxoI{GLJrKp#KPuBOw-yIAQX@x@L!dXA)w1uB9~ljO44Y1UzN!T9Yk7GpgB1?Ew~*@ zh2y`0169J0dhHUA+wq}Q8!+PfP%X;x-FpZ3zZmqr`^3(S=XEO5eb1fJ4B9r+i+S$AX*pdP_Vf-m94{;{!lckF zTPc{0K*7c+1)A0$6eg!PPt*{lf7D2~nS9&5ay}xv+Hi9}$^I(4d__b{M>igdLwDAv z^6+`x&=3>{|L7z2%DcXHjUc+xjJkRBQh$1DlVdcRQTCzw9{ubB3jCc9aIw5e;bLRP z>;$S?3H}?;3SN)v&%l6Jc4u2=>$>stny`G@+b5uDccfWKTl>&v6 z%U2^#EC&TPuDc9YE@`K0VeQkP0*XrfS(u0{&s{wI{a1E-P)cGplGeEDO!)VM@;}`x zSRBysn0#;Byy$B3RnoRmBUVPH-E({2iBepMo@DL9J!vrC>wft`JoY+$nxQ5E_U}*5 zdB?=0zpJ-gj9)5C1@`v>V1RIIsWUk>dK{a^#RKIEEG#TKaoZ^_AqCmLKa<6SK*1H0 zmq#i70~G7Fw0Ul+#TYZE4u2aRoksI=4x`)|WYvbN_x~VcIHhav%cy3ypt2#%xM4z;wbYeU)1j`(J7Zkcdtg5WD8J< zNz_>GtKHp%F;IFCTVG@KPgwoaiChEK`Ibhm@XCYwE! zDz!4TX3O+tGKE-B73F5Tef+vmd2dS-iS zIcIEYZmy)MnXad&hmDI{_~{cMQe)#}0y84Vzx{nWM$8XO7=zt1nDTh4>n>E66ijDgegg#En z?o^r8DeK>p;$0USJ8&jhWLA!kM@;-WZouAFC|S`fRf;-g;!J}iN|q;1W}d&GcGzIV zy|u1RnJd)sSBs}dt;y*Y8*EJMhCOGk!2qU+m^$znm0zC?fE6*3BNE19F<0>()RemK z%~UQcD%e(S4ZDpVVjsoEVwP#Os+pMVa8_vZo&TDZ?295EUw*iEy1T!4GchwWQ@{!R zclRwEnUmf$ZytQojUP5*=MFc=#KOvLYHG@A#r^vYWH~8ArTF-?Ffsp3#*40ZfByst zBnyZ6`$wJskG=PfYU=CSMX_K*u!3|{nt*_G2~rd(p?9PNRC?&W1{4b_O}f;G)KCQy zdJ&N>y#zv&UIK&`kbYP6_kQnv@BPj_|J?J>87Bh<8JnG*wbz>Ena`YS?sNaY9sK&b z>ZFa0&C+;Hm6n#)bub#;T)o8~@2^c3ka`P&*NdAJ%Ct5`Puv=*^%nqE)%WlyZMFr8 zPWFEdr}Ab)Lkq}LEoMm{+5}%Nh60(X7y0LopPrU&XdyYB<6Y!2SMC5e6j$S0Htsb| z33DINrH8m>v!SZ1M_ai6(WiUwo-qu8-6ATUWA6H{m${&Fa&ov5t8h`fp_@>M*V<=p zZtjqrtAEEvc8X3=aN|1#fB#C^1YgOp(0=v7Mte+bY$oUlp%f|VFlznS{hw1`KkRO3 z2?@D4yyDT=*vKU+st5FhUd+Kb_@)-5BBBH2YS^I;Wk&6>yc8b)`jFezhm@>91`AuG zILq*(>QcLp&t77_X!&3Nzx~iisMk&uA8X`bgRKA=v0kw5J5P20v0gHq*U=FX`o_+0 zDm@9~bDM4L?L;u#?9x)vC%Wf(%Zwa6UjcPu13ezzR6GJBg&|wbBU>5SQK5)Jp*;t1 z(2gK^+ld;4*GWt>#*8ZfF$_RB5T2rlw}`@YJk_GSF|hwbSl$6}d&i zwrAZwY}l@JZl0Q&IzPn~RD2xHiPiOhE&G9*B}GMmm2#Fns(=F>=>CNeRo3|40%|F0*1l6PTO9vxmS$_Uy1D3d zKhW70T!HjD^A+5-i%51+k)|!SKwDP!RTztO0pJ94YHLk&bqALEIMvc)Q1nn zG}v>vGos_aDcD#o8|eRDflNtV9a~l_q7qDl$Oa!2C1gywRG=-|MKdxomcCPfB5+Kb zWiWPqT45HxHmZ1UW>ZgUcjj7Q)s4aa{?~hh|6C^Uo1JKGaj_O2>(~tn$Jkm~H*Kx0 zHB?kWfHl|E^LRu_8JbgA=;)mMF)SoH9n@1{hMX(G2i)D=Q)@;3UggD2Yhca|hR^Hj z=;WlP_TUA82oAobOq_A4(9zPGB7XW99>Vy*%V~4EsY#v}R;g3>yOF;-EcUM@X@aV% zuDB&4-0`u@tK4L3-^~SE9m;H5J_cKq3|BMv4Gm?p0te+ukkZxD^Ooe8LEZ;S6PXUm zxEDdaH8#<)7L~(Ih&e&_+Qi4F@(A+_SqV0Neuq(m;l0-EN|6+v&Y){+YZIRF2O_tJ z4Ge$-80;UuUtjLLM>sx;zvX_iThzcH9(W}UP!)^CjyaT_p}N~&RwrrfQ8k7a&=idK z=X&E1J*97o$XFRE!zX)=YXY0lvO2B$N1XdwZKkr0MB`NiQHvE1n2|ml$)foOTU>Jv zo@v0zNv=B2U3Vk15{qh02OWSGU!EmkS3E}xcnZpmIik>r5szUz(HiMvmdce*jODUoPm?(=;cYDh70BK4 z%4!*2v^>>8Dn)`iZQ{hvvC1`$(R`MpH5zKFyE@M64#@!d4fv#tpS@73u~T@LCOE}^ zj;jMHZJMi}@8EzP8oHlyTGqtjkzNmGs&wjX47bF(2z;k~6~F@EEFszM72CL6U5|s;QhN)my88OIVEZbr z#5cTIpteocnIG5~8M=Y2523+*j>rQ{TRV^NW{g2(y0chlUzGZx-m!PHI+*3forAu8 znR=R~qk#c^Q`DzVPY$O8j<29?O9$EutMiy8U3B#I=X(YR2B;2K93An}N7I~!)m}wd z%6nw56@d1uCEdT3*pYeS)CC<~T@lL7!x5Fb4IZ3NyG11X^ z!;T$;Al|qO1#C)oNO z&<*-95l1(>;KRih^upGkj}EfL&`;~Xf88#u6>>?}^E;|A=F^w4pT(y{68yig!Mu9p z5}bCDmDHd;z2R{_#+!hD@3~=`*CToC9uCGoy^hjXA*(y>%`*ty_`a69f#U4Ta0=h*-Mg9bfvJqn9#F4x}Pmj+=nfl_y zu}S{^YGS%$!2U=_v$0F7Mu)#`X{_Jxh+2uID$)uD;sBnySxh^eMOxf#g7oJ^#|k-p zQ&-H@t5?zPcs(7IuK?}uTr7u$iOGWo3s?~H2M+#g?!Xv*)?S`<#{)YJ+JTU4%@&A0i~@WPyFddJ z)5JX)luiCl+4%2GWTL|Eu-*DIyeaU@tWWZ&;zkst|VZ_e9pRSA20-E7fl=vL=Y@mKhB8RGIs-{7IRQIubeTP!He0iRoCr z!^bQ?k^FJF51Yf7w1LL5af$E_%!@me&C8oPNHc^dd)s{m8=N!dQxGO*j*Pi@$8wYX z?Uexr#{)TlC-vPivfjJ932qZ??)X7fn)cRX#ACVLiO07^h%@}B0LFy#Nw_V1_SxyQ z*PO_7?7lnrO*9keEi5Vm!M!+Rc@cEgC?>5=CFm-#HH{3xmUO9Nlcj5vEi5cnmQL;V zdf^TmhU-9($Z2}X=|gFq@18FUS1T9uJO+a_G}8BUsf4`Xdt-wmyUZq0O3*MfOFKig zm87(>C8UlYkUmP?^|o0Qu*abdMoR5%SA~^(*R>~9vXZwuJOM-jGV69QBoJ;}Iy)o- zz7dr^^KP{k?{GNNe4Gl(Ms3=nTS>P2SN#p^@2{&w2xB{2uL$9Ned^NJx<$gH86(&% zEG$k=kJuG@#ghu&JhW5aq(Z3$$YnNJp>4L1#&}XzOhA- z>|7+kguKS(8YFFl!{KN%g@`j3?`S-mB&*pAl1BR+0sMt|)*z1{_=;uc6f0#kD~Y(w zC<0dn5EHtiSx;BIKg~*=l3`obexf#QV1IsgesMA=AXWOOk1hy2qc+hn3j6ISrh&ZW zyNGFq8+ZU@quAm*IO=u`Yv34?>F`~R&#rKt%XupE(#6%c2uevtw$|DrHBg#A;zp=B z&ft28t3xKjwh2`f;v~c+u*3 z>G%3{PIk72o}OMXR!~Uj!QiL7?CdpP^#FnyafyG%>Avt$^R$T7HY4Tn;MW)UJ5T^j z_FOvR>>+1m=9@`#^Cix*IfQY!W*=5nIJ_D)e239zu&ne2Oh_;Q?aBY?ICDgg)6Eba|E4&4;|QiY_T zP^@k@A8rfZ+OQ^Q%GY@<+{nq#ALu%spRYXF z;MrQrErhRsX*@9ih=Zi$kR@$kw$-&cj*Ue`sKDkrHp%yXO~Fs*f>pEIHDexZCBw^t0BXjo z^rGriIighHK78TLSt*PC*OL8y6Q`TbWL>@bASe!b1OdwWD7Sdy^-s;g7h%Lf&r@Vc zZYIZ@Tm9PVOP$DZ6ad!WVaYTc<3gGh6*a=U^<0w$7hX_V_jn4$)6xB*OTGRw13YuSeT77?TKIwO7v`By-PySg;|I4 zRh|SYPhGOtWPRxPOdxTW1wQ@xBmU@>C?EpPoKMg?x+I^APF9Ma5+x<&q&4s}FVx=G z0@hMyK@JeX88cL#l8-=35krnBDENU8%Rqb%RqdKx|J^T??DHk+Ql0;@hZr{p$KW(` zR-J!lWhI2y5Ut40t^iDk=hl(*cpMug{e4BzL{$(n=mKwCIX?p!*x0eyUl>1ZC#(Z& zhp(XIi~C~sT*n8kLry-=!z&4^3J%6>qHO_*^7zBvo?-k?%@a)ce8Su%QZWDG7n~#| zPlN;+0r#Xk?IbK+SdgQq^*pj`BP^-PZ#S#0tGzGW-z_or?vV!Q7t$ZZ;(z_S=UFnc zZcI0jM6z?tnKqtR9az$|5iw_d91w^a%b#A*NMVaMb6&%8jZ2ckx&u@}mbHS~All^Z zZ13uFl5!r+5;w(hA-rC#3V<>tPC!$1IMk<5Z*N>yM8 z0{=EdJ1+DzrBdf@Qz}e^-V?kCa6GWsV$x-tL+)$c4|Vl)ym>D*RmcXJd;yzN7K+>X z`w|_GfuZMkFS*5-&2_2b?FiPe|!~crCoUC-cT(f zac@W36eOKLyno+AazpKN`BLrdTp}Wvv3Yp~1^kk2L2tG9ra8uN*6F^^bhf67o5~Q5 zq!L>HOp(0&+lRnj0%VfE*d@hUW@K3YG=hq8h#zXwUQ$v*TrdlVe;>Ps^DAH*z+!Fp z2q`h`v%P0GrMnT*iEL#?K9jri4TGdk-4TI*L~8Ai9dWn_MC!X`$ys-?l|S|(QgU_D zQOw?rX?J(`io}8V)$Ur-y)xS~g88$SGu+qHOZh0?M&DgDzUOe?_c5-^gY8SGZ16TE zFHUUBdb;~V^tL@oApYn$NvprnzW#?lB)jy9f-{5I^%Oez)iO!4@^`g*&kM{UY3^bjO%>o0HCoGk?M)tucZLe zhiWOoKQIRzoJSA`^i8@{p{fJr&eGjK&hLO~M35ihhtIafFp8~zjN<7_&&)p&mxpDY*Vl1HWDvz2@d_W3!8&6HFM6z1rkES~u9P&gWkzUdgt(R{MwE|}FGt-|- zx?QxDe>+S$v7a&?aC|s7kK$@g*x2Oq!RL#)AD`L*T$c|c5=>iycCsEc=U+x>wq07+ z-z3F-|6Y5*e;lSYlc5_!qm{xat{zBz2rA8_5(W&=kwi-F~Aw2Hc#G8Y}>SN<*|$DzlGuP7V`Qho42}($bP)QXCRF@zQfZ zWOgQrVU^p>1}F`%N&i@&dWQ*+{rqfFD3nT|G5EcFP7-$?Z9UQb1tA+QN|pZ<<9L71 z;dR6_VV|GYye3K>Tt^}&Alri)Qt1^!pMNpC=#K@{AMw|5Sk@%vl1O}>)wZ2# zz&2gQSnqf8V7Fi2{Gl5LCLnfRtn<|au9xNKvCPh)UF4|4;+uKorFLK7dfo~D<#8L> zqK0`r&C2mtu8JnY{y+Tc=+G~nM~{k|CDj6$VEX?4{y(S!!cPCkTt*Dm#>8-%hZUFl zf3DscbJ~7~My@D6{txHFwKDRBU@0i1`cE$Mg~0y`bo0O8H344xHoR0&wFkYJo(oYKBT4N;6icz zVU+-1m#i0?<2>#E`Te!k2;ezG;!OI%PIrvsM6F&p>VIzgZZSGC0WqEW>#nK3x7-8d zG>LjuFqn`+U_kR{aYia=FWA-R4uJOq2)Z^5b*FI4aMcY$eT^aQ(LwBzq7yuXoD?yPNK>IicJvmEzOkP4;gi^nGBSJI6bWcZFm2L;dJ!$ z=7_XVsdhXbFLLPF1-9%hzS?D3s70zqj_}riDx*m|AQ>;-$xMJS>ZTWn0LNU!-c~W+ zdR8(F*lu$G2>v6%+=t?X?5^RPLn+nR+1PwXr?;>@?qK;8me=MLW&Q&@+pHKFct4uE zV=L3_gRh!gzG5Ri67lXo)=mpoCO6SxZkZrTx?S)x1BBGyclS;|13AAxFWLY0;FIq# zpn<^lb77u-Y~DJ z(xio-cC0iNPLHD4(S}$DUc>6r>qY~=y2i*mgleYPxYM7zrGXo&o zrXKPqU4A<$LK|ab#9s`xwKXuDvZr?=n9qh*XfB>Q<9z_-J-5ufmXF!Kv^8;s?8Q^o zzaKI7laCb)x+pYb^70H4TCppY1vdhp6t%DP`Au76g#iUy3a15QQbf;?y{7mh4c+AC z4EG?#`K&a);dCQkNR#+hHF4EbQZ}G4)qNaysPY~yKz37wM4^+BEu17Rt^cu#A|XN& zZJ);PDsMb|dkZX06efmvAhT@5|5{){Zyi?zRX2$f{H9R)vcHMp4*=l#P= zevI_{{tV@9+d-f;L=?h*OYUrS*e)qu!jGIa-S$GnENaDHQ1IzTN!ng(x)O~sn)~=H z|7}gAVR92CnN0Ry$%Q=zg5clK^25Zt(ranjm7b*j>$ z6r#21MqZ}A_UWnvACGQNN!^N*Bt%HruURy;==8n7tN&i9Pc<{j;_7cA^b%nRNBY9v zP`qT=qNJtWH-^z4R^~P>{iYZn6q>8 zR!)@+QxHpq0`Y4W%Xd7K0lDvYe-!oIQz%iw&FAhAVQziN>KkG#bRhC4vjt@g8J1EWZd1%*~!RO>V<)gAiMe90Dq~) z=GP(pB|S&v59+bO@6@BNq?pEwEcUjuodek*kSVG6RKmK#61sU6|)6W!IZb% zdt!O5MTv|I_bP%E(aO9IRz~T=P&+n#sZYg=umt>YY%)_yjy-1h)HpAswfT;c=`9QW z9>bxf@vo0rFG*#rv_hE!7^&Gin!Su1uaUjbxc!e($+|?M_bD1yZ|HiC67aKQzO6dw zoQ9qA<~bhlmju+%I-S9Zvk4I`OIGIQdkI)K+v!|fv_bcji0@(TT=UD2g41L-KmFy= z+)i)};Agt6Wml;Q57=6ljz4Ws=3qS!n6*m+vRt;Sa~858;Ty1BX?;7N*eM}FcfY9P zO0+2Co`9J2FIWZZ`@+kE2C$5ABU9{9f7h0tg|tXhQ|WyU?t+J^JpQE?SQj7yFYgJe z_ycm)c4*D9%`U8$6M1*QC?K8}3dl1F5qQ7ntXl;el#~cDEO$z6uD}}mh63;g0aRsV zBg+yfkl8j!0Rk+1srTg)H!^K!sWeZ=vg3AzOJ%EjlPz(!S?x_keqG$)_m=e0YCgJBkrh#?nOp_Q zHs3~O?FrX#JaKG3doqcwa6Z#8d~t{ll*@g)1H!C(08K1S96uNnTCX!Nda~8q&Tli@ zrpt}fOporrE_w9(+8$tjt5Xsq6aE`$sH+QG!Bm6XChy_zYxzcHYM$!l%88=(gZ)97 zODImH7NFPa35-DCHXbWYfs3eyGx4All`+zrC0XvdS%!PKuGB=}6(` z9y0602JlRP-xnwn0OXV=Am8M}>GE}X^shb!PW|+q$UTSH9Q(+Eg&_t9x#`K|991aA z!qDpte%&IWv68$Q<&5DEh_exqcSETo4OuW&@5^>at23cs#7*!L6fk)955^ZX{kyN^ z*VIrLN!Aq{e0r{?`sOUzH$|`l8tAm_U1w055gi$sqnfCuRu5H$4n`x@HI-ZH^kxNV&jfPImU3Ih`!s$ei4qnCMSzQ7R_w69LDFG4FYG-TO~p(C`}O^_a=yv3>q4 zW&XRneiA;JNlC5^us3|6|HVy#0GDTI#Yjh|fA8Uy%#xikwYVs)FQ-JPaEi@)p>^>L+NeAZR6jl1vKz~yPgo>SEC*HHnSvvd0^iB0oFdkj@Rj8)4b#WdT? z6%%VoWrN*q4U>fHyqpp$#*In&N&R<(V+g3PBT8#$E$z;yI3AVm&+1#+SZPKq;qk<7 z{!BRNujWIj**P$29;-fZYu^CuL&vA}NH$XX>c3+eu+#zX&z-@s|C>Pgi+jJh4c3k< z!dFD&>-JK35YNm$cg%=c|X=Mpz>#im{b7|2JSxl}7-B z3YAo){u-D2cSP(mH|ClbPX&s((2#YDN&jm^M1#5fUartj*7fF}KHxJQqsS z^<8qdWgtI6_K3B>D6^}(Xrf~pb383~2HOI7<7aD&8|p;xTT@$VH|)Y|R63fr3|e@D z+Ktz9^At39`$6@KQ4&7aIFg%3eO0j>Bu?#zz82N!0*>cC-tA4Tk(piIau}tHJ?s|n z-*(6DR0JJZl7P04Dx{)bJt_QK5$_2zeTH4EaIes?kIJl)^^SW^uxLxHX*T-At7yYt z!}8RPaxf2{n`CY@NAl85R_3UE>a-Bu?`6x zh_eu&FGt8%6;8S(oqHa7(5|@eqbjTH*KYH}9(Go+v)PTH7%lqrBS1Sk_DY691=9sv zHAhxj4z*{-Br0UFXKzE{^*1vvtsV54{fMbtt}xh}yVhqE!h8n8jd`k(+a8+kOII;Y zeNT`C&+jWUcCDll^a}@jL_k;Z>x6aOdw(-SCV}`>98cWTA@emAkqm|FaVe-Dqkc>i z=<#U`S~M(#bdwPS$g?Fr&!|`K=QO23h9@5w%rBxRREeXB|6l+r(I{ zg3+1%iC-K=>tDliJ+pHL152uPoEPKUAgpX4lCw{Nr+{hLDtkJ}=hu7oVx;T9h}K!F zpHa@ja3I5Kg6V!FxQhaA?!T2ZY`*qjlL0P!ojbo#A8c$aPKeU_KwogsboZL#4btHBZ zpj!AlQID9crsC&N@-_z7w2h|Gp;TLpyP>DmfUw2b>I2-Qt%btMKoMCO9s~s9xw$$f z%M}hL(G^7v?qrZxAVSSoiz7C|OYnhE?XAYBaPkVhg|?HFw-dLjysHPvpGEtCsDd|e zhZOQn;!COj#9vOa28h*0-nI*#c|GC#dEE@QGa?`D9#m_K2sEko#HMFAwS9)Q!%K1r zs?(BfV3@d_p6~DId-~DAwcSZ z1L}Vg>O1Q&;Msl;t)?mgKdW4q*zC(Y@*CHFrNjs|;NfC!4mW!bAE=Em&dL?9Rv4@f z^hjo)zES8e7Ovmo&3dZKUenF#%ldv-KRM7?aRo~fG7eg{yWKV5G z`Jz=`%;-m~LTp_J)o%*SxS7N9^@r(?|7h>8Xi7DE7cZg6UkUr+wmuq5aPyheG>_7*bs67# zwDa`VpE#h~&Y)MOk!4q@Mo2P?EDoZg^Dt7*ow9L)-YPo6JXmu=z&`4{_hv_*7 zv0O`Uy;b8+x*yx4s{Ermk~PEC@WJo=4STHzX&G|K>9bdjJiVghn%>@eA3X!9)EB%N zvIPV$7wE~rbid`Part8=6Jq=a=}%UQh$+G1^bOXBiGs-PaAx{2G2&61Kx5j~VLm7zQ0|b*)Mx|j zxZ5E4ice8|&%>%%X~2Y#I1nug5_7T~W7Tq4(@N)@0qo4iZ!;aQ7oE3UTkgqH2E3B>RFZ^7bth3fP5zo+I$52Kp2G+^DF#41e9(=RPH<^J_na zm8ZqHWqLuPJ|I#2(Hi)X`>H~oTTj9@jhyLJ)A*7v%(0cdZ8M8}#3Ju=ZNbN?K33fP zQx_I!K3)IblPSOkZA(9BYYPrWFCSIC^0t=p7|1;6^aCi)Db`!yT`G zxi^r#*}^E8*N8p<49nXLzC#D*iRa=k`tUAq+$(x?H{Smx2D9`qbx*ceD?q|ONZ9_x zHN~4Y@5>gaP<0#P#T>dpy1A}_dSafl^`Bw6k|)O~ML!*c8eZ@~RIdz@EP3Dvr!^?v zIz7v5T{Yxd$xvD41oRfH7r8|=y2nL61in6@(zjV6{t^9SMA)nMx@_j&+1pk>Vt5DZ z?R(=ht(E^KtRWR|+q9q&O!7)E$s>VRcU$joK77kIN9(D9x3I}Ow?xEoGp4cDkiSmN z2`~Irvh#^Sx6-)=K;%(8{#*g!r%!wLT2CVIW2}N9TOxsOYw27$cg9g4#v~{UH0M8x zxOrurzire<`bN8_049=sj#9QaM(y#N5`SxB>h6yYz~h)%3;I9%a{40jf$|F5;oRz8 zLRXHGbAl;W`RiJX6lRQfOPi4ihfQxOYMMhRcju&4N!$^=5|LR*)S4meH-*$$^@Qg` z`%;PI8;%aI0cFU#Ixzz5l>p_g8?Yaj@bWs7*P#q1?wMS;tDrm+jKj_4{CT31V)o8m zyQHkC!nb!Uew8(7qo@`Hrj*!c=P*Mv93Lq!skrOH71b`l5mVZ#AFw9j#X~S245vr& z^WM<&PI;nLSYqd+dei$}K8Qd8NHJzCAz!qLFJ-Gi9hf*-fUdRveSz4lrjA9<$b z>-kwHCH%otk0cLXe;E^Do+UIM>12M7I00+OPR=_uHY-YNt8k{+NCd`t0MCG}cp0>{ggz_j;7bx*9gNC-UWZE)bmoKw~mnh@9O<&3N{8 z=JZ&mUd@Y$Sfy0nQY^zhXwzen&_BlF<4zPZS!0G{fb9IRJLrulEtw?_Safyt7VK_b zgk2NuCy6=xFAo4@er?lL3RKnL->m{15Qf>K8yo&3+6y<8t5x4y`Q;jE8ei{s^uD_; znn6iBLmZjjn7XZdov<7^fldmoK75eOH6(3H_M%jIIfgSh+_dtx^E3a)wWBiw+(vbp zoo@>RYKk6nwmtZjd2f(gI1=uuR7~%}-z+{RGV^U7S2gK}Ok)@W6Zqj!w?l*ozw%hA zcYOQ}edRuTe#mac^QyAi0^`niJhOLfhRgW*`LtA9BdE^00=J1nPm%~(UU@MHz(pi} zyGwOB2AgV=HUJ z3A|;yaL;}ppU4}V_3nqi?D(f5rrNi*_zPnU`#HTOwuaI};Vj6iM~-E{imOACq9g9QVA9XI_(mIw?Z8&hBA zI-i2D-FSeBV%I3w(f0WGevOnBJq}+e@X;OG5d+>x=K`oUUSdg+9Amsbd3HqLkik90c@dhcG?NjjH>*x`2MN zEE^5YXQ6v(Ct3w44)Ei)4bFs#1 zVhk^wQ`nBT4K6i8*I>!>O<0#6R}{P-&(hDA-t@cG#H=Ah_l}f3SbEo}j7+AB=C}6j z2q+s6VXhOx=YOWsyN_Nfp)rMsA-D#H<>QDoHq~7k71iY}-ITt3<(|cFIqmz(?jl>A zaZ*p9qPZ?w3|pi1cpbzfuYaLtnQM&$1DG9M z*9MuvDjM!g30)~-t0&)4bLjJl7|+~BE;<>cjTd1qR;(yoLRmF96Y+ze^RBz*?)dC< z-&}Fu*}79Tcw`fN^GHROlhK}tW9s=+8}TR5l)*-S)F>P?_4_>I7y{}>1Doic(z2+? z8O$#xikNcU>fDT*L^P+Jyl|Tx1;P`G@If0~g?g!%{rAedONGQO?3dzob^ts~LvedK~X9TOEI>pUG1+q~zs2*0tBK6;>rVSD(z@BuKd>F5)@aqiZ5 z!w9j)YV48#GG-{lxU0Bn=!x#k4E&V~=AqWs%c}-SMCNAmgyhQ7=AyZIPHf7uv8l@M zM}Oj%&+QOSlp<$N?)9;H3vI}&%4Ga(ixhTkU$m)dsRl6?y_~mkhre}UEEC`F+1?%E z)3rD{hcmQb191LLac3q!-97N+AOrmN-B4FK;1jPY^A-z(9}%NOs##hOD7_VXa7B~l0+pg z?cm4DRTtx>L#tUg5O$qf{bc>SGGQF;ju;4o8%F;6X+aC3)0Kp#WW!&FR+Tox@sC;B zyH2~#yW=YZvK?cOHQ&dGghC?6_N;6WWfupKntFF_Iz{9BhYBD3g5>jg;GJN2>Gsq= z`PCPq&U*T&NvHNJf<-4*V<~94Aej%tc$KM{UiXmLKz7H(!jtVZ;Ty7@ZcJkC+8LO{ zrRkcpHui^vG~ml&8teSw?oE+tjE43d4!(`AKHuilr_N^?vuQw6M{dmE1fJu;Kh-kL}nnJ8-Q-RRTZ3q30gVgP{thM%*U-jjWsi3fb-oI9KT zy}{s%S-3-qjLN?6^R!iG)DymdWN=;IyJ>*phNwpw(jJuocB;Q>ZvyfW*Dx=@;A6tJ zVvzQ7fLAOP#UpueygcUVb$u4sygre>BY*D1Rkm}A6uKu+Cws4d_1$8m_@EO$mFi9A zt6*An`&|u>#{;Eni*j5idn0Hd>70ovh-1ZD${taw!((iI(%9LUv3M)XlnS9d&sD@6 zox2e4KnjPwogK{3MDeT2t$LbvKS80G@^wz{k#+Hz58fjAsMP>|UGahmZGQNQY}5+3 zqR;PLeDR`sdET*{OK+br-%OCp$f3s&7M?n4a$bOUYd8flOT(ruv_TwO-&1-YR#d!g zoe^$oaGh0C32Pr7J(B6#t@&m@`taV%iD?ENuCBsSuuEq*CIKs>qKHLpZY=)de+X8AVsw_pm%1ybi0I<)XL0Iea9U)Uh=%7oF|0V^O$K zu(MQN8$7g^hoYk14atT}bMJ|puXq|nOqdy9V~sJ#;UR?->Q0xFi~#ryv^g}L+Utw z^2$uw#fzj##iVOB1h8?T*Xjgm1x{=nZ&GF$3u+-uQ;ZIo(BE;g3VwL8Y|JI9pHTHa zc8PHfhfdAe*iTo#vGObIYdQ9@myO<!eQLU%?_)Dle^eaaE-c~60XLCa2ZyrLC zE9dndsSF%B{e*V*6yncImTC&D+{EGly&XOKsk&$AkymYLQMXh`LhN z%AfPRECR{D>4=vr-kd*HmMS+DU!1?dmU1*8CT0PhOdRJWT8z9*3U(`P&xQ{N9BpUN z#}eNUp2cX?sxL+u?ppP^b5EIWgjPOLh~?$h{dm!Ku*)UejgNF^7*(~kf?7Ssj;iN_!);6)- z>Rq8r?La2xW;W-p<~IeY54Ypo_0u`1Z@c#PxlP3Hw=;diJV$EY6D&)3%`}>|?f;&p z-6Qd&yI|(>OCuKbgACl>Q_4$UALe`f(y49i>pG}Tx1?+f_;Q_$EL#diw+rzb9H=4V z=7VgMQ`sN1K95&$sD#Wq3`!@kaM8Z8w*OIhpu9e;sQ9L{ z&ga{)9imEgrJCM!@{XEL(n)jMl+5z?OGxS#fjUgCgTx_RvaV`K!-s8%(Y?rtIw#>J z?~|?vThBV{W@vmf*kTM0TEMM^UOFz+xgsFarEat|VeTow>CI^c5-iBfBNxL5!AP9Z|u*9**J*!2k`%vmuUAk z^!5fKu2H`GNhNfpF1LC7i-m=^Jb8<6pKlvM`{-I3f5j}70YjI!`S9(P!zosaB(D8u za+Hp4vHa2+0Ctz$7P!R=z@M5Ix>_T13g8?)NuOiO1FBrk8`PSm9rc>emL ztYPJhKB*cEH2g_;U=jJ?!Gk}$k_ziQqN|WamzrWLYHv?aRdBMmk3_@ujf~p%SptnL zGySf1wsG>vo|yZ2fgs(nrkPph7;j%)T^(aT&ufQ~E69o~#*ZL1^qUDuLlN#n6 zNWWsie(Bqq3L9y}y3FtxgO5KaYIVxI0z4fmL$7iRE{Pr6Rd$*PwoYDp1KG#w9j-64 z#xS3!9dP{k*5g6O&&<%A3M*aDxyRRL-;|-BSe_Qk*3cxh6w5&>o~fcVDLbru?}%TVo4EE94?2MPECAUH18Dg06!@!MN9^h~(~% zKKMv!XmN2daUK}}AN^hAvpQ^$AZ1o*DJiDx?d`3%Blidt)pG|wPe-(jkzqePC|4FU z(W-257ZfmUR|~@Fexmpwk&UNM?%~r$Hy$j_cD(ej1s=>=H|>1J1#xdW4obv9P|hQ5Rmw1X67H%~Hii{J$Dg#4xCqtHjaJ8)0`^3bzSbX1;yL28}ai#A+k zGA=Zv(8}!NBsQ0X$NJDlNcv28y-C>S^X-PJPNhWOEqwhT`TSzeR<;8l|MF~l7OEqb zlbYp<&Rm>;e{=ZKg^6iBg5iUwAG$dg8-f+CU_~>SXV|!I5w-Iu zu@&_GAY%+4rF6=V638bGF7|q!H-GzvN4lw2^D9@6z~23m)GwV+q^_@|r%!1AgkODL zx@W$&N%>(YtJ6jZy<2Ya?2ETdakQ@%%RE}&A2EVx+gb04uq=YaM(2T@TzWOvVv#qR zRZaFJgyp>l?qA>$?oZd?y^9&;y`_CiVk-Je(+1@^zRg=+@fB|cH_4HT-+tYB@w7)Y@jlOX;FXS@UeVZr<;kThwJ!SNw-Rus z4H3(q){UjMS&hMQ)lqNb&D?!m+rXq~c)baLLW zAAO=(Q5ymq{Vj#R1eQb5&Ckb*em=DiU50y_ZafOaNT%;GztyDMm6~TRHdlh=KJBa5 zvj32IxE6i=3EVzn%w=LbRBcASJ=*c`{KjNQVzz(czJx8~y9R|YkHNk3^>x{9e%PcIdxQ^2tT)xPvNUg(| zh#MRwllDb8Wh?65pS4w&Kd}8kckfQ04p+qG1ol*2iEpPqYWB=1O1z7Wdud04PEII$%fpsxeJL9z;i()6+>-EC z&aFMrF?4oukqb0Exv2fNx~95)n1`ggFGoT7vT?Rof(Mg#LMa<1o!}Gfgj4cQOPkiI zj6JYxIT%ZR<_y>*Cs$Oo;x?@+)@B!Lf|#Xd=isRL+6=Bht*xJu$r}N-V7{pA!u6rK zB1#_Pco-B;OSrR?qBsJ6P>9RuLnOw>>L{1fVo9vs>B<@$3XiR6tg6MTw-FCUfz=e zb*UqrNH@^@!{*1kD}ono79&kX0*?HGkP520!@ud@rV5zJ=eovqx(Hp&e^L?o+fXwG zkJV$CvPE$VFiUuZfM?5RCyg3Id-n;5&vL;uSEjGpHDdLam5QkqrhL_Or)+JFrh*e* z9CWO=Ofm*Db{4fvFouIaeCdo~&)N9y*Vor4Jn$|Ct)|kye)Y*1Y;G#m&gx8#zKqm> z!$8}*fl7CtF^S2EjkUGz#FGDuxVM0+vg_J~F)%>HKvLjA8l*b}6=?}YI;Et$TPY=_ zySt?uqy+>7>6GqGcl~qg^PKlPiy$5m>#77Uf@U211$W@637*u-RK#!3x21X(}+s3@GBM z%jSdy2a7qO%aYfXje2-_-krLajX(S2i!}7ig^ZRND}T~yt`Je&D#hmM+#8&Y43C%m z^X9VoV?l|h=L>doW&wh?7Zv7A1BBr(9HrZB#-kO#$S5fdx96yj_l9)R2Eq9)$M>e} z!O<%m246Th37qMVMSY|=bAU5d9jHb+zJKpu{+ThixSA18r)03$tqMn>ksji2w~-Mi zBk?-y&}8_FniStw=y%;>C+9P=V#V}bs~P3zFq%0STcvLXaJ{{}sF|30HF-3hB$4-A z9lTZg|25zx4+DaNp5C+_H?IzVtd?B-t{a;H4kvsja$PeQ?K^iq@r_WodUfB^%S#_f z8IU}3%rDbtte5qd(Qc#JsI?h>+MZFKyExxKE!7)&PF!=5)E&!}1EOR1)p>^Rs~Z}_ zKK!vDZtat~g5JmUC!7*VBn_iA^HR0vqpQusl7pJ^%ynfmdf)!4v!PFMwPA7G> z@Yts z9~u$y&e~Nsig0jfH1p9`9J5Zg&?MLECNE4GU^5vwyQ?m~-wcLo_|a5vrGRiaCW6;J%A+T|X8GSv_JR6y5^!-jekGaJ;zh0PVK6;#i4 z9sU|42iVx@9PjG$+hr(nMq0XdH%T$W3>N3-(~!iD3C_n+HXD<4qGHG$ zO5>?{jN4Zfa!N{`yp7WLPqyH&bQXQxp{D3-P0e7S6iI@U8Z(C19sf*f;1eeHTLRTK zh&N(lK0rlKs(FN_3e%x!bX~cS_V*ngt^K(}y#MJAiI8_V@cRybpJ88e6k46Ud9d7a z;rQzMIbR}Wf}NPxTPoJaFJCuYZ|K*TDb;MT$WtoKwj29T_rU6vHwI*mGX!{eS+Mpy zo0Z#c}a zAdmCj)b_}Ib>)u?2F!n^_Azpz|IBxG4%b6Vq094nFe2y9Q}t~2ecF3#12RBYf}Fkl zsw$18^FjU#P&>8U8_7smX^Bw#Jn|S$Rx5yITATM6dIxW*!jcCDw44#sT9RVLV^E{9 zn)`NF^if^d8?ZBAlD#)YLxraWJNUt*vpk3#*NjhKAh}nd1q%otu$kz+k+*89I7G8W z7xQGR)U)SCztG>S+lh|{`$&0@1Zk1cO?b@X-SLZ}QqwcS0PuQW71wlIn1Lo1h?R$4kmd)LIz{IWp{B!e8MRMvQx*(^J< zp&R+?e#V4sPlA`5jP&nj3i$CC7S=y}0-UHV|4T>vR??~w__8%lz-?DGAMOfjdOv_*hV5PVSXxJBA!1jflH(Ke&}dvOi?ZWeC{HC$T}dhlDr4xN8EDP~MfwC1 zxNshq(~9($_CFB&Kq(J5?j0Hxm39r|kxzuThlgH&X0`Q{^L^_f)q_l}w%aTHSt%`L z`z6bRc{$Onp9VqK)aqoNbfCuGQq@5p$Co`QD5&aafq>0nmvh49M1;*`Tm;rik>N$a zo1dDBsj@lZ{TWgT1|wyLL!JE~Jv?FAqu~FLOANY4<)AZooo_KaTc4@Snl4oPJjY%Zs@pjt9me{OBKwJn!N z-wW-N;O8D84=KlKYK`f4hV#?*>o$2!0YA$|aH<=M>(SvMRyjj3gmR}&#Wl6Hp)oPE zK<%5Bo<3T7QN()7Ql`lPvG4KPan>R%UcXF1?WZQSXncHpH0v_I!`U8()s*u^{&rb= zdwVp?yL>nUMlK3$%(742)GL{2XO!#TJCQzGH;4f!$U;*rwAd0++byJH8Kk75N7IY@se1jX5g2q9wi#LQpw>TRnjGpl9GCIymTMw8bhS` z@OiYmtPlaG;6U`_ zwYu}3CuFH51_K62q6*MSEpogVbmC=s^WJ42V_)@DIGxCaq4YaD8%enBq`h}IgL)Eh z7`^E%0|F*Gmx;t@9!ljVc@gR^Y2%hOeFyWE+q;-wa@GrF>-BfUr6NXJ2mYC3pzO)* z+e1UijQck6ZNjKD2>u`eqKMEJ@OWzyy5#L+0cBfTTP;(U^OZ`KRI9AyG9(Ef09`0d zu>Qh#?{&WT5u?GBKt7e4^^UNd4r7^;ixY(7Q*e0>djkL!{m8cluDNG^+aFinXm*bSan5^$dtjUx>Lg6YRD#CPr_0zG!^{wP69C)}Bw zNemqGfbgv!I7OjnKAHb~I1@~@aH%Ow{`QaVxN0n7Vg^yAN{cjZTSRVG42SYWeQg<{ zN6uG|hhD$9Q|8a3aRKjo4|;d-BoN#~vW3EPm>`Aye_db+AUEzn?P3 z81@pkU@D=i;h7%SI{WP zL}JscXb;)I!QN;d+juqSJsp^^ft(>dpj~=kQmosh1ngaM8}%0f5NkpZnvawgE*U8| zO+bz!oPsAsk{$h9Y~44xi~-T&sAhmwVw|QPKD()y60a; zTEX2~mfsTG;iD)~rFIFWG$iLjT?D!0@8${wvo;%b`v1+S{+!7`~jK?GG@oUfc++kO7xF<~I? z6%-m$Ha1Zc#D3S99-VED=%MS|J(BPMVZTc#VXtUxS;n<5a(j^u7GBb*etx+EzuSUV zs;XF#1>3}YE$x3&s)(fjG^P44WMBUaZEy?VCe*vS-D>*$7WehE6a-tD zOSO;HaOz~H2_Dq&9*o8&(DK(mg$P4Y6-due+S+@1bzm8<^4|$$8u?6}l#F zaR>>?c|ev6#05@T@XlY__V$$>9n9@bO?A&xUm)N(@-G^ld6VyOPE8jt_&d4iEGphDSS5mEW(H~k z0-oO9fCShO2As7fai-@99_)|<#$ zy$O{};LBsEKNaIY-E=2w@9Y!@CaB){CypgfEoiE)6V_;GXnjaF&F?HdD{%U)&w5CW zR-K>Uc%n*zp=R^L45;NCJaIZSo3NYpHyE!70`|SV&PMg!UTiWR)atf*V zR6wQ_OAJSf(oAZmudR<)ih~PX0ltt)=L5nCr=70RGUsZz)&7*W`KpV$6QA;t|aQ zYr{;yh~HmfK5Kovt1tNC#Zqw(-vFG2%-893p`A4KYEON2;3xOospb(W!BS2^-D?Gf z$kqVD9AKlO`s=T!aNI49+ZsV8y1lnI@GZ!;l)%VpGhU7 z=HXGcopKJGnF$aH2DvAQH7{NTb~=hUyaL`5gQ5Jq0HP<^5M^;PMgMt*%$s0|61SRu z=tS5AWxQKHHVL8VHRvO&dVbOgAj>hvs`YYO1Pv?e3qC%+m5J)|^Ha+l04!)R11;P1 zC#p)px0g5$(ZfbU&CLAT#Kgph$Kee~-e4R5l%OETnsjjnSVu=&o9YR$iP_>{UH{qF z7aAO!`}y-7_)M0nQ7$-jAVRB}S%>0%iCwQIc-G$JSQDzI#A z5_~H+Gv4B&-9!r~i=$(5O(kog#HumXSc= z2yRvGn%ZnfmaFU#83;*4zgS&}G3k@vsh68=^|x+aSd3X6`;hDA;D`0-2N@9b;1a=mjLHfiL``hLqdgJ3v(E5(2E{Ni|# z4k%swj4!HWK}oR1^P8Y=_!~V}068crg*#o8^9_Y=$phhQ_}M;47K8vSMK_9=EESK6 zit;h(d%i+Ci8 zN=lAHr=nddp`;Yece+IC>EW?-b}$7DSuDGaJ3XtVL+U?MFJ5Nv7W(d=wYMcuF)~VT z)a(T4cE={!PDN!D%90IM0TFi+{SNpRWD3ceSD#Gtc5BnL=nfA-Wr?yo6d40HX=Z1OWaHIVOIA!}K` zYRikDA9NR568wiwHbA3^?|c|x35opxEZ#gIXdf*F%p3VyArkP4QCx76KWHL3J-ecf zE7TDC(UTve>2p%@L6M#6VRtg8{GgMMPTs-(z8t(tfgqSZXQ$SZu0YQJCXh{-TD*BFoAq=UjM&?pZpg99DrJ$=4r8a8g++^ z7=@GJc}>sUhP&@~_Imrx%rfFl!}w|qvf_XeATNi2+Q;QvQRD*Kdl;W%VjLToe}@mU zy0~6Yw9v+Vt*aY8J~1)Bu|Zj^2-A(^C5O|I+UzkpWyOE#|Bhsy4#<+*T1LF(r5NZk z^+ErjiG_6w-4)-9L_lyuXRB&6?oIMng`(lega}H0{!Z=i-<630)a%!CbULw-6$KFN zf(vAj2wZD#YZC_jN$?rr)z#H-A@q7326Y!TldRu2EGxe}-o+TnQl$Ca!je=Q9$|ov zmR5Xk)tmOOuj&W{5z-F>awuD%7707?-1Ibw?Sao!zW_2WlRiVUC_fJCnD`Ss($B&r z$H#er{>88}JP$M=WS&3&0DF1L+L{rN6Xyu}`i6yt<(-{e$cc$1=TzTKm|%@R;h1 zoT}fK&(0tB|6Qs6G0s4EMN^iJ7Dou5SP0{$c zKVB`&|F3(nmx1?GtyZIOb#=`#-rnhg3ROwK!%s2>2GJpoH&AfGzkK<^ee$c|c&|xy z0yS~4n8CJ_fx|W@CV?4r`0m2vs$-w$8Wz+J4yay|78Vp@mxgcwTx)JVM@*N&6rUU* zT5p~{bqfJ*F`NKjdV^$+Nhg7zAPkj>2i_IS3o&~M7reh#Oi)VItmjM+%zkaDRa`eK zA|fJgHBm*rEJivP<~O@)*ahfQS)2Z7YVnT7?2DaaV{J3IoY7u9w^bnBWuyuS`7 zm@QPH_XPI(!3Ov14<0_uwVjO10Bj+S&sk|m-8GKgcsv!@v#}WXK_DRQL~B(+c7)Px zl$MK2F)Irc)Wt(UhMt#qJS-%H7F;jnnH=RGR92UIwSZp$DwdFl+E(uy!_tsGTM_Ff z69xBw+>?5p)#a4)Y$u_drp!4PKoQWL`39p?y0EsMdQ88)Mj;R zc1v>AS;epGpsAq0K3a<1DLl#c8c8%Pm#0K7iqEH6kuDjR?1;ci1KD6vFQ0-+=oMJX zuNJ1^pp*hiYs+IU6l=jSOO7TD$VP{?Hh(o>Zvba>3XoCZ?4=*8I`2vEZA?i8gB(cy ze5d>hC?jGU8`0S|7owdp$WNh0So#@)&Vq z%JvTN(0X-JBDgS8mFLLxqu=n4JMb>f-~)4DSyPzMgHc5EmY zgh_)5S=>9roYJ4<*BGeVFnoA#H(>pZvU0y`wBj^b`STFr(NS-(X{R0cPN#_k!=Zhv=hxvhsfhvjB znKFRxde=rwSK3hd^}5Hos&@Ot*SW*##<`{I74RcFN8AyABLqfKQ< z(>_XeF^>_a*H6RbJCvrC&jpTYd4unfz0n8V&?GE%AyW_VQi?Dp>eZ`DQSTCh$|EA; z9c{wT4`*#9V(sc34iKs}2@74ykdo9PH`Ee?c`?Bk{qj55UF4!5bTc%VHwJj4=D`v0 zU7z{*@+8-vAB*4LNYEQ%hma$h)gH+xlA$k?fy_RT1Q6rnKuXgmucx0MKB;!`s^bky`I+?SGqNa$= z&dgvD5Eu?Ip$yA~{dQNOq%5=ot3!E1iH&0cpeaGh!NHNIYHQLR4VpwR!F#W+SDG3& z9b-7`_>TalD_xG5@EvnHn6OnUcJQw|-4++^3*Y4EsPxG@$h~?&!J-5( zQGa~va=w#O8KfnE`=RHo7hB*g3tdZ~-N}F&7(hr(n)yB3nuRbikoCcotQO^~mbzX{ zf%YIWPXaVSd3~~07*YgyqC3?%18yM*#i5LT&g{Z# z7(^v2MRIUg?VfSX|BFNYEeei5zg1hjo}6QIGJ;FGxY^ zh`NeCtDKsv5X)_?1Pez7D2n>CWGJBkEFypVVIneN20f1B`G^EsYc(jVz~Z3Wi6vj& zCW@d1HsNY?VaxRNRM;q=f1n#hy^e!L^0Tko->vI=M=&TtRkor0;siQ5Hw(}P23~83 zt+rDUcx=e>U0jXJ~#T1CsCp~FV@29+bb)NS?b_x^L)B|e6O*|Hv;lUTFB^bZf!kSQ4azQZ-`vP zW?KV(e&gBbn{G0XhJGj~C+C4I8DW27e#t49oj8-KEnTE0iOqU3^CIxI8w9`xqRO=? zR(m{}V*pqUtOd{xaHzeQbOfMG8bl6gLT9mXB-djveuN##K%xoFij0YNiuQ${(S(He z*UT%GO7!wE2nLbs7T;k&*5_kp0bR^R?xsE1F5DC}CxwqKUb=FO6X+9~3Sl7)fPVS(ydKExBs5N8=#=xvB zgQGic0|AUy|CIh3K}(5v=B&dp17Q7{wry!^Tb*>I%!IP3)zAd#gZn?<_+m4ZRdz)W z7$W0gFO0mY@mm|rRM#Bs0RHRD1a3|pzrsgjJEt}B%SM^$^sm?;>H z&v$RlvS$U!fDgf*MNzTu(9v})vZ5diQX~Mh9Bn8h5VCxqhB69bkES$TYvOVDRk+$;x1DOlu&q{K8QM zvn!G=2l_K0c3KbcBtz1GzUuAeHv8py30! z(WC-c#!#++y0|_miDYHCFeRLhN8axx04E1i&IRXJjtyt79OSrOd`PH0*#(vxby^nE zlnP^Q?JI=bcHa<(D48yhSbmj64mqq)U^i7Thy#Ra_o7=`%%Uh=GA0?Wg6t^b2I&n? zX!Qw7f{qs&(1^r5KX|5UF31lKMM@l-p|DZ{9-+*5On1kIRMcT%Iiu8Jz+l4aCk8ZP zA!Cc(=KE3xC;RneqDX@T&&E1EdDCXkvTKFSVKL6uYTbHhWnf6{6+~c2$d!hLRKRd3 zcc6#=B6+&W+pw{A*80~5@C+T_y{ki%^T&$gN26JgaxtU**TaI|_BDSxFzo2328Y*w zKaf8QHgIL!7Tb^r|!RL3W$zK`*z$&@`2~rjpIG?B?HC2{i3&HN3a@=sP zePSCwTIxG+MhmP#Ip!QkFDb&6PMFr$|qoShfFZe~}BhXMd-xwHx1wwhk5Hbz3 zx1{fi@?}@$qS;otVDA&kkc_kGW>PNggMc$NBf&)oWXZ}LtB2HFB|y)&82kKn)v#*( z%>zl`X_AqMW6o1fkU5!mmY);K)$Wo+u#A9n~TSzAj$Sl%})+bAq z;a>*M?o(p(Yp16pF=SfVkAB8eYVs<*@}_l6W0z_#OWy^vO2jsV@D)*aLVpH5j4eC- zfUbPK$0c*5+v6e*H<-J58doZKvR?>|wC}8-_e0(mwc?j~4Eyi8M?d~JQWZnm(Jba4 ze%1e$avD4+yngpRCgvOVXC{F|%mo9{TM5-~=`5%^rt!yRdhSI+r@9S?020&}wu5Pk zjHRVNQ%FUTVPaxBpN}lU;}UjrnM2V?=^L~<*#=gLLuGUDw4e@Xnl3`#9F5bJIXj-s zF2wBq5WvJ?K*z*{45Y3=CPT6lYRWu3GC&0Q6N(S;w!Kn$WG55s8{CJvkjBS)Yo=ZS z@C5n;+X0jdX>441z<>!I^<5lOssEv)K2N<~O@Db=E&>iy%Q`^nH>P)FO1t}ab3G8z zXQcN;<}3Dey@rKP+X!G1e+2h0E6^%VUZG2L3RwQ=GBWASHP`w%*@j%m-Isw9w>OpM3u{oWOD}d z0OvNawr}?+OCDSJ&l=g$ke#ZFTRS^48+^6a2dcI>IFHh@=S|2lh?!p2I_&p}Gt?Rt z@^W%2P*b;Kcf!16Zq%Qs1tT{S$e8Gnp_VYQODZIOa${ToX>tP2f^*~c?bkwY`)EY} zv>b+$G_7oB8=cGs@*~p%ALAiOUmS*B=%wG?m677vT!VWs$Vdk|CN?$+n2%Q}t^VK{ zg=$Z_d1N~#e0C8N${=>!+fV&-Zgn*yG4U_h0OjM=)IeG8=bWgF&)D(taW;HvwEn`h z)a0j9m!_uGfoclY-QK4cQLI5o^d@rvI-l?8M$wjxPZ}dND_RLm#e(6QNNElIR=;I2 zjwsf}JT9PgrzuqhxkK6?JLtkzR$47BgqGGEE3BGp3e7`>9}8yhYPDtOA0npe>+v0C z@d0Yu-renA8?g+Fj7-dkcX$ob$7Q2sk1CaCGlim=t&1q6qH}kxLZJN<*)!3b(k2!R zU8*T~EEhSiYy;&WgG5T!0qLP=@Jv866hWnCDgA~3XMtUxsA$C1f^+q7&Ft~a>YwXt z-BoYP?n1&@IKTU+HJ)yrxP3(-CVGeE#ub zizdNyi;D%G=s3$|W2Qq|fqc-33_(6%cm+^F0Z!*k^^0?d{it_jqTn#2IkzrB-u=w^ zc)xd`DSvP_@QF$31P0O8Cn)KCVP!s*u>AG)ncxdnX;=ji z1lbfCTOWR#7MoWmUmvTMGVO8^hoy$kce?U=m7#L(-uz+=%u{CTD}T_&%}!1Iz(sbl zx0(U{k%kcLI~eXO`}q5Zy~Vv92)eayDU8g0s7T_(e`{R~Bg^R|U^mJXCUUk3XYl-| z7vRNgn}5M(S#T&cW#L?G05i$LN(L&wK12kafBMHGTEN&7)1g4vOUqE7>rHGg0yj8X zXIw4!Se>MJL}mK4Gy1}DTCjF|g_hfbOcL2QbWm$cE~*$66}>lZ2{lXt@$4|PVqnWdW+@csPZ3s9aHa3 zu}7RPaPcz;zkg4H-S#VH8C@6*-*ElT(&z_V9_%e^Fgm;D;X(s(CmETLxdnv2ys}}X zvIGYJGjys&Z=iaXj}VJ{rJTP$>T}5b=L1M9g%60&n|KlYZqjgz)|_TQ2WS%>+*`Bz z|4Jw1e+|ip8%2?p@gw9Ba=(W0M$#moZ29bf?+29a{=L6mw5YO1#OP%IxUGw0Imw%d zUJV-%TTDJDb?~-G(M%hgbPz+y00UPtI$1F+FxXCQ@1R?QV^vR8pZ|v!V6s zH8>2~!pAQ;rkp~M*$ND=!)(_O2yi;2rDZ6q65-&;Bg{dZnso*7DmWF^px&v+7CwiK z$`VF=((9_&R(PY#u18SvTAibN~LzyQ&i{&p&L zi?Q?X1Yc`?>Y`k!Tt82jiK*%&|CP^2y~K-`(V5^_xamC&nN5EM@B}XnJlXyvrFII-F?0OomqKe za&#M3KC8}(wnRLu;_@=Z(eW`9p#-5e7Dk3FlCtQY%Y*05J3V#UTOSk7k;@MzYa#rE zOh}M^nBV0>faM*pcJT%{pn~tzL-xJB{@D1-0%o_80dTT{U$jUmG?^+w)A@v)_;~iC1yF zE%e(Lz!J_K#aCZI4&~R2V%v>ri2?ibuTx3-(c&>B%W>FZ=%LEu4N>}pcyHfEa5ovd z=B(}xUlI|AtAUW6gN;k zwX(9B8LupkWYVI574SJU)T7dJ@z+b|E30;2S}pyD%d83BWWRWK zA@7<~UWLb3DjWvn1uUTYT5)lHVg}tGMq^bNP;Qc@!s>MjRXU+;`4homtvZJu|3ZRCn~(|c$~$B9w5yRF6` zty+o`7e|QlmAiZHXpvp3Q(XDS;Nb0AwZJ%2#8ni|YuC5dSbDy$?rCn#v<%mqUThje z6TV+j(a3Ps9$O}xLCx0j>ZU_f#K-$vdc{RrPdqD_B=bJ)Vcj6{XvRYmx(D~+amS*o z!QInH!2qYh0$;#5%jc@E`$#6QaV4)y)+8pqw^u+Q4%wvnl zk^f^Z%@~Xv0T_n-UL0oIu5=^OuhN`R=CN7f$iZp!`VjPch~N)9@|EBt+egTsq`O4^ zeCS3cMT$q!X`>ERtlwl+w1SlTXC_L{)nS)P^!<1QZkLxI>Jd6E)7xHPK~?9gl-#zL zW@NtRa@a(ydH&(3sh;4L!$V%PJ7eV=U$0<}2X=I{n5pJG*d2_i7qGV0c(qF|9--0V z^FW%)?d*iF;X1L?SML0w)j~v}aMBx^!qr;U%)Go1#HT-l$1Fs)5@{%UuOpI-44*Y> zB(hxJ^6T)(!BlJIg$s`m%OkD#?6ktz>Q!)%{Tc|Cda732U|iJc|18*Sr*yr7l8X0p zbN6R`W0s*Rj(ZxOf5mLTSfNdqyIJ*w-Ck*gIUHF~$dzl|QY_Y;DVD2yX+=a33h-4XqLksCLCgUPQ1f1vRA7)b5tfBAy!wUK&-s6YO2^YFz{glhxzn0vc& zq^D^%xf*>1p>1{|)dMPscdqPn;O>4=N9VlkHlXSG%v6IA>T-saO0uLQpbp?G)#*W52zE_`n_EU07WEwdiG&C0>n zuxvH6m0;Qo)r3vR8_XvIanuIQ;m+j?CN8`%Rtw9k@B{hl?#jhrjFlUEH%;HDoCu{Z zx$4>OanrsqzA;j`z}%{Ll|Xg(Sctf`wU>Nwb*#UitmEv?J=!-Hj84~-8YMq2WwKuV zbgk0K{zbz#T%|H6y#4jFs|dp`11vPu?*1VyGqXD=X1|KBP|?w0RQQjK2hN4Ay8cC; zK{h*+=8L;j&KPha`)2&(;2^rfCMNpk{@8{FNTYvOXCyw^N1#XHsJ{4;GDLB_#1=sB z;GE*CHno&8#96hpM?PhBEV`x`WI=UEY*_Ll^v*J)j(#Wk=NB^(Ds{ZcI-^X``0}x% zL+o^~;DU!Yy;tvxbDIs6BvDXc;|8GoCJH0IKQuiqc5Al!MTv-tK70N8-a>brJIfwM z?kq~;>#R+8@~Jj&VwaClI~_J+A2zlj%L*oQ8T>O3-*A*AUd*J-D?BoY6>Ly?-kjXO zf?HfNe^2BQ*=v{)Qh=Nq;n3QTAzN!Z=n|gpisRWDTXgaDxQeQz)U5@GJ9LY*$kf}R z0l@W4U7dK+@j%tcWtz*b_Nn!Nnw5}-5l*`#tEg6nSY-@BlEUH)0h>0HGgoOS>AM#4;C4qs^E)!4q2m-A_l^#^rjU*iE$zdj((kk-u`->f9h&^ z7k&z!KOvUi-^P1qlfnO#VkmSDK0%IvPpqo!O7LNdN~m+9?ywu*#yvUxWNjXUN}u7U zY@i|VXJL^fN|pwF{<+Qne1;J9VE+BG^T)*lAxXToe|@cV#;Az&UlDh<%mxbl7c~@W z-Z~IxmVM#Wi(>!1>5@Ei*$yF2NlZiud{Rb)@jt5Z2+gaE>CMCU6e{!b0gs8h!Fod3Ad55R0J|H^!tByKD?diS2Csnsi%6U3^ z*pIhy9RPC<{Iq|wm4uHHsBbgKBXs_VeF=v9hYoiT=pydYU*(F zppcyWA1A~7Twnj@!XzV3t-J$lKSHF4idman)t$e_O*qt?*L5CpSzN`=2{VUOE$)En z_~fK;Wl|Q!jmxs*i|#spu8YRr=Ke_WyOgA)w=b_gy7X7MCK2-o`VVu19`B*y*s0hZ z1>r>)6H7FAT2 zsmA7G#IgoFt>_W|*k+I^cU@YEN?h5eib&?h`F&G)&biQO$%en3D$2_^?guiyzFVyX zJWS5qj$WiFcGC}d<0JTq0`t_ptpoGj2MpYWQUqOH`4QC{pf&-G{ac4Uu3oYSlh*^= z+)>dNW`8sYpt$Xq+31xjLGUhU@a+={%!G*7TIi^Z{OPU4^1-jF;cs*b8)7|{o7Hw>2!uS>aot0 z<`q0mdrkUe;v;2(!)3>u15&Pe2VHdwF=l$Ts*mqo&}Z!!^W?mZi=w zj;g9pN|Xqpsi${r%-7Qwd%E6bhXBQ`MYEv^^8yQ0A0z8E_`>OxeKK5+SZ7aTqN5)< zp(bl}W1f4O*OZsb7jdE6$CkKC5S+UTTVnVv9J{R#61d!uQt~oVU)1ZYd3Sk(Pu|$w z{NUJWQGMe!Ua?`Y&K~a^Yeb=lNx_C+iBU=X`ir9-lVKx> zBQ*RM@e7St*-sbhUHs=eBW8fT18tbkc$C$SR}De}5(a)u>`s(?wY`DQDivlV&}jI6 zNNvmqWOr+JyloSusebJ-ztiowI~FWZN#{$Xc>TLxejDlRm8njN!=!#0AD*pYJNM>Y)vC5qok&q}1M=9IfjJw$By1a!i{;@kwxaM@N?NrMGPQ3dN`WyLm z!FTVjqkIWqu|Kcu8s6^1E-5V)C>YXOjN!S2643CB_vAavrbDeljUChe#u&d-Op`Ze zO0E%mnTa!ofj_=!&AEdL>La$Ok=!oaxxuR%+JDhajNLDC{vyB^S3fUif>Z6}SKNWS z8r6}NkAI}wd*j9u3t{mKC%lL1VtPduGxWIB9RF;$WFi6rSBM;q&Neq>Wd|~|vu}?C z8viRco+_FPvn{xa@>sbl*8CzrpZ+0_jnkoRM5ia6vKsz?<>xE;w^%CKF5%*_c1f^$ z8p(4$el1B!-@g82K18`;GH#MH=T@yX+r4hDJ7^vuOF_y)bpOoi@7TG;0qs(#5}{A{ z6gHzS-;@U;rIFjgQYZY|aHhY+izmv zw#+h5+coW}HP0SBGr=qU=b1bhX%tJb?_*EgW7Mr4C+Eb?icqAu7V~SqWqvu1DO&VY z#=zQaY;Nim(3P8ClbR0~d{Kwz@%ZQdx^SmQihoY%$MMUK&%e=G#{Cnef4*}p8X|Bx z$jy|?^1M;Z%Bd=1XWyxT=t66^+(+KG@US~9(q^c`jmhyHj}pgeB+FBZxkq#jEwOZ> zmzU8qngv^>C0lsA*SGG8P-=qEsHls4qx(YXp{Co0{m+o#x%+IW;1%848+Ls{ z&S_{om|dDXmOfD$%z;atN?I5xVbtMLM06kTLk8p`6;Kj#+G^$=T;yX$`K6bC2{)ew zqfAa=OSN}FT+`QdgbUU4W50)0A4T7EhfAok1_P15@7~qF-SA9jR>e&`a-B>39LA%! z(!%*uNUKjNx;6ayd4hdE<3s;*1XU^bch|Q(T7UKRNOKBqS}yKL+`{$PiJeW$Pyeh@ zX)fo)i^I(Bu`~a&#;JQo){Jnv@G0$=t8QV7k3he&UW0p>QwsZXj4ZR z?dCo%TE>lh{+ov%G)WnT?^|<+<|+~VxEZrrI2?cSt&#$>gjrYU_)}EZO^<6eYZ2Aj z^5~Lliw_N90#J!qwuVwaGzV^6O!t^8I&Q}NRS{suSc$q18D4kDz5g+cS{2MyhnIs} zZ1b_KdXu|4-kZze8GfO_?^LNT2A)v*%v+u^Uw*X^eo;l-#;=b*%Gef>CwLtaLu ze%)_`-gY|Vx(n9+*;RGkHkyyvnEvV>S3BMx+R>Bo=`$Bo-=WrWQarIhUuz-#AHKhJ z6ZI8*E2H6nQh&|1%GU}C76QRe(+8)VKM_iw7!p))nCY$T4J$^-HC0dyiY+)dC@036 zIMtGQ-3uPM>?XEp-tf6;e22BqmYt9A-su^HPE0pNS_QDl)pzju4KxLk$@{c?FVFX~ z4H~p|6xQ7M*TZ`BgU-kjb~m^0b7F65p4WO$SnATuNJVAq88~{ID^+8{{3p51iG1l2 zrHc&cs_Kqaq>Gbgc;uPYI2Yx{$&?tGW4{UuiQM=y~M(Px*X53G7HR1a4hYKl!XUhs4~N1Z*_E(&}Bzjwt38tHGYyus*=&KUVFjt zr>s$~YE9xks^+cH3aMHM)=QifoAjvRx45<0a=+v6&{O-?D+BA`!Qu`oY5^hXi6%1> z+R!(h3qE_b!MwM>o}K!JPxPeF z>TkdMEs|M|#U!=j&_MyL=Pvs|Y`R+q~xX#Hx4~&rXwh486HG-+YfwmGb& zlLJlsq=i#A^ApBh0@R96{i!sT_iq1d=suOBt4hcPB&Ic6*sCp*>wI+;cHw`X&=c#; z{%h^zy?7*y9!+L|mXK%^6^26UjjP_(PD(eJ0`QFGZnG7VBnNj^8j(rp+I(FcH!f|K zu*5FBxz0-{B|a~lA1QLM-)u%P^y+W)<1>?XuC9gP=nKIcDV0Q@souX>I>8c^ko+1! zII^`o`D#jyrguq&ZjdR3A@*KGC(|e1PY>)ky6>4rZNlAnXb-3#uUPi4kJIE-6~DeS z8Y$$%pr1^_(TlZ8jgt6J+1aosR%hKvx@o->6gGsv%TJ+V*`wq@Bq(gqc)p z?)`OyFnndjxCzQvWz+N90|NsJMOHU~Fy@ly<>C5``%aq;*GbBC#=P}az;{n*1wPMu zqdDYQEv>9fxx9nSky5o|WxZa+-S%pv{(&PIKtP?8u&C z22Tq`x+VUpTVwO|!}PW(7fv$6Q37Ilw~rgC8C4u5BI$py;bSnb+f1G?Z`R~Hp3f&V zM#scbVWEwKkXFUCGKWU&bLdaPoWb0vLI10!zFvqX?g@eIq&ZK#-He0%AxI0IA`VQG zs!y0})EHvBpREm_a~l|3W>JCOBl28FE5h-ib2Y3F(L*oNpPIk7i3_(K_%to zI|!{pCA_wGUD2O<_@1~ywN`U4`nt5de4u}zX)|E9zDY?*3cS|$ynX=zajVsImu=AZ z7cpT^vhS-oRNbAeBR4TWwA0M?M{E-tyAb(_&YZsD8$I9*XT7|*hI#7YNc{W?*Yq#+ zW2gIFANQ_)_!u-2nkl}3p|If7@9`n5qz;)tTc*;jlrGBS;^{X9; z3Oj?k^Bxb~VI695Gq&|}QWNzsSdcEfYnSF!4AmKZUe`QBi_>7!wUo_&la)=7l|Sp| z**2}(g^+sez_D0Xn0v=A^dO?4o$B**%8CAaTzS*Wu!N)z$pzq$)Kb~Iv<;y@i zc>iOP{ykoy_g4d%u>IIKcO)?9?%dm`J6^eg{EY&c++{TeBRRuiSJy&R(ue{DuZ^w~ zA~n0|7B_SE2Dft;%dZOabiP}Qf#5NVe7uKIYL*~94K>7I@A?D-SjoNiOaLEE+snnf=12oS<$EcO0yE=IJK6% zF**#y5D{nlwR)d&tE02vtHzHelj?Kh+}~i`jS9jsJ~S%aHrtI!|C5b+%rtWH;c8oA zp9m;x_F?z#ah^85Z5|vG0=`pO2rI z=6ntd^M}%CBnvK{RA*}=NlDVvRG+=R+#jE-+L1A2e{H2iU)#gHSRZ@Ni!JZFQ&J^? zQFgaZz1`Qjr9;958ou55w)K#(iz};b(c3u&IT&2$sYZ_;NZMa#in$@^`{u&M%lAVN zQ_;WSiW)kZT=Sk02X({yTaD;QXi?o61LX@e+O6f~x6THddJO0`?Hi9xn{hsJMB>#q zNVL9d1?CAWDbd7F)!+dyjn=sL>PCTv{n6~!@UgYR!4kJZ9=jf^((b{*Y~2YF3d;9r zw)=q>r|%T3H=H-^con{A@ZBcly^5k|YUnMO(-K{3XtuV*bb7i)sj&E@xaJXkguhN` z=-Un~gwBW8+V`pow%-XZaP>4@JwLJ#{h>$3U=872!}o18PRFAJtuI&oDbQFnIhI>v zKL0TDaTO-Q+rdE_HT`Dd-iQ_1wbm8scGW+8$0n#IeL9kngdzK&hoZoWeb#R4)RU1b zwOu;oLuf&PgHv7R;?nmTUb{z}I7d1!W2hUABFXhisG=?~achQ2ocL}nI#LH#n}zo+ zE-p6UHuboFo|TVrbxue^us>Y=F|$_jJ+ao|MLZrqpm%Q1lfFhBJKVZ``!dRHCmR%~ zo*Zs>7JQDiIGYQN4fgYmpuaXgXoPq-{w{H<(SFRwhoE=V;S&1!+e?!!CxqYWWBiZM z7qp-czSkl8_n@2W^rec5h=@G3v}8Fx&QGqZyA375l=G8LDX3`!BFJBLVsg5TQ-5nj zO@U4f}*+LBYY7Q2^Z*6c(n0 z6H&f+EV_@#$x-2~X(W&B?e2cyF!B>3MfrkgvQ`cg^CW(J#K(8KDF94MjR3Wo6qAaE zVa4FMA(zGrFX&P=yEr>|mi6WlHT6xI4D8uPPxKV<9jX5tWjDW_i0t-ndeC8n66iNY zxH-#cjbZxEAbgrq)_e0Rj_VG7y@-KNOZI}DNel1GYC{Nl+Otcg4d&%Ns9r@p7V~FG zYLiIMVblJufljE!9KPCC94FPP^rTjU>siA6i$L5))!XZg)}FB(?)CkjLb3L)HXX%& zS*MlN_cgny4ce)%wh)dAVuEVQCV(!7>z4I`&p%wIqgyP5x(4UN+cgJ>>fzHy%T0&( zE7`S6YimzUOh726MMk060Rx(ia1Snh7akqq{IaRu+KLa{+LR*xFoa%Hq9guT z{n~HbT9M0Q?uM9{?%UNe`RTpO#IC1IqoboJUoPTW*#EswcwAKY(B0h*8fkv*ITYG$ zX|7F`yB@N8V-gD~ju;tD9OL)nvrXEaS=-fq7jK?dX+pGy7ALtcnEcg*GyE%CfKt|< z%{}jdi2wQQw`J$xqT9-x7BmY<^=w58Vb6JHCO=~xqdxz%eamBxr(?Mm>%P33!Q<;1 zmlhTyWSB7R<|{vL@T|?Yj1vau#Ua+;NBVw9yRq%LQ!}zZ0PZVmKywOW43;NwP(<$g$tlpawZjPuM^vf8 zF(Q?KYqKnG$;{riL<`M>a#a<0h1cU+k7a~1=_SB3k@7%@H#iVAw z4Wd-yfh}jmHAYjGb<-!p?E@2sfyXqqCUz zrz+sTQGa5|JKLhgj15o>B=K!BiDxy+F=`m0Nf2zI$RJW?zHq&KJ&lCJU#9S+r_DS7 zVe!F%?tIy_CJz8Arb7+i-hL}&Y_?aDh-M~qQ(JFybW@gZ7>6j^JJP&HjMqsM_`5KS zSdjW=UyJ^*iwqw|T!K8;9j?5TJl^r{>jcTQKP-`NZA=jQp#PhHAb__rasWms;z1LS zO+!VTdVo;b3H*SEwmaK9;}J4z{mS{VS&9KvnGNdKtK8N+;cpMhRHe@{ABeWA_?f8U zcRpQX51y6yJb0|*j^NQW?a^}>sK(ssa~mzw<>=zS=UJfIZo2&T7)!i(^%;YzrII~A z!YNg_4;Ekpy^NK9ZjzVhFUp&;|3V(2mmcf#0t$H(S38Ne~*JoWHg zm86Nl1yr1h*D>%R5cS4I;xB~)bV?A5QiI(6h8ryOFHZB%H1EljKv3-C%ToGEa)IY< zD`S;Qga=$O^7FB3Wn*rxyR6>0N>hs#<%cWM$C#)kQ&_Br(Sl)vt^lQXlQGrGXU^2` z%I&4PBGvUU8=O7~B|x)OD`VMU7}!Fzfp3F(TZZVOwmv}fWS+>=njKZMO>u~82-i*C zQbiiHtvG)B2D`NNF>m4+-$|FX^bto;i5Ym_{^e73$AsO##WP|`c+Lc9674Z8fvU>* zmsplBg!j@A-UQ@{jd%dC@i|Ualw21ICV(vM-TIm|B;!3nCIyz4A1>RyON53yKJNY` ztU1+i6_J(!`l}Gw%iR@l)DAcsy9<|)WM-ilEqSUb;`#WE&%L^t9x9B(v`y9J_8(b; z$gHbtL@M3uf~c` zk;TE>Mf@*XLdO=pgHA~;>9UY1#dlSx7Hw6Ro~mO+NWeqtew%dJR|C7x{n4Guw&a+{ zj8aeyV$bisjK?xft3pHNxX2q*QKDXQOk}POB%pLTB`sE%Fqut4Mz)Awr=xOp45)tF zWY`|g-5|0@q*rPo^VG>UUdW3=0N*Ws-;8%uUjLRt8=O)*q8jb7yLOROM|RJDH6w}^ z(q@gyWMvJ|8xZ}T7Aw(;&}MCu{)bDVH*)(`O~CZqJ~zwjfUaUZ&DCz9i!<5*q#Z{0 zwo{s#&D-*{xvgg!d}7rIDa;N%RrYWXpCaT3&E%`_&#ALy&}~Qe%2bbjx41*=xzb5> zj?O&v+9Dr|C`8Gj@Iy?IReo$ArdXc%Hyt-IFoF0-7=5I}pDIrAJZdDQC!$S7kQy$% z+gBN{GJBgoRYtbUb>Jzg?S}k-o`x!&A=4}09E8e!e6P2goMkC_BZjKtZTq-*6h*^9 zBVP7d(Bl9T3dzqN;6G+5JnqLa;^z*|;CiF;FPMZSpY{y(T)~hfW9;s!=B*1H2)x42 zs#1R5*eHz&k>fADDckpNeb%6RC-ec+p6z?mCs(H@;TUy={V%Z6^i_9A6}m*q@`c84 zUPcDN(bIJn448Jj_5MI#ie=QB&&i0{ls(ID=5P&j7OOAN!@3lQ-3wD_jcBafu839) zpz8(})b?Lv;09aqOW&J{MF1&fJ`^W>fSPp;iybFZed-T}_3G_oRWJ3AEu+tu{pgC4 z*Bq#Qlk`;O-65vr%Qs*#LbI?`s?3nEF9<&HoR^$a zNR9N&3A@-#j8;|mh(d6%tsQL+3^Zr5k~|!*PrZ~<^cHA|6DLR2nQrSinFKidue)Zo z!5ksB*0jS{MT4*$v7@`-KHYyDn;BbOhw<2Vfgs;lDL5m~a_$p;M0Tkvy;Ctuch6qu z7N?~vy=FWR6@E3Z9wb(PW6m%Uib6(6QxOyj;O9+?S*{-({$e5G{d*^g(&*#MNJlIZ z@2^hml2}aq@Lg&1;gsO7%vi+!Jsm?Bk_OR=D5MTQa%dvrzN(SG8p;Yg60$}hxl?0WJ%iDs=#wWQ(*3kMeJ2uWz}H{uFAvNpvVtUff3WBbhX!TV!Z(j`zob9U zIy1^JDq@0r2feg)49Mj+=XcfEVOlW^8b|0BQc6q%0_o5P%IW`*7WBkV`0}?|o(oY< zUbMNo@V>&GiGWw53P+l@J;ka<`vJp53SdkeYndOfl<90sGO>9b9&oUKIS$=1j-Rij ztxpYJLCU*MPEfisk+qirg`n&ilILcTz)P1D8$-iZdTWZ+?m&euJ1~UQ55}G!FItt} z5kBshVk7Z8hzJm?$Es4eOL`?|HPXFLy7f$9QmQY2&_xvsKe>ottDeIy8tq(QAiAJ% z{Cgzni64ykpAU9!z*A_aN&LO|;KS{N=!qI#( z9AJf=Fa1mo6JOFfY~{`v_K0LCnQ2u(jbu;GZvxk#T>(Z~xGSYuQ@@%q#(}`|Wnx4& zY(@~3GHHyp?^pyrldPldFgb;u#v)aKWe?F6jJ81DfXW6qF^{Ub#taQIFNa$A;=u7; zG(KrfYY;nU_Y6Ua5X2OvhS=ZW<+ycXy5TX0_rWlLGoHx`K1VkH<@}{^Pjkfamw)1n z5Qf?yeoV+We67rwPL)vhk)HVmcqsUvW6%c*vu`Kq;p;k>v@g>1ZsmvEIGKjS)q+A_*AzOsmnX%hx9vxr{wdg&Le|8~4@4BrM<_AwrMvLB~+=-C6Tfc|9c*X?rP2msUS_WdWnodE?T@GT*0*nFnV-%0r zQ=VN5Oj3MwW;EXK2BCfQw2@3r9==RKN7Gj5GjawQ^$!$mP*82aJO`%=l$*Cv)d177 zi{2aGLY4yO&{)qrN9xe;jv3Kmm=xa&828t8MYae(f^5nt^F^YZG7$}^F^Ow>yxHWb zfPQ2RT?QH5ld`rl$FDl|8(!;L=?_ZM2G!}i8Lj;cv4e9veVy={0zO7)x*?lC{=Ybs zA#b0_LG;5a84ty{OzMb;GY7TGBMY5FwLoIBr|=RcbinwlKM0R$Uk~Xo98ldJV%;ZG zP*Er0d=~G2h$|3rM$<|~DrjXFafj=y-NPG(UXmT+{!ZIEjRf+wZ$AAH;%qBM)%?M-wMUWLXkFzsa6mVPy%*LJW>R zdv^zN$=1v2Mrb{MD=|3;3$8!xn>u!!(CK-k#Z4_eJh_*N9B3BL?`%L7iN6c%NwBFe z|63)ohm`^*!uDS>L035JPiL>jS1H>kvpoJ3YfCU)-(B2M`KJ@6E-T|_$~}EZYBCZh zN~_tJR<(`T<2CO|DZ#RS(q$J3FQb_Y?!35*1al%=l~0@AwnD{H!0`>fiVd&`!;#A#<0YTn9yUVbZEs`-OE!C*F%#6gjG zxnKTkKtapNZ^;G|#Qm)OsSRnNJ2!GCI|5a@-m%kV)ds#x?CY1s4)s-GamBJZFX!RZ zsDCZuE02wlxQo;NQ?b`aId&>N*g0+Kh~#R=2$Z4nm*yD6n_Mfze?T009sL@Dytv5% zMNd=un*XNlI#r2zyvlWiH$B{rK&z1EV8-8$flF!0JJu3yHdqvZt`2SRU+HI_FI|j6@+02fl?7X7uolV_cc1vBn(Vx>9XLw3@LvS z=6th3`@w;Dt=3S7`O2)9(P0d~SKG|*^_Zc<5c{!s3;G7e#m6|yOQ&yYAH!Iw$adl@ z)$>u>xRoTL*^hdN*+##VbLGFz`u9N>)o!J@TK{Y~)Sa(^#brOjzCRkOhw}PQ<}Sy9 z(tB6dgN8LnuH%lVc0Fut?8}A2C4Q$1gU^kc?8&~=QB?qGp*7fbk z&oIc{k}DpndBfP4B1DAD5G;qA0$)k!x0alHoNuyt-`f>IB&7XVhnkz4b0)}dZYIb-(13%A zW6D7>n=ESGK~O}qO67KC#lgX8d-m}OU(nFSrCU6Ux%t}Z@sKwBW};-ee%m%or23%Q z?HB5pv&4e+BHCJz{}A^}i(4A#wj2VE05anC)4}m$dzKqh)3R~D^3dV#(NoBiShKL$ zj5}s}jU*IGzxFm)xS6s24;EU>H>#0_LNcrmrR!+FgF6Z0&?YCgyoI{wbB&3$UM8|) zfmHroRcm?BSIgFpPt}h@3f<2Q-Qrk_jF);~P`lx<0{bq$tnJHqpV&Cb8VI#GU?NL% zF|p}Tsuahcl{k&TLn-lREPor)jkn-^9vRP*V7~deEk=bVz>dFWuSj{A@cBQ$1GIJa z)R*c-7i7!oH{cgS=OmCn`p(@H{tx%?)ZjRoUB>qp)A!8B0@M&jh*p0e zOqecIc$n*(L1?|0YU-|3-fJc(?R{Uf<9#bnWWc<)dbqc61EMQQ!|hsY!Bpk`VM2vn zMmx3@%t>9rznlG49gASeUMP8wEoMU=2p2!$3G+IcJVs#3{_SPyO|@FKC62zqJ< zDu7y{dhQk&%kAkihmr89pm(4PP&sJGpCfuPP>CBCay5BnHCda(7k7jo?62%)p?9}HN3&6>!9R8YBK)Bkj}5~hE*!kxh8)Cak+n}i_9#ns1KeWT0drGW6LNh z1$f?edwyAPylWdaEp1C<f-n#353;`W7b7PZFWZfj+(~5iiU=wM00{#-*%=mcfNYN#-H*vwSGm6 z{8Qh$`aoG$L;u^`&$Y%%>9+hg=b62}^Hf_{+bZIFJsEP$3Vw5}a%xB6VM+aEoLN^2~{Q!T`J+y_cCyN^!0QMF}g6w9?@D|8h=W_7p zSL&2WL3Rr)1;|~0gM0l>6BU9ik=!-STcrpPK zvPKzm#AgXccj*R@RvV^Y&==to;FOT^d<6+%K)Oi7O-zoHkEc!(yXDkFs6*o8Xv(I& zX{kN;(lA?4z!S}k|GeA%*J8~+C$ZAyZ54WrbD;-dg$K|9}0q0QwunVDD%nyWY z`_6h)0ULF|kZJe7Gf}Y#%8QEdRSTcxwp=OG6ubmJGT_0fM+W&g@m82P)8n5FM&+ zz!FO#qPYfH)@;;PW!mhwk}JL=8!_B%!$Q)~V+7nEKP4AGc72aO>UPI0;9f8X{#A!Q z-gWqx|2B`OTk~^@%UyU`uKVQ`E zHZHoL;U<8A=X4WOq?@gU=w5R$m5(UO;~f3uO_v_A2ur!p5T+BwAhT9XS?p!!`I}5F z&Y@-Fl+=53rm3eaQivXrIi~^2d-fbmnCfT8)M7nZpOH3W+2{FS&PMk&UfnQ5}ga4D+}Gmv1Kve>;cCM>-U#fbi|y7 zK=q!|8#bDsq<^ZSuq_RTgYBVgiiEtp1Y)tYFi_)3TtWMDPRsv;dYeOM7HE;wdpc~< z6$`0KD=frwX~gGTnn#4PMtQb6W0c}OwcEMc`L)z@>3(Gnm&Xmuz@WU;Vb@IBZ2U7| zt*@Lh(!D?^n(BEYE*5rq_Pi}V-cN&@Z-{T1ak5_!pt@d%8THG-B}FByPV33pL=K}x z{D^Ib);y78w%EdO;+o!g^6y60#XoTjG=1H9TEEEXpmaX>GZtPdy#z{2O@>0XF3i?( zO0fzTTJ(WpkKLoi8dO@HEppUDC)O2!&k~vpXAJodmww=4#`QTiHICVwj*sRp` z1uN=Fso$RPaS0(;O?%~tMX923|5iPq0?_cfRs;gDs}&R!GJpI~J*rvyQ*|1rfa3d8 zb!tS1Dlz|M+jUo8S7*I9eY~=yL5o>s?kqBL8vwbSY45)cj%H#)wxFKT9GE@u1pnmN9iK>(ank*WMiH+gG@RucQ z0VjhNC4!3Ta9%3gEYnVA;!5~Q;@?gW@NUrQtRFE#sAaC}v=EGR6UKD}-?id@VNKZ8 z>BK@2RCg`*9VU7v!3dJ8F7;)TtY14-;ibCnR@0#|Aldg$)7H74SW}KZsVv4u@4sL{ zaPc!(aNro1p2?WoSDCSe#g4zl8^<(aS+$1U`lwQOj%`gP02x3Y+=jf2DMT<-5CwE^ zuYVwizXdJNH8V^pO!4R0QC0GF2$xhs*y!flG95fvzc57vlX!(mv*tc$ahbBH)^U^f z;Root8w+h~eR;tL5_vfLmc&AD3kUA&BVSj6Joss!*o+iYmbYYsm`C*t8`S&){3{gF zS|0$&4NmA6i!Vpg`&thCTU_Fik@xf%iBM`3F}&XjSdmY7>AtoLu;mzWZ=_mw64NDp zrK3L$_-2!`u;u#*(sU&{5&fpEQVY$3{e$-~`iz!LX2{Eu;~(%=Y|~mdamY0kvCJ{~t4x@2ruV1lK$9c> zI;np%mX*{;CgKQgtgDKDxB!5M0G)iK`o)$($N|2P>n1wKDP!_o1B`9OW$d0t@HWrw zhcasjQrKirelK~S;SdihS?-UMX%K*T(}*UA*T>|wRKm4%UTd<0;!a=BO4AE4;B?AbBAyAw@?EAq#U)zSR3s@ z*6C)ap{9t!x{#Z3311C9DEm^{^KgCbi(@|h#;Q*miC2WEEJ^E%eV|8k6o3sF zRo3^mW4P4ua2Cee$6?=?3G$cN-3eAagP-g`u4#Pc!}xst6JHggYcH$LZ82hhXDk-x z{(NJK2=i-lx2*$cD>pczt4l~~_u5#EHmInM6UIMRie!R{?Yp)$~&%xm*9f6;MHhtMgXZSz0z-L@T{pI_Q8A*0;!0{v2 zpQCw6WX3K0_DjlE2MV>b)ZtbEs&kr*clwh}=SpQ3a&H&_nqKx}{dg0fXx!GBzo)ea z-X0+?wTw+h-@s0iz;n0QGEM`8gL%7Q?i!OtCk9LPC(m#%Q9M6yX+Ztk9$f2wCcEfz z>Xcjbj5@gKKU(UPyU$#>C%MnbLAvbNSdCm+>Cj*|m*pxr9hIgH_t=&fXnxs@685!Mhu=-m$q|;Cu_P+Y!l4rU8yA0o=iy0B03C4nv zk@~*Px^V3DU>G%j{8}CRb|&IJ5-xUq8;|*(2wul$JqKw&V7pX(E zT&br}4hPsZ?=3j2tj*4Al_Dbg@r4s1zn%6r5DBkohD+L5XtN@P7jv3t= zkcTT#E(}=%b~7DDsrW6uQDmhqgD|=I`Q2UJY>#0_=g3HS$8gf0*zaT1cydK{+R3!` z*LGetyzfP3Gw$gb*u;8c5z&a>RUzP+`@||i5s;G1rKJ{$oKf0%Q{EaR@wyt)Xm591!_ z70c}nhmd9pfhOacF|#%`t46r$In?U8(0)jL`ov|Efw&k`Msac2SyiXpR~ao&c-F+j z7G9eS_@74NLe0)zHuD>{Z$*;7fEbuPS&~|<2}KK!gWvm4t{l!lzWCk*rpX`mvBp`d zkAp!@`_bLmn*9qatuTGEk%kEIS)`qsB|d<*+Rw0r(B)5$W@WF8r}lA0Sx~?Xv^BE~ zO%orQ|0t)dVwkz1l|i}AGVb1p90ogqj`BE!04AwmpJs~CKnx<6)_TE0WDbDSu47O` zc`RevB?la1_FwQhviktW9Wp#_)L5TU}S&Kc-a+;2wTehZzt?5Cm@92~0eRKVK zuYV)U(C*1|`ea-+JUx2ZU_$IDDLP~6too)Th5RSMW!*OLK6jA_swDYyDswp>WLRBu zaElDeJnYW!a?Qf-8P*PqhN{h%M%CM=$B6|jFn_yc$*0voMTIuDM$)jMb^ch(;9k^5 z@Rab-N;W?DPJa)T%DLhK)w{giZSm7QZiDBWOG=uR_v(|nYp$d1xbl}mj@d?(nAYWI zQa(GgXw%%&6P$=ch!InVewg}EmoA%LB7Jh1r-WmQ*GQZg_fGaT;{Bt^DrCSS^-RD7YnVNlW8{< zW%L4C0u|qHA`xd;v!7#q40HGCoio&y(4Q-^MHaUPhi?&OKOVA%_Zw|1_IaC;fbY&; zkLE@9)0$uV`^xe%p_l4_6leF1b8@QV5N3-ZBKbGrZX{ z+BX$H+PgM&u?Q7?N4y@;5*hM(V0x2;moRhKrS)8^RT#v&8j@&M*0og&FarEk1&r6X zT`xCpozNX9W00h{&xjXYFSwbRFYl#g$X~I`X@voGFF<)+Z81WSmZoP|b~Lo(jg%{* z3lo+U@t-j_#JkAPt$9&ch*`2sfUyk3ouNDw;v4ytR)v z*o5OF>kdTwx#RK=wHJ7bud0*cJ;lbP1oc!iA9Gf<`jV=JwOwUepFGXYVGs866ghv; zdEJt!xCQ{K$4c>`Oi$F1tao_7H4&MvtQ?FRoq8H1AN3#_K$gji=g{WM5vk1 zmh0~Z%tk*sxP`D{OX5>ACM8dgOD=qXPI1#C#+3~gdI^pzN{)h@$J#)eCm;D1Fw$ZE zXM&Ft3P0y$E+iUUT(4U9&(x@se{MnZ1>gc^4Zwuj*U)GHI zsBtZJ(0z(@W%*&v&*GzRy_t(hS?@_=Wex2UPRYEfusG;`9Zr8GV6>!?213rI(c2E^ zQ=tts)$mX^vf8v@t?Jh2mgC?7daj1%z210+wB2Oyt3EBdvB&{8^D_*|PEKFnxUosE zK9hs+eFBYvjuF^{wXr3{danAHu2^;2FiqJHMs=^+TRB*p(4Q3ci0El)tIi#~XlUS& zE&B3t@uF2{$ERqGirD$dunJK>Oj1n(?lC4$Q-SQJ_IfCV5!@H+jZ8jwxBwQx#XI;3 zRgiy{!PjndkYtitd;AM2qXbS#&Z>W*g+$>@!I3L93RC#%^Iw|T*s_#@x(>|ydqDwR zPM5{6C;Xf6cpJa=ytBc9syZ-6LxJE1og0amS?m>?Ue*3cWXYb#OU;xpCUlZ6^!xB7 zq$F>V39H1F9YW_iZkj}u4n$YCzamS} zHuLab+m7+WWO`_YBoc3XbSx{9wuNiA6AEw-+~#+el5d#3+I4k)1AA2_M6_0%*ZV4$ z2GD3)Pn=^iL2}IVCa$MGwfI#B2_R?@@2tyBvB!<7)8(HK{3)~LwGsy{{2 zpd}+|GJGF={hg=W$cCVE?kvm#3t#I$x)zhxJhID;)`mc-^Q9CQ z2Ul!Fg!j_jJ&nDSmjBeQ-%pM}yg4l)35jLddWfb6;x4`hLUp3M$g;3rVHB%%;m;CA zrpcdAzXEnolnP0H7pM8Hio3alc~((tPxept7hc6MN4FJrxoC^6^LSCzlAa5~q%;dn zq*Ma&>Cj`#*@Xb>_4+7Q8AJ9!mi8@ebPi@6@j*5dK*YpOOoll~>2>nR`X>msohJ@L zPcX#KRwIt<1*#^hT-*9o4f~*?f0#&=#99tpJMJ5+4<-}Ns@LoYGJ0qV^%_)aKjwkx z@|4SI$0^}^*VcLF-1MmYM^{6eiY=fRuh50+GSwJ(t@D52$AT(syIJX^dfN2!mY z-mfs|$~bZuH#E})-GMJF?3IuJ;D>80uiYvSLJvpzNpAmqfvI{okTvU0x+mvZu6ql@ z?UWhWd9Vsi5=QIhRq=+enbo`$-G<<>lCi%bIHs_k3^@B7vAT1PYw2j%sDqTChVZ1z z-0vzgS@olNrbhiUEe%Dfl+O>ADA_niKThYM^_(^KFCjq-ps1wMeDt&%6zUb@s?)jH zm=kaOTea4*Hv@=VLiCkBI|SF`nqnz%*PzQ+%jxvQN`wx~e|$OAXT1T)%zrgtptV^O zODz}ldiwghcxjsOpOdGkhFo7WmCU%QVEuRDt@8h^Q}Dj!>w2FW#@aB!9rhn3n#=R* zW;mcTQSJG_$^*pUM^nnxYtYeU;wJ+8%v7vAY1oTdx@q1@T1_{4-mje>}N|ilH;!cijLag;qb% zPGml1%)BlAnxBSW;yq--C%dp&A7%X#@lJLvLjz912en9MXah8jG|9r7jJXA^_Cx7E z6ZB6^=7=`+3e8dXl{dHCB$eW4=E!{T>y6PT^0I{DOS!@l&qUFLY2sUd#90OFl5Jf5 zXTNE!VBhZird%|9q|cqu-okwf!hF(2MxymWp!w*1Cx#!+#kxDs0K>&bY-O_xmq}xo3}ZJ9B3n+KOLa`#%gXx+4fGWIqr*%!|87 z@x2#@&2mzjSJ3%n9#~Mck>6ZdNK2WvI;Jiy^HHNd*QtG)fx>o+8wm(e?A$g-RdIi312B)hyo(V@>A+| zm0h8D#ksl0K-}K!d`|@l8ST-tuPX#!ZIKPg!PBVzkyjqC6*EePL}s|&yA=b=%pc9) z@FxhrMl8biZL}JlC_)JjiAL$_n?z(6i5jIGDAkYL#RNwfm|?+>)g_-}6#h2bd>iJp zA0ui+c_mP70y6?tmg7=_1$^@(L??;r%FDvIomf`TrMyUxE1&Rg?--)Y_rMsej0i&ZpI1cp*H zM>j}yOsX#Z?y0oN!w*W#>551}j(E*&GsNk#zopl=Yp0gIh2&%*eG}C$h8(DQKci&8 zdf}`N7cqLGxbBg*A&bX*p3F1t=qKseFtm1P)atDf8E_U!_2b}cU`B)bjAPEuWZ#upyDVt|TIVLc+??~+LRI6v8U`FDmynZ_y+>aSN5?9E)OxMR z@k%pH3Aa6~x{DMY+xN*hf6PvlDMUn2 zhan^p+0UOhGL%xc7l2(cVEFxz`i0G^F{&?=-6m_?^?^cMR=y%<;nvVW(rzK#zGKa9 z2C)ef{Yj^Pd&JI6Ls}nCDA6hRa0_J=kwH@6sGw``x4MMwFZ0N;)g9E*wwVo-T0~1n z?(j;E{C1V!e!F7_1*1zIMC)dUJT-%8PanP)TCi^`w2(5i=kr)u$p`*U6W_6FSFE#= zLJKJ7t!TJ}wtjt~HMf`6+L|-SUp>uQ@9{t#Ovqb2cAZbnT^Vcjv|!NMt+JA@I{nRS z=_aiC%O?NkqNsICL&sLnS&dGS@~-jD?CX{6kQ+s}K!CJ@R?LlpDkM2Wpi?5+vckdj zhl+CSbwIKF3CE&mbvfP(=qvo3M>kt2`L|S5YNa+g55i%gUnkPr1EQmncT4ZYM@Jb{ zYpf%Hr_nR+vvs)`(CNw{0p1i97nc_V)sr{u(w?3N93sFzvq&)1IUn~ zqGL!a!}o%`9mrDJgSN=Y9!gd-D2HwPOZ~2NK01^$aen09P4`d0P04{?Ni=PISeZxf z?Y>T1_$7aMd3UX^v{6ZxpO`&EylX_E_};Mm{N%wsqIV3tvj?U&9a<&#c%qo3lIoZF z(6?~vx}08UwUEA5{IXNmVm1>bjR!w`;amRDwfxk(?XRM65(E?Lr$xkV2^ZFfm)tvr z+_E&A?>KS2uO7(%IKrhN%~f0d>_HGBr=^vA79apzGsZAehan3oTVUKcs}4uhm=?$E zJUtbGn`)tAvEzeY25>pBeXr3sFd*NrNCVuOGNZ`E=x*xv%KFE%#f&6Uc?1&KtR@i% zJP6zA*KM6s;;{#8Pm*JT#k}pEtv}TYX5EvmGVR|Pd~#OH7}UJ)cK?-K-MUu8BpRs^ zHRRlV7Zlf`Ui}jrJ8#0Z-C^S@#%MDJwnx>2#fCXA+rG;s?_N4;sk*HHCN;>p=_HDS zG8L|hn^VOVc|Kp$9S1Mh8fEu$U8|_XK?Yu6aKo!5yOZQuC-wIcR&ufBClDeswK&fp zN9=A639uJwvWOv#)_!B&=UpR}m{^eS6|KB#ICZ?@{qg9xH$u3_vs&;I4{n!|i|d_s zv$Hw5Tab;)m^&Avh~KEqhby69@#bXZuXKiG+;qOJD&%5}R=Q?F7HEEd){hKsZs;w#y6ba}wB`iZq?iNNUx`GW@Pe2d=pz3y1c1j zXQnr*ef?2z_x7?qd)F>GopkpAF1VlvR>U!E7;dL7xgK54-VmA&OOGxFqt|25-bDl9 zCnd|B`{{4Ew)mm2VPpDcGVh+nhV1VgT+|-Ma(tcDdqu~-ZwjY|o5gKii==L$Su8bq z%ir$)U`=%*Lo$zqWUD>-AaRIc^*T5rS6{BKd@;;~j&-LYuA=>^g5$@esVxDq;Amy} zv(@4WyiCg14^J)0J(pMaUdc+Q^`J;y^m7#w)|L)C$)R9*JHTN}TlHB6YeRnLXYJqP zkEkqfj}n26(RE=KIKO8B{dI+%-NMF!E-jBy@F#*8INELxaUMhyqt>3k54(G`k-7tB zJ{xnuzb@hZ*%?l%fPl`~K;HH9T+d5|Y{I;VhC)|Ia$b5m4muI>5znel@_7dr$Ms(N z-2+v%rM_U6d-k%*IzJQY3mg7|Ps!)*m!t!xgv89m9pp5w)tCC<@ z#p}=U1JykD*;31*pa$u~@?^`82A8zoL(8p34yS7Y$lvpCr2P*Eo-I`L-%mG%^W^uGDHpKgjJ*y%;tZyK~U8@{K&QQ0<@6-#ywu+|!! zi%koOes;#0W-4FlXsYmy)3@3 z-(TV##jGWf{!pPP8~2RPFr2H}{=v$jpn1?pv5*(@0#9L#^|NJ}*dL8_DAS&ws7^i( zjly~M3*}6B!>>Dqkz1x24V#uUzOJ6u#c$AV|Wj|dD z2^mjnR4Bf~(u!Mi?6wnmpMcxYOqTn`s@xM|#a2PpU=qVNGheJYOJ`C@92V1vv*<;k ze48+)Lh$NU?%jQ%>5F}jRgx;>yLDldzQicmKQiN9I7r{$99)_%R>5}GA@x;D`NM5) zxEn=920ooptBBN)_0PHkix~R{mCyKnhMhB4;`BdU0OpTE38^_?(i&-F3UuyelUC=QhAAZNxun30cFfxNst;Ftb2*1cYRcdB>_ru}egbI>G#-o{7s^F3Tlbo8Yw7(Xb~;OZo5 zZtjIbEf7H-G;V2%$O4zYN7Ec`>j#*P;fw820kvw&Y*3$v!odxWK<&7~8&Xo8#hBT5 z6d9{?cr%`7NA~*lYl?S_{4ZX-uw}=^v0ZpxC?fIa8`@8-IA-EIdxC*gJ1}tAm~`Aw zP})8n=7A80VsY?es*fmvt<~ir^>DZT5DtkKm#;Uln^d~b%TkIwu|17`o|<-b zy0zPMMuImlm~?c*@dX#s%{5){;kwR3;(&?^FhVjz-ALCvKOoFG7CY<^`2 zoD5Az{|NPG1M8>VhUixdr(3QQSt4?G4!_AC9};1%Oo(}lx^3@@wef7lugI+9wQiH+ zj86&dHa;hNj)QJy-+^TY)qeH$Af6)HYhVJ=YTK)jM*~a-+x-{lL>`Bjl;@mn!F8CP zBUus^YeWmUHAWL>A4)0N+gJF9VHFWf-luhw@*JEw^sImfOrM4|!>4axK*dU1<;KZM zs~GT%fO6?z>hK*f?T~`;;N?4Pe(-kry_f#zd)2#267#LnZc9?E3pnSFjL1WxoXec; z5;8LNqJilKP;k;hA2@Dbo|VC~o`!TrkC9eV==x zL7nN{oLf8^%&h`0`z81G-@uAyrx4pJ4^#{pQQCv$fRzY6923!-K;U{ln8>MA z+;APKPHQsS*7^SPkry!-@$DFhmI9cr7;d$EnfT7iw2+z1UYmNg<|B3xY0&1=2}|kmnF}NzbtMR}2iL zj3}AE=ho!@m~od`)N=Dwk|?EmcgkKT|OG%lGY%7c3T!-iEEV>@9 zF2A@|6?jBFUSV^In)w8z_>UL@G4px^ai95anmC|4BXq%#u@LHV8 zz=I%0M_3HvmI-Vmr+=S5)vpXK)xoMeY|_~Q-UR$80{uz_gkAXK8{B(^Xvf@zw8xD= zp5J~WMk#lDe8GAt`4o7gseW`DUZ&|+PWW zy4B`8C#I>HXkcF2p+W)t5cIl2M!|oYwZ1MGlk#Zv>2mR_Fytk6Ov)~pxef>OB1A%5 zxg#Ue5~&Otd3k2`s~`O>Yc~u-v3O*GwOIn|9cuRX@04Jp36?`*cL+N6!#tOZ`Y!Ri z<*wbJeCGG$0JdtaZA@9YUEnc=Lr`%wk4SHC$3=8q7aIkK2oU)irE9*GO%J_PlvfDh z?i{{z9xfdi>?i8V87FC4Jm26B;p2N|M0>*a?=4tvzMr#Yqy5>XLIJ$B!SzqDE!|d+ z*{!UsT=$pgBrt6)oK!_ezmjmjV-Xn(3R!ajMx6@k>IFVNPw=>1K90(azHe^75*3?! zSHD0T(4OwF%bF{PBh@T?4|) zC{Byc0kPDY8Ef6&Ak|<~P>M=Tpca{})iZLwsokoQk(DiDwOT9y;dB&b7{> zfl?)w%OA4puprJ?8<6pp+h;C^3!cF2Y7JPHz5EDIAMV6pr za3R1x%E_$9LEzE9I1G3t9k@L{=9K4JPEAdoMhz@t0Fw~+yy|xc3G_(#TLCWouuGRs z3Xc>XBFYmCF6gxq`HG+D3ou?!4XHb1eeruUelT zpKSgX_~_fFr~3anW(kibz?7f&EAw#9Av!o5t8W`mPN-%Z&8ok1-G!jEOUv4pWsRZv z+JM97>cjx71cR$4kVDG>$HG^bm^p!gPtOh(#MH*Bu6O01#L_WXP8P%;)LX>?L&Dc! z8-Ip3G}YvO6IhD&XG}Y)U79)`;WZ|sL3wrcB;v2Un+!@KCb(nE3obZ z>7HP7B>j1||1}ls$3z7yV%Xa zvAT-BpqaUP3N~tbCCMQXzv(u&om~niu=9I4P;OWXK82Wo*N;CX@c$pfzW@JXNlRc@ znl2KB{}To+BygM(#GIdH z4+p+zcNqPINl8rnk@x;G>bsbifFczol0#9XB2&D%vHPAr_xAbkci(&e`_K1w z(|(BTU3;&+)(m5exu)F)i=HcNx0(c9BPT*M_0>@*AH^DBxzp9T4bUMZ1zy|a=0-*D zWEnZywLkv)(_M9@dE?M8VPTnU6eO?Fr=_Py|O3)0I(@ycuo%Zsj{=}Ee3?C#mS9qcKRtj`C z8ZN}}qE{)UN;W8R2ht9q#w>%(3I^inwlK6yOY<@>SC#o51s2kiGf z?&kZxXwZX^pN2-|$HvyJYlKMp@)a}E2<&h0TgVGfI$tI5IvtxRS9dKuE8)rgq7fmR zp0VQ5zDv93V%096u0Wk?1&JwVn^lV@V)l)yGS;u9(eG5+==t-SzU0R zx!u;#6L5BXS=}P$RKkeAHTIcGFU~ha^M`{KZN8SQik4QmbO1G@-_tRI*?sJb2sskk z(!S?6gAfZEfu6b&*VQ#;L`y#WX3)5tNg2Q50*7}KYYo;Z=r5$mZ9(4K1Y zolZf#Zq~|tMdo7Ho;`c&>P4eHhek%yq0P78wCYHsa7sF;QdXBLBs!rFd0$NZ_wOwg zREpgOd6Oy!OC53hUaR9~dd?jo;w;2qT4p&F-F&-XG~K3B9LLCzS~tg``)&Xf!h0U> zsqpqLpDMNKU7V|d8Kew#FO(iD)eNze%iHxMKYR?=id|*aUCHwDqI6$aTNeG}9d|?H zaNU$8E}GL<0!sr+-6mP#52@*0G^wxdC&jKrG{~wvqeUx5T~VfT|ELSeXfLv3K@2X^ z!0+Kq9J9yNo>+tCVTbWrj@a`=*Q(bc9gCczR4H1q1aNuf)Kx3(JX3$$PXnuS;&#}p zdcyq=pW2pF6zdurJGh{8sV~pOsPlg8oZ$PAq=zi4tD>PujF0pv3}j_dnIBR2PSnGp z6B82!P3my&gZ{z*fQ}BlirKm|Eo%yTgUQH_T$PbslG}FD)90>yKZfSfkDJIs%c3o2 zgxUI8_!#TP)c5TD>_x9J*u?q|MjnctV_ia#^5C`RhB!MG8|&&+baf$Ou(pP-nEcSJ z5m(pCQ<7>>fL2lIxga2LN7Q9X36emst$G&Qjqc3SiA^%Z8|F0QNXBL07B5+ z1^uqyUhO^(^M+IPoE3GQHzcAYS=#wi2q@KuX;NRb21B>6 z@m84{cyY>CuNIdq|0F^q%DaNoxP)8#2rj)@?n8mCm-kCS*LB=fw_PcvyU4BrTDvOT zl9o>Y_)!Vko3^4c5uL&+WL=1lAz>geGllG zg$0~qn%lf7M3)HwL$2f#LG}xM*{qSd1oxMrfldK~-2SNsS2_V^FtmQS&3m)- zj>KkJQ&uzdsuS?0hDC^ZB(sf=#^T2(F?4m~P8uGbTq~lX&R4O*VY&`EyAGD!fwsgk zyACk0p?dPn0u+hnkza^g%RV(dd}rI*ahgL>>l@x@U>fPj!-^TQrD?fz^70K`hpQW6 z2UWj)`&M^pKmCg{(W@3GU+fRDsVB{*aYy$kjaT9VqSUm^b^65Z<%*BgLvr(njc6oo)anR+hgPtb0>6AN zP5U;;7F!yio8hAu*=HING{t-@{6BIlyZ`qS$FfK!Gm^SDyS2kuGI?`vbXQFi!JqKeEs_MLVy({&!nQwjvS*JCvY6^ zg}h({;P}jFD!nVUBSX`wYVLSlok*I*x?!qmGxep2^Tb#u2j|th;H)9C@M2*9b;1u( zvNn$3(VU%HTHzdN$B?L=m@S}b}o?S$7xNV1}bD)jT6b?<{s zct0HH3Y}aTjSVwAnH&o(R*|C#-t5zF5$zC@_LrB_(Z!-i=nnvX>a{*A(`fKfawEE>SyQWuMS>YAH{HxWz{ zjt8BIC+{lmuWZ?gUNhf#aR5fd#UoojvNI@YV+|+ou>rl{nXWqAWNISEy<~P*;?l@$ zfu3ynPU9-53l;A~MMSiNV;<^)M+O|_*_%^W@AriNy7HsfpYD{#tb0j%>v^q658yXt+0lr6<@4k(QAv7m;k@d19)PbS@Jb#hUe8ma3y@0 zwENVbMJ#pfS65JYGhIB|EV9%mRKR@n*Fz~CJ;pkTGBt%xt!rTpj*cBA&Rhg)N^jl1 zz{ySwKH-*}C3@}dC)gu7y z%_&Fl+jlH1bjlWazc=GSf1L?^sK?>oTjhlbnNyjk#8!q>wM$*HYS6IDzDf7|+$UYc z9K#WM$@tl)A%|6uQ^~JKB&WKejTk+-Sj|q77N(tUr zc{FsrMUF%5euqq@Bc9)2`}Ox9Z~hx%;@{E&{)0mCQSzoeJ@!f0p;&gvp_mQz!f@G5 z5+3b9tRVy^)j})zU2!(zm>b3l$xsb{8^W5@0Lw(e{XnlFD-4D;q9F} zpM%|c?c)<#kMxg*Ce>QcpRXXy%8J(8g?Gqh;-)8pO1q6uDxwa|1Hl(}^25%`e@2YY zU(&cJBqAq!jENM}utiy=mOgv7Bih?*w==2d*43+5;R8B5G!4?ef4|KnzMcV8yR}dx z&<6_WdVMi7l>LV-_vEZO+1GdpCabCL-aTiG;h88|5PFVA;j|$C%k93=k!!LK(!Mn1 zMB}RJYKY9$gmfBtC(a;1MXI-RE#a5{_Bwj>Xck}^Ksz8kZlS|{9i&#^2ZkP8c^4U( za*(Mf?PE)Pw5Thms3?vo1-SSXs}7lIM1(*?e_|TqQ zlSl?ZS&dlnawJ+04>vb4KBa1ps?@#zJFq1!|2Y5fks}FFQKx{EQl_WpV5=o%qkEd$#y zNWBgS;3WsYeH({#rLL%$TB0nL%vY@6fQIMJR;9LM&Ji1n5(`!9&4M0_7xjdlUe%!M zkBnttuytAJei=tf`%nrS>M4dP6WViVT&1px@EYjv``leM>|n7pE!I=wTnc6wLH@?Z z&ZCh09ww4?;h_&HDab(O_L97q`5sBq`CW}<`;e4mK8I5rCvL6*XwuE$KGMx8i$cwg z7jTVeQT*2rNZz7^Ao$}WhGl482ht4 zA(zb`^gqd`3$$McBGiFQL-78$-ywanDG=!|!0RVI-p(65gkGmO>toAKJjzo9%t#^T zTH42tmmup`C-p>gL^dXW`?ll2$p>k`{1uor>qlR)zoC#aTHr(TVFv5syPgOP&+x$vA4n4U^R_YBo6O1d2kNE$16_!6{1`G;J<h7RUx$~lP- zuq{ulWyQuDAD>o#VM3~1gJ z?pvLZi1pO9SU0hB&CO{Ees`zTqxErIv%mYGJIRbL`r-9WN1tNkWs)?4a|0oV)N zcBwpmKUUh_-X4}z1~YVBv_bW%OrENb1Srb9zs%bLR@0Cjjb#>3u}g>RaeybRf#J(ISo^u9qX+R6Sb-)Oh4Ii&Nx)Z zqZ_c5VSBaVACI3xbf{u@J^mo`H6-@bELBt=Rabh8@i}_Vl8D_vaG{^KeA`nL?F!6W zZ}w~cE2I$~`%gQv@vn32^iu|D&( z9|*uU9M0;U5xa1@c6^T0WOt~)re0+t`AvIdo)l%5P#N)6*? zs)X+5?hh$sJRr4?VR#8e9!s`CvUIDITM903*f{xG#jf+nWjZ?M5qk6-+{7Rh-zZL3 zbJllrb1ASQ>eB|eYmG)ZzPTYKx9OYHZLQACf3-M}ZM-GEa@Yp87J^>0_rf|D{cFE)mYOXY3y=|JYtq323W(=_n7wFG)s5Wy8d(=&UG@7rsM zG|Sro!I$KAp6XIY+$rcDFbld+Snaehb{LT}K)(kGDysSYt{CyLaOv@sSTBtL zW&+{_{prOD>nMWIdSYS4EEX|yp^4w){VS|hmY4}n7MpUEftQZzZ)3@w60EQZqOfh- z0byNCUz9GBguU+5T?bS^krB$Ra}HJ4Sb(Ib6Rs6&S%r2)WzveKvp!!=PEe9*W90ag zeO`%3rpd<1_APa;pfme!#!(IhbLix~YThq1(UX+{VB-V?Nf|B(R1mX6wW$y~a*8dq zJZ0D%dx(a9uCR34D8hYl2?=LO{pE6;qaoRR<8p68LPL{({=AQvR)-SZ63-u>FAUzK z?)uDvAVh)>zY9%Ujk&l z!2O{Ir2Yb(ASZ?3bLviZ}0b}%`sW*zG2G(AOU*Bzf-j- zG$tl{c92=!3#*qXEtZ)xSPg2?JBo@i3DD=c<9v;X+hsX;-EyO`L}jqHZLLN%>8Ci5 zxw4f*%z2fP?wvS#l&kWR@yQoX6Dbfhs}*t=)ua?USxSANRg`cEJ;uqy(@GM&AWjP} z!f|*gO6^r+k#q00Ux1Cmhb_S7k8Fa*>K~E$U&KxSM{>{+d0BD1criAVRo&4Pk7Qsa91!K0fV2>-PW)wWm8)LppT6Xi%fa8yaPjW?sgrnQ@12$Mp7#2DF7!2Da6?l(MqdBQi8<3z|*IxEe8V=318G00zu{O~{i^-B~) zunSK!Gp(ztt3vH+s;eei2Bhk0nZ{ZP_l?BwTl4(1g0HP_Ch3cHX3h$YuvU+p#Ufwu&EC#shxdgn)a{nIC{eOTQ%<$!A!!{pr26Kel_MJef5ay%&&HHNRl!@Jze zBMbAAZrj`Y*kY7B2>Goa^X-V@<|&f9_x}J*K$9?fT8R@$m0bc